?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
s390: use signed tag comparisons, so that we could use
[ajla.git] / codegen.c
blobac6b244cd60427fea632465170825bf0f1cdc010
1 /*
2 * Copyright (C) 2024 Mikulas Patocka
3 *
4 * This file is part of Ajla.
5 *
6 * Ajla is free software: you can redistribute it and/or modify it under the
7 * terms of the GNU General Public License as published by the Free Software
8 * Foundation, either version 3 of the License, or (at your option) any later
9 * version.
10 *
11 * Ajla is distributed in the hope that it will be useful, but WITHOUT ANY
12 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
13 * A PARTICULAR PURPOSE. See the GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License along with
16 * Ajla. If not, see <https://www.gnu.org/licenses/>.
17 */
18
19 #include "ajla.h"
20
21 #ifndef FILE_OMIT
22
23 #include "data.h"
24 #include "os.h"
25 #include "os_util.h"
26 #include "util.h"
27 #include "ipfn.h"
28 #include "ipret.h"
29 #include "funct.h"
30 #include "thread.h"
31 #include "task.h"
32 #include "save.h"
33
34 #include "codegen.h"
35
36 #ifdef HAVE_CODEGEN
37
38 #define flag_cache_chicken 0
39 #define must_be_flat_chicken 0
40 #define ra_chicken 0
41
42 #define INLINE_BITMAP_SLOTS 16
43 #define INLINE_COPY_SIZE 64
44
45 /*#define DEBUG_INSNS*/
46 /*#define DEBUG_GARBAGE*/
47
48 #if (defined(ARCH_X86_64) || defined(ARCH_X86_X32)) && !defined(ARCH_X86_WIN_ABI)
49 #if defined(HAVE_SYSCALL) && defined(HAVE_ASM_PRCTL_H) && defined(HAVE_SYS_SYSCALL_H)
50 #include <asm/prctl.h>
51 #include <sys/syscall.h>
52 #include <unistd.h>
53 #endif
54 #if (defined(HAVE_AMD64_SET_GSBASE) || defined(HAVE_SYSARCH)) && defined(HAVE_X86_SYSARCH_H)
55 #include <x86/sysarch.h>
56 #endif
57 #endif
58
59 code_return_t (*codegen_entry)(frame_s *, struct cg_upcall_vector_s *, tick_stamp_t, void *);
60 static void *codegen_ptr;
61 static size_t codegen_size;
62
63 #ifdef DEBUG_ENV
64 static mutex_t dump_mutex;
65 static uint64_t dump_seq = 0;
66 #endif
67
68 /*
69 * insn:
70 * opcode - 16 bits
71 * op_size - 3 bits
72 * aux - 7 bits
73 * writes flags - 2 bit
74 * jmp size - 2 bits
75 */
76
77 #define INSN_OPCODE 0x0000ffffUL
78 #define INSN_OP_SIZE 0x00070000UL
79 #define INSN_AUX 0x03f80000UL
80 #define INSN_WRITES_FLAGS 0x0c000000UL
81 #define INSN_JUMP_SIZE 0x30000000UL
82
83 #define INSN_OPCODE_SHIFT 0
84 #define INSN_OP_SIZE_SHIFT 16
85 #define INSN_AUX_SHIFT 19
86 #define INSN_WRITES_FLAGS_SHIFT 26
87 #define INSN_JUMP_SIZE_SHIFT 28
88
89 #define insn_opcode(insn) (((insn) >> INSN_OPCODE_SHIFT) & (INSN_OPCODE >> INSN_OPCODE_SHIFT))
90 #define insn_op_size(insn) (((insn) >> INSN_OP_SIZE_SHIFT) & (INSN_OP_SIZE >> INSN_OP_SIZE_SHIFT))
91 #define insn_aux(insn) (((insn) >> INSN_AUX_SHIFT) & (INSN_AUX >> INSN_AUX_SHIFT))
92 #define insn_writes_flags(insn) (((insn) >> INSN_WRITES_FLAGS_SHIFT) & (INSN_WRITES_FLAGS >> INSN_WRITES_FLAGS_SHIFT))
93 #define insn_jump_size(insn) (((insn) >> INSN_JUMP_SIZE_SHIFT) & (INSN_JUMP_SIZE >> INSN_JUMP_SIZE_SHIFT))
94
95 #define ALU_ADD 0x00
96 #define ALU_OR 0x01
97 #define ALU_ADC 0x02
98 #define ALU_SBB 0x03
99 #define ALU_AND 0x04
100 #define ALU_SUB 0x05
101 #define ALU_XOR 0x06
102 #define ALU_ORN 0x08
103 #define ALU_ANDN 0x09
104 #define ALU_XORN 0x0a
105 #define ALU_MUL 0x10
106 #define ALU_UMULH 0x11
107 #define ALU_SMULH 0x12
108 #define ALU_UDIV 0x13
109 #define ALU_SDIV 0x14
110 #define ALU_UREM 0x15
111 #define ALU_SREM 0x16
112 #define ALU_SAVE 0x17
113 #define ALU_EXTBL 0x18
114 #define ALU_EXTWL 0x19
115 #define ALU_EXTLL 0x1a
116 #define ALU_EXTLH 0x1b
117 #define ALU_INSBL 0x1c
118 #define ALU_MSKBL 0x1d
119 #define ALU_ZAP 0x20
120 #define ALU_ZAPNOT 0x21
121
122 #define ALU1_NOT 0x00
123 #define ALU1_NEG 0x01
124 #define ALU1_NGC 0x02
125 #define ALU1_INC 0x03
126 #define ALU1_DEC 0x04
127 #define ALU1_BSWAP 0x05
128 #define ALU1_BSWAP16 0x06
129 #define ALU1_BREV 0x07
130 #define ALU1_BSF 0x08
131 #define ALU1_BSR 0x09
132 #define ALU1_LZCNT 0x0a
133 #define ALU1_POPCNT 0x0b
134
135 #define FP_ALU_ADD 0
136 #define FP_ALU_SUB 1
137 #define FP_ALU_MUL 2
138 #define FP_ALU_DIV 3
139 #define FP_ALU1_NEG 0
140 #define FP_ALU1_SQRT 1
141 #define FP_ALU1_ROUND 2
142 #define FP_ALU1_FLOOR 3
143 #define FP_ALU1_CEIL 4
144 #define FP_ALU1_TRUNC 5
145 #define FP_ALU1_VCNT8 6
146 #define FP_ALU1_VPADDL 7
147 #define FP_ALU1_ADDV 8
148
149 #define COND_O 0x0
150 #define COND_NO 0x1
151 #define COND_B 0x2
152 #define COND_AE 0x3
153 #define COND_E 0x4
154 #define COND_NE 0x5
155 #define COND_BE 0x6
156 #define COND_A 0x7
157 #define COND_S 0x8
158 #define COND_NS 0x9
159 #define COND_P 0xa
160 #define COND_NP 0xb
161 #define COND_L 0xc
162 #define COND_GE 0xd
163 #define COND_LE 0xe
164 #define COND_G 0xf
165 #define COND_BLBC 0x10
166 #define COND_BLBS 0x11
167 #define COND_ALWAYS 0x12
168
169 #define COND_FP 0x20
170 #define FP_COND_P (COND_FP | COND_P)
171 #define FP_COND_NP (COND_FP | COND_NP)
172 #define FP_COND_E (COND_FP | COND_E)
173 #define FP_COND_NE (COND_FP | COND_NE)
174 #define FP_COND_A (COND_FP | COND_A)
175 #define FP_COND_BE (COND_FP | COND_BE)
176 #define FP_COND_B (COND_FP | COND_B)
177 #define FP_COND_AE (COND_FP | COND_AE)
178
179 #define ROT_ROL 0x0
180 #define ROT_ROR 0x1
181 #define ROT_RCL 0x2
182 #define ROT_RCR 0x3
183 #define ROT_SHL 0x4
184 #define ROT_SHR 0x5
185 #define ROT_SAR 0x7
186 #define ROT_SAL 0x8
187
188 #define BTX_BT 0x0
189 #define BTX_BTS 0x1
190 #define BTX_BTR 0x2
191 #define BTX_BTC 0x3
192 #define BTX_BTEXT 0x4
193
194 #define OP_SIZE_1 0
195 #define OP_SIZE_2 1
196 #define OP_SIZE_4 2
197 #define OP_SIZE_8 3
198 #define OP_SIZE_16 4
199 #define OP_SIZE_10 7
200
201 #define MOV_MASK_0_16 0x0
202 #define MOV_MASK_16_32 0x1
203 #define MOV_MASK_32_48 0x2
204 #define MOV_MASK_48_64 0x3
205 #define MOV_MASK_0_8 0x4
206 #define MOV_MASK_32_64 0x5
207 #define MOV_MASK_52_64 0x6
208
209 #define JMP_SHORTEST 0x0000
210 #define JMP_SHORT 0x0001
211 #define JMP_LONG 0x0002
212 #define JMP_EXTRA_LONG 0x0003
213
214 enum {
215 INSN_ENTRY,
216 INSN_LABEL,
217 INSN_RET,
218 INSN_RET_IMM,
219 INSN_ARM_PUSH,
220 INSN_ARM_POP,
221 INSN_S390_PUSH,
222 INSN_S390_POP,
223 INSN_IA64_ALLOC,
224 INSN_IA64_DEALLOC,
225 INSN_PUSH,
226 INSN_POP,
227 INSN_CALL,
228 INSN_CALL_INDIRECT,
229 INSN_MOV,
230 INSN_MOVSX,
231 INSN_MOV_U,
232 INSN_MOV_LR,
233 INSN_CMP,
234 INSN_CMP_DEST_REG,
235 INSN_CMN,
236 INSN_TEST,
237 INSN_TEST_DEST_REG,
238 INSN_ALU,
239 INSN_ALU_PARTIAL,
240 INSN_ALU_FLAGS,
241 INSN_ALU_FLAGS_PARTIAL,
242 INSN_ALU_TRAP,
243 INSN_ALU_FLAGS_TRAP,
244 INSN_ALU1,
245 INSN_ALU1_PARTIAL,
246 INSN_ALU1_FLAGS,
247 INSN_ALU1_FLAGS_PARTIAL,
248 INSN_ALU1_TRAP,
249 INSN_LEA3,
250 INSN_ROT,
251 INSN_ROT_PARTIAL,
252 INSN_BT,
253 INSN_BTX,
254 INSN_MUL_L,
255 INSN_DIV_L,
256 INSN_MADD,
257 INSN_CBW,
258 INSN_CBW_PARTIAL,
259 INSN_CWD,
260 INSN_CWD_PARTIAL,
261 INSN_SET_COND,
262 INSN_SET_COND_PARTIAL,
263 INSN_CMOV,
264 INSN_CMOV_XCC,
265 INSN_MOVR,
266 INSN_CSEL_SEL,
267 INSN_CSEL_INC,
268 INSN_CSEL_INV,
269 INSN_CSEL_NEG,
270 INSN_STP,
271 INSN_LDP,
272 INSN_MOV_MASK,
273 INSN_MEMCPY,
274 INSN_MEMSET,
275 INSN_FP_CMP,
276 INSN_FP_CMP_DEST_REG,
277 INSN_FP_CMP_DEST_REG_TRAP,
278 INSN_FP_CMP_UNORDERED_DEST_REG,
279 INSN_FP_CMP_COND,
280 INSN_FP_TEST_REG,
281 INSN_FP_TO_INT_FLAGS,
282 INSN_FP_ALU,
283 INSN_FP_ALU1,
284 INSN_FP_TO_INT32,
285 INSN_FP_TO_INT64,
286 INSN_FP_TO_INT64_TRAP,
287 INSN_FP_FROM_INT32,
288 INSN_FP_FROM_INT64,
289 INSN_FP_INT64_TO_INT32_TRAP,
290 INSN_FP_CVT,
291 INSN_X87_FLD,
292 INSN_X87_FILD,
293 INSN_X87_FSTP,
294 INSN_X87_FISTP,
295 INSN_X87_FISTTP,
296 INSN_X87_FCOMP,
297 INSN_X87_FCOMPP,
298 INSN_X87_FCOMIP,
299 INSN_X87_ALU,
300 INSN_X87_ALUP,
301 INSN_X87_FCHS,
302 INSN_X87_FSQRT,
303 INSN_X87_FRNDINT,
304 INSN_X87_FNSTSW,
305 INSN_X87_FLDCW,
306 INSN_JMP,
307 INSN_JMP_COND,
308 INSN_JMP_COND_LOGICAL,
309 INSN_JMP_REG,
310 INSN_JMP_REG_BIT,
311 INSN_JMP_2REGS,
312 INSN_JMP_FP_TEST,
313 INSN_JMP_INDIRECT,
314 INSN_MB,
315 INSN_CALL_MILLICODE,
316 };
317
318 #define ARG_REGS_MAX 0xc0
319 #define ARG_SHIFTED_REGISTER 0xc0
320 #define ARG_SHIFT_AMOUNT 0x3f
321 #define ARG_SHIFT_MODE 0xc0
322 #define ARG_SHIFT_LSL 0x00
323 #define ARG_SHIFT_LSR 0x40
324 #define ARG_SHIFT_ASR 0x80
325 #define ARG_SHIFT_ROR 0xc0
326 #define ARG_EXTENDED_REGISTER 0xc1
327 #define ARG_EXTEND_SHIFT 0x07
328 #define ARG_EXTEND_MODE 0x38
329 #define ARG_EXTEND_UXTB 0x00
330 #define ARG_EXTEND_UXTH 0x08
331 #define ARG_EXTEND_UXTW 0x10
332 #define ARG_EXTEND_UXTX 0x18
333 #define ARG_EXTEND_SXTB 0x20
334 #define ARG_EXTEND_SXTH 0x28
335 #define ARG_EXTEND_SXTW 0x30
336 #define ARG_EXTEND_SXTX 0x38
337 #define ARG_ADDRESS_0 0xd0
338 #define ARG_ADDRESS_1 0xd1
339 #define ARG_ADDRESS_1_2 0xd2
340 #define ARG_ADDRESS_1_4 0xd3
341 #define ARG_ADDRESS_1_8 0xd4
342 #define ARG_ADDRESS_1_PRE_I 0xd5
343 #define ARG_ADDRESS_1_POST_I 0xd6
344 #define ARG_ADDRESS_2 0xd7
345 #define ARG_ADDRESS_2_2 0xd8
346 #define ARG_ADDRESS_2_4 0xd9
347 #define ARG_ADDRESS_2_8 0xda
348 #define ARG_ADDRESS_2_UXTW 0xdb
349 #define ARG_ADDRESS_2_SXTW 0xdc
350 #define ARG_IMM 0xe0
351
352 #define ARG_IS_ADDRESS(a) ((a) >= ARG_ADDRESS_0 && (a) <= ARG_ADDRESS_2_SXTW)
353
354 #ifdef POINTER_COMPRESSION
355 #define OP_SIZE_SLOT OP_SIZE_4
356 #else
357 #define OP_SIZE_SLOT OP_SIZE_ADDRESS
358 #endif
359
360 #define OP_SIZE_BITMAP (bitmap_64bit ? OP_SIZE_8 : OP_SIZE_4)
361
362 #define OP_SIZE_INT log_2(sizeof(int_default_t))
363
364 #define check_insn(insn) \
365 do { \
366 /*if ((insn_opcode(insn) == INSN_ALU || insn_opcode(insn) == INSN_ALU1) && insn_op_size(insn) != OP_SIZE_NATIVE) internal(file_line, "invalid insn %08x", (unsigned)(insn));*/\
367 /*if (insn == 0x001a000e) internal(file_line, "invalid insn %08x", insn);*/\
368 } while (0)
369
370 #ifdef DEBUG_INSNS
371 #define gen_line() gen_four(__LINE__)
372 #else
373 #define gen_line() do { } while (0)
374 #endif
375
376 #ifdef ARCH_IA64
377 #define ARCH_CONTEXT struct { \
378 uint64_t insns[3]; \
379 uint8_t insn_units[3]; \
380 bool insn_stops[3]; \
381 uint64_t wr_mask[4]; \
382 }
383 #endif
384
385 #define gen_insn(opcode, op_size, aux, writes_flags) \
386 do { \
387 uint32_t dword = \
388 (uint32_t)(opcode) << INSN_OPCODE_SHIFT | \
389 (uint32_t)(op_size) << INSN_OP_SIZE_SHIFT | \
390 (uint32_t)(aux) << INSN_AUX_SHIFT | \
391 (uint32_t)(writes_flags) << INSN_WRITES_FLAGS_SHIFT; \
392 check_insn(dword); \
393 gen_line(); \
394 gen_four(dword); \
395 } while (0)
396
397 static size_t arg_size(uint8_t arg)
398 {
399 if (arg < ARG_REGS_MAX)
400 return 1;
401 if (arg >= ARG_SHIFTED_REGISTER && arg <= ARG_EXTENDED_REGISTER)
402 return 3;
403 if (arg == ARG_ADDRESS_0)
404 return 9;
405 if (arg >= ARG_ADDRESS_1 && arg <= ARG_ADDRESS_1_POST_I)
406 return 10;
407 if (arg >= ARG_ADDRESS_2 && arg <= ARG_ADDRESS_2_SXTW)
408 return 11;
409 if (arg == ARG_IMM)
410 return 9;
411 internal(file_line, "arg_size: invalid argument %02x", arg);
412 return 0;
413 }
414
415 struct relocation {
416 uint32_t label_id;
417 uint8_t length;
418 size_t position;
419 size_t jmp_instr;
420 };
421
422 struct code_arg {
423 frame_t slot;
424 frame_t flags;
425 frame_t type;
426 };
427
428 struct cg_entry {
429 size_t entry_to_pos;
430 frame_t *variables;
431 size_t n_variables;
432 uint32_t entry_label;
433 uint32_t nonflat_label;
434 };
435
436 struct cg_exit {
437 uint32_t undo_label;
438 uint8_t undo_opcode;
439 uint8_t undo_op_size;
440 uint8_t undo_aux;
441 uint8_t undo_writes_flags;
442 uint8_t undo_parameters[35];
443 uint8_t undo_parameters_len;
444 uint32_t escape_label;
445 };
446
447 #define FLAG_CACHE_IS_FLAT 0x01
448 #define FLAG_CACHE_IS_NOT_FLAT 0x02
449 #define FLAG_CACHE_IS_NOT_THUNK 0x04
450
451 struct codegen_context {
452 struct data *fn;
453 struct data **local_directory;
454
455 const code_t *instr_start;
456 const code_t *current_position;
457 uchar_efficient_t arg_mode;
458
459 uint32_t label_id;
460 struct cg_entry *entries;
461 frame_t n_entries;
462
463 uint8_t *code;
464 size_t code_size;
465
466 uint8_t *code_position;
467
468 uint32_t *code_labels;
469 struct cg_exit **code_exits;
470 uint32_t escape_nospill_label;
471 uint32_t call_label;
472 uint32_t reload_label;
473
474 uint8_t *mcode;
475 size_t mcode_size;
476
477 size_t *label_to_pos;
478 struct relocation *reloc;
479 size_t reloc_size;
480
481 struct trap_record *trap_records;
482 size_t trap_records_size;
483
484 struct code_arg *args;
485 size_t args_l;
486 const code_t *return_values;
487
488 uint8_t *flag_cache;
489 short *registers;
490 frame_t *need_spill;
491 size_t need_spill_l;
492
493 unsigned base_reg;
494 bool offset_reg;
495 int64_t offset_imm;
496
497 bool const_reg;
498 int64_t const_imm;
499
500 struct data *codegen;
501
502 int upcall_args;
503 frame_t *var_aux;
504
505 ajla_error_t err;
506
507 #ifdef ARCH_CONTEXT
508 ARCH_CONTEXT a;
509 #endif
510 };
511
512 static void init_ctx(struct codegen_context *ctx)
513 {
514 ctx->local_directory = NULL;
515 ctx->label_id = 0;
516 ctx->entries = NULL;
517 ctx->n_entries = 0;
518 ctx->code = NULL;
519 ctx->code_labels = NULL;
520 ctx->code_exits = NULL;
521 ctx->escape_nospill_label = 0;
522 ctx->call_label = 0;
523 ctx->reload_label = 0;
524 ctx->mcode = NULL;
525 ctx->label_to_pos = NULL;
526 ctx->reloc = NULL;
527 ctx->trap_records = NULL;
528 ctx->args = NULL;
529 ctx->flag_cache = NULL;
530 ctx->registers = NULL;
531 ctx->need_spill = NULL;
532 ctx->codegen = NULL;
533 ctx->upcall_args = -1;
534 ctx->var_aux = NULL;
535 }
536
537 static void done_ctx(struct codegen_context *ctx)
538 {
539 if (ctx->local_directory)
540 mem_free(ctx->local_directory);
541 if (ctx->entries) {
542 size_t i;
543 for (i = 0; i < ctx->n_entries; i++) {
544 struct cg_entry *ce = &ctx->entries[i];
545 if (ce->variables)
546 mem_free(ce->variables);
547 }
548 mem_free(ctx->entries);
549 }
550 if (ctx->code)
551 mem_free(ctx->code);
552 if (ctx->code_labels)
553 mem_free(ctx->code_labels);
554 if (ctx->code_exits) {
555 ip_t ip;
556 ip_t cs = da(ctx->fn,function)->code_size;
557 for (ip = 0; ip < cs; ip++) {
558 if (ctx->code_exits[ip])
559 mem_free(ctx->code_exits[ip]);
560 }
561 mem_free(ctx->code_exits);
562 }
563 if (ctx->mcode)
564 mem_free(ctx->mcode);
565 if (ctx->label_to_pos)
566 mem_free(ctx->label_to_pos);
567 if (ctx->reloc)
568 mem_free(ctx->reloc);
569 if (ctx->trap_records)
570 mem_free(ctx->trap_records);
571 if (ctx->args)
572 mem_free(ctx->args);
573 if (ctx->flag_cache)
574 mem_free(ctx->flag_cache);
575 if (ctx->registers)
576 mem_free(ctx->registers);
577 if (ctx->need_spill)
578 mem_free(ctx->need_spill);
579 if (ctx->codegen)
580 data_free(ctx->codegen);
581 if (ctx->var_aux)
582 mem_free(ctx->var_aux);
583 }
584
585
586 static inline code_t get_code(struct codegen_context *ctx)
587 {
588 ajla_assert(ctx->current_position < da(ctx->fn,function)->code + da(ctx->fn,function)->code_size, (file_line, "get_code: ran out of code"));
589 return *ctx->current_position++;
590 }
591
592 static inline uint32_t get_uint32(struct codegen_context *ctx)
593 {
594 uint32_t a1 = get_code(ctx);
595 uint32_t a2 = get_code(ctx);
596 #if !CODE_ENDIAN
597 return a1 + (a2 << 16);
598 #else
599 return a2 + (a1 << 16);
600 #endif
601 }
602
603 static int32_t get_jump_offset(struct codegen_context *ctx)
604 {
605 if (SIZEOF_IP_T == 2) {
606 return (int32_t)(int16_t)get_code(ctx);
607 } else if (SIZEOF_IP_T == 4) {
608 return (int32_t)get_uint32(ctx);
609 } else {
610 not_reached();
611 return -1;
612 }
613 }
614
615 static inline void get_one(struct codegen_context *ctx, frame_t *v)
616 {
617 if (!ctx->arg_mode) {
618 code_t c = get_code(ctx);
619 ajla_assert(!(c & ~0xff), (file_line, "get_one: high byte is not cleared: %u", (unsigned)c));
620 *v = c & 0xff;
621 } else if (ctx->arg_mode == 1) {
622 *v = get_code(ctx);
623 #if ARG_MODE_N >= 2
624 } else if (ctx->arg_mode == 2) {
625 *v = get_uint32(ctx);
626 #endif
627 } else {
628 internal(file_line, "get_one: invalid arg mode %u", ctx->arg_mode);
629 }
630 }
631
632 static inline void get_two(struct codegen_context *ctx, frame_t *v1, frame_t *v2)
633 {
634 if (!ctx->arg_mode) {
635 code_t c = get_code(ctx);
636 *v1 = c & 0xff;
637 *v2 = c >> 8;
638 } else if (ctx->arg_mode == 1) {
639 *v1 = get_code(ctx);
640 *v2 = get_code(ctx);
641 #if ARG_MODE_N >= 2
642 } else if (ctx->arg_mode == 2) {
643 *v1 = get_uint32(ctx);
644 *v2 = get_uint32(ctx);
645 #endif
646 } else {
647 internal(file_line, "get_two: invalid arg mode %u", ctx->arg_mode);
648 }
649 }
650
651
652 static uint32_t alloc_label(struct codegen_context *ctx)
653 {
654 return ++ctx->label_id;
655 }
656
657 static struct cg_exit *alloc_cg_exit_for_ip(struct codegen_context *ctx, const code_t *code)
658 {
659 ip_t ip = code - da(ctx->fn,function)->code;
660 struct cg_exit *ce = ctx->code_exits[ip];
661 if (!ce) {
662 ce = mem_calloc_mayfail(struct cg_exit *, sizeof(struct cg_exit), &ctx->err);
663 if (unlikely(!ce))
664 return NULL;
665 ctx->code_exits[ip] = ce;
666 }
667 return ce;
668 }
669
670 static struct cg_exit *alloc_undo_label(struct codegen_context *ctx)
671 {
672 struct cg_exit *ce = alloc_cg_exit_for_ip(ctx, ctx->instr_start);
673 if (unlikely(!ce))
674 return NULL;
675 if (unlikely(ce->undo_label != 0))
676 internal(file_line, "alloc_cg_exit: undo label already allocated");
677 ce->undo_label = alloc_label(ctx);
678 if (unlikely(!ce->undo_label))
679 return NULL;
680 return ce;
681 }
682
683 static uint32_t alloc_escape_label_for_ip(struct codegen_context *ctx, const code_t *code)
684 {
685 struct cg_exit *ce = alloc_cg_exit_for_ip(ctx, code);
686 if (!ce)
687 return 0;
688 if (!ce->escape_label)
689 ce->escape_label = alloc_label(ctx);
690 return ce->escape_label;
691 }
692
693 static uint32_t alloc_escape_label(struct codegen_context *ctx)
694 {
695 return alloc_escape_label_for_ip(ctx, ctx->instr_start);
696 }
697
698 static uint32_t attr_unused alloc_call_label(struct codegen_context *ctx)
699 {
700 if (!ctx->call_label) {
701 ctx->call_label = alloc_label(ctx);
702 }
703 return ctx->call_label;
704 }
705
706 static uint32_t alloc_reload_label(struct codegen_context *ctx)
707 {
708 if (!ctx->reload_label) {
709 ctx->reload_label = alloc_label(ctx);
710 }
711 return ctx->reload_label;
712 }
713
714 static size_t attr_unused mark_params(struct codegen_context *ctx)
715 {
716 return ctx->code_size;
717 }
718
719 static void attr_unused copy_params(struct codegen_context *ctx, struct cg_exit *ce, size_t mark)
720 {
721 if (ctx->code_size - mark > n_array_elements(ce->undo_parameters))
722 internal(file_line, "undo_parameters is too small: %"PRIuMAX" > %"PRIuMAX"", (uintmax_t)(ctx->code_size - mark), (uintmax_t)n_array_elements(ce->undo_parameters));
723 memcpy(ce->undo_parameters, ctx->code + mark, ctx->code_size - mark);
724 ce->undo_parameters_len = ctx->code_size - mark;
725 ctx->code_size = mark;
726 }
727
728 #define g(call) \
729 do { \
730 if (unlikely(!call)) \
731 return false; \
732 } while (0)
733
734 #define gen_one(byte) \
735 do { \
736 /*debug("gen %d: %02x", __LINE__, (uint8_t)(byte))*/; \
737 if (unlikely(!array_add_mayfail(uint8_t, &ctx->code, &ctx->code_size, byte, NULL, &ctx->err)))\
738 return false; \
739 } while (0)
740
741 #if defined(C_LITTLE_ENDIAN)
742 #define gen_two(word) \
743 do { \
744 uint16_t word_ = (word); \
745 /*debug("gen %d: %04x", __LINE__, (uint16_t)(word_));*/ \
746 if (unlikely(!array_add_multiple_mayfail(uint8_t, &ctx->code, &ctx->code_size, cast_ptr(uint8_t *, &word_), 2, NULL, &ctx->err)))\
747 return false; \
748 } while (0)
749 #define gen_four(dword) \
750 do { \
751 uint32_t dword_ = (dword); \
752 /*debug("gen %d: %08x", __LINE__, (uint32_t)(dword_));*/ \
753 if (unlikely(!array_add_multiple_mayfail(uint8_t, &ctx->code, &ctx->code_size, cast_ptr(uint8_t *, &dword_), 4, NULL, &ctx->err)))\
754 return false; \
755 } while (0)
756 #define gen_eight(qword) \
757 do { \
758 uint64_t qword_ = (qword); \
759 /*debug("gen %d: %016lx", __LINE__, (uint64_t)(qword_));*/ \
760 if (unlikely(!array_add_multiple_mayfail(uint8_t, &ctx->code, &ctx->code_size, cast_ptr(uint8_t *, &qword_), 8, NULL, &ctx->err)))\
761 return false; \
762 } while (0)
763 #else
764 #define gen_two(word) \
765 do { \
766 uint16_t word_ = (word); \
767 gen_one(word_ & 0xffU); \
768 gen_one(word_ >> 8); \
769 } while (0)
770 #define gen_four(dword) \
771 do { \
772 uint32_t dword_ = (dword); \
773 gen_two(dword_ & 0xffffU); \
774 gen_two(dword_ >> 15 >> 1); \
775 } while (0)
776 #define gen_eight(qword) \
777 do { \
778 uint64_t qword_ = (qword); \
779 gen_four(qword_ & 0xffffffffUL); \
780 gen_four(qword_ >> 15 >> 15 >> 2); \
781 } while (0)
782 #endif
783
784 #define gen_label(label_id) \
785 do { \
786 gen_insn(INSN_LABEL, 0, 0, 0); \
787 gen_four(label_id); \
788 } while (0)
789
790
791 static uint8_t attr_unused cget_one(struct codegen_context *ctx)
792 {
793 ajla_assert(ctx->code_position < ctx->code + ctx->code_size, (file_line, "cget_one: ran out of code"));
794 return *ctx->code_position++;
795 }
796
797 static uint16_t attr_unused cget_two(struct codegen_context *ctx)
798 {
799 #if defined(C_LITTLE_ENDIAN)
800 uint16_t r;
801 ajla_assert(ctx->code_position < ctx->code + ctx->code_size, (file_line, "cget_two: ran out of code"));
802 memcpy(&r, ctx->code_position, 2);
803 ctx->code_position += 2;
804 return r;
805 #else
806 uint16_t r = cget_one(ctx);
807 r |= cget_one(ctx) << 8;
808 return r;
809 #endif
810 }
811
812 static uint32_t cget_four(struct codegen_context *ctx)
813 {
814 #if defined(C_LITTLE_ENDIAN)
815 uint32_t r;
816 ajla_assert(ctx->code_position < ctx->code + ctx->code_size, (file_line, "cget_four: ran out of code"));
817 memcpy(&r, ctx->code_position, 4);
818 ctx->code_position += 4;
819 return r;
820 #else
821 uint32_t r = cget_two(ctx);
822 r |= (uint32_t)cget_two(ctx) << 16;
823 return r;
824 #endif
825 }
826
827 static uint64_t attr_unused cget_eight(struct codegen_context *ctx)
828 {
829 #if defined(C_LITTLE_ENDIAN)
830 uint64_t r;
831 ajla_assert(ctx->code_position < ctx->code + ctx->code_size, (file_line, "cget_eight: ran out of code"));
832 memcpy(&r, ctx->code_position, 8);
833 ctx->code_position += 8;
834 return r;
835 #else
836 uint64_t r = cget_four(ctx);
837 r |= (uint64_t)cget_four(ctx) << 32;
838 return r;
839 #endif
840 }
841
842 static int64_t get_imm(uint8_t *ptr)
843 {
844 #if defined(C_LITTLE_ENDIAN)
845 int64_t r;
846 memcpy(&r, ptr, 8);
847 return r;
848 #else
849 int64_t r;
850 r = (uint64_t)ptr[0] |
851 ((uint64_t)ptr[1] << 8) |
852 ((uint64_t)ptr[2] << 16) |
853 ((uint64_t)ptr[3] << 24) |
854 ((uint64_t)ptr[4] << 32) |
855 ((uint64_t)ptr[5] << 40) |
856 ((uint64_t)ptr[6] << 48) |
857 ((uint64_t)ptr[7] << 56);
858 return r;
859 #endif
860 }
861
862 #define cgen_one(byte) \
863 do { \
864 if (unlikely(!array_add_mayfail(uint8_t, &ctx->mcode, &ctx->mcode_size, byte, NULL, &ctx->err)))\
865 return false; \
866 } while (0)
867
868 #if defined(C_LITTLE_ENDIAN) || 1
869 #define cgen_two(word) \
870 do { \
871 uint16_t word_ = (word); \
872 if (unlikely(!array_add_multiple_mayfail(uint8_t, &ctx->mcode, &ctx->mcode_size, cast_ptr(uint8_t *, &word_), 2, NULL, &ctx->err)))\
873 return false; \
874 } while (0)
875 #define cgen_four(dword) \
876 do { \
877 uint32_t dword_ = (dword); \
878 /*if (dword_ == 0x1ee02000) internal(file_line, "invalid instruction");*/\
879 if (unlikely(!array_add_multiple_mayfail(uint8_t, &ctx->mcode, &ctx->mcode_size, cast_ptr(uint8_t *, &dword_), 4, NULL, &ctx->err)))\
880 return false; \
881 } while (0)
882 #define cgen_eight(qword) \
883 do { \
884 uint64_t qword_ = (qword); \
885 if (unlikely(!array_add_multiple_mayfail(uint8_t, &ctx->mcode, &ctx->mcode_size, cast_ptr(uint8_t *, &qword_), 8, NULL, &ctx->err)))\
886 return false; \
887 } while (0)
888 #else
889 #define cgen_two(word) \
890 do { \
891 cgen_one((word) & 0xff); \
892 cgen_one((word) >> 8); \
893 } while (0)
894 #define cgen_four(dword) \
895 do { \
896 cgen_two((dword) & 0xffff); \
897 cgen_two((dword) >> 15 >> 1); \
898 } while (0)
899 #define cgen_eight(qword) \
900 do { \
901 cgen_four((qword) & 0xffffffff); \
902 cgen_four((qword) >> 15 >> 15 >> 2); \
903 } while (0)
904 #endif
905
906
907 #define IMM_PURPOSE_LDR_OFFSET 1
908 #define IMM_PURPOSE_LDR_SX_OFFSET 2
909 #define IMM_PURPOSE_STR_OFFSET 3
910 #define IMM_PURPOSE_LDP_STP_OFFSET 4
911 #define IMM_PURPOSE_VLDR_VSTR_OFFSET 5
912 #define IMM_PURPOSE_MVI_CLI_OFFSET 6
913 #define IMM_PURPOSE_STORE_VALUE 7
914 #define IMM_PURPOSE_ADD 8
915 #define IMM_PURPOSE_SUB 9
916 #define IMM_PURPOSE_CMP 10
917 #define IMM_PURPOSE_CMP_LOGICAL 11
918 #define IMM_PURPOSE_AND 12
919 #define IMM_PURPOSE_OR 13
920 #define IMM_PURPOSE_XOR 14
921 #define IMM_PURPOSE_ANDN 15
922 #define IMM_PURPOSE_TEST 16
923 #define IMM_PURPOSE_JMP_2REGS 17
924 #define IMM_PURPOSE_MUL 18
925 #define IMM_PURPOSE_CMOV 19
926 #define IMM_PURPOSE_MOVR 20
927 #define IMM_PURPOSE_BITWISE 21
928
929
930 static bool attr_w gen_upcall_end(struct codegen_context *ctx, unsigned args);
931
932 #define gen_address_offset() \
933 do { \
934 if (likely(!ctx->offset_reg)) { \
935 gen_one(ARG_ADDRESS_1); \
936 gen_one(ctx->base_reg); \
937 gen_eight(ctx->offset_imm); \
938 } else { \
939 gen_one(ARG_ADDRESS_2); \
940 gen_one(ctx->base_reg); \
941 gen_one(R_OFFSET_IMM); \
942 gen_eight(0); \
943 } \
944 } while (0)
945
946 #define gen_imm_offset() \
947 do { \
948 if (likely(!ctx->const_reg)) { \
949 gen_one(ARG_IMM); \
950 gen_eight(ctx->const_imm); \
951 } else { \
952 gen_one(R_CONST_IMM); \
953 } \
954 } while (0)
955
956 #define is_imm() (!ctx->const_reg)
957
958
959 #if defined(ARCH_ALPHA)
960 #include "c1-alpha.inc"
961 #elif defined(ARCH_ARM32)
962 #include "c1-arm.inc"
963 #elif defined(ARCH_ARM64)
964 #include "c1-arm64.inc"
965 #elif defined(ARCH_IA64)
966 #include "c1-ia64.inc"
967 #elif defined(ARCH_LOONGARCH64)
968 #include "c1-loong.inc"
969 #elif defined(ARCH_MIPS)
970 #include "c1-mips.inc"
971 #elif defined(ARCH_PARISC)
972 #include "c1-hppa.inc"
973 #elif defined(ARCH_POWER)
974 #include "c1-power.inc"
975 #elif defined(ARCH_S390)
976 #include "c1-s390.inc"
977 #elif defined(ARCH_SPARC)
978 #include "c1-sparc.inc"
979 #elif defined(ARCH_RISCV64)
980 #include "c1-riscv.inc"
981 #elif defined(ARCH_X86)
982 #include "c1-x86.inc"
983 #endif
984
985
986 #ifndef ARCH_SUPPORTS_TRAPS
987 #define ARCH_SUPPORTS_TRAPS 0
988 #define ARCH_TRAP_BEFORE 0
989 #endif
990
991
992 #include "cg-util.inc"
993
994 #include "cg-frame.inc"
995
996 #include "cg-flags.inc"
997
998 #include "cg-flcch.inc"
999
1000 #include "cg-ptr.inc"
1001
1002 #include "cg-alu.inc"
1003
1004 #include "cg-ops.inc"
1005
1006
1007 #ifndef n_regs_saved
1008 #define n_regs_saved n_array_elements(regs_saved)
1009 #endif
1010
1011 #ifndef n_regs_volatile
1012 #define n_regs_volatile n_array_elements(regs_volatile)
1013 #endif
1014
1015 #ifndef n_fp_saved
1016 #define n_fp_saved n_array_elements(fp_saved)
1017 #endif
1018
1019 #ifndef n_fp_volatile
1020 #define n_fp_volatile n_array_elements(fp_volatile)
1021 #endif
1022
1023 #ifndef n_vector_volatile
1024 #define n_vector_volatile n_array_elements(vector_volatile)
1025 #endif
1026
1027 static bool attr_w gen_registers(struct codegen_context *ctx)
1028 {
1029 frame_t v;
1030 size_t index_saved = 0;
1031 size_t index_volatile = 0;
1032 size_t index_fp_saved = 0;
1033 size_t index_fp_volatile = 0;
1034 size_t attr_unused index_vector_volatile = 0;
1035 #ifdef ARCH_S390
1036 bool uses_x = false;
1037 for (v = MIN_USEABLE_SLOT; v < function_n_variables(ctx->fn); v++) {
1038 const struct type *t = get_type_of_local(ctx, v);
1039 if (t && TYPE_TAG_IS_REAL(t->tag) && TYPE_TAG_IDX_REAL(t->tag) == 4) {
1040 uses_x = true;
1041 break;
1042 }
1043 }
1044 #endif
1045 /*for (v = function_n_variables(ctx->fn) - 1; v >= MIN_USEABLE_SLOT; v--)*/
1046 for (v = MIN_USEABLE_SLOT; v < function_n_variables(ctx->fn); v++) {
1047 const struct type *t;
1048 ctx->registers[v] = -1;
1049 if (ra_chicken)
1050 continue;
1051 t = get_type_of_local(ctx, v);
1052 if (unlikely(!t))
1053 continue;
1054 if (!da(ctx->fn,function)->local_variables_flags[v].must_be_flat)
1055 continue;
1056 if (!ARCH_HAS_BWX && t->size < 1U << OP_SIZE_4)
1057 continue;
1058 if (TYPE_TAG_IS_FIXED(t->tag) || TYPE_TAG_IS_INT(t->tag) || t->tag == TYPE_TAG_flat_option) {
1059 if (!is_power_of_2(t->size) || t->size > 1U << OP_SIZE_NATIVE)
1060 continue;
1061 if (index_saved < n_regs_saved + zero
1062 #if defined(ARCH_PARISC) || defined(ARCH_SPARC)
1063 && t->size <= 1U << OP_SIZE_ADDRESS
1064 #endif
1065 ) {
1066 ctx->registers[v] = regs_saved[index_saved++];
1067 } else if (index_volatile < n_regs_volatile + zero) {
1068 ctx->registers[v] = regs_volatile[index_volatile++];
1069 } else {
1070 continue;
1071 }
1072 } else if (TYPE_TAG_IS_REAL(t->tag)) {
1073 unsigned real_type = TYPE_TAG_IDX_REAL(t->tag);
1074 if ((SUPPORTED_FP >> real_type) & 1) {
1075 #ifdef ARCH_POWER
1076 if (real_type == 4) {
1077 if (index_vector_volatile < n_vector_volatile + zero) {
1078 ctx->registers[v] = vector_volatile[index_vector_volatile++];
1079 goto success;
1080 }
1081 continue;
1082 }
1083 #endif
1084 #ifdef ARCH_S390
1085 if (real_type == 4) {
1086 if (!(index_fp_saved & 1) && index_fp_saved + 1 < n_fp_saved + zero) {
1087 ctx->registers[v] = fp_saved[index_fp_saved++];
1088 index_fp_saved++;
1089 goto success;
1090 }
1091 if (index_fp_saved & 1 && index_fp_saved + 2 < n_fp_saved + zero) {
1092 index_fp_saved++;
1093 ctx->registers[v] = fp_saved[index_fp_saved++];
1094 index_fp_saved++;
1095 goto success;
1096 }
1097 if (!(index_fp_volatile & 1) && index_fp_volatile + 1 < n_fp_volatile + zero) {
1098 ctx->registers[v] = fp_volatile[index_fp_volatile++];
1099 index_fp_volatile++;
1100 goto success;
1101 }
1102 if (index_fp_volatile & 1 && index_fp_volatile + 2 < n_fp_volatile + zero) {
1103 index_fp_volatile++;
1104 ctx->registers[v] = fp_volatile[index_fp_volatile++];
1105 index_fp_volatile++;
1106 goto success;
1107 }
1108 continue;
1109 }
1110 #endif
1111 if (index_fp_saved < n_fp_saved + zero) {
1112 ctx->registers[v] = fp_saved[index_fp_saved++];
1113 } else if (index_fp_volatile < n_fp_volatile + zero) {
1114 ctx->registers[v] = fp_volatile[index_fp_volatile++];
1115 } else {
1116 continue;
1117 }
1118 } else {
1119 continue;
1120 }
1121 } else {
1122 continue;
1123 }
1124 goto success;
1125 success:
1126 if (!reg_is_saved(ctx->registers[v])) {
1127 if (unlikely(!array_add_mayfail(frame_t, &ctx->need_spill, &ctx->need_spill_l, v, NULL, &ctx->err)))
1128 return false;
1129 }
1130 }
1131
1132 return true;
1133 }
1134
1135 static bool attr_w gen_fused_binary(struct codegen_context *ctx, unsigned mode, unsigned op_size, unsigned op, uint32_t escape_label, frame_t slot_1, frame_t slot_2, frame_t slot_r, bool *failed)
1136 {
1137 const code_t *backup = ctx->current_position;
1138 code_t code;
1139 frame_t slot_dr, slot_test;
1140 int32_t offs_false;
1141
1142 *failed = false;
1143
1144 next_code:
1145 code = get_code(ctx);
1146 ctx->arg_mode = code / OPCODE_MODE_MULT;
1147 code %= OPCODE_MODE_MULT;
1148 ajla_assert_lo(ctx->arg_mode < ARG_MODE_N, (file_line, "gen_fused_binary: invalid opcode %04x", (unsigned)*ctx->instr_start));
1149
1150 if (code == OPCODE_DEREFERENCE) {
1151 get_one(ctx, &slot_dr);
1152 const struct type *t = get_type_of_local(ctx, slot_dr);
1153 if (!TYPE_TAG_IS_BUILTIN(t->tag)) {
1154 *failed = true;
1155 goto fail;
1156 }
1157 if (unlikely(!flag_is_clear(ctx, slot_dr))) {
1158 *failed = true;
1159 goto fail;
1160 }
1161 goto next_code;
1162 }
1163 if (code == OPCODE_DEREFERENCE_CLEAR) {
1164 *failed = true;
1165 goto fail;
1166 }
1167 if (unlikely(code != OPCODE_JMP_FALSE))
1168 internal(file_line, "gen_fused_binary: binary operation is not followed by jmp false: %x, %s", code, decode_opcode(code, true));
1169 get_one(ctx, &slot_test);
1170 if (unlikely(slot_test != slot_r))
1171 internal(file_line, "gen_fused_binary: the result of the binary operation and the tested variable do not match");
1172 offs_false = get_jump_offset(ctx);
1173 get_jump_offset(ctx);
1174
1175 if (mode == MODE_ARRAY_LEN_GT) {
1176 g(gen_array_len(ctx, slot_1, slot_2, slot_r, true, offs_false));
1177 } else if (mode == MODE_REAL) {
1178 g(gen_fp_alu_jmp(ctx, op_size, op, escape_label, slot_1, slot_2, offs_false, failed));
1179 } else {
1180 g(gen_alu_jmp(ctx, mode, op_size, op, slot_1, slot_2, offs_false, failed));
1181 }
1182
1183 fail:
1184 if (*failed)
1185 ctx->current_position = backup;
1186
1187 return true;
1188 }
1189
1190 static bool attr_w gen_function(struct codegen_context *ctx)
1191 {
1192 ctx->current_position = da(ctx->fn,function)->code;
1193
1194 ctx->escape_nospill_label = alloc_label(ctx);
1195 if (unlikely(!ctx->escape_nospill_label))
1196 return false;
1197
1198 while (ctx->current_position != da(ctx->fn,function)->code + da(ctx->fn,function)->code_size) {
1199 ip_t ip;
1200 code_t code;
1201 unsigned op, type;
1202 frame_t slot_1, slot_2, slot_3, slot_r, flags, fn_idx, opt;
1203 arg_t n_args, n_ret, i_arg;
1204 uint32_t label_id;
1205 uint32_t escape_label;
1206 bool failed;
1207
1208 ajla_assert_lo(ctx->current_position < da(ctx->fn,function)->code + da(ctx->fn,function)->code_size, (file_line, "gen_function: ran out of code in %s", da(ctx->fn,function)->function_name));
1209
1210 ctx->instr_start = ctx->current_position;
1211
1212 /*debug("%s: %04x, %s", da(ctx->fn,function)->function_name, *ctx->instr_start, decode_opcode(*ctx->instr_start, true));*/
1213
1214 ip = ctx->instr_start - da(ctx->fn,function)->code;
1215 if (likely(!ctx->code_labels[ip])) {
1216 ctx->code_labels[ip] = alloc_label(ctx);
1217 if (unlikely(!ctx->code_labels[ip]))
1218 return false;
1219 }
1220 gen_label(ctx->code_labels[ip]);
1221
1222 code = get_code(ctx);
1223 ctx->arg_mode = code / OPCODE_MODE_MULT;
1224 code %= OPCODE_MODE_MULT;
1225 ajla_assert_lo(ctx->arg_mode < ARG_MODE_N, (file_line, "gen_function: invalid opcode %04x", (unsigned)*ctx->instr_start));
1226
1227 if (code >= OPCODE_FIXED_OP + uzero && code < OPCODE_INT_OP) {
1228 code -= OPCODE_FIXED_OP;
1229 op = (code / OPCODE_FIXED_OP_MULT) % OPCODE_FIXED_TYPE_MULT;
1230 type = code / OPCODE_FIXED_TYPE_MULT;
1231 if (op < OPCODE_FIXED_OP_UNARY) {
1232 get_two(ctx, &slot_1, &slot_2);
1233 get_two(ctx, &slot_r, &flags);
1234 escape_label = alloc_escape_label(ctx);
1235 if (unlikely(!escape_label))
1236 return false;
1237 g(gen_test_2_cached(ctx, slot_1, slot_2, escape_label));
1238 flag_set(ctx, slot_1, false);
1239 flag_set(ctx, slot_2, false);
1240 flag_set(ctx, slot_r, false);
1241 if (flags & OPCODE_FLAG_FUSED) {
1242 g(gen_fused_binary(ctx, MODE_FIXED, type, op, escape_label, slot_1, slot_2, slot_r, &failed));
1243 if (unlikely(!failed))
1244 continue;
1245 }
1246 g(gen_alu(ctx, MODE_FIXED, type, op, escape_label, slot_1, slot_2, slot_r));
1247 continue;
1248 } else if (op < OPCODE_FIXED_OP_N) {
1249 get_two(ctx, &slot_1, &slot_r);
1250 get_one(ctx, &flags);
1251 escape_label = alloc_escape_label(ctx);
1252 if (unlikely(!escape_label))
1253 return false;
1254 g(gen_test_1_cached(ctx, slot_1, escape_label));
1255 flag_set(ctx, slot_1, false);
1256 flag_set(ctx, slot_r, false);
1257 g(gen_alu1(ctx, MODE_FIXED, type, op, escape_label, slot_1, slot_r));
1258 continue;
1259 } else if (op == OPCODE_FIXED_OP_ldc) {
1260 unsigned i;
1261 get_one(ctx, &slot_r);
1262 g(gen_constant(ctx, false, type, false, slot_r));
1263 for (i = 0; i < 1U << type; i += 2)
1264 get_code(ctx);
1265 flag_set(ctx, slot_r, false);
1266 continue;
1267 } else if (op == OPCODE_FIXED_OP_ldc16) {
1268 get_one(ctx, &slot_r);
1269 g(gen_constant(ctx, false, type, true, slot_r));
1270 get_code(ctx);
1271 flag_set(ctx, slot_r, false);
1272 continue;
1273 } else if (op == OPCODE_FIXED_OP_move || op == OPCODE_FIXED_OP_copy) {
1274 get_two(ctx, &slot_1, &slot_r);
1275 escape_label = alloc_escape_label(ctx);
1276 if (unlikely(!escape_label))
1277 return false;
1278 g(gen_test_1_cached(ctx, slot_1, escape_label));
1279 flag_set(ctx, slot_1, false);
1280 flag_set(ctx, slot_r, false);
1281 g(gen_copy(ctx, type, slot_1, slot_r));
1282 continue;
1283 } else {
1284 internal(file_line, "gen_function: bad fixed code %04x", *ctx->instr_start);
1285 }
1286 } else if (code >= OPCODE_INT_OP && code < OPCODE_REAL_OP) {
1287 code -= OPCODE_INT_OP;
1288 op = (code / OPCODE_INT_OP_MULT) % OPCODE_INT_TYPE_MULT;
1289 type = code / OPCODE_INT_TYPE_MULT;
1290 if (op < OPCODE_INT_OP_UNARY) {
1291 get_two(ctx, &slot_1, &slot_2);
1292 get_two(ctx, &slot_r, &flags);
1293 escape_label = alloc_escape_label(ctx);
1294 if (unlikely(!escape_label))
1295 return false;
1296 g(gen_test_2_cached(ctx, slot_1, slot_2, escape_label));
1297 flag_set(ctx, slot_1, false);
1298 flag_set(ctx, slot_2, false);
1299 flag_set(ctx, slot_r, false);
1300 if (flags & OPCODE_FLAG_FUSED) {
1301 g(gen_fused_binary(ctx, MODE_INT, type, op, escape_label, slot_1, slot_2, slot_r, &failed));
1302 if (unlikely(!failed))
1303 continue;
1304 }
1305 g(gen_alu(ctx, MODE_INT, type, op, escape_label, slot_1, slot_2, slot_r));
1306 continue;
1307 } else if (op < OPCODE_INT_OP_N) {
1308 get_two(ctx, &slot_1, &slot_r);
1309 get_one(ctx, &flags);
1310 if ((op == OPCODE_INT_OP_to_int || op == OPCODE_INT_OP_from_int) && slot_1 == slot_r)
1311 continue;
1312 escape_label = alloc_escape_label(ctx);
1313 if (unlikely(!escape_label))
1314 return false;
1315 g(gen_test_1_cached(ctx, slot_1, escape_label));
1316 flag_set(ctx, slot_1, false);
1317 flag_set(ctx, slot_r, false);
1318 g(gen_alu1(ctx, MODE_INT, type, op, escape_label, slot_1, slot_r));
1319 continue;
1320 } else if (op == OPCODE_INT_OP_ldc) {
1321 unsigned i;
1322 get_one(ctx, &slot_r);
1323 g(gen_constant(ctx, false, type, false, slot_r));
1324 for (i = 0; i < 1U << type; i += 2)
1325 get_code(ctx);
1326 flag_set(ctx, slot_r, false);
1327 continue;
1328 } else if (op == OPCODE_INT_OP_ldc16) {
1329 get_one(ctx, &slot_r);
1330 g(gen_constant(ctx, false, type, true, slot_r));
1331 get_code(ctx);
1332 flag_set(ctx, slot_r, false);
1333 continue;
1334 } else if (op == OPCODE_INT_OP_move || op == OPCODE_INT_OP_copy) {
1335 get_two(ctx, &slot_1, &slot_r);
1336 escape_label = alloc_escape_label(ctx);
1337 if (unlikely(!escape_label))
1338 return false;
1339 g(gen_test_1_cached(ctx, slot_1, escape_label));
1340 flag_set(ctx, slot_1, false);
1341 flag_set(ctx, slot_r, false);
1342 g(gen_copy(ctx, type, slot_1, slot_r));
1343 continue;
1344 } else {
1345 internal(file_line, "gen_function: bad integer code %04x", *ctx->instr_start);
1346 }
1347 } else if (code >= OPCODE_REAL_OP && code < OPCODE_BOOL_OP) {
1348 code -= OPCODE_REAL_OP;
1349 op = (code / OPCODE_REAL_OP_MULT) % OPCODE_REAL_TYPE_MULT;
1350 type = code / OPCODE_REAL_TYPE_MULT;
1351 if (op < OPCODE_REAL_OP_UNARY) {
1352 get_two(ctx, &slot_1, &slot_2);
1353 get_two(ctx, &slot_r, &flags);
1354 escape_label = alloc_escape_label(ctx);
1355 if (unlikely(!escape_label))
1356 return false;
1357 g(gen_test_2_cached(ctx, slot_1, slot_2, escape_label));
1358 flag_set(ctx, slot_1, false);
1359 flag_set(ctx, slot_2, false);
1360 flag_set(ctx, slot_r, false);
1361 if (flags & OPCODE_FLAG_FUSED) {
1362 g(gen_fused_binary(ctx, MODE_REAL, type, op, escape_label, slot_1, slot_2, slot_r, &failed));
1363 if (unlikely(!failed))
1364 continue;
1365 }
1366 g(gen_fp_alu(ctx, type, op, escape_label, slot_1, slot_2, slot_r));
1367 continue;
1368 } else if (op < OPCODE_REAL_OP_N) {
1369 get_two(ctx, &slot_1, &slot_r);
1370 get_one(ctx, &flags);
1371 escape_label = alloc_escape_label(ctx);
1372 if (unlikely(!escape_label))
1373 return false;
1374 g(gen_test_1_cached(ctx, slot_1, escape_label));
1375 flag_set(ctx, slot_1, false);
1376 flag_set(ctx, slot_r, false);
1377 g(gen_fp_alu1(ctx, type, op, escape_label, slot_1, slot_r));
1378 continue;
1379 } else if (op == OPCODE_REAL_OP_ldc) {
1380 const struct type *t;
1381 unsigned i;
1382 get_one(ctx, &slot_r);
1383 t = type_get_real(type);
1384 g(gen_real_constant(ctx, t, slot_r));
1385 for (i = 0; i < t->size; i += 2)
1386 get_code(ctx);
1387 flag_set(ctx, slot_r, false);
1388 continue;
1389 } else if (op == OPCODE_REAL_OP_move || op == OPCODE_REAL_OP_copy) {
1390 get_two(ctx, &slot_1, &slot_r);
1391 escape_label = alloc_escape_label(ctx);
1392 if (unlikely(!escape_label))
1393 return false;
1394 g(gen_test_1_cached(ctx, slot_1, escape_label));
1395 flag_set(ctx, slot_1, false);
1396 flag_set(ctx, slot_r, false);
1397 g(gen_memcpy_slots(ctx, slot_r, slot_1));
1398 continue;
1399 } else {
1400 internal(file_line, "gen_function: bad real code %04x", *ctx->instr_start);
1401 }
1402 } else if (code >= OPCODE_BOOL_OP && code < OPCODE_EXTRA) {
1403 code -= OPCODE_BOOL_OP;
1404 op = (code / OPCODE_BOOL_OP_MULT) % OPCODE_BOOL_TYPE_MULT;
1405 type = log_2(sizeof(ajla_flat_option_t));
1406 if (op < OPCODE_BOOL_OP_UNARY) {
1407 get_two(ctx, &slot_1, &slot_2);
1408 get_two(ctx, &slot_r, &flags);
1409 escape_label = alloc_escape_label(ctx);
1410 if (unlikely(!escape_label))
1411 return false;
1412 g(gen_test_2_cached(ctx, slot_1, slot_2, escape_label));
1413 flag_set(ctx, slot_1, false);
1414 flag_set(ctx, slot_2, false);
1415 flag_set(ctx, slot_r, false);
1416 if (flags & OPCODE_FLAG_FUSED) {
1417 g(gen_fused_binary(ctx, MODE_BOOL, type, op, escape_label, slot_1, slot_2, slot_r, &failed));
1418 if (unlikely(!failed))
1419 continue;
1420 }
1421 g(gen_alu(ctx, MODE_BOOL, type, op, escape_label, slot_1, slot_2, slot_r));
1422 continue;
1423 } else if (op < OPCODE_BOOL_OP_N) {
1424 get_two(ctx, &slot_1, &slot_r);
1425 get_one(ctx, &flags);
1426 escape_label = alloc_escape_label(ctx);
1427 if (unlikely(!escape_label))
1428 return false;
1429 g(gen_test_1_cached(ctx, slot_1, escape_label));
1430 flag_set(ctx, slot_1, false);
1431 flag_set(ctx, slot_r, false);
1432 g(gen_alu1(ctx, MODE_BOOL, type, op, escape_label, slot_1, slot_r));
1433 continue;
1434 } else if (op == OPCODE_BOOL_OP_move || op == OPCODE_BOOL_OP_copy) {
1435 get_two(ctx, &slot_1, &slot_r);
1436 escape_label = alloc_escape_label(ctx);
1437 if (unlikely(!escape_label))
1438 return false;
1439 g(gen_test_1_cached(ctx, slot_1, escape_label));
1440 flag_set(ctx, slot_1, false);
1441 flag_set(ctx, slot_r, false);
1442 g(gen_copy(ctx, type, slot_1, slot_r));
1443 continue;
1444 } else {
1445 internal(file_line, "gen_function: bad boolean code %04x", *ctx->instr_start);
1446 }
1447 } else switch (code) {
1448 case OPCODE_INT_LDC_LONG: {
1449 uint32_t words, w;
1450 get_one(ctx, &slot_r);
1451 words = get_uint32(ctx);
1452 for (w = 0; w < words; w++)
1453 get_code(ctx);
1454 unconditional_escape:
1455 escape_label = alloc_escape_label(ctx);
1456 if (unlikely(!escape_label))
1457 return false;
1458 gen_insn(INSN_JMP, 0, 0, 0);
1459 gen_four(escape_label);
1460 continue;
1461 }
1462 case OPCODE_IS_EXCEPTION: {
1463 get_two(ctx, &slot_1, &slot_r);
1464 get_one(ctx, &flags);
1465 g(gen_is_exception(ctx, slot_1, slot_r));
1466 continue;
1467 }
1468 case OPCODE_EXCEPTION_CLASS:
1469 case OPCODE_EXCEPTION_TYPE:
1470 case OPCODE_EXCEPTION_AUX: {
1471 get_two(ctx, &slot_1, &slot_r);
1472 get_one(ctx, &flags);
1473 goto unconditional_escape;
1474 }
1475 case OPCODE_SYSTEM_PROPERTY: {
1476 get_two(ctx, &slot_1, &slot_r);
1477 get_one(ctx, &flags);
1478 g(gen_system_property(ctx, slot_1, slot_r));
1479 continue;
1480 }
1481 case OPCODE_FLAT_MOVE:
1482 case OPCODE_FLAT_COPY: {
1483 get_two(ctx, &slot_1, &slot_r);
1484 g(gen_flat_move_copy(ctx, slot_1, slot_r));
1485 continue;
1486 }
1487 case OPCODE_REF_MOVE:
1488 case OPCODE_REF_MOVE_CLEAR:
1489 case OPCODE_REF_COPY: {
1490 get_two(ctx, &slot_1, &slot_r);
1491 g(gen_ref_move_copy(ctx, code, slot_1, slot_r));
1492 continue;
1493 }
1494 case OPCODE_BOX_MOVE_CLEAR:
1495 case OPCODE_BOX_COPY: {
1496 get_two(ctx, &slot_1, &slot_r);
1497 g(gen_box_move_copy(ctx, code, slot_1, slot_r));
1498 continue;
1499 }
1500 case OPCODE_TAKE_BORROWED:
1501 get_one(ctx, &slot_1);
1502 if (!da(ctx->fn,function)->local_variables_flags[slot_1].may_be_borrowed)
1503 continue;
1504 if (unlikely(!(label_id = alloc_label(ctx))))
1505 return false;
1506 if (flag_is_set(ctx, slot_1))
1507 goto take_borrowed_done;
1508 if (flag_is_clear(ctx, slot_1)) {
1509 g(gen_set_1(ctx, R_FRAME, slot_1, 0, true));
1510 goto do_take_borrowed;
1511 }
1512 g(gen_test_1(ctx, R_FRAME, slot_1, 0, label_id, false, TEST_SET));
1513 do_take_borrowed:
1514 g(gen_upcall_start(ctx, 1));
1515 g(gen_frame_load(ctx, OP_SIZE_SLOT, garbage, slot_1, 0, R_ARG0));
1516 g(gen_upcall_argument(ctx, 0));
1517 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_pointer_reference_owned), 1));
1518 flag_set(ctx, slot_1, true);
1519 take_borrowed_done:
1520 gen_label(label_id);
1521 continue;
1522 case OPCODE_DEREFERENCE:
1523 case OPCODE_DEREFERENCE_CLEAR: {
1524 bool need_bit_test;
1525 /*const struct type *type;*/
1526 get_one(ctx, &slot_1);
1527 if (flag_is_clear(ctx, slot_1))
1528 goto skip_dereference;
1529 /*type = get_type_of_local(ctx, slot_1);*/
1530 /*need_bit_test = 1 || TYPE_IS_FLAT(type) || da(ctx->fn,function)->local_variables[slot_1].may_be_borrowed;*/
1531 need_bit_test = !flag_is_set(ctx, slot_1);
1532 if (need_bit_test) {
1533 if (unlikely(!(label_id = alloc_label(ctx))))
1534 return false;
1535 g(gen_test_1(ctx, R_FRAME, slot_1, 0, label_id, true, TEST_CLEAR));
1536 } else {
1537 g(gen_set_1(ctx, R_FRAME, slot_1, 0, false));
1538 label_id = 0; /* avoid warning */
1539 }
1540 g(gen_upcall_start(ctx, 1));
1541 g(gen_frame_load(ctx, OP_SIZE_SLOT, garbage, slot_1, 0, R_ARG0));
1542 g(gen_upcall_argument(ctx, 0));
1543 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_pointer_dereference), 1));
1544 if (need_bit_test)
1545 gen_label(label_id);
1546 skip_dereference:
1547 if (code == OPCODE_DEREFERENCE_CLEAR)
1548 g(gen_frame_clear(ctx, OP_SIZE_SLOT, slot_1));
1549 flag_set_unknown(ctx, slot_1);
1550 flag_set(ctx, slot_1, false);
1551 continue;
1552 }
1553 case OPCODE_EVAL: {
1554 get_one(ctx, &slot_1);
1555 g(gen_eval(ctx, slot_1));
1556 continue;
1557 }
1558 case OPCODE_ESCAPE_NONFLAT: {
1559 frame_t n, i;
1560 frame_t *vars;
1561
1562 get_one(ctx, &n);
1563 vars = mem_alloc_array_mayfail(mem_alloc_mayfail, frame_t *, 0, 0, n, sizeof(frame_t), &ctx->err);
1564 if (unlikely(!vars))
1565 return false;
1566 for (i = 0; i < n; i++) {
1567 get_one(ctx, &vars[i]);
1568 }
1569
1570 escape_label = alloc_escape_label(ctx);
1571 if (unlikely(!escape_label)) {
1572 mem_free(vars);
1573 return false;
1574 }
1575
1576 if (unlikely(!gen_test_multiple(ctx, vars, n, escape_label))) {
1577 mem_free(vars);
1578 return false;
1579 }
1580 mem_free(vars);
1581
1582 continue;
1583 }
1584 case OPCODE_CHECKPOINT: {
1585 frame_t n_vars;
1586
1587 g(clear_flag_cache(ctx));
1588
1589 if (SIZEOF_IP_T == 2) {
1590 slot_1 = get_code(ctx);
1591 } else if (SIZEOF_IP_T == 4) {
1592 slot_1 = get_uint32(ctx);
1593 } else {
1594 not_reached();
1595 continue;
1596 }
1597
1598 if (unlikely(!(slot_1 + 1)))
1599 return false;
1600 while (slot_1 >= ctx->n_entries) {
1601 void *err_entries;
1602 struct cg_entry e;
1603 if (unlikely(!ctx->entries)) {
1604 if (unlikely(!array_init_mayfail(struct cg_entry, &ctx->entries, &ctx->n_entries, &ctx->err)))
1605 return false;
1606 }
1607 memset(&e, 0, sizeof(struct cg_entry));
1608 if (unlikely(!array_add_mayfail(struct cg_entry, &ctx->entries, &ctx->n_entries, e, &err_entries, &ctx->err))) {
1609 ctx->entries = err_entries;
1610 return false;
1611 }
1612 }
1613
1614 get_one(ctx, &n_vars);
1615
1616 escape_label = 0; /* avoid warning */
1617 if (likely(slot_1 != 0)) {
1618 escape_label = alloc_escape_label(ctx);
1619 if (unlikely(!escape_label))
1620 return false;
1621 }
1622
1623 if (n_vars || !slot_1) {
1624 frame_t i;
1625 uint32_t entry_label, nonflat_label;
1626 struct cg_entry *ce = &ctx->entries[slot_1];
1627
1628 if (unlikely(!array_init_mayfail(frame_t, &ce->variables, &ce->n_variables, &ctx->err)))
1629 return false;
1630 for (i = 0; i < n_vars; i++) {
1631 frame_t v;
1632 get_one(ctx, &v);
1633 if (unlikely(!array_add_mayfail(frame_t, &ce->variables, &ce->n_variables, v, NULL, &ctx->err)))
1634 return false;
1635 }
1636 if (!slot_1) {
1637 g(gen_test_multiple(ctx, ce->variables, ce->n_variables, ctx->escape_nospill_label));
1638 }
1639 entry_label = alloc_label(ctx);
1640 if (unlikely(!entry_label))
1641 return false;
1642 gen_label(entry_label);
1643 ce->entry_label = entry_label;
1644
1645 nonflat_label = alloc_escape_label_for_ip(ctx, ctx->current_position);
1646 if (unlikely(!nonflat_label))
1647 return false;
1648 ce->nonflat_label = nonflat_label;
1649
1650 if (unlikely(!slot_1))
1651 g(gen_timestamp_test(ctx, ctx->escape_nospill_label));
1652 else
1653 g(gen_timestamp_test(ctx, escape_label));
1654 } else {
1655 g(gen_timestamp_test(ctx, escape_label));
1656
1657 gen_insn(INSN_ENTRY, 0, 0, 0);
1658 gen_four(slot_1);
1659 }
1660 continue;
1661 }
1662 case OPCODE_JMP: {
1663 int32_t x = get_jump_offset(ctx);
1664 g(gen_jump(ctx, x, OP_SIZE_NATIVE, COND_ALWAYS, -1U, -1U));
1665 continue;
1666 }
1667 case OPCODE_JMP_BACK_16: {
1668 int32_t x = get_code(ctx);
1669 g(gen_jump(ctx, -x - (int)(2 * sizeof(code_t)), OP_SIZE_NATIVE, COND_ALWAYS, -1U, -1U));
1670 continue;
1671 }
1672 case OPCODE_JMP_FALSE: {
1673 int32_t offs_false;
1674 get_one(ctx, &slot_1);
1675 offs_false = get_jump_offset(ctx);
1676 get_jump_offset(ctx);
1677 escape_label = alloc_escape_label(ctx);
1678 if (unlikely(!escape_label))
1679 return false;
1680 g(gen_test_1_cached(ctx, slot_1, escape_label));
1681 flag_set(ctx, slot_1, false);
1682 g(gen_cond_jump(ctx, slot_1, offs_false));
1683 continue;
1684 }
1685 case OPCODE_LABEL: {
1686 g(clear_flag_cache(ctx));
1687 continue;
1688 }
1689 #define init_args \
1690 do { \
1691 if (ctx->args != NULL) \
1692 mem_free(ctx->args); \
1693 g(array_init_mayfail(struct code_arg, &ctx->args, &ctx->args_l, &ctx->err));\
1694 } while (0)
1695 #define load_args \
1696 do { \
1697 init_args; \
1698 for (i_arg = 0; i_arg < n_args; i_arg++) { \
1699 struct code_arg a; \
1700 get_two(ctx, &a.slot, &a.flags); \
1701 a.type = 0; \
1702 g(array_add_mayfail(struct code_arg, &ctx->args, &ctx->args_l, a, NULL, &ctx->err));\
1703 } \
1704 } while (0)
1705 case OPCODE_LOAD_FN:
1706 get_two(ctx, &n_args, &slot_r);
1707 get_one(ctx, &fn_idx);
1708 load_args;
1709 g(gen_load_fn_or_curry(ctx, fn_idx, NO_FRAME_T, slot_r, 0));
1710 continue;
1711 case OPCODE_CURRY:
1712 get_two(ctx, &n_args, &slot_r);
1713 get_two(ctx, &slot_1, &flags);
1714 load_args;
1715 g(gen_load_fn_or_curry(ctx, NO_FRAME_T, slot_1, slot_r, flags));
1716 continue;
1717 case OPCODE_CALL:
1718 case OPCODE_CALL_STRICT:
1719 case OPCODE_CALL_SPARK:
1720 case OPCODE_CALL_LAZY:
1721 case OPCODE_CALL_CACHE:
1722 case OPCODE_CALL_SAVE: {
1723 get_two(ctx, &n_args, &n_ret);
1724 get_one(ctx, &fn_idx);
1725 jump_over_arguments_and_return:
1726 load_args;
1727 ctx->return_values = ctx->current_position;
1728 for (i_arg = 0; i_arg < n_ret; i_arg++) {
1729 #if ARG_MODE_N >= 3
1730 get_uint32(ctx);
1731 #else
1732 get_code(ctx);
1733 #endif
1734 get_code(ctx);
1735 }
1736 if (unlikely(profiling))
1737 goto unconditional_escape;
1738 if (code == OPCODE_CALL || code == OPCODE_CALL_STRICT) {
1739 g(gen_call(ctx, code, fn_idx));
1740 continue;
1741 }
1742 /*if (code == OPCODE_CALL_INDIRECT || code == OPCODE_CALL_INDIRECT_STRICT) {
1743 if (unlikely(!gen_call_indirect(ctx, code, slot_1, flags)))
1744 return false;
1745 continue;
1746 }*/
1747 goto unconditional_escape;
1748 }
1749 case OPCODE_CALL_INDIRECT:
1750 case OPCODE_CALL_INDIRECT_STRICT:
1751 case OPCODE_CALL_INDIRECT_SPARK:
1752 case OPCODE_CALL_INDIRECT_LAZY:
1753 case OPCODE_CALL_INDIRECT_CACHE:
1754 case OPCODE_CALL_INDIRECT_SAVE: {
1755 fn_idx = 0; /* avoid warning */
1756 get_two(ctx, &n_args, &n_ret);
1757 get_two(ctx, &slot_1, &flags);
1758 goto jump_over_arguments_and_return;
1759 }
1760 case OPCODE_RETURN: {
1761 n_args = da(ctx->fn,function)->n_return_values;
1762 load_args;
1763 if (unlikely(profiling))
1764 goto unconditional_escape;
1765 g(gen_return(ctx));
1766 continue;
1767 }
1768 case OPCODE_STRUCTURED: {
1769 init_args;
1770 get_two(ctx, &slot_1, &slot_2);
1771 do {
1772 struct code_arg a;
1773 get_two(ctx, &flags, &slot_r);
1774 get_one(ctx, &opt);
1775 a.slot = slot_r;
1776 a.flags = flags;
1777 a.type = opt;
1778 g(array_add_mayfail(struct code_arg, &ctx->args, &ctx->args_l, a, NULL, &ctx->err));
1779 } while (!(flags & OPCODE_STRUCTURED_FLAG_END));
1780 g(gen_structured(ctx, slot_1, slot_2));
1781 continue;
1782 }
1783 case OPCODE_RECORD_CREATE: {
1784 init_args;
1785 get_two(ctx, &slot_r, &n_args);
1786 for (i_arg = 0; i_arg < n_args; i_arg++) {
1787 struct code_arg a;
1788 get_two(ctx, &slot_1, &flags);
1789 a.slot = slot_1;
1790 a.flags = flags;
1791 a.type = 0;
1792 g(array_add_mayfail(struct code_arg, &ctx->args, &ctx->args_l, a, NULL, &ctx->err));
1793 }
1794 g(gen_record_create(ctx, slot_r));
1795 continue;
1796 }
1797 case OPCODE_RECORD_LOAD: {
1798 get_two(ctx, &slot_1, &opt);
1799 get_two(ctx, &slot_r, &flags);
1800 g(gen_record_load(ctx, slot_1, slot_r, opt, flags));
1801 continue;
1802 }
1803 case OPCODE_OPTION_CREATE_EMPTY_FLAT: {
1804 get_two(ctx, &slot_r, &opt);
1805 g(gen_option_create_empty_flat(ctx, opt, slot_r));
1806 continue;
1807 }
1808 case OPCODE_OPTION_CREATE_EMPTY: {
1809 get_two(ctx, &slot_r, &opt);
1810 g(gen_option_create_empty(ctx, opt, slot_r));
1811 continue;
1812 }
1813 case OPCODE_OPTION_CREATE: {
1814 get_two(ctx, &slot_r, &opt);
1815 get_two(ctx, &slot_1, &flags);
1816 g(gen_option_create(ctx, opt, slot_1, slot_r, flags));
1817 continue;
1818 }
1819 case OPCODE_OPTION_LOAD: {
1820 get_two(ctx, &slot_1, &opt);
1821 get_two(ctx, &slot_r, &flags);
1822 g(gen_option_load(ctx, slot_1, slot_r, opt, flags));
1823 continue;
1824 }
1825 case OPCODE_OPTION_TEST_FLAT: {
1826 get_two(ctx, &slot_1, &opt);
1827 get_one(ctx, &slot_r);
1828 g(gen_option_test_flat(ctx, slot_1, opt, slot_r));
1829 continue;
1830 }
1831 case OPCODE_OPTION_TEST: {
1832 get_two(ctx, &slot_1, &opt);
1833 get_one(ctx, &slot_r);
1834 g(gen_option_test(ctx, slot_1, opt, slot_r));
1835 continue;
1836 }
1837 case OPCODE_OPTION_ORD_FLAT: {
1838 get_two(ctx, &slot_1, &slot_r);
1839 g(gen_option_ord(ctx, slot_1, slot_r, true));
1840 continue;
1841 }
1842 case OPCODE_OPTION_ORD: {
1843 get_two(ctx, &slot_1, &slot_r);
1844 g(gen_option_ord(ctx, slot_1, slot_r, false));
1845 continue;
1846 }
1847 case OPCODE_ARRAY_CREATE: {
1848 init_args;
1849 get_two(ctx, &slot_r, &n_args);
1850 for (i_arg = 0; i_arg < n_args; i_arg++) {
1851 struct code_arg a;
1852 get_two(ctx, &slot_1, &flags);
1853 a.slot = slot_1;
1854 a.flags = flags;
1855 a.type = 0;
1856 g(array_add_mayfail(struct code_arg, &ctx->args, &ctx->args_l, a, NULL, &ctx->err));
1857 }
1858 g(gen_array_create(ctx, slot_r));
1859 continue;
1860 }
1861 case OPCODE_ARRAY_CREATE_EMPTY_FLAT: {
1862 get_two(ctx, &slot_r, &flags);
1863 g(gen_array_create_empty_flat(ctx, slot_r, flags));
1864 continue;
1865 }
1866 case OPCODE_ARRAY_CREATE_EMPTY: {
1867 get_one(ctx, &slot_r);
1868 g(gen_array_create_empty(ctx, slot_r));
1869 continue;
1870 }
1871 case OPCODE_ARRAY_FILL: {
1872 get_two(ctx, &slot_1, &flags);
1873 get_two(ctx, &slot_2, &slot_r);
1874 g(gen_array_fill(ctx, slot_1, flags, slot_2, slot_r));
1875 continue;
1876 }
1877 case OPCODE_ARRAY_STRING: {
1878 frame_t i;
1879 get_two(ctx, &slot_r, &i);
1880 g(gen_array_string(ctx, type_get_fixed(0, true)->tag, cast_ptr(uint8_t *, ctx->current_position), i, slot_r));
1881 ctx->current_position += (i + 1) >> 1;
1882 continue;
1883 }
1884 case OPCODE_ARRAY_UNICODE: {
1885 frame_t i;
1886 get_two(ctx, &slot_r, &i);
1887 g(gen_array_string(ctx, type_get_int(2)->tag, cast_ptr(uint8_t *, ctx->current_position), i, slot_r));
1888 ctx->current_position += i * 2;
1889 continue;
1890 }
1891 case OPCODE_ARRAY_LOAD: {
1892 get_two(ctx, &slot_1, &slot_2);
1893 get_two(ctx, &slot_r, &flags);
1894 g(gen_array_load(ctx, slot_1, slot_2, slot_r, flags));
1895 continue;
1896 }
1897 case OPCODE_ARRAY_LEN: {
1898 get_two(ctx, &slot_1, &slot_r);
1899 get_one(ctx, &flags);
1900 g(gen_array_len(ctx, slot_1, NO_FRAME_T, slot_r, false, 0));
1901 continue;
1902 }
1903 case OPCODE_ARRAY_LEN_GREATER_THAN: {
1904 get_two(ctx, &slot_1, &slot_2);
1905 get_two(ctx, &slot_r, &flags);
1906 escape_label = alloc_escape_label(ctx);
1907 if (unlikely(!escape_label))
1908 return false;
1909 if (flags & OPCODE_FLAG_FUSED) {
1910 g(gen_fused_binary(ctx, MODE_ARRAY_LEN_GT, 0, 0, escape_label, slot_1, slot_2, slot_r, &failed));
1911 if (unlikely(!failed))
1912 continue;
1913 }
1914 g(gen_array_len(ctx, slot_1, slot_2, slot_r, false, 0));
1915 continue;
1916 }
1917 case OPCODE_ARRAY_SUB: {
1918 get_two(ctx, &slot_1, &slot_2);
1919 get_two(ctx, &slot_3, &slot_r);
1920 get_one(ctx, &flags);
1921 g(gen_array_sub(ctx, slot_1, slot_2, slot_3, slot_r, flags));
1922 continue;
1923 }
1924 case OPCODE_ARRAY_SKIP: {
1925 get_two(ctx, &slot_1, &slot_2);
1926 get_two(ctx, &slot_r, &flags);
1927 g(gen_array_skip(ctx, slot_1, slot_2, slot_r, flags));
1928 continue;
1929 }
1930 case OPCODE_ARRAY_APPEND: {
1931 get_two(ctx, &slot_r, &flags);
1932 get_two(ctx, &slot_1, &slot_2);
1933 g(gen_array_append(ctx, slot_1, slot_2, slot_r, flags));
1934 continue;
1935 }
1936 case OPCODE_ARRAY_APPEND_ONE_FLAT: {
1937 get_two(ctx, &slot_r, &flags);
1938 get_two(ctx, &slot_1, &slot_2);
1939 g(gen_array_append_one_flat(ctx, slot_1, slot_2, slot_r, flags));
1940 continue;
1941 }
1942 case OPCODE_ARRAY_APPEND_ONE: {
1943 get_two(ctx, &slot_r, &flags);
1944 get_two(ctx, &slot_1, &slot_2);
1945 g(gen_array_append_one(ctx, slot_1, slot_2, slot_r, flags));
1946 continue;
1947 }
1948 case OPCODE_ARRAY_FLATTEN: {
1949 get_two(ctx, &slot_r, &flags);
1950 get_one(ctx, &slot_1);
1951 goto unconditional_escape;
1952 }
1953 case OPCODE_IO: {
1954 get_two(ctx, &flags, &slot_1);
1955 get_two(ctx, &slot_2, &slot_3);
1956 g(gen_io(ctx, flags, slot_1, slot_2, slot_3));
1957 continue;
1958 }
1959 case OPCODE_INTERNAL_FUNCTION:
1960 case OPCODE_EXIT_THREAD:
1961 case OPCODE_UNREACHABLE: {
1962 goto unconditional_escape;
1963 }
1964 default: {
1965 #if 1
1966 /*if (getenv("DUMP") && !strcmp(da(ctx->fn,function)->function_name, getenv("DUMP")))*/
1967 warning("gen_function: %s: unknown opcode %04x, %s", da(ctx->fn,function)->function_name, *ctx->instr_start, decode_opcode(*ctx->instr_start, false));
1968 #endif
1969 return false;
1970 }
1971 }
1972 }
1973
1974 return true;
1975 }
1976
1977 static bool attr_w gen_entries(struct codegen_context *ctx)
1978 {
1979 size_t i;
1980 for (i = 0; i < ctx->n_entries; i++) {
1981 struct cg_entry *ce = &ctx->entries[i];
1982 if (ce->entry_label) {
1983 gen_insn(INSN_ENTRY, 0, 0, 0);
1984 gen_four(i);
1985
1986 g(gen_test_multiple(ctx, ce->variables, ce->n_variables, ce->nonflat_label));
1987
1988 gen_insn(INSN_JMP, 0, 0, 0);
1989 gen_four(ce->entry_label);
1990 }
1991 }
1992 return true;
1993 }
1994
1995 static bool attr_w gen_epilogues(struct codegen_context *ctx)
1996 {
1997 frame_t v;
1998 ip_t ip;
1999 uint32_t escape_label, nospill_label;
2000 escape_label = alloc_label(ctx);
2001 if (unlikely(!escape_label))
2002 return false;
2003 nospill_label = alloc_label(ctx);
2004 if (unlikely(!nospill_label))
2005 return false;
2006 #if defined(ARCH_PARISC)
2007 if (ctx->call_label) {
2008 gen_label(ctx->call_label);
2009 g(gen_call_millicode(ctx));
2010 }
2011 #endif
2012 if (ctx->reload_label) {
2013 gen_label(ctx->reload_label);
2014 g(gen_mov(ctx, i_size(OP_SIZE_ADDRESS), R_FRAME, R_RET0));
2015 g(gen_escape_arg(ctx, (ip_t)-1, escape_label));
2016 }
2017 gen_label(ctx->escape_nospill_label);
2018 g(gen_escape_arg(ctx, 0, nospill_label));
2019 for (ip = 0; ip < da(ctx->fn,function)->code_size; ip++) {
2020 struct cg_exit *ce = ctx->code_exits[ip];
2021 if (ce && (ce->undo_label || ce->escape_label)) {
2022 if (ce->undo_label) {
2023 size_t i;
2024 gen_label(ce->undo_label);
2025 gen_insn(ce->undo_opcode, ce->undo_op_size, ce->undo_aux, ce->undo_writes_flags);
2026 for (i = 0; i < ce->undo_parameters_len; i++)
2027 gen_one(ce->undo_parameters[i]);
2028 }
2029 if (ce->escape_label) {
2030 gen_label(ce->escape_label);
2031 }
2032 g(gen_escape_arg(ctx, ip, escape_label));
2033 }
2034 }
2035 gen_label(escape_label);
2036 for (v = MIN_USEABLE_SLOT; v < function_n_variables(ctx->fn); v++) {
2037 if (ctx->registers[v] >= 0) {
2038 g(spill(ctx, v));
2039 }
2040 }
2041 gen_label(nospill_label);
2042 g(gen_escape(ctx));
2043 return true;
2044 }
2045
2046 static bool attr_w cgen_entry(struct codegen_context *ctx)
2047 {
2048 uint32_t entry_id = cget_four(ctx);
2049 ajla_assert_lo(entry_id < ctx->n_entries, (file_line, "cgen_entry: invalid entry %lx", (unsigned long)entry_id));
2050 ctx->entries[entry_id].entry_to_pos = ctx->mcode_size;
2051 return true;
2052 }
2053
2054 static bool attr_w cgen_label(struct codegen_context *ctx)
2055 {
2056 uint32_t label_id = cget_four(ctx);
2057 ctx->label_to_pos[label_id] = ctx->mcode_size;
2058 return true;
2059 }
2060
2061 static bool attr_w attr_unused cgen_trap(struct codegen_context *ctx, uint32_t label)
2062 {
2063 struct trap_record tr;
2064 tr.source_ip = ctx->mcode_size;
2065 tr.destination_ip = label;
2066 if (unlikely(!array_add_mayfail(struct trap_record, &ctx->trap_records, &ctx->trap_records_size, tr, NULL, &ctx->err)))
2067 return false;
2068 return true;
2069 }
2070
2071 static bool attr_w add_relocation(struct codegen_context *ctx, unsigned length, int offset, bool *known)
2072 {
2073 struct relocation rel;
2074 rel.label_id = cget_four(ctx);
2075 rel.length = length;
2076 rel.position = ctx->mcode_size;
2077 rel.jmp_instr = ctx->code_position - 8 - offset - ctx->code;
2078 if (unlikely(!array_add_mayfail(struct relocation, &ctx->reloc, &ctx->reloc_size, rel, NULL, &ctx->err)))
2079 return false;
2080 if (known)
2081 *known = ctx->label_to_pos[rel.label_id] != (size_t)-1;
2082 return true;
2083 }
2084
2085
2086 #if defined(ARCH_ALPHA)
2087 #include "c2-alpha.inc"
2088 #elif defined(ARCH_ARM32)
2089 #include "c2-arm.inc"
2090 #elif defined(ARCH_ARM64)
2091 #include "c2-arm64.inc"
2092 #elif defined(ARCH_IA64)
2093 #include "c2-ia64.inc"
2094 #elif defined(ARCH_LOONGARCH64)
2095 #include "c2-loong.inc"
2096 #elif defined(ARCH_MIPS)
2097 #include "c2-mips.inc"
2098 #elif defined(ARCH_PARISC)
2099 #include "c2-hppa.inc"
2100 #elif defined(ARCH_POWER)
2101 #include "c2-power.inc"
2102 #elif defined(ARCH_S390)
2103 #include "c2-s390.inc"
2104 #elif defined(ARCH_SPARC)
2105 #include "c2-sparc.inc"
2106 #elif defined(ARCH_RISCV64)
2107 #include "c2-riscv.inc"
2108 #elif defined(ARCH_X86)
2109 #include "c2-x86.inc"
2110 #endif
2111
2112
2113 static bool attr_w gen_mcode(struct codegen_context *ctx)
2114 {
2115 ctx->code_position = ctx->code;
2116
2117 while (ctx->code_position != ctx->code + ctx->code_size) {
2118 uint32_t insn;
2119 ajla_assert_lo(ctx->code_position < ctx->code + ctx->code_size, (file_line, "gen_mcode: ran out of code"));
2120 #ifdef DEBUG_INSNS
2121 insn = cget_four(ctx);
2122 debug("line: %u", insn);
2123 #endif
2124 insn = cget_four(ctx);
2125 g(cgen_insn(ctx, insn));
2126 }
2127
2128 return true;
2129 }
2130
2131 #define RELOCS_RETRY -1
2132 #define RELOCS_FAIL 0
2133 #define RELOCS_OK 1
2134
2135 static int8_t resolve_relocs(struct codegen_context *ctx)
2136 {
2137 size_t i;
2138 int8_t status = RELOCS_OK;
2139 for (i = 0; i < ctx->reloc_size; i++) {
2140 struct relocation *reloc = &ctx->reloc[i];
2141 if (!resolve_relocation(ctx, reloc)) {
2142 uint8_t *jmp_instr;
2143 uint32_t insn;
2144 uint32_t new_length;
2145 status = RELOCS_RETRY;
2146 if (unlikely(reloc->length + zero >= JMP_LIMIT))
2147 return RELOCS_FAIL;
2148 new_length = reloc->length + 1;
2149 jmp_instr = ctx->code + reloc->jmp_instr;
2150 insn = (uint32_t)jmp_instr[0] +
2151 ((uint32_t)jmp_instr[1] << 8) +
2152 ((uint32_t)jmp_instr[2] << 16) +
2153 ((uint32_t)jmp_instr[3] << 24);
2154 insn &= ~INSN_JUMP_SIZE;
2155 insn |= (uint32_t)new_length << INSN_JUMP_SIZE_SHIFT;
2156 jmp_instr[0] = insn;
2157 jmp_instr[1] = insn >> 8;
2158 jmp_instr[2] = insn >> 16;
2159 jmp_instr[3] = insn >> 24;
2160 }
2161 }
2162 return status;
2163 }
2164
2165 static void resolve_traps(struct codegen_context *ctx)
2166 {
2167 size_t i;
2168 for (i = 0; i < ctx->trap_records_size; i++) {
2169 struct trap_record *tr = &ctx->trap_records[i];
2170 tr->destination_ip = ctx->label_to_pos[tr->destination_ip];
2171 }
2172 }
2173
2174
2175 static bool attr_w codegen_map(struct codegen_context *ctx)
2176 {
2177 void *ptr;
2178 frame_t i;
2179 array_finish(uint8_t, &ctx->mcode, &ctx->mcode_size);
2180 ptr = os_code_map(ctx->mcode, ctx->mcode_size, &ctx->err);
2181 ctx->mcode = NULL;
2182 if (unlikely(!ptr)) {
2183 return false;
2184 }
2185 for (i = 0; i < ctx->n_entries; i++) {
2186 char *entry = cast_ptr(char *, ptr) + ctx->entries[i].entry_to_pos;
2187 da(ctx->codegen,codegen)->unoptimized_code[i] = entry;
2188 da(ctx->codegen,codegen)->n_entries++;
2189 }
2190 da(ctx->codegen,codegen)->unoptimized_code_base = ptr;
2191 da(ctx->codegen,codegen)->unoptimized_code_size = ctx->mcode_size;
2192
2193 return true;
2194 }
2195
2196
2197 void *codegen_fn(frame_s *fp, const code_t *ip, union internal_arg ia[])
2198 {
2199 struct codegen_context ctx_;
2200 struct codegen_context *ctx = &ctx_;
2201 frame_t i;
2202 int8_t rr;
2203 struct data *codegen;
2204 uint32_t l;
2205
2206 init_ctx(ctx);
2207 ctx->fn = ia[0].ptr;
2208
2209 #ifdef DEBUG_ENV
2210 if (getenv("CG") && strcmp(da(ctx->fn,function)->function_name, getenv("CG")))
2211 goto fail;
2212 #endif
2213
2214 ctx->local_directory = mem_alloc_array_mayfail(mem_calloc_mayfail, struct data **, 0, 0, da(ctx->fn,function)->local_directory_size, sizeof(struct data *), &ctx->err);
2215 if (unlikely(!ctx->local_directory))
2216 goto fail;
2217
2218 if (0) for (i = 0; i < da(ctx->fn,function)->local_directory_size; i++) {
2219 struct data *callee;
2220 pointer_t *ptr;
2221 ptr = da(ctx->fn,function)->local_directory[i];
2222 pointer_follow(ptr, false, callee, PF_SPARK, NULL, 0,
2223 SUBMIT_EX(ex_);
2224 goto next_one,
2225 goto next_one;
2226 );
2227 ctx->local_directory[i] = callee;
2228 next_one:;
2229 }
2230 for (i = 0; i < da(ctx->fn,function)->local_directory_size; i++) {
2231 struct data *callee;
2232 pointer_t *ptr;
2233 if (ctx->local_directory[i])
2234 continue;
2235 ptr = da(ctx->fn,function)->local_directory[i];
2236 pointer_follow(ptr, false, callee, PF_WAIT, fp, ip,
2237 done_ctx(ctx);
2238 return ex_,
2239 goto fail
2240 );
2241 ctx->local_directory[i] = callee;
2242 /*debug("processing call: %s -> %s", da(ctx->fn,function)->function_name, da(callee,function)->function_name);*/
2243 }
2244
2245 if (da(ctx->fn,function)->module_designator) {
2246 struct function_descriptor *sfd = save_find_function_descriptor(da(ctx->fn,function)->module_designator, da(ctx->fn,function)->function_designator);
2247 if (sfd && sfd->unoptimized_code_size) {
2248 codegen = data_alloc_flexible(codegen, unoptimized_code, sfd->n_entries, &ctx->err);
2249 if (unlikely(!codegen))
2250 goto fail;
2251 da(codegen,codegen)->unoptimized_code_base = sfd->unoptimized_code_base;
2252 da(codegen,codegen)->unoptimized_code_size = sfd->unoptimized_code_size;
2253 da(codegen,codegen)->function = ctx->fn;
2254 da(codegen,codegen)->is_saved = true;
2255 da(codegen,codegen)->n_entries = sfd->n_entries;
2256 da(codegen,codegen)->offsets = NULL;
2257 for (i = 0; i < sfd->n_entries; i++) {
2258 da(codegen,codegen)->unoptimized_code[i] = cast_ptr(char *, da(codegen,codegen)->unoptimized_code_base) + sfd->entries[i];
2259 /*debug("%s: %p + %lx -> %p", da(ctx->fn,function)->function_name, da(codegen,codegen)->unoptimized_code_base, sfd->entries[i], da(codegen,codegen)->unoptimized_code[i]);*/
2260 }
2261 #ifdef HAVE_CODEGEN_TRAPS
2262 da(codegen,codegen)->trap_records = sfd->trap_records;
2263 da(codegen,codegen)->trap_records_size = sfd->trap_records_size;
2264 data_trap_insert(codegen);
2265 #endif
2266 goto have_codegen;
2267 }
2268 }
2269
2270 /*debug("trying: %s", da(ctx->fn,function)->function_name);*/
2271 if (unlikely(!array_init_mayfail(uint8_t, &ctx->code, &ctx->code_size, &ctx->err)))
2272 goto fail;
2273
2274 ctx->code_labels = mem_alloc_array_mayfail(mem_calloc_mayfail, uint32_t *, 0, 0, da(ctx->fn,function)->code_size, sizeof(uint32_t), &ctx->err);
2275 if (unlikely(!ctx->code_labels))
2276 goto fail;
2277
2278 ctx->code_exits = mem_alloc_array_mayfail(mem_calloc_mayfail, struct cg_exit **, 0, 0, da(ctx->fn,function)->code_size, sizeof(struct cg_exit *), &ctx->err);
2279 if (unlikely(!ctx->code_exits))
2280 goto fail;
2281
2282 ctx->flag_cache = mem_alloc_array_mayfail(mem_calloc_mayfail, uint8_t *, 0, 0, function_n_variables(ctx->fn), sizeof(int8_t), &ctx->err);
2283 if (unlikely(!ctx->flag_cache))
2284 goto fail;
2285
2286 ctx->registers = mem_alloc_array_mayfail(mem_alloc_mayfail, short *, 0, 0, function_n_variables(ctx->fn), sizeof(short), &ctx->err);
2287 if (unlikely(!ctx->registers))
2288 goto fail;
2289
2290 if (unlikely(!array_init_mayfail(frame_t, &ctx->need_spill, &ctx->need_spill_l, &ctx->err)))
2291 goto fail;
2292
2293 if (unlikely(!gen_registers(ctx)))
2294 goto fail;
2295
2296 if (unlikely(!gen_function(ctx)))
2297 goto fail;
2298
2299 if (unlikely(!gen_entries(ctx)))
2300 goto fail;
2301
2302 if (unlikely(!gen_epilogues(ctx)))
2303 goto fail;
2304
2305 if (unlikely(!(ctx->label_id + 1)))
2306 goto fail;
2307 if (unlikely(!(ctx->label_to_pos = mem_alloc_array_mayfail(mem_alloc_mayfail, size_t *, 0, 0, ctx->label_id + 1, sizeof(size_t), &ctx->err))))
2308 goto fail;
2309
2310 again:
2311 for (l = 0; l < ctx->label_id + 1; l++)
2312 ctx->label_to_pos[l] = (size_t)-1;
2313
2314 if (unlikely(!array_init_mayfail(uint8_t, &ctx->mcode, &ctx->mcode_size, &ctx->err)))
2315 goto fail;
2316
2317 if (unlikely(!array_init_mayfail(struct relocation, &ctx->reloc, &ctx->reloc_size, &ctx->err)))
2318 goto fail;
2319
2320 if (unlikely(!array_init_mayfail(struct trap_record, &ctx->trap_records, &ctx->trap_records_size, &ctx->err)))
2321 goto fail;
2322
2323 #ifdef ARCH_CONTEXT
2324 init_arch_context(ctx);
2325 #endif
2326
2327 if (unlikely(!gen_mcode(ctx)))
2328 goto fail;
2329
2330 rr = resolve_relocs(ctx);
2331 if (unlikely(rr == RELOCS_FAIL)) {
2332 /*debug("relocation fail: %s", da(ctx->fn,function)->function_name);*/
2333 goto fail;
2334 }
2335 if (rr == RELOCS_RETRY) {
2336 mem_free(ctx->mcode);
2337 ctx->mcode = NULL;
2338 mem_free(ctx->reloc);
2339 ctx->reloc = NULL;
2340 mem_free(ctx->trap_records);
2341 ctx->trap_records = NULL;
2342 goto again;
2343 }
2344
2345 resolve_traps(ctx);
2346
2347 #ifdef DEBUG_ENV
2348 if ((getenv("DUMP") && !strcmp(getenv("DUMP"), da(ctx->fn,function)->function_name)) || getenv("DUMP_ALL")) {
2349 char *hex;
2350 size_t hexl;
2351 size_t i;
2352 handle_t h;
2353
2354 mutex_lock(&dump_mutex);
2355 str_init(&hex, &hexl);
2356 str_add_string(&hex, &hexl, "_");
2357 str_add_unsigned(&hex, &hexl, dump_seq++, 10);
2358 str_add_string(&hex, &hexl, "_");
2359 str_add_string(&hex, &hexl, da(ctx->fn,function)->function_name);
2360 str_add_string(&hex, &hexl, ":");
2361 for (i = 0; i < hexl; i++)
2362 if (hex[i] == '/')
2363 hex[i] = '_';
2364 for (i = 0; i < ctx->mcode_size; i++) {
2365 uint8_t a = ctx->mcode[i];
2366 if (!(i & 0xff))
2367 str_add_string(&hex, &hexl, "\n .byte 0x");
2368 else
2369 str_add_string(&hex, &hexl, ",0x");
2370 if (a < 16)
2371 str_add_char(&hex, &hexl, '0');
2372 str_add_unsigned(&hex, &hexl, a, 16);
2373 }
2374 str_add_string(&hex, &hexl, "\n");
2375 h = os_open(os_cwd, "dump.s", O_WRONLY | O_APPEND, 0600, NULL);
2376 os_write_all(h, hex, hexl, NULL);
2377 os_close(h);
2378 mem_free(hex);
2379 mutex_unlock(&dump_mutex);
2380 }
2381 #endif
2382
2383 ctx->codegen = data_alloc_flexible(codegen, unoptimized_code, ctx->n_entries, &ctx->err);
2384 if (unlikely(!ctx->codegen))
2385 goto fail;
2386 da(ctx->codegen,codegen)->function = ctx->fn;
2387 da(ctx->codegen,codegen)->is_saved = false;
2388 da(ctx->codegen,codegen)->n_entries = 0;
2389 da(ctx->codegen,codegen)->offsets = NULL;
2390
2391 if (unlikely(!codegen_map(ctx)))
2392 goto fail;
2393
2394 codegen = ctx->codegen;
2395 ctx->codegen = NULL;
2396
2397 #ifdef HAVE_CODEGEN_TRAPS
2398 da(codegen,codegen)->trap_records = ctx->trap_records;
2399 da(codegen,codegen)->trap_records_size = ctx->trap_records_size;
2400 ctx->trap_records = NULL;
2401 data_trap_insert(codegen);
2402 #endif
2403
2404 have_codegen:
2405 done_ctx(ctx);
2406 return function_return(fp, pointer_data(codegen));
2407
2408 fail:
2409 /*debug("FAILED: %s", da(ctx->fn,function)->function_name);*/
2410 done_ctx(ctx);
2411 return function_return(fp, pointer_thunk(thunk_alloc_exception_error(error_ajla(EC_SYNC, AJLA_ERROR_NOT_SUPPORTED), NULL, NULL, NULL pass_file_line)));
2412 }
2413
2414 void codegen_free(struct data *codegen)
2415 {
2416 if (unlikely(da(codegen,codegen)->offsets != NULL))
2417 mem_free(da(codegen,codegen)->offsets);
2418 if (likely(da(codegen,codegen)->is_saved))
2419 return;
2420 #ifdef HAVE_CODEGEN_TRAPS
2421 mem_free(da(codegen,codegen)->trap_records);
2422 #endif
2423 os_code_unmap(da(codegen,codegen)->unoptimized_code_base, da(codegen,codegen)->unoptimized_code_size);
2424 }
2425
2426 #if defined(ARCH_IA64)
2427 static uintptr_t ia64_stub[2];
2428 #endif
2429 #if defined(ARCH_PARISC32) && defined(ARCH_PARISC_USE_STUBS)
2430 static uintptr_t parisc_stub[2];
2431 #endif
2432 #if defined(ARCH_PARISC64) && defined(ARCH_PARISC_USE_STUBS)
2433 static uintptr_t parisc_stub[4];
2434 #endif
2435 #if defined(ARCH_POWER) && defined(AIX_CALL)
2436 static uintptr_t ppc_stub[3];
2437 #endif
2438
2439 void name(codegen_init)(void)
2440 {
2441 struct codegen_context ctx_;
2442 struct codegen_context *ctx = &ctx_;
2443 void *ptr;
2444
2445 #if (defined(ARCH_X86_64) || defined(ARCH_X86_X32)) && !defined(ARCH_X86_WIN_ABI)
2446 #if defined(HAVE_SYSCALL) && defined(HAVE_ASM_PRCTL_H) && defined(HAVE_SYS_SYSCALL_H)
2447 if (!dll) {
2448 int r;
2449 EINTR_LOOP(r, syscall(SYS_arch_prctl, ARCH_SET_GS, &cg_upcall_vector));
2450 if (!r)
2451 upcall_register = R_GS;
2452 }
2453 #elif defined(HAVE_AMD64_SET_GSBASE) && defined(HAVE_X86_SYSARCH_H)
2454 if (!dll) {
2455 int r;
2456 EINTR_LOOP(r, amd64_set_gsbase(&cg_upcall_vector));
2457 if (!r)
2458 upcall_register = R_GS;
2459 }
2460 #elif defined(HAVE_SYSARCH) && defined(HAVE_X86_SYSARCH_H) && defined(X86_64_SET_GSBASE)
2461 if (!dll) {
2462 int r;
2463 void *ptr = &cg_upcall_vector;
2464 EINTR_LOOP(r, sysarch(X86_64_SET_GSBASE, &ptr));
2465 if (!r)
2466 upcall_register = R_GS;
2467 }
2468 #endif
2469 #endif
2470
2471 init_ctx(ctx);
2472 ctx->fn = NULL;
2473
2474 array_init(uint8_t, &ctx->code, &ctx->code_size);
2475
2476 if (unlikely(!gen_entry(ctx)))
2477 goto fail;
2478
2479 array_init(uint8_t, &ctx->mcode, &ctx->mcode_size);
2480
2481 #ifdef ARCH_CONTEXT
2482 init_arch_context(ctx);
2483 #endif
2484
2485 if (unlikely(!gen_mcode(ctx)))
2486 goto fail;
2487
2488 array_finish(uint8_t, &ctx->mcode, &ctx->mcode_size);
2489 ptr = os_code_map(ctx->mcode, ctx->mcode_size, NULL);
2490 codegen_ptr = ptr;
2491 codegen_size = ctx->mcode_size;
2492 ctx->mcode = NULL;
2493 #if defined(ARCH_IA64)
2494 ia64_stub[0] = ptr_to_num(ptr);
2495 ia64_stub[1] = 0;
2496 codegen_entry = cast_ptr(codegen_type, ia64_stub);
2497 #elif defined(ARCH_PARISC32) && defined(ARCH_PARISC_USE_STUBS)
2498 parisc_stub[0] = ptr_to_num(ptr);
2499 parisc_stub[1] = 0;
2500 codegen_entry = cast_ptr(codegen_type, cast_ptr(char *, parisc_stub) + 2);
2501 #elif defined(ARCH_PARISC64) && defined(ARCH_PARISC_USE_STUBS)
2502 parisc_stub[0] = 0;
2503 parisc_stub[1] = 0;
2504 parisc_stub[2] = ptr_to_num(ptr);
2505 parisc_stub[3] = 0;
2506 codegen_entry = cast_ptr(codegen_type, parisc_stub);
2507 #elif defined(ARCH_POWER) && defined(AIX_CALL)
2508 ppc_stub[0] = ptr_to_num(ptr);
2509 ppc_stub[1] = 0;
2510 ppc_stub[2] = 0;
2511 codegen_entry = cast_ptr(codegen_type, ppc_stub);
2512 #else
2513 codegen_entry = ptr;
2514 #endif
2515 done_ctx(ctx);
2516
2517 #ifdef DEBUG_ENV
2518 mutex_init(&dump_mutex);
2519 if (getenv("DUMP") || getenv("DUMP_ALL")) {
2520 size_t i;
2521 char *hex;
2522 size_t hexl;
2523 str_init(&hex, &hexl);
2524 #if defined(ARCH_RISCV64)
2525 str_add_string(&hex, &hexl, " .attribute arch, \"rv64i2p1_m2p0_a2p1_f2p2_d2p2_c2p0_zicsr2p0_zifencei2p0_zba1p0_zbb1p0_zbc1p0_zbs1p0\"\n");
2526 #endif
2527 for (i = 0; i < codegen_size; i++) {
2528 uint8_t a = cast_ptr(uint8_t *, codegen_ptr)[i];
2529 str_add_string(&hex, &hexl, " .byte 0x");
2530 if (a < 16)
2531 str_add_char(&hex, &hexl, '0');
2532 str_add_unsigned(&hex, &hexl, a, 16);
2533 str_add_char(&hex, &hexl, '\n');
2534 }
2535 os_write_atomic(".", "dump.s", hex, hexl, NULL);
2536 mem_free(hex);
2537 }
2538 #endif
2539
2540 return;
2541
2542 fail:
2543 fatal("couldn't compile global entry");
2544 }
2545
2546 void name(codegen_done)(void)
2547 {
2548 os_code_unmap(codegen_ptr, codegen_size);
2549 #ifdef DEBUG_ENV
2550 mutex_done(&dump_mutex);
2551 #endif
2552 }
2553
2554 #else
2555
2556 void name(codegen_init)(void)
2557 {
2558 }
2559
2560 void name(codegen_done)(void)
2561 {
2562 }
2563
2564 #endif
2565
2566 #endif
Ajla - a purely functional programming language