?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
codegen: gen_io: fix double-free of ctx->var_aux if
[ajla.git] / codegen.c
blob165d27f50482f1155229a3b22d975ef4707daa06
1 /*
2 * Copyright (C) 2024 Mikulas Patocka
3 *
4 * This file is part of Ajla.
5 *
6 * Ajla is free software: you can redistribute it and/or modify it under the
7 * terms of the GNU General Public License as published by the Free Software
8 * Foundation, either version 3 of the License, or (at your option) any later
9 * version.
10 *
11 * Ajla is distributed in the hope that it will be useful, but WITHOUT ANY
12 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
13 * A PARTICULAR PURPOSE. See the GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License along with
16 * Ajla. If not, see <https://www.gnu.org/licenses/>.
17 */
18
19 #include "ajla.h"
20
21 #ifndef FILE_OMIT
22
23 #include "data.h"
24 #include "os.h"
25 #include "os_util.h"
26 #include "util.h"
27 #include "ipfn.h"
28 #include "ipret.h"
29 #include "funct.h"
30 #include "thread.h"
31 #include "task.h"
32 #include "save.h"
33
34 #include "codegen.h"
35
36 #ifdef HAVE_CODEGEN
37
38 #define flag_cache_chicken 0
39 #define must_be_flat_chicken 0
40 #define ra_chicken 0
41
42 #define INLINE_BITMAP_SLOTS 16
43 #define INLINE_COPY_SIZE 64
44
45 /*#define DEBUG_INSNS*/
46 /*#define DEBUG_GARBAGE*/
47
48 #if (defined(ARCH_X86_64) || defined(ARCH_X86_X32)) && !defined(ARCH_X86_WIN_ABI)
49 #if defined(HAVE_SYSCALL) && defined(HAVE_ASM_PRCTL_H) && defined(HAVE_SYS_SYSCALL_H)
50 #include <asm/prctl.h>
51 #include <sys/syscall.h>
52 #include <unistd.h>
53 #endif
54 #if (defined(HAVE_AMD64_SET_GSBASE) || defined(HAVE_SYSARCH)) && defined(HAVE_X86_SYSARCH_H)
55 #include <x86/sysarch.h>
56 #endif
57 #endif
58
59 code_return_t (*codegen_entry)(frame_s *, struct cg_upcall_vector_s *, tick_stamp_t, void *);
60 static void *codegen_ptr;
61 static size_t codegen_size;
62
63 #ifdef DEBUG_ENV
64 static mutex_t dump_mutex;
65 static uint64_t dump_seq = 0;
66 #endif
67
68 /*
69 * insn:
70 * opcode - 16 bits
71 * op_size - 3 bits
72 * aux - 7 bits
73 * writes flags - 2 bit
74 * jmp size - 2 bits
75 */
76
77 #define INSN_OPCODE 0x0000ffffUL
78 #define INSN_OP_SIZE 0x00070000UL
79 #define INSN_AUX 0x03f80000UL
80 #define INSN_WRITES_FLAGS 0x0c000000UL
81 #define INSN_JUMP_SIZE 0x30000000UL
82
83 #define INSN_OPCODE_SHIFT 0
84 #define INSN_OP_SIZE_SHIFT 16
85 #define INSN_AUX_SHIFT 19
86 #define INSN_WRITES_FLAGS_SHIFT 26
87 #define INSN_JUMP_SIZE_SHIFT 28
88
89 #define insn_opcode(insn) (((insn) >> INSN_OPCODE_SHIFT) & (INSN_OPCODE >> INSN_OPCODE_SHIFT))
90 #define insn_op_size(insn) (((insn) >> INSN_OP_SIZE_SHIFT) & (INSN_OP_SIZE >> INSN_OP_SIZE_SHIFT))
91 #define insn_aux(insn) (((insn) >> INSN_AUX_SHIFT) & (INSN_AUX >> INSN_AUX_SHIFT))
92 #define insn_writes_flags(insn) (((insn) >> INSN_WRITES_FLAGS_SHIFT) & (INSN_WRITES_FLAGS >> INSN_WRITES_FLAGS_SHIFT))
93 #define insn_jump_size(insn) (((insn) >> INSN_JUMP_SIZE_SHIFT) & (INSN_JUMP_SIZE >> INSN_JUMP_SIZE_SHIFT))
94
95 #define ALU_ADD 0x00
96 #define ALU_OR 0x01
97 #define ALU_ADC 0x02
98 #define ALU_SBB 0x03
99 #define ALU_AND 0x04
100 #define ALU_SUB 0x05
101 #define ALU_XOR 0x06
102 #define ALU_ORN 0x08
103 #define ALU_ANDN 0x09
104 #define ALU_XORN 0x0a
105 #define ALU_MUL 0x10
106 #define ALU_UMULH 0x11
107 #define ALU_SMULH 0x12
108 #define ALU_UDIV 0x13
109 #define ALU_SDIV 0x14
110 #define ALU_UREM 0x15
111 #define ALU_SREM 0x16
112 #define ALU_SAVE 0x17
113 #define ALU_EXTBL 0x18
114 #define ALU_EXTWL 0x19
115 #define ALU_EXTLL 0x1a
116 #define ALU_EXTLH 0x1b
117 #define ALU_INSBL 0x1c
118 #define ALU_MSKBL 0x1d
119 #define ALU_ZAP 0x20
120 #define ALU_ZAPNOT 0x21
121
122 #define ALU1_NOT 0x00
123 #define ALU1_NEG 0x01
124 #define ALU1_NGC 0x02
125 #define ALU1_INC 0x03
126 #define ALU1_DEC 0x04
127 #define ALU1_BSWAP 0x05
128 #define ALU1_BSWAP16 0x06
129 #define ALU1_BREV 0x07
130 #define ALU1_BSF 0x08
131 #define ALU1_BSR 0x09
132 #define ALU1_LZCNT 0x0a
133 #define ALU1_POPCNT 0x0b
134
135 #define FP_ALU_ADD 0
136 #define FP_ALU_SUB 1
137 #define FP_ALU_MUL 2
138 #define FP_ALU_DIV 3
139 #define FP_ALU1_NEG 0
140 #define FP_ALU1_SQRT 1
141 #define FP_ALU1_ROUND 2
142 #define FP_ALU1_FLOOR 3
143 #define FP_ALU1_CEIL 4
144 #define FP_ALU1_TRUNC 5
145 #define FP_ALU1_VCNT8 6
146 #define FP_ALU1_VPADDL 7
147 #define FP_ALU1_ADDV 8
148
149 #define COND_O 0x0
150 #define COND_NO 0x1
151 #define COND_B 0x2
152 #define COND_AE 0x3
153 #define COND_E 0x4
154 #define COND_NE 0x5
155 #define COND_BE 0x6
156 #define COND_A 0x7
157 #define COND_S 0x8
158 #define COND_NS 0x9
159 #define COND_P 0xa
160 #define COND_NP 0xb
161 #define COND_L 0xc
162 #define COND_GE 0xd
163 #define COND_LE 0xe
164 #define COND_G 0xf
165 #define COND_BLBC 0x10
166 #define COND_BLBS 0x11
167 #define COND_ALWAYS 0x12
168
169 #define COND_FP 0x20
170 #define FP_COND_P (COND_FP | COND_P)
171 #define FP_COND_NP (COND_FP | COND_NP)
172 #define FP_COND_E (COND_FP | COND_E)
173 #define FP_COND_NE (COND_FP | COND_NE)
174 #define FP_COND_A (COND_FP | COND_A)
175 #define FP_COND_BE (COND_FP | COND_BE)
176 #define FP_COND_B (COND_FP | COND_B)
177 #define FP_COND_AE (COND_FP | COND_AE)
178
179 #define ROT_ROL 0x0
180 #define ROT_ROR 0x1
181 #define ROT_RCL 0x2
182 #define ROT_RCR 0x3
183 #define ROT_SHL 0x4
184 #define ROT_SHR 0x5
185 #define ROT_SAR 0x7
186 #define ROT_SAL 0x8
187
188 #define BTX_BT 0x0
189 #define BTX_BTS 0x1
190 #define BTX_BTR 0x2
191 #define BTX_BTC 0x3
192 #define BTX_BTEXT 0x4
193
194 #define OP_SIZE_1 0
195 #define OP_SIZE_2 1
196 #define OP_SIZE_4 2
197 #define OP_SIZE_8 3
198 #define OP_SIZE_16 4
199 #define OP_SIZE_10 7
200
201 #define MOV_MASK_0_16 0x0
202 #define MOV_MASK_16_32 0x1
203 #define MOV_MASK_32_48 0x2
204 #define MOV_MASK_48_64 0x3
205 #define MOV_MASK_0_8 0x4
206 #define MOV_MASK_32_64 0x5
207 #define MOV_MASK_52_64 0x6
208
209 #define JMP_SHORTEST 0x0000
210 #define JMP_SHORT 0x0001
211 #define JMP_LONG 0x0002
212 #define JMP_EXTRA_LONG 0x0003
213
214 enum {
215 INSN_ENTRY,
216 INSN_LABEL,
217 INSN_RET,
218 INSN_RET_IMM,
219 INSN_ARM_PUSH,
220 INSN_ARM_POP,
221 INSN_S390_PUSH,
222 INSN_S390_POP,
223 INSN_IA64_ALLOC,
224 INSN_IA64_DEALLOC,
225 INSN_PUSH,
226 INSN_POP,
227 INSN_CALL,
228 INSN_CALL_INDIRECT,
229 INSN_MOV,
230 INSN_MOVSX,
231 INSN_MOV_U,
232 INSN_MOV_LR,
233 INSN_CMP,
234 INSN_CMP_DEST_REG,
235 INSN_CMN,
236 INSN_TEST,
237 INSN_TEST_DEST_REG,
238 INSN_ALU,
239 INSN_ALU_PARTIAL,
240 INSN_ALU_FLAGS,
241 INSN_ALU_FLAGS_PARTIAL,
242 INSN_ALU_TRAP,
243 INSN_ALU_FLAGS_TRAP,
244 INSN_ALU1,
245 INSN_ALU1_PARTIAL,
246 INSN_ALU1_FLAGS,
247 INSN_ALU1_FLAGS_PARTIAL,
248 INSN_ALU1_TRAP,
249 INSN_LEA3,
250 INSN_ROT,
251 INSN_ROT_PARTIAL,
252 INSN_BT,
253 INSN_BTX,
254 INSN_MUL_L,
255 INSN_DIV_L,
256 INSN_MADD,
257 INSN_CBW,
258 INSN_CBW_PARTIAL,
259 INSN_CWD,
260 INSN_CWD_PARTIAL,
261 INSN_SET_COND,
262 INSN_SET_COND_PARTIAL,
263 INSN_CMOV,
264 INSN_CMOV_XCC,
265 INSN_CMP_CMOV,
266 INSN_MOVR,
267 INSN_CSEL_SEL,
268 INSN_CSEL_INC,
269 INSN_CSEL_INV,
270 INSN_CSEL_NEG,
271 INSN_STP,
272 INSN_LDP,
273 INSN_MOV_MASK,
274 INSN_MEMCPY,
275 INSN_MEMSET,
276 INSN_FP_CMP,
277 INSN_FP_CMP_DEST_REG,
278 INSN_FP_CMP_DEST_REG_TRAP,
279 INSN_FP_CMP_UNORDERED_DEST_REG,
280 INSN_FP_CMP_COND,
281 INSN_FP_TEST_REG,
282 INSN_FP_TO_INT_FLAGS,
283 INSN_FP_ALU,
284 INSN_FP_ALU1,
285 INSN_FP_TO_INT32,
286 INSN_FP_TO_INT64,
287 INSN_FP_TO_INT64_TRAP,
288 INSN_FP_FROM_INT32,
289 INSN_FP_FROM_INT64,
290 INSN_FP_INT64_TO_INT32_TRAP,
291 INSN_FP_CVT,
292 INSN_X87_FLD,
293 INSN_X87_FILD,
294 INSN_X87_FSTP,
295 INSN_X87_FISTP,
296 INSN_X87_FISTTP,
297 INSN_X87_FCOMP,
298 INSN_X87_FCOMPP,
299 INSN_X87_FCOMIP,
300 INSN_X87_ALU,
301 INSN_X87_ALUP,
302 INSN_X87_FCHS,
303 INSN_X87_FSQRT,
304 INSN_X87_FRNDINT,
305 INSN_X87_FNSTSW,
306 INSN_X87_FLDCW,
307 INSN_JMP,
308 INSN_JMP_COND,
309 INSN_JMP_COND_LOGICAL,
310 INSN_JMP_REG,
311 INSN_JMP_REG_BIT,
312 INSN_JMP_2REGS,
313 INSN_JMP_FP_TEST,
314 INSN_JMP_INDIRECT,
315 INSN_MB,
316 INSN_CALL_MILLICODE,
317 };
318
319 #define ARG_REGS_MAX 0xc0
320 #define ARG_SHIFTED_REGISTER 0xc0
321 #define ARG_SHIFT_AMOUNT 0x3f
322 #define ARG_SHIFT_MODE 0xc0
323 #define ARG_SHIFT_LSL 0x00
324 #define ARG_SHIFT_LSR 0x40
325 #define ARG_SHIFT_ASR 0x80
326 #define ARG_SHIFT_ROR 0xc0
327 #define ARG_EXTENDED_REGISTER 0xc1
328 #define ARG_EXTEND_SHIFT 0x07
329 #define ARG_EXTEND_MODE 0x38
330 #define ARG_EXTEND_UXTB 0x00
331 #define ARG_EXTEND_UXTH 0x08
332 #define ARG_EXTEND_UXTW 0x10
333 #define ARG_EXTEND_UXTX 0x18
334 #define ARG_EXTEND_SXTB 0x20
335 #define ARG_EXTEND_SXTH 0x28
336 #define ARG_EXTEND_SXTW 0x30
337 #define ARG_EXTEND_SXTX 0x38
338 #define ARG_ADDRESS_0 0xd0
339 #define ARG_ADDRESS_1 0xd1
340 #define ARG_ADDRESS_1_2 0xd2
341 #define ARG_ADDRESS_1_4 0xd3
342 #define ARG_ADDRESS_1_8 0xd4
343 #define ARG_ADDRESS_1_PRE_I 0xd5
344 #define ARG_ADDRESS_1_POST_I 0xd6
345 #define ARG_ADDRESS_2 0xd7
346 #define ARG_ADDRESS_2_2 0xd8
347 #define ARG_ADDRESS_2_4 0xd9
348 #define ARG_ADDRESS_2_8 0xda
349 #define ARG_ADDRESS_2_UXTW 0xdb
350 #define ARG_ADDRESS_2_SXTW 0xdc
351 #define ARG_IMM 0xe0
352
353 #define ARG_IS_ADDRESS(a) ((a) >= ARG_ADDRESS_0 && (a) <= ARG_ADDRESS_2_SXTW)
354
355 #ifdef POINTER_COMPRESSION
356 #define OP_SIZE_SLOT OP_SIZE_4
357 #else
358 #define OP_SIZE_SLOT OP_SIZE_ADDRESS
359 #endif
360
361 #define OP_SIZE_BITMAP (bitmap_64bit ? OP_SIZE_8 : OP_SIZE_4)
362
363 #define OP_SIZE_INT log_2(sizeof(int_default_t))
364
365 #define check_insn(insn) \
366 do { \
367 /*if ((insn_opcode(insn) == INSN_ALU || insn_opcode(insn) == INSN_ALU1) && insn_op_size(insn) != OP_SIZE_NATIVE) internal(file_line, "invalid insn %08x", (unsigned)(insn));*/\
368 /*if (insn == 0x001a000e) internal(file_line, "invalid insn %08x", insn);*/\
369 } while (0)
370
371 #ifdef DEBUG_INSNS
372 #define gen_line() gen_four(__LINE__)
373 #else
374 #define gen_line() do { } while (0)
375 #endif
376
377 #ifdef ARCH_IA64
378 #define ARCH_CONTEXT struct { \
379 uint64_t insns[3]; \
380 uint8_t insn_units[3]; \
381 bool insn_stops[3]; \
382 uint64_t wr_mask[4]; \
383 }
384 #endif
385
386 #define gen_insn(opcode, op_size, aux, writes_flags) \
387 do { \
388 uint32_t dword = \
389 (uint32_t)(opcode) << INSN_OPCODE_SHIFT | \
390 (uint32_t)(op_size) << INSN_OP_SIZE_SHIFT | \
391 (uint32_t)(aux) << INSN_AUX_SHIFT | \
392 (uint32_t)(writes_flags) << INSN_WRITES_FLAGS_SHIFT; \
393 check_insn(dword); \
394 gen_line(); \
395 gen_four(dword); \
396 } while (0)
397
398 static size_t arg_size(uint8_t arg)
399 {
400 if (arg < ARG_REGS_MAX)
401 return 1;
402 if (arg >= ARG_SHIFTED_REGISTER && arg <= ARG_EXTENDED_REGISTER)
403 return 3;
404 if (arg == ARG_ADDRESS_0)
405 return 9;
406 if (arg >= ARG_ADDRESS_1 && arg <= ARG_ADDRESS_1_POST_I)
407 return 10;
408 if (arg >= ARG_ADDRESS_2 && arg <= ARG_ADDRESS_2_SXTW)
409 return 11;
410 if (arg == ARG_IMM)
411 return 9;
412 internal(file_line, "arg_size: invalid argument %02x", arg);
413 return 0;
414 }
415
416 struct relocation {
417 uint32_t label_id;
418 uint8_t length;
419 size_t position;
420 size_t jmp_instr;
421 };
422
423 struct code_arg {
424 frame_t slot;
425 frame_t flags;
426 frame_t type;
427 };
428
429 struct cg_entry {
430 size_t entry_to_pos;
431 frame_t *variables;
432 size_t n_variables;
433 uint32_t entry_label;
434 uint32_t nonflat_label;
435 };
436
437 struct cg_exit {
438 uint32_t undo_label;
439 uint8_t undo_opcode;
440 uint8_t undo_op_size;
441 uint8_t undo_aux;
442 uint8_t undo_writes_flags;
443 uint8_t undo_parameters[35];
444 uint8_t undo_parameters_len;
445 uint32_t escape_label;
446 };
447
448 #define FLAG_CACHE_IS_FLAT 0x01
449 #define FLAG_CACHE_IS_NOT_FLAT 0x02
450 #define FLAG_CACHE_IS_NOT_THUNK 0x04
451
452 struct codegen_context {
453 struct data *fn;
454 struct data **local_directory;
455
456 const code_t *instr_start;
457 const code_t *current_position;
458 uchar_efficient_t arg_mode;
459
460 uint32_t label_id;
461 struct cg_entry *entries;
462 frame_t n_entries;
463
464 uint8_t *code;
465 size_t code_size;
466
467 uint8_t *code_position;
468
469 uint32_t *code_labels;
470 struct cg_exit **code_exits;
471 uint32_t escape_nospill_label;
472 uint32_t call_label;
473 uint32_t reload_label;
474
475 uint8_t *mcode;
476 size_t mcode_size;
477
478 size_t *label_to_pos;
479 struct relocation *reloc;
480 size_t reloc_size;
481
482 struct trap_record *trap_records;
483 size_t trap_records_size;
484
485 struct code_arg *args;
486 size_t args_l;
487 const code_t *return_values;
488
489 uint8_t *flag_cache;
490 short *registers;
491 frame_t *need_spill;
492 size_t need_spill_l;
493
494 unsigned base_reg;
495 bool offset_reg;
496 int64_t offset_imm;
497
498 bool const_reg;
499 int64_t const_imm;
500
501 struct data *codegen;
502
503 int upcall_args;
504 frame_t *var_aux;
505
506 ajla_error_t err;
507
508 #ifdef ARCH_CONTEXT
509 ARCH_CONTEXT a;
510 #endif
511 };
512
513 static void init_ctx(struct codegen_context *ctx)
514 {
515 ctx->local_directory = NULL;
516 ctx->label_id = 0;
517 ctx->entries = NULL;
518 ctx->n_entries = 0;
519 ctx->code = NULL;
520 ctx->code_labels = NULL;
521 ctx->code_exits = NULL;
522 ctx->escape_nospill_label = 0;
523 ctx->call_label = 0;
524 ctx->reload_label = 0;
525 ctx->mcode = NULL;
526 ctx->label_to_pos = NULL;
527 ctx->reloc = NULL;
528 ctx->trap_records = NULL;
529 ctx->args = NULL;
530 ctx->flag_cache = NULL;
531 ctx->registers = NULL;
532 ctx->need_spill = NULL;
533 ctx->codegen = NULL;
534 ctx->upcall_args = -1;
535 ctx->var_aux = NULL;
536 }
537
538 static void done_ctx(struct codegen_context *ctx)
539 {
540 if (ctx->local_directory)
541 mem_free(ctx->local_directory);
542 if (ctx->entries) {
543 size_t i;
544 for (i = 0; i < ctx->n_entries; i++) {
545 struct cg_entry *ce = &ctx->entries[i];
546 if (ce->variables)
547 mem_free(ce->variables);
548 }
549 mem_free(ctx->entries);
550 }
551 if (ctx->code)
552 mem_free(ctx->code);
553 if (ctx->code_labels)
554 mem_free(ctx->code_labels);
555 if (ctx->code_exits) {
556 ip_t ip;
557 ip_t cs = da(ctx->fn,function)->code_size;
558 for (ip = 0; ip < cs; ip++) {
559 if (ctx->code_exits[ip])
560 mem_free(ctx->code_exits[ip]);
561 }
562 mem_free(ctx->code_exits);
563 }
564 if (ctx->mcode)
565 mem_free(ctx->mcode);
566 if (ctx->label_to_pos)
567 mem_free(ctx->label_to_pos);
568 if (ctx->reloc)
569 mem_free(ctx->reloc);
570 if (ctx->trap_records)
571 mem_free(ctx->trap_records);
572 if (ctx->args)
573 mem_free(ctx->args);
574 if (ctx->flag_cache)
575 mem_free(ctx->flag_cache);
576 if (ctx->registers)
577 mem_free(ctx->registers);
578 if (ctx->need_spill)
579 mem_free(ctx->need_spill);
580 if (ctx->codegen)
581 data_free(ctx->codegen);
582 if (ctx->var_aux)
583 mem_free(ctx->var_aux);
584 }
585
586
587 static inline code_t get_code(struct codegen_context *ctx)
588 {
589 ajla_assert(ctx->current_position < da(ctx->fn,function)->code + da(ctx->fn,function)->code_size, (file_line, "get_code: ran out of code"));
590 return *ctx->current_position++;
591 }
592
593 static inline uint32_t get_uint32(struct codegen_context *ctx)
594 {
595 uint32_t a1 = get_code(ctx);
596 uint32_t a2 = get_code(ctx);
597 #if !CODE_ENDIAN
598 return a1 + (a2 << 16);
599 #else
600 return a2 + (a1 << 16);
601 #endif
602 }
603
604 static int32_t get_jump_offset(struct codegen_context *ctx)
605 {
606 if (SIZEOF_IP_T == 2) {
607 return (int32_t)(int16_t)get_code(ctx);
608 } else if (SIZEOF_IP_T == 4) {
609 return (int32_t)get_uint32(ctx);
610 } else {
611 not_reached();
612 return -1;
613 }
614 }
615
616 static inline void get_one(struct codegen_context *ctx, frame_t *v)
617 {
618 if (!ctx->arg_mode) {
619 code_t c = get_code(ctx);
620 ajla_assert(!(c & ~0xff), (file_line, "get_one: high byte is not cleared: %u", (unsigned)c));
621 *v = c & 0xff;
622 } else if (ctx->arg_mode == 1) {
623 *v = get_code(ctx);
624 #if ARG_MODE_N >= 2
625 } else if (ctx->arg_mode == 2) {
626 *v = get_uint32(ctx);
627 #endif
628 } else {
629 internal(file_line, "get_one: invalid arg mode %u", ctx->arg_mode);
630 }
631 }
632
633 static inline void get_two(struct codegen_context *ctx, frame_t *v1, frame_t *v2)
634 {
635 if (!ctx->arg_mode) {
636 code_t c = get_code(ctx);
637 *v1 = c & 0xff;
638 *v2 = c >> 8;
639 } else if (ctx->arg_mode == 1) {
640 *v1 = get_code(ctx);
641 *v2 = get_code(ctx);
642 #if ARG_MODE_N >= 2
643 } else if (ctx->arg_mode == 2) {
644 *v1 = get_uint32(ctx);
645 *v2 = get_uint32(ctx);
646 #endif
647 } else {
648 internal(file_line, "get_two: invalid arg mode %u", ctx->arg_mode);
649 }
650 }
651
652
653 static uint32_t alloc_label(struct codegen_context *ctx)
654 {
655 return ++ctx->label_id;
656 }
657
658 static struct cg_exit *alloc_cg_exit_for_ip(struct codegen_context *ctx, const code_t *code)
659 {
660 ip_t ip = code - da(ctx->fn,function)->code;
661 struct cg_exit *ce = ctx->code_exits[ip];
662 if (!ce) {
663 ce = mem_calloc_mayfail(struct cg_exit *, sizeof(struct cg_exit), &ctx->err);
664 if (unlikely(!ce))
665 return NULL;
666 ctx->code_exits[ip] = ce;
667 }
668 return ce;
669 }
670
671 static struct cg_exit *alloc_undo_label(struct codegen_context *ctx)
672 {
673 struct cg_exit *ce = alloc_cg_exit_for_ip(ctx, ctx->instr_start);
674 if (unlikely(!ce))
675 return NULL;
676 if (unlikely(ce->undo_label != 0))
677 internal(file_line, "alloc_cg_exit: undo label already allocated");
678 ce->undo_label = alloc_label(ctx);
679 if (unlikely(!ce->undo_label))
680 return NULL;
681 return ce;
682 }
683
684 static uint32_t alloc_escape_label_for_ip(struct codegen_context *ctx, const code_t *code)
685 {
686 struct cg_exit *ce = alloc_cg_exit_for_ip(ctx, code);
687 if (!ce)
688 return 0;
689 if (!ce->escape_label)
690 ce->escape_label = alloc_label(ctx);
691 return ce->escape_label;
692 }
693
694 static uint32_t alloc_escape_label(struct codegen_context *ctx)
695 {
696 return alloc_escape_label_for_ip(ctx, ctx->instr_start);
697 }
698
699 static uint32_t attr_unused alloc_call_label(struct codegen_context *ctx)
700 {
701 if (!ctx->call_label) {
702 ctx->call_label = alloc_label(ctx);
703 }
704 return ctx->call_label;
705 }
706
707 static uint32_t alloc_reload_label(struct codegen_context *ctx)
708 {
709 if (!ctx->reload_label) {
710 ctx->reload_label = alloc_label(ctx);
711 }
712 return ctx->reload_label;
713 }
714
715 static size_t attr_unused mark_params(struct codegen_context *ctx)
716 {
717 return ctx->code_size;
718 }
719
720 static void attr_unused copy_params(struct codegen_context *ctx, struct cg_exit *ce, size_t mark)
721 {
722 if (ctx->code_size - mark > n_array_elements(ce->undo_parameters))
723 internal(file_line, "undo_parameters is too small: %"PRIuMAX" > %"PRIuMAX"", (uintmax_t)(ctx->code_size - mark), (uintmax_t)n_array_elements(ce->undo_parameters));
724 memcpy(ce->undo_parameters, ctx->code + mark, ctx->code_size - mark);
725 ce->undo_parameters_len = ctx->code_size - mark;
726 ctx->code_size = mark;
727 }
728
729 #define g(call) \
730 do { \
731 if (unlikely(!call)) \
732 return false; \
733 } while (0)
734
735 #define gen_one(byte) \
736 do { \
737 /*debug("gen %d: %02x", __LINE__, (uint8_t)(byte))*/; \
738 if (unlikely(!array_add_mayfail(uint8_t, &ctx->code, &ctx->code_size, byte, NULL, &ctx->err)))\
739 return false; \
740 } while (0)
741
742 #if defined(C_LITTLE_ENDIAN)
743 #define gen_two(word) \
744 do { \
745 uint16_t word_ = (word); \
746 /*debug("gen %d: %04x", __LINE__, (uint16_t)(word_));*/ \
747 if (unlikely(!array_add_multiple_mayfail(uint8_t, &ctx->code, &ctx->code_size, cast_ptr(uint8_t *, &word_), 2, NULL, &ctx->err)))\
748 return false; \
749 } while (0)
750 #define gen_four(dword) \
751 do { \
752 uint32_t dword_ = (dword); \
753 /*debug("gen %d: %08x", __LINE__, (uint32_t)(dword_));*/ \
754 if (unlikely(!array_add_multiple_mayfail(uint8_t, &ctx->code, &ctx->code_size, cast_ptr(uint8_t *, &dword_), 4, NULL, &ctx->err)))\
755 return false; \
756 } while (0)
757 #define gen_eight(qword) \
758 do { \
759 uint64_t qword_ = (qword); \
760 /*debug("gen %d: %016lx", __LINE__, (uint64_t)(qword_));*/ \
761 if (unlikely(!array_add_multiple_mayfail(uint8_t, &ctx->code, &ctx->code_size, cast_ptr(uint8_t *, &qword_), 8, NULL, &ctx->err)))\
762 return false; \
763 } while (0)
764 #else
765 #define gen_two(word) \
766 do { \
767 uint16_t word_ = (word); \
768 gen_one(word_ & 0xffU); \
769 gen_one(word_ >> 8); \
770 } while (0)
771 #define gen_four(dword) \
772 do { \
773 uint32_t dword_ = (dword); \
774 gen_two(dword_ & 0xffffU); \
775 gen_two(dword_ >> 15 >> 1); \
776 } while (0)
777 #define gen_eight(qword) \
778 do { \
779 uint64_t qword_ = (qword); \
780 gen_four(qword_ & 0xffffffffUL); \
781 gen_four(qword_ >> 15 >> 15 >> 2); \
782 } while (0)
783 #endif
784
785 #define gen_label(label_id) \
786 do { \
787 gen_insn(INSN_LABEL, 0, 0, 0); \
788 gen_four(label_id); \
789 } while (0)
790
791
792 static uint8_t attr_unused cget_one(struct codegen_context *ctx)
793 {
794 ajla_assert(ctx->code_position < ctx->code + ctx->code_size, (file_line, "cget_one: ran out of code"));
795 return *ctx->code_position++;
796 }
797
798 static uint16_t attr_unused cget_two(struct codegen_context *ctx)
799 {
800 #if defined(C_LITTLE_ENDIAN)
801 uint16_t r;
802 ajla_assert(ctx->code_position < ctx->code + ctx->code_size, (file_line, "cget_two: ran out of code"));
803 memcpy(&r, ctx->code_position, 2);
804 ctx->code_position += 2;
805 return r;
806 #else
807 uint16_t r = cget_one(ctx);
808 r |= cget_one(ctx) << 8;
809 return r;
810 #endif
811 }
812
813 static uint32_t cget_four(struct codegen_context *ctx)
814 {
815 #if defined(C_LITTLE_ENDIAN)
816 uint32_t r;
817 ajla_assert(ctx->code_position < ctx->code + ctx->code_size, (file_line, "cget_four: ran out of code"));
818 memcpy(&r, ctx->code_position, 4);
819 ctx->code_position += 4;
820 return r;
821 #else
822 uint32_t r = cget_two(ctx);
823 r |= (uint32_t)cget_two(ctx) << 16;
824 return r;
825 #endif
826 }
827
828 static uint64_t attr_unused cget_eight(struct codegen_context *ctx)
829 {
830 #if defined(C_LITTLE_ENDIAN)
831 uint64_t r;
832 ajla_assert(ctx->code_position < ctx->code + ctx->code_size, (file_line, "cget_eight: ran out of code"));
833 memcpy(&r, ctx->code_position, 8);
834 ctx->code_position += 8;
835 return r;
836 #else
837 uint64_t r = cget_four(ctx);
838 r |= (uint64_t)cget_four(ctx) << 32;
839 return r;
840 #endif
841 }
842
843 static int64_t get_imm(uint8_t *ptr)
844 {
845 #if defined(C_LITTLE_ENDIAN)
846 int64_t r;
847 memcpy(&r, ptr, 8);
848 return r;
849 #else
850 int64_t r;
851 r = (uint64_t)ptr[0] |
852 ((uint64_t)ptr[1] << 8) |
853 ((uint64_t)ptr[2] << 16) |
854 ((uint64_t)ptr[3] << 24) |
855 ((uint64_t)ptr[4] << 32) |
856 ((uint64_t)ptr[5] << 40) |
857 ((uint64_t)ptr[6] << 48) |
858 ((uint64_t)ptr[7] << 56);
859 return r;
860 #endif
861 }
862
863 #define cgen_one(byte) \
864 do { \
865 if (unlikely(!array_add_mayfail(uint8_t, &ctx->mcode, &ctx->mcode_size, byte, NULL, &ctx->err)))\
866 return false; \
867 } while (0)
868
869 #if defined(C_LITTLE_ENDIAN) || 1
870 #define cgen_two(word) \
871 do { \
872 uint16_t word_ = (word); \
873 if (unlikely(!array_add_multiple_mayfail(uint8_t, &ctx->mcode, &ctx->mcode_size, cast_ptr(uint8_t *, &word_), 2, NULL, &ctx->err)))\
874 return false; \
875 } while (0)
876 #define cgen_four(dword) \
877 do { \
878 uint32_t dword_ = (dword); \
879 /*if (dword_ == 0x1ee02000) internal(file_line, "invalid instruction");*/\
880 if (unlikely(!array_add_multiple_mayfail(uint8_t, &ctx->mcode, &ctx->mcode_size, cast_ptr(uint8_t *, &dword_), 4, NULL, &ctx->err)))\
881 return false; \
882 } while (0)
883 #define cgen_eight(qword) \
884 do { \
885 uint64_t qword_ = (qword); \
886 if (unlikely(!array_add_multiple_mayfail(uint8_t, &ctx->mcode, &ctx->mcode_size, cast_ptr(uint8_t *, &qword_), 8, NULL, &ctx->err)))\
887 return false; \
888 } while (0)
889 #else
890 #define cgen_two(word) \
891 do { \
892 cgen_one((word) & 0xff); \
893 cgen_one((word) >> 8); \
894 } while (0)
895 #define cgen_four(dword) \
896 do { \
897 cgen_two((dword) & 0xffff); \
898 cgen_two((dword) >> 15 >> 1); \
899 } while (0)
900 #define cgen_eight(qword) \
901 do { \
902 cgen_four((qword) & 0xffffffff); \
903 cgen_four((qword) >> 15 >> 15 >> 2); \
904 } while (0)
905 #endif
906
907
908 #define IMM_PURPOSE_LDR_OFFSET 1
909 #define IMM_PURPOSE_LDR_SX_OFFSET 2
910 #define IMM_PURPOSE_STR_OFFSET 3
911 #define IMM_PURPOSE_LDP_STP_OFFSET 4
912 #define IMM_PURPOSE_VLDR_VSTR_OFFSET 5
913 #define IMM_PURPOSE_MVI_CLI_OFFSET 6
914 #define IMM_PURPOSE_STORE_VALUE 7
915 #define IMM_PURPOSE_ADD 8
916 #define IMM_PURPOSE_SUB 9
917 #define IMM_PURPOSE_CMP 10
918 #define IMM_PURPOSE_CMP_LOGICAL 11
919 #define IMM_PURPOSE_AND 12
920 #define IMM_PURPOSE_OR 13
921 #define IMM_PURPOSE_XOR 14
922 #define IMM_PURPOSE_ANDN 15
923 #define IMM_PURPOSE_TEST 16
924 #define IMM_PURPOSE_JMP_2REGS 17
925 #define IMM_PURPOSE_MUL 18
926 #define IMM_PURPOSE_CMOV 19
927 #define IMM_PURPOSE_MOVR 20
928 #define IMM_PURPOSE_BITWISE 21
929
930
931 static bool attr_w gen_upcall_end(struct codegen_context *ctx, unsigned args);
932
933 #define gen_address_offset() \
934 do { \
935 if (likely(!ctx->offset_reg)) { \
936 gen_one(ARG_ADDRESS_1); \
937 gen_one(ctx->base_reg); \
938 gen_eight(ctx->offset_imm); \
939 } else { \
940 gen_one(ARG_ADDRESS_2); \
941 gen_one(ctx->base_reg); \
942 gen_one(R_OFFSET_IMM); \
943 gen_eight(0); \
944 } \
945 } while (0)
946
947 #define gen_imm_offset() \
948 do { \
949 if (likely(!ctx->const_reg)) { \
950 gen_one(ARG_IMM); \
951 gen_eight(ctx->const_imm); \
952 } else { \
953 gen_one(R_CONST_IMM); \
954 } \
955 } while (0)
956
957 #define is_imm() (!ctx->const_reg)
958
959
960 #if defined(ARCH_ALPHA)
961 #include "c1-alpha.inc"
962 #elif defined(ARCH_ARM32)
963 #include "c1-arm.inc"
964 #elif defined(ARCH_ARM64)
965 #include "c1-arm64.inc"
966 #elif defined(ARCH_IA64)
967 #include "c1-ia64.inc"
968 #elif defined(ARCH_LOONGARCH64)
969 #include "c1-loong.inc"
970 #elif defined(ARCH_MIPS)
971 #include "c1-mips.inc"
972 #elif defined(ARCH_PARISC)
973 #include "c1-hppa.inc"
974 #elif defined(ARCH_POWER)
975 #include "c1-power.inc"
976 #elif defined(ARCH_S390)
977 #include "c1-s390.inc"
978 #elif defined(ARCH_SPARC)
979 #include "c1-sparc.inc"
980 #elif defined(ARCH_RISCV64)
981 #include "c1-riscv.inc"
982 #elif defined(ARCH_X86)
983 #include "c1-x86.inc"
984 #endif
985
986
987 #ifndef ARCH_SUPPORTS_TRAPS
988 #define ARCH_SUPPORTS_TRAPS 0
989 #define ARCH_TRAP_BEFORE 0
990 #endif
991
992
993 #include "cg-util.inc"
994
995 #include "cg-frame.inc"
996
997 #include "cg-flags.inc"
998
999 #include "cg-flcch.inc"
1000
1001 #include "cg-ptr.inc"
1002
1003 #include "cg-alu.inc"
1004
1005 #include "cg-ops.inc"
1006
1007
1008 #ifndef n_regs_saved
1009 #define n_regs_saved n_array_elements(regs_saved)
1010 #endif
1011
1012 #ifndef n_regs_volatile
1013 #define n_regs_volatile n_array_elements(regs_volatile)
1014 #endif
1015
1016 #ifndef n_fp_saved
1017 #define n_fp_saved n_array_elements(fp_saved)
1018 #endif
1019
1020 #ifndef n_fp_volatile
1021 #define n_fp_volatile n_array_elements(fp_volatile)
1022 #endif
1023
1024 #ifndef n_vector_volatile
1025 #define n_vector_volatile n_array_elements(vector_volatile)
1026 #endif
1027
1028 static bool attr_w gen_registers(struct codegen_context *ctx)
1029 {
1030 frame_t v;
1031 size_t index_saved = 0;
1032 size_t index_volatile = 0;
1033 size_t index_fp_saved = 0;
1034 size_t index_fp_volatile = 0;
1035 size_t attr_unused index_vector_volatile = 0;
1036 #ifdef ARCH_S390
1037 bool uses_x = false;
1038 for (v = MIN_USEABLE_SLOT; v < function_n_variables(ctx->fn); v++) {
1039 const struct type *t = get_type_of_local(ctx, v);
1040 if (t && TYPE_TAG_IS_REAL(t->tag) && TYPE_TAG_IDX_REAL(t->tag) == 4) {
1041 uses_x = true;
1042 break;
1043 }
1044 }
1045 #endif
1046 /*for (v = function_n_variables(ctx->fn) - 1; v >= MIN_USEABLE_SLOT; v--)*/
1047 for (v = MIN_USEABLE_SLOT; v < function_n_variables(ctx->fn); v++) {
1048 const struct type *t;
1049 ctx->registers[v] = -1;
1050 if (ra_chicken)
1051 continue;
1052 t = get_type_of_local(ctx, v);
1053 if (unlikely(!t))
1054 continue;
1055 if (!da(ctx->fn,function)->local_variables_flags[v].must_be_flat &&
1056 !da(ctx->fn,function)->local_variables_flags[v].must_be_data)
1057 continue;
1058 if (!ARCH_HAS_BWX && t->size < 1U << OP_SIZE_4)
1059 continue;
1060 if (TYPE_TAG_IS_FIXED(t->tag) || TYPE_TAG_IS_INT(t->tag) || t->tag == TYPE_TAG_flat_option || t->tag == TYPE_TAG_unknown || t->tag == TYPE_TAG_record) {
1061 if (TYPE_TAG_IS_BUILTIN(t->tag)) {
1062 if (!is_power_of_2(t->size) || t->size > 1U << OP_SIZE_NATIVE)
1063 continue;
1064 }
1065 if (index_saved < n_regs_saved + zero
1066 #if defined(ARCH_PARISC) || defined(ARCH_SPARC)
1067 && t->size <= 1U << OP_SIZE_ADDRESS
1068 #endif
1069 ) {
1070 ctx->registers[v] = regs_saved[index_saved++];
1071 } else if (index_volatile < n_regs_volatile + zero) {
1072 ctx->registers[v] = regs_volatile[index_volatile++];
1073 } else {
1074 continue;
1075 }
1076 } else if (TYPE_TAG_IS_REAL(t->tag)) {
1077 unsigned real_type = TYPE_TAG_IDX_REAL(t->tag);
1078 if ((SUPPORTED_FP >> real_type) & 1) {
1079 #ifdef ARCH_POWER
1080 if (real_type == 4) {
1081 if (index_vector_volatile < n_vector_volatile + zero) {
1082 ctx->registers[v] = vector_volatile[index_vector_volatile++];
1083 goto success;
1084 }
1085 continue;
1086 }
1087 #endif
1088 #ifdef ARCH_S390
1089 if (real_type == 4) {
1090 if (!(index_fp_saved & 1) && index_fp_saved + 1 < n_fp_saved + zero) {
1091 ctx->registers[v] = fp_saved[index_fp_saved++];
1092 index_fp_saved++;
1093 goto success;
1094 }
1095 if (index_fp_saved & 1 && index_fp_saved + 2 < n_fp_saved + zero) {
1096 index_fp_saved++;
1097 ctx->registers[v] = fp_saved[index_fp_saved++];
1098 index_fp_saved++;
1099 goto success;
1100 }
1101 if (!(index_fp_volatile & 1) && index_fp_volatile + 1 < n_fp_volatile + zero) {
1102 ctx->registers[v] = fp_volatile[index_fp_volatile++];
1103 index_fp_volatile++;
1104 goto success;
1105 }
1106 if (index_fp_volatile & 1 && index_fp_volatile + 2 < n_fp_volatile + zero) {
1107 index_fp_volatile++;
1108 ctx->registers[v] = fp_volatile[index_fp_volatile++];
1109 index_fp_volatile++;
1110 goto success;
1111 }
1112 continue;
1113 }
1114 #endif
1115 if (index_fp_saved < n_fp_saved + zero) {
1116 ctx->registers[v] = fp_saved[index_fp_saved++];
1117 } else if (index_fp_volatile < n_fp_volatile + zero) {
1118 ctx->registers[v] = fp_volatile[index_fp_volatile++];
1119 } else {
1120 continue;
1121 }
1122 } else {
1123 continue;
1124 }
1125 } else {
1126 continue;
1127 }
1128 goto success;
1129 success:
1130 if (!reg_is_saved(ctx->registers[v])) {
1131 if (unlikely(!array_add_mayfail(frame_t, &ctx->need_spill, &ctx->need_spill_l, v, NULL, &ctx->err)))
1132 return false;
1133 }
1134 }
1135
1136 return true;
1137 }
1138
1139 static bool attr_w gen_fused_binary(struct codegen_context *ctx, unsigned mode, unsigned op_size, unsigned op, uint32_t escape_label, frame_t slot_1, frame_t slot_2, frame_t slot_r, bool *failed)
1140 {
1141 const code_t *backup = ctx->current_position;
1142 code_t code;
1143 frame_t slot_dr, slot_test;
1144 int32_t offs_false;
1145
1146 *failed = false;
1147
1148 next_code:
1149 code = get_code(ctx);
1150 ctx->arg_mode = code / OPCODE_MODE_MULT;
1151 code %= OPCODE_MODE_MULT;
1152 ajla_assert_lo(ctx->arg_mode < ARG_MODE_N, (file_line, "gen_fused_binary: invalid opcode %04x", (unsigned)*ctx->instr_start));
1153
1154 if (code == OPCODE_DEREFERENCE) {
1155 get_one(ctx, &slot_dr);
1156 const struct type *t = get_type_of_local(ctx, slot_dr);
1157 if (!TYPE_TAG_IS_BUILTIN(t->tag)) {
1158 *failed = true;
1159 goto fail;
1160 }
1161 if (unlikely(!flag_is_clear(ctx, slot_dr))) {
1162 *failed = true;
1163 goto fail;
1164 }
1165 goto next_code;
1166 }
1167 if (code == OPCODE_DEREFERENCE_CLEAR) {
1168 *failed = true;
1169 goto fail;
1170 }
1171 if (unlikely(code != OPCODE_JMP_FALSE))
1172 internal(file_line, "gen_fused_binary: binary operation is not followed by jmp false: %x, %s", code, decode_opcode(code, true));
1173 get_one(ctx, &slot_test);
1174 if (unlikely(slot_test != slot_r))
1175 internal(file_line, "gen_fused_binary: the result of the binary operation and the tested variable do not match");
1176 offs_false = get_jump_offset(ctx);
1177 get_jump_offset(ctx);
1178
1179 if (mode == MODE_ARRAY_LEN_GT) {
1180 g(gen_array_len(ctx, slot_1, slot_2, slot_r, true, offs_false));
1181 } else if (mode == MODE_REAL) {
1182 g(gen_fp_alu_jmp(ctx, op_size, op, escape_label, slot_1, slot_2, offs_false, failed));
1183 } else {
1184 g(gen_alu_jmp(ctx, mode, op_size, op, slot_1, slot_2, offs_false, failed));
1185 }
1186
1187 fail:
1188 if (*failed)
1189 ctx->current_position = backup;
1190
1191 return true;
1192 }
1193
1194 static bool attr_w gen_test_variables(struct codegen_context *ctx, frame_t *variables, size_t n_variables, uint32_t label)
1195 {
1196 size_t i;
1197 frame_t *vars;
1198
1199 for (i = 0; i < n_variables; i++) {
1200 frame_t v = variables[i];
1201 if (ctx->registers[v] >= 0)
1202 g(unspill(ctx, v));
1203 }
1204
1205 vars = mem_alloc_array_mayfail(mem_alloc_mayfail, frame_t *, 0, 0, n_variables, sizeof(frame_t), &ctx->err);
1206 if (unlikely(!vars))
1207 return false;
1208
1209 for (i = 0; i < 2; i++) {
1210 size_t n_vars = 0;
1211 size_t n;
1212 for (n = 0; n < n_variables; n++) {
1213 frame_t v = variables[n];
1214 if (!i ? da(ctx->fn,function)->local_variables_flags[v].must_be_flat : da(ctx->fn,function)->local_variables_flags[v].must_be_data)
1215 vars[n_vars++] = v;
1216 }
1217 if (!i) {
1218 if (!gen_test_multiple(ctx, vars, n_vars, label)) {
1219 mem_free(vars);
1220 return false;
1221 }
1222 } else {
1223 if (!gen_test_multiple_thunks(ctx, vars, n_vars, label)) {
1224 mem_free(vars);
1225 return false;
1226 }
1227 }
1228 }
1229
1230 mem_free(vars);
1231 return true;
1232 }
1233
1234 static bool attr_w gen_function(struct codegen_context *ctx)
1235 {
1236 ctx->current_position = da(ctx->fn,function)->code;
1237
1238 ctx->escape_nospill_label = alloc_label(ctx);
1239 if (unlikely(!ctx->escape_nospill_label))
1240 return false;
1241
1242 while (ctx->current_position != da(ctx->fn,function)->code + da(ctx->fn,function)->code_size) {
1243 ip_t ip;
1244 code_t code;
1245 unsigned op, type;
1246 frame_t slot_1, slot_2, slot_3, slot_r, flags, fn_idx, opt;
1247 arg_t n_args, n_ret, i_arg;
1248 uint32_t label_id;
1249 uint32_t escape_label;
1250 bool failed;
1251
1252 ajla_assert_lo(ctx->current_position < da(ctx->fn,function)->code + da(ctx->fn,function)->code_size, (file_line, "gen_function: ran out of code in %s", da(ctx->fn,function)->function_name));
1253
1254 ctx->instr_start = ctx->current_position;
1255
1256 /*debug("%s: %04x, %s", da(ctx->fn,function)->function_name, *ctx->instr_start, decode_opcode(*ctx->instr_start, true));*/
1257
1258 ip = ctx->instr_start - da(ctx->fn,function)->code;
1259 if (likely(!ctx->code_labels[ip])) {
1260 ctx->code_labels[ip] = alloc_label(ctx);
1261 if (unlikely(!ctx->code_labels[ip]))
1262 return false;
1263 }
1264 gen_label(ctx->code_labels[ip]);
1265
1266 code = get_code(ctx);
1267 ctx->arg_mode = code / OPCODE_MODE_MULT;
1268 code %= OPCODE_MODE_MULT;
1269 ajla_assert_lo(ctx->arg_mode < ARG_MODE_N, (file_line, "gen_function: invalid opcode %04x", (unsigned)*ctx->instr_start));
1270
1271 if (code >= OPCODE_FIXED_OP + uzero && code < OPCODE_INT_OP) {
1272 code -= OPCODE_FIXED_OP;
1273 op = (code / OPCODE_FIXED_OP_MULT) % OPCODE_FIXED_TYPE_MULT;
1274 type = code / OPCODE_FIXED_TYPE_MULT;
1275 if (op < OPCODE_FIXED_OP_UNARY) {
1276 get_two(ctx, &slot_1, &slot_2);
1277 get_two(ctx, &slot_r, &flags);
1278 escape_label = alloc_escape_label(ctx);
1279 if (unlikely(!escape_label))
1280 return false;
1281 g(gen_test_2_cached(ctx, slot_1, slot_2, escape_label));
1282 flag_set(ctx, slot_1, false);
1283 flag_set(ctx, slot_2, false);
1284 flag_set(ctx, slot_r, false);
1285 if (flags & OPCODE_FLAG_FUSED) {
1286 g(gen_fused_binary(ctx, MODE_FIXED, type, op, escape_label, slot_1, slot_2, slot_r, &failed));
1287 if (unlikely(!failed))
1288 continue;
1289 }
1290 g(gen_alu(ctx, MODE_FIXED, type, op, escape_label, slot_1, slot_2, slot_r));
1291 continue;
1292 } else if (op < OPCODE_FIXED_OP_N) {
1293 get_two(ctx, &slot_1, &slot_r);
1294 get_one(ctx, &flags);
1295 escape_label = alloc_escape_label(ctx);
1296 if (unlikely(!escape_label))
1297 return false;
1298 g(gen_test_1_cached(ctx, slot_1, escape_label));
1299 flag_set(ctx, slot_1, false);
1300 flag_set(ctx, slot_r, false);
1301 g(gen_alu1(ctx, MODE_FIXED, type, op, escape_label, slot_1, slot_r));
1302 continue;
1303 } else if (op == OPCODE_FIXED_OP_ldc) {
1304 unsigned i;
1305 get_one(ctx, &slot_r);
1306 g(gen_constant(ctx, false, type, false, slot_r));
1307 for (i = 0; i < 1U << type; i += 2)
1308 get_code(ctx);
1309 flag_set(ctx, slot_r, false);
1310 continue;
1311 } else if (op == OPCODE_FIXED_OP_ldc16) {
1312 get_one(ctx, &slot_r);
1313 g(gen_constant(ctx, false, type, true, slot_r));
1314 get_code(ctx);
1315 flag_set(ctx, slot_r, false);
1316 continue;
1317 } else if (op == OPCODE_FIXED_OP_move || op == OPCODE_FIXED_OP_copy) {
1318 get_two(ctx, &slot_1, &slot_r);
1319 escape_label = alloc_escape_label(ctx);
1320 if (unlikely(!escape_label))
1321 return false;
1322 g(gen_test_1_cached(ctx, slot_1, escape_label));
1323 flag_set(ctx, slot_1, false);
1324 flag_set(ctx, slot_r, false);
1325 g(gen_copy(ctx, type, slot_1, slot_r));
1326 continue;
1327 } else {
1328 internal(file_line, "gen_function: bad fixed code %04x", *ctx->instr_start);
1329 }
1330 } else if (code >= OPCODE_INT_OP && code < OPCODE_REAL_OP) {
1331 code -= OPCODE_INT_OP;
1332 op = (code / OPCODE_INT_OP_MULT) % OPCODE_INT_TYPE_MULT;
1333 type = code / OPCODE_INT_TYPE_MULT;
1334 if (op < OPCODE_INT_OP_UNARY) {
1335 get_two(ctx, &slot_1, &slot_2);
1336 get_two(ctx, &slot_r, &flags);
1337 escape_label = alloc_escape_label(ctx);
1338 if (unlikely(!escape_label))
1339 return false;
1340 g(gen_test_2_cached(ctx, slot_1, slot_2, escape_label));
1341 flag_set(ctx, slot_1, false);
1342 flag_set(ctx, slot_2, false);
1343 flag_set(ctx, slot_r, false);
1344 if (flags & OPCODE_FLAG_FUSED) {
1345 g(gen_fused_binary(ctx, MODE_INT, type, op, escape_label, slot_1, slot_2, slot_r, &failed));
1346 if (unlikely(!failed))
1347 continue;
1348 }
1349 g(gen_alu(ctx, MODE_INT, type, op, escape_label, slot_1, slot_2, slot_r));
1350 continue;
1351 } else if (op < OPCODE_INT_OP_N) {
1352 get_two(ctx, &slot_1, &slot_r);
1353 get_one(ctx, &flags);
1354 if ((op == OPCODE_INT_OP_to_int || op == OPCODE_INT_OP_from_int) && slot_1 == slot_r)
1355 continue;
1356 escape_label = alloc_escape_label(ctx);
1357 if (unlikely(!escape_label))
1358 return false;
1359 g(gen_test_1_cached(ctx, slot_1, escape_label));
1360 flag_set(ctx, slot_1, false);
1361 flag_set(ctx, slot_r, false);
1362 g(gen_alu1(ctx, MODE_INT, type, op, escape_label, slot_1, slot_r));
1363 continue;
1364 } else if (op == OPCODE_INT_OP_ldc) {
1365 unsigned i;
1366 get_one(ctx, &slot_r);
1367 g(gen_constant(ctx, false, type, false, slot_r));
1368 for (i = 0; i < 1U << type; i += 2)
1369 get_code(ctx);
1370 flag_set(ctx, slot_r, false);
1371 continue;
1372 } else if (op == OPCODE_INT_OP_ldc16) {
1373 get_one(ctx, &slot_r);
1374 g(gen_constant(ctx, false, type, true, slot_r));
1375 get_code(ctx);
1376 flag_set(ctx, slot_r, false);
1377 continue;
1378 } else if (op == OPCODE_INT_OP_move || op == OPCODE_INT_OP_copy) {
1379 get_two(ctx, &slot_1, &slot_r);
1380 escape_label = alloc_escape_label(ctx);
1381 if (unlikely(!escape_label))
1382 return false;
1383 g(gen_test_1_cached(ctx, slot_1, escape_label));
1384 flag_set(ctx, slot_1, false);
1385 flag_set(ctx, slot_r, false);
1386 g(gen_copy(ctx, type, slot_1, slot_r));
1387 continue;
1388 } else {
1389 internal(file_line, "gen_function: bad integer code %04x", *ctx->instr_start);
1390 }
1391 } else if (code >= OPCODE_REAL_OP && code < OPCODE_BOOL_OP) {
1392 code -= OPCODE_REAL_OP;
1393 op = (code / OPCODE_REAL_OP_MULT) % OPCODE_REAL_TYPE_MULT;
1394 type = code / OPCODE_REAL_TYPE_MULT;
1395 if (op < OPCODE_REAL_OP_UNARY) {
1396 get_two(ctx, &slot_1, &slot_2);
1397 get_two(ctx, &slot_r, &flags);
1398 escape_label = alloc_escape_label(ctx);
1399 if (unlikely(!escape_label))
1400 return false;
1401 g(gen_test_2_cached(ctx, slot_1, slot_2, escape_label));
1402 flag_set(ctx, slot_1, false);
1403 flag_set(ctx, slot_2, false);
1404 flag_set(ctx, slot_r, false);
1405 if (flags & OPCODE_FLAG_FUSED) {
1406 g(gen_fused_binary(ctx, MODE_REAL, type, op, escape_label, slot_1, slot_2, slot_r, &failed));
1407 if (unlikely(!failed))
1408 continue;
1409 }
1410 g(gen_fp_alu(ctx, type, op, escape_label, slot_1, slot_2, slot_r));
1411 continue;
1412 } else if (op < OPCODE_REAL_OP_N) {
1413 get_two(ctx, &slot_1, &slot_r);
1414 get_one(ctx, &flags);
1415 escape_label = alloc_escape_label(ctx);
1416 if (unlikely(!escape_label))
1417 return false;
1418 g(gen_test_1_cached(ctx, slot_1, escape_label));
1419 flag_set(ctx, slot_1, false);
1420 flag_set(ctx, slot_r, false);
1421 g(gen_fp_alu1(ctx, type, op, escape_label, slot_1, slot_r));
1422 continue;
1423 } else if (op == OPCODE_REAL_OP_ldc) {
1424 const struct type *t;
1425 unsigned i;
1426 get_one(ctx, &slot_r);
1427 t = type_get_real(type);
1428 g(gen_real_constant(ctx, t, slot_r));
1429 for (i = 0; i < t->size; i += 2)
1430 get_code(ctx);
1431 flag_set(ctx, slot_r, false);
1432 continue;
1433 } else if (op == OPCODE_REAL_OP_move || op == OPCODE_REAL_OP_copy) {
1434 get_two(ctx, &slot_1, &slot_r);
1435 escape_label = alloc_escape_label(ctx);
1436 if (unlikely(!escape_label))
1437 return false;
1438 g(gen_test_1_cached(ctx, slot_1, escape_label));
1439 flag_set(ctx, slot_1, false);
1440 flag_set(ctx, slot_r, false);
1441 g(gen_memcpy_slots(ctx, slot_r, slot_1));
1442 continue;
1443 } else {
1444 internal(file_line, "gen_function: bad real code %04x", *ctx->instr_start);
1445 }
1446 } else if (code >= OPCODE_BOOL_OP && code < OPCODE_EXTRA) {
1447 code -= OPCODE_BOOL_OP;
1448 op = (code / OPCODE_BOOL_OP_MULT) % OPCODE_BOOL_TYPE_MULT;
1449 type = log_2(sizeof(ajla_flat_option_t));
1450 if (op < OPCODE_BOOL_OP_UNARY) {
1451 get_two(ctx, &slot_1, &slot_2);
1452 get_two(ctx, &slot_r, &flags);
1453 escape_label = alloc_escape_label(ctx);
1454 if (unlikely(!escape_label))
1455 return false;
1456 g(gen_test_2_cached(ctx, slot_1, slot_2, escape_label));
1457 flag_set(ctx, slot_1, false);
1458 flag_set(ctx, slot_2, false);
1459 flag_set(ctx, slot_r, false);
1460 if (flags & OPCODE_FLAG_FUSED) {
1461 g(gen_fused_binary(ctx, MODE_BOOL, type, op, escape_label, slot_1, slot_2, slot_r, &failed));
1462 if (unlikely(!failed))
1463 continue;
1464 }
1465 g(gen_alu(ctx, MODE_BOOL, type, op, escape_label, slot_1, slot_2, slot_r));
1466 continue;
1467 } else if (op < OPCODE_BOOL_OP_N) {
1468 get_two(ctx, &slot_1, &slot_r);
1469 get_one(ctx, &flags);
1470 escape_label = alloc_escape_label(ctx);
1471 if (unlikely(!escape_label))
1472 return false;
1473 g(gen_test_1_cached(ctx, slot_1, escape_label));
1474 flag_set(ctx, slot_1, false);
1475 flag_set(ctx, slot_r, false);
1476 g(gen_alu1(ctx, MODE_BOOL, type, op, escape_label, slot_1, slot_r));
1477 continue;
1478 } else if (op == OPCODE_BOOL_OP_move || op == OPCODE_BOOL_OP_copy) {
1479 get_two(ctx, &slot_1, &slot_r);
1480 escape_label = alloc_escape_label(ctx);
1481 if (unlikely(!escape_label))
1482 return false;
1483 g(gen_test_1_cached(ctx, slot_1, escape_label));
1484 flag_set(ctx, slot_1, false);
1485 flag_set(ctx, slot_r, false);
1486 g(gen_copy(ctx, type, slot_1, slot_r));
1487 continue;
1488 } else {
1489 internal(file_line, "gen_function: bad boolean code %04x", *ctx->instr_start);
1490 }
1491 } else switch (code) {
1492 case OPCODE_INT_LDC_LONG: {
1493 uint32_t words, w;
1494 get_one(ctx, &slot_r);
1495 words = get_uint32(ctx);
1496 for (w = 0; w < words; w++)
1497 get_code(ctx);
1498 unconditional_escape:
1499 escape_label = alloc_escape_label(ctx);
1500 if (unlikely(!escape_label))
1501 return false;
1502 gen_insn(INSN_JMP, 0, 0, 0);
1503 gen_four(escape_label);
1504 continue;
1505 }
1506 case OPCODE_IS_EXCEPTION: {
1507 get_two(ctx, &slot_1, &slot_r);
1508 get_one(ctx, &flags);
1509 g(gen_is_exception(ctx, slot_1, slot_r));
1510 continue;
1511 }
1512 case OPCODE_EXCEPTION_CLASS:
1513 case OPCODE_EXCEPTION_TYPE:
1514 case OPCODE_EXCEPTION_AUX: {
1515 get_two(ctx, &slot_1, &slot_r);
1516 get_one(ctx, &flags);
1517 goto unconditional_escape;
1518 }
1519 case OPCODE_SYSTEM_PROPERTY: {
1520 get_two(ctx, &slot_1, &slot_r);
1521 get_one(ctx, &flags);
1522 g(gen_system_property(ctx, slot_1, slot_r));
1523 continue;
1524 }
1525 case OPCODE_FLAT_MOVE:
1526 case OPCODE_FLAT_COPY: {
1527 get_two(ctx, &slot_1, &slot_r);
1528 g(gen_flat_move_copy(ctx, slot_1, slot_r));
1529 continue;
1530 }
1531 case OPCODE_REF_MOVE:
1532 case OPCODE_REF_MOVE_CLEAR:
1533 case OPCODE_REF_COPY: {
1534 get_two(ctx, &slot_1, &slot_r);
1535 g(gen_ref_move_copy(ctx, code, slot_1, slot_r));
1536 continue;
1537 }
1538 case OPCODE_BOX_MOVE_CLEAR:
1539 case OPCODE_BOX_COPY: {
1540 get_two(ctx, &slot_1, &slot_r);
1541 g(gen_box_move_copy(ctx, code, slot_1, slot_r));
1542 continue;
1543 }
1544 case OPCODE_TAKE_BORROWED:
1545 get_one(ctx, &slot_1);
1546 if (!da(ctx->fn,function)->local_variables_flags[slot_1].may_be_borrowed)
1547 continue;
1548 if (unlikely(!(label_id = alloc_label(ctx))))
1549 return false;
1550 if (flag_is_set(ctx, slot_1))
1551 goto take_borrowed_done;
1552 if (flag_is_clear(ctx, slot_1)) {
1553 g(gen_set_1(ctx, R_FRAME, slot_1, 0, true));
1554 goto do_take_borrowed;
1555 }
1556 g(gen_test_1(ctx, R_FRAME, slot_1, 0, label_id, false, TEST_SET));
1557 do_take_borrowed:
1558 g(gen_upcall_start(ctx, 1));
1559 g(gen_frame_load(ctx, OP_SIZE_SLOT, garbage, slot_1, 0, R_ARG0));
1560 g(gen_upcall_argument(ctx, 0));
1561 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_pointer_reference_owned), 1));
1562 flag_set(ctx, slot_1, true);
1563 take_borrowed_done:
1564 gen_label(label_id);
1565 continue;
1566 case OPCODE_DEREFERENCE:
1567 case OPCODE_DEREFERENCE_CLEAR: {
1568 bool need_bit_test;
1569 /*const struct type *type;*/
1570 get_one(ctx, &slot_1);
1571 if (flag_is_clear(ctx, slot_1))
1572 goto skip_dereference;
1573 /*type = get_type_of_local(ctx, slot_1);*/
1574 /*need_bit_test = 1 || TYPE_IS_FLAT(type) || da(ctx->fn,function)->local_variables[slot_1].may_be_borrowed;*/
1575 need_bit_test = !flag_is_set(ctx, slot_1);
1576 if (need_bit_test) {
1577 if (unlikely(!(label_id = alloc_label(ctx))))
1578 return false;
1579 g(gen_test_1(ctx, R_FRAME, slot_1, 0, label_id, true, TEST_CLEAR));
1580 } else {
1581 g(gen_set_1(ctx, R_FRAME, slot_1, 0, false));
1582 label_id = 0; /* avoid warning */
1583 }
1584 g(gen_upcall_start(ctx, 1));
1585 g(gen_frame_load(ctx, OP_SIZE_SLOT, garbage, slot_1, 0, R_ARG0));
1586 g(gen_upcall_argument(ctx, 0));
1587 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_pointer_dereference), 1));
1588 if (need_bit_test)
1589 gen_label(label_id);
1590 skip_dereference:
1591 if (code == OPCODE_DEREFERENCE_CLEAR)
1592 g(gen_frame_clear(ctx, OP_SIZE_SLOT, slot_1));
1593 flag_set_unknown(ctx, slot_1);
1594 flag_set(ctx, slot_1, false);
1595 continue;
1596 }
1597 case OPCODE_EVAL: {
1598 get_one(ctx, &slot_1);
1599 g(gen_eval(ctx, slot_1));
1600 continue;
1601 }
1602 case OPCODE_ESCAPE_NONFLAT: {
1603 frame_t n, i;
1604 frame_t *vars;
1605
1606 get_one(ctx, &n);
1607 vars = mem_alloc_array_mayfail(mem_alloc_mayfail, frame_t *, 0, 0, n, sizeof(frame_t), &ctx->err);
1608 if (unlikely(!vars))
1609 return false;
1610 for (i = 0; i < n; i++) {
1611 get_one(ctx, &vars[i]);
1612 }
1613
1614 escape_label = alloc_escape_label(ctx);
1615 if (unlikely(!escape_label)) {
1616 mem_free(vars);
1617 return false;
1618 }
1619
1620 if (unlikely(!gen_test_variables(ctx, vars, n, escape_label))) {
1621 mem_free(vars);
1622 return false;
1623 }
1624 mem_free(vars);
1625
1626 continue;
1627 }
1628 case OPCODE_CHECKPOINT: {
1629 frame_t n_vars;
1630
1631 g(clear_flag_cache(ctx));
1632
1633 if (SIZEOF_IP_T == 2) {
1634 slot_1 = get_code(ctx);
1635 } else if (SIZEOF_IP_T == 4) {
1636 slot_1 = get_uint32(ctx);
1637 } else {
1638 not_reached();
1639 continue;
1640 }
1641
1642 if (unlikely(!(slot_1 + 1)))
1643 return false;
1644 while (slot_1 >= ctx->n_entries) {
1645 void *err_entries;
1646 struct cg_entry e;
1647 if (unlikely(!ctx->entries)) {
1648 if (unlikely(!array_init_mayfail(struct cg_entry, &ctx->entries, &ctx->n_entries, &ctx->err)))
1649 return false;
1650 }
1651 memset(&e, 0, sizeof(struct cg_entry));
1652 if (unlikely(!array_add_mayfail(struct cg_entry, &ctx->entries, &ctx->n_entries, e, &err_entries, &ctx->err))) {
1653 ctx->entries = err_entries;
1654 return false;
1655 }
1656 }
1657
1658 get_one(ctx, &n_vars);
1659
1660 escape_label = 0; /* avoid warning */
1661 if (likely(slot_1 != 0)) {
1662 escape_label = alloc_escape_label(ctx);
1663 if (unlikely(!escape_label))
1664 return false;
1665 }
1666
1667 if (n_vars || !slot_1) {
1668 frame_t i;
1669 uint32_t entry_label, nonflat_label;
1670 struct cg_entry *ce = &ctx->entries[slot_1];
1671
1672 if (unlikely(!array_init_mayfail(frame_t, &ce->variables, &ce->n_variables, &ctx->err)))
1673 return false;
1674 for (i = 0; i < n_vars; i++) {
1675 frame_t v;
1676 get_one(ctx, &v);
1677 if (unlikely(!array_add_mayfail(frame_t, &ce->variables, &ce->n_variables, v, NULL, &ctx->err)))
1678 return false;
1679 }
1680 if (!slot_1) {
1681 g(gen_test_variables(ctx, ce->variables, ce->n_variables, ctx->escape_nospill_label));
1682 }
1683 entry_label = alloc_label(ctx);
1684 if (unlikely(!entry_label))
1685 return false;
1686 gen_label(entry_label);
1687 ce->entry_label = entry_label;
1688
1689 nonflat_label = alloc_escape_label_for_ip(ctx, ctx->current_position);
1690 if (unlikely(!nonflat_label))
1691 return false;
1692 ce->nonflat_label = nonflat_label;
1693
1694 if (unlikely(!slot_1))
1695 g(gen_timestamp_test(ctx, ctx->escape_nospill_label));
1696 else
1697 g(gen_timestamp_test(ctx, escape_label));
1698 } else {
1699 g(gen_timestamp_test(ctx, escape_label));
1700
1701 gen_insn(INSN_ENTRY, 0, 0, 0);
1702 gen_four(slot_1);
1703 }
1704 continue;
1705 }
1706 case OPCODE_JMP: {
1707 int32_t x = get_jump_offset(ctx);
1708 g(gen_jump(ctx, x, OP_SIZE_NATIVE, COND_ALWAYS, -1U, -1U));
1709 continue;
1710 }
1711 case OPCODE_JMP_BACK_16: {
1712 int32_t x = get_code(ctx);
1713 g(gen_jump(ctx, -x - (int)(2 * sizeof(code_t)), OP_SIZE_NATIVE, COND_ALWAYS, -1U, -1U));
1714 continue;
1715 }
1716 case OPCODE_JMP_FALSE: {
1717 int32_t offs_false;
1718 get_one(ctx, &slot_1);
1719 offs_false = get_jump_offset(ctx);
1720 get_jump_offset(ctx);
1721 escape_label = alloc_escape_label(ctx);
1722 if (unlikely(!escape_label))
1723 return false;
1724 g(gen_test_1_cached(ctx, slot_1, escape_label));
1725 flag_set(ctx, slot_1, false);
1726 g(gen_cond_jump(ctx, slot_1, offs_false));
1727 continue;
1728 }
1729 case OPCODE_LABEL: {
1730 g(clear_flag_cache(ctx));
1731 continue;
1732 }
1733 #define init_args \
1734 do { \
1735 if (ctx->args != NULL) \
1736 mem_free(ctx->args); \
1737 g(array_init_mayfail(struct code_arg, &ctx->args, &ctx->args_l, &ctx->err));\
1738 } while (0)
1739 #define load_args \
1740 do { \
1741 init_args; \
1742 for (i_arg = 0; i_arg < n_args; i_arg++) { \
1743 struct code_arg a; \
1744 get_two(ctx, &a.slot, &a.flags); \
1745 a.type = 0; \
1746 g(array_add_mayfail(struct code_arg, &ctx->args, &ctx->args_l, a, NULL, &ctx->err));\
1747 } \
1748 } while (0)
1749 case OPCODE_LOAD_FN:
1750 get_two(ctx, &n_args, &slot_r);
1751 get_one(ctx, &fn_idx);
1752 load_args;
1753 g(gen_load_fn_or_curry(ctx, fn_idx, NO_FRAME_T, slot_r, 0));
1754 continue;
1755 case OPCODE_CURRY:
1756 get_two(ctx, &n_args, &slot_r);
1757 get_two(ctx, &slot_1, &flags);
1758 load_args;
1759 g(gen_load_fn_or_curry(ctx, NO_FRAME_T, slot_1, slot_r, flags));
1760 continue;
1761 case OPCODE_CALL:
1762 case OPCODE_CALL_STRICT:
1763 case OPCODE_CALL_SPARK:
1764 case OPCODE_CALL_LAZY:
1765 case OPCODE_CALL_CACHE:
1766 case OPCODE_CALL_SAVE: {
1767 get_two(ctx, &n_args, &n_ret);
1768 get_one(ctx, &fn_idx);
1769 jump_over_arguments_and_return:
1770 load_args;
1771 ctx->return_values = ctx->current_position;
1772 for (i_arg = 0; i_arg < n_ret; i_arg++) {
1773 #if ARG_MODE_N >= 3
1774 get_uint32(ctx);
1775 #else
1776 get_code(ctx);
1777 #endif
1778 get_code(ctx);
1779 }
1780 if (unlikely(profiling))
1781 goto unconditional_escape;
1782 if (code == OPCODE_CALL || code == OPCODE_CALL_STRICT) {
1783 g(gen_call(ctx, code, fn_idx));
1784 continue;
1785 }
1786 /*if (code == OPCODE_CALL_INDIRECT || code == OPCODE_CALL_INDIRECT_STRICT) {
1787 if (unlikely(!gen_call_indirect(ctx, code, slot_1, flags)))
1788 return false;
1789 continue;
1790 }*/
1791 goto unconditional_escape;
1792 }
1793 case OPCODE_CALL_INDIRECT:
1794 case OPCODE_CALL_INDIRECT_STRICT:
1795 case OPCODE_CALL_INDIRECT_SPARK:
1796 case OPCODE_CALL_INDIRECT_LAZY:
1797 case OPCODE_CALL_INDIRECT_CACHE:
1798 case OPCODE_CALL_INDIRECT_SAVE: {
1799 fn_idx = 0; /* avoid warning */
1800 get_two(ctx, &n_args, &n_ret);
1801 get_two(ctx, &slot_1, &flags);
1802 goto jump_over_arguments_and_return;
1803 }
1804 case OPCODE_RETURN: {
1805 n_args = da(ctx->fn,function)->n_return_values;
1806 load_args;
1807 if (unlikely(profiling))
1808 goto unconditional_escape;
1809 g(gen_return(ctx));
1810 continue;
1811 }
1812 case OPCODE_STRUCTURED: {
1813 init_args;
1814 get_two(ctx, &slot_1, &slot_2);
1815 do {
1816 struct code_arg a;
1817 get_two(ctx, &flags, &slot_r);
1818 get_one(ctx, &opt);
1819 a.slot = slot_r;
1820 a.flags = flags;
1821 a.type = opt;
1822 g(array_add_mayfail(struct code_arg, &ctx->args, &ctx->args_l, a, NULL, &ctx->err));
1823 } while (!(flags & OPCODE_STRUCTURED_FLAG_END));
1824 g(gen_structured(ctx, slot_1, slot_2));
1825 continue;
1826 }
1827 case OPCODE_RECORD_CREATE: {
1828 init_args;
1829 get_two(ctx, &slot_r, &n_args);
1830 for (i_arg = 0; i_arg < n_args; i_arg++) {
1831 struct code_arg a;
1832 get_two(ctx, &slot_1, &flags);
1833 a.slot = slot_1;
1834 a.flags = flags;
1835 a.type = 0;
1836 g(array_add_mayfail(struct code_arg, &ctx->args, &ctx->args_l, a, NULL, &ctx->err));
1837 }
1838 g(gen_record_create(ctx, slot_r));
1839 continue;
1840 }
1841 case OPCODE_RECORD_LOAD: {
1842 get_two(ctx, &slot_1, &opt);
1843 get_two(ctx, &slot_r, &flags);
1844 g(gen_record_load(ctx, slot_1, slot_r, opt, flags));
1845 continue;
1846 }
1847 case OPCODE_OPTION_CREATE_EMPTY_FLAT: {
1848 get_two(ctx, &slot_r, &opt);
1849 g(gen_option_create_empty_flat(ctx, opt, slot_r));
1850 continue;
1851 }
1852 case OPCODE_OPTION_CREATE_EMPTY: {
1853 get_two(ctx, &slot_r, &opt);
1854 g(gen_option_create_empty(ctx, opt, slot_r));
1855 continue;
1856 }
1857 case OPCODE_OPTION_CREATE: {
1858 get_two(ctx, &slot_r, &opt);
1859 get_two(ctx, &slot_1, &flags);
1860 g(gen_option_create(ctx, opt, slot_1, slot_r, flags));
1861 continue;
1862 }
1863 case OPCODE_OPTION_LOAD: {
1864 get_two(ctx, &slot_1, &opt);
1865 get_two(ctx, &slot_r, &flags);
1866 g(gen_option_load(ctx, slot_1, slot_r, opt, flags));
1867 continue;
1868 }
1869 case OPCODE_OPTION_TEST_FLAT: {
1870 get_two(ctx, &slot_1, &opt);
1871 get_one(ctx, &slot_r);
1872 g(gen_option_test_flat(ctx, slot_1, opt, slot_r));
1873 continue;
1874 }
1875 case OPCODE_OPTION_TEST: {
1876 get_two(ctx, &slot_1, &opt);
1877 get_one(ctx, &slot_r);
1878 g(gen_option_test(ctx, slot_1, opt, slot_r));
1879 continue;
1880 }
1881 case OPCODE_OPTION_ORD_FLAT: {
1882 get_two(ctx, &slot_1, &slot_r);
1883 g(gen_option_ord(ctx, slot_1, slot_r, true));
1884 continue;
1885 }
1886 case OPCODE_OPTION_ORD: {
1887 get_two(ctx, &slot_1, &slot_r);
1888 g(gen_option_ord(ctx, slot_1, slot_r, false));
1889 continue;
1890 }
1891 case OPCODE_ARRAY_CREATE: {
1892 init_args;
1893 get_two(ctx, &slot_r, &n_args);
1894 for (i_arg = 0; i_arg < n_args; i_arg++) {
1895 struct code_arg a;
1896 get_two(ctx, &slot_1, &flags);
1897 a.slot = slot_1;
1898 a.flags = flags;
1899 a.type = 0;
1900 g(array_add_mayfail(struct code_arg, &ctx->args, &ctx->args_l, a, NULL, &ctx->err));
1901 }
1902 g(gen_array_create(ctx, slot_r));
1903 continue;
1904 }
1905 case OPCODE_ARRAY_CREATE_EMPTY_FLAT: {
1906 get_two(ctx, &slot_r, &flags);
1907 g(gen_array_create_empty_flat(ctx, slot_r, flags));
1908 continue;
1909 }
1910 case OPCODE_ARRAY_CREATE_EMPTY: {
1911 get_one(ctx, &slot_r);
1912 g(gen_array_create_empty(ctx, slot_r));
1913 continue;
1914 }
1915 case OPCODE_ARRAY_FILL: {
1916 get_two(ctx, &slot_1, &flags);
1917 get_two(ctx, &slot_2, &slot_r);
1918 g(gen_array_fill(ctx, slot_1, flags, slot_2, slot_r));
1919 continue;
1920 }
1921 case OPCODE_ARRAY_STRING: {
1922 frame_t i;
1923 get_two(ctx, &slot_r, &i);
1924 g(gen_array_string(ctx, type_get_fixed(0, true)->tag, cast_ptr(uint8_t *, ctx->current_position), i, slot_r));
1925 ctx->current_position += (i + 1) >> 1;
1926 continue;
1927 }
1928 case OPCODE_ARRAY_UNICODE: {
1929 frame_t i;
1930 get_two(ctx, &slot_r, &i);
1931 g(gen_array_string(ctx, type_get_int(2)->tag, cast_ptr(uint8_t *, ctx->current_position), i, slot_r));
1932 ctx->current_position += i * 2;
1933 continue;
1934 }
1935 case OPCODE_ARRAY_LOAD: {
1936 get_two(ctx, &slot_1, &slot_2);
1937 get_two(ctx, &slot_r, &flags);
1938 g(gen_array_load(ctx, slot_1, slot_2, slot_r, flags));
1939 continue;
1940 }
1941 case OPCODE_ARRAY_LEN: {
1942 get_two(ctx, &slot_1, &slot_r);
1943 get_one(ctx, &flags);
1944 g(gen_array_len(ctx, slot_1, NO_FRAME_T, slot_r, false, 0));
1945 continue;
1946 }
1947 case OPCODE_ARRAY_LEN_GREATER_THAN: {
1948 get_two(ctx, &slot_1, &slot_2);
1949 get_two(ctx, &slot_r, &flags);
1950 escape_label = alloc_escape_label(ctx);
1951 if (unlikely(!escape_label))
1952 return false;
1953 if (flags & OPCODE_FLAG_FUSED) {
1954 g(gen_fused_binary(ctx, MODE_ARRAY_LEN_GT, 0, 0, escape_label, slot_1, slot_2, slot_r, &failed));
1955 if (unlikely(!failed))
1956 continue;
1957 }
1958 g(gen_array_len(ctx, slot_1, slot_2, slot_r, false, 0));
1959 continue;
1960 }
1961 case OPCODE_ARRAY_SUB: {
1962 get_two(ctx, &slot_1, &slot_2);
1963 get_two(ctx, &slot_3, &slot_r);
1964 get_one(ctx, &flags);
1965 g(gen_array_sub(ctx, slot_1, slot_2, slot_3, slot_r, flags));
1966 continue;
1967 }
1968 case OPCODE_ARRAY_SKIP: {
1969 get_two(ctx, &slot_1, &slot_2);
1970 get_two(ctx, &slot_r, &flags);
1971 g(gen_array_skip(ctx, slot_1, slot_2, slot_r, flags));
1972 continue;
1973 }
1974 case OPCODE_ARRAY_APPEND: {
1975 get_two(ctx, &slot_r, &flags);
1976 get_two(ctx, &slot_1, &slot_2);
1977 g(gen_array_append(ctx, slot_1, slot_2, slot_r, flags));
1978 continue;
1979 }
1980 case OPCODE_ARRAY_APPEND_ONE_FLAT: {
1981 get_two(ctx, &slot_r, &flags);
1982 get_two(ctx, &slot_1, &slot_2);
1983 g(gen_array_append_one_flat(ctx, slot_1, slot_2, slot_r, flags));
1984 continue;
1985 }
1986 case OPCODE_ARRAY_APPEND_ONE: {
1987 get_two(ctx, &slot_r, &flags);
1988 get_two(ctx, &slot_1, &slot_2);
1989 g(gen_array_append_one(ctx, slot_1, slot_2, slot_r, flags));
1990 continue;
1991 }
1992 case OPCODE_ARRAY_FLATTEN: {
1993 get_two(ctx, &slot_r, &flags);
1994 get_one(ctx, &slot_1);
1995 goto unconditional_escape;
1996 }
1997 case OPCODE_IO: {
1998 get_two(ctx, &flags, &slot_1);
1999 get_two(ctx, &slot_2, &slot_3);
2000 g(gen_io(ctx, flags, slot_1, slot_2, slot_3));
2001 continue;
2002 }
2003 case OPCODE_INTERNAL_FUNCTION:
2004 case OPCODE_EXIT_THREAD:
2005 case OPCODE_UNREACHABLE: {
2006 goto unconditional_escape;
2007 }
2008 default: {
2009 #if 1
2010 /*if (getenv("DUMP") && !strcmp(da(ctx->fn,function)->function_name, getenv("DUMP")))*/
2011 warning("gen_function: %s: unknown opcode %04x, %s", da(ctx->fn,function)->function_name, *ctx->instr_start, decode_opcode(*ctx->instr_start, false));
2012 #endif
2013 return false;
2014 }
2015 }
2016 }
2017
2018 return true;
2019 }
2020
2021 static bool attr_w gen_entries(struct codegen_context *ctx)
2022 {
2023 size_t i;
2024 for (i = 0; i < ctx->n_entries; i++) {
2025 struct cg_entry *ce = &ctx->entries[i];
2026 if (ce->entry_label) {
2027 gen_insn(INSN_ENTRY, 0, 0, 0);
2028 gen_four(i);
2029
2030 g(gen_test_variables(ctx, ce->variables, ce->n_variables, ce->nonflat_label));
2031
2032 gen_insn(INSN_JMP, 0, 0, 0);
2033 gen_four(ce->entry_label);
2034 }
2035 }
2036 return true;
2037 }
2038
2039 static bool attr_w gen_epilogues(struct codegen_context *ctx)
2040 {
2041 frame_t v;
2042 ip_t ip;
2043 uint32_t escape_label, nospill_label;
2044 escape_label = alloc_label(ctx);
2045 if (unlikely(!escape_label))
2046 return false;
2047 nospill_label = alloc_label(ctx);
2048 if (unlikely(!nospill_label))
2049 return false;
2050 #if defined(ARCH_PARISC)
2051 if (ctx->call_label) {
2052 gen_label(ctx->call_label);
2053 g(gen_call_millicode(ctx));
2054 }
2055 #endif
2056 if (ctx->reload_label) {
2057 gen_label(ctx->reload_label);
2058 g(gen_mov(ctx, i_size(OP_SIZE_ADDRESS), R_FRAME, R_RET0));
2059 g(gen_escape_arg(ctx, (ip_t)-1, nospill_label));
2060 }
2061 gen_label(ctx->escape_nospill_label);
2062 g(gen_escape_arg(ctx, 0, nospill_label));
2063 for (ip = 0; ip < da(ctx->fn,function)->code_size; ip++) {
2064 struct cg_exit *ce = ctx->code_exits[ip];
2065 if (ce && (ce->undo_label || ce->escape_label)) {
2066 if (ce->undo_label) {
2067 size_t i;
2068 gen_label(ce->undo_label);
2069 gen_insn(ce->undo_opcode, ce->undo_op_size, ce->undo_aux, ce->undo_writes_flags);
2070 for (i = 0; i < ce->undo_parameters_len; i++)
2071 gen_one(ce->undo_parameters[i]);
2072 }
2073 if (ce->escape_label) {
2074 gen_label(ce->escape_label);
2075 }
2076 g(gen_escape_arg(ctx, ip, escape_label));
2077 }
2078 }
2079 gen_label(escape_label);
2080 for (v = MIN_USEABLE_SLOT; v < function_n_variables(ctx->fn); v++) {
2081 if (ctx->registers[v] >= 0) {
2082 g(spill(ctx, v));
2083 }
2084 }
2085 gen_label(nospill_label);
2086 g(gen_escape(ctx));
2087 return true;
2088 }
2089
2090 static bool attr_w cgen_entry(struct codegen_context *ctx)
2091 {
2092 uint32_t entry_id = cget_four(ctx);
2093 ajla_assert_lo(entry_id < ctx->n_entries, (file_line, "cgen_entry: invalid entry %lx", (unsigned long)entry_id));
2094 ctx->entries[entry_id].entry_to_pos = ctx->mcode_size;
2095 return true;
2096 }
2097
2098 static bool attr_w cgen_label(struct codegen_context *ctx)
2099 {
2100 uint32_t label_id = cget_four(ctx);
2101 ctx->label_to_pos[label_id] = ctx->mcode_size;
2102 return true;
2103 }
2104
2105 static bool attr_w attr_unused cgen_trap(struct codegen_context *ctx, uint32_t label)
2106 {
2107 struct trap_record tr;
2108 tr.source_ip = ctx->mcode_size;
2109 tr.destination_ip = label;
2110 if (unlikely(!array_add_mayfail(struct trap_record, &ctx->trap_records, &ctx->trap_records_size, tr, NULL, &ctx->err)))
2111 return false;
2112 return true;
2113 }
2114
2115 static bool attr_w add_relocation(struct codegen_context *ctx, unsigned length, int offset, bool *known)
2116 {
2117 struct relocation rel;
2118 rel.label_id = cget_four(ctx);
2119 rel.length = length;
2120 rel.position = ctx->mcode_size;
2121 rel.jmp_instr = ctx->code_position - 8 - offset - ctx->code;
2122 if (unlikely(!array_add_mayfail(struct relocation, &ctx->reloc, &ctx->reloc_size, rel, NULL, &ctx->err)))
2123 return false;
2124 if (known)
2125 *known = ctx->label_to_pos[rel.label_id] != (size_t)-1;
2126 return true;
2127 }
2128
2129
2130 #if defined(ARCH_ALPHA)
2131 #include "c2-alpha.inc"
2132 #elif defined(ARCH_ARM32)
2133 #include "c2-arm.inc"
2134 #elif defined(ARCH_ARM64)
2135 #include "c2-arm64.inc"
2136 #elif defined(ARCH_IA64)
2137 #include "c2-ia64.inc"
2138 #elif defined(ARCH_LOONGARCH64)
2139 #include "c2-loong.inc"
2140 #elif defined(ARCH_MIPS)
2141 #include "c2-mips.inc"
2142 #elif defined(ARCH_PARISC)
2143 #include "c2-hppa.inc"
2144 #elif defined(ARCH_POWER)
2145 #include "c2-power.inc"
2146 #elif defined(ARCH_S390)
2147 #include "c2-s390.inc"
2148 #elif defined(ARCH_SPARC)
2149 #include "c2-sparc.inc"
2150 #elif defined(ARCH_RISCV64)
2151 #include "c2-riscv.inc"
2152 #elif defined(ARCH_X86)
2153 #include "c2-x86.inc"
2154 #endif
2155
2156
2157 static bool attr_w gen_mcode(struct codegen_context *ctx)
2158 {
2159 ctx->code_position = ctx->code;
2160
2161 while (ctx->code_position != ctx->code + ctx->code_size) {
2162 uint32_t insn;
2163 ajla_assert_lo(ctx->code_position < ctx->code + ctx->code_size, (file_line, "gen_mcode: ran out of code"));
2164 #ifdef DEBUG_INSNS
2165 insn = cget_four(ctx);
2166 debug("line: %u", insn);
2167 #endif
2168 insn = cget_four(ctx);
2169 g(cgen_insn(ctx, insn));
2170 }
2171
2172 return true;
2173 }
2174
2175 #define RELOCS_RETRY -1
2176 #define RELOCS_FAIL 0
2177 #define RELOCS_OK 1
2178
2179 static int8_t resolve_relocs(struct codegen_context *ctx)
2180 {
2181 size_t i;
2182 int8_t status = RELOCS_OK;
2183 for (i = 0; i < ctx->reloc_size; i++) {
2184 struct relocation *reloc = &ctx->reloc[i];
2185 if (!resolve_relocation(ctx, reloc)) {
2186 uint8_t *jmp_instr;
2187 uint32_t insn;
2188 uint32_t new_length;
2189 status = RELOCS_RETRY;
2190 if (unlikely(reloc->length + zero >= JMP_LIMIT))
2191 return RELOCS_FAIL;
2192 new_length = reloc->length + 1;
2193 jmp_instr = ctx->code + reloc->jmp_instr;
2194 insn = (uint32_t)jmp_instr[0] +
2195 ((uint32_t)jmp_instr[1] << 8) +
2196 ((uint32_t)jmp_instr[2] << 16) +
2197 ((uint32_t)jmp_instr[3] << 24);
2198 insn &= ~INSN_JUMP_SIZE;
2199 insn |= (uint32_t)new_length << INSN_JUMP_SIZE_SHIFT;
2200 jmp_instr[0] = insn;
2201 jmp_instr[1] = insn >> 8;
2202 jmp_instr[2] = insn >> 16;
2203 jmp_instr[3] = insn >> 24;
2204 }
2205 }
2206 return status;
2207 }
2208
2209 static void resolve_traps(struct codegen_context *ctx)
2210 {
2211 size_t i;
2212 for (i = 0; i < ctx->trap_records_size; i++) {
2213 struct trap_record *tr = &ctx->trap_records[i];
2214 tr->destination_ip = ctx->label_to_pos[tr->destination_ip];
2215 }
2216 }
2217
2218
2219 static bool attr_w codegen_map(struct codegen_context *ctx)
2220 {
2221 void *ptr;
2222 frame_t i;
2223 array_finish(uint8_t, &ctx->mcode, &ctx->mcode_size);
2224 ptr = os_code_map(ctx->mcode, ctx->mcode_size, &ctx->err);
2225 ctx->mcode = NULL;
2226 if (unlikely(!ptr)) {
2227 return false;
2228 }
2229 for (i = 0; i < ctx->n_entries; i++) {
2230 char *entry = cast_ptr(char *, ptr) + ctx->entries[i].entry_to_pos;
2231 da(ctx->codegen,codegen)->unoptimized_code[i] = entry;
2232 da(ctx->codegen,codegen)->n_entries++;
2233 }
2234 da(ctx->codegen,codegen)->unoptimized_code_base = ptr;
2235 da(ctx->codegen,codegen)->unoptimized_code_size = ctx->mcode_size;
2236
2237 return true;
2238 }
2239
2240
2241 void *codegen_fn(frame_s *fp, const code_t *ip, union internal_arg ia[])
2242 {
2243 struct codegen_context ctx_;
2244 struct codegen_context *ctx = &ctx_;
2245 frame_t i;
2246 int8_t rr;
2247 struct data *codegen;
2248 uint32_t l;
2249
2250 init_ctx(ctx);
2251 ctx->fn = ia[0].ptr;
2252
2253 #ifdef DEBUG_ENV
2254 if (getenv("CG") && strcmp(da(ctx->fn,function)->function_name, getenv("CG")))
2255 goto fail;
2256 #endif
2257
2258 ctx->local_directory = mem_alloc_array_mayfail(mem_calloc_mayfail, struct data **, 0, 0, da(ctx->fn,function)->local_directory_size, sizeof(struct data *), &ctx->err);
2259 if (unlikely(!ctx->local_directory))
2260 goto fail;
2261
2262 if (0) for (i = 0; i < da(ctx->fn,function)->local_directory_size; i++) {
2263 struct data *callee;
2264 pointer_t *ptr;
2265 ptr = da(ctx->fn,function)->local_directory[i];
2266 pointer_follow(ptr, false, callee, PF_SPARK, NULL, 0,
2267 SUBMIT_EX(ex_);
2268 goto next_one,
2269 goto next_one;
2270 );
2271 ctx->local_directory[i] = callee;
2272 next_one:;
2273 }
2274 for (i = 0; i < da(ctx->fn,function)->local_directory_size; i++) {
2275 struct data *callee;
2276 pointer_t *ptr;
2277 if (ctx->local_directory[i])
2278 continue;
2279 ptr = da(ctx->fn,function)->local_directory[i];
2280 pointer_follow(ptr, false, callee, PF_WAIT, fp, ip,
2281 done_ctx(ctx);
2282 return ex_,
2283 goto fail
2284 );
2285 ctx->local_directory[i] = callee;
2286 /*debug("processing call: %s -> %s", da(ctx->fn,function)->function_name, da(callee,function)->function_name);*/
2287 }
2288
2289 if (da(ctx->fn,function)->module_designator) {
2290 struct function_descriptor *sfd = save_find_function_descriptor(da(ctx->fn,function)->module_designator, da(ctx->fn,function)->function_designator);
2291 if (sfd && sfd->unoptimized_code_size) {
2292 codegen = data_alloc_flexible(codegen, unoptimized_code, sfd->n_entries, &ctx->err);
2293 if (unlikely(!codegen))
2294 goto fail;
2295 da(codegen,codegen)->unoptimized_code_base = sfd->unoptimized_code_base;
2296 da(codegen,codegen)->unoptimized_code_size = sfd->unoptimized_code_size;
2297 da(codegen,codegen)->function = ctx->fn;
2298 da(codegen,codegen)->is_saved = true;
2299 da(codegen,codegen)->n_entries = sfd->n_entries;
2300 da(codegen,codegen)->offsets = NULL;
2301 for (i = 0; i < sfd->n_entries; i++) {
2302 da(codegen,codegen)->unoptimized_code[i] = cast_ptr(char *, da(codegen,codegen)->unoptimized_code_base) + sfd->entries[i];
2303 /*debug("%s: %p + %lx -> %p", da(ctx->fn,function)->function_name, da(codegen,codegen)->unoptimized_code_base, sfd->entries[i], da(codegen,codegen)->unoptimized_code[i]);*/
2304 }
2305 #ifdef HAVE_CODEGEN_TRAPS
2306 da(codegen,codegen)->trap_records = sfd->trap_records;
2307 da(codegen,codegen)->trap_records_size = sfd->trap_records_size;
2308 data_trap_insert(codegen);
2309 #endif
2310 goto have_codegen;
2311 }
2312 }
2313
2314 /*debug("trying: %s", da(ctx->fn,function)->function_name);*/
2315 if (unlikely(!array_init_mayfail(uint8_t, &ctx->code, &ctx->code_size, &ctx->err)))
2316 goto fail;
2317
2318 ctx->code_labels = mem_alloc_array_mayfail(mem_calloc_mayfail, uint32_t *, 0, 0, da(ctx->fn,function)->code_size, sizeof(uint32_t), &ctx->err);
2319 if (unlikely(!ctx->code_labels))
2320 goto fail;
2321
2322 ctx->code_exits = mem_alloc_array_mayfail(mem_calloc_mayfail, struct cg_exit **, 0, 0, da(ctx->fn,function)->code_size, sizeof(struct cg_exit *), &ctx->err);
2323 if (unlikely(!ctx->code_exits))
2324 goto fail;
2325
2326 ctx->flag_cache = mem_alloc_array_mayfail(mem_calloc_mayfail, uint8_t *, 0, 0, function_n_variables(ctx->fn), sizeof(int8_t), &ctx->err);
2327 if (unlikely(!ctx->flag_cache))
2328 goto fail;
2329
2330 ctx->registers = mem_alloc_array_mayfail(mem_alloc_mayfail, short *, 0, 0, function_n_variables(ctx->fn), sizeof(short), &ctx->err);
2331 if (unlikely(!ctx->registers))
2332 goto fail;
2333
2334 if (unlikely(!array_init_mayfail(frame_t, &ctx->need_spill, &ctx->need_spill_l, &ctx->err)))
2335 goto fail;
2336
2337 if (unlikely(!gen_registers(ctx)))
2338 goto fail;
2339
2340 if (unlikely(!gen_function(ctx)))
2341 goto fail;
2342
2343 if (unlikely(!gen_entries(ctx)))
2344 goto fail;
2345
2346 if (unlikely(!gen_epilogues(ctx)))
2347 goto fail;
2348
2349 if (unlikely(!(ctx->label_id + 1)))
2350 goto fail;
2351 if (unlikely(!(ctx->label_to_pos = mem_alloc_array_mayfail(mem_alloc_mayfail, size_t *, 0, 0, ctx->label_id + 1, sizeof(size_t), &ctx->err))))
2352 goto fail;
2353
2354 again:
2355 for (l = 0; l < ctx->label_id + 1; l++)
2356 ctx->label_to_pos[l] = (size_t)-1;
2357
2358 if (unlikely(!array_init_mayfail(uint8_t, &ctx->mcode, &ctx->mcode_size, &ctx->err)))
2359 goto fail;
2360
2361 if (unlikely(!array_init_mayfail(struct relocation, &ctx->reloc, &ctx->reloc_size, &ctx->err)))
2362 goto fail;
2363
2364 if (unlikely(!array_init_mayfail(struct trap_record, &ctx->trap_records, &ctx->trap_records_size, &ctx->err)))
2365 goto fail;
2366
2367 #ifdef ARCH_CONTEXT
2368 init_arch_context(ctx);
2369 #endif
2370
2371 if (unlikely(!gen_mcode(ctx)))
2372 goto fail;
2373
2374 rr = resolve_relocs(ctx);
2375 if (unlikely(rr == RELOCS_FAIL)) {
2376 /*debug("relocation fail: %s", da(ctx->fn,function)->function_name);*/
2377 goto fail;
2378 }
2379 if (rr == RELOCS_RETRY) {
2380 mem_free(ctx->mcode);
2381 ctx->mcode = NULL;
2382 mem_free(ctx->reloc);
2383 ctx->reloc = NULL;
2384 mem_free(ctx->trap_records);
2385 ctx->trap_records = NULL;
2386 goto again;
2387 }
2388
2389 resolve_traps(ctx);
2390
2391 #ifdef DEBUG_ENV
2392 if ((getenv("DUMP") && !strcmp(getenv("DUMP"), da(ctx->fn,function)->function_name)) || getenv("DUMP_ALL")) {
2393 char *hex;
2394 size_t hexl;
2395 size_t i;
2396 handle_t h;
2397
2398 mutex_lock(&dump_mutex);
2399 str_init(&hex, &hexl);
2400 str_add_string(&hex, &hexl, "_");
2401 str_add_unsigned(&hex, &hexl, dump_seq++, 10);
2402 str_add_string(&hex, &hexl, "_");
2403 str_add_string(&hex, &hexl, da(ctx->fn,function)->function_name);
2404 str_add_string(&hex, &hexl, ":");
2405 for (i = 0; i < hexl; i++)
2406 if (hex[i] == '/')
2407 hex[i] = '_';
2408 for (i = 0; i < ctx->mcode_size; i++) {
2409 uint8_t a = ctx->mcode[i];
2410 if (!(i & 0xff))
2411 str_add_string(&hex, &hexl, "\n .byte 0x");
2412 else
2413 str_add_string(&hex, &hexl, ",0x");
2414 if (a < 16)
2415 str_add_char(&hex, &hexl, '0');
2416 str_add_unsigned(&hex, &hexl, a, 16);
2417 }
2418 str_add_string(&hex, &hexl, "\n");
2419 h = os_open(os_cwd, "dump.s", O_WRONLY | O_APPEND, 0600, NULL);
2420 os_write_all(h, hex, hexl, NULL);
2421 os_close(h);
2422 mem_free(hex);
2423 mutex_unlock(&dump_mutex);
2424 }
2425 #endif
2426
2427 ctx->codegen = data_alloc_flexible(codegen, unoptimized_code, ctx->n_entries, &ctx->err);
2428 if (unlikely(!ctx->codegen))
2429 goto fail;
2430 da(ctx->codegen,codegen)->function = ctx->fn;
2431 da(ctx->codegen,codegen)->is_saved = false;
2432 da(ctx->codegen,codegen)->n_entries = 0;
2433 da(ctx->codegen,codegen)->offsets = NULL;
2434
2435 if (unlikely(!codegen_map(ctx)))
2436 goto fail;
2437
2438 codegen = ctx->codegen;
2439 ctx->codegen = NULL;
2440
2441 #ifdef HAVE_CODEGEN_TRAPS
2442 da(codegen,codegen)->trap_records = ctx->trap_records;
2443 da(codegen,codegen)->trap_records_size = ctx->trap_records_size;
2444 ctx->trap_records = NULL;
2445 data_trap_insert(codegen);
2446 #endif
2447
2448 have_codegen:
2449 done_ctx(ctx);
2450 return function_return(fp, pointer_data(codegen));
2451
2452 fail:
2453 /*debug("FAILED: %s", da(ctx->fn,function)->function_name);*/
2454 done_ctx(ctx);
2455 return function_return(fp, pointer_thunk(thunk_alloc_exception_error(error_ajla(EC_SYNC, AJLA_ERROR_NOT_SUPPORTED), NULL, NULL, NULL pass_file_line)));
2456 }
2457
2458 void codegen_free(struct data *codegen)
2459 {
2460 if (unlikely(da(codegen,codegen)->offsets != NULL))
2461 mem_free(da(codegen,codegen)->offsets);
2462 if (likely(da(codegen,codegen)->is_saved))
2463 return;
2464 #ifdef HAVE_CODEGEN_TRAPS
2465 mem_free(da(codegen,codegen)->trap_records);
2466 #endif
2467 os_code_unmap(da(codegen,codegen)->unoptimized_code_base, da(codegen,codegen)->unoptimized_code_size);
2468 }
2469
2470 #if defined(ARCH_IA64)
2471 static uintptr_t ia64_stub[2];
2472 #endif
2473 #if defined(ARCH_PARISC32) && defined(ARCH_PARISC_USE_STUBS)
2474 static uintptr_t parisc_stub[2];
2475 #endif
2476 #if defined(ARCH_PARISC64) && defined(ARCH_PARISC_USE_STUBS)
2477 static uintptr_t parisc_stub[4];
2478 #endif
2479 #if defined(ARCH_POWER) && defined(AIX_CALL)
2480 static uintptr_t ppc_stub[3];
2481 #endif
2482
2483 void name(codegen_init)(void)
2484 {
2485 struct codegen_context ctx_;
2486 struct codegen_context *ctx = &ctx_;
2487 void *ptr;
2488
2489 #if (defined(ARCH_X86_64) || defined(ARCH_X86_X32)) && !defined(ARCH_X86_WIN_ABI)
2490 #if defined(HAVE_SYSCALL) && defined(HAVE_ASM_PRCTL_H) && defined(HAVE_SYS_SYSCALL_H)
2491 if (!dll) {
2492 int r;
2493 EINTR_LOOP(r, syscall(SYS_arch_prctl, ARCH_SET_GS, &cg_upcall_vector));
2494 if (!r)
2495 upcall_register = R_GS;
2496 }
2497 #elif defined(HAVE_AMD64_SET_GSBASE) && defined(HAVE_X86_SYSARCH_H)
2498 if (!dll) {
2499 int r;
2500 EINTR_LOOP(r, amd64_set_gsbase(&cg_upcall_vector));
2501 if (!r)
2502 upcall_register = R_GS;
2503 }
2504 #elif defined(HAVE_SYSARCH) && defined(HAVE_X86_SYSARCH_H) && defined(X86_64_SET_GSBASE)
2505 if (!dll) {
2506 int r;
2507 void *ptr = &cg_upcall_vector;
2508 EINTR_LOOP(r, sysarch(X86_64_SET_GSBASE, &ptr));
2509 if (!r)
2510 upcall_register = R_GS;
2511 }
2512 #endif
2513 #endif
2514
2515 init_ctx(ctx);
2516 ctx->fn = NULL;
2517
2518 array_init(uint8_t, &ctx->code, &ctx->code_size);
2519
2520 if (unlikely(!gen_entry(ctx)))
2521 goto fail;
2522
2523 array_init(uint8_t, &ctx->mcode, &ctx->mcode_size);
2524
2525 #ifdef ARCH_CONTEXT
2526 init_arch_context(ctx);
2527 #endif
2528
2529 if (unlikely(!gen_mcode(ctx)))
2530 goto fail;
2531
2532 array_finish(uint8_t, &ctx->mcode, &ctx->mcode_size);
2533 ptr = os_code_map(ctx->mcode, ctx->mcode_size, NULL);
2534 codegen_ptr = ptr;
2535 codegen_size = ctx->mcode_size;
2536 ctx->mcode = NULL;
2537 #if defined(ARCH_IA64)
2538 ia64_stub[0] = ptr_to_num(ptr);
2539 ia64_stub[1] = 0;
2540 codegen_entry = cast_ptr(codegen_type, ia64_stub);
2541 #elif defined(ARCH_PARISC32) && defined(ARCH_PARISC_USE_STUBS)
2542 parisc_stub[0] = ptr_to_num(ptr);
2543 parisc_stub[1] = 0;
2544 codegen_entry = cast_ptr(codegen_type, cast_ptr(char *, parisc_stub) + 2);
2545 #elif defined(ARCH_PARISC64) && defined(ARCH_PARISC_USE_STUBS)
2546 parisc_stub[0] = 0;
2547 parisc_stub[1] = 0;
2548 parisc_stub[2] = ptr_to_num(ptr);
2549 parisc_stub[3] = 0;
2550 codegen_entry = cast_ptr(codegen_type, parisc_stub);
2551 #elif defined(ARCH_POWER) && defined(AIX_CALL)
2552 ppc_stub[0] = ptr_to_num(ptr);
2553 ppc_stub[1] = 0;
2554 ppc_stub[2] = 0;
2555 codegen_entry = cast_ptr(codegen_type, ppc_stub);
2556 #else
2557 codegen_entry = ptr;
2558 #endif
2559 done_ctx(ctx);
2560
2561 #ifdef DEBUG_ENV
2562 mutex_init(&dump_mutex);
2563 if (getenv("DUMP") || getenv("DUMP_ALL")) {
2564 size_t i;
2565 char *hex;
2566 size_t hexl;
2567 str_init(&hex, &hexl);
2568 #if defined(ARCH_RISCV64)
2569 str_add_string(&hex, &hexl, " .attribute arch, \"rv64i2p1_m2p0_a2p1_f2p2_d2p2_c2p0_zicsr2p0_zifencei2p0_zba1p0_zbb1p0_zbc1p0_zbs1p0\"\n");
2570 #endif
2571 for (i = 0; i < codegen_size; i++) {
2572 uint8_t a = cast_ptr(uint8_t *, codegen_ptr)[i];
2573 str_add_string(&hex, &hexl, " .byte 0x");
2574 if (a < 16)
2575 str_add_char(&hex, &hexl, '0');
2576 str_add_unsigned(&hex, &hexl, a, 16);
2577 str_add_char(&hex, &hexl, '\n');
2578 }
2579 os_write_atomic(".", "dump.s", hex, hexl, NULL);
2580 mem_free(hex);
2581 }
2582 #endif
2583
2584 return;
2585
2586 fail:
2587 fatal("couldn't compile global entry");
2588 }
2589
2590 void name(codegen_done)(void)
2591 {
2592 os_code_unmap(codegen_ptr, codegen_size);
2593 #ifdef DEBUG_ENV
2594 mutex_done(&dump_mutex);
2595 #endif
2596 }
2597
2598 #else
2599
2600 void name(codegen_init)(void)
2601 {
2602 }
2603
2604 void name(codegen_done)(void)
2605 {
2606 }
2607
2608 #endif
2609
2610 #endif
Ajla - a purely functional programming language