?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
verify: implement P_Array_Append_One
[ajla.git] / codegen.c
blob5858ddfb3e4ebddac51dc503130a324d930f3e89
1 /*
2 * Copyright (C) 2024 Mikulas Patocka
3 *
4 * This file is part of Ajla.
5 *
6 * Ajla is free software: you can redistribute it and/or modify it under the
7 * terms of the GNU General Public License as published by the Free Software
8 * Foundation, either version 3 of the License, or (at your option) any later
9 * version.
10 *
11 * Ajla is distributed in the hope that it will be useful, but WITHOUT ANY
12 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
13 * A PARTICULAR PURPOSE. See the GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License along with
16 * Ajla. If not, see <https://www.gnu.org/licenses/>.
17 */
18
19 #include "ajla.h"
20
21 #ifndef FILE_OMIT
22
23 #include "data.h"
24 #include "os.h"
25 #include "os_util.h"
26 #include "util.h"
27 #include "ipfn.h"
28 #include "ipret.h"
29 #include "funct.h"
30 #include "thread.h"
31 #include "task.h"
32 #include "save.h"
33
34 #include "codegen.h"
35
36 #ifdef HAVE_CODEGEN
37
38 #define flag_cache_chicken 0
39 #define must_be_flat_chicken 0
40 #define ra_chicken 0
41
42 #define INLINE_BITMAP_SLOTS 16
43 #define INLINE_COPY_SIZE 64
44
45 /*#define DEBUG_INSNS*/
46 /*#define DEBUG_GARBAGE*/
47
48 #if (defined(ARCH_X86_64) || defined(ARCH_X86_X32)) && !defined(ARCH_X86_WIN_ABI)
49 #if defined(HAVE_SYSCALL) && defined(HAVE_ASM_PRCTL_H) && defined(HAVE_SYS_SYSCALL_H)
50 #include <asm/prctl.h>
51 #include <sys/syscall.h>
52 #include <unistd.h>
53 #endif
54 #if (defined(HAVE_AMD64_SET_GSBASE) || defined(HAVE_SYSARCH)) && defined(HAVE_X86_SYSARCH_H)
55 #include <x86/sysarch.h>
56 #endif
57 #endif
58
59 code_return_t (*codegen_entry)(frame_s *, struct cg_upcall_vector_s *, tick_stamp_t, void *);
60 static void *codegen_ptr;
61 static size_t codegen_size;
62
63 #ifdef DEBUG_ENV
64 static mutex_t dump_mutex;
65 static uint64_t dump_seq = 0;
66 #endif
67
68 /*
69 * insn:
70 * opcode - 16 bits
71 * op_size - 3 bits
72 * aux - 7 bits
73 * writes flags - 2 bit
74 * jmp size - 2 bits
75 */
76
77 #define INSN_OPCODE 0x0000ffffUL
78 #define INSN_OP_SIZE 0x00070000UL
79 #define INSN_AUX 0x03f80000UL
80 #define INSN_WRITES_FLAGS 0x0c000000UL
81 #define INSN_JUMP_SIZE 0x30000000UL
82
83 #define INSN_OPCODE_SHIFT 0
84 #define INSN_OP_SIZE_SHIFT 16
85 #define INSN_AUX_SHIFT 19
86 #define INSN_WRITES_FLAGS_SHIFT 26
87 #define INSN_JUMP_SIZE_SHIFT 28
88
89 #define insn_opcode(insn) (((insn) >> INSN_OPCODE_SHIFT) & (INSN_OPCODE >> INSN_OPCODE_SHIFT))
90 #define insn_op_size(insn) (((insn) >> INSN_OP_SIZE_SHIFT) & (INSN_OP_SIZE >> INSN_OP_SIZE_SHIFT))
91 #define insn_aux(insn) (((insn) >> INSN_AUX_SHIFT) & (INSN_AUX >> INSN_AUX_SHIFT))
92 #define insn_writes_flags(insn) (((insn) >> INSN_WRITES_FLAGS_SHIFT) & (INSN_WRITES_FLAGS >> INSN_WRITES_FLAGS_SHIFT))
93 #define insn_jump_size(insn) (((insn) >> INSN_JUMP_SIZE_SHIFT) & (INSN_JUMP_SIZE >> INSN_JUMP_SIZE_SHIFT))
94
95 #define ALU_ADD 0x00
96 #define ALU_OR 0x01
97 #define ALU_ADC 0x02
98 #define ALU_SBB 0x03
99 #define ALU_AND 0x04
100 #define ALU_SUB 0x05
101 #define ALU_XOR 0x06
102 #define ALU_ORN 0x08
103 #define ALU_ANDN 0x09
104 #define ALU_XORN 0x0a
105 #define ALU_MUL 0x10
106 #define ALU_UMULH 0x11
107 #define ALU_SMULH 0x12
108 #define ALU_UDIV 0x13
109 #define ALU_SDIV 0x14
110 #define ALU_UREM 0x15
111 #define ALU_SREM 0x16
112 #define ALU_SAVE 0x17
113 #define ALU_EXTBL 0x18
114 #define ALU_EXTWL 0x19
115 #define ALU_EXTLL 0x1a
116 #define ALU_EXTLH 0x1b
117 #define ALU_INSBL 0x1c
118 #define ALU_MSKBL 0x1d
119 #define ALU_ZAP 0x20
120 #define ALU_ZAPNOT 0x21
121
122 #define ALU1_NOT 0x00
123 #define ALU1_NEG 0x01
124 #define ALU1_NGC 0x02
125 #define ALU1_BSWAP 0x03
126 #define ALU1_BSWAP16 0x04
127 #define ALU1_BREV 0x05
128 #define ALU1_BSF 0x06
129 #define ALU1_BSR 0x07
130 #define ALU1_LZCNT 0x08
131 #define ALU1_POPCNT 0x09
132
133 #define FP_ALU_ADD 0
134 #define FP_ALU_SUB 1
135 #define FP_ALU_MUL 2
136 #define FP_ALU_DIV 3
137 #define FP_ALU1_NEG 0
138 #define FP_ALU1_SQRT 1
139 #define FP_ALU1_ROUND 2
140 #define FP_ALU1_FLOOR 3
141 #define FP_ALU1_CEIL 4
142 #define FP_ALU1_TRUNC 5
143 #define FP_ALU1_VCNT8 6
144 #define FP_ALU1_VPADDL 7
145 #define FP_ALU1_ADDV 8
146
147 #define COND_O 0x0
148 #define COND_NO 0x1
149 #define COND_B 0x2
150 #define COND_AE 0x3
151 #define COND_E 0x4
152 #define COND_NE 0x5
153 #define COND_BE 0x6
154 #define COND_A 0x7
155 #define COND_S 0x8
156 #define COND_NS 0x9
157 #define COND_P 0xa
158 #define COND_NP 0xb
159 #define COND_L 0xc
160 #define COND_GE 0xd
161 #define COND_LE 0xe
162 #define COND_G 0xf
163 #define COND_BLBC 0x10
164 #define COND_BLBS 0x11
165 #define COND_ALWAYS 0x12
166
167 #define COND_FP 0x20
168 #define FP_COND_P (COND_FP | COND_P)
169 #define FP_COND_NP (COND_FP | COND_NP)
170 #define FP_COND_E (COND_FP | COND_E)
171 #define FP_COND_NE (COND_FP | COND_NE)
172 #define FP_COND_A (COND_FP | COND_A)
173 #define FP_COND_BE (COND_FP | COND_BE)
174 #define FP_COND_B (COND_FP | COND_B)
175 #define FP_COND_AE (COND_FP | COND_AE)
176
177 #define ROT_ROL 0x0
178 #define ROT_ROR 0x1
179 #define ROT_RCL 0x2
180 #define ROT_RCR 0x3
181 #define ROT_SHL 0x4
182 #define ROT_SHR 0x5
183 #define ROT_SAR 0x7
184 #define ROT_SAL 0x8
185
186 #define BTX_BT 0x0
187 #define BTX_BTS 0x1
188 #define BTX_BTR 0x2
189 #define BTX_BTC 0x3
190 #define BTX_BTEXT 0x4
191
192 #define OP_SIZE_1 0
193 #define OP_SIZE_2 1
194 #define OP_SIZE_4 2
195 #define OP_SIZE_8 3
196 #define OP_SIZE_16 4
197 #define OP_SIZE_10 7
198
199 #define MOV_MASK_0_16 0x0
200 #define MOV_MASK_16_32 0x1
201 #define MOV_MASK_32_48 0x2
202 #define MOV_MASK_48_64 0x3
203 #define MOV_MASK_0_8 0x4
204 #define MOV_MASK_32_64 0x5
205 #define MOV_MASK_52_64 0x6
206
207 #define JMP_SHORTEST 0x0000
208 #define JMP_SHORT 0x0001
209 #define JMP_LONG 0x0002
210 #define JMP_EXTRA_LONG 0x0003
211
212 enum {
213 INSN_ENTRY,
214 INSN_LABEL,
215 INSN_RET,
216 INSN_RET_IMM,
217 INSN_ARM_PUSH,
218 INSN_ARM_POP,
219 INSN_S390_PUSH,
220 INSN_S390_POP,
221 INSN_IA64_ALLOC,
222 INSN_IA64_DEALLOC,
223 INSN_PUSH,
224 INSN_PUSH2,
225 INSN_POP,
226 INSN_POP2,
227 INSN_CALL,
228 INSN_CALL_INDIRECT,
229 INSN_MOV,
230 INSN_MOVSX,
231 INSN_MOV_U,
232 INSN_MOV_LR,
233 INSN_CMP,
234 INSN_CMP_DEST_REG,
235 INSN_CMN,
236 INSN_TEST,
237 INSN_TEST_DEST_REG,
238 INSN_ALU,
239 INSN_ALU_PARTIAL,
240 INSN_ALU_FLAGS,
241 INSN_ALU_FLAGS_PARTIAL,
242 INSN_ALU_TRAP,
243 INSN_ALU_FLAGS_TRAP,
244 INSN_ALU1,
245 INSN_ALU1_PARTIAL,
246 INSN_ALU1_FLAGS,
247 INSN_ALU1_FLAGS_PARTIAL,
248 INSN_ALU1_TRAP,
249 INSN_LEA3,
250 INSN_ROT,
251 INSN_ROT_PARTIAL,
252 INSN_BT,
253 INSN_BTX,
254 INSN_MUL_L,
255 INSN_DIV_L,
256 INSN_MADD,
257 INSN_CBW,
258 INSN_CBW_PARTIAL,
259 INSN_CWD,
260 INSN_CWD_PARTIAL,
261 INSN_SET_COND,
262 INSN_SET_COND_PARTIAL,
263 INSN_CMOV,
264 INSN_CMOV_XCC,
265 INSN_CMP_CMOV,
266 INSN_MOVR,
267 INSN_CSEL_SEL,
268 INSN_CSEL_INC,
269 INSN_CSEL_INV,
270 INSN_CSEL_NEG,
271 INSN_STP,
272 INSN_LDP,
273 INSN_MOV_MASK,
274 INSN_MEMCPY,
275 INSN_MEMSET,
276 INSN_FP_CMP,
277 INSN_FP_CMP_DEST_REG,
278 INSN_FP_CMP_DEST_REG_TRAP,
279 INSN_FP_CMP_UNORDERED_DEST_REG,
280 INSN_FP_CMP_COND,
281 INSN_FP_TEST_REG,
282 INSN_FP_TO_INT_FLAGS,
283 INSN_FP_ALU,
284 INSN_FP_ALU1,
285 INSN_FP_TO_INT32,
286 INSN_FP_TO_INT64,
287 INSN_FP_TO_INT64_TRAP,
288 INSN_FP_FROM_INT32,
289 INSN_FP_FROM_INT64,
290 INSN_FP_INT64_TO_INT32_TRAP,
291 INSN_FP_CVT,
292 INSN_X87_FLD,
293 INSN_X87_FILD,
294 INSN_X87_FSTP,
295 INSN_X87_FISTP,
296 INSN_X87_FISTTP,
297 INSN_X87_FCOMP,
298 INSN_X87_FCOMPP,
299 INSN_X87_FCOMIP,
300 INSN_X87_ALU,
301 INSN_X87_ALUP,
302 INSN_X87_FCHS,
303 INSN_X87_FSQRT,
304 INSN_X87_FRNDINT,
305 INSN_X87_FNSTSW,
306 INSN_X87_FLDCW,
307 INSN_JMP,
308 INSN_JMP_COND,
309 INSN_JMP_COND_LOGICAL,
310 INSN_JMP_REG,
311 INSN_JMP_REG_BIT,
312 INSN_JMP_2REGS,
313 INSN_JMP_FP_TEST,
314 INSN_JMP_INDIRECT,
315 INSN_MB,
316 INSN_CALL_MILLICODE,
317 };
318
319 #define ARG_REGS_MAX 0xc0
320 #define ARG_SHIFTED_REGISTER 0xc0
321 #define ARG_SHIFT_AMOUNT 0x3f
322 #define ARG_SHIFT_MODE 0xc0
323 #define ARG_SHIFT_LSL 0x00
324 #define ARG_SHIFT_LSR 0x40
325 #define ARG_SHIFT_ASR 0x80
326 #define ARG_SHIFT_ROR 0xc0
327 #define ARG_EXTENDED_REGISTER 0xc1
328 #define ARG_EXTEND_SHIFT 0x07
329 #define ARG_EXTEND_MODE 0x38
330 #define ARG_EXTEND_UXTB 0x00
331 #define ARG_EXTEND_UXTH 0x08
332 #define ARG_EXTEND_UXTW 0x10
333 #define ARG_EXTEND_UXTX 0x18
334 #define ARG_EXTEND_SXTB 0x20
335 #define ARG_EXTEND_SXTH 0x28
336 #define ARG_EXTEND_SXTW 0x30
337 #define ARG_EXTEND_SXTX 0x38
338 #define ARG_ADDRESS_0 0xd0
339 #define ARG_ADDRESS_1 0xd1
340 #define ARG_ADDRESS_1_2 0xd2
341 #define ARG_ADDRESS_1_4 0xd3
342 #define ARG_ADDRESS_1_8 0xd4
343 #define ARG_ADDRESS_1_PRE_I 0xd5
344 #define ARG_ADDRESS_1_POST_I 0xd6
345 #define ARG_ADDRESS_2 0xd7
346 #define ARG_ADDRESS_2_2 0xd8
347 #define ARG_ADDRESS_2_4 0xd9
348 #define ARG_ADDRESS_2_8 0xda
349 #define ARG_ADDRESS_2_UXTW 0xdb
350 #define ARG_ADDRESS_2_SXTW 0xdc
351 #define ARG_IMM 0xe0
352
353 #define ARG_IS_ADDRESS(a) ((a) >= ARG_ADDRESS_0 && (a) <= ARG_ADDRESS_2_SXTW)
354
355 #ifdef POINTER_COMPRESSION
356 #define OP_SIZE_SLOT OP_SIZE_4
357 #else
358 #define OP_SIZE_SLOT OP_SIZE_ADDRESS
359 #endif
360
361 #define OP_SIZE_BITMAP (bitmap_64bit ? OP_SIZE_8 : OP_SIZE_4)
362
363 #define OP_SIZE_INT log_2(sizeof(int_default_t))
364
365 #define check_insn(insn) \
366 do { \
367 /*if ((insn_opcode(insn) == INSN_ALU || insn_opcode(insn) == INSN_ALU1) && insn_op_size(insn) != OP_SIZE_NATIVE) internal(file_line, "invalid insn %08x", (unsigned)(insn));*/\
368 /*if (insn == 0x001a000e) internal(file_line, "invalid insn %08x", insn);*/\
369 } while (0)
370
371 #ifdef DEBUG_INSNS
372 #define gen_line() gen_four(__LINE__ + (insn_file << 24))
373 #else
374 #define gen_line() do { } while (0)
375 #endif
376
377 #ifdef ARCH_IA64
378 #define ARCH_CONTEXT struct { \
379 uint64_t insns[3]; \
380 uint8_t insn_units[3]; \
381 bool insn_stops[3]; \
382 uint64_t wr_mask[4]; \
383 }
384 #endif
385
386 #define gen_insn(opcode, op_size, aux, writes_flags) \
387 do { \
388 uint32_t dword = \
389 (uint32_t)(opcode) << INSN_OPCODE_SHIFT | \
390 (uint32_t)(op_size) << INSN_OP_SIZE_SHIFT | \
391 (uint32_t)(aux) << INSN_AUX_SHIFT | \
392 (uint32_t)(writes_flags) << INSN_WRITES_FLAGS_SHIFT; \
393 check_insn(dword); \
394 gen_line(); \
395 gen_four(dword); \
396 } while (0)
397
398 static size_t arg_size(uint8_t arg)
399 {
400 if (arg < ARG_REGS_MAX)
401 return 1;
402 if (arg >= ARG_SHIFTED_REGISTER && arg <= ARG_EXTENDED_REGISTER)
403 return 3;
404 if (arg == ARG_ADDRESS_0)
405 return 9;
406 if (arg >= ARG_ADDRESS_1 && arg <= ARG_ADDRESS_1_POST_I)
407 return 10;
408 if (arg >= ARG_ADDRESS_2 && arg <= ARG_ADDRESS_2_SXTW)
409 return 11;
410 if (arg == ARG_IMM)
411 return 9;
412 internal(file_line, "arg_size: invalid argument %02x", arg);
413 return 0;
414 }
415
416 struct relocation {
417 uint32_t label_id;
418 uint8_t length;
419 size_t position;
420 size_t jmp_instr;
421 };
422
423 struct code_arg {
424 frame_t slot;
425 frame_t flags;
426 frame_t type;
427 };
428
429 struct cg_entry {
430 size_t entry_to_pos;
431 frame_t *variables;
432 size_t n_variables;
433 uint32_t entry_label;
434 uint32_t nonflat_label;
435 };
436
437 struct cg_exit {
438 uint32_t undo_label;
439 uint8_t undo_opcode;
440 uint8_t undo_op_size;
441 uint8_t undo_aux;
442 uint8_t undo_writes_flags;
443 uint8_t undo_parameters[35];
444 uint8_t undo_parameters_len;
445 uint32_t escape_label;
446 };
447
448 #define FLAG_CACHE_IS_FLAT 0x01
449 #define FLAG_CACHE_IS_NOT_FLAT 0x02
450 #define FLAG_CACHE_IS_NOT_THUNK 0x04
451
452 struct codegen_context {
453 struct data *fn;
454 struct data **local_directory;
455
456 const code_t *instr_start;
457 const code_t *current_position;
458 uchar_efficient_t arg_mode;
459
460 uint32_t label_id;
461 struct cg_entry *entries;
462 frame_t n_entries;
463
464 uint8_t *code;
465 size_t code_size;
466
467 uint8_t *code_position;
468
469 uint32_t *code_labels;
470 struct cg_exit **code_exits;
471 uint32_t escape_nospill_label;
472 uint32_t call_label;
473 uint32_t reload_label;
474
475 uint8_t *mcode;
476 size_t mcode_size;
477
478 size_t *label_to_pos;
479 struct relocation *reloc;
480 size_t reloc_size;
481
482 struct trap_record *trap_records;
483 size_t trap_records_size;
484
485 struct code_arg *args;
486 size_t args_l;
487 const code_t *return_values;
488
489 uint8_t *flag_cache;
490 short *registers;
491 frame_t *need_spill;
492 size_t need_spill_l;
493
494 unsigned base_reg;
495 bool offset_reg;
496 int64_t offset_imm;
497
498 bool const_reg;
499 int64_t const_imm;
500
501 struct data *codegen;
502
503 int upcall_offset;
504 int8_t upcall_args;
505 uint8_t n_pushes;
506 bool upcall_hacked_abi;
507 frame_t *var_aux;
508
509 ajla_error_t err;
510
511 #ifdef ARCH_CONTEXT
512 ARCH_CONTEXT a;
513 #endif
514 };
515
516 static void init_ctx(struct codegen_context *ctx)
517 {
518 ctx->local_directory = NULL;
519 ctx->label_id = 0;
520 ctx->entries = NULL;
521 ctx->n_entries = 0;
522 ctx->code = NULL;
523 ctx->code_labels = NULL;
524 ctx->code_exits = NULL;
525 ctx->escape_nospill_label = 0;
526 ctx->call_label = 0;
527 ctx->reload_label = 0;
528 ctx->mcode = NULL;
529 ctx->label_to_pos = NULL;
530 ctx->reloc = NULL;
531 ctx->trap_records = NULL;
532 ctx->args = NULL;
533 ctx->flag_cache = NULL;
534 ctx->registers = NULL;
535 ctx->need_spill = NULL;
536 ctx->codegen = NULL;
537 ctx->upcall_offset = -1;
538 ctx->upcall_args = -1;
539 ctx->upcall_hacked_abi = false;
540 ctx->var_aux = NULL;
541 }
542
543 static void done_ctx(struct codegen_context *ctx)
544 {
545 if (ctx->local_directory)
546 mem_free(ctx->local_directory);
547 if (ctx->entries) {
548 size_t i;
549 for (i = 0; i < ctx->n_entries; i++) {
550 struct cg_entry *ce = &ctx->entries[i];
551 if (ce->variables)
552 mem_free(ce->variables);
553 }
554 mem_free(ctx->entries);
555 }
556 if (ctx->code)
557 mem_free(ctx->code);
558 if (ctx->code_labels)
559 mem_free(ctx->code_labels);
560 if (ctx->code_exits) {
561 ip_t ip;
562 ip_t cs = da(ctx->fn,function)->code_size;
563 for (ip = 0; ip < cs; ip++) {
564 if (ctx->code_exits[ip])
565 mem_free(ctx->code_exits[ip]);
566 }
567 mem_free(ctx->code_exits);
568 }
569 if (ctx->mcode)
570 mem_free(ctx->mcode);
571 if (ctx->label_to_pos)
572 mem_free(ctx->label_to_pos);
573 if (ctx->reloc)
574 mem_free(ctx->reloc);
575 if (ctx->trap_records)
576 mem_free(ctx->trap_records);
577 if (ctx->args)
578 mem_free(ctx->args);
579 if (ctx->flag_cache)
580 mem_free(ctx->flag_cache);
581 if (ctx->registers)
582 mem_free(ctx->registers);
583 if (ctx->need_spill)
584 mem_free(ctx->need_spill);
585 if (ctx->codegen)
586 data_free(ctx->codegen);
587 if (ctx->var_aux)
588 mem_free(ctx->var_aux);
589 }
590
591
592 static inline code_t get_code(struct codegen_context *ctx)
593 {
594 ajla_assert(ctx->current_position < da(ctx->fn,function)->code + da(ctx->fn,function)->code_size, (file_line, "get_code: ran out of code"));
595 return *ctx->current_position++;
596 }
597
598 static inline uint32_t get_uint32(struct codegen_context *ctx)
599 {
600 uint32_t a1 = get_code(ctx);
601 uint32_t a2 = get_code(ctx);
602 #if !CODE_ENDIAN
603 return a1 + (a2 << 16);
604 #else
605 return a2 + (a1 << 16);
606 #endif
607 }
608
609 static int32_t get_jump_offset(struct codegen_context *ctx)
610 {
611 if (SIZEOF_IP_T == 2) {
612 return (int32_t)(int16_t)get_code(ctx);
613 } else if (SIZEOF_IP_T == 4) {
614 return (int32_t)get_uint32(ctx);
615 } else {
616 not_reached();
617 return -1;
618 }
619 }
620
621 static inline void get_one(struct codegen_context *ctx, frame_t *v)
622 {
623 if (!ctx->arg_mode) {
624 code_t c = get_code(ctx);
625 ajla_assert(!(c & ~0xff), (file_line, "get_one: high byte is not cleared: %u", (unsigned)c));
626 *v = c & 0xff;
627 } else if (ctx->arg_mode == 1) {
628 *v = get_code(ctx);
629 #if ARG_MODE_N >= 2
630 } else if (ctx->arg_mode == 2) {
631 *v = get_uint32(ctx);
632 #endif
633 } else {
634 internal(file_line, "get_one: invalid arg mode %u", ctx->arg_mode);
635 }
636 }
637
638 static inline void get_two(struct codegen_context *ctx, frame_t *v1, frame_t *v2)
639 {
640 if (!ctx->arg_mode) {
641 code_t c = get_code(ctx);
642 *v1 = c & 0xff;
643 *v2 = c >> 8;
644 } else if (ctx->arg_mode == 1) {
645 *v1 = get_code(ctx);
646 *v2 = get_code(ctx);
647 #if ARG_MODE_N >= 2
648 } else if (ctx->arg_mode == 2) {
649 *v1 = get_uint32(ctx);
650 *v2 = get_uint32(ctx);
651 #endif
652 } else {
653 internal(file_line, "get_two: invalid arg mode %u", ctx->arg_mode);
654 }
655 }
656
657
658 static uint32_t alloc_label(struct codegen_context *ctx)
659 {
660 return ++ctx->label_id;
661 }
662
663 static struct cg_exit *alloc_cg_exit_for_ip(struct codegen_context *ctx, const code_t *code)
664 {
665 ip_t ip = code - da(ctx->fn,function)->code;
666 struct cg_exit *ce = ctx->code_exits[ip];
667 if (!ce) {
668 ce = mem_calloc_mayfail(struct cg_exit *, sizeof(struct cg_exit), &ctx->err);
669 if (unlikely(!ce))
670 return NULL;
671 ctx->code_exits[ip] = ce;
672 }
673 return ce;
674 }
675
676 static struct cg_exit *alloc_undo_label(struct codegen_context *ctx)
677 {
678 struct cg_exit *ce = alloc_cg_exit_for_ip(ctx, ctx->instr_start);
679 if (unlikely(!ce))
680 return NULL;
681 if (unlikely(ce->undo_label != 0))
682 internal(file_line, "alloc_cg_exit: undo label already allocated");
683 ce->undo_label = alloc_label(ctx);
684 if (unlikely(!ce->undo_label))
685 return NULL;
686 return ce;
687 }
688
689 static uint32_t alloc_escape_label_for_ip(struct codegen_context *ctx, const code_t *code)
690 {
691 struct cg_exit *ce = alloc_cg_exit_for_ip(ctx, code);
692 if (!ce)
693 return 0;
694 if (!ce->escape_label)
695 ce->escape_label = alloc_label(ctx);
696 return ce->escape_label;
697 }
698
699 static uint32_t alloc_escape_label(struct codegen_context *ctx)
700 {
701 return alloc_escape_label_for_ip(ctx, ctx->instr_start);
702 }
703
704 static uint32_t attr_unused alloc_call_label(struct codegen_context *ctx)
705 {
706 if (!ctx->call_label) {
707 ctx->call_label = alloc_label(ctx);
708 }
709 return ctx->call_label;
710 }
711
712 static uint32_t alloc_reload_label(struct codegen_context *ctx)
713 {
714 if (!ctx->reload_label) {
715 ctx->reload_label = alloc_label(ctx);
716 }
717 return ctx->reload_label;
718 }
719
720 static size_t attr_unused mark_params(struct codegen_context *ctx)
721 {
722 return ctx->code_size;
723 }
724
725 static void attr_unused copy_params(struct codegen_context *ctx, struct cg_exit *ce, size_t mark)
726 {
727 if (ctx->code_size - mark > n_array_elements(ce->undo_parameters))
728 internal(file_line, "undo_parameters is too small: %"PRIuMAX" > %"PRIuMAX"", (uintmax_t)(ctx->code_size - mark), (uintmax_t)n_array_elements(ce->undo_parameters));
729 memcpy(ce->undo_parameters, ctx->code + mark, ctx->code_size - mark);
730 ce->undo_parameters_len = ctx->code_size - mark;
731 ctx->code_size = mark;
732 }
733
734 #define g(call) \
735 do { \
736 if (unlikely(!call)) \
737 return false; \
738 } while (0)
739
740 #define gen_one(byte) \
741 do { \
742 /*debug("gen %d: %02x", __LINE__, (uint8_t)(byte))*/; \
743 if (unlikely(!array_add_mayfail(uint8_t, &ctx->code, &ctx->code_size, byte, NULL, &ctx->err)))\
744 return false; \
745 } while (0)
746
747 #if defined(C_LITTLE_ENDIAN)
748 #define gen_two(word) \
749 do { \
750 uint16_t word_ = (word); \
751 /*debug("gen %d: %04x", __LINE__, (uint16_t)(word_));*/ \
752 if (unlikely(!array_add_multiple_mayfail(uint8_t, &ctx->code, &ctx->code_size, cast_ptr(uint8_t *, &word_), 2, NULL, &ctx->err)))\
753 return false; \
754 } while (0)
755 #define gen_four(dword) \
756 do { \
757 uint32_t dword_ = (dword); \
758 /*debug("gen %d: %08x", __LINE__, (uint32_t)(dword_));*/ \
759 if (unlikely(!array_add_multiple_mayfail(uint8_t, &ctx->code, &ctx->code_size, cast_ptr(uint8_t *, &dword_), 4, NULL, &ctx->err)))\
760 return false; \
761 } while (0)
762 #define gen_eight(qword) \
763 do { \
764 uint64_t qword_ = (qword); \
765 /*debug("gen %d: %016lx", __LINE__, (uint64_t)(qword_));*/ \
766 if (unlikely(!array_add_multiple_mayfail(uint8_t, &ctx->code, &ctx->code_size, cast_ptr(uint8_t *, &qword_), 8, NULL, &ctx->err)))\
767 return false; \
768 } while (0)
769 #else
770 #define gen_two(word) \
771 do { \
772 uint16_t word_ = (word); \
773 gen_one(word_ & 0xffU); \
774 gen_one(word_ >> 8); \
775 } while (0)
776 #define gen_four(dword) \
777 do { \
778 uint32_t dword_ = (dword); \
779 gen_two(dword_ & 0xffffU); \
780 gen_two(dword_ >> 15 >> 1); \
781 } while (0)
782 #define gen_eight(qword) \
783 do { \
784 uint64_t qword_ = (qword); \
785 gen_four(qword_ & 0xffffffffUL); \
786 gen_four(qword_ >> 15 >> 15 >> 2); \
787 } while (0)
788 #endif
789
790 #define gen_label(label_id) \
791 do { \
792 gen_insn(INSN_LABEL, 0, 0, 0); \
793 gen_four(label_id); \
794 } while (0)
795
796
797 static uint8_t attr_unused cget_one(struct codegen_context *ctx)
798 {
799 ajla_assert(ctx->code_position < ctx->code + ctx->code_size, (file_line, "cget_one: ran out of code"));
800 return *ctx->code_position++;
801 }
802
803 static uint16_t attr_unused cget_two(struct codegen_context *ctx)
804 {
805 #if defined(C_LITTLE_ENDIAN)
806 uint16_t r;
807 ajla_assert(ctx->code_position < ctx->code + ctx->code_size, (file_line, "cget_two: ran out of code"));
808 memcpy(&r, ctx->code_position, 2);
809 ctx->code_position += 2;
810 return r;
811 #else
812 uint16_t r = cget_one(ctx);
813 r |= cget_one(ctx) << 8;
814 return r;
815 #endif
816 }
817
818 static uint32_t cget_four(struct codegen_context *ctx)
819 {
820 #if defined(C_LITTLE_ENDIAN)
821 uint32_t r;
822 ajla_assert(ctx->code_position < ctx->code + ctx->code_size, (file_line, "cget_four: ran out of code"));
823 memcpy(&r, ctx->code_position, 4);
824 ctx->code_position += 4;
825 return r;
826 #else
827 uint32_t r = cget_two(ctx);
828 r |= (uint32_t)cget_two(ctx) << 16;
829 return r;
830 #endif
831 }
832
833 static uint64_t attr_unused cget_eight(struct codegen_context *ctx)
834 {
835 #if defined(C_LITTLE_ENDIAN)
836 uint64_t r;
837 ajla_assert(ctx->code_position < ctx->code + ctx->code_size, (file_line, "cget_eight: ran out of code"));
838 memcpy(&r, ctx->code_position, 8);
839 ctx->code_position += 8;
840 return r;
841 #else
842 uint64_t r = cget_four(ctx);
843 r |= (uint64_t)cget_four(ctx) << 32;
844 return r;
845 #endif
846 }
847
848 static int64_t get_imm(uint8_t *ptr)
849 {
850 #if defined(C_LITTLE_ENDIAN)
851 int64_t r;
852 memcpy(&r, ptr, 8);
853 return r;
854 #else
855 int64_t r;
856 r = (uint64_t)ptr[0] |
857 ((uint64_t)ptr[1] << 8) |
858 ((uint64_t)ptr[2] << 16) |
859 ((uint64_t)ptr[3] << 24) |
860 ((uint64_t)ptr[4] << 32) |
861 ((uint64_t)ptr[5] << 40) |
862 ((uint64_t)ptr[6] << 48) |
863 ((uint64_t)ptr[7] << 56);
864 return r;
865 #endif
866 }
867
868 #define cgen_one(byte) \
869 do { \
870 if (unlikely(!array_add_mayfail(uint8_t, &ctx->mcode, &ctx->mcode_size, byte, NULL, &ctx->err)))\
871 return false; \
872 } while (0)
873
874 #if defined(C_LITTLE_ENDIAN) || 1
875 #define cgen_two(word) \
876 do { \
877 uint16_t word_ = (word); \
878 if (unlikely(!array_add_multiple_mayfail(uint8_t, &ctx->mcode, &ctx->mcode_size, cast_ptr(uint8_t *, &word_), 2, NULL, &ctx->err)))\
879 return false; \
880 } while (0)
881 #define cgen_four(dword) \
882 do { \
883 uint32_t dword_ = (dword); \
884 /*if (dword_ == 0x1ee02000) internal(file_line, "invalid instruction");*/\
885 if (unlikely(!array_add_multiple_mayfail(uint8_t, &ctx->mcode, &ctx->mcode_size, cast_ptr(uint8_t *, &dword_), 4, NULL, &ctx->err)))\
886 return false; \
887 } while (0)
888 #define cgen_eight(qword) \
889 do { \
890 uint64_t qword_ = (qword); \
891 if (unlikely(!array_add_multiple_mayfail(uint8_t, &ctx->mcode, &ctx->mcode_size, cast_ptr(uint8_t *, &qword_), 8, NULL, &ctx->err)))\
892 return false; \
893 } while (0)
894 #else
895 #define cgen_two(word) \
896 do { \
897 cgen_one((word) & 0xff); \
898 cgen_one((word) >> 8); \
899 } while (0)
900 #define cgen_four(dword) \
901 do { \
902 cgen_two((dword) & 0xffff); \
903 cgen_two((dword) >> 15 >> 1); \
904 } while (0)
905 #define cgen_eight(qword) \
906 do { \
907 cgen_four((qword) & 0xffffffff); \
908 cgen_four((qword) >> 15 >> 15 >> 2); \
909 } while (0)
910 #endif
911
912
913 #define IMM_PURPOSE_LDR_OFFSET 1
914 #define IMM_PURPOSE_LDR_SX_OFFSET 2
915 #define IMM_PURPOSE_STR_OFFSET 3
916 #define IMM_PURPOSE_LDP_STP_OFFSET 4
917 #define IMM_PURPOSE_VLDR_VSTR_OFFSET 5
918 #define IMM_PURPOSE_MVI_CLI_OFFSET 6
919 #define IMM_PURPOSE_STORE_VALUE 7
920 #define IMM_PURPOSE_ADD 8
921 #define IMM_PURPOSE_SUB 9
922 #define IMM_PURPOSE_CMP 10
923 #define IMM_PURPOSE_CMP_LOGICAL 11
924 #define IMM_PURPOSE_AND 12
925 #define IMM_PURPOSE_OR 13
926 #define IMM_PURPOSE_XOR 14
927 #define IMM_PURPOSE_ANDN 15
928 #define IMM_PURPOSE_TEST 16
929 #define IMM_PURPOSE_JMP_2REGS 17
930 #define IMM_PURPOSE_MUL 18
931 #define IMM_PURPOSE_CMOV 19
932 #define IMM_PURPOSE_MOVR 20
933 #define IMM_PURPOSE_BITWISE 21
934 #define IMM_PURPOSE_ADD_TRAP 22
935 #define IMM_PURPOSE_SUB_TRAP 23
936
937
938 static unsigned alu_purpose(unsigned alu)
939 {
940 unsigned purpose =
941 alu == ALU_ADD ? IMM_PURPOSE_ADD :
942 alu == ALU_ADC ? IMM_PURPOSE_ADD :
943 alu == ALU_SUB ? IMM_PURPOSE_SUB :
944 alu == ALU_SBB ? IMM_PURPOSE_SUB :
945 alu == ALU_MUL ? IMM_PURPOSE_MUL :
946 alu == ALU_UMULH ? IMM_PURPOSE_MUL :
947 alu == ALU_SMULH ? IMM_PURPOSE_MUL :
948 alu == ALU_ANDN ? IMM_PURPOSE_ANDN :
949 alu == ALU_AND ? IMM_PURPOSE_AND :
950 alu == ALU_OR ? IMM_PURPOSE_OR :
951 alu == ALU_XOR ? IMM_PURPOSE_XOR :
952 alu == ALU_EXTBL ? IMM_PURPOSE_OR :
953 alu == ALU_EXTWL ? IMM_PURPOSE_OR :
954 alu == ALU_EXTLL ? IMM_PURPOSE_OR :
955 alu == ALU_EXTLH ? IMM_PURPOSE_OR :
956 alu == ALU_INSBL ? IMM_PURPOSE_OR :
957 alu == ALU_MSKBL ? IMM_PURPOSE_OR :
958 alu == ALU_ZAP ? IMM_PURPOSE_ANDN :
959 alu == ALU_ZAPNOT ? IMM_PURPOSE_AND :
960 -1U;
961 if (unlikely(purpose == -1U))
962 internal(file_line, "alu_purpose: invalid alu %u", alu);
963 return purpose;
964 }
965
966 static unsigned alu_trap_purpose(unsigned alu)
967 {
968 unsigned purpose =
969 alu == ALU_ADD ? IMM_PURPOSE_ADD_TRAP :
970 alu == ALU_SUB ? IMM_PURPOSE_SUB_TRAP :
971 -1U;
972 if (unlikely(purpose == -1U))
973 internal(file_line, "alu_trap_purpose: invalid alu %u", alu);
974 return purpose;
975 }
976
977
978 static bool attr_w gen_imm(struct codegen_context *ctx, int64_t imm, unsigned purpose, unsigned size);
979 static bool attr_w gen_upcall_end(struct codegen_context *ctx, unsigned offset, unsigned args);
980
981 #define gen_address_offset() \
982 do { \
983 if (likely(!ctx->offset_reg)) { \
984 gen_one(ARG_ADDRESS_1); \
985 gen_one(ctx->base_reg); \
986 gen_eight(ctx->offset_imm); \
987 } else { \
988 gen_one(ARG_ADDRESS_2); \
989 gen_one(ctx->base_reg); \
990 gen_one(R_OFFSET_IMM); \
991 gen_eight(0); \
992 } \
993 } while (0)
994
995 #define gen_imm_offset() \
996 do { \
997 if (likely(!ctx->const_reg)) { \
998 gen_one(ARG_IMM); \
999 gen_eight(ctx->const_imm); \
1000 } else { \
1001 gen_one(R_CONST_IMM); \
1002 } \
1003 } while (0)
1004
1005 #define is_imm() (!ctx->const_reg)
1006
1007
1008 static inline bool slot_is_register(struct codegen_context *ctx, frame_t slot)
1009 {
1010 if (frame_t_is_const(slot))
1011 return false;
1012 if (unlikely(slot >= function_n_variables(ctx->fn)))
1013 internal(file_line, "slot_is_register: invalid slot %lu", (unsigned long)slot);
1014 return ctx->registers[slot] >= 0;
1015 }
1016
1017
1018 #define insn_file 1
1019 #if defined(ARCH_ALPHA)
1020 #include "c1-alpha.inc"
1021 #elif defined(ARCH_ARM32)
1022 #include "c1-arm.inc"
1023 #elif defined(ARCH_ARM64)
1024 #include "c1-arm64.inc"
1025 #elif defined(ARCH_IA64)
1026 #include "c1-ia64.inc"
1027 #elif defined(ARCH_LOONGARCH64)
1028 #include "c1-loong.inc"
1029 #elif defined(ARCH_MIPS)
1030 #include "c1-mips.inc"
1031 #elif defined(ARCH_PARISC)
1032 #include "c1-hppa.inc"
1033 #elif defined(ARCH_POWER)
1034 #include "c1-power.inc"
1035 #elif defined(ARCH_S390)
1036 #include "c1-s390.inc"
1037 #elif defined(ARCH_SPARC)
1038 #include "c1-sparc.inc"
1039 #elif defined(ARCH_RISCV64)
1040 #include "c1-riscv.inc"
1041 #elif defined(ARCH_X86)
1042 #include "c1-x86.inc"
1043 #endif
1044 #undef insn_file
1045
1046
1047 #ifndef ARCH_SUPPORTS_TRAPS
1048 #define ARCH_SUPPORTS_TRAPS(size) 0
1049 #define ARCH_TRAP_BEFORE 0
1050 #endif
1051
1052
1053 static bool attr_w gen_imm(struct codegen_context *ctx, int64_t imm, unsigned purpose, unsigned size)
1054 {
1055 if (!is_direct_const(imm, purpose & 0xff, size))
1056 goto load_const;
1057 if (purpose >> 8 && !is_direct_const(imm, purpose >> 8, size))
1058 goto load_const;
1059 ctx->const_imm = imm;
1060 ctx->const_reg = false;
1061 return true;
1062 load_const:
1063 g(gen_load_constant(ctx, R_CONST_IMM, imm));
1064 ctx->const_reg = true;
1065 return true;
1066 }
1067
1068
1069 #define insn_file 2
1070 #include "cg-util.inc"
1071 #undef insn_file
1072
1073 #define insn_file 3
1074 #include "cg-frame.inc"
1075 #undef insn_file
1076
1077 #define insn_file 4
1078 #include "cg-flags.inc"
1079 #undef insn_file
1080
1081 #define insn_file 5
1082 #include "cg-flcch.inc"
1083 #undef insn_file
1084
1085 #define insn_file 6
1086 #include "cg-ptr.inc"
1087 #undef insn_file
1088
1089 #define insn_file 7
1090 #include "cg-alu.inc"
1091 #undef insn_file
1092
1093 #define insn_file 8
1094 #include "cg-ops.inc"
1095 #undef insn_file
1096
1097 #define insn_file 0
1098
1099
1100 #ifndef n_regs_saved
1101 #define n_regs_saved n_array_elements(regs_saved)
1102 #endif
1103
1104 #ifndef n_regs_volatile
1105 #define n_regs_volatile n_array_elements(regs_volatile)
1106 #endif
1107
1108 #ifndef n_fp_saved
1109 #define n_fp_saved n_array_elements(fp_saved)
1110 #endif
1111
1112 #ifndef n_fp_volatile
1113 #define n_fp_volatile n_array_elements(fp_volatile)
1114 #endif
1115
1116 #ifndef n_vector_volatile
1117 #define n_vector_volatile n_array_elements(vector_volatile)
1118 #endif
1119
1120 static bool attr_w gen_registers(struct codegen_context *ctx)
1121 {
1122 frame_t v;
1123 size_t index_saved = 0;
1124 size_t index_volatile = 0;
1125 size_t index_fp_saved = 0;
1126 size_t index_fp_volatile = 0;
1127 size_t attr_unused index_vector_volatile = 0;
1128 #ifdef ARCH_S390
1129 bool uses_x = false;
1130 for (v = MIN_USEABLE_SLOT; v < function_n_variables(ctx->fn); v++) {
1131 const struct type *t = get_type_of_local(ctx, v);
1132 if (t && TYPE_TAG_IS_REAL(t->tag) && TYPE_TAG_IDX_REAL(t->tag) == 4) {
1133 uses_x = true;
1134 break;
1135 }
1136 }
1137 #endif
1138 /*for (v = function_n_variables(ctx->fn) - 1; v >= MIN_USEABLE_SLOT; v--)*/
1139 for (v = MIN_USEABLE_SLOT; v < function_n_variables(ctx->fn); v++) {
1140 const struct type *t;
1141 ctx->registers[v] = -1;
1142 if (ra_chicken)
1143 continue;
1144 t = get_type_of_local(ctx, v);
1145 if (unlikely(!t))
1146 continue;
1147 if (!da(ctx->fn,function)->local_variables_flags[v].must_be_flat &&
1148 !da(ctx->fn,function)->local_variables_flags[v].must_be_data)
1149 continue;
1150 if (!ARCH_HAS_BWX && t->size < 1U << OP_SIZE_4)
1151 continue;
1152 if (TYPE_TAG_IS_FIXED(t->tag) || TYPE_TAG_IS_INT(t->tag) || t->tag == TYPE_TAG_flat_option || t->tag == TYPE_TAG_unknown || t->tag == TYPE_TAG_record) {
1153 if (TYPE_TAG_IS_BUILTIN(t->tag)) {
1154 if (!is_power_of_2(t->size) || t->size > 1U << OP_SIZE_NATIVE)
1155 continue;
1156 }
1157 if (index_saved < n_regs_saved + zero
1158 #if defined(ARCH_PARISC) || defined(ARCH_SPARC)
1159 && t->size <= 1U << OP_SIZE_ADDRESS
1160 #endif
1161 ) {
1162 ctx->registers[v] = regs_saved[index_saved++];
1163 } else if (index_volatile < n_regs_volatile + zero) {
1164 ctx->registers[v] = regs_volatile[index_volatile++];
1165 } else {
1166 continue;
1167 }
1168 } else if (TYPE_TAG_IS_REAL(t->tag)) {
1169 unsigned real_type = TYPE_TAG_IDX_REAL(t->tag);
1170 if ((SUPPORTED_FP >> real_type) & 1) {
1171 #ifdef ARCH_POWER
1172 if (real_type == 4) {
1173 if (index_vector_volatile < n_vector_volatile + zero) {
1174 ctx->registers[v] = vector_volatile[index_vector_volatile++];
1175 goto success;
1176 }
1177 continue;
1178 }
1179 #endif
1180 #ifdef ARCH_S390
1181 if (real_type == 4) {
1182 if (!(index_fp_saved & 1) && index_fp_saved + 1 < n_fp_saved + zero) {
1183 ctx->registers[v] = fp_saved[index_fp_saved++];
1184 index_fp_saved++;
1185 goto success;
1186 }
1187 if (index_fp_saved & 1 && index_fp_saved + 2 < n_fp_saved + zero) {
1188 index_fp_saved++;
1189 ctx->registers[v] = fp_saved[index_fp_saved++];
1190 index_fp_saved++;
1191 goto success;
1192 }
1193 if (!(index_fp_volatile & 1) && index_fp_volatile + 1 < n_fp_volatile + zero) {
1194 ctx->registers[v] = fp_volatile[index_fp_volatile++];
1195 index_fp_volatile++;
1196 goto success;
1197 }
1198 if (index_fp_volatile & 1 && index_fp_volatile + 2 < n_fp_volatile + zero) {
1199 index_fp_volatile++;
1200 ctx->registers[v] = fp_volatile[index_fp_volatile++];
1201 index_fp_volatile++;
1202 goto success;
1203 }
1204 continue;
1205 }
1206 #endif
1207 if (index_fp_saved < n_fp_saved + zero) {
1208 ctx->registers[v] = fp_saved[index_fp_saved++];
1209 } else if (index_fp_volatile < n_fp_volatile + zero) {
1210 ctx->registers[v] = fp_volatile[index_fp_volatile++];
1211 } else {
1212 continue;
1213 }
1214 } else {
1215 continue;
1216 }
1217 } else {
1218 continue;
1219 }
1220 goto success;
1221 success:
1222 if (!reg_is_saved(ctx->registers[v])) {
1223 if (unlikely(!array_add_mayfail(frame_t, &ctx->need_spill, &ctx->need_spill_l, v, NULL, &ctx->err)))
1224 return false;
1225 }
1226 }
1227
1228 return true;
1229 }
1230
1231 static bool attr_w gen_fused_binary(struct codegen_context *ctx, unsigned mode, unsigned op_size, unsigned op, uint32_t escape_label, frame_t slot_1, frame_t slot_2, frame_t slot_r, bool *failed)
1232 {
1233 const code_t *backup = ctx->current_position;
1234 code_t code;
1235 frame_t slot_dr, slot_test;
1236 int32_t offs_false;
1237
1238 *failed = false;
1239
1240 next_code:
1241 code = get_code(ctx);
1242 ctx->arg_mode = code / OPCODE_MODE_MULT;
1243 code %= OPCODE_MODE_MULT;
1244 ajla_assert_lo(ctx->arg_mode < ARG_MODE_N, (file_line, "gen_fused_binary: invalid opcode %04x", (unsigned)*ctx->instr_start));
1245
1246 if (code == OPCODE_DEREFERENCE) {
1247 const struct type *t;
1248 get_one(ctx, &slot_dr);
1249 t = get_type_of_local(ctx, slot_dr);
1250 if (!TYPE_TAG_IS_BUILTIN(t->tag)) {
1251 *failed = true;
1252 goto fail;
1253 }
1254 if (unlikely(!flag_is_clear(ctx, slot_dr))) {
1255 *failed = true;
1256 goto fail;
1257 }
1258 goto next_code;
1259 }
1260 if (code == OPCODE_DEREFERENCE_CLEAR) {
1261 *failed = true;
1262 goto fail;
1263 }
1264 if (unlikely(code != OPCODE_JMP_FALSE))
1265 internal(file_line, "gen_fused_binary: binary operation is not followed by jmp false: %x, %s", code, decode_opcode(code, true));
1266 get_one(ctx, &slot_test);
1267 if (unlikely(slot_test != slot_r))
1268 internal(file_line, "gen_fused_binary: the result of the binary operation and the tested variable do not match");
1269 offs_false = get_jump_offset(ctx);
1270 get_jump_offset(ctx);
1271
1272 if (mode == MODE_ARRAY_LEN_GT) {
1273 g(gen_array_len(ctx, slot_1, slot_2, slot_r, true, offs_false));
1274 } else if (mode == MODE_REAL) {
1275 g(gen_fp_alu_jmp(ctx, op_size, op, escape_label, slot_1, slot_2, offs_false, failed));
1276 } else {
1277 g(gen_alu_jmp(ctx, mode, op_size, op, slot_1, slot_2, offs_false, failed));
1278 }
1279
1280 fail:
1281 if (*failed)
1282 ctx->current_position = backup;
1283
1284 return true;
1285 }
1286
1287 static bool attr_w gen_function(struct codegen_context *ctx)
1288 {
1289 ctx->current_position = da(ctx->fn,function)->code;
1290
1291 ctx->escape_nospill_label = alloc_label(ctx);
1292 if (unlikely(!ctx->escape_nospill_label))
1293 return false;
1294
1295 while (ctx->current_position != da(ctx->fn,function)->code + da(ctx->fn,function)->code_size) {
1296 ip_t ip;
1297 code_t code;
1298 unsigned op, type;
1299 frame_t slot_1, slot_2, slot_3, slot_r, flags, fn_idx, opt;
1300 arg_t n_args, n_ret, i_arg;
1301 uint32_t label_id;
1302 uint32_t escape_label;
1303 bool failed;
1304
1305 ajla_assert_lo(ctx->current_position < da(ctx->fn,function)->code + da(ctx->fn,function)->code_size, (file_line, "gen_function: ran out of code in %s", da(ctx->fn,function)->function_name));
1306
1307 ctx->instr_start = ctx->current_position;
1308
1309 /*debug("%s: %04x, %s", da(ctx->fn,function)->function_name, *ctx->instr_start, decode_opcode(*ctx->instr_start, true));*/
1310
1311 ip = ctx->instr_start - da(ctx->fn,function)->code;
1312 if (likely(!ctx->code_labels[ip])) {
1313 ctx->code_labels[ip] = alloc_label(ctx);
1314 if (unlikely(!ctx->code_labels[ip]))
1315 return false;
1316 }
1317 gen_label(ctx->code_labels[ip]);
1318
1319 code = get_code(ctx);
1320 ctx->arg_mode = code / OPCODE_MODE_MULT;
1321 code %= OPCODE_MODE_MULT;
1322 ajla_assert_lo(ctx->arg_mode < ARG_MODE_N, (file_line, "gen_function: invalid opcode %04x", (unsigned)*ctx->instr_start));
1323
1324 if (code >= OPCODE_FIXED_OP + uzero && code < OPCODE_INT_OP) {
1325 code -= OPCODE_FIXED_OP;
1326 op = (code / OPCODE_FIXED_OP_MULT) % OPCODE_FIXED_TYPE_MULT;
1327 type = code / OPCODE_FIXED_TYPE_MULT;
1328 if (op < OPCODE_FIXED_OP_C) {
1329 get_two(ctx, &slot_1, &slot_2);
1330 get_two(ctx, &slot_r, &flags);
1331 escape_label = alloc_escape_label(ctx);
1332 if (unlikely(!escape_label))
1333 return false;
1334 g(gen_test_2_cached(ctx, slot_1, slot_2, escape_label));
1335 flag_set(ctx, slot_1, false);
1336 flag_set(ctx, slot_2, false);
1337 flag_set(ctx, slot_r, false);
1338 if (flags & OPCODE_FLAG_FUSED) {
1339 g(gen_fused_binary(ctx, MODE_FIXED, type, op, escape_label, slot_1, slot_2, slot_r, &failed));
1340 if (unlikely(!failed))
1341 continue;
1342 }
1343 g(gen_alu(ctx, MODE_FIXED, type, op, escape_label, slot_1, slot_2, slot_r));
1344 continue;
1345 } else if (op < OPCODE_FIXED_OP_UNARY) {
1346 op -= OPCODE_FIXED_OP_C;
1347 get_two(ctx, &slot_1, &slot_2);
1348 get_two(ctx, &slot_r, &flags);
1349 escape_label = alloc_escape_label(ctx);
1350 if (unlikely(!escape_label))
1351 return false;
1352 g(gen_test_1_cached(ctx, slot_1, escape_label));
1353 flag_set(ctx, slot_1, false);
1354 flag_set(ctx, slot_r, false);
1355 slot_2 = frame_t_from_const((int32_t)slot_2);
1356 if (flags & OPCODE_FLAG_FUSED) {
1357 g(gen_fused_binary(ctx, MODE_FIXED, type, op, escape_label, slot_1, slot_2, slot_r, &failed));
1358 if (unlikely(!failed))
1359 continue;
1360 }
1361 g(gen_alu(ctx, MODE_FIXED, type, op, escape_label, slot_1, slot_2, slot_r));
1362 continue;
1363 } else if (op < OPCODE_FIXED_OP_N) {
1364 get_two(ctx, &slot_1, &slot_r);
1365 get_one(ctx, &flags);
1366 escape_label = alloc_escape_label(ctx);
1367 if (unlikely(!escape_label))
1368 return false;
1369 g(gen_test_1_cached(ctx, slot_1, escape_label));
1370 flag_set(ctx, slot_1, false);
1371 flag_set(ctx, slot_r, false);
1372 g(gen_alu1(ctx, MODE_FIXED, type, op, escape_label, slot_1, slot_r));
1373 continue;
1374 } else if (op == OPCODE_FIXED_OP_ldc) {
1375 unsigned i;
1376 get_one(ctx, &slot_r);
1377 g(gen_constant(ctx, false, type, false, slot_r));
1378 for (i = 0; i < 1U << type; i += 2)
1379 get_code(ctx);
1380 flag_set(ctx, slot_r, false);
1381 continue;
1382 } else if (op == OPCODE_FIXED_OP_ldc16) {
1383 get_one(ctx, &slot_r);
1384 g(gen_constant(ctx, false, type, true, slot_r));
1385 get_code(ctx);
1386 flag_set(ctx, slot_r, false);
1387 continue;
1388 } else if (op == OPCODE_FIXED_OP_move || op == OPCODE_FIXED_OP_copy) {
1389 get_two(ctx, &slot_1, &slot_r);
1390 escape_label = alloc_escape_label(ctx);
1391 if (unlikely(!escape_label))
1392 return false;
1393 g(gen_test_1_cached(ctx, slot_1, escape_label));
1394 flag_set(ctx, slot_1, false);
1395 flag_set(ctx, slot_r, false);
1396 g(gen_copy(ctx, type, slot_1, slot_r));
1397 continue;
1398 } else {
1399 internal(file_line, "gen_function: bad fixed code %04x", *ctx->instr_start);
1400 }
1401 } else if (code >= OPCODE_INT_OP && code < OPCODE_REAL_OP) {
1402 code -= OPCODE_INT_OP;
1403 op = (code / OPCODE_INT_OP_MULT) % OPCODE_INT_TYPE_MULT;
1404 type = code / OPCODE_INT_TYPE_MULT;
1405 if (op < OPCODE_INT_OP_C) {
1406 get_two(ctx, &slot_1, &slot_2);
1407 get_two(ctx, &slot_r, &flags);
1408 escape_label = alloc_escape_label(ctx);
1409 if (unlikely(!escape_label))
1410 return false;
1411 g(gen_test_2_cached(ctx, slot_1, slot_2, escape_label));
1412 flag_set(ctx, slot_1, false);
1413 flag_set(ctx, slot_2, false);
1414 flag_set(ctx, slot_r, false);
1415 if (flags & OPCODE_FLAG_FUSED) {
1416 g(gen_fused_binary(ctx, MODE_INT, type, op, escape_label, slot_1, slot_2, slot_r, &failed));
1417 if (unlikely(!failed))
1418 continue;
1419 }
1420 g(gen_alu(ctx, MODE_INT, type, op, escape_label, slot_1, slot_2, slot_r));
1421 continue;
1422 } else if (op < OPCODE_INT_OP_UNARY) {
1423 op -= OPCODE_INT_OP_C;
1424 get_two(ctx, &slot_1, &slot_2);
1425 get_two(ctx, &slot_r, &flags);
1426 escape_label = alloc_escape_label(ctx);
1427 if (unlikely(!escape_label))
1428 return false;
1429 g(gen_test_1_cached(ctx, slot_1, escape_label));
1430 flag_set(ctx, slot_1, false);
1431 flag_set(ctx, slot_r, false);
1432 slot_2 = frame_t_from_const((int32_t)slot_2);
1433 if (flags & OPCODE_FLAG_FUSED) {
1434 g(gen_fused_binary(ctx, MODE_INT, type, op, escape_label, slot_1, slot_2, slot_r, &failed));
1435 if (unlikely(!failed))
1436 continue;
1437 }
1438 g(gen_alu(ctx, MODE_INT, type, op, escape_label, slot_1, slot_2, slot_r));
1439 continue;
1440 } else if (op < OPCODE_INT_OP_N) {
1441 get_two(ctx, &slot_1, &slot_r);
1442 get_one(ctx, &flags);
1443 if ((op == OPCODE_INT_OP_to_int || op == OPCODE_INT_OP_from_int) && slot_1 == slot_r)
1444 continue;
1445 escape_label = alloc_escape_label(ctx);
1446 if (unlikely(!escape_label))
1447 return false;
1448 g(gen_test_1_cached(ctx, slot_1, escape_label));
1449 flag_set(ctx, slot_1, false);
1450 flag_set(ctx, slot_r, false);
1451 g(gen_alu1(ctx, MODE_INT, type, op, escape_label, slot_1, slot_r));
1452 continue;
1453 } else if (op == OPCODE_INT_OP_ldc) {
1454 unsigned i;
1455 get_one(ctx, &slot_r);
1456 g(gen_constant(ctx, false, type, false, slot_r));
1457 for (i = 0; i < 1U << type; i += 2)
1458 get_code(ctx);
1459 flag_set(ctx, slot_r, false);
1460 continue;
1461 } else if (op == OPCODE_INT_OP_ldc16) {
1462 get_one(ctx, &slot_r);
1463 g(gen_constant(ctx, false, type, true, slot_r));
1464 get_code(ctx);
1465 flag_set(ctx, slot_r, false);
1466 continue;
1467 } else if (op == OPCODE_INT_OP_move || op == OPCODE_INT_OP_copy) {
1468 get_two(ctx, &slot_1, &slot_r);
1469 escape_label = alloc_escape_label(ctx);
1470 if (unlikely(!escape_label))
1471 return false;
1472 g(gen_test_1_cached(ctx, slot_1, escape_label));
1473 flag_set(ctx, slot_1, false);
1474 flag_set(ctx, slot_r, false);
1475 g(gen_copy(ctx, type, slot_1, slot_r));
1476 continue;
1477 } else {
1478 internal(file_line, "gen_function: bad integer code %04x", *ctx->instr_start);
1479 }
1480 } else if (code >= OPCODE_REAL_OP && code < OPCODE_BOOL_OP) {
1481 code -= OPCODE_REAL_OP;
1482 op = (code / OPCODE_REAL_OP_MULT) % OPCODE_REAL_TYPE_MULT;
1483 type = code / OPCODE_REAL_TYPE_MULT;
1484 if (op < OPCODE_REAL_OP_UNARY) {
1485 get_two(ctx, &slot_1, &slot_2);
1486 get_two(ctx, &slot_r, &flags);
1487 escape_label = alloc_escape_label(ctx);
1488 if (unlikely(!escape_label))
1489 return false;
1490 g(gen_test_2_cached(ctx, slot_1, slot_2, escape_label));
1491 flag_set(ctx, slot_1, false);
1492 flag_set(ctx, slot_2, false);
1493 flag_set(ctx, slot_r, false);
1494 if (flags & OPCODE_FLAG_FUSED) {
1495 g(gen_fused_binary(ctx, MODE_REAL, type, op, escape_label, slot_1, slot_2, slot_r, &failed));
1496 if (unlikely(!failed))
1497 continue;
1498 }
1499 g(gen_fp_alu(ctx, type, op, escape_label, slot_1, slot_2, slot_r));
1500 continue;
1501 } else if (op < OPCODE_REAL_OP_N) {
1502 get_two(ctx, &slot_1, &slot_r);
1503 get_one(ctx, &flags);
1504 escape_label = alloc_escape_label(ctx);
1505 if (unlikely(!escape_label))
1506 return false;
1507 g(gen_test_1_cached(ctx, slot_1, escape_label));
1508 flag_set(ctx, slot_1, false);
1509 flag_set(ctx, slot_r, false);
1510 g(gen_fp_alu1(ctx, type, op, escape_label, slot_1, slot_r));
1511 continue;
1512 } else if (op == OPCODE_REAL_OP_ldc) {
1513 const struct type *t;
1514 unsigned i;
1515 get_one(ctx, &slot_r);
1516 t = type_get_real(type);
1517 g(gen_real_constant(ctx, t, slot_r));
1518 for (i = 0; i < t->size; i += 2)
1519 get_code(ctx);
1520 flag_set(ctx, slot_r, false);
1521 continue;
1522 } else if (op == OPCODE_REAL_OP_move || op == OPCODE_REAL_OP_copy) {
1523 get_two(ctx, &slot_1, &slot_r);
1524 escape_label = alloc_escape_label(ctx);
1525 if (unlikely(!escape_label))
1526 return false;
1527 g(gen_test_1_cached(ctx, slot_1, escape_label));
1528 flag_set(ctx, slot_1, false);
1529 flag_set(ctx, slot_r, false);
1530 g(gen_memcpy_slots(ctx, slot_r, slot_1));
1531 continue;
1532 } else {
1533 internal(file_line, "gen_function: bad real code %04x", *ctx->instr_start);
1534 }
1535 } else if (code >= OPCODE_BOOL_OP && code < OPCODE_EXTRA) {
1536 code -= OPCODE_BOOL_OP;
1537 op = (code / OPCODE_BOOL_OP_MULT) % OPCODE_BOOL_TYPE_MULT;
1538 type = log_2(sizeof(ajla_flat_option_t));
1539 if (op < OPCODE_BOOL_OP_UNARY) {
1540 get_two(ctx, &slot_1, &slot_2);
1541 get_two(ctx, &slot_r, &flags);
1542 escape_label = alloc_escape_label(ctx);
1543 if (unlikely(!escape_label))
1544 return false;
1545 g(gen_test_2_cached(ctx, slot_1, slot_2, escape_label));
1546 flag_set(ctx, slot_1, false);
1547 flag_set(ctx, slot_2, false);
1548 flag_set(ctx, slot_r, false);
1549 if (flags & OPCODE_FLAG_FUSED) {
1550 g(gen_fused_binary(ctx, MODE_BOOL, type, op, escape_label, slot_1, slot_2, slot_r, &failed));
1551 if (unlikely(!failed))
1552 continue;
1553 }
1554 g(gen_alu(ctx, MODE_BOOL, type, op, escape_label, slot_1, slot_2, slot_r));
1555 continue;
1556 } else if (op < OPCODE_BOOL_OP_N) {
1557 get_two(ctx, &slot_1, &slot_r);
1558 get_one(ctx, &flags);
1559 escape_label = alloc_escape_label(ctx);
1560 if (unlikely(!escape_label))
1561 return false;
1562 g(gen_test_1_cached(ctx, slot_1, escape_label));
1563 flag_set(ctx, slot_1, false);
1564 flag_set(ctx, slot_r, false);
1565 g(gen_alu1(ctx, MODE_BOOL, type, op, escape_label, slot_1, slot_r));
1566 continue;
1567 } else if (op == OPCODE_BOOL_OP_move || op == OPCODE_BOOL_OP_copy) {
1568 get_two(ctx, &slot_1, &slot_r);
1569 escape_label = alloc_escape_label(ctx);
1570 if (unlikely(!escape_label))
1571 return false;
1572 g(gen_test_1_cached(ctx, slot_1, escape_label));
1573 flag_set(ctx, slot_1, false);
1574 flag_set(ctx, slot_r, false);
1575 g(gen_copy(ctx, type, slot_1, slot_r));
1576 continue;
1577 } else {
1578 internal(file_line, "gen_function: bad boolean code %04x", *ctx->instr_start);
1579 }
1580 } else switch (code) {
1581 case OPCODE_INT_LDC_LONG: {
1582 uint32_t words, w;
1583 get_one(ctx, &slot_r);
1584 words = get_uint32(ctx);
1585 for (w = 0; w < words; w++)
1586 get_code(ctx);
1587 unconditional_escape:
1588 escape_label = alloc_escape_label(ctx);
1589 if (unlikely(!escape_label))
1590 return false;
1591 gen_insn(INSN_JMP, 0, 0, 0);
1592 gen_four(escape_label);
1593 continue;
1594 }
1595 case OPCODE_IS_EXCEPTION: {
1596 get_two(ctx, &slot_1, &slot_r);
1597 get_one(ctx, &flags);
1598 g(gen_is_exception(ctx, slot_1, slot_r));
1599 continue;
1600 }
1601 case OPCODE_EXCEPTION_CLASS:
1602 case OPCODE_EXCEPTION_TYPE:
1603 case OPCODE_EXCEPTION_AUX: {
1604 get_two(ctx, &slot_1, &slot_r);
1605 get_one(ctx, &flags);
1606 goto unconditional_escape;
1607 }
1608 case OPCODE_SYSTEM_PROPERTY: {
1609 get_two(ctx, &slot_1, &slot_r);
1610 get_one(ctx, &flags);
1611 g(gen_system_property(ctx, slot_1, slot_r));
1612 continue;
1613 }
1614 case OPCODE_FLAT_MOVE:
1615 case OPCODE_FLAT_COPY: {
1616 get_two(ctx, &slot_1, &slot_r);
1617 g(gen_flat_move_copy(ctx, slot_1, slot_r));
1618 continue;
1619 }
1620 case OPCODE_REF_MOVE:
1621 case OPCODE_REF_MOVE_CLEAR:
1622 case OPCODE_REF_COPY: {
1623 get_two(ctx, &slot_1, &slot_r);
1624 g(gen_ref_move_copy(ctx, code, slot_1, slot_r));
1625 continue;
1626 }
1627 case OPCODE_BOX_MOVE_CLEAR:
1628 case OPCODE_BOX_COPY: {
1629 get_two(ctx, &slot_1, &slot_r);
1630 g(gen_box_move_copy(ctx, code, slot_1, slot_r));
1631 continue;
1632 }
1633 case OPCODE_TAKE_BORROWED:
1634 get_one(ctx, &slot_1);
1635 if (!da(ctx->fn,function)->local_variables_flags[slot_1].may_be_borrowed)
1636 continue;
1637 if (unlikely(!(label_id = alloc_label(ctx))))
1638 return false;
1639 if (flag_is_set(ctx, slot_1))
1640 goto take_borrowed_done;
1641 if (flag_is_clear(ctx, slot_1)) {
1642 g(gen_set_1(ctx, R_FRAME, slot_1, 0, true));
1643 goto do_take_borrowed;
1644 }
1645 g(gen_test_1(ctx, R_FRAME, slot_1, 0, label_id, false, TEST_SET));
1646 do_take_borrowed:
1647 g(gen_upcall_start(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_pointer_reference_owned), 1));
1648 g(gen_frame_load(ctx, OP_SIZE_SLOT, garbage, slot_1, 0, false, R_ARG0));
1649 g(gen_upcall_argument(ctx, 0));
1650 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_pointer_reference_owned), 1));
1651 flag_set(ctx, slot_1, true);
1652 take_borrowed_done:
1653 gen_label(label_id);
1654 continue;
1655 case OPCODE_DEREFERENCE:
1656 case OPCODE_DEREFERENCE_CLEAR: {
1657 bool need_bit_test;
1658 /*const struct type *type;*/
1659 get_one(ctx, &slot_1);
1660 if (flag_is_clear(ctx, slot_1))
1661 goto skip_dereference;
1662 /*type = get_type_of_local(ctx, slot_1);*/
1663 /*need_bit_test = 1 || TYPE_IS_FLAT(type) || da(ctx->fn,function)->local_variables[slot_1].may_be_borrowed;*/
1664 need_bit_test = !flag_is_set(ctx, slot_1);
1665 if (need_bit_test) {
1666 if (unlikely(!(label_id = alloc_label(ctx))))
1667 return false;
1668 g(gen_test_1(ctx, R_FRAME, slot_1, 0, label_id, true, TEST_CLEAR));
1669 } else {
1670 g(gen_set_1(ctx, R_FRAME, slot_1, 0, false));
1671 label_id = 0; /* avoid warning */
1672 }
1673 g(gen_upcall_start(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_pointer_dereference), 1));
1674 g(gen_frame_load(ctx, OP_SIZE_SLOT, garbage, slot_1, 0, false, R_ARG0));
1675 g(gen_upcall_argument(ctx, 0));
1676 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_pointer_dereference), 1));
1677 if (need_bit_test)
1678 gen_label(label_id);
1679 skip_dereference:
1680 if (code == OPCODE_DEREFERENCE_CLEAR)
1681 g(gen_frame_clear(ctx, OP_SIZE_SLOT, slot_1));
1682 flag_set_unknown(ctx, slot_1);
1683 flag_set(ctx, slot_1, false);
1684 continue;
1685 }
1686 case OPCODE_EVAL: {
1687 get_one(ctx, &slot_1);
1688 g(gen_eval(ctx, slot_1));
1689 continue;
1690 }
1691 case OPCODE_ESCAPE_NONFLAT: {
1692 frame_t n, i;
1693 frame_t *vars;
1694
1695 get_one(ctx, &n);
1696 vars = mem_alloc_array_mayfail(mem_alloc_mayfail, frame_t *, 0, 0, n, sizeof(frame_t), &ctx->err);
1697 if (unlikely(!vars))
1698 return false;
1699 for (i = 0; i < n; i++) {
1700 get_one(ctx, &vars[i]);
1701 }
1702
1703 escape_label = alloc_escape_label(ctx);
1704 if (unlikely(!escape_label)) {
1705 mem_free(vars);
1706 return false;
1707 }
1708
1709 if (unlikely(!gen_test_variables(ctx, vars, n, escape_label))) {
1710 mem_free(vars);
1711 return false;
1712 }
1713 mem_free(vars);
1714
1715 continue;
1716 }
1717 case OPCODE_CHECKPOINT: {
1718 frame_t n_vars;
1719
1720 g(clear_flag_cache(ctx));
1721
1722 if (SIZEOF_IP_T == 2) {
1723 slot_1 = get_code(ctx);
1724 } else if (SIZEOF_IP_T == 4) {
1725 slot_1 = get_uint32(ctx);
1726 } else {
1727 not_reached();
1728 continue;
1729 }
1730
1731 if (unlikely(!(slot_1 + 1)))
1732 return false;
1733 while (slot_1 >= ctx->n_entries) {
1734 void *err_entries;
1735 struct cg_entry e;
1736 if (unlikely(!ctx->entries)) {
1737 if (unlikely(!array_init_mayfail(struct cg_entry, &ctx->entries, &ctx->n_entries, &ctx->err)))
1738 return false;
1739 }
1740 memset(&e, 0, sizeof(struct cg_entry));
1741 if (unlikely(!array_add_mayfail(struct cg_entry, &ctx->entries, &ctx->n_entries, e, &err_entries, &ctx->err))) {
1742 ctx->entries = err_entries;
1743 return false;
1744 }
1745 }
1746
1747 get_one(ctx, &n_vars);
1748
1749 escape_label = 0; /* avoid warning */
1750 if (likely(slot_1 != 0)) {
1751 escape_label = alloc_escape_label(ctx);
1752 if (unlikely(!escape_label))
1753 return false;
1754 }
1755
1756 if (n_vars || !slot_1) {
1757 frame_t i;
1758 uint32_t entry_label, nonflat_label;
1759 struct cg_entry *ce = &ctx->entries[slot_1];
1760
1761 if (unlikely(!array_init_mayfail(frame_t, &ce->variables, &ce->n_variables, &ctx->err)))
1762 return false;
1763 for (i = 0; i < n_vars; i++) {
1764 frame_t v;
1765 get_one(ctx, &v);
1766 if (unlikely(!array_add_mayfail(frame_t, &ce->variables, &ce->n_variables, v, NULL, &ctx->err)))
1767 return false;
1768 }
1769 if (!slot_1) {
1770 g(gen_test_variables(ctx, ce->variables, ce->n_variables, ctx->escape_nospill_label));
1771 }
1772 entry_label = alloc_label(ctx);
1773 if (unlikely(!entry_label))
1774 return false;
1775 gen_label(entry_label);
1776 ce->entry_label = entry_label;
1777
1778 nonflat_label = alloc_escape_label_for_ip(ctx, ctx->current_position);
1779 if (unlikely(!nonflat_label))
1780 return false;
1781 ce->nonflat_label = nonflat_label;
1782
1783 if (unlikely(!slot_1))
1784 g(gen_timestamp_test(ctx, ctx->escape_nospill_label));
1785 else
1786 g(gen_timestamp_test(ctx, escape_label));
1787 } else {
1788 g(gen_timestamp_test(ctx, escape_label));
1789
1790 gen_insn(INSN_ENTRY, 0, 0, 0);
1791 gen_four(slot_1);
1792 }
1793 continue;
1794 }
1795 case OPCODE_JMP: {
1796 int32_t x = get_jump_offset(ctx);
1797 g(gen_jump(ctx, x, OP_SIZE_NATIVE, COND_ALWAYS, -1U, -1U));
1798 continue;
1799 }
1800 case OPCODE_JMP_BACK_16: {
1801 int32_t x = get_code(ctx);
1802 g(gen_jump(ctx, -x - (int)(2 * sizeof(code_t)), OP_SIZE_NATIVE, COND_ALWAYS, -1U, -1U));
1803 continue;
1804 }
1805 case OPCODE_JMP_FALSE: {
1806 int32_t offs_false;
1807 get_one(ctx, &slot_1);
1808 offs_false = get_jump_offset(ctx);
1809 get_jump_offset(ctx);
1810 escape_label = alloc_escape_label(ctx);
1811 if (unlikely(!escape_label))
1812 return false;
1813 g(gen_test_1_cached(ctx, slot_1, escape_label));
1814 flag_set(ctx, slot_1, false);
1815 g(gen_cond_jump(ctx, slot_1, offs_false));
1816 continue;
1817 }
1818 case OPCODE_LABEL: {
1819 g(clear_flag_cache(ctx));
1820 continue;
1821 }
1822 #define init_args \
1823 do { \
1824 if (ctx->args != NULL) \
1825 mem_free(ctx->args); \
1826 g(array_init_mayfail(struct code_arg, &ctx->args, &ctx->args_l, &ctx->err));\
1827 } while (0)
1828 #define load_args \
1829 do { \
1830 init_args; \
1831 for (i_arg = 0; i_arg < n_args; i_arg++) { \
1832 struct code_arg a; \
1833 get_two(ctx, &a.slot, &a.flags); \
1834 a.type = 0; \
1835 g(array_add_mayfail(struct code_arg, &ctx->args, &ctx->args_l, a, NULL, &ctx->err));\
1836 } \
1837 } while (0)
1838 case OPCODE_LOAD_FN:
1839 get_two(ctx, &n_args, &slot_r);
1840 get_one(ctx, &fn_idx);
1841 load_args;
1842 g(gen_load_fn_or_curry(ctx, fn_idx, NO_FRAME_T, slot_r, 0));
1843 continue;
1844 case OPCODE_CURRY:
1845 get_two(ctx, &n_args, &slot_r);
1846 get_two(ctx, &slot_1, &flags);
1847 load_args;
1848 g(gen_load_fn_or_curry(ctx, NO_FRAME_T, slot_1, slot_r, flags));
1849 continue;
1850 case OPCODE_CALL:
1851 case OPCODE_CALL_STRICT:
1852 case OPCODE_CALL_SPARK:
1853 case OPCODE_CALL_LAZY:
1854 case OPCODE_CALL_CACHE:
1855 case OPCODE_CALL_SAVE: {
1856 get_two(ctx, &n_args, &n_ret);
1857 get_one(ctx, &fn_idx);
1858 jump_over_arguments_and_return:
1859 load_args;
1860 ctx->return_values = ctx->current_position;
1861 for (i_arg = 0; i_arg < n_ret; i_arg++) {
1862 #if ARG_MODE_N >= 3
1863 get_uint32(ctx);
1864 #else
1865 get_code(ctx);
1866 #endif
1867 get_code(ctx);
1868 }
1869 if (unlikely(profiling))
1870 goto unconditional_escape;
1871 if (code == OPCODE_CALL || code == OPCODE_CALL_STRICT) {
1872 g(gen_call(ctx, code, fn_idx));
1873 continue;
1874 }
1875 /*if (code == OPCODE_CALL_INDIRECT || code == OPCODE_CALL_INDIRECT_STRICT) {
1876 if (unlikely(!gen_call_indirect(ctx, code, slot_1, flags)))
1877 return false;
1878 continue;
1879 }*/
1880 goto unconditional_escape;
1881 }
1882 case OPCODE_CALL_INDIRECT:
1883 case OPCODE_CALL_INDIRECT_STRICT:
1884 case OPCODE_CALL_INDIRECT_SPARK:
1885 case OPCODE_CALL_INDIRECT_LAZY:
1886 case OPCODE_CALL_INDIRECT_CACHE:
1887 case OPCODE_CALL_INDIRECT_SAVE: {
1888 fn_idx = 0; /* avoid warning */
1889 get_two(ctx, &n_args, &n_ret);
1890 get_two(ctx, &slot_1, &flags);
1891 goto jump_over_arguments_and_return;
1892 }
1893 case OPCODE_RETURN: {
1894 n_args = da(ctx->fn,function)->n_return_values;
1895 load_args;
1896 if (unlikely(profiling))
1897 goto unconditional_escape;
1898 g(gen_return(ctx));
1899 continue;
1900 }
1901 case OPCODE_STRUCTURED: {
1902 init_args;
1903 get_two(ctx, &slot_1, &slot_2);
1904 do {
1905 struct code_arg a;
1906 get_two(ctx, &flags, &slot_r);
1907 get_one(ctx, &opt);
1908 a.slot = slot_r;
1909 a.flags = flags;
1910 a.type = opt;
1911 g(array_add_mayfail(struct code_arg, &ctx->args, &ctx->args_l, a, NULL, &ctx->err));
1912 } while (!(flags & OPCODE_STRUCTURED_FLAG_END));
1913 g(gen_structured(ctx, slot_1, slot_2));
1914 continue;
1915 }
1916 case OPCODE_RECORD_CREATE: {
1917 init_args;
1918 get_two(ctx, &slot_r, &n_args);
1919 for (i_arg = 0; i_arg < n_args; i_arg++) {
1920 struct code_arg a;
1921 get_two(ctx, &slot_1, &flags);
1922 a.slot = slot_1;
1923 a.flags = flags;
1924 a.type = 0;
1925 g(array_add_mayfail(struct code_arg, &ctx->args, &ctx->args_l, a, NULL, &ctx->err));
1926 }
1927 g(gen_record_create(ctx, slot_r));
1928 continue;
1929 }
1930 case OPCODE_RECORD_LOAD: {
1931 get_two(ctx, &slot_1, &opt);
1932 get_two(ctx, &slot_r, &flags);
1933 g(gen_record_load(ctx, slot_1, slot_r, opt, flags));
1934 continue;
1935 }
1936 case OPCODE_OPTION_CREATE_EMPTY_FLAT: {
1937 get_two(ctx, &slot_r, &opt);
1938 g(gen_option_create_empty_flat(ctx, opt, slot_r));
1939 continue;
1940 }
1941 case OPCODE_OPTION_CREATE_EMPTY: {
1942 get_two(ctx, &slot_r, &opt);
1943 g(gen_option_create_empty(ctx, opt, slot_r));
1944 continue;
1945 }
1946 case OPCODE_OPTION_CREATE: {
1947 get_two(ctx, &slot_r, &opt);
1948 get_two(ctx, &slot_1, &flags);
1949 g(gen_option_create(ctx, opt, slot_1, slot_r, flags));
1950 continue;
1951 }
1952 case OPCODE_OPTION_LOAD: {
1953 get_two(ctx, &slot_1, &opt);
1954 get_two(ctx, &slot_r, &flags);
1955 g(gen_option_load(ctx, slot_1, slot_r, opt, flags));
1956 continue;
1957 }
1958 case OPCODE_OPTION_TEST_FLAT: {
1959 get_two(ctx, &slot_1, &opt);
1960 get_one(ctx, &slot_r);
1961 g(gen_option_test_flat(ctx, slot_1, opt, slot_r));
1962 continue;
1963 }
1964 case OPCODE_OPTION_TEST: {
1965 get_two(ctx, &slot_1, &opt);
1966 get_one(ctx, &slot_r);
1967 g(gen_option_test(ctx, slot_1, opt, slot_r));
1968 continue;
1969 }
1970 case OPCODE_OPTION_ORD_FLAT: {
1971 get_two(ctx, &slot_1, &slot_r);
1972 g(gen_option_ord(ctx, slot_1, slot_r, true));
1973 continue;
1974 }
1975 case OPCODE_OPTION_ORD: {
1976 get_two(ctx, &slot_1, &slot_r);
1977 g(gen_option_ord(ctx, slot_1, slot_r, false));
1978 continue;
1979 }
1980 case OPCODE_ARRAY_CREATE: {
1981 init_args;
1982 get_two(ctx, &slot_r, &n_args);
1983 for (i_arg = 0; i_arg < n_args; i_arg++) {
1984 struct code_arg a;
1985 get_two(ctx, &slot_1, &flags);
1986 a.slot = slot_1;
1987 a.flags = flags;
1988 a.type = 0;
1989 g(array_add_mayfail(struct code_arg, &ctx->args, &ctx->args_l, a, NULL, &ctx->err));
1990 }
1991 g(gen_array_create(ctx, slot_r));
1992 continue;
1993 }
1994 case OPCODE_ARRAY_CREATE_EMPTY_FLAT: {
1995 get_two(ctx, &slot_r, &flags);
1996 g(gen_array_create_empty_flat(ctx, slot_r, flags));
1997 continue;
1998 }
1999 case OPCODE_ARRAY_CREATE_EMPTY: {
2000 get_one(ctx, &slot_r);
2001 g(gen_array_create_empty(ctx, slot_r));
2002 continue;
2003 }
2004 case OPCODE_ARRAY_FILL: {
2005 get_two(ctx, &slot_1, &flags);
2006 get_two(ctx, &slot_2, &slot_r);
2007 g(gen_array_fill(ctx, slot_1, flags, slot_2, slot_r));
2008 continue;
2009 }
2010 case OPCODE_ARRAY_STRING: {
2011 frame_t i;
2012 get_two(ctx, &slot_r, &i);
2013 g(gen_array_string(ctx, type_get_fixed(0, true)->tag, cast_ptr(uint8_t *, ctx->current_position), i, slot_r));
2014 ctx->current_position += (i + 1) >> 1;
2015 continue;
2016 }
2017 case OPCODE_ARRAY_UNICODE: {
2018 frame_t i;
2019 get_two(ctx, &slot_r, &i);
2020 g(gen_array_string(ctx, type_get_int(2)->tag, cast_ptr(uint8_t *, ctx->current_position), i, slot_r));
2021 ctx->current_position += i * 2;
2022 continue;
2023 }
2024 case OPCODE_ARRAY_LOAD: {
2025 get_two(ctx, &slot_1, &slot_2);
2026 get_two(ctx, &slot_r, &flags);
2027 g(gen_array_load(ctx, slot_1, slot_2, slot_r, flags));
2028 continue;
2029 }
2030 case OPCODE_ARRAY_LEN: {
2031 get_two(ctx, &slot_1, &slot_r);
2032 get_one(ctx, &flags);
2033 g(gen_array_len(ctx, slot_1, NO_FRAME_T, slot_r, false, 0));
2034 continue;
2035 }
2036 case OPCODE_ARRAY_LEN_GREATER_THAN: {
2037 get_two(ctx, &slot_1, &slot_2);
2038 get_two(ctx, &slot_r, &flags);
2039 escape_label = alloc_escape_label(ctx);
2040 if (unlikely(!escape_label))
2041 return false;
2042 if (flags & OPCODE_FLAG_FUSED) {
2043 g(gen_fused_binary(ctx, MODE_ARRAY_LEN_GT, 0, 0, escape_label, slot_1, slot_2, slot_r, &failed));
2044 if (unlikely(!failed))
2045 continue;
2046 }
2047 g(gen_array_len(ctx, slot_1, slot_2, slot_r, false, 0));
2048 continue;
2049 }
2050 case OPCODE_ARRAY_SUB: {
2051 get_two(ctx, &slot_1, &slot_2);
2052 get_two(ctx, &slot_3, &slot_r);
2053 get_one(ctx, &flags);
2054 g(gen_array_sub(ctx, slot_1, slot_2, slot_3, slot_r, flags));
2055 continue;
2056 }
2057 case OPCODE_ARRAY_SKIP: {
2058 get_two(ctx, &slot_1, &slot_2);
2059 get_two(ctx, &slot_r, &flags);
2060 g(gen_array_skip(ctx, slot_1, slot_2, slot_r, flags));
2061 continue;
2062 }
2063 case OPCODE_ARRAY_APPEND: {
2064 get_two(ctx, &slot_r, &flags);
2065 get_two(ctx, &slot_1, &slot_2);
2066 g(gen_array_append(ctx, slot_1, slot_2, slot_r, flags));
2067 continue;
2068 }
2069 case OPCODE_ARRAY_APPEND_ONE_FLAT: {
2070 get_two(ctx, &slot_r, &flags);
2071 get_two(ctx, &slot_1, &slot_2);
2072 g(gen_array_append_one_flat(ctx, slot_1, slot_2, slot_r, flags));
2073 continue;
2074 }
2075 case OPCODE_ARRAY_APPEND_ONE: {
2076 get_two(ctx, &slot_r, &flags);
2077 get_two(ctx, &slot_1, &slot_2);
2078 g(gen_array_append_one(ctx, slot_1, slot_2, slot_r, flags));
2079 continue;
2080 }
2081 case OPCODE_ARRAY_FLATTEN: {
2082 get_two(ctx, &slot_r, &flags);
2083 get_one(ctx, &slot_1);
2084 goto unconditional_escape;
2085 }
2086 case OPCODE_IO: {
2087 get_two(ctx, &flags, &slot_1);
2088 get_two(ctx, &slot_2, &slot_3);
2089 g(gen_io(ctx, flags, slot_1, slot_2, slot_3));
2090 continue;
2091 }
2092 case OPCODE_INTERNAL_FUNCTION:
2093 case OPCODE_EXIT_THREAD:
2094 case OPCODE_UNREACHABLE: {
2095 goto unconditional_escape;
2096 }
2097 default: {
2098 #if 1
2099 /*if (getenv("DUMP") && !strcmp(da(ctx->fn,function)->function_name, getenv("DUMP")))*/
2100 warning("gen_function: %s: unknown opcode %04x, %s", da(ctx->fn,function)->function_name, *ctx->instr_start, decode_opcode(*ctx->instr_start, false));
2101 #endif
2102 return false;
2103 }
2104 }
2105 }
2106
2107 return true;
2108 }
2109
2110 static bool attr_w gen_entries(struct codegen_context *ctx)
2111 {
2112 size_t i;
2113 for (i = 0; i < ctx->n_entries; i++) {
2114 struct cg_entry *ce = &ctx->entries[i];
2115 if (ce->entry_label) {
2116 gen_insn(INSN_ENTRY, 0, 0, 0);
2117 gen_four(i);
2118
2119 g(gen_test_variables(ctx, ce->variables, ce->n_variables, ce->nonflat_label));
2120
2121 gen_insn(INSN_JMP, 0, 0, 0);
2122 gen_four(ce->entry_label);
2123 }
2124 }
2125 return true;
2126 }
2127
2128 static bool attr_w gen_epilogues(struct codegen_context *ctx)
2129 {
2130 frame_t v;
2131 ip_t ip;
2132 uint32_t escape_label, nospill_label;
2133 escape_label = alloc_label(ctx);
2134 if (unlikely(!escape_label))
2135 return false;
2136 nospill_label = alloc_label(ctx);
2137 if (unlikely(!nospill_label))
2138 return false;
2139 #if defined(ARCH_PARISC)
2140 if (ctx->call_label) {
2141 gen_label(ctx->call_label);
2142 g(gen_call_millicode(ctx));
2143 }
2144 #endif
2145 if (ctx->reload_label) {
2146 gen_label(ctx->reload_label);
2147 g(gen_mov(ctx, i_size(OP_SIZE_ADDRESS), R_FRAME, R_RET0));
2148 g(gen_escape_arg(ctx, (ip_t)-1, nospill_label));
2149 }
2150 gen_label(ctx->escape_nospill_label);
2151 g(gen_escape_arg(ctx, 0, nospill_label));
2152 for (ip = 0; ip < da(ctx->fn,function)->code_size; ip++) {
2153 struct cg_exit *ce = ctx->code_exits[ip];
2154 if (ce && (ce->undo_label || ce->escape_label)) {
2155 if (ce->undo_label) {
2156 size_t i;
2157 gen_label(ce->undo_label);
2158 gen_insn(ce->undo_opcode, ce->undo_op_size, ce->undo_aux, ce->undo_writes_flags);
2159 for (i = 0; i < ce->undo_parameters_len; i++)
2160 gen_one(ce->undo_parameters[i]);
2161 }
2162 if (ce->escape_label) {
2163 gen_label(ce->escape_label);
2164 }
2165 g(gen_escape_arg(ctx, ip, escape_label));
2166 }
2167 }
2168 gen_label(escape_label);
2169 for (v = MIN_USEABLE_SLOT; v < function_n_variables(ctx->fn); v++) {
2170 if (slot_is_register(ctx, v)) {
2171 g(spill(ctx, v));
2172 }
2173 }
2174 gen_label(nospill_label);
2175 g(gen_escape(ctx));
2176 return true;
2177 }
2178
2179 static bool attr_w cgen_entry(struct codegen_context *ctx)
2180 {
2181 uint32_t entry_id = cget_four(ctx);
2182 ajla_assert_lo(entry_id < ctx->n_entries, (file_line, "cgen_entry: invalid entry %lx", (unsigned long)entry_id));
2183 ctx->entries[entry_id].entry_to_pos = ctx->mcode_size;
2184 return true;
2185 }
2186
2187 static bool attr_w cgen_label(struct codegen_context *ctx)
2188 {
2189 uint32_t label_id = cget_four(ctx);
2190 ctx->label_to_pos[label_id] = ctx->mcode_size;
2191 return true;
2192 }
2193
2194 static bool attr_w attr_unused cgen_trap(struct codegen_context *ctx, uint32_t label)
2195 {
2196 struct trap_record tr;
2197 tr.source_ip = ctx->mcode_size;
2198 tr.destination_ip = label;
2199 if (unlikely(!array_add_mayfail(struct trap_record, &ctx->trap_records, &ctx->trap_records_size, tr, NULL, &ctx->err)))
2200 return false;
2201 return true;
2202 }
2203
2204 static bool attr_w add_relocation(struct codegen_context *ctx, unsigned length, int offset, bool *known)
2205 {
2206 struct relocation rel;
2207 rel.label_id = cget_four(ctx);
2208 rel.length = length;
2209 rel.position = ctx->mcode_size;
2210 rel.jmp_instr = ctx->code_position - 8 - offset - ctx->code;
2211 if (unlikely(!array_add_mayfail(struct relocation, &ctx->reloc, &ctx->reloc_size, rel, NULL, &ctx->err)))
2212 return false;
2213 if (known)
2214 *known = ctx->label_to_pos[rel.label_id] != (size_t)-1;
2215 return true;
2216 }
2217
2218
2219 #if defined(ARCH_ALPHA)
2220 #include "c2-alpha.inc"
2221 #elif defined(ARCH_ARM32)
2222 #include "c2-arm.inc"
2223 #elif defined(ARCH_ARM64)
2224 #include "c2-arm64.inc"
2225 #elif defined(ARCH_IA64)
2226 #include "c2-ia64.inc"
2227 #elif defined(ARCH_LOONGARCH64)
2228 #include "c2-loong.inc"
2229 #elif defined(ARCH_MIPS)
2230 #include "c2-mips.inc"
2231 #elif defined(ARCH_PARISC)
2232 #include "c2-hppa.inc"
2233 #elif defined(ARCH_POWER)
2234 #include "c2-power.inc"
2235 #elif defined(ARCH_S390)
2236 #include "c2-s390.inc"
2237 #elif defined(ARCH_SPARC)
2238 #include "c2-sparc.inc"
2239 #elif defined(ARCH_RISCV64)
2240 #include "c2-riscv.inc"
2241 #elif defined(ARCH_X86)
2242 #include "c2-x86.inc"
2243 #endif
2244
2245
2246 static bool attr_w gen_mcode(struct codegen_context *ctx)
2247 {
2248 ctx->code_position = ctx->code;
2249
2250 while (ctx->code_position != ctx->code + ctx->code_size) {
2251 uint32_t insn;
2252 ajla_assert_lo(ctx->code_position < ctx->code + ctx->code_size, (file_line, "gen_mcode: ran out of code"));
2253 #ifdef DEBUG_INSNS
2254 insn = cget_four(ctx);
2255 debug("line: %u/%u", insn >> 24, insn & 0x00FFFFFFU);
2256 #endif
2257 insn = cget_four(ctx);
2258 g(cgen_insn(ctx, insn));
2259 }
2260
2261 return true;
2262 }
2263
2264 #define RELOCS_RETRY -1
2265 #define RELOCS_FAIL 0
2266 #define RELOCS_OK 1
2267
2268 static int8_t resolve_relocs(struct codegen_context *ctx)
2269 {
2270 size_t i;
2271 int8_t status = RELOCS_OK;
2272 for (i = 0; i < ctx->reloc_size; i++) {
2273 struct relocation *reloc = &ctx->reloc[i];
2274 if (!resolve_relocation(ctx, reloc)) {
2275 uint8_t *jmp_instr;
2276 uint32_t insn;
2277 uint32_t new_length;
2278 status = RELOCS_RETRY;
2279 if (unlikely(reloc->length + zero >= JMP_LIMIT))
2280 return RELOCS_FAIL;
2281 new_length = reloc->length + 1;
2282 jmp_instr = ctx->code + reloc->jmp_instr;
2283 insn = (uint32_t)jmp_instr[0] +
2284 ((uint32_t)jmp_instr[1] << 8) +
2285 ((uint32_t)jmp_instr[2] << 16) +
2286 ((uint32_t)jmp_instr[3] << 24);
2287 insn &= ~INSN_JUMP_SIZE;
2288 insn |= (uint32_t)new_length << INSN_JUMP_SIZE_SHIFT;
2289 jmp_instr[0] = insn;
2290 jmp_instr[1] = insn >> 8;
2291 jmp_instr[2] = insn >> 16;
2292 jmp_instr[3] = insn >> 24;
2293 }
2294 }
2295 return status;
2296 }
2297
2298 static void resolve_traps(struct codegen_context *ctx)
2299 {
2300 size_t i;
2301 for (i = 0; i < ctx->trap_records_size; i++) {
2302 struct trap_record *tr = &ctx->trap_records[i];
2303 tr->destination_ip = ctx->label_to_pos[tr->destination_ip];
2304 }
2305 }
2306
2307
2308 static bool attr_w codegen_map(struct codegen_context *ctx)
2309 {
2310 void *ptr;
2311 frame_t i;
2312 array_finish(uint8_t, &ctx->mcode, &ctx->mcode_size);
2313 ptr = os_code_map(ctx->mcode, ctx->mcode_size, &ctx->err);
2314 ctx->mcode = NULL;
2315 if (unlikely(!ptr)) {
2316 return false;
2317 }
2318 for (i = 0; i < ctx->n_entries; i++) {
2319 char *entry = cast_ptr(char *, ptr) + ctx->entries[i].entry_to_pos;
2320 da(ctx->codegen,codegen)->unoptimized_code[i] = entry;
2321 da(ctx->codegen,codegen)->n_entries++;
2322 }
2323 da(ctx->codegen,codegen)->unoptimized_code_base = ptr;
2324 da(ctx->codegen,codegen)->unoptimized_code_size = ctx->mcode_size;
2325
2326 return true;
2327 }
2328
2329
2330 void *codegen_fn(frame_s *fp, const code_t *ip, union internal_arg ia[])
2331 {
2332 struct codegen_context ctx_;
2333 struct codegen_context *ctx = &ctx_;
2334 frame_t i;
2335 int8_t rr;
2336 struct data *codegen;
2337 uint32_t l;
2338
2339 init_ctx(ctx);
2340 ctx->fn = ia[0].ptr;
2341
2342 #ifdef DEBUG_ENV
2343 if (getenv("CG") && strcmp(da(ctx->fn,function)->function_name, getenv("CG")))
2344 goto fail;
2345 #endif
2346
2347 ctx->local_directory = mem_alloc_array_mayfail(mem_calloc_mayfail, struct data **, 0, 0, da(ctx->fn,function)->local_directory_size, sizeof(struct data *), &ctx->err);
2348 if (unlikely(!ctx->local_directory))
2349 goto fail;
2350
2351 if (0) for (i = 0; i < da(ctx->fn,function)->local_directory_size; i++) {
2352 struct data *callee;
2353 pointer_t *ptr;
2354 ptr = da(ctx->fn,function)->local_directory[i];
2355 pointer_follow(ptr, false, callee, PF_SPARK, NULL, 0,
2356 SUBMIT_EX(ex_);
2357 goto next_one,
2358 goto next_one;
2359 );
2360 ctx->local_directory[i] = callee;
2361 next_one:;
2362 }
2363 for (i = 0; i < da(ctx->fn,function)->local_directory_size; i++) {
2364 struct data *callee;
2365 pointer_t *ptr;
2366 if (ctx->local_directory[i])
2367 continue;
2368 ptr = da(ctx->fn,function)->local_directory[i];
2369 pointer_follow(ptr, false, callee, PF_WAIT, fp, ip,
2370 done_ctx(ctx);
2371 return ex_,
2372 goto fail
2373 );
2374 ctx->local_directory[i] = callee;
2375 /*debug("processing call: %s -> %s", da(ctx->fn,function)->function_name, da(callee,function)->function_name);*/
2376 }
2377
2378 if (da(ctx->fn,function)->module_designator) {
2379 struct function_descriptor *sfd = save_find_function_descriptor(da(ctx->fn,function)->module_designator, da(ctx->fn,function)->function_designator);
2380 if (sfd && sfd->unoptimized_code_size) {
2381 codegen = data_alloc_flexible(codegen, unoptimized_code, sfd->n_entries, &ctx->err);
2382 if (unlikely(!codegen))
2383 goto fail;
2384 da(codegen,codegen)->unoptimized_code_base = sfd->unoptimized_code_base;
2385 da(codegen,codegen)->unoptimized_code_size = sfd->unoptimized_code_size;
2386 da(codegen,codegen)->function = ctx->fn;
2387 da(codegen,codegen)->is_saved = true;
2388 da(codegen,codegen)->n_entries = sfd->n_entries;
2389 da(codegen,codegen)->offsets = NULL;
2390 for (i = 0; i < sfd->n_entries; i++) {
2391 da(codegen,codegen)->unoptimized_code[i] = cast_ptr(char *, da(codegen,codegen)->unoptimized_code_base) + sfd->entries[i];
2392 /*debug("%s: %p + %lx -> %p", da(ctx->fn,function)->function_name, da(codegen,codegen)->unoptimized_code_base, sfd->entries[i], da(codegen,codegen)->unoptimized_code[i]);*/
2393 }
2394 #ifdef HAVE_CODEGEN_TRAPS
2395 da(codegen,codegen)->trap_records = sfd->trap_records;
2396 da(codegen,codegen)->trap_records_size = sfd->trap_records_size;
2397 data_trap_insert(codegen);
2398 #endif
2399 goto have_codegen;
2400 }
2401 }
2402
2403 /*debug("trying: %s", da(ctx->fn,function)->function_name);*/
2404 if (unlikely(!array_init_mayfail(uint8_t, &ctx->code, &ctx->code_size, &ctx->err)))
2405 goto fail;
2406
2407 ctx->code_labels = mem_alloc_array_mayfail(mem_calloc_mayfail, uint32_t *, 0, 0, da(ctx->fn,function)->code_size, sizeof(uint32_t), &ctx->err);
2408 if (unlikely(!ctx->code_labels))
2409 goto fail;
2410
2411 ctx->code_exits = mem_alloc_array_mayfail(mem_calloc_mayfail, struct cg_exit **, 0, 0, da(ctx->fn,function)->code_size, sizeof(struct cg_exit *), &ctx->err);
2412 if (unlikely(!ctx->code_exits))
2413 goto fail;
2414
2415 ctx->flag_cache = mem_alloc_array_mayfail(mem_calloc_mayfail, uint8_t *, 0, 0, function_n_variables(ctx->fn), sizeof(int8_t), &ctx->err);
2416 if (unlikely(!ctx->flag_cache))
2417 goto fail;
2418
2419 ctx->registers = mem_alloc_array_mayfail(mem_alloc_mayfail, short *, 0, 0, function_n_variables(ctx->fn), sizeof(short), &ctx->err);
2420 if (unlikely(!ctx->registers))
2421 goto fail;
2422
2423 if (unlikely(!array_init_mayfail(frame_t, &ctx->need_spill, &ctx->need_spill_l, &ctx->err)))
2424 goto fail;
2425
2426 if (unlikely(!gen_registers(ctx)))
2427 goto fail;
2428
2429 if (unlikely(!gen_function(ctx)))
2430 goto fail;
2431
2432 if (unlikely(!gen_entries(ctx)))
2433 goto fail;
2434
2435 if (unlikely(!gen_epilogues(ctx)))
2436 goto fail;
2437
2438 if (unlikely(!(ctx->label_id + 1)))
2439 goto fail;
2440 if (unlikely(!(ctx->label_to_pos = mem_alloc_array_mayfail(mem_alloc_mayfail, size_t *, 0, 0, ctx->label_id + 1, sizeof(size_t), &ctx->err))))
2441 goto fail;
2442
2443 again:
2444 for (l = 0; l < ctx->label_id + 1; l++)
2445 ctx->label_to_pos[l] = (size_t)-1;
2446
2447 if (unlikely(!array_init_mayfail(uint8_t, &ctx->mcode, &ctx->mcode_size, &ctx->err)))
2448 goto fail;
2449
2450 if (unlikely(!array_init_mayfail(struct relocation, &ctx->reloc, &ctx->reloc_size, &ctx->err)))
2451 goto fail;
2452
2453 if (unlikely(!array_init_mayfail(struct trap_record, &ctx->trap_records, &ctx->trap_records_size, &ctx->err)))
2454 goto fail;
2455
2456 #ifdef ARCH_CONTEXT
2457 init_arch_context(ctx);
2458 #endif
2459
2460 if (unlikely(!gen_mcode(ctx)))
2461 goto fail;
2462
2463 rr = resolve_relocs(ctx);
2464 if (unlikely(rr == RELOCS_FAIL)) {
2465 /*debug("relocation fail: %s", da(ctx->fn,function)->function_name);*/
2466 goto fail;
2467 }
2468 if (rr == RELOCS_RETRY) {
2469 mem_free(ctx->mcode);
2470 ctx->mcode = NULL;
2471 mem_free(ctx->reloc);
2472 ctx->reloc = NULL;
2473 mem_free(ctx->trap_records);
2474 ctx->trap_records = NULL;
2475 goto again;
2476 }
2477
2478 resolve_traps(ctx);
2479
2480 #ifdef DEBUG_ENV
2481 if ((getenv("DUMP") && !strcmp(getenv("DUMP"), da(ctx->fn,function)->function_name)) || getenv("DUMP_ALL")) {
2482 char *hex;
2483 size_t hexl;
2484 size_t i;
2485 handle_t h;
2486
2487 mutex_lock(&dump_mutex);
2488 str_init(&hex, &hexl);
2489 str_add_string(&hex, &hexl, "_");
2490 str_add_unsigned(&hex, &hexl, dump_seq++, 10);
2491 str_add_string(&hex, &hexl, "_");
2492 str_add_string(&hex, &hexl, da(ctx->fn,function)->function_name);
2493 str_add_string(&hex, &hexl, ":");
2494 for (i = 0; i < hexl; i++)
2495 if (hex[i] == '/')
2496 hex[i] = '_';
2497 for (i = 0; i < ctx->mcode_size; i++) {
2498 uint8_t a = ctx->mcode[i];
2499 if (!(i & 0xff))
2500 str_add_string(&hex, &hexl, "\n .byte 0x");
2501 else
2502 str_add_string(&hex, &hexl, ",0x");
2503 if (a < 16)
2504 str_add_char(&hex, &hexl, '0');
2505 str_add_unsigned(&hex, &hexl, a, 16);
2506 }
2507 str_add_string(&hex, &hexl, "\n");
2508 h = os_open(os_cwd, "dump.s", O_WRONLY | O_APPEND, 0600, NULL);
2509 os_write_all(h, hex, hexl, NULL);
2510 os_close(h);
2511 mem_free(hex);
2512 mutex_unlock(&dump_mutex);
2513 }
2514 #endif
2515
2516 ctx->codegen = data_alloc_flexible(codegen, unoptimized_code, ctx->n_entries, &ctx->err);
2517 if (unlikely(!ctx->codegen))
2518 goto fail;
2519 da(ctx->codegen,codegen)->function = ctx->fn;
2520 da(ctx->codegen,codegen)->is_saved = false;
2521 da(ctx->codegen,codegen)->n_entries = 0;
2522 da(ctx->codegen,codegen)->offsets = NULL;
2523
2524 if (unlikely(!codegen_map(ctx)))
2525 goto fail;
2526
2527 codegen = ctx->codegen;
2528 ctx->codegen = NULL;
2529
2530 #ifdef HAVE_CODEGEN_TRAPS
2531 da(codegen,codegen)->trap_records = ctx->trap_records;
2532 da(codegen,codegen)->trap_records_size = ctx->trap_records_size;
2533 ctx->trap_records = NULL;
2534 data_trap_insert(codegen);
2535 #endif
2536
2537 have_codegen:
2538 done_ctx(ctx);
2539 return function_return(fp, pointer_data(codegen));
2540
2541 fail:
2542 /*debug("FAILED: %s", da(ctx->fn,function)->function_name);*/
2543 done_ctx(ctx);
2544 return function_return(fp, pointer_thunk(thunk_alloc_exception_error(error_ajla(EC_SYNC, AJLA_ERROR_NOT_SUPPORTED), NULL, NULL, NULL pass_file_line)));
2545 }
2546
2547 void codegen_free(struct data *codegen)
2548 {
2549 if (unlikely(da(codegen,codegen)->offsets != NULL))
2550 mem_free(da(codegen,codegen)->offsets);
2551 if (likely(da(codegen,codegen)->is_saved))
2552 return;
2553 #ifdef HAVE_CODEGEN_TRAPS
2554 mem_free(da(codegen,codegen)->trap_records);
2555 #endif
2556 os_code_unmap(da(codegen,codegen)->unoptimized_code_base, da(codegen,codegen)->unoptimized_code_size);
2557 }
2558
2559 #if defined(ARCH_IA64)
2560 static uintptr_t ia64_stub[2];
2561 #endif
2562 #if defined(ARCH_PARISC32) && defined(ARCH_PARISC_USE_STUBS)
2563 static uintptr_t parisc_stub[2];
2564 #endif
2565 #if defined(ARCH_PARISC64) && defined(ARCH_PARISC_USE_STUBS)
2566 static uintptr_t parisc_stub[4];
2567 #endif
2568 #if defined(ARCH_POWER) && defined(AIX_CALL)
2569 static uintptr_t ppc_stub[3];
2570 #endif
2571
2572 void name(codegen_init)(void)
2573 {
2574 struct codegen_context ctx_;
2575 struct codegen_context *ctx = &ctx_;
2576 void *ptr;
2577
2578 #if (defined(ARCH_X86_64) || defined(ARCH_X86_X32)) && !defined(ARCH_X86_WIN_ABI)
2579 #if defined(HAVE_SYSCALL) && defined(HAVE_ASM_PRCTL_H) && defined(HAVE_SYS_SYSCALL_H)
2580 if (!dll) {
2581 int r;
2582 EINTR_LOOP(r, syscall(SYS_arch_prctl, ARCH_SET_GS, &cg_upcall_vector));
2583 if (!r)
2584 upcall_register = R_GS;
2585 }
2586 #elif defined(HAVE_AMD64_SET_GSBASE) && defined(HAVE_X86_SYSARCH_H)
2587 if (!dll) {
2588 int r;
2589 EINTR_LOOP(r, amd64_set_gsbase(&cg_upcall_vector));
2590 if (!r)
2591 upcall_register = R_GS;
2592 }
2593 #elif defined(HAVE_SYSARCH) && defined(HAVE_X86_SYSARCH_H) && defined(X86_64_SET_GSBASE)
2594 if (!dll) {
2595 int r;
2596 void *ptr = &cg_upcall_vector;
2597 EINTR_LOOP(r, sysarch(X86_64_SET_GSBASE, &ptr));
2598 if (!r)
2599 upcall_register = R_GS;
2600 }
2601 #endif
2602 #endif
2603
2604 init_ctx(ctx);
2605 ctx->fn = NULL;
2606
2607 array_init(uint8_t, &ctx->code, &ctx->code_size);
2608
2609 if (unlikely(!gen_entry(ctx)))
2610 goto fail;
2611
2612 array_init(uint8_t, &ctx->mcode, &ctx->mcode_size);
2613
2614 #ifdef ARCH_CONTEXT
2615 init_arch_context(ctx);
2616 #endif
2617
2618 if (unlikely(!gen_mcode(ctx)))
2619 goto fail;
2620
2621 array_finish(uint8_t, &ctx->mcode, &ctx->mcode_size);
2622 ptr = os_code_map(ctx->mcode, ctx->mcode_size, NULL);
2623 codegen_ptr = ptr;
2624 codegen_size = ctx->mcode_size;
2625 ctx->mcode = NULL;
2626 #if defined(ARCH_IA64)
2627 ia64_stub[0] = ptr_to_num(ptr);
2628 ia64_stub[1] = 0;
2629 codegen_entry = cast_ptr(codegen_type, ia64_stub);
2630 #elif defined(ARCH_PARISC32) && defined(ARCH_PARISC_USE_STUBS)
2631 parisc_stub[0] = ptr_to_num(ptr);
2632 parisc_stub[1] = 0;
2633 codegen_entry = cast_ptr(codegen_type, cast_ptr(char *, parisc_stub) + 2);
2634 #elif defined(ARCH_PARISC64) && defined(ARCH_PARISC_USE_STUBS)
2635 parisc_stub[0] = 0;
2636 parisc_stub[1] = 0;
2637 parisc_stub[2] = ptr_to_num(ptr);
2638 parisc_stub[3] = 0;
2639 codegen_entry = cast_ptr(codegen_type, parisc_stub);
2640 #elif defined(ARCH_POWER) && defined(AIX_CALL)
2641 ppc_stub[0] = ptr_to_num(ptr);
2642 ppc_stub[1] = 0;
2643 ppc_stub[2] = 0;
2644 codegen_entry = cast_ptr(codegen_type, ppc_stub);
2645 #else
2646 codegen_entry = ptr;
2647 #endif
2648 done_ctx(ctx);
2649
2650 #ifdef DEBUG_ENV
2651 mutex_init(&dump_mutex);
2652 if (getenv("DUMP") || getenv("DUMP_ALL")) {
2653 size_t i;
2654 char *hex;
2655 size_t hexl;
2656 str_init(&hex, &hexl);
2657 #if defined(ARCH_RISCV64)
2658 str_add_string(&hex, &hexl, " .attribute arch, \"rv64i2p1_m2p0_a2p1_f2p2_d2p2_c2p0_zicsr2p0_zifencei2p0_zba1p0_zbb1p0_zbc1p0_zbs1p0\"\n");
2659 #endif
2660 for (i = 0; i < codegen_size; i++) {
2661 uint8_t a = cast_ptr(uint8_t *, codegen_ptr)[i];
2662 str_add_string(&hex, &hexl, " .byte 0x");
2663 if (a < 16)
2664 str_add_char(&hex, &hexl, '0');
2665 str_add_unsigned(&hex, &hexl, a, 16);
2666 str_add_char(&hex, &hexl, '\n');
2667 }
2668 os_write_atomic(".", "dump.s", hex, hexl, NULL);
2669 mem_free(hex);
2670 }
2671 #endif
2672
2673 return;
2674
2675 fail:
2676 fatal("couldn't compile global entry");
2677 }
2678
2679 void name(codegen_done)(void)
2680 {
2681 os_code_unmap(codegen_ptr, codegen_size);
2682 #ifdef DEBUG_ENV
2683 mutex_done(&dump_mutex);
2684 #endif
2685 }
2686
2687 #else
2688
2689 void name(codegen_init)(void)
2690 {
2691 }
2692
2693 void name(codegen_done)(void)
2694 {
2695 }
2696
2697 #endif
2698
2699 #endif
Ajla - a purely functional programming language