?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
x86: generate 2-byte lea
[ajla.git] / codegen.c
blob5b0447d2d62c860809957f06331b7fb5f405c65e
1 /*
2 * Copyright (C) 2024 Mikulas Patocka
3 *
4 * This file is part of Ajla.
5 *
6 * Ajla is free software: you can redistribute it and/or modify it under the
7 * terms of the GNU General Public License as published by the Free Software
8 * Foundation, either version 3 of the License, or (at your option) any later
9 * version.
10 *
11 * Ajla is distributed in the hope that it will be useful, but WITHOUT ANY
12 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
13 * A PARTICULAR PURPOSE. See the GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License along with
16 * Ajla. If not, see <https://www.gnu.org/licenses/>.
17 */
18
19 #include "ajla.h"
20
21 #ifndef FILE_OMIT
22
23 #include "data.h"
24 #include "os.h"
25 #include "os_util.h"
26 #include "util.h"
27 #include "ipfn.h"
28 #include "ipret.h"
29 #include "funct.h"
30 #include "thread.h"
31 #include "task.h"
32 #include "save.h"
33
34 #include "codegen.h"
35
36 #ifdef HAVE_CODEGEN
37
38 #define flag_cache_chicken 0
39 #define must_be_flat_chicken 0
40 #define ra_chicken 0
41
42 #define INLINE_BITMAP_SLOTS 16
43 #define INLINE_COPY_SIZE 64
44
45 /*#define DEBUG_INSNS*/
46 /*#define DEBUG_GARBAGE*/
47
48 #if (defined(ARCH_X86_64) || defined(ARCH_X86_X32)) && !defined(ARCH_X86_WIN_ABI)
49 #if defined(HAVE_SYSCALL) && defined(HAVE_ASM_PRCTL_H) && defined(HAVE_SYS_SYSCALL_H)
50 #include <asm/prctl.h>
51 #include <sys/syscall.h>
52 #include <unistd.h>
53 #endif
54 #if (defined(HAVE_AMD64_SET_GSBASE) || defined(HAVE_SYSARCH)) && defined(HAVE_X86_SYSARCH_H)
55 #include <x86/sysarch.h>
56 #endif
57 #endif
58
59 code_return_t (*codegen_entry)(frame_s *, struct cg_upcall_vector_s *, tick_stamp_t, void *);
60 static void *codegen_ptr;
61 static size_t codegen_size;
62
63 #ifdef DEBUG_ENV
64 static mutex_t dump_mutex;
65 static uint64_t dump_seq = 0;
66 #endif
67
68 /*
69 * insn:
70 * opcode - 16 bits
71 * op_size - 3 bits
72 * aux - 7 bits
73 * writes flags - 2 bit
74 * jmp size - 2 bits
75 */
76
77 #define INSN_OPCODE 0x0000ffffUL
78 #define INSN_OP_SIZE 0x00070000UL
79 #define INSN_AUX 0x03f80000UL
80 #define INSN_WRITES_FLAGS 0x0c000000UL
81 #define INSN_JUMP_SIZE 0x30000000UL
82
83 #define INSN_OPCODE_SHIFT 0
84 #define INSN_OP_SIZE_SHIFT 16
85 #define INSN_AUX_SHIFT 19
86 #define INSN_WRITES_FLAGS_SHIFT 26
87 #define INSN_JUMP_SIZE_SHIFT 28
88
89 #define insn_opcode(insn) (((insn) >> INSN_OPCODE_SHIFT) & (INSN_OPCODE >> INSN_OPCODE_SHIFT))
90 #define insn_op_size(insn) (((insn) >> INSN_OP_SIZE_SHIFT) & (INSN_OP_SIZE >> INSN_OP_SIZE_SHIFT))
91 #define insn_aux(insn) (((insn) >> INSN_AUX_SHIFT) & (INSN_AUX >> INSN_AUX_SHIFT))
92 #define insn_writes_flags(insn) (((insn) >> INSN_WRITES_FLAGS_SHIFT) & (INSN_WRITES_FLAGS >> INSN_WRITES_FLAGS_SHIFT))
93 #define insn_jump_size(insn) (((insn) >> INSN_JUMP_SIZE_SHIFT) & (INSN_JUMP_SIZE >> INSN_JUMP_SIZE_SHIFT))
94
95 #define ALU_ADD 0x00
96 #define ALU_OR 0x01
97 #define ALU_ADC 0x02
98 #define ALU_SBB 0x03
99 #define ALU_AND 0x04
100 #define ALU_SUB 0x05
101 #define ALU_XOR 0x06
102 #define ALU_ORN 0x08
103 #define ALU_ANDN 0x09
104 #define ALU_XORN 0x0a
105 #define ALU_MUL 0x10
106 #define ALU_UMULH 0x11
107 #define ALU_SMULH 0x12
108 #define ALU_UDIV 0x13
109 #define ALU_SDIV 0x14
110 #define ALU_UREM 0x15
111 #define ALU_SREM 0x16
112 #define ALU_SAVE 0x17
113 #define ALU_EXTBL 0x18
114 #define ALU_EXTWL 0x19
115 #define ALU_EXTLL 0x1a
116 #define ALU_EXTLH 0x1b
117 #define ALU_INSBL 0x1c
118 #define ALU_MSKBL 0x1d
119 #define ALU_ZAP 0x20
120 #define ALU_ZAPNOT 0x21
121
122 #define ALU1_NOT 0x00
123 #define ALU1_NEG 0x01
124 #define ALU1_NGC 0x02
125 #define ALU1_BSWAP 0x03
126 #define ALU1_BSWAP16 0x04
127 #define ALU1_BREV 0x05
128 #define ALU1_BSF 0x06
129 #define ALU1_BSR 0x07
130 #define ALU1_LZCNT 0x08
131 #define ALU1_POPCNT 0x09
132
133 #define FP_ALU_ADD 0
134 #define FP_ALU_SUB 1
135 #define FP_ALU_MUL 2
136 #define FP_ALU_DIV 3
137 #define FP_ALU1_NEG 0
138 #define FP_ALU1_SQRT 1
139 #define FP_ALU1_ROUND 2
140 #define FP_ALU1_FLOOR 3
141 #define FP_ALU1_CEIL 4
142 #define FP_ALU1_TRUNC 5
143 #define FP_ALU1_VCNT8 6
144 #define FP_ALU1_VPADDL 7
145 #define FP_ALU1_ADDV 8
146
147 #define COND_O 0x0
148 #define COND_NO 0x1
149 #define COND_B 0x2
150 #define COND_AE 0x3
151 #define COND_E 0x4
152 #define COND_NE 0x5
153 #define COND_BE 0x6
154 #define COND_A 0x7
155 #define COND_S 0x8
156 #define COND_NS 0x9
157 #define COND_P 0xa
158 #define COND_NP 0xb
159 #define COND_L 0xc
160 #define COND_GE 0xd
161 #define COND_LE 0xe
162 #define COND_G 0xf
163 #define COND_BLBC 0x10
164 #define COND_BLBS 0x11
165 #define COND_ALWAYS 0x12
166
167 #define COND_FP 0x20
168 #define FP_COND_P (COND_FP | COND_P)
169 #define FP_COND_NP (COND_FP | COND_NP)
170 #define FP_COND_E (COND_FP | COND_E)
171 #define FP_COND_NE (COND_FP | COND_NE)
172 #define FP_COND_A (COND_FP | COND_A)
173 #define FP_COND_BE (COND_FP | COND_BE)
174 #define FP_COND_B (COND_FP | COND_B)
175 #define FP_COND_AE (COND_FP | COND_AE)
176
177 #define ROT_ROL 0x0
178 #define ROT_ROR 0x1
179 #define ROT_RCL 0x2
180 #define ROT_RCR 0x3
181 #define ROT_SHL 0x4
182 #define ROT_SHR 0x5
183 #define ROT_SAR 0x7
184 #define ROT_SAL 0x8
185
186 #define BTX_BT 0x0
187 #define BTX_BTS 0x1
188 #define BTX_BTR 0x2
189 #define BTX_BTC 0x3
190 #define BTX_BTEXT 0x4
191
192 #define OP_SIZE_1 0
193 #define OP_SIZE_2 1
194 #define OP_SIZE_4 2
195 #define OP_SIZE_8 3
196 #define OP_SIZE_16 4
197 #define OP_SIZE_10 7
198
199 #define MOV_MASK_0_16 0x0
200 #define MOV_MASK_16_32 0x1
201 #define MOV_MASK_32_48 0x2
202 #define MOV_MASK_48_64 0x3
203 #define MOV_MASK_0_8 0x4
204 #define MOV_MASK_32_64 0x5
205 #define MOV_MASK_52_64 0x6
206
207 #define JMP_SHORTEST 0x0000
208 #define JMP_SHORT 0x0001
209 #define JMP_LONG 0x0002
210 #define JMP_EXTRA_LONG 0x0003
211
212 enum {
213 INSN_ENTRY,
214 INSN_LABEL,
215 INSN_RET,
216 INSN_RET_IMM,
217 INSN_ARM_PUSH,
218 INSN_ARM_POP,
219 INSN_S390_PUSH,
220 INSN_S390_POP,
221 INSN_IA64_ALLOC,
222 INSN_IA64_DEALLOC,
223 INSN_PUSH,
224 INSN_POP,
225 INSN_CALL,
226 INSN_CALL_INDIRECT,
227 INSN_MOV,
228 INSN_MOVSX,
229 INSN_MOV_U,
230 INSN_MOV_LR,
231 INSN_CMP,
232 INSN_CMP_DEST_REG,
233 INSN_CMN,
234 INSN_TEST,
235 INSN_TEST_DEST_REG,
236 INSN_ALU,
237 INSN_ALU_PARTIAL,
238 INSN_ALU_FLAGS,
239 INSN_ALU_FLAGS_PARTIAL,
240 INSN_ALU_TRAP,
241 INSN_ALU_FLAGS_TRAP,
242 INSN_ALU1,
243 INSN_ALU1_PARTIAL,
244 INSN_ALU1_FLAGS,
245 INSN_ALU1_FLAGS_PARTIAL,
246 INSN_ALU1_TRAP,
247 INSN_LEA3,
248 INSN_ROT,
249 INSN_ROT_PARTIAL,
250 INSN_BT,
251 INSN_BTX,
252 INSN_MUL_L,
253 INSN_DIV_L,
254 INSN_MADD,
255 INSN_CBW,
256 INSN_CBW_PARTIAL,
257 INSN_CWD,
258 INSN_CWD_PARTIAL,
259 INSN_SET_COND,
260 INSN_SET_COND_PARTIAL,
261 INSN_CMOV,
262 INSN_CMOV_XCC,
263 INSN_CMP_CMOV,
264 INSN_MOVR,
265 INSN_CSEL_SEL,
266 INSN_CSEL_INC,
267 INSN_CSEL_INV,
268 INSN_CSEL_NEG,
269 INSN_STP,
270 INSN_LDP,
271 INSN_MOV_MASK,
272 INSN_MEMCPY,
273 INSN_MEMSET,
274 INSN_FP_CMP,
275 INSN_FP_CMP_DEST_REG,
276 INSN_FP_CMP_DEST_REG_TRAP,
277 INSN_FP_CMP_UNORDERED_DEST_REG,
278 INSN_FP_CMP_COND,
279 INSN_FP_TEST_REG,
280 INSN_FP_TO_INT_FLAGS,
281 INSN_FP_ALU,
282 INSN_FP_ALU1,
283 INSN_FP_TO_INT32,
284 INSN_FP_TO_INT64,
285 INSN_FP_TO_INT64_TRAP,
286 INSN_FP_FROM_INT32,
287 INSN_FP_FROM_INT64,
288 INSN_FP_INT64_TO_INT32_TRAP,
289 INSN_FP_CVT,
290 INSN_X87_FLD,
291 INSN_X87_FILD,
292 INSN_X87_FSTP,
293 INSN_X87_FISTP,
294 INSN_X87_FISTTP,
295 INSN_X87_FCOMP,
296 INSN_X87_FCOMPP,
297 INSN_X87_FCOMIP,
298 INSN_X87_ALU,
299 INSN_X87_ALUP,
300 INSN_X87_FCHS,
301 INSN_X87_FSQRT,
302 INSN_X87_FRNDINT,
303 INSN_X87_FNSTSW,
304 INSN_X87_FLDCW,
305 INSN_JMP,
306 INSN_JMP_COND,
307 INSN_JMP_COND_LOGICAL,
308 INSN_JMP_REG,
309 INSN_JMP_REG_BIT,
310 INSN_JMP_2REGS,
311 INSN_JMP_FP_TEST,
312 INSN_JMP_INDIRECT,
313 INSN_MB,
314 INSN_CALL_MILLICODE,
315 };
316
317 #define ARG_REGS_MAX 0xc0
318 #define ARG_SHIFTED_REGISTER 0xc0
319 #define ARG_SHIFT_AMOUNT 0x3f
320 #define ARG_SHIFT_MODE 0xc0
321 #define ARG_SHIFT_LSL 0x00
322 #define ARG_SHIFT_LSR 0x40
323 #define ARG_SHIFT_ASR 0x80
324 #define ARG_SHIFT_ROR 0xc0
325 #define ARG_EXTENDED_REGISTER 0xc1
326 #define ARG_EXTEND_SHIFT 0x07
327 #define ARG_EXTEND_MODE 0x38
328 #define ARG_EXTEND_UXTB 0x00
329 #define ARG_EXTEND_UXTH 0x08
330 #define ARG_EXTEND_UXTW 0x10
331 #define ARG_EXTEND_UXTX 0x18
332 #define ARG_EXTEND_SXTB 0x20
333 #define ARG_EXTEND_SXTH 0x28
334 #define ARG_EXTEND_SXTW 0x30
335 #define ARG_EXTEND_SXTX 0x38
336 #define ARG_ADDRESS_0 0xd0
337 #define ARG_ADDRESS_1 0xd1
338 #define ARG_ADDRESS_1_2 0xd2
339 #define ARG_ADDRESS_1_4 0xd3
340 #define ARG_ADDRESS_1_8 0xd4
341 #define ARG_ADDRESS_1_PRE_I 0xd5
342 #define ARG_ADDRESS_1_POST_I 0xd6
343 #define ARG_ADDRESS_2 0xd7
344 #define ARG_ADDRESS_2_2 0xd8
345 #define ARG_ADDRESS_2_4 0xd9
346 #define ARG_ADDRESS_2_8 0xda
347 #define ARG_ADDRESS_2_UXTW 0xdb
348 #define ARG_ADDRESS_2_SXTW 0xdc
349 #define ARG_IMM 0xe0
350
351 #define ARG_IS_ADDRESS(a) ((a) >= ARG_ADDRESS_0 && (a) <= ARG_ADDRESS_2_SXTW)
352
353 #ifdef POINTER_COMPRESSION
354 #define OP_SIZE_SLOT OP_SIZE_4
355 #else
356 #define OP_SIZE_SLOT OP_SIZE_ADDRESS
357 #endif
358
359 #define OP_SIZE_BITMAP (bitmap_64bit ? OP_SIZE_8 : OP_SIZE_4)
360
361 #define OP_SIZE_INT log_2(sizeof(int_default_t))
362
363 #define check_insn(insn) \
364 do { \
365 /*if ((insn_opcode(insn) == INSN_ALU || insn_opcode(insn) == INSN_ALU1) && insn_op_size(insn) != OP_SIZE_NATIVE) internal(file_line, "invalid insn %08x", (unsigned)(insn));*/\
366 /*if (insn == 0x001a000e) internal(file_line, "invalid insn %08x", insn);*/\
367 } while (0)
368
369 #ifdef DEBUG_INSNS
370 #define gen_line() gen_four(__LINE__ + (insn_file << 24))
371 #else
372 #define gen_line() do { } while (0)
373 #endif
374
375 #ifdef ARCH_IA64
376 #define ARCH_CONTEXT struct { \
377 uint64_t insns[3]; \
378 uint8_t insn_units[3]; \
379 bool insn_stops[3]; \
380 uint64_t wr_mask[4]; \
381 }
382 #endif
383
384 #define gen_insn(opcode, op_size, aux, writes_flags) \
385 do { \
386 uint32_t dword = \
387 (uint32_t)(opcode) << INSN_OPCODE_SHIFT | \
388 (uint32_t)(op_size) << INSN_OP_SIZE_SHIFT | \
389 (uint32_t)(aux) << INSN_AUX_SHIFT | \
390 (uint32_t)(writes_flags) << INSN_WRITES_FLAGS_SHIFT; \
391 check_insn(dword); \
392 gen_line(); \
393 gen_four(dword); \
394 } while (0)
395
396 static size_t arg_size(uint8_t arg)
397 {
398 if (arg < ARG_REGS_MAX)
399 return 1;
400 if (arg >= ARG_SHIFTED_REGISTER && arg <= ARG_EXTENDED_REGISTER)
401 return 3;
402 if (arg == ARG_ADDRESS_0)
403 return 9;
404 if (arg >= ARG_ADDRESS_1 && arg <= ARG_ADDRESS_1_POST_I)
405 return 10;
406 if (arg >= ARG_ADDRESS_2 && arg <= ARG_ADDRESS_2_SXTW)
407 return 11;
408 if (arg == ARG_IMM)
409 return 9;
410 internal(file_line, "arg_size: invalid argument %02x", arg);
411 return 0;
412 }
413
414 struct relocation {
415 uint32_t label_id;
416 uint8_t length;
417 size_t position;
418 size_t jmp_instr;
419 };
420
421 struct code_arg {
422 frame_t slot;
423 frame_t flags;
424 frame_t type;
425 };
426
427 struct cg_entry {
428 size_t entry_to_pos;
429 frame_t *variables;
430 size_t n_variables;
431 uint32_t entry_label;
432 uint32_t nonflat_label;
433 };
434
435 struct cg_exit {
436 uint32_t undo_label;
437 uint8_t undo_opcode;
438 uint8_t undo_op_size;
439 uint8_t undo_aux;
440 uint8_t undo_writes_flags;
441 uint8_t undo_parameters[35];
442 uint8_t undo_parameters_len;
443 uint32_t escape_label;
444 };
445
446 #define FLAG_CACHE_IS_FLAT 0x01
447 #define FLAG_CACHE_IS_NOT_FLAT 0x02
448 #define FLAG_CACHE_IS_NOT_THUNK 0x04
449
450 struct codegen_context {
451 struct data *fn;
452 struct data **local_directory;
453
454 const code_t *instr_start;
455 const code_t *current_position;
456 uchar_efficient_t arg_mode;
457
458 uint32_t label_id;
459 struct cg_entry *entries;
460 frame_t n_entries;
461
462 uint8_t *code;
463 size_t code_size;
464
465 uint8_t *code_position;
466
467 uint32_t *code_labels;
468 struct cg_exit **code_exits;
469 uint32_t escape_nospill_label;
470 uint32_t call_label;
471 uint32_t reload_label;
472
473 uint8_t *mcode;
474 size_t mcode_size;
475
476 size_t *label_to_pos;
477 struct relocation *reloc;
478 size_t reloc_size;
479
480 struct trap_record *trap_records;
481 size_t trap_records_size;
482
483 struct code_arg *args;
484 size_t args_l;
485 const code_t *return_values;
486
487 uint8_t *flag_cache;
488 short *registers;
489 frame_t *need_spill;
490 size_t need_spill_l;
491
492 unsigned base_reg;
493 bool offset_reg;
494 int64_t offset_imm;
495
496 bool const_reg;
497 int64_t const_imm;
498
499 struct data *codegen;
500
501 int upcall_args;
502 frame_t *var_aux;
503
504 ajla_error_t err;
505
506 #ifdef ARCH_CONTEXT
507 ARCH_CONTEXT a;
508 #endif
509 };
510
511 static void init_ctx(struct codegen_context *ctx)
512 {
513 ctx->local_directory = NULL;
514 ctx->label_id = 0;
515 ctx->entries = NULL;
516 ctx->n_entries = 0;
517 ctx->code = NULL;
518 ctx->code_labels = NULL;
519 ctx->code_exits = NULL;
520 ctx->escape_nospill_label = 0;
521 ctx->call_label = 0;
522 ctx->reload_label = 0;
523 ctx->mcode = NULL;
524 ctx->label_to_pos = NULL;
525 ctx->reloc = NULL;
526 ctx->trap_records = NULL;
527 ctx->args = NULL;
528 ctx->flag_cache = NULL;
529 ctx->registers = NULL;
530 ctx->need_spill = NULL;
531 ctx->codegen = NULL;
532 ctx->upcall_args = -1;
533 ctx->var_aux = NULL;
534 }
535
536 static void done_ctx(struct codegen_context *ctx)
537 {
538 if (ctx->local_directory)
539 mem_free(ctx->local_directory);
540 if (ctx->entries) {
541 size_t i;
542 for (i = 0; i < ctx->n_entries; i++) {
543 struct cg_entry *ce = &ctx->entries[i];
544 if (ce->variables)
545 mem_free(ce->variables);
546 }
547 mem_free(ctx->entries);
548 }
549 if (ctx->code)
550 mem_free(ctx->code);
551 if (ctx->code_labels)
552 mem_free(ctx->code_labels);
553 if (ctx->code_exits) {
554 ip_t ip;
555 ip_t cs = da(ctx->fn,function)->code_size;
556 for (ip = 0; ip < cs; ip++) {
557 if (ctx->code_exits[ip])
558 mem_free(ctx->code_exits[ip]);
559 }
560 mem_free(ctx->code_exits);
561 }
562 if (ctx->mcode)
563 mem_free(ctx->mcode);
564 if (ctx->label_to_pos)
565 mem_free(ctx->label_to_pos);
566 if (ctx->reloc)
567 mem_free(ctx->reloc);
568 if (ctx->trap_records)
569 mem_free(ctx->trap_records);
570 if (ctx->args)
571 mem_free(ctx->args);
572 if (ctx->flag_cache)
573 mem_free(ctx->flag_cache);
574 if (ctx->registers)
575 mem_free(ctx->registers);
576 if (ctx->need_spill)
577 mem_free(ctx->need_spill);
578 if (ctx->codegen)
579 data_free(ctx->codegen);
580 if (ctx->var_aux)
581 mem_free(ctx->var_aux);
582 }
583
584
585 static inline code_t get_code(struct codegen_context *ctx)
586 {
587 ajla_assert(ctx->current_position < da(ctx->fn,function)->code + da(ctx->fn,function)->code_size, (file_line, "get_code: ran out of code"));
588 return *ctx->current_position++;
589 }
590
591 static inline uint32_t get_uint32(struct codegen_context *ctx)
592 {
593 uint32_t a1 = get_code(ctx);
594 uint32_t a2 = get_code(ctx);
595 #if !CODE_ENDIAN
596 return a1 + (a2 << 16);
597 #else
598 return a2 + (a1 << 16);
599 #endif
600 }
601
602 static int32_t get_jump_offset(struct codegen_context *ctx)
603 {
604 if (SIZEOF_IP_T == 2) {
605 return (int32_t)(int16_t)get_code(ctx);
606 } else if (SIZEOF_IP_T == 4) {
607 return (int32_t)get_uint32(ctx);
608 } else {
609 not_reached();
610 return -1;
611 }
612 }
613
614 static inline void get_one(struct codegen_context *ctx, frame_t *v)
615 {
616 if (!ctx->arg_mode) {
617 code_t c = get_code(ctx);
618 ajla_assert(!(c & ~0xff), (file_line, "get_one: high byte is not cleared: %u", (unsigned)c));
619 *v = c & 0xff;
620 } else if (ctx->arg_mode == 1) {
621 *v = get_code(ctx);
622 #if ARG_MODE_N >= 2
623 } else if (ctx->arg_mode == 2) {
624 *v = get_uint32(ctx);
625 #endif
626 } else {
627 internal(file_line, "get_one: invalid arg mode %u", ctx->arg_mode);
628 }
629 }
630
631 static inline void get_two(struct codegen_context *ctx, frame_t *v1, frame_t *v2)
632 {
633 if (!ctx->arg_mode) {
634 code_t c = get_code(ctx);
635 *v1 = c & 0xff;
636 *v2 = c >> 8;
637 } else if (ctx->arg_mode == 1) {
638 *v1 = get_code(ctx);
639 *v2 = get_code(ctx);
640 #if ARG_MODE_N >= 2
641 } else if (ctx->arg_mode == 2) {
642 *v1 = get_uint32(ctx);
643 *v2 = get_uint32(ctx);
644 #endif
645 } else {
646 internal(file_line, "get_two: invalid arg mode %u", ctx->arg_mode);
647 }
648 }
649
650
651 static uint32_t alloc_label(struct codegen_context *ctx)
652 {
653 return ++ctx->label_id;
654 }
655
656 static struct cg_exit *alloc_cg_exit_for_ip(struct codegen_context *ctx, const code_t *code)
657 {
658 ip_t ip = code - da(ctx->fn,function)->code;
659 struct cg_exit *ce = ctx->code_exits[ip];
660 if (!ce) {
661 ce = mem_calloc_mayfail(struct cg_exit *, sizeof(struct cg_exit), &ctx->err);
662 if (unlikely(!ce))
663 return NULL;
664 ctx->code_exits[ip] = ce;
665 }
666 return ce;
667 }
668
669 static struct cg_exit *alloc_undo_label(struct codegen_context *ctx)
670 {
671 struct cg_exit *ce = alloc_cg_exit_for_ip(ctx, ctx->instr_start);
672 if (unlikely(!ce))
673 return NULL;
674 if (unlikely(ce->undo_label != 0))
675 internal(file_line, "alloc_cg_exit: undo label already allocated");
676 ce->undo_label = alloc_label(ctx);
677 if (unlikely(!ce->undo_label))
678 return NULL;
679 return ce;
680 }
681
682 static uint32_t alloc_escape_label_for_ip(struct codegen_context *ctx, const code_t *code)
683 {
684 struct cg_exit *ce = alloc_cg_exit_for_ip(ctx, code);
685 if (!ce)
686 return 0;
687 if (!ce->escape_label)
688 ce->escape_label = alloc_label(ctx);
689 return ce->escape_label;
690 }
691
692 static uint32_t alloc_escape_label(struct codegen_context *ctx)
693 {
694 return alloc_escape_label_for_ip(ctx, ctx->instr_start);
695 }
696
697 static uint32_t attr_unused alloc_call_label(struct codegen_context *ctx)
698 {
699 if (!ctx->call_label) {
700 ctx->call_label = alloc_label(ctx);
701 }
702 return ctx->call_label;
703 }
704
705 static uint32_t alloc_reload_label(struct codegen_context *ctx)
706 {
707 if (!ctx->reload_label) {
708 ctx->reload_label = alloc_label(ctx);
709 }
710 return ctx->reload_label;
711 }
712
713 static size_t attr_unused mark_params(struct codegen_context *ctx)
714 {
715 return ctx->code_size;
716 }
717
718 static void attr_unused copy_params(struct codegen_context *ctx, struct cg_exit *ce, size_t mark)
719 {
720 if (ctx->code_size - mark > n_array_elements(ce->undo_parameters))
721 internal(file_line, "undo_parameters is too small: %"PRIuMAX" > %"PRIuMAX"", (uintmax_t)(ctx->code_size - mark), (uintmax_t)n_array_elements(ce->undo_parameters));
722 memcpy(ce->undo_parameters, ctx->code + mark, ctx->code_size - mark);
723 ce->undo_parameters_len = ctx->code_size - mark;
724 ctx->code_size = mark;
725 }
726
727 #define g(call) \
728 do { \
729 if (unlikely(!call)) \
730 return false; \
731 } while (0)
732
733 #define gen_one(byte) \
734 do { \
735 /*debug("gen %d: %02x", __LINE__, (uint8_t)(byte))*/; \
736 if (unlikely(!array_add_mayfail(uint8_t, &ctx->code, &ctx->code_size, byte, NULL, &ctx->err)))\
737 return false; \
738 } while (0)
739
740 #if defined(C_LITTLE_ENDIAN)
741 #define gen_two(word) \
742 do { \
743 uint16_t word_ = (word); \
744 /*debug("gen %d: %04x", __LINE__, (uint16_t)(word_));*/ \
745 if (unlikely(!array_add_multiple_mayfail(uint8_t, &ctx->code, &ctx->code_size, cast_ptr(uint8_t *, &word_), 2, NULL, &ctx->err)))\
746 return false; \
747 } while (0)
748 #define gen_four(dword) \
749 do { \
750 uint32_t dword_ = (dword); \
751 /*debug("gen %d: %08x", __LINE__, (uint32_t)(dword_));*/ \
752 if (unlikely(!array_add_multiple_mayfail(uint8_t, &ctx->code, &ctx->code_size, cast_ptr(uint8_t *, &dword_), 4, NULL, &ctx->err)))\
753 return false; \
754 } while (0)
755 #define gen_eight(qword) \
756 do { \
757 uint64_t qword_ = (qword); \
758 /*debug("gen %d: %016lx", __LINE__, (uint64_t)(qword_));*/ \
759 if (unlikely(!array_add_multiple_mayfail(uint8_t, &ctx->code, &ctx->code_size, cast_ptr(uint8_t *, &qword_), 8, NULL, &ctx->err)))\
760 return false; \
761 } while (0)
762 #else
763 #define gen_two(word) \
764 do { \
765 uint16_t word_ = (word); \
766 gen_one(word_ & 0xffU); \
767 gen_one(word_ >> 8); \
768 } while (0)
769 #define gen_four(dword) \
770 do { \
771 uint32_t dword_ = (dword); \
772 gen_two(dword_ & 0xffffU); \
773 gen_two(dword_ >> 15 >> 1); \
774 } while (0)
775 #define gen_eight(qword) \
776 do { \
777 uint64_t qword_ = (qword); \
778 gen_four(qword_ & 0xffffffffUL); \
779 gen_four(qword_ >> 15 >> 15 >> 2); \
780 } while (0)
781 #endif
782
783 #define gen_label(label_id) \
784 do { \
785 gen_insn(INSN_LABEL, 0, 0, 0); \
786 gen_four(label_id); \
787 } while (0)
788
789
790 static uint8_t attr_unused cget_one(struct codegen_context *ctx)
791 {
792 ajla_assert(ctx->code_position < ctx->code + ctx->code_size, (file_line, "cget_one: ran out of code"));
793 return *ctx->code_position++;
794 }
795
796 static uint16_t attr_unused cget_two(struct codegen_context *ctx)
797 {
798 #if defined(C_LITTLE_ENDIAN)
799 uint16_t r;
800 ajla_assert(ctx->code_position < ctx->code + ctx->code_size, (file_line, "cget_two: ran out of code"));
801 memcpy(&r, ctx->code_position, 2);
802 ctx->code_position += 2;
803 return r;
804 #else
805 uint16_t r = cget_one(ctx);
806 r |= cget_one(ctx) << 8;
807 return r;
808 #endif
809 }
810
811 static uint32_t cget_four(struct codegen_context *ctx)
812 {
813 #if defined(C_LITTLE_ENDIAN)
814 uint32_t r;
815 ajla_assert(ctx->code_position < ctx->code + ctx->code_size, (file_line, "cget_four: ran out of code"));
816 memcpy(&r, ctx->code_position, 4);
817 ctx->code_position += 4;
818 return r;
819 #else
820 uint32_t r = cget_two(ctx);
821 r |= (uint32_t)cget_two(ctx) << 16;
822 return r;
823 #endif
824 }
825
826 static uint64_t attr_unused cget_eight(struct codegen_context *ctx)
827 {
828 #if defined(C_LITTLE_ENDIAN)
829 uint64_t r;
830 ajla_assert(ctx->code_position < ctx->code + ctx->code_size, (file_line, "cget_eight: ran out of code"));
831 memcpy(&r, ctx->code_position, 8);
832 ctx->code_position += 8;
833 return r;
834 #else
835 uint64_t r = cget_four(ctx);
836 r |= (uint64_t)cget_four(ctx) << 32;
837 return r;
838 #endif
839 }
840
841 static int64_t get_imm(uint8_t *ptr)
842 {
843 #if defined(C_LITTLE_ENDIAN)
844 int64_t r;
845 memcpy(&r, ptr, 8);
846 return r;
847 #else
848 int64_t r;
849 r = (uint64_t)ptr[0] |
850 ((uint64_t)ptr[1] << 8) |
851 ((uint64_t)ptr[2] << 16) |
852 ((uint64_t)ptr[3] << 24) |
853 ((uint64_t)ptr[4] << 32) |
854 ((uint64_t)ptr[5] << 40) |
855 ((uint64_t)ptr[6] << 48) |
856 ((uint64_t)ptr[7] << 56);
857 return r;
858 #endif
859 }
860
861 #define cgen_one(byte) \
862 do { \
863 if (unlikely(!array_add_mayfail(uint8_t, &ctx->mcode, &ctx->mcode_size, byte, NULL, &ctx->err)))\
864 return false; \
865 } while (0)
866
867 #if defined(C_LITTLE_ENDIAN) || 1
868 #define cgen_two(word) \
869 do { \
870 uint16_t word_ = (word); \
871 if (unlikely(!array_add_multiple_mayfail(uint8_t, &ctx->mcode, &ctx->mcode_size, cast_ptr(uint8_t *, &word_), 2, NULL, &ctx->err)))\
872 return false; \
873 } while (0)
874 #define cgen_four(dword) \
875 do { \
876 uint32_t dword_ = (dword); \
877 /*if (dword_ == 0x1ee02000) internal(file_line, "invalid instruction");*/\
878 if (unlikely(!array_add_multiple_mayfail(uint8_t, &ctx->mcode, &ctx->mcode_size, cast_ptr(uint8_t *, &dword_), 4, NULL, &ctx->err)))\
879 return false; \
880 } while (0)
881 #define cgen_eight(qword) \
882 do { \
883 uint64_t qword_ = (qword); \
884 if (unlikely(!array_add_multiple_mayfail(uint8_t, &ctx->mcode, &ctx->mcode_size, cast_ptr(uint8_t *, &qword_), 8, NULL, &ctx->err)))\
885 return false; \
886 } while (0)
887 #else
888 #define cgen_two(word) \
889 do { \
890 cgen_one((word) & 0xff); \
891 cgen_one((word) >> 8); \
892 } while (0)
893 #define cgen_four(dword) \
894 do { \
895 cgen_two((dword) & 0xffff); \
896 cgen_two((dword) >> 15 >> 1); \
897 } while (0)
898 #define cgen_eight(qword) \
899 do { \
900 cgen_four((qword) & 0xffffffff); \
901 cgen_four((qword) >> 15 >> 15 >> 2); \
902 } while (0)
903 #endif
904
905
906 #define IMM_PURPOSE_LDR_OFFSET 1
907 #define IMM_PURPOSE_LDR_SX_OFFSET 2
908 #define IMM_PURPOSE_STR_OFFSET 3
909 #define IMM_PURPOSE_LDP_STP_OFFSET 4
910 #define IMM_PURPOSE_VLDR_VSTR_OFFSET 5
911 #define IMM_PURPOSE_MVI_CLI_OFFSET 6
912 #define IMM_PURPOSE_STORE_VALUE 7
913 #define IMM_PURPOSE_ADD 8
914 #define IMM_PURPOSE_SUB 9
915 #define IMM_PURPOSE_CMP 10
916 #define IMM_PURPOSE_CMP_LOGICAL 11
917 #define IMM_PURPOSE_AND 12
918 #define IMM_PURPOSE_OR 13
919 #define IMM_PURPOSE_XOR 14
920 #define IMM_PURPOSE_ANDN 15
921 #define IMM_PURPOSE_TEST 16
922 #define IMM_PURPOSE_JMP_2REGS 17
923 #define IMM_PURPOSE_MUL 18
924 #define IMM_PURPOSE_CMOV 19
925 #define IMM_PURPOSE_MOVR 20
926 #define IMM_PURPOSE_BITWISE 21
927 #define IMM_PURPOSE_ADD_TRAP 22
928 #define IMM_PURPOSE_SUB_TRAP 23
929
930
931 static unsigned alu_purpose(unsigned alu)
932 {
933 unsigned purpose =
934 alu == ALU_ADD ? IMM_PURPOSE_ADD :
935 alu == ALU_ADC ? IMM_PURPOSE_ADD :
936 alu == ALU_SUB ? IMM_PURPOSE_SUB :
937 alu == ALU_SBB ? IMM_PURPOSE_SUB :
938 alu == ALU_MUL ? IMM_PURPOSE_MUL :
939 alu == ALU_UMULH ? IMM_PURPOSE_MUL :
940 alu == ALU_SMULH ? IMM_PURPOSE_MUL :
941 alu == ALU_ANDN ? IMM_PURPOSE_ANDN :
942 alu == ALU_AND ? IMM_PURPOSE_AND :
943 alu == ALU_OR ? IMM_PURPOSE_OR :
944 alu == ALU_XOR ? IMM_PURPOSE_XOR :
945 alu == ALU_EXTBL ? IMM_PURPOSE_OR :
946 alu == ALU_EXTWL ? IMM_PURPOSE_OR :
947 alu == ALU_EXTLL ? IMM_PURPOSE_OR :
948 alu == ALU_EXTLH ? IMM_PURPOSE_OR :
949 alu == ALU_INSBL ? IMM_PURPOSE_OR :
950 alu == ALU_MSKBL ? IMM_PURPOSE_OR :
951 alu == ALU_ZAP ? IMM_PURPOSE_ANDN :
952 alu == ALU_ZAPNOT ? IMM_PURPOSE_AND :
953 -1U;
954 if (unlikely(purpose == -1U))
955 internal(file_line, "alu_purpose: invalid alu %u", alu);
956 return purpose;
957 }
958
959 static unsigned alu_trap_purpose(unsigned alu)
960 {
961 unsigned purpose =
962 alu == ALU_ADD ? IMM_PURPOSE_ADD_TRAP :
963 alu == ALU_SUB ? IMM_PURPOSE_SUB_TRAP :
964 -1U;
965 if (unlikely(purpose == -1U))
966 internal(file_line, "alu_trap_purpose: invalid alu %u", alu);
967 return purpose;
968 }
969
970
971 static bool attr_w gen_imm(struct codegen_context *ctx, int64_t imm, unsigned purpose, unsigned size);
972 static bool attr_w gen_upcall_end(struct codegen_context *ctx, unsigned args);
973
974 #define gen_address_offset() \
975 do { \
976 if (likely(!ctx->offset_reg)) { \
977 gen_one(ARG_ADDRESS_1); \
978 gen_one(ctx->base_reg); \
979 gen_eight(ctx->offset_imm); \
980 } else { \
981 gen_one(ARG_ADDRESS_2); \
982 gen_one(ctx->base_reg); \
983 gen_one(R_OFFSET_IMM); \
984 gen_eight(0); \
985 } \
986 } while (0)
987
988 #define gen_imm_offset() \
989 do { \
990 if (likely(!ctx->const_reg)) { \
991 gen_one(ARG_IMM); \
992 gen_eight(ctx->const_imm); \
993 } else { \
994 gen_one(R_CONST_IMM); \
995 } \
996 } while (0)
997
998 #define is_imm() (!ctx->const_reg)
999
1000
1001 static inline bool slot_is_register(struct codegen_context *ctx, frame_t slot)
1002 {
1003 if (frame_t_is_const(slot))
1004 return false;
1005 if (unlikely(slot >= function_n_variables(ctx->fn)))
1006 internal(file_line, "slot_is_register: invalid slot %lu", (unsigned long)slot);
1007 return ctx->registers[slot] >= 0;
1008 }
1009
1010
1011 #define insn_file 1
1012 #if defined(ARCH_ALPHA)
1013 #include "c1-alpha.inc"
1014 #elif defined(ARCH_ARM32)
1015 #include "c1-arm.inc"
1016 #elif defined(ARCH_ARM64)
1017 #include "c1-arm64.inc"
1018 #elif defined(ARCH_IA64)
1019 #include "c1-ia64.inc"
1020 #elif defined(ARCH_LOONGARCH64)
1021 #include "c1-loong.inc"
1022 #elif defined(ARCH_MIPS)
1023 #include "c1-mips.inc"
1024 #elif defined(ARCH_PARISC)
1025 #include "c1-hppa.inc"
1026 #elif defined(ARCH_POWER)
1027 #include "c1-power.inc"
1028 #elif defined(ARCH_S390)
1029 #include "c1-s390.inc"
1030 #elif defined(ARCH_SPARC)
1031 #include "c1-sparc.inc"
1032 #elif defined(ARCH_RISCV64)
1033 #include "c1-riscv.inc"
1034 #elif defined(ARCH_X86)
1035 #include "c1-x86.inc"
1036 #endif
1037 #undef insn_file
1038
1039
1040 #ifndef ARCH_SUPPORTS_TRAPS
1041 #define ARCH_SUPPORTS_TRAPS(size) 0
1042 #define ARCH_TRAP_BEFORE 0
1043 #endif
1044
1045
1046 static bool attr_w gen_imm(struct codegen_context *ctx, int64_t imm, unsigned purpose, unsigned size)
1047 {
1048 if (!is_direct_const(imm, purpose & 0xff, size))
1049 goto load_const;
1050 if (purpose >> 8 && !is_direct_const(imm, purpose >> 8, size))
1051 goto load_const;
1052 ctx->const_imm = imm;
1053 ctx->const_reg = false;
1054 return true;
1055 load_const:
1056 g(gen_load_constant(ctx, R_CONST_IMM, imm));
1057 ctx->const_reg = true;
1058 return true;
1059 }
1060
1061
1062 #define insn_file 2
1063 #include "cg-util.inc"
1064 #undef insn_file
1065
1066 #define insn_file 3
1067 #include "cg-frame.inc"
1068 #undef insn_file
1069
1070 #define insn_file 4
1071 #include "cg-flags.inc"
1072 #undef insn_file
1073
1074 #define insn_file 5
1075 #include "cg-flcch.inc"
1076 #undef insn_file
1077
1078 #define insn_file 6
1079 #include "cg-ptr.inc"
1080 #undef insn_file
1081
1082 #define insn_file 7
1083 #include "cg-alu.inc"
1084 #undef insn_file
1085
1086 #define insn_file 8
1087 #include "cg-ops.inc"
1088 #undef insn_file
1089
1090 #define insn_file 0
1091
1092
1093 #ifndef n_regs_saved
1094 #define n_regs_saved n_array_elements(regs_saved)
1095 #endif
1096
1097 #ifndef n_regs_volatile
1098 #define n_regs_volatile n_array_elements(regs_volatile)
1099 #endif
1100
1101 #ifndef n_fp_saved
1102 #define n_fp_saved n_array_elements(fp_saved)
1103 #endif
1104
1105 #ifndef n_fp_volatile
1106 #define n_fp_volatile n_array_elements(fp_volatile)
1107 #endif
1108
1109 #ifndef n_vector_volatile
1110 #define n_vector_volatile n_array_elements(vector_volatile)
1111 #endif
1112
1113 static bool attr_w gen_registers(struct codegen_context *ctx)
1114 {
1115 frame_t v;
1116 size_t index_saved = 0;
1117 size_t index_volatile = 0;
1118 size_t index_fp_saved = 0;
1119 size_t index_fp_volatile = 0;
1120 size_t attr_unused index_vector_volatile = 0;
1121 #ifdef ARCH_S390
1122 bool uses_x = false;
1123 for (v = MIN_USEABLE_SLOT; v < function_n_variables(ctx->fn); v++) {
1124 const struct type *t = get_type_of_local(ctx, v);
1125 if (t && TYPE_TAG_IS_REAL(t->tag) && TYPE_TAG_IDX_REAL(t->tag) == 4) {
1126 uses_x = true;
1127 break;
1128 }
1129 }
1130 #endif
1131 /*for (v = function_n_variables(ctx->fn) - 1; v >= MIN_USEABLE_SLOT; v--)*/
1132 for (v = MIN_USEABLE_SLOT; v < function_n_variables(ctx->fn); v++) {
1133 const struct type *t;
1134 ctx->registers[v] = -1;
1135 if (ra_chicken)
1136 continue;
1137 t = get_type_of_local(ctx, v);
1138 if (unlikely(!t))
1139 continue;
1140 if (!da(ctx->fn,function)->local_variables_flags[v].must_be_flat &&
1141 !da(ctx->fn,function)->local_variables_flags[v].must_be_data)
1142 continue;
1143 if (!ARCH_HAS_BWX && t->size < 1U << OP_SIZE_4)
1144 continue;
1145 if (TYPE_TAG_IS_FIXED(t->tag) || TYPE_TAG_IS_INT(t->tag) || t->tag == TYPE_TAG_flat_option || t->tag == TYPE_TAG_unknown || t->tag == TYPE_TAG_record) {
1146 if (TYPE_TAG_IS_BUILTIN(t->tag)) {
1147 if (!is_power_of_2(t->size) || t->size > 1U << OP_SIZE_NATIVE)
1148 continue;
1149 }
1150 if (index_saved < n_regs_saved + zero
1151 #if defined(ARCH_PARISC) || defined(ARCH_SPARC)
1152 && t->size <= 1U << OP_SIZE_ADDRESS
1153 #endif
1154 ) {
1155 ctx->registers[v] = regs_saved[index_saved++];
1156 } else if (index_volatile < n_regs_volatile + zero) {
1157 ctx->registers[v] = regs_volatile[index_volatile++];
1158 } else {
1159 continue;
1160 }
1161 } else if (TYPE_TAG_IS_REAL(t->tag)) {
1162 unsigned real_type = TYPE_TAG_IDX_REAL(t->tag);
1163 if ((SUPPORTED_FP >> real_type) & 1) {
1164 #ifdef ARCH_POWER
1165 if (real_type == 4) {
1166 if (index_vector_volatile < n_vector_volatile + zero) {
1167 ctx->registers[v] = vector_volatile[index_vector_volatile++];
1168 goto success;
1169 }
1170 continue;
1171 }
1172 #endif
1173 #ifdef ARCH_S390
1174 if (real_type == 4) {
1175 if (!(index_fp_saved & 1) && index_fp_saved + 1 < n_fp_saved + zero) {
1176 ctx->registers[v] = fp_saved[index_fp_saved++];
1177 index_fp_saved++;
1178 goto success;
1179 }
1180 if (index_fp_saved & 1 && index_fp_saved + 2 < n_fp_saved + zero) {
1181 index_fp_saved++;
1182 ctx->registers[v] = fp_saved[index_fp_saved++];
1183 index_fp_saved++;
1184 goto success;
1185 }
1186 if (!(index_fp_volatile & 1) && index_fp_volatile + 1 < n_fp_volatile + zero) {
1187 ctx->registers[v] = fp_volatile[index_fp_volatile++];
1188 index_fp_volatile++;
1189 goto success;
1190 }
1191 if (index_fp_volatile & 1 && index_fp_volatile + 2 < n_fp_volatile + zero) {
1192 index_fp_volatile++;
1193 ctx->registers[v] = fp_volatile[index_fp_volatile++];
1194 index_fp_volatile++;
1195 goto success;
1196 }
1197 continue;
1198 }
1199 #endif
1200 if (index_fp_saved < n_fp_saved + zero) {
1201 ctx->registers[v] = fp_saved[index_fp_saved++];
1202 } else if (index_fp_volatile < n_fp_volatile + zero) {
1203 ctx->registers[v] = fp_volatile[index_fp_volatile++];
1204 } else {
1205 continue;
1206 }
1207 } else {
1208 continue;
1209 }
1210 } else {
1211 continue;
1212 }
1213 goto success;
1214 success:
1215 if (!reg_is_saved(ctx->registers[v])) {
1216 if (unlikely(!array_add_mayfail(frame_t, &ctx->need_spill, &ctx->need_spill_l, v, NULL, &ctx->err)))
1217 return false;
1218 }
1219 }
1220
1221 return true;
1222 }
1223
1224 static bool attr_w gen_fused_binary(struct codegen_context *ctx, unsigned mode, unsigned op_size, unsigned op, uint32_t escape_label, frame_t slot_1, frame_t slot_2, frame_t slot_r, bool *failed)
1225 {
1226 const code_t *backup = ctx->current_position;
1227 code_t code;
1228 frame_t slot_dr, slot_test;
1229 int32_t offs_false;
1230
1231 *failed = false;
1232
1233 next_code:
1234 code = get_code(ctx);
1235 ctx->arg_mode = code / OPCODE_MODE_MULT;
1236 code %= OPCODE_MODE_MULT;
1237 ajla_assert_lo(ctx->arg_mode < ARG_MODE_N, (file_line, "gen_fused_binary: invalid opcode %04x", (unsigned)*ctx->instr_start));
1238
1239 if (code == OPCODE_DEREFERENCE) {
1240 get_one(ctx, &slot_dr);
1241 const struct type *t = get_type_of_local(ctx, slot_dr);
1242 if (!TYPE_TAG_IS_BUILTIN(t->tag)) {
1243 *failed = true;
1244 goto fail;
1245 }
1246 if (unlikely(!flag_is_clear(ctx, slot_dr))) {
1247 *failed = true;
1248 goto fail;
1249 }
1250 goto next_code;
1251 }
1252 if (code == OPCODE_DEREFERENCE_CLEAR) {
1253 *failed = true;
1254 goto fail;
1255 }
1256 if (unlikely(code != OPCODE_JMP_FALSE))
1257 internal(file_line, "gen_fused_binary: binary operation is not followed by jmp false: %x, %s", code, decode_opcode(code, true));
1258 get_one(ctx, &slot_test);
1259 if (unlikely(slot_test != slot_r))
1260 internal(file_line, "gen_fused_binary: the result of the binary operation and the tested variable do not match");
1261 offs_false = get_jump_offset(ctx);
1262 get_jump_offset(ctx);
1263
1264 if (mode == MODE_ARRAY_LEN_GT) {
1265 g(gen_array_len(ctx, slot_1, slot_2, slot_r, true, offs_false));
1266 } else if (mode == MODE_REAL) {
1267 g(gen_fp_alu_jmp(ctx, op_size, op, escape_label, slot_1, slot_2, offs_false, failed));
1268 } else {
1269 g(gen_alu_jmp(ctx, mode, op_size, op, slot_1, slot_2, offs_false, failed));
1270 }
1271
1272 fail:
1273 if (*failed)
1274 ctx->current_position = backup;
1275
1276 return true;
1277 }
1278
1279 static bool attr_w gen_function(struct codegen_context *ctx)
1280 {
1281 ctx->current_position = da(ctx->fn,function)->code;
1282
1283 ctx->escape_nospill_label = alloc_label(ctx);
1284 if (unlikely(!ctx->escape_nospill_label))
1285 return false;
1286
1287 while (ctx->current_position != da(ctx->fn,function)->code + da(ctx->fn,function)->code_size) {
1288 ip_t ip;
1289 code_t code;
1290 unsigned op, type;
1291 frame_t slot_1, slot_2, slot_3, slot_r, flags, fn_idx, opt;
1292 arg_t n_args, n_ret, i_arg;
1293 uint32_t label_id;
1294 uint32_t escape_label;
1295 bool failed;
1296
1297 ajla_assert_lo(ctx->current_position < da(ctx->fn,function)->code + da(ctx->fn,function)->code_size, (file_line, "gen_function: ran out of code in %s", da(ctx->fn,function)->function_name));
1298
1299 ctx->instr_start = ctx->current_position;
1300
1301 /*debug("%s: %04x, %s", da(ctx->fn,function)->function_name, *ctx->instr_start, decode_opcode(*ctx->instr_start, true));*/
1302
1303 ip = ctx->instr_start - da(ctx->fn,function)->code;
1304 if (likely(!ctx->code_labels[ip])) {
1305 ctx->code_labels[ip] = alloc_label(ctx);
1306 if (unlikely(!ctx->code_labels[ip]))
1307 return false;
1308 }
1309 gen_label(ctx->code_labels[ip]);
1310
1311 code = get_code(ctx);
1312 ctx->arg_mode = code / OPCODE_MODE_MULT;
1313 code %= OPCODE_MODE_MULT;
1314 ajla_assert_lo(ctx->arg_mode < ARG_MODE_N, (file_line, "gen_function: invalid opcode %04x", (unsigned)*ctx->instr_start));
1315
1316 if (code >= OPCODE_FIXED_OP + uzero && code < OPCODE_INT_OP) {
1317 code -= OPCODE_FIXED_OP;
1318 op = (code / OPCODE_FIXED_OP_MULT) % OPCODE_FIXED_TYPE_MULT;
1319 type = code / OPCODE_FIXED_TYPE_MULT;
1320 if (op < OPCODE_FIXED_OP_C) {
1321 get_two(ctx, &slot_1, &slot_2);
1322 get_two(ctx, &slot_r, &flags);
1323 escape_label = alloc_escape_label(ctx);
1324 if (unlikely(!escape_label))
1325 return false;
1326 g(gen_test_2_cached(ctx, slot_1, slot_2, escape_label));
1327 flag_set(ctx, slot_1, false);
1328 flag_set(ctx, slot_2, false);
1329 flag_set(ctx, slot_r, false);
1330 if (flags & OPCODE_FLAG_FUSED) {
1331 g(gen_fused_binary(ctx, MODE_FIXED, type, op, escape_label, slot_1, slot_2, slot_r, &failed));
1332 if (unlikely(!failed))
1333 continue;
1334 }
1335 g(gen_alu(ctx, MODE_FIXED, type, op, escape_label, slot_1, slot_2, slot_r));
1336 continue;
1337 } else if (op < OPCODE_FIXED_OP_UNARY) {
1338 op -= OPCODE_FIXED_OP_C;
1339 get_two(ctx, &slot_1, &slot_2);
1340 get_two(ctx, &slot_r, &flags);
1341 escape_label = alloc_escape_label(ctx);
1342 if (unlikely(!escape_label))
1343 return false;
1344 g(gen_test_1_cached(ctx, slot_1, escape_label));
1345 flag_set(ctx, slot_1, false);
1346 flag_set(ctx, slot_r, false);
1347 slot_2 = frame_t_from_const((int32_t)slot_2);
1348 if (flags & OPCODE_FLAG_FUSED) {
1349 g(gen_fused_binary(ctx, MODE_FIXED, type, op, escape_label, slot_1, slot_2, slot_r, &failed));
1350 if (unlikely(!failed))
1351 continue;
1352 }
1353 g(gen_alu(ctx, MODE_FIXED, type, op, escape_label, slot_1, slot_2, slot_r));
1354 continue;
1355 } else if (op < OPCODE_FIXED_OP_N) {
1356 get_two(ctx, &slot_1, &slot_r);
1357 get_one(ctx, &flags);
1358 escape_label = alloc_escape_label(ctx);
1359 if (unlikely(!escape_label))
1360 return false;
1361 g(gen_test_1_cached(ctx, slot_1, escape_label));
1362 flag_set(ctx, slot_1, false);
1363 flag_set(ctx, slot_r, false);
1364 g(gen_alu1(ctx, MODE_FIXED, type, op, escape_label, slot_1, slot_r));
1365 continue;
1366 } else if (op == OPCODE_FIXED_OP_ldc) {
1367 unsigned i;
1368 get_one(ctx, &slot_r);
1369 g(gen_constant(ctx, false, type, false, slot_r));
1370 for (i = 0; i < 1U << type; i += 2)
1371 get_code(ctx);
1372 flag_set(ctx, slot_r, false);
1373 continue;
1374 } else if (op == OPCODE_FIXED_OP_ldc16) {
1375 get_one(ctx, &slot_r);
1376 g(gen_constant(ctx, false, type, true, slot_r));
1377 get_code(ctx);
1378 flag_set(ctx, slot_r, false);
1379 continue;
1380 } else if (op == OPCODE_FIXED_OP_move || op == OPCODE_FIXED_OP_copy) {
1381 get_two(ctx, &slot_1, &slot_r);
1382 escape_label = alloc_escape_label(ctx);
1383 if (unlikely(!escape_label))
1384 return false;
1385 g(gen_test_1_cached(ctx, slot_1, escape_label));
1386 flag_set(ctx, slot_1, false);
1387 flag_set(ctx, slot_r, false);
1388 g(gen_copy(ctx, type, slot_1, slot_r));
1389 continue;
1390 } else {
1391 internal(file_line, "gen_function: bad fixed code %04x", *ctx->instr_start);
1392 }
1393 } else if (code >= OPCODE_INT_OP && code < OPCODE_REAL_OP) {
1394 code -= OPCODE_INT_OP;
1395 op = (code / OPCODE_INT_OP_MULT) % OPCODE_INT_TYPE_MULT;
1396 type = code / OPCODE_INT_TYPE_MULT;
1397 if (op < OPCODE_INT_OP_C) {
1398 get_two(ctx, &slot_1, &slot_2);
1399 get_two(ctx, &slot_r, &flags);
1400 escape_label = alloc_escape_label(ctx);
1401 if (unlikely(!escape_label))
1402 return false;
1403 g(gen_test_2_cached(ctx, slot_1, slot_2, escape_label));
1404 flag_set(ctx, slot_1, false);
1405 flag_set(ctx, slot_2, false);
1406 flag_set(ctx, slot_r, false);
1407 if (flags & OPCODE_FLAG_FUSED) {
1408 g(gen_fused_binary(ctx, MODE_INT, type, op, escape_label, slot_1, slot_2, slot_r, &failed));
1409 if (unlikely(!failed))
1410 continue;
1411 }
1412 g(gen_alu(ctx, MODE_INT, type, op, escape_label, slot_1, slot_2, slot_r));
1413 continue;
1414 } else if (op < OPCODE_INT_OP_UNARY) {
1415 op -= OPCODE_INT_OP_C;
1416 get_two(ctx, &slot_1, &slot_2);
1417 get_two(ctx, &slot_r, &flags);
1418 escape_label = alloc_escape_label(ctx);
1419 if (unlikely(!escape_label))
1420 return false;
1421 g(gen_test_1_cached(ctx, slot_1, escape_label));
1422 flag_set(ctx, slot_1, false);
1423 flag_set(ctx, slot_r, false);
1424 slot_2 = frame_t_from_const((int32_t)slot_2);
1425 if (flags & OPCODE_FLAG_FUSED) {
1426 g(gen_fused_binary(ctx, MODE_INT, type, op, escape_label, slot_1, slot_2, slot_r, &failed));
1427 if (unlikely(!failed))
1428 continue;
1429 }
1430 g(gen_alu(ctx, MODE_INT, type, op, escape_label, slot_1, slot_2, slot_r));
1431 continue;
1432 } else if (op < OPCODE_INT_OP_N) {
1433 get_two(ctx, &slot_1, &slot_r);
1434 get_one(ctx, &flags);
1435 if ((op == OPCODE_INT_OP_to_int || op == OPCODE_INT_OP_from_int) && slot_1 == slot_r)
1436 continue;
1437 escape_label = alloc_escape_label(ctx);
1438 if (unlikely(!escape_label))
1439 return false;
1440 g(gen_test_1_cached(ctx, slot_1, escape_label));
1441 flag_set(ctx, slot_1, false);
1442 flag_set(ctx, slot_r, false);
1443 g(gen_alu1(ctx, MODE_INT, type, op, escape_label, slot_1, slot_r));
1444 continue;
1445 } else if (op == OPCODE_INT_OP_ldc) {
1446 unsigned i;
1447 get_one(ctx, &slot_r);
1448 g(gen_constant(ctx, false, type, false, slot_r));
1449 for (i = 0; i < 1U << type; i += 2)
1450 get_code(ctx);
1451 flag_set(ctx, slot_r, false);
1452 continue;
1453 } else if (op == OPCODE_INT_OP_ldc16) {
1454 get_one(ctx, &slot_r);
1455 g(gen_constant(ctx, false, type, true, slot_r));
1456 get_code(ctx);
1457 flag_set(ctx, slot_r, false);
1458 continue;
1459 } else if (op == OPCODE_INT_OP_move || op == OPCODE_INT_OP_copy) {
1460 get_two(ctx, &slot_1, &slot_r);
1461 escape_label = alloc_escape_label(ctx);
1462 if (unlikely(!escape_label))
1463 return false;
1464 g(gen_test_1_cached(ctx, slot_1, escape_label));
1465 flag_set(ctx, slot_1, false);
1466 flag_set(ctx, slot_r, false);
1467 g(gen_copy(ctx, type, slot_1, slot_r));
1468 continue;
1469 } else {
1470 internal(file_line, "gen_function: bad integer code %04x", *ctx->instr_start);
1471 }
1472 } else if (code >= OPCODE_REAL_OP && code < OPCODE_BOOL_OP) {
1473 code -= OPCODE_REAL_OP;
1474 op = (code / OPCODE_REAL_OP_MULT) % OPCODE_REAL_TYPE_MULT;
1475 type = code / OPCODE_REAL_TYPE_MULT;
1476 if (op < OPCODE_REAL_OP_UNARY) {
1477 get_two(ctx, &slot_1, &slot_2);
1478 get_two(ctx, &slot_r, &flags);
1479 escape_label = alloc_escape_label(ctx);
1480 if (unlikely(!escape_label))
1481 return false;
1482 g(gen_test_2_cached(ctx, slot_1, slot_2, escape_label));
1483 flag_set(ctx, slot_1, false);
1484 flag_set(ctx, slot_2, false);
1485 flag_set(ctx, slot_r, false);
1486 if (flags & OPCODE_FLAG_FUSED) {
1487 g(gen_fused_binary(ctx, MODE_REAL, type, op, escape_label, slot_1, slot_2, slot_r, &failed));
1488 if (unlikely(!failed))
1489 continue;
1490 }
1491 g(gen_fp_alu(ctx, type, op, escape_label, slot_1, slot_2, slot_r));
1492 continue;
1493 } else if (op < OPCODE_REAL_OP_N) {
1494 get_two(ctx, &slot_1, &slot_r);
1495 get_one(ctx, &flags);
1496 escape_label = alloc_escape_label(ctx);
1497 if (unlikely(!escape_label))
1498 return false;
1499 g(gen_test_1_cached(ctx, slot_1, escape_label));
1500 flag_set(ctx, slot_1, false);
1501 flag_set(ctx, slot_r, false);
1502 g(gen_fp_alu1(ctx, type, op, escape_label, slot_1, slot_r));
1503 continue;
1504 } else if (op == OPCODE_REAL_OP_ldc) {
1505 const struct type *t;
1506 unsigned i;
1507 get_one(ctx, &slot_r);
1508 t = type_get_real(type);
1509 g(gen_real_constant(ctx, t, slot_r));
1510 for (i = 0; i < t->size; i += 2)
1511 get_code(ctx);
1512 flag_set(ctx, slot_r, false);
1513 continue;
1514 } else if (op == OPCODE_REAL_OP_move || op == OPCODE_REAL_OP_copy) {
1515 get_two(ctx, &slot_1, &slot_r);
1516 escape_label = alloc_escape_label(ctx);
1517 if (unlikely(!escape_label))
1518 return false;
1519 g(gen_test_1_cached(ctx, slot_1, escape_label));
1520 flag_set(ctx, slot_1, false);
1521 flag_set(ctx, slot_r, false);
1522 g(gen_memcpy_slots(ctx, slot_r, slot_1));
1523 continue;
1524 } else {
1525 internal(file_line, "gen_function: bad real code %04x", *ctx->instr_start);
1526 }
1527 } else if (code >= OPCODE_BOOL_OP && code < OPCODE_EXTRA) {
1528 code -= OPCODE_BOOL_OP;
1529 op = (code / OPCODE_BOOL_OP_MULT) % OPCODE_BOOL_TYPE_MULT;
1530 type = log_2(sizeof(ajla_flat_option_t));
1531 if (op < OPCODE_BOOL_OP_UNARY) {
1532 get_two(ctx, &slot_1, &slot_2);
1533 get_two(ctx, &slot_r, &flags);
1534 escape_label = alloc_escape_label(ctx);
1535 if (unlikely(!escape_label))
1536 return false;
1537 g(gen_test_2_cached(ctx, slot_1, slot_2, escape_label));
1538 flag_set(ctx, slot_1, false);
1539 flag_set(ctx, slot_2, false);
1540 flag_set(ctx, slot_r, false);
1541 if (flags & OPCODE_FLAG_FUSED) {
1542 g(gen_fused_binary(ctx, MODE_BOOL, type, op, escape_label, slot_1, slot_2, slot_r, &failed));
1543 if (unlikely(!failed))
1544 continue;
1545 }
1546 g(gen_alu(ctx, MODE_BOOL, type, op, escape_label, slot_1, slot_2, slot_r));
1547 continue;
1548 } else if (op < OPCODE_BOOL_OP_N) {
1549 get_two(ctx, &slot_1, &slot_r);
1550 get_one(ctx, &flags);
1551 escape_label = alloc_escape_label(ctx);
1552 if (unlikely(!escape_label))
1553 return false;
1554 g(gen_test_1_cached(ctx, slot_1, escape_label));
1555 flag_set(ctx, slot_1, false);
1556 flag_set(ctx, slot_r, false);
1557 g(gen_alu1(ctx, MODE_BOOL, type, op, escape_label, slot_1, slot_r));
1558 continue;
1559 } else if (op == OPCODE_BOOL_OP_move || op == OPCODE_BOOL_OP_copy) {
1560 get_two(ctx, &slot_1, &slot_r);
1561 escape_label = alloc_escape_label(ctx);
1562 if (unlikely(!escape_label))
1563 return false;
1564 g(gen_test_1_cached(ctx, slot_1, escape_label));
1565 flag_set(ctx, slot_1, false);
1566 flag_set(ctx, slot_r, false);
1567 g(gen_copy(ctx, type, slot_1, slot_r));
1568 continue;
1569 } else {
1570 internal(file_line, "gen_function: bad boolean code %04x", *ctx->instr_start);
1571 }
1572 } else switch (code) {
1573 case OPCODE_INT_LDC_LONG: {
1574 uint32_t words, w;
1575 get_one(ctx, &slot_r);
1576 words = get_uint32(ctx);
1577 for (w = 0; w < words; w++)
1578 get_code(ctx);
1579 unconditional_escape:
1580 escape_label = alloc_escape_label(ctx);
1581 if (unlikely(!escape_label))
1582 return false;
1583 gen_insn(INSN_JMP, 0, 0, 0);
1584 gen_four(escape_label);
1585 continue;
1586 }
1587 case OPCODE_IS_EXCEPTION: {
1588 get_two(ctx, &slot_1, &slot_r);
1589 get_one(ctx, &flags);
1590 g(gen_is_exception(ctx, slot_1, slot_r));
1591 continue;
1592 }
1593 case OPCODE_EXCEPTION_CLASS:
1594 case OPCODE_EXCEPTION_TYPE:
1595 case OPCODE_EXCEPTION_AUX: {
1596 get_two(ctx, &slot_1, &slot_r);
1597 get_one(ctx, &flags);
1598 goto unconditional_escape;
1599 }
1600 case OPCODE_SYSTEM_PROPERTY: {
1601 get_two(ctx, &slot_1, &slot_r);
1602 get_one(ctx, &flags);
1603 g(gen_system_property(ctx, slot_1, slot_r));
1604 continue;
1605 }
1606 case OPCODE_FLAT_MOVE:
1607 case OPCODE_FLAT_COPY: {
1608 get_two(ctx, &slot_1, &slot_r);
1609 g(gen_flat_move_copy(ctx, slot_1, slot_r));
1610 continue;
1611 }
1612 case OPCODE_REF_MOVE:
1613 case OPCODE_REF_MOVE_CLEAR:
1614 case OPCODE_REF_COPY: {
1615 get_two(ctx, &slot_1, &slot_r);
1616 g(gen_ref_move_copy(ctx, code, slot_1, slot_r));
1617 continue;
1618 }
1619 case OPCODE_BOX_MOVE_CLEAR:
1620 case OPCODE_BOX_COPY: {
1621 get_two(ctx, &slot_1, &slot_r);
1622 g(gen_box_move_copy(ctx, code, slot_1, slot_r));
1623 continue;
1624 }
1625 case OPCODE_TAKE_BORROWED:
1626 get_one(ctx, &slot_1);
1627 if (!da(ctx->fn,function)->local_variables_flags[slot_1].may_be_borrowed)
1628 continue;
1629 if (unlikely(!(label_id = alloc_label(ctx))))
1630 return false;
1631 if (flag_is_set(ctx, slot_1))
1632 goto take_borrowed_done;
1633 if (flag_is_clear(ctx, slot_1)) {
1634 g(gen_set_1(ctx, R_FRAME, slot_1, 0, true));
1635 goto do_take_borrowed;
1636 }
1637 g(gen_test_1(ctx, R_FRAME, slot_1, 0, label_id, false, TEST_SET));
1638 do_take_borrowed:
1639 g(gen_upcall_start(ctx, 1));
1640 g(gen_frame_load(ctx, OP_SIZE_SLOT, garbage, slot_1, 0, false, R_ARG0));
1641 g(gen_upcall_argument(ctx, 0));
1642 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_pointer_reference_owned), 1));
1643 flag_set(ctx, slot_1, true);
1644 take_borrowed_done:
1645 gen_label(label_id);
1646 continue;
1647 case OPCODE_DEREFERENCE:
1648 case OPCODE_DEREFERENCE_CLEAR: {
1649 bool need_bit_test;
1650 /*const struct type *type;*/
1651 get_one(ctx, &slot_1);
1652 if (flag_is_clear(ctx, slot_1))
1653 goto skip_dereference;
1654 /*type = get_type_of_local(ctx, slot_1);*/
1655 /*need_bit_test = 1 || TYPE_IS_FLAT(type) || da(ctx->fn,function)->local_variables[slot_1].may_be_borrowed;*/
1656 need_bit_test = !flag_is_set(ctx, slot_1);
1657 if (need_bit_test) {
1658 if (unlikely(!(label_id = alloc_label(ctx))))
1659 return false;
1660 g(gen_test_1(ctx, R_FRAME, slot_1, 0, label_id, true, TEST_CLEAR));
1661 } else {
1662 g(gen_set_1(ctx, R_FRAME, slot_1, 0, false));
1663 label_id = 0; /* avoid warning */
1664 }
1665 g(gen_upcall_start(ctx, 1));
1666 g(gen_frame_load(ctx, OP_SIZE_SLOT, garbage, slot_1, 0, false, R_ARG0));
1667 g(gen_upcall_argument(ctx, 0));
1668 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_pointer_dereference), 1));
1669 if (need_bit_test)
1670 gen_label(label_id);
1671 skip_dereference:
1672 if (code == OPCODE_DEREFERENCE_CLEAR)
1673 g(gen_frame_clear(ctx, OP_SIZE_SLOT, slot_1));
1674 flag_set_unknown(ctx, slot_1);
1675 flag_set(ctx, slot_1, false);
1676 continue;
1677 }
1678 case OPCODE_EVAL: {
1679 get_one(ctx, &slot_1);
1680 g(gen_eval(ctx, slot_1));
1681 continue;
1682 }
1683 case OPCODE_ESCAPE_NONFLAT: {
1684 frame_t n, i;
1685 frame_t *vars;
1686
1687 get_one(ctx, &n);
1688 vars = mem_alloc_array_mayfail(mem_alloc_mayfail, frame_t *, 0, 0, n, sizeof(frame_t), &ctx->err);
1689 if (unlikely(!vars))
1690 return false;
1691 for (i = 0; i < n; i++) {
1692 get_one(ctx, &vars[i]);
1693 }
1694
1695 escape_label = alloc_escape_label(ctx);
1696 if (unlikely(!escape_label)) {
1697 mem_free(vars);
1698 return false;
1699 }
1700
1701 if (unlikely(!gen_test_variables(ctx, vars, n, escape_label))) {
1702 mem_free(vars);
1703 return false;
1704 }
1705 mem_free(vars);
1706
1707 continue;
1708 }
1709 case OPCODE_CHECKPOINT: {
1710 frame_t n_vars;
1711
1712 g(clear_flag_cache(ctx));
1713
1714 if (SIZEOF_IP_T == 2) {
1715 slot_1 = get_code(ctx);
1716 } else if (SIZEOF_IP_T == 4) {
1717 slot_1 = get_uint32(ctx);
1718 } else {
1719 not_reached();
1720 continue;
1721 }
1722
1723 if (unlikely(!(slot_1 + 1)))
1724 return false;
1725 while (slot_1 >= ctx->n_entries) {
1726 void *err_entries;
1727 struct cg_entry e;
1728 if (unlikely(!ctx->entries)) {
1729 if (unlikely(!array_init_mayfail(struct cg_entry, &ctx->entries, &ctx->n_entries, &ctx->err)))
1730 return false;
1731 }
1732 memset(&e, 0, sizeof(struct cg_entry));
1733 if (unlikely(!array_add_mayfail(struct cg_entry, &ctx->entries, &ctx->n_entries, e, &err_entries, &ctx->err))) {
1734 ctx->entries = err_entries;
1735 return false;
1736 }
1737 }
1738
1739 get_one(ctx, &n_vars);
1740
1741 escape_label = 0; /* avoid warning */
1742 if (likely(slot_1 != 0)) {
1743 escape_label = alloc_escape_label(ctx);
1744 if (unlikely(!escape_label))
1745 return false;
1746 }
1747
1748 if (n_vars || !slot_1) {
1749 frame_t i;
1750 uint32_t entry_label, nonflat_label;
1751 struct cg_entry *ce = &ctx->entries[slot_1];
1752
1753 if (unlikely(!array_init_mayfail(frame_t, &ce->variables, &ce->n_variables, &ctx->err)))
1754 return false;
1755 for (i = 0; i < n_vars; i++) {
1756 frame_t v;
1757 get_one(ctx, &v);
1758 if (unlikely(!array_add_mayfail(frame_t, &ce->variables, &ce->n_variables, v, NULL, &ctx->err)))
1759 return false;
1760 }
1761 if (!slot_1) {
1762 g(gen_test_variables(ctx, ce->variables, ce->n_variables, ctx->escape_nospill_label));
1763 }
1764 entry_label = alloc_label(ctx);
1765 if (unlikely(!entry_label))
1766 return false;
1767 gen_label(entry_label);
1768 ce->entry_label = entry_label;
1769
1770 nonflat_label = alloc_escape_label_for_ip(ctx, ctx->current_position);
1771 if (unlikely(!nonflat_label))
1772 return false;
1773 ce->nonflat_label = nonflat_label;
1774
1775 if (unlikely(!slot_1))
1776 g(gen_timestamp_test(ctx, ctx->escape_nospill_label));
1777 else
1778 g(gen_timestamp_test(ctx, escape_label));
1779 } else {
1780 g(gen_timestamp_test(ctx, escape_label));
1781
1782 gen_insn(INSN_ENTRY, 0, 0, 0);
1783 gen_four(slot_1);
1784 }
1785 continue;
1786 }
1787 case OPCODE_JMP: {
1788 int32_t x = get_jump_offset(ctx);
1789 g(gen_jump(ctx, x, OP_SIZE_NATIVE, COND_ALWAYS, -1U, -1U));
1790 continue;
1791 }
1792 case OPCODE_JMP_BACK_16: {
1793 int32_t x = get_code(ctx);
1794 g(gen_jump(ctx, -x - (int)(2 * sizeof(code_t)), OP_SIZE_NATIVE, COND_ALWAYS, -1U, -1U));
1795 continue;
1796 }
1797 case OPCODE_JMP_FALSE: {
1798 int32_t offs_false;
1799 get_one(ctx, &slot_1);
1800 offs_false = get_jump_offset(ctx);
1801 get_jump_offset(ctx);
1802 escape_label = alloc_escape_label(ctx);
1803 if (unlikely(!escape_label))
1804 return false;
1805 g(gen_test_1_cached(ctx, slot_1, escape_label));
1806 flag_set(ctx, slot_1, false);
1807 g(gen_cond_jump(ctx, slot_1, offs_false));
1808 continue;
1809 }
1810 case OPCODE_LABEL: {
1811 g(clear_flag_cache(ctx));
1812 continue;
1813 }
1814 #define init_args \
1815 do { \
1816 if (ctx->args != NULL) \
1817 mem_free(ctx->args); \
1818 g(array_init_mayfail(struct code_arg, &ctx->args, &ctx->args_l, &ctx->err));\
1819 } while (0)
1820 #define load_args \
1821 do { \
1822 init_args; \
1823 for (i_arg = 0; i_arg < n_args; i_arg++) { \
1824 struct code_arg a; \
1825 get_two(ctx, &a.slot, &a.flags); \
1826 a.type = 0; \
1827 g(array_add_mayfail(struct code_arg, &ctx->args, &ctx->args_l, a, NULL, &ctx->err));\
1828 } \
1829 } while (0)
1830 case OPCODE_LOAD_FN:
1831 get_two(ctx, &n_args, &slot_r);
1832 get_one(ctx, &fn_idx);
1833 load_args;
1834 g(gen_load_fn_or_curry(ctx, fn_idx, NO_FRAME_T, slot_r, 0));
1835 continue;
1836 case OPCODE_CURRY:
1837 get_two(ctx, &n_args, &slot_r);
1838 get_two(ctx, &slot_1, &flags);
1839 load_args;
1840 g(gen_load_fn_or_curry(ctx, NO_FRAME_T, slot_1, slot_r, flags));
1841 continue;
1842 case OPCODE_CALL:
1843 case OPCODE_CALL_STRICT:
1844 case OPCODE_CALL_SPARK:
1845 case OPCODE_CALL_LAZY:
1846 case OPCODE_CALL_CACHE:
1847 case OPCODE_CALL_SAVE: {
1848 get_two(ctx, &n_args, &n_ret);
1849 get_one(ctx, &fn_idx);
1850 jump_over_arguments_and_return:
1851 load_args;
1852 ctx->return_values = ctx->current_position;
1853 for (i_arg = 0; i_arg < n_ret; i_arg++) {
1854 #if ARG_MODE_N >= 3
1855 get_uint32(ctx);
1856 #else
1857 get_code(ctx);
1858 #endif
1859 get_code(ctx);
1860 }
1861 if (unlikely(profiling))
1862 goto unconditional_escape;
1863 if (code == OPCODE_CALL || code == OPCODE_CALL_STRICT) {
1864 g(gen_call(ctx, code, fn_idx));
1865 continue;
1866 }
1867 /*if (code == OPCODE_CALL_INDIRECT || code == OPCODE_CALL_INDIRECT_STRICT) {
1868 if (unlikely(!gen_call_indirect(ctx, code, slot_1, flags)))
1869 return false;
1870 continue;
1871 }*/
1872 goto unconditional_escape;
1873 }
1874 case OPCODE_CALL_INDIRECT:
1875 case OPCODE_CALL_INDIRECT_STRICT:
1876 case OPCODE_CALL_INDIRECT_SPARK:
1877 case OPCODE_CALL_INDIRECT_LAZY:
1878 case OPCODE_CALL_INDIRECT_CACHE:
1879 case OPCODE_CALL_INDIRECT_SAVE: {
1880 fn_idx = 0; /* avoid warning */
1881 get_two(ctx, &n_args, &n_ret);
1882 get_two(ctx, &slot_1, &flags);
1883 goto jump_over_arguments_and_return;
1884 }
1885 case OPCODE_RETURN: {
1886 n_args = da(ctx->fn,function)->n_return_values;
1887 load_args;
1888 if (unlikely(profiling))
1889 goto unconditional_escape;
1890 g(gen_return(ctx));
1891 continue;
1892 }
1893 case OPCODE_STRUCTURED: {
1894 init_args;
1895 get_two(ctx, &slot_1, &slot_2);
1896 do {
1897 struct code_arg a;
1898 get_two(ctx, &flags, &slot_r);
1899 get_one(ctx, &opt);
1900 a.slot = slot_r;
1901 a.flags = flags;
1902 a.type = opt;
1903 g(array_add_mayfail(struct code_arg, &ctx->args, &ctx->args_l, a, NULL, &ctx->err));
1904 } while (!(flags & OPCODE_STRUCTURED_FLAG_END));
1905 g(gen_structured(ctx, slot_1, slot_2));
1906 continue;
1907 }
1908 case OPCODE_RECORD_CREATE: {
1909 init_args;
1910 get_two(ctx, &slot_r, &n_args);
1911 for (i_arg = 0; i_arg < n_args; i_arg++) {
1912 struct code_arg a;
1913 get_two(ctx, &slot_1, &flags);
1914 a.slot = slot_1;
1915 a.flags = flags;
1916 a.type = 0;
1917 g(array_add_mayfail(struct code_arg, &ctx->args, &ctx->args_l, a, NULL, &ctx->err));
1918 }
1919 g(gen_record_create(ctx, slot_r));
1920 continue;
1921 }
1922 case OPCODE_RECORD_LOAD: {
1923 get_two(ctx, &slot_1, &opt);
1924 get_two(ctx, &slot_r, &flags);
1925 g(gen_record_load(ctx, slot_1, slot_r, opt, flags));
1926 continue;
1927 }
1928 case OPCODE_OPTION_CREATE_EMPTY_FLAT: {
1929 get_two(ctx, &slot_r, &opt);
1930 g(gen_option_create_empty_flat(ctx, opt, slot_r));
1931 continue;
1932 }
1933 case OPCODE_OPTION_CREATE_EMPTY: {
1934 get_two(ctx, &slot_r, &opt);
1935 g(gen_option_create_empty(ctx, opt, slot_r));
1936 continue;
1937 }
1938 case OPCODE_OPTION_CREATE: {
1939 get_two(ctx, &slot_r, &opt);
1940 get_two(ctx, &slot_1, &flags);
1941 g(gen_option_create(ctx, opt, slot_1, slot_r, flags));
1942 continue;
1943 }
1944 case OPCODE_OPTION_LOAD: {
1945 get_two(ctx, &slot_1, &opt);
1946 get_two(ctx, &slot_r, &flags);
1947 g(gen_option_load(ctx, slot_1, slot_r, opt, flags));
1948 continue;
1949 }
1950 case OPCODE_OPTION_TEST_FLAT: {
1951 get_two(ctx, &slot_1, &opt);
1952 get_one(ctx, &slot_r);
1953 g(gen_option_test_flat(ctx, slot_1, opt, slot_r));
1954 continue;
1955 }
1956 case OPCODE_OPTION_TEST: {
1957 get_two(ctx, &slot_1, &opt);
1958 get_one(ctx, &slot_r);
1959 g(gen_option_test(ctx, slot_1, opt, slot_r));
1960 continue;
1961 }
1962 case OPCODE_OPTION_ORD_FLAT: {
1963 get_two(ctx, &slot_1, &slot_r);
1964 g(gen_option_ord(ctx, slot_1, slot_r, true));
1965 continue;
1966 }
1967 case OPCODE_OPTION_ORD: {
1968 get_two(ctx, &slot_1, &slot_r);
1969 g(gen_option_ord(ctx, slot_1, slot_r, false));
1970 continue;
1971 }
1972 case OPCODE_ARRAY_CREATE: {
1973 init_args;
1974 get_two(ctx, &slot_r, &n_args);
1975 for (i_arg = 0; i_arg < n_args; i_arg++) {
1976 struct code_arg a;
1977 get_two(ctx, &slot_1, &flags);
1978 a.slot = slot_1;
1979 a.flags = flags;
1980 a.type = 0;
1981 g(array_add_mayfail(struct code_arg, &ctx->args, &ctx->args_l, a, NULL, &ctx->err));
1982 }
1983 g(gen_array_create(ctx, slot_r));
1984 continue;
1985 }
1986 case OPCODE_ARRAY_CREATE_EMPTY_FLAT: {
1987 get_two(ctx, &slot_r, &flags);
1988 g(gen_array_create_empty_flat(ctx, slot_r, flags));
1989 continue;
1990 }
1991 case OPCODE_ARRAY_CREATE_EMPTY: {
1992 get_one(ctx, &slot_r);
1993 g(gen_array_create_empty(ctx, slot_r));
1994 continue;
1995 }
1996 case OPCODE_ARRAY_FILL: {
1997 get_two(ctx, &slot_1, &flags);
1998 get_two(ctx, &slot_2, &slot_r);
1999 g(gen_array_fill(ctx, slot_1, flags, slot_2, slot_r));
2000 continue;
2001 }
2002 case OPCODE_ARRAY_STRING: {
2003 frame_t i;
2004 get_two(ctx, &slot_r, &i);
2005 g(gen_array_string(ctx, type_get_fixed(0, true)->tag, cast_ptr(uint8_t *, ctx->current_position), i, slot_r));
2006 ctx->current_position += (i + 1) >> 1;
2007 continue;
2008 }
2009 case OPCODE_ARRAY_UNICODE: {
2010 frame_t i;
2011 get_two(ctx, &slot_r, &i);
2012 g(gen_array_string(ctx, type_get_int(2)->tag, cast_ptr(uint8_t *, ctx->current_position), i, slot_r));
2013 ctx->current_position += i * 2;
2014 continue;
2015 }
2016 case OPCODE_ARRAY_LOAD: {
2017 get_two(ctx, &slot_1, &slot_2);
2018 get_two(ctx, &slot_r, &flags);
2019 g(gen_array_load(ctx, slot_1, slot_2, slot_r, flags));
2020 continue;
2021 }
2022 case OPCODE_ARRAY_LEN: {
2023 get_two(ctx, &slot_1, &slot_r);
2024 get_one(ctx, &flags);
2025 g(gen_array_len(ctx, slot_1, NO_FRAME_T, slot_r, false, 0));
2026 continue;
2027 }
2028 case OPCODE_ARRAY_LEN_GREATER_THAN: {
2029 get_two(ctx, &slot_1, &slot_2);
2030 get_two(ctx, &slot_r, &flags);
2031 escape_label = alloc_escape_label(ctx);
2032 if (unlikely(!escape_label))
2033 return false;
2034 if (flags & OPCODE_FLAG_FUSED) {
2035 g(gen_fused_binary(ctx, MODE_ARRAY_LEN_GT, 0, 0, escape_label, slot_1, slot_2, slot_r, &failed));
2036 if (unlikely(!failed))
2037 continue;
2038 }
2039 g(gen_array_len(ctx, slot_1, slot_2, slot_r, false, 0));
2040 continue;
2041 }
2042 case OPCODE_ARRAY_SUB: {
2043 get_two(ctx, &slot_1, &slot_2);
2044 get_two(ctx, &slot_3, &slot_r);
2045 get_one(ctx, &flags);
2046 g(gen_array_sub(ctx, slot_1, slot_2, slot_3, slot_r, flags));
2047 continue;
2048 }
2049 case OPCODE_ARRAY_SKIP: {
2050 get_two(ctx, &slot_1, &slot_2);
2051 get_two(ctx, &slot_r, &flags);
2052 g(gen_array_skip(ctx, slot_1, slot_2, slot_r, flags));
2053 continue;
2054 }
2055 case OPCODE_ARRAY_APPEND: {
2056 get_two(ctx, &slot_r, &flags);
2057 get_two(ctx, &slot_1, &slot_2);
2058 g(gen_array_append(ctx, slot_1, slot_2, slot_r, flags));
2059 continue;
2060 }
2061 case OPCODE_ARRAY_APPEND_ONE_FLAT: {
2062 get_two(ctx, &slot_r, &flags);
2063 get_two(ctx, &slot_1, &slot_2);
2064 g(gen_array_append_one_flat(ctx, slot_1, slot_2, slot_r, flags));
2065 continue;
2066 }
2067 case OPCODE_ARRAY_APPEND_ONE: {
2068 get_two(ctx, &slot_r, &flags);
2069 get_two(ctx, &slot_1, &slot_2);
2070 g(gen_array_append_one(ctx, slot_1, slot_2, slot_r, flags));
2071 continue;
2072 }
2073 case OPCODE_ARRAY_FLATTEN: {
2074 get_two(ctx, &slot_r, &flags);
2075 get_one(ctx, &slot_1);
2076 goto unconditional_escape;
2077 }
2078 case OPCODE_IO: {
2079 get_two(ctx, &flags, &slot_1);
2080 get_two(ctx, &slot_2, &slot_3);
2081 g(gen_io(ctx, flags, slot_1, slot_2, slot_3));
2082 continue;
2083 }
2084 case OPCODE_INTERNAL_FUNCTION:
2085 case OPCODE_EXIT_THREAD:
2086 case OPCODE_UNREACHABLE: {
2087 goto unconditional_escape;
2088 }
2089 default: {
2090 #if 1
2091 /*if (getenv("DUMP") && !strcmp(da(ctx->fn,function)->function_name, getenv("DUMP")))*/
2092 warning("gen_function: %s: unknown opcode %04x, %s", da(ctx->fn,function)->function_name, *ctx->instr_start, decode_opcode(*ctx->instr_start, false));
2093 #endif
2094 return false;
2095 }
2096 }
2097 }
2098
2099 return true;
2100 }
2101
2102 static bool attr_w gen_entries(struct codegen_context *ctx)
2103 {
2104 size_t i;
2105 for (i = 0; i < ctx->n_entries; i++) {
2106 struct cg_entry *ce = &ctx->entries[i];
2107 if (ce->entry_label) {
2108 gen_insn(INSN_ENTRY, 0, 0, 0);
2109 gen_four(i);
2110
2111 g(gen_test_variables(ctx, ce->variables, ce->n_variables, ce->nonflat_label));
2112
2113 gen_insn(INSN_JMP, 0, 0, 0);
2114 gen_four(ce->entry_label);
2115 }
2116 }
2117 return true;
2118 }
2119
2120 static bool attr_w gen_epilogues(struct codegen_context *ctx)
2121 {
2122 frame_t v;
2123 ip_t ip;
2124 uint32_t escape_label, nospill_label;
2125 escape_label = alloc_label(ctx);
2126 if (unlikely(!escape_label))
2127 return false;
2128 nospill_label = alloc_label(ctx);
2129 if (unlikely(!nospill_label))
2130 return false;
2131 #if defined(ARCH_PARISC)
2132 if (ctx->call_label) {
2133 gen_label(ctx->call_label);
2134 g(gen_call_millicode(ctx));
2135 }
2136 #endif
2137 if (ctx->reload_label) {
2138 gen_label(ctx->reload_label);
2139 g(gen_mov(ctx, i_size(OP_SIZE_ADDRESS), R_FRAME, R_RET0));
2140 g(gen_escape_arg(ctx, (ip_t)-1, nospill_label));
2141 }
2142 gen_label(ctx->escape_nospill_label);
2143 g(gen_escape_arg(ctx, 0, nospill_label));
2144 for (ip = 0; ip < da(ctx->fn,function)->code_size; ip++) {
2145 struct cg_exit *ce = ctx->code_exits[ip];
2146 if (ce && (ce->undo_label || ce->escape_label)) {
2147 if (ce->undo_label) {
2148 size_t i;
2149 gen_label(ce->undo_label);
2150 gen_insn(ce->undo_opcode, ce->undo_op_size, ce->undo_aux, ce->undo_writes_flags);
2151 for (i = 0; i < ce->undo_parameters_len; i++)
2152 gen_one(ce->undo_parameters[i]);
2153 }
2154 if (ce->escape_label) {
2155 gen_label(ce->escape_label);
2156 }
2157 g(gen_escape_arg(ctx, ip, escape_label));
2158 }
2159 }
2160 gen_label(escape_label);
2161 for (v = MIN_USEABLE_SLOT; v < function_n_variables(ctx->fn); v++) {
2162 if (slot_is_register(ctx, v)) {
2163 g(spill(ctx, v));
2164 }
2165 }
2166 gen_label(nospill_label);
2167 g(gen_escape(ctx));
2168 return true;
2169 }
2170
2171 static bool attr_w cgen_entry(struct codegen_context *ctx)
2172 {
2173 uint32_t entry_id = cget_four(ctx);
2174 ajla_assert_lo(entry_id < ctx->n_entries, (file_line, "cgen_entry: invalid entry %lx", (unsigned long)entry_id));
2175 ctx->entries[entry_id].entry_to_pos = ctx->mcode_size;
2176 return true;
2177 }
2178
2179 static bool attr_w cgen_label(struct codegen_context *ctx)
2180 {
2181 uint32_t label_id = cget_four(ctx);
2182 ctx->label_to_pos[label_id] = ctx->mcode_size;
2183 return true;
2184 }
2185
2186 static bool attr_w attr_unused cgen_trap(struct codegen_context *ctx, uint32_t label)
2187 {
2188 struct trap_record tr;
2189 tr.source_ip = ctx->mcode_size;
2190 tr.destination_ip = label;
2191 if (unlikely(!array_add_mayfail(struct trap_record, &ctx->trap_records, &ctx->trap_records_size, tr, NULL, &ctx->err)))
2192 return false;
2193 return true;
2194 }
2195
2196 static bool attr_w add_relocation(struct codegen_context *ctx, unsigned length, int offset, bool *known)
2197 {
2198 struct relocation rel;
2199 rel.label_id = cget_four(ctx);
2200 rel.length = length;
2201 rel.position = ctx->mcode_size;
2202 rel.jmp_instr = ctx->code_position - 8 - offset - ctx->code;
2203 if (unlikely(!array_add_mayfail(struct relocation, &ctx->reloc, &ctx->reloc_size, rel, NULL, &ctx->err)))
2204 return false;
2205 if (known)
2206 *known = ctx->label_to_pos[rel.label_id] != (size_t)-1;
2207 return true;
2208 }
2209
2210
2211 #if defined(ARCH_ALPHA)
2212 #include "c2-alpha.inc"
2213 #elif defined(ARCH_ARM32)
2214 #include "c2-arm.inc"
2215 #elif defined(ARCH_ARM64)
2216 #include "c2-arm64.inc"
2217 #elif defined(ARCH_IA64)
2218 #include "c2-ia64.inc"
2219 #elif defined(ARCH_LOONGARCH64)
2220 #include "c2-loong.inc"
2221 #elif defined(ARCH_MIPS)
2222 #include "c2-mips.inc"
2223 #elif defined(ARCH_PARISC)
2224 #include "c2-hppa.inc"
2225 #elif defined(ARCH_POWER)
2226 #include "c2-power.inc"
2227 #elif defined(ARCH_S390)
2228 #include "c2-s390.inc"
2229 #elif defined(ARCH_SPARC)
2230 #include "c2-sparc.inc"
2231 #elif defined(ARCH_RISCV64)
2232 #include "c2-riscv.inc"
2233 #elif defined(ARCH_X86)
2234 #include "c2-x86.inc"
2235 #endif
2236
2237
2238 static bool attr_w gen_mcode(struct codegen_context *ctx)
2239 {
2240 ctx->code_position = ctx->code;
2241
2242 while (ctx->code_position != ctx->code + ctx->code_size) {
2243 uint32_t insn;
2244 ajla_assert_lo(ctx->code_position < ctx->code + ctx->code_size, (file_line, "gen_mcode: ran out of code"));
2245 #ifdef DEBUG_INSNS
2246 insn = cget_four(ctx);
2247 debug("line: %u/%u", insn >> 24, insn & 0x00FFFFFFU);
2248 #endif
2249 insn = cget_four(ctx);
2250 g(cgen_insn(ctx, insn));
2251 }
2252
2253 return true;
2254 }
2255
2256 #define RELOCS_RETRY -1
2257 #define RELOCS_FAIL 0
2258 #define RELOCS_OK 1
2259
2260 static int8_t resolve_relocs(struct codegen_context *ctx)
2261 {
2262 size_t i;
2263 int8_t status = RELOCS_OK;
2264 for (i = 0; i < ctx->reloc_size; i++) {
2265 struct relocation *reloc = &ctx->reloc[i];
2266 if (!resolve_relocation(ctx, reloc)) {
2267 uint8_t *jmp_instr;
2268 uint32_t insn;
2269 uint32_t new_length;
2270 status = RELOCS_RETRY;
2271 if (unlikely(reloc->length + zero >= JMP_LIMIT))
2272 return RELOCS_FAIL;
2273 new_length = reloc->length + 1;
2274 jmp_instr = ctx->code + reloc->jmp_instr;
2275 insn = (uint32_t)jmp_instr[0] +
2276 ((uint32_t)jmp_instr[1] << 8) +
2277 ((uint32_t)jmp_instr[2] << 16) +
2278 ((uint32_t)jmp_instr[3] << 24);
2279 insn &= ~INSN_JUMP_SIZE;
2280 insn |= (uint32_t)new_length << INSN_JUMP_SIZE_SHIFT;
2281 jmp_instr[0] = insn;
2282 jmp_instr[1] = insn >> 8;
2283 jmp_instr[2] = insn >> 16;
2284 jmp_instr[3] = insn >> 24;
2285 }
2286 }
2287 return status;
2288 }
2289
2290 static void resolve_traps(struct codegen_context *ctx)
2291 {
2292 size_t i;
2293 for (i = 0; i < ctx->trap_records_size; i++) {
2294 struct trap_record *tr = &ctx->trap_records[i];
2295 tr->destination_ip = ctx->label_to_pos[tr->destination_ip];
2296 }
2297 }
2298
2299
2300 static bool attr_w codegen_map(struct codegen_context *ctx)
2301 {
2302 void *ptr;
2303 frame_t i;
2304 array_finish(uint8_t, &ctx->mcode, &ctx->mcode_size);
2305 ptr = os_code_map(ctx->mcode, ctx->mcode_size, &ctx->err);
2306 ctx->mcode = NULL;
2307 if (unlikely(!ptr)) {
2308 return false;
2309 }
2310 for (i = 0; i < ctx->n_entries; i++) {
2311 char *entry = cast_ptr(char *, ptr) + ctx->entries[i].entry_to_pos;
2312 da(ctx->codegen,codegen)->unoptimized_code[i] = entry;
2313 da(ctx->codegen,codegen)->n_entries++;
2314 }
2315 da(ctx->codegen,codegen)->unoptimized_code_base = ptr;
2316 da(ctx->codegen,codegen)->unoptimized_code_size = ctx->mcode_size;
2317
2318 return true;
2319 }
2320
2321
2322 void *codegen_fn(frame_s *fp, const code_t *ip, union internal_arg ia[])
2323 {
2324 struct codegen_context ctx_;
2325 struct codegen_context *ctx = &ctx_;
2326 frame_t i;
2327 int8_t rr;
2328 struct data *codegen;
2329 uint32_t l;
2330
2331 init_ctx(ctx);
2332 ctx->fn = ia[0].ptr;
2333
2334 #ifdef DEBUG_ENV
2335 if (getenv("CG") && strcmp(da(ctx->fn,function)->function_name, getenv("CG")))
2336 goto fail;
2337 #endif
2338
2339 ctx->local_directory = mem_alloc_array_mayfail(mem_calloc_mayfail, struct data **, 0, 0, da(ctx->fn,function)->local_directory_size, sizeof(struct data *), &ctx->err);
2340 if (unlikely(!ctx->local_directory))
2341 goto fail;
2342
2343 if (0) for (i = 0; i < da(ctx->fn,function)->local_directory_size; i++) {
2344 struct data *callee;
2345 pointer_t *ptr;
2346 ptr = da(ctx->fn,function)->local_directory[i];
2347 pointer_follow(ptr, false, callee, PF_SPARK, NULL, 0,
2348 SUBMIT_EX(ex_);
2349 goto next_one,
2350 goto next_one;
2351 );
2352 ctx->local_directory[i] = callee;
2353 next_one:;
2354 }
2355 for (i = 0; i < da(ctx->fn,function)->local_directory_size; i++) {
2356 struct data *callee;
2357 pointer_t *ptr;
2358 if (ctx->local_directory[i])
2359 continue;
2360 ptr = da(ctx->fn,function)->local_directory[i];
2361 pointer_follow(ptr, false, callee, PF_WAIT, fp, ip,
2362 done_ctx(ctx);
2363 return ex_,
2364 goto fail
2365 );
2366 ctx->local_directory[i] = callee;
2367 /*debug("processing call: %s -> %s", da(ctx->fn,function)->function_name, da(callee,function)->function_name);*/
2368 }
2369
2370 if (da(ctx->fn,function)->module_designator) {
2371 struct function_descriptor *sfd = save_find_function_descriptor(da(ctx->fn,function)->module_designator, da(ctx->fn,function)->function_designator);
2372 if (sfd && sfd->unoptimized_code_size) {
2373 codegen = data_alloc_flexible(codegen, unoptimized_code, sfd->n_entries, &ctx->err);
2374 if (unlikely(!codegen))
2375 goto fail;
2376 da(codegen,codegen)->unoptimized_code_base = sfd->unoptimized_code_base;
2377 da(codegen,codegen)->unoptimized_code_size = sfd->unoptimized_code_size;
2378 da(codegen,codegen)->function = ctx->fn;
2379 da(codegen,codegen)->is_saved = true;
2380 da(codegen,codegen)->n_entries = sfd->n_entries;
2381 da(codegen,codegen)->offsets = NULL;
2382 for (i = 0; i < sfd->n_entries; i++) {
2383 da(codegen,codegen)->unoptimized_code[i] = cast_ptr(char *, da(codegen,codegen)->unoptimized_code_base) + sfd->entries[i];
2384 /*debug("%s: %p + %lx -> %p", da(ctx->fn,function)->function_name, da(codegen,codegen)->unoptimized_code_base, sfd->entries[i], da(codegen,codegen)->unoptimized_code[i]);*/
2385 }
2386 #ifdef HAVE_CODEGEN_TRAPS
2387 da(codegen,codegen)->trap_records = sfd->trap_records;
2388 da(codegen,codegen)->trap_records_size = sfd->trap_records_size;
2389 data_trap_insert(codegen);
2390 #endif
2391 goto have_codegen;
2392 }
2393 }
2394
2395 /*debug("trying: %s", da(ctx->fn,function)->function_name);*/
2396 if (unlikely(!array_init_mayfail(uint8_t, &ctx->code, &ctx->code_size, &ctx->err)))
2397 goto fail;
2398
2399 ctx->code_labels = mem_alloc_array_mayfail(mem_calloc_mayfail, uint32_t *, 0, 0, da(ctx->fn,function)->code_size, sizeof(uint32_t), &ctx->err);
2400 if (unlikely(!ctx->code_labels))
2401 goto fail;
2402
2403 ctx->code_exits = mem_alloc_array_mayfail(mem_calloc_mayfail, struct cg_exit **, 0, 0, da(ctx->fn,function)->code_size, sizeof(struct cg_exit *), &ctx->err);
2404 if (unlikely(!ctx->code_exits))
2405 goto fail;
2406
2407 ctx->flag_cache = mem_alloc_array_mayfail(mem_calloc_mayfail, uint8_t *, 0, 0, function_n_variables(ctx->fn), sizeof(int8_t), &ctx->err);
2408 if (unlikely(!ctx->flag_cache))
2409 goto fail;
2410
2411 ctx->registers = mem_alloc_array_mayfail(mem_alloc_mayfail, short *, 0, 0, function_n_variables(ctx->fn), sizeof(short), &ctx->err);
2412 if (unlikely(!ctx->registers))
2413 goto fail;
2414
2415 if (unlikely(!array_init_mayfail(frame_t, &ctx->need_spill, &ctx->need_spill_l, &ctx->err)))
2416 goto fail;
2417
2418 if (unlikely(!gen_registers(ctx)))
2419 goto fail;
2420
2421 if (unlikely(!gen_function(ctx)))
2422 goto fail;
2423
2424 if (unlikely(!gen_entries(ctx)))
2425 goto fail;
2426
2427 if (unlikely(!gen_epilogues(ctx)))
2428 goto fail;
2429
2430 if (unlikely(!(ctx->label_id + 1)))
2431 goto fail;
2432 if (unlikely(!(ctx->label_to_pos = mem_alloc_array_mayfail(mem_alloc_mayfail, size_t *, 0, 0, ctx->label_id + 1, sizeof(size_t), &ctx->err))))
2433 goto fail;
2434
2435 again:
2436 for (l = 0; l < ctx->label_id + 1; l++)
2437 ctx->label_to_pos[l] = (size_t)-1;
2438
2439 if (unlikely(!array_init_mayfail(uint8_t, &ctx->mcode, &ctx->mcode_size, &ctx->err)))
2440 goto fail;
2441
2442 if (unlikely(!array_init_mayfail(struct relocation, &ctx->reloc, &ctx->reloc_size, &ctx->err)))
2443 goto fail;
2444
2445 if (unlikely(!array_init_mayfail(struct trap_record, &ctx->trap_records, &ctx->trap_records_size, &ctx->err)))
2446 goto fail;
2447
2448 #ifdef ARCH_CONTEXT
2449 init_arch_context(ctx);
2450 #endif
2451
2452 if (unlikely(!gen_mcode(ctx)))
2453 goto fail;
2454
2455 rr = resolve_relocs(ctx);
2456 if (unlikely(rr == RELOCS_FAIL)) {
2457 /*debug("relocation fail: %s", da(ctx->fn,function)->function_name);*/
2458 goto fail;
2459 }
2460 if (rr == RELOCS_RETRY) {
2461 mem_free(ctx->mcode);
2462 ctx->mcode = NULL;
2463 mem_free(ctx->reloc);
2464 ctx->reloc = NULL;
2465 mem_free(ctx->trap_records);
2466 ctx->trap_records = NULL;
2467 goto again;
2468 }
2469
2470 resolve_traps(ctx);
2471
2472 #ifdef DEBUG_ENV
2473 if ((getenv("DUMP") && !strcmp(getenv("DUMP"), da(ctx->fn,function)->function_name)) || getenv("DUMP_ALL")) {
2474 char *hex;
2475 size_t hexl;
2476 size_t i;
2477 handle_t h;
2478
2479 mutex_lock(&dump_mutex);
2480 str_init(&hex, &hexl);
2481 str_add_string(&hex, &hexl, "_");
2482 str_add_unsigned(&hex, &hexl, dump_seq++, 10);
2483 str_add_string(&hex, &hexl, "_");
2484 str_add_string(&hex, &hexl, da(ctx->fn,function)->function_name);
2485 str_add_string(&hex, &hexl, ":");
2486 for (i = 0; i < hexl; i++)
2487 if (hex[i] == '/')
2488 hex[i] = '_';
2489 for (i = 0; i < ctx->mcode_size; i++) {
2490 uint8_t a = ctx->mcode[i];
2491 if (!(i & 0xff))
2492 str_add_string(&hex, &hexl, "\n .byte 0x");
2493 else
2494 str_add_string(&hex, &hexl, ",0x");
2495 if (a < 16)
2496 str_add_char(&hex, &hexl, '0');
2497 str_add_unsigned(&hex, &hexl, a, 16);
2498 }
2499 str_add_string(&hex, &hexl, "\n");
2500 h = os_open(os_cwd, "dump.s", O_WRONLY | O_APPEND, 0600, NULL);
2501 os_write_all(h, hex, hexl, NULL);
2502 os_close(h);
2503 mem_free(hex);
2504 mutex_unlock(&dump_mutex);
2505 }
2506 #endif
2507
2508 ctx->codegen = data_alloc_flexible(codegen, unoptimized_code, ctx->n_entries, &ctx->err);
2509 if (unlikely(!ctx->codegen))
2510 goto fail;
2511 da(ctx->codegen,codegen)->function = ctx->fn;
2512 da(ctx->codegen,codegen)->is_saved = false;
2513 da(ctx->codegen,codegen)->n_entries = 0;
2514 da(ctx->codegen,codegen)->offsets = NULL;
2515
2516 if (unlikely(!codegen_map(ctx)))
2517 goto fail;
2518
2519 codegen = ctx->codegen;
2520 ctx->codegen = NULL;
2521
2522 #ifdef HAVE_CODEGEN_TRAPS
2523 da(codegen,codegen)->trap_records = ctx->trap_records;
2524 da(codegen,codegen)->trap_records_size = ctx->trap_records_size;
2525 ctx->trap_records = NULL;
2526 data_trap_insert(codegen);
2527 #endif
2528
2529 have_codegen:
2530 done_ctx(ctx);
2531 return function_return(fp, pointer_data(codegen));
2532
2533 fail:
2534 /*debug("FAILED: %s", da(ctx->fn,function)->function_name);*/
2535 done_ctx(ctx);
2536 return function_return(fp, pointer_thunk(thunk_alloc_exception_error(error_ajla(EC_SYNC, AJLA_ERROR_NOT_SUPPORTED), NULL, NULL, NULL pass_file_line)));
2537 }
2538
2539 void codegen_free(struct data *codegen)
2540 {
2541 if (unlikely(da(codegen,codegen)->offsets != NULL))
2542 mem_free(da(codegen,codegen)->offsets);
2543 if (likely(da(codegen,codegen)->is_saved))
2544 return;
2545 #ifdef HAVE_CODEGEN_TRAPS
2546 mem_free(da(codegen,codegen)->trap_records);
2547 #endif
2548 os_code_unmap(da(codegen,codegen)->unoptimized_code_base, da(codegen,codegen)->unoptimized_code_size);
2549 }
2550
2551 #if defined(ARCH_IA64)
2552 static uintptr_t ia64_stub[2];
2553 #endif
2554 #if defined(ARCH_PARISC32) && defined(ARCH_PARISC_USE_STUBS)
2555 static uintptr_t parisc_stub[2];
2556 #endif
2557 #if defined(ARCH_PARISC64) && defined(ARCH_PARISC_USE_STUBS)
2558 static uintptr_t parisc_stub[4];
2559 #endif
2560 #if defined(ARCH_POWER) && defined(AIX_CALL)
2561 static uintptr_t ppc_stub[3];
2562 #endif
2563
2564 void name(codegen_init)(void)
2565 {
2566 struct codegen_context ctx_;
2567 struct codegen_context *ctx = &ctx_;
2568 void *ptr;
2569
2570 #if (defined(ARCH_X86_64) || defined(ARCH_X86_X32)) && !defined(ARCH_X86_WIN_ABI)
2571 #if defined(HAVE_SYSCALL) && defined(HAVE_ASM_PRCTL_H) && defined(HAVE_SYS_SYSCALL_H)
2572 if (!dll) {
2573 int r;
2574 EINTR_LOOP(r, syscall(SYS_arch_prctl, ARCH_SET_GS, &cg_upcall_vector));
2575 if (!r)
2576 upcall_register = R_GS;
2577 }
2578 #elif defined(HAVE_AMD64_SET_GSBASE) && defined(HAVE_X86_SYSARCH_H)
2579 if (!dll) {
2580 int r;
2581 EINTR_LOOP(r, amd64_set_gsbase(&cg_upcall_vector));
2582 if (!r)
2583 upcall_register = R_GS;
2584 }
2585 #elif defined(HAVE_SYSARCH) && defined(HAVE_X86_SYSARCH_H) && defined(X86_64_SET_GSBASE)
2586 if (!dll) {
2587 int r;
2588 void *ptr = &cg_upcall_vector;
2589 EINTR_LOOP(r, sysarch(X86_64_SET_GSBASE, &ptr));
2590 if (!r)
2591 upcall_register = R_GS;
2592 }
2593 #endif
2594 #endif
2595
2596 init_ctx(ctx);
2597 ctx->fn = NULL;
2598
2599 array_init(uint8_t, &ctx->code, &ctx->code_size);
2600
2601 if (unlikely(!gen_entry(ctx)))
2602 goto fail;
2603
2604 array_init(uint8_t, &ctx->mcode, &ctx->mcode_size);
2605
2606 #ifdef ARCH_CONTEXT
2607 init_arch_context(ctx);
2608 #endif
2609
2610 if (unlikely(!gen_mcode(ctx)))
2611 goto fail;
2612
2613 array_finish(uint8_t, &ctx->mcode, &ctx->mcode_size);
2614 ptr = os_code_map(ctx->mcode, ctx->mcode_size, NULL);
2615 codegen_ptr = ptr;
2616 codegen_size = ctx->mcode_size;
2617 ctx->mcode = NULL;
2618 #if defined(ARCH_IA64)
2619 ia64_stub[0] = ptr_to_num(ptr);
2620 ia64_stub[1] = 0;
2621 codegen_entry = cast_ptr(codegen_type, ia64_stub);
2622 #elif defined(ARCH_PARISC32) && defined(ARCH_PARISC_USE_STUBS)
2623 parisc_stub[0] = ptr_to_num(ptr);
2624 parisc_stub[1] = 0;
2625 codegen_entry = cast_ptr(codegen_type, cast_ptr(char *, parisc_stub) + 2);
2626 #elif defined(ARCH_PARISC64) && defined(ARCH_PARISC_USE_STUBS)
2627 parisc_stub[0] = 0;
2628 parisc_stub[1] = 0;
2629 parisc_stub[2] = ptr_to_num(ptr);
2630 parisc_stub[3] = 0;
2631 codegen_entry = cast_ptr(codegen_type, parisc_stub);
2632 #elif defined(ARCH_POWER) && defined(AIX_CALL)
2633 ppc_stub[0] = ptr_to_num(ptr);
2634 ppc_stub[1] = 0;
2635 ppc_stub[2] = 0;
2636 codegen_entry = cast_ptr(codegen_type, ppc_stub);
2637 #else
2638 codegen_entry = ptr;
2639 #endif
2640 done_ctx(ctx);
2641
2642 #ifdef DEBUG_ENV
2643 mutex_init(&dump_mutex);
2644 if (getenv("DUMP") || getenv("DUMP_ALL")) {
2645 size_t i;
2646 char *hex;
2647 size_t hexl;
2648 str_init(&hex, &hexl);
2649 #if defined(ARCH_RISCV64)
2650 str_add_string(&hex, &hexl, " .attribute arch, \"rv64i2p1_m2p0_a2p1_f2p2_d2p2_c2p0_zicsr2p0_zifencei2p0_zba1p0_zbb1p0_zbc1p0_zbs1p0\"\n");
2651 #endif
2652 for (i = 0; i < codegen_size; i++) {
2653 uint8_t a = cast_ptr(uint8_t *, codegen_ptr)[i];
2654 str_add_string(&hex, &hexl, " .byte 0x");
2655 if (a < 16)
2656 str_add_char(&hex, &hexl, '0');
2657 str_add_unsigned(&hex, &hexl, a, 16);
2658 str_add_char(&hex, &hexl, '\n');
2659 }
2660 os_write_atomic(".", "dump.s", hex, hexl, NULL);
2661 mem_free(hex);
2662 }
2663 #endif
2664
2665 return;
2666
2667 fail:
2668 fatal("couldn't compile global entry");
2669 }
2670
2671 void name(codegen_done)(void)
2672 {
2673 os_code_unmap(codegen_ptr, codegen_size);
2674 #ifdef DEBUG_ENV
2675 mutex_done(&dump_mutex);
2676 #endif
2677 }
2678
2679 #else
2680
2681 void name(codegen_init)(void)
2682 {
2683 }
2684
2685 void name(codegen_done)(void)
2686 {
2687 }
2688
2689 #endif
2690
2691 #endif
Ajla - a purely functional programming language