?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
ssa: don't swap the arguments of comparison operators
[ajla.git] / codegen.c
blobdf7fc7f2dc2693dcc1c61f45325d17f5ebf2e7cc
1 /*
2 * Copyright (C) 2024 Mikulas Patocka
3 *
4 * This file is part of Ajla.
5 *
6 * Ajla is free software: you can redistribute it and/or modify it under the
7 * terms of the GNU General Public License as published by the Free Software
8 * Foundation, either version 3 of the License, or (at your option) any later
9 * version.
10 *
11 * Ajla is distributed in the hope that it will be useful, but WITHOUT ANY
12 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
13 * A PARTICULAR PURPOSE. See the GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License along with
16 * Ajla. If not, see <https://www.gnu.org/licenses/>.
17 */
18
19 #include "ajla.h"
20
21 #ifndef FILE_OMIT
22
23 #include "data.h"
24 #include "os.h"
25 #include "os_util.h"
26 #include "util.h"
27 #include "ipfn.h"
28 #include "ipret.h"
29 #include "funct.h"
30 #include "thread.h"
31 #include "task.h"
32 #include "save.h"
33
34 #include "codegen.h"
35
36 #ifdef HAVE_CODEGEN
37
38 #define flag_cache_chicken 0
39 #define must_be_flat_chicken 0
40 #define ra_chicken 0
41
42 #define INLINE_BITMAP_SLOTS 16
43 #define INLINE_COPY_SIZE 64
44
45 /*#define DEBUG_INSNS*/
46 /*#define DEBUG_GARBAGE*/
47
48 #if (defined(ARCH_X86_64) || defined(ARCH_X86_X32)) && !defined(ARCH_X86_WIN_ABI)
49 #if defined(HAVE_SYSCALL) && defined(HAVE_ASM_PRCTL_H) && defined(HAVE_SYS_SYSCALL_H)
50 #include <asm/prctl.h>
51 #include <sys/syscall.h>
52 #include <unistd.h>
53 #endif
54 #if (defined(HAVE_AMD64_SET_GSBASE) || defined(HAVE_SYSARCH)) && defined(HAVE_X86_SYSARCH_H)
55 #include <x86/sysarch.h>
56 #endif
57 #endif
58
59 code_return_t (*codegen_entry)(frame_s *, struct cg_upcall_vector_s *, tick_stamp_t, void *);
60 static void *codegen_ptr;
61 static size_t codegen_size;
62
63 #ifdef DEBUG_ENV
64 static mutex_t dump_mutex;
65 static uint64_t dump_seq = 0;
66 #endif
67
68 /*
69 * insn:
70 * opcode - 16 bits
71 * op_size - 3 bits
72 * aux - 7 bits
73 * writes flags - 2 bit
74 * jmp size - 2 bits
75 */
76
77 #define INSN_OPCODE 0x0000ffffUL
78 #define INSN_OP_SIZE 0x00070000UL
79 #define INSN_AUX 0x03f80000UL
80 #define INSN_WRITES_FLAGS 0x0c000000UL
81 #define INSN_JUMP_SIZE 0x30000000UL
82
83 #define INSN_OPCODE_SHIFT 0
84 #define INSN_OP_SIZE_SHIFT 16
85 #define INSN_AUX_SHIFT 19
86 #define INSN_WRITES_FLAGS_SHIFT 26
87 #define INSN_JUMP_SIZE_SHIFT 28
88
89 #define insn_opcode(insn) (((insn) >> INSN_OPCODE_SHIFT) & (INSN_OPCODE >> INSN_OPCODE_SHIFT))
90 #define insn_op_size(insn) (((insn) >> INSN_OP_SIZE_SHIFT) & (INSN_OP_SIZE >> INSN_OP_SIZE_SHIFT))
91 #define insn_aux(insn) (((insn) >> INSN_AUX_SHIFT) & (INSN_AUX >> INSN_AUX_SHIFT))
92 #define insn_writes_flags(insn) (((insn) >> INSN_WRITES_FLAGS_SHIFT) & (INSN_WRITES_FLAGS >> INSN_WRITES_FLAGS_SHIFT))
93 #define insn_jump_size(insn) (((insn) >> INSN_JUMP_SIZE_SHIFT) & (INSN_JUMP_SIZE >> INSN_JUMP_SIZE_SHIFT))
94
95 #define ALU_ADD 0x00
96 #define ALU_OR 0x01
97 #define ALU_ADC 0x02
98 #define ALU_SBB 0x03
99 #define ALU_AND 0x04
100 #define ALU_SUB 0x05
101 #define ALU_XOR 0x06
102 #define ALU_ORN 0x08
103 #define ALU_ANDN 0x09
104 #define ALU_XORN 0x0a
105 #define ALU_MUL 0x10
106 #define ALU_UMULH 0x11
107 #define ALU_SMULH 0x12
108 #define ALU_UDIV 0x13
109 #define ALU_SDIV 0x14
110 #define ALU_UREM 0x15
111 #define ALU_SREM 0x16
112 #define ALU_SAVE 0x17
113 #define ALU_EXTBL 0x18
114 #define ALU_EXTWL 0x19
115 #define ALU_EXTLL 0x1a
116 #define ALU_EXTLH 0x1b
117 #define ALU_INSBL 0x1c
118 #define ALU_MSKBL 0x1d
119 #define ALU_ZAP 0x20
120 #define ALU_ZAPNOT 0x21
121
122 #define ALU1_NOT 0x00
123 #define ALU1_NEG 0x01
124 #define ALU1_NGC 0x02
125 #define ALU1_INC 0x03
126 #define ALU1_DEC 0x04
127 #define ALU1_BSWAP 0x05
128 #define ALU1_BSWAP16 0x06
129 #define ALU1_BREV 0x07
130 #define ALU1_BSF 0x08
131 #define ALU1_BSR 0x09
132 #define ALU1_LZCNT 0x0a
133 #define ALU1_POPCNT 0x0b
134
135 #define FP_ALU_ADD 0
136 #define FP_ALU_SUB 1
137 #define FP_ALU_MUL 2
138 #define FP_ALU_DIV 3
139 #define FP_ALU1_NEG 0
140 #define FP_ALU1_SQRT 1
141 #define FP_ALU1_ROUND 2
142 #define FP_ALU1_FLOOR 3
143 #define FP_ALU1_CEIL 4
144 #define FP_ALU1_TRUNC 5
145 #define FP_ALU1_VCNT8 6
146 #define FP_ALU1_VPADDL 7
147 #define FP_ALU1_ADDV 8
148
149 #define COND_O 0x0
150 #define COND_NO 0x1
151 #define COND_B 0x2
152 #define COND_AE 0x3
153 #define COND_E 0x4
154 #define COND_NE 0x5
155 #define COND_BE 0x6
156 #define COND_A 0x7
157 #define COND_S 0x8
158 #define COND_NS 0x9
159 #define COND_P 0xa
160 #define COND_NP 0xb
161 #define COND_L 0xc
162 #define COND_GE 0xd
163 #define COND_LE 0xe
164 #define COND_G 0xf
165 #define COND_BLBC 0x10
166 #define COND_BLBS 0x11
167 #define COND_ALWAYS 0x12
168
169 #define COND_FP 0x20
170 #define FP_COND_P (COND_FP | COND_P)
171 #define FP_COND_NP (COND_FP | COND_NP)
172 #define FP_COND_E (COND_FP | COND_E)
173 #define FP_COND_NE (COND_FP | COND_NE)
174 #define FP_COND_A (COND_FP | COND_A)
175 #define FP_COND_BE (COND_FP | COND_BE)
176 #define FP_COND_B (COND_FP | COND_B)
177 #define FP_COND_AE (COND_FP | COND_AE)
178
179 #define ROT_ROL 0x0
180 #define ROT_ROR 0x1
181 #define ROT_RCL 0x2
182 #define ROT_RCR 0x3
183 #define ROT_SHL 0x4
184 #define ROT_SHR 0x5
185 #define ROT_SAR 0x7
186 #define ROT_SAL 0x8
187
188 #define BTX_BT 0x0
189 #define BTX_BTS 0x1
190 #define BTX_BTR 0x2
191 #define BTX_BTC 0x3
192 #define BTX_BTEXT 0x4
193
194 #define OP_SIZE_1 0
195 #define OP_SIZE_2 1
196 #define OP_SIZE_4 2
197 #define OP_SIZE_8 3
198 #define OP_SIZE_16 4
199 #define OP_SIZE_10 7
200
201 #define MOV_MASK_0_16 0x0
202 #define MOV_MASK_16_32 0x1
203 #define MOV_MASK_32_48 0x2
204 #define MOV_MASK_48_64 0x3
205 #define MOV_MASK_0_8 0x4
206 #define MOV_MASK_32_64 0x5
207 #define MOV_MASK_52_64 0x6
208
209 #define JMP_SHORTEST 0x0000
210 #define JMP_SHORT 0x0001
211 #define JMP_LONG 0x0002
212 #define JMP_EXTRA_LONG 0x0003
213
214 enum {
215 INSN_ENTRY,
216 INSN_LABEL,
217 INSN_RET,
218 INSN_RET_IMM,
219 INSN_ARM_PUSH,
220 INSN_ARM_POP,
221 INSN_S390_PUSH,
222 INSN_S390_POP,
223 INSN_IA64_ALLOC,
224 INSN_IA64_DEALLOC,
225 INSN_PUSH,
226 INSN_POP,
227 INSN_CALL,
228 INSN_CALL_INDIRECT,
229 INSN_MOV,
230 INSN_MOVSX,
231 INSN_MOV_U,
232 INSN_MOV_LR,
233 INSN_CMP,
234 INSN_CMP_DEST_REG,
235 INSN_CMN,
236 INSN_TEST,
237 INSN_TEST_DEST_REG,
238 INSN_ALU,
239 INSN_ALU_PARTIAL,
240 INSN_ALU_FLAGS,
241 INSN_ALU_FLAGS_PARTIAL,
242 INSN_ALU_TRAP,
243 INSN_ALU_FLAGS_TRAP,
244 INSN_ALU1,
245 INSN_ALU1_PARTIAL,
246 INSN_ALU1_FLAGS,
247 INSN_ALU1_FLAGS_PARTIAL,
248 INSN_ALU1_TRAP,
249 INSN_LEA3,
250 INSN_ROT,
251 INSN_ROT_PARTIAL,
252 INSN_BT,
253 INSN_BTX,
254 INSN_MUL_L,
255 INSN_DIV_L,
256 INSN_MADD,
257 INSN_CBW,
258 INSN_CBW_PARTIAL,
259 INSN_CWD,
260 INSN_CWD_PARTIAL,
261 INSN_SET_COND,
262 INSN_SET_COND_PARTIAL,
263 INSN_CMOV,
264 INSN_CMOV_XCC,
265 INSN_CMP_CMOV,
266 INSN_MOVR,
267 INSN_CSEL_SEL,
268 INSN_CSEL_INC,
269 INSN_CSEL_INV,
270 INSN_CSEL_NEG,
271 INSN_STP,
272 INSN_LDP,
273 INSN_MOV_MASK,
274 INSN_MEMCPY,
275 INSN_MEMSET,
276 INSN_FP_CMP,
277 INSN_FP_CMP_DEST_REG,
278 INSN_FP_CMP_DEST_REG_TRAP,
279 INSN_FP_CMP_UNORDERED_DEST_REG,
280 INSN_FP_CMP_COND,
281 INSN_FP_TEST_REG,
282 INSN_FP_TO_INT_FLAGS,
283 INSN_FP_ALU,
284 INSN_FP_ALU1,
285 INSN_FP_TO_INT32,
286 INSN_FP_TO_INT64,
287 INSN_FP_TO_INT64_TRAP,
288 INSN_FP_FROM_INT32,
289 INSN_FP_FROM_INT64,
290 INSN_FP_INT64_TO_INT32_TRAP,
291 INSN_FP_CVT,
292 INSN_X87_FLD,
293 INSN_X87_FILD,
294 INSN_X87_FSTP,
295 INSN_X87_FISTP,
296 INSN_X87_FISTTP,
297 INSN_X87_FCOMP,
298 INSN_X87_FCOMPP,
299 INSN_X87_FCOMIP,
300 INSN_X87_ALU,
301 INSN_X87_ALUP,
302 INSN_X87_FCHS,
303 INSN_X87_FSQRT,
304 INSN_X87_FRNDINT,
305 INSN_X87_FNSTSW,
306 INSN_X87_FLDCW,
307 INSN_JMP,
308 INSN_JMP_COND,
309 INSN_JMP_COND_LOGICAL,
310 INSN_JMP_REG,
311 INSN_JMP_REG_BIT,
312 INSN_JMP_2REGS,
313 INSN_JMP_FP_TEST,
314 INSN_JMP_INDIRECT,
315 INSN_MB,
316 INSN_CALL_MILLICODE,
317 };
318
319 #define ARG_REGS_MAX 0xc0
320 #define ARG_SHIFTED_REGISTER 0xc0
321 #define ARG_SHIFT_AMOUNT 0x3f
322 #define ARG_SHIFT_MODE 0xc0
323 #define ARG_SHIFT_LSL 0x00
324 #define ARG_SHIFT_LSR 0x40
325 #define ARG_SHIFT_ASR 0x80
326 #define ARG_SHIFT_ROR 0xc0
327 #define ARG_EXTENDED_REGISTER 0xc1
328 #define ARG_EXTEND_SHIFT 0x07
329 #define ARG_EXTEND_MODE 0x38
330 #define ARG_EXTEND_UXTB 0x00
331 #define ARG_EXTEND_UXTH 0x08
332 #define ARG_EXTEND_UXTW 0x10
333 #define ARG_EXTEND_UXTX 0x18
334 #define ARG_EXTEND_SXTB 0x20
335 #define ARG_EXTEND_SXTH 0x28
336 #define ARG_EXTEND_SXTW 0x30
337 #define ARG_EXTEND_SXTX 0x38
338 #define ARG_ADDRESS_0 0xd0
339 #define ARG_ADDRESS_1 0xd1
340 #define ARG_ADDRESS_1_2 0xd2
341 #define ARG_ADDRESS_1_4 0xd3
342 #define ARG_ADDRESS_1_8 0xd4
343 #define ARG_ADDRESS_1_PRE_I 0xd5
344 #define ARG_ADDRESS_1_POST_I 0xd6
345 #define ARG_ADDRESS_2 0xd7
346 #define ARG_ADDRESS_2_2 0xd8
347 #define ARG_ADDRESS_2_4 0xd9
348 #define ARG_ADDRESS_2_8 0xda
349 #define ARG_ADDRESS_2_UXTW 0xdb
350 #define ARG_ADDRESS_2_SXTW 0xdc
351 #define ARG_IMM 0xe0
352
353 #define ARG_IS_ADDRESS(a) ((a) >= ARG_ADDRESS_0 && (a) <= ARG_ADDRESS_2_SXTW)
354
355 #ifdef POINTER_COMPRESSION
356 #define OP_SIZE_SLOT OP_SIZE_4
357 #else
358 #define OP_SIZE_SLOT OP_SIZE_ADDRESS
359 #endif
360
361 #define OP_SIZE_BITMAP (bitmap_64bit ? OP_SIZE_8 : OP_SIZE_4)
362
363 #define OP_SIZE_INT log_2(sizeof(int_default_t))
364
365 #define check_insn(insn) \
366 do { \
367 /*if ((insn_opcode(insn) == INSN_ALU || insn_opcode(insn) == INSN_ALU1) && insn_op_size(insn) != OP_SIZE_NATIVE) internal(file_line, "invalid insn %08x", (unsigned)(insn));*/\
368 /*if (insn == 0x001a000e) internal(file_line, "invalid insn %08x", insn);*/\
369 } while (0)
370
371 #ifdef DEBUG_INSNS
372 #define gen_line() gen_four(__LINE__ + (insn_file << 24))
373 #else
374 #define gen_line() do { } while (0)
375 #endif
376
377 #ifdef ARCH_IA64
378 #define ARCH_CONTEXT struct { \
379 uint64_t insns[3]; \
380 uint8_t insn_units[3]; \
381 bool insn_stops[3]; \
382 uint64_t wr_mask[4]; \
383 }
384 #endif
385
386 #define gen_insn(opcode, op_size, aux, writes_flags) \
387 do { \
388 uint32_t dword = \
389 (uint32_t)(opcode) << INSN_OPCODE_SHIFT | \
390 (uint32_t)(op_size) << INSN_OP_SIZE_SHIFT | \
391 (uint32_t)(aux) << INSN_AUX_SHIFT | \
392 (uint32_t)(writes_flags) << INSN_WRITES_FLAGS_SHIFT; \
393 check_insn(dword); \
394 gen_line(); \
395 gen_four(dword); \
396 } while (0)
397
398 static size_t arg_size(uint8_t arg)
399 {
400 if (arg < ARG_REGS_MAX)
401 return 1;
402 if (arg >= ARG_SHIFTED_REGISTER && arg <= ARG_EXTENDED_REGISTER)
403 return 3;
404 if (arg == ARG_ADDRESS_0)
405 return 9;
406 if (arg >= ARG_ADDRESS_1 && arg <= ARG_ADDRESS_1_POST_I)
407 return 10;
408 if (arg >= ARG_ADDRESS_2 && arg <= ARG_ADDRESS_2_SXTW)
409 return 11;
410 if (arg == ARG_IMM)
411 return 9;
412 internal(file_line, "arg_size: invalid argument %02x", arg);
413 return 0;
414 }
415
416 struct relocation {
417 uint32_t label_id;
418 uint8_t length;
419 size_t position;
420 size_t jmp_instr;
421 };
422
423 struct code_arg {
424 frame_t slot;
425 frame_t flags;
426 frame_t type;
427 };
428
429 struct cg_entry {
430 size_t entry_to_pos;
431 frame_t *variables;
432 size_t n_variables;
433 uint32_t entry_label;
434 uint32_t nonflat_label;
435 };
436
437 struct cg_exit {
438 uint32_t undo_label;
439 uint8_t undo_opcode;
440 uint8_t undo_op_size;
441 uint8_t undo_aux;
442 uint8_t undo_writes_flags;
443 uint8_t undo_parameters[35];
444 uint8_t undo_parameters_len;
445 uint32_t escape_label;
446 };
447
448 #define FLAG_CACHE_IS_FLAT 0x01
449 #define FLAG_CACHE_IS_NOT_FLAT 0x02
450 #define FLAG_CACHE_IS_NOT_THUNK 0x04
451
452 struct codegen_context {
453 struct data *fn;
454 struct data **local_directory;
455
456 const code_t *instr_start;
457 const code_t *current_position;
458 uchar_efficient_t arg_mode;
459
460 uint32_t label_id;
461 struct cg_entry *entries;
462 frame_t n_entries;
463
464 uint8_t *code;
465 size_t code_size;
466
467 uint8_t *code_position;
468
469 uint32_t *code_labels;
470 struct cg_exit **code_exits;
471 uint32_t escape_nospill_label;
472 uint32_t call_label;
473 uint32_t reload_label;
474
475 uint8_t *mcode;
476 size_t mcode_size;
477
478 size_t *label_to_pos;
479 struct relocation *reloc;
480 size_t reloc_size;
481
482 struct trap_record *trap_records;
483 size_t trap_records_size;
484
485 struct code_arg *args;
486 size_t args_l;
487 const code_t *return_values;
488
489 uint8_t *flag_cache;
490 short *registers;
491 frame_t *need_spill;
492 size_t need_spill_l;
493
494 unsigned base_reg;
495 bool offset_reg;
496 int64_t offset_imm;
497
498 bool const_reg;
499 int64_t const_imm;
500
501 struct data *codegen;
502
503 int upcall_args;
504 frame_t *var_aux;
505
506 ajla_error_t err;
507
508 #ifdef ARCH_CONTEXT
509 ARCH_CONTEXT a;
510 #endif
511 };
512
513 static void init_ctx(struct codegen_context *ctx)
514 {
515 ctx->local_directory = NULL;
516 ctx->label_id = 0;
517 ctx->entries = NULL;
518 ctx->n_entries = 0;
519 ctx->code = NULL;
520 ctx->code_labels = NULL;
521 ctx->code_exits = NULL;
522 ctx->escape_nospill_label = 0;
523 ctx->call_label = 0;
524 ctx->reload_label = 0;
525 ctx->mcode = NULL;
526 ctx->label_to_pos = NULL;
527 ctx->reloc = NULL;
528 ctx->trap_records = NULL;
529 ctx->args = NULL;
530 ctx->flag_cache = NULL;
531 ctx->registers = NULL;
532 ctx->need_spill = NULL;
533 ctx->codegen = NULL;
534 ctx->upcall_args = -1;
535 ctx->var_aux = NULL;
536 }
537
538 static void done_ctx(struct codegen_context *ctx)
539 {
540 if (ctx->local_directory)
541 mem_free(ctx->local_directory);
542 if (ctx->entries) {
543 size_t i;
544 for (i = 0; i < ctx->n_entries; i++) {
545 struct cg_entry *ce = &ctx->entries[i];
546 if (ce->variables)
547 mem_free(ce->variables);
548 }
549 mem_free(ctx->entries);
550 }
551 if (ctx->code)
552 mem_free(ctx->code);
553 if (ctx->code_labels)
554 mem_free(ctx->code_labels);
555 if (ctx->code_exits) {
556 ip_t ip;
557 ip_t cs = da(ctx->fn,function)->code_size;
558 for (ip = 0; ip < cs; ip++) {
559 if (ctx->code_exits[ip])
560 mem_free(ctx->code_exits[ip]);
561 }
562 mem_free(ctx->code_exits);
563 }
564 if (ctx->mcode)
565 mem_free(ctx->mcode);
566 if (ctx->label_to_pos)
567 mem_free(ctx->label_to_pos);
568 if (ctx->reloc)
569 mem_free(ctx->reloc);
570 if (ctx->trap_records)
571 mem_free(ctx->trap_records);
572 if (ctx->args)
573 mem_free(ctx->args);
574 if (ctx->flag_cache)
575 mem_free(ctx->flag_cache);
576 if (ctx->registers)
577 mem_free(ctx->registers);
578 if (ctx->need_spill)
579 mem_free(ctx->need_spill);
580 if (ctx->codegen)
581 data_free(ctx->codegen);
582 if (ctx->var_aux)
583 mem_free(ctx->var_aux);
584 }
585
586
587 static inline code_t get_code(struct codegen_context *ctx)
588 {
589 ajla_assert(ctx->current_position < da(ctx->fn,function)->code + da(ctx->fn,function)->code_size, (file_line, "get_code: ran out of code"));
590 return *ctx->current_position++;
591 }
592
593 static inline uint32_t get_uint32(struct codegen_context *ctx)
594 {
595 uint32_t a1 = get_code(ctx);
596 uint32_t a2 = get_code(ctx);
597 #if !CODE_ENDIAN
598 return a1 + (a2 << 16);
599 #else
600 return a2 + (a1 << 16);
601 #endif
602 }
603
604 static int32_t get_jump_offset(struct codegen_context *ctx)
605 {
606 if (SIZEOF_IP_T == 2) {
607 return (int32_t)(int16_t)get_code(ctx);
608 } else if (SIZEOF_IP_T == 4) {
609 return (int32_t)get_uint32(ctx);
610 } else {
611 not_reached();
612 return -1;
613 }
614 }
615
616 static inline void get_one(struct codegen_context *ctx, frame_t *v)
617 {
618 if (!ctx->arg_mode) {
619 code_t c = get_code(ctx);
620 ajla_assert(!(c & ~0xff), (file_line, "get_one: high byte is not cleared: %u", (unsigned)c));
621 *v = c & 0xff;
622 } else if (ctx->arg_mode == 1) {
623 *v = get_code(ctx);
624 #if ARG_MODE_N >= 2
625 } else if (ctx->arg_mode == 2) {
626 *v = get_uint32(ctx);
627 #endif
628 } else {
629 internal(file_line, "get_one: invalid arg mode %u", ctx->arg_mode);
630 }
631 }
632
633 static inline void get_two(struct codegen_context *ctx, frame_t *v1, frame_t *v2)
634 {
635 if (!ctx->arg_mode) {
636 code_t c = get_code(ctx);
637 *v1 = c & 0xff;
638 *v2 = c >> 8;
639 } else if (ctx->arg_mode == 1) {
640 *v1 = get_code(ctx);
641 *v2 = get_code(ctx);
642 #if ARG_MODE_N >= 2
643 } else if (ctx->arg_mode == 2) {
644 *v1 = get_uint32(ctx);
645 *v2 = get_uint32(ctx);
646 #endif
647 } else {
648 internal(file_line, "get_two: invalid arg mode %u", ctx->arg_mode);
649 }
650 }
651
652
653 static uint32_t alloc_label(struct codegen_context *ctx)
654 {
655 return ++ctx->label_id;
656 }
657
658 static struct cg_exit *alloc_cg_exit_for_ip(struct codegen_context *ctx, const code_t *code)
659 {
660 ip_t ip = code - da(ctx->fn,function)->code;
661 struct cg_exit *ce = ctx->code_exits[ip];
662 if (!ce) {
663 ce = mem_calloc_mayfail(struct cg_exit *, sizeof(struct cg_exit), &ctx->err);
664 if (unlikely(!ce))
665 return NULL;
666 ctx->code_exits[ip] = ce;
667 }
668 return ce;
669 }
670
671 static struct cg_exit *alloc_undo_label(struct codegen_context *ctx)
672 {
673 struct cg_exit *ce = alloc_cg_exit_for_ip(ctx, ctx->instr_start);
674 if (unlikely(!ce))
675 return NULL;
676 if (unlikely(ce->undo_label != 0))
677 internal(file_line, "alloc_cg_exit: undo label already allocated");
678 ce->undo_label = alloc_label(ctx);
679 if (unlikely(!ce->undo_label))
680 return NULL;
681 return ce;
682 }
683
684 static uint32_t alloc_escape_label_for_ip(struct codegen_context *ctx, const code_t *code)
685 {
686 struct cg_exit *ce = alloc_cg_exit_for_ip(ctx, code);
687 if (!ce)
688 return 0;
689 if (!ce->escape_label)
690 ce->escape_label = alloc_label(ctx);
691 return ce->escape_label;
692 }
693
694 static uint32_t alloc_escape_label(struct codegen_context *ctx)
695 {
696 return alloc_escape_label_for_ip(ctx, ctx->instr_start);
697 }
698
699 static uint32_t attr_unused alloc_call_label(struct codegen_context *ctx)
700 {
701 if (!ctx->call_label) {
702 ctx->call_label = alloc_label(ctx);
703 }
704 return ctx->call_label;
705 }
706
707 static uint32_t alloc_reload_label(struct codegen_context *ctx)
708 {
709 if (!ctx->reload_label) {
710 ctx->reload_label = alloc_label(ctx);
711 }
712 return ctx->reload_label;
713 }
714
715 static size_t attr_unused mark_params(struct codegen_context *ctx)
716 {
717 return ctx->code_size;
718 }
719
720 static void attr_unused copy_params(struct codegen_context *ctx, struct cg_exit *ce, size_t mark)
721 {
722 if (ctx->code_size - mark > n_array_elements(ce->undo_parameters))
723 internal(file_line, "undo_parameters is too small: %"PRIuMAX" > %"PRIuMAX"", (uintmax_t)(ctx->code_size - mark), (uintmax_t)n_array_elements(ce->undo_parameters));
724 memcpy(ce->undo_parameters, ctx->code + mark, ctx->code_size - mark);
725 ce->undo_parameters_len = ctx->code_size - mark;
726 ctx->code_size = mark;
727 }
728
729 #define g(call) \
730 do { \
731 if (unlikely(!call)) \
732 return false; \
733 } while (0)
734
735 #define gen_one(byte) \
736 do { \
737 /*debug("gen %d: %02x", __LINE__, (uint8_t)(byte))*/; \
738 if (unlikely(!array_add_mayfail(uint8_t, &ctx->code, &ctx->code_size, byte, NULL, &ctx->err)))\
739 return false; \
740 } while (0)
741
742 #if defined(C_LITTLE_ENDIAN)
743 #define gen_two(word) \
744 do { \
745 uint16_t word_ = (word); \
746 /*debug("gen %d: %04x", __LINE__, (uint16_t)(word_));*/ \
747 if (unlikely(!array_add_multiple_mayfail(uint8_t, &ctx->code, &ctx->code_size, cast_ptr(uint8_t *, &word_), 2, NULL, &ctx->err)))\
748 return false; \
749 } while (0)
750 #define gen_four(dword) \
751 do { \
752 uint32_t dword_ = (dword); \
753 /*debug("gen %d: %08x", __LINE__, (uint32_t)(dword_));*/ \
754 if (unlikely(!array_add_multiple_mayfail(uint8_t, &ctx->code, &ctx->code_size, cast_ptr(uint8_t *, &dword_), 4, NULL, &ctx->err)))\
755 return false; \
756 } while (0)
757 #define gen_eight(qword) \
758 do { \
759 uint64_t qword_ = (qword); \
760 /*debug("gen %d: %016lx", __LINE__, (uint64_t)(qword_));*/ \
761 if (unlikely(!array_add_multiple_mayfail(uint8_t, &ctx->code, &ctx->code_size, cast_ptr(uint8_t *, &qword_), 8, NULL, &ctx->err)))\
762 return false; \
763 } while (0)
764 #else
765 #define gen_two(word) \
766 do { \
767 uint16_t word_ = (word); \
768 gen_one(word_ & 0xffU); \
769 gen_one(word_ >> 8); \
770 } while (0)
771 #define gen_four(dword) \
772 do { \
773 uint32_t dword_ = (dword); \
774 gen_two(dword_ & 0xffffU); \
775 gen_two(dword_ >> 15 >> 1); \
776 } while (0)
777 #define gen_eight(qword) \
778 do { \
779 uint64_t qword_ = (qword); \
780 gen_four(qword_ & 0xffffffffUL); \
781 gen_four(qword_ >> 15 >> 15 >> 2); \
782 } while (0)
783 #endif
784
785 #define gen_label(label_id) \
786 do { \
787 gen_insn(INSN_LABEL, 0, 0, 0); \
788 gen_four(label_id); \
789 } while (0)
790
791
792 static uint8_t attr_unused cget_one(struct codegen_context *ctx)
793 {
794 ajla_assert(ctx->code_position < ctx->code + ctx->code_size, (file_line, "cget_one: ran out of code"));
795 return *ctx->code_position++;
796 }
797
798 static uint16_t attr_unused cget_two(struct codegen_context *ctx)
799 {
800 #if defined(C_LITTLE_ENDIAN)
801 uint16_t r;
802 ajla_assert(ctx->code_position < ctx->code + ctx->code_size, (file_line, "cget_two: ran out of code"));
803 memcpy(&r, ctx->code_position, 2);
804 ctx->code_position += 2;
805 return r;
806 #else
807 uint16_t r = cget_one(ctx);
808 r |= cget_one(ctx) << 8;
809 return r;
810 #endif
811 }
812
813 static uint32_t cget_four(struct codegen_context *ctx)
814 {
815 #if defined(C_LITTLE_ENDIAN)
816 uint32_t r;
817 ajla_assert(ctx->code_position < ctx->code + ctx->code_size, (file_line, "cget_four: ran out of code"));
818 memcpy(&r, ctx->code_position, 4);
819 ctx->code_position += 4;
820 return r;
821 #else
822 uint32_t r = cget_two(ctx);
823 r |= (uint32_t)cget_two(ctx) << 16;
824 return r;
825 #endif
826 }
827
828 static uint64_t attr_unused cget_eight(struct codegen_context *ctx)
829 {
830 #if defined(C_LITTLE_ENDIAN)
831 uint64_t r;
832 ajla_assert(ctx->code_position < ctx->code + ctx->code_size, (file_line, "cget_eight: ran out of code"));
833 memcpy(&r, ctx->code_position, 8);
834 ctx->code_position += 8;
835 return r;
836 #else
837 uint64_t r = cget_four(ctx);
838 r |= (uint64_t)cget_four(ctx) << 32;
839 return r;
840 #endif
841 }
842
843 static int64_t get_imm(uint8_t *ptr)
844 {
845 #if defined(C_LITTLE_ENDIAN)
846 int64_t r;
847 memcpy(&r, ptr, 8);
848 return r;
849 #else
850 int64_t r;
851 r = (uint64_t)ptr[0] |
852 ((uint64_t)ptr[1] << 8) |
853 ((uint64_t)ptr[2] << 16) |
854 ((uint64_t)ptr[3] << 24) |
855 ((uint64_t)ptr[4] << 32) |
856 ((uint64_t)ptr[5] << 40) |
857 ((uint64_t)ptr[6] << 48) |
858 ((uint64_t)ptr[7] << 56);
859 return r;
860 #endif
861 }
862
863 #define cgen_one(byte) \
864 do { \
865 if (unlikely(!array_add_mayfail(uint8_t, &ctx->mcode, &ctx->mcode_size, byte, NULL, &ctx->err)))\
866 return false; \
867 } while (0)
868
869 #if defined(C_LITTLE_ENDIAN) || 1
870 #define cgen_two(word) \
871 do { \
872 uint16_t word_ = (word); \
873 if (unlikely(!array_add_multiple_mayfail(uint8_t, &ctx->mcode, &ctx->mcode_size, cast_ptr(uint8_t *, &word_), 2, NULL, &ctx->err)))\
874 return false; \
875 } while (0)
876 #define cgen_four(dword) \
877 do { \
878 uint32_t dword_ = (dword); \
879 /*if (dword_ == 0x1ee02000) internal(file_line, "invalid instruction");*/\
880 if (unlikely(!array_add_multiple_mayfail(uint8_t, &ctx->mcode, &ctx->mcode_size, cast_ptr(uint8_t *, &dword_), 4, NULL, &ctx->err)))\
881 return false; \
882 } while (0)
883 #define cgen_eight(qword) \
884 do { \
885 uint64_t qword_ = (qword); \
886 if (unlikely(!array_add_multiple_mayfail(uint8_t, &ctx->mcode, &ctx->mcode_size, cast_ptr(uint8_t *, &qword_), 8, NULL, &ctx->err)))\
887 return false; \
888 } while (0)
889 #else
890 #define cgen_two(word) \
891 do { \
892 cgen_one((word) & 0xff); \
893 cgen_one((word) >> 8); \
894 } while (0)
895 #define cgen_four(dword) \
896 do { \
897 cgen_two((dword) & 0xffff); \
898 cgen_two((dword) >> 15 >> 1); \
899 } while (0)
900 #define cgen_eight(qword) \
901 do { \
902 cgen_four((qword) & 0xffffffff); \
903 cgen_four((qword) >> 15 >> 15 >> 2); \
904 } while (0)
905 #endif
906
907
908 #define IMM_PURPOSE_LDR_OFFSET 1
909 #define IMM_PURPOSE_LDR_SX_OFFSET 2
910 #define IMM_PURPOSE_STR_OFFSET 3
911 #define IMM_PURPOSE_LDP_STP_OFFSET 4
912 #define IMM_PURPOSE_VLDR_VSTR_OFFSET 5
913 #define IMM_PURPOSE_MVI_CLI_OFFSET 6
914 #define IMM_PURPOSE_STORE_VALUE 7
915 #define IMM_PURPOSE_ADD 8
916 #define IMM_PURPOSE_SUB 9
917 #define IMM_PURPOSE_CMP 10
918 #define IMM_PURPOSE_CMP_LOGICAL 11
919 #define IMM_PURPOSE_AND 12
920 #define IMM_PURPOSE_OR 13
921 #define IMM_PURPOSE_XOR 14
922 #define IMM_PURPOSE_ANDN 15
923 #define IMM_PURPOSE_TEST 16
924 #define IMM_PURPOSE_JMP_2REGS 17
925 #define IMM_PURPOSE_MUL 18
926 #define IMM_PURPOSE_CMOV 19
927 #define IMM_PURPOSE_MOVR 20
928 #define IMM_PURPOSE_BITWISE 21
929 #define IMM_PURPOSE_ADD_TRAP 22
930 #define IMM_PURPOSE_SUB_TRAP 23
931
932
933 static unsigned alu_purpose(unsigned alu)
934 {
935 unsigned purpose =
936 alu == ALU_ADD ? IMM_PURPOSE_ADD :
937 alu == ALU_ADC ? IMM_PURPOSE_ADD :
938 alu == ALU_SUB ? IMM_PURPOSE_SUB :
939 alu == ALU_SBB ? IMM_PURPOSE_SUB :
940 alu == ALU_MUL ? IMM_PURPOSE_MUL :
941 alu == ALU_UMULH ? IMM_PURPOSE_MUL :
942 alu == ALU_SMULH ? IMM_PURPOSE_MUL :
943 alu == ALU_ANDN ? IMM_PURPOSE_ANDN :
944 alu == ALU_AND ? IMM_PURPOSE_AND :
945 alu == ALU_OR ? IMM_PURPOSE_OR :
946 alu == ALU_XOR ? IMM_PURPOSE_XOR :
947 alu == ALU_EXTBL ? IMM_PURPOSE_OR :
948 alu == ALU_EXTWL ? IMM_PURPOSE_OR :
949 alu == ALU_EXTLL ? IMM_PURPOSE_OR :
950 alu == ALU_EXTLH ? IMM_PURPOSE_OR :
951 alu == ALU_INSBL ? IMM_PURPOSE_OR :
952 alu == ALU_MSKBL ? IMM_PURPOSE_OR :
953 alu == ALU_ZAP ? IMM_PURPOSE_ANDN :
954 alu == ALU_ZAPNOT ? IMM_PURPOSE_AND :
955 -1U;
956 if (unlikely(purpose == -1U))
957 internal(file_line, "alu_purpose: invalid alu %u", alu);
958 return purpose;
959 }
960
961 static unsigned alu_trap_purpose(unsigned alu)
962 {
963 unsigned purpose =
964 alu == ALU_ADD ? IMM_PURPOSE_ADD_TRAP :
965 alu == ALU_SUB ? IMM_PURPOSE_SUB_TRAP :
966 -1U;
967 if (unlikely(purpose == -1U))
968 internal(file_line, "alu_trap_purpose: invalid alu %u", alu);
969 return purpose;
970 }
971
972
973 static bool attr_w gen_upcall_end(struct codegen_context *ctx, unsigned args);
974
975 #define gen_address_offset() \
976 do { \
977 if (likely(!ctx->offset_reg)) { \
978 gen_one(ARG_ADDRESS_1); \
979 gen_one(ctx->base_reg); \
980 gen_eight(ctx->offset_imm); \
981 } else { \
982 gen_one(ARG_ADDRESS_2); \
983 gen_one(ctx->base_reg); \
984 gen_one(R_OFFSET_IMM); \
985 gen_eight(0); \
986 } \
987 } while (0)
988
989 #define gen_imm_offset() \
990 do { \
991 if (likely(!ctx->const_reg)) { \
992 gen_one(ARG_IMM); \
993 gen_eight(ctx->const_imm); \
994 } else { \
995 gen_one(R_CONST_IMM); \
996 } \
997 } while (0)
998
999 #define is_imm() (!ctx->const_reg)
1000
1001
1002 static inline bool slot_is_register(struct codegen_context *ctx, frame_t slot)
1003 {
1004 if (frame_t_is_const(slot))
1005 return false;
1006 if (unlikely(slot >= function_n_variables(ctx->fn)))
1007 internal(file_line, "slot_is_register: invalid slot %lu", (unsigned long)slot);
1008 return ctx->registers[slot] >= 0;
1009 }
1010
1011
1012 #define insn_file 1
1013 #if defined(ARCH_ALPHA)
1014 #include "c1-alpha.inc"
1015 #elif defined(ARCH_ARM32)
1016 #include "c1-arm.inc"
1017 #elif defined(ARCH_ARM64)
1018 #include "c1-arm64.inc"
1019 #elif defined(ARCH_IA64)
1020 #include "c1-ia64.inc"
1021 #elif defined(ARCH_LOONGARCH64)
1022 #include "c1-loong.inc"
1023 #elif defined(ARCH_MIPS)
1024 #include "c1-mips.inc"
1025 #elif defined(ARCH_PARISC)
1026 #include "c1-hppa.inc"
1027 #elif defined(ARCH_POWER)
1028 #include "c1-power.inc"
1029 #elif defined(ARCH_S390)
1030 #include "c1-s390.inc"
1031 #elif defined(ARCH_SPARC)
1032 #include "c1-sparc.inc"
1033 #elif defined(ARCH_RISCV64)
1034 #include "c1-riscv.inc"
1035 #elif defined(ARCH_X86)
1036 #include "c1-x86.inc"
1037 #endif
1038 #undef insn_file
1039
1040
1041 #ifndef ARCH_SUPPORTS_TRAPS
1042 #define ARCH_SUPPORTS_TRAPS 0
1043 #define ARCH_TRAP_BEFORE 0
1044 #endif
1045
1046
1047 #define insn_file 2
1048 #include "cg-util.inc"
1049 #undef insn_file
1050
1051 #define insn_file 3
1052 #include "cg-frame.inc"
1053 #undef insn_file
1054
1055 #define insn_file 4
1056 #include "cg-flags.inc"
1057 #undef insn_file
1058
1059 #define insn_file 5
1060 #include "cg-flcch.inc"
1061 #undef insn_file
1062
1063 #define insn_file 6
1064 #include "cg-ptr.inc"
1065 #undef insn_file
1066
1067 #define insn_file 7
1068 #include "cg-alu.inc"
1069 #undef insn_file
1070
1071 #define insn_file 8
1072 #include "cg-ops.inc"
1073 #undef insn_file
1074
1075 #define insn_file 0
1076
1077
1078 #ifndef n_regs_saved
1079 #define n_regs_saved n_array_elements(regs_saved)
1080 #endif
1081
1082 #ifndef n_regs_volatile
1083 #define n_regs_volatile n_array_elements(regs_volatile)
1084 #endif
1085
1086 #ifndef n_fp_saved
1087 #define n_fp_saved n_array_elements(fp_saved)
1088 #endif
1089
1090 #ifndef n_fp_volatile
1091 #define n_fp_volatile n_array_elements(fp_volatile)
1092 #endif
1093
1094 #ifndef n_vector_volatile
1095 #define n_vector_volatile n_array_elements(vector_volatile)
1096 #endif
1097
1098 static bool attr_w gen_registers(struct codegen_context *ctx)
1099 {
1100 frame_t v;
1101 size_t index_saved = 0;
1102 size_t index_volatile = 0;
1103 size_t index_fp_saved = 0;
1104 size_t index_fp_volatile = 0;
1105 size_t attr_unused index_vector_volatile = 0;
1106 #ifdef ARCH_S390
1107 bool uses_x = false;
1108 for (v = MIN_USEABLE_SLOT; v < function_n_variables(ctx->fn); v++) {
1109 const struct type *t = get_type_of_local(ctx, v);
1110 if (t && TYPE_TAG_IS_REAL(t->tag) && TYPE_TAG_IDX_REAL(t->tag) == 4) {
1111 uses_x = true;
1112 break;
1113 }
1114 }
1115 #endif
1116 /*for (v = function_n_variables(ctx->fn) - 1; v >= MIN_USEABLE_SLOT; v--)*/
1117 for (v = MIN_USEABLE_SLOT; v < function_n_variables(ctx->fn); v++) {
1118 const struct type *t;
1119 ctx->registers[v] = -1;
1120 if (ra_chicken)
1121 continue;
1122 t = get_type_of_local(ctx, v);
1123 if (unlikely(!t))
1124 continue;
1125 if (!da(ctx->fn,function)->local_variables_flags[v].must_be_flat &&
1126 !da(ctx->fn,function)->local_variables_flags[v].must_be_data)
1127 continue;
1128 if (!ARCH_HAS_BWX && t->size < 1U << OP_SIZE_4)
1129 continue;
1130 if (TYPE_TAG_IS_FIXED(t->tag) || TYPE_TAG_IS_INT(t->tag) || t->tag == TYPE_TAG_flat_option || t->tag == TYPE_TAG_unknown || t->tag == TYPE_TAG_record) {
1131 if (TYPE_TAG_IS_BUILTIN(t->tag)) {
1132 if (!is_power_of_2(t->size) || t->size > 1U << OP_SIZE_NATIVE)
1133 continue;
1134 }
1135 if (index_saved < n_regs_saved + zero
1136 #if defined(ARCH_PARISC) || defined(ARCH_SPARC)
1137 && t->size <= 1U << OP_SIZE_ADDRESS
1138 #endif
1139 ) {
1140 ctx->registers[v] = regs_saved[index_saved++];
1141 } else if (index_volatile < n_regs_volatile + zero) {
1142 ctx->registers[v] = regs_volatile[index_volatile++];
1143 } else {
1144 continue;
1145 }
1146 } else if (TYPE_TAG_IS_REAL(t->tag)) {
1147 unsigned real_type = TYPE_TAG_IDX_REAL(t->tag);
1148 if ((SUPPORTED_FP >> real_type) & 1) {
1149 #ifdef ARCH_POWER
1150 if (real_type == 4) {
1151 if (index_vector_volatile < n_vector_volatile + zero) {
1152 ctx->registers[v] = vector_volatile[index_vector_volatile++];
1153 goto success;
1154 }
1155 continue;
1156 }
1157 #endif
1158 #ifdef ARCH_S390
1159 if (real_type == 4) {
1160 if (!(index_fp_saved & 1) && index_fp_saved + 1 < n_fp_saved + zero) {
1161 ctx->registers[v] = fp_saved[index_fp_saved++];
1162 index_fp_saved++;
1163 goto success;
1164 }
1165 if (index_fp_saved & 1 && index_fp_saved + 2 < n_fp_saved + zero) {
1166 index_fp_saved++;
1167 ctx->registers[v] = fp_saved[index_fp_saved++];
1168 index_fp_saved++;
1169 goto success;
1170 }
1171 if (!(index_fp_volatile & 1) && index_fp_volatile + 1 < n_fp_volatile + zero) {
1172 ctx->registers[v] = fp_volatile[index_fp_volatile++];
1173 index_fp_volatile++;
1174 goto success;
1175 }
1176 if (index_fp_volatile & 1 && index_fp_volatile + 2 < n_fp_volatile + zero) {
1177 index_fp_volatile++;
1178 ctx->registers[v] = fp_volatile[index_fp_volatile++];
1179 index_fp_volatile++;
1180 goto success;
1181 }
1182 continue;
1183 }
1184 #endif
1185 if (index_fp_saved < n_fp_saved + zero) {
1186 ctx->registers[v] = fp_saved[index_fp_saved++];
1187 } else if (index_fp_volatile < n_fp_volatile + zero) {
1188 ctx->registers[v] = fp_volatile[index_fp_volatile++];
1189 } else {
1190 continue;
1191 }
1192 } else {
1193 continue;
1194 }
1195 } else {
1196 continue;
1197 }
1198 goto success;
1199 success:
1200 if (!reg_is_saved(ctx->registers[v])) {
1201 if (unlikely(!array_add_mayfail(frame_t, &ctx->need_spill, &ctx->need_spill_l, v, NULL, &ctx->err)))
1202 return false;
1203 }
1204 }
1205
1206 return true;
1207 }
1208
1209 static bool attr_w gen_fused_binary(struct codegen_context *ctx, unsigned mode, unsigned op_size, unsigned op, uint32_t escape_label, frame_t slot_1, frame_t slot_2, frame_t slot_r, bool *failed)
1210 {
1211 const code_t *backup = ctx->current_position;
1212 code_t code;
1213 frame_t slot_dr, slot_test;
1214 int32_t offs_false;
1215
1216 *failed = false;
1217
1218 next_code:
1219 code = get_code(ctx);
1220 ctx->arg_mode = code / OPCODE_MODE_MULT;
1221 code %= OPCODE_MODE_MULT;
1222 ajla_assert_lo(ctx->arg_mode < ARG_MODE_N, (file_line, "gen_fused_binary: invalid opcode %04x", (unsigned)*ctx->instr_start));
1223
1224 if (code == OPCODE_DEREFERENCE) {
1225 get_one(ctx, &slot_dr);
1226 const struct type *t = get_type_of_local(ctx, slot_dr);
1227 if (!TYPE_TAG_IS_BUILTIN(t->tag)) {
1228 *failed = true;
1229 goto fail;
1230 }
1231 if (unlikely(!flag_is_clear(ctx, slot_dr))) {
1232 *failed = true;
1233 goto fail;
1234 }
1235 goto next_code;
1236 }
1237 if (code == OPCODE_DEREFERENCE_CLEAR) {
1238 *failed = true;
1239 goto fail;
1240 }
1241 if (unlikely(code != OPCODE_JMP_FALSE))
1242 internal(file_line, "gen_fused_binary: binary operation is not followed by jmp false: %x, %s", code, decode_opcode(code, true));
1243 get_one(ctx, &slot_test);
1244 if (unlikely(slot_test != slot_r))
1245 internal(file_line, "gen_fused_binary: the result of the binary operation and the tested variable do not match");
1246 offs_false = get_jump_offset(ctx);
1247 get_jump_offset(ctx);
1248
1249 if (mode == MODE_ARRAY_LEN_GT) {
1250 g(gen_array_len(ctx, slot_1, slot_2, slot_r, true, offs_false));
1251 } else if (mode == MODE_REAL) {
1252 g(gen_fp_alu_jmp(ctx, op_size, op, escape_label, slot_1, slot_2, offs_false, failed));
1253 } else {
1254 g(gen_alu_jmp(ctx, mode, op_size, op, slot_1, slot_2, offs_false, failed));
1255 }
1256
1257 fail:
1258 if (*failed)
1259 ctx->current_position = backup;
1260
1261 return true;
1262 }
1263
1264 static bool attr_w gen_function(struct codegen_context *ctx)
1265 {
1266 ctx->current_position = da(ctx->fn,function)->code;
1267
1268 ctx->escape_nospill_label = alloc_label(ctx);
1269 if (unlikely(!ctx->escape_nospill_label))
1270 return false;
1271
1272 while (ctx->current_position != da(ctx->fn,function)->code + da(ctx->fn,function)->code_size) {
1273 ip_t ip;
1274 code_t code;
1275 unsigned op, type;
1276 frame_t slot_1, slot_2, slot_3, slot_r, flags, fn_idx, opt;
1277 arg_t n_args, n_ret, i_arg;
1278 uint32_t label_id;
1279 uint32_t escape_label;
1280 bool failed;
1281
1282 ajla_assert_lo(ctx->current_position < da(ctx->fn,function)->code + da(ctx->fn,function)->code_size, (file_line, "gen_function: ran out of code in %s", da(ctx->fn,function)->function_name));
1283
1284 ctx->instr_start = ctx->current_position;
1285
1286 /*debug("%s: %04x, %s", da(ctx->fn,function)->function_name, *ctx->instr_start, decode_opcode(*ctx->instr_start, true));*/
1287
1288 ip = ctx->instr_start - da(ctx->fn,function)->code;
1289 if (likely(!ctx->code_labels[ip])) {
1290 ctx->code_labels[ip] = alloc_label(ctx);
1291 if (unlikely(!ctx->code_labels[ip]))
1292 return false;
1293 }
1294 gen_label(ctx->code_labels[ip]);
1295
1296 code = get_code(ctx);
1297 ctx->arg_mode = code / OPCODE_MODE_MULT;
1298 code %= OPCODE_MODE_MULT;
1299 ajla_assert_lo(ctx->arg_mode < ARG_MODE_N, (file_line, "gen_function: invalid opcode %04x", (unsigned)*ctx->instr_start));
1300
1301 if (code >= OPCODE_FIXED_OP + uzero && code < OPCODE_INT_OP) {
1302 code -= OPCODE_FIXED_OP;
1303 op = (code / OPCODE_FIXED_OP_MULT) % OPCODE_FIXED_TYPE_MULT;
1304 type = code / OPCODE_FIXED_TYPE_MULT;
1305 if (op < OPCODE_FIXED_OP_C) {
1306 get_two(ctx, &slot_1, &slot_2);
1307 get_two(ctx, &slot_r, &flags);
1308 escape_label = alloc_escape_label(ctx);
1309 if (unlikely(!escape_label))
1310 return false;
1311 g(gen_test_2_cached(ctx, slot_1, slot_2, escape_label));
1312 flag_set(ctx, slot_1, false);
1313 flag_set(ctx, slot_2, false);
1314 flag_set(ctx, slot_r, false);
1315 if (flags & OPCODE_FLAG_FUSED) {
1316 g(gen_fused_binary(ctx, MODE_FIXED, type, op, escape_label, slot_1, slot_2, slot_r, &failed));
1317 if (unlikely(!failed))
1318 continue;
1319 }
1320 g(gen_alu(ctx, MODE_FIXED, type, op, escape_label, slot_1, slot_2, slot_r));
1321 continue;
1322 } else if (op < OPCODE_FIXED_OP_UNARY) {
1323 op -= OPCODE_FIXED_OP_C;
1324 get_two(ctx, &slot_1, &slot_2);
1325 get_two(ctx, &slot_r, &flags);
1326 escape_label = alloc_escape_label(ctx);
1327 if (unlikely(!escape_label))
1328 return false;
1329 g(gen_test_1_cached(ctx, slot_1, escape_label));
1330 flag_set(ctx, slot_1, false);
1331 flag_set(ctx, slot_r, false);
1332 slot_2 = frame_t_from_const((int32_t)slot_2);
1333 if (flags & OPCODE_FLAG_FUSED) {
1334 g(gen_fused_binary(ctx, MODE_FIXED, type, op, escape_label, slot_1, slot_2, slot_r, &failed));
1335 if (unlikely(!failed))
1336 continue;
1337 }
1338 g(gen_alu(ctx, MODE_FIXED, type, op, escape_label, slot_1, slot_2, slot_r));
1339 continue;
1340 } else if (op < OPCODE_FIXED_OP_N) {
1341 get_two(ctx, &slot_1, &slot_r);
1342 get_one(ctx, &flags);
1343 escape_label = alloc_escape_label(ctx);
1344 if (unlikely(!escape_label))
1345 return false;
1346 g(gen_test_1_cached(ctx, slot_1, escape_label));
1347 flag_set(ctx, slot_1, false);
1348 flag_set(ctx, slot_r, false);
1349 g(gen_alu1(ctx, MODE_FIXED, type, op, escape_label, slot_1, slot_r));
1350 continue;
1351 } else if (op == OPCODE_FIXED_OP_ldc) {
1352 unsigned i;
1353 get_one(ctx, &slot_r);
1354 g(gen_constant(ctx, false, type, false, slot_r));
1355 for (i = 0; i < 1U << type; i += 2)
1356 get_code(ctx);
1357 flag_set(ctx, slot_r, false);
1358 continue;
1359 } else if (op == OPCODE_FIXED_OP_ldc16) {
1360 get_one(ctx, &slot_r);
1361 g(gen_constant(ctx, false, type, true, slot_r));
1362 get_code(ctx);
1363 flag_set(ctx, slot_r, false);
1364 continue;
1365 } else if (op == OPCODE_FIXED_OP_move || op == OPCODE_FIXED_OP_copy) {
1366 get_two(ctx, &slot_1, &slot_r);
1367 escape_label = alloc_escape_label(ctx);
1368 if (unlikely(!escape_label))
1369 return false;
1370 g(gen_test_1_cached(ctx, slot_1, escape_label));
1371 flag_set(ctx, slot_1, false);
1372 flag_set(ctx, slot_r, false);
1373 g(gen_copy(ctx, type, slot_1, slot_r));
1374 continue;
1375 } else {
1376 internal(file_line, "gen_function: bad fixed code %04x", *ctx->instr_start);
1377 }
1378 } else if (code >= OPCODE_INT_OP && code < OPCODE_REAL_OP) {
1379 code -= OPCODE_INT_OP;
1380 op = (code / OPCODE_INT_OP_MULT) % OPCODE_INT_TYPE_MULT;
1381 type = code / OPCODE_INT_TYPE_MULT;
1382 if (op < OPCODE_INT_OP_C) {
1383 get_two(ctx, &slot_1, &slot_2);
1384 get_two(ctx, &slot_r, &flags);
1385 escape_label = alloc_escape_label(ctx);
1386 if (unlikely(!escape_label))
1387 return false;
1388 g(gen_test_2_cached(ctx, slot_1, slot_2, escape_label));
1389 flag_set(ctx, slot_1, false);
1390 flag_set(ctx, slot_2, false);
1391 flag_set(ctx, slot_r, false);
1392 if (flags & OPCODE_FLAG_FUSED) {
1393 g(gen_fused_binary(ctx, MODE_INT, type, op, escape_label, slot_1, slot_2, slot_r, &failed));
1394 if (unlikely(!failed))
1395 continue;
1396 }
1397 g(gen_alu(ctx, MODE_INT, type, op, escape_label, slot_1, slot_2, slot_r));
1398 continue;
1399 } else if (op < OPCODE_INT_OP_UNARY) {
1400 op -= OPCODE_INT_OP_C;
1401 get_two(ctx, &slot_1, &slot_2);
1402 get_two(ctx, &slot_r, &flags);
1403 escape_label = alloc_escape_label(ctx);
1404 if (unlikely(!escape_label))
1405 return false;
1406 g(gen_test_1_cached(ctx, slot_1, escape_label));
1407 flag_set(ctx, slot_1, false);
1408 flag_set(ctx, slot_r, false);
1409 slot_2 = frame_t_from_const((int32_t)slot_2);
1410 if (flags & OPCODE_FLAG_FUSED) {
1411 g(gen_fused_binary(ctx, MODE_INT, type, op, escape_label, slot_1, slot_2, slot_r, &failed));
1412 if (unlikely(!failed))
1413 continue;
1414 }
1415 g(gen_alu(ctx, MODE_INT, type, op, escape_label, slot_1, slot_2, slot_r));
1416 continue;
1417 } else if (op < OPCODE_INT_OP_N) {
1418 get_two(ctx, &slot_1, &slot_r);
1419 get_one(ctx, &flags);
1420 if ((op == OPCODE_INT_OP_to_int || op == OPCODE_INT_OP_from_int) && slot_1 == slot_r)
1421 continue;
1422 escape_label = alloc_escape_label(ctx);
1423 if (unlikely(!escape_label))
1424 return false;
1425 g(gen_test_1_cached(ctx, slot_1, escape_label));
1426 flag_set(ctx, slot_1, false);
1427 flag_set(ctx, slot_r, false);
1428 g(gen_alu1(ctx, MODE_INT, type, op, escape_label, slot_1, slot_r));
1429 continue;
1430 } else if (op == OPCODE_INT_OP_ldc) {
1431 unsigned i;
1432 get_one(ctx, &slot_r);
1433 g(gen_constant(ctx, false, type, false, slot_r));
1434 for (i = 0; i < 1U << type; i += 2)
1435 get_code(ctx);
1436 flag_set(ctx, slot_r, false);
1437 continue;
1438 } else if (op == OPCODE_INT_OP_ldc16) {
1439 get_one(ctx, &slot_r);
1440 g(gen_constant(ctx, false, type, true, slot_r));
1441 get_code(ctx);
1442 flag_set(ctx, slot_r, false);
1443 continue;
1444 } else if (op == OPCODE_INT_OP_move || op == OPCODE_INT_OP_copy) {
1445 get_two(ctx, &slot_1, &slot_r);
1446 escape_label = alloc_escape_label(ctx);
1447 if (unlikely(!escape_label))
1448 return false;
1449 g(gen_test_1_cached(ctx, slot_1, escape_label));
1450 flag_set(ctx, slot_1, false);
1451 flag_set(ctx, slot_r, false);
1452 g(gen_copy(ctx, type, slot_1, slot_r));
1453 continue;
1454 } else {
1455 internal(file_line, "gen_function: bad integer code %04x", *ctx->instr_start);
1456 }
1457 } else if (code >= OPCODE_REAL_OP && code < OPCODE_BOOL_OP) {
1458 code -= OPCODE_REAL_OP;
1459 op = (code / OPCODE_REAL_OP_MULT) % OPCODE_REAL_TYPE_MULT;
1460 type = code / OPCODE_REAL_TYPE_MULT;
1461 if (op < OPCODE_REAL_OP_UNARY) {
1462 get_two(ctx, &slot_1, &slot_2);
1463 get_two(ctx, &slot_r, &flags);
1464 escape_label = alloc_escape_label(ctx);
1465 if (unlikely(!escape_label))
1466 return false;
1467 g(gen_test_2_cached(ctx, slot_1, slot_2, escape_label));
1468 flag_set(ctx, slot_1, false);
1469 flag_set(ctx, slot_2, false);
1470 flag_set(ctx, slot_r, false);
1471 if (flags & OPCODE_FLAG_FUSED) {
1472 g(gen_fused_binary(ctx, MODE_REAL, type, op, escape_label, slot_1, slot_2, slot_r, &failed));
1473 if (unlikely(!failed))
1474 continue;
1475 }
1476 g(gen_fp_alu(ctx, type, op, escape_label, slot_1, slot_2, slot_r));
1477 continue;
1478 } else if (op < OPCODE_REAL_OP_N) {
1479 get_two(ctx, &slot_1, &slot_r);
1480 get_one(ctx, &flags);
1481 escape_label = alloc_escape_label(ctx);
1482 if (unlikely(!escape_label))
1483 return false;
1484 g(gen_test_1_cached(ctx, slot_1, escape_label));
1485 flag_set(ctx, slot_1, false);
1486 flag_set(ctx, slot_r, false);
1487 g(gen_fp_alu1(ctx, type, op, escape_label, slot_1, slot_r));
1488 continue;
1489 } else if (op == OPCODE_REAL_OP_ldc) {
1490 const struct type *t;
1491 unsigned i;
1492 get_one(ctx, &slot_r);
1493 t = type_get_real(type);
1494 g(gen_real_constant(ctx, t, slot_r));
1495 for (i = 0; i < t->size; i += 2)
1496 get_code(ctx);
1497 flag_set(ctx, slot_r, false);
1498 continue;
1499 } else if (op == OPCODE_REAL_OP_move || op == OPCODE_REAL_OP_copy) {
1500 get_two(ctx, &slot_1, &slot_r);
1501 escape_label = alloc_escape_label(ctx);
1502 if (unlikely(!escape_label))
1503 return false;
1504 g(gen_test_1_cached(ctx, slot_1, escape_label));
1505 flag_set(ctx, slot_1, false);
1506 flag_set(ctx, slot_r, false);
1507 g(gen_memcpy_slots(ctx, slot_r, slot_1));
1508 continue;
1509 } else {
1510 internal(file_line, "gen_function: bad real code %04x", *ctx->instr_start);
1511 }
1512 } else if (code >= OPCODE_BOOL_OP && code < OPCODE_EXTRA) {
1513 code -= OPCODE_BOOL_OP;
1514 op = (code / OPCODE_BOOL_OP_MULT) % OPCODE_BOOL_TYPE_MULT;
1515 type = log_2(sizeof(ajla_flat_option_t));
1516 if (op < OPCODE_BOOL_OP_UNARY) {
1517 get_two(ctx, &slot_1, &slot_2);
1518 get_two(ctx, &slot_r, &flags);
1519 escape_label = alloc_escape_label(ctx);
1520 if (unlikely(!escape_label))
1521 return false;
1522 g(gen_test_2_cached(ctx, slot_1, slot_2, escape_label));
1523 flag_set(ctx, slot_1, false);
1524 flag_set(ctx, slot_2, false);
1525 flag_set(ctx, slot_r, false);
1526 if (flags & OPCODE_FLAG_FUSED) {
1527 g(gen_fused_binary(ctx, MODE_BOOL, type, op, escape_label, slot_1, slot_2, slot_r, &failed));
1528 if (unlikely(!failed))
1529 continue;
1530 }
1531 g(gen_alu(ctx, MODE_BOOL, type, op, escape_label, slot_1, slot_2, slot_r));
1532 continue;
1533 } else if (op < OPCODE_BOOL_OP_N) {
1534 get_two(ctx, &slot_1, &slot_r);
1535 get_one(ctx, &flags);
1536 escape_label = alloc_escape_label(ctx);
1537 if (unlikely(!escape_label))
1538 return false;
1539 g(gen_test_1_cached(ctx, slot_1, escape_label));
1540 flag_set(ctx, slot_1, false);
1541 flag_set(ctx, slot_r, false);
1542 g(gen_alu1(ctx, MODE_BOOL, type, op, escape_label, slot_1, slot_r));
1543 continue;
1544 } else if (op == OPCODE_BOOL_OP_move || op == OPCODE_BOOL_OP_copy) {
1545 get_two(ctx, &slot_1, &slot_r);
1546 escape_label = alloc_escape_label(ctx);
1547 if (unlikely(!escape_label))
1548 return false;
1549 g(gen_test_1_cached(ctx, slot_1, escape_label));
1550 flag_set(ctx, slot_1, false);
1551 flag_set(ctx, slot_r, false);
1552 g(gen_copy(ctx, type, slot_1, slot_r));
1553 continue;
1554 } else {
1555 internal(file_line, "gen_function: bad boolean code %04x", *ctx->instr_start);
1556 }
1557 } else switch (code) {
1558 case OPCODE_INT_LDC_LONG: {
1559 uint32_t words, w;
1560 get_one(ctx, &slot_r);
1561 words = get_uint32(ctx);
1562 for (w = 0; w < words; w++)
1563 get_code(ctx);
1564 unconditional_escape:
1565 escape_label = alloc_escape_label(ctx);
1566 if (unlikely(!escape_label))
1567 return false;
1568 gen_insn(INSN_JMP, 0, 0, 0);
1569 gen_four(escape_label);
1570 continue;
1571 }
1572 case OPCODE_IS_EXCEPTION: {
1573 get_two(ctx, &slot_1, &slot_r);
1574 get_one(ctx, &flags);
1575 g(gen_is_exception(ctx, slot_1, slot_r));
1576 continue;
1577 }
1578 case OPCODE_EXCEPTION_CLASS:
1579 case OPCODE_EXCEPTION_TYPE:
1580 case OPCODE_EXCEPTION_AUX: {
1581 get_two(ctx, &slot_1, &slot_r);
1582 get_one(ctx, &flags);
1583 goto unconditional_escape;
1584 }
1585 case OPCODE_SYSTEM_PROPERTY: {
1586 get_two(ctx, &slot_1, &slot_r);
1587 get_one(ctx, &flags);
1588 g(gen_system_property(ctx, slot_1, slot_r));
1589 continue;
1590 }
1591 case OPCODE_FLAT_MOVE:
1592 case OPCODE_FLAT_COPY: {
1593 get_two(ctx, &slot_1, &slot_r);
1594 g(gen_flat_move_copy(ctx, slot_1, slot_r));
1595 continue;
1596 }
1597 case OPCODE_REF_MOVE:
1598 case OPCODE_REF_MOVE_CLEAR:
1599 case OPCODE_REF_COPY: {
1600 get_two(ctx, &slot_1, &slot_r);
1601 g(gen_ref_move_copy(ctx, code, slot_1, slot_r));
1602 continue;
1603 }
1604 case OPCODE_BOX_MOVE_CLEAR:
1605 case OPCODE_BOX_COPY: {
1606 get_two(ctx, &slot_1, &slot_r);
1607 g(gen_box_move_copy(ctx, code, slot_1, slot_r));
1608 continue;
1609 }
1610 case OPCODE_TAKE_BORROWED:
1611 get_one(ctx, &slot_1);
1612 if (!da(ctx->fn,function)->local_variables_flags[slot_1].may_be_borrowed)
1613 continue;
1614 if (unlikely(!(label_id = alloc_label(ctx))))
1615 return false;
1616 if (flag_is_set(ctx, slot_1))
1617 goto take_borrowed_done;
1618 if (flag_is_clear(ctx, slot_1)) {
1619 g(gen_set_1(ctx, R_FRAME, slot_1, 0, true));
1620 goto do_take_borrowed;
1621 }
1622 g(gen_test_1(ctx, R_FRAME, slot_1, 0, label_id, false, TEST_SET));
1623 do_take_borrowed:
1624 g(gen_upcall_start(ctx, 1));
1625 g(gen_frame_load(ctx, OP_SIZE_SLOT, garbage, slot_1, 0, false, R_ARG0));
1626 g(gen_upcall_argument(ctx, 0));
1627 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_pointer_reference_owned), 1));
1628 flag_set(ctx, slot_1, true);
1629 take_borrowed_done:
1630 gen_label(label_id);
1631 continue;
1632 case OPCODE_DEREFERENCE:
1633 case OPCODE_DEREFERENCE_CLEAR: {
1634 bool need_bit_test;
1635 /*const struct type *type;*/
1636 get_one(ctx, &slot_1);
1637 if (flag_is_clear(ctx, slot_1))
1638 goto skip_dereference;
1639 /*type = get_type_of_local(ctx, slot_1);*/
1640 /*need_bit_test = 1 || TYPE_IS_FLAT(type) || da(ctx->fn,function)->local_variables[slot_1].may_be_borrowed;*/
1641 need_bit_test = !flag_is_set(ctx, slot_1);
1642 if (need_bit_test) {
1643 if (unlikely(!(label_id = alloc_label(ctx))))
1644 return false;
1645 g(gen_test_1(ctx, R_FRAME, slot_1, 0, label_id, true, TEST_CLEAR));
1646 } else {
1647 g(gen_set_1(ctx, R_FRAME, slot_1, 0, false));
1648 label_id = 0; /* avoid warning */
1649 }
1650 g(gen_upcall_start(ctx, 1));
1651 g(gen_frame_load(ctx, OP_SIZE_SLOT, garbage, slot_1, 0, false, R_ARG0));
1652 g(gen_upcall_argument(ctx, 0));
1653 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_pointer_dereference), 1));
1654 if (need_bit_test)
1655 gen_label(label_id);
1656 skip_dereference:
1657 if (code == OPCODE_DEREFERENCE_CLEAR)
1658 g(gen_frame_clear(ctx, OP_SIZE_SLOT, slot_1));
1659 flag_set_unknown(ctx, slot_1);
1660 flag_set(ctx, slot_1, false);
1661 continue;
1662 }
1663 case OPCODE_EVAL: {
1664 get_one(ctx, &slot_1);
1665 g(gen_eval(ctx, slot_1));
1666 continue;
1667 }
1668 case OPCODE_ESCAPE_NONFLAT: {
1669 frame_t n, i;
1670 frame_t *vars;
1671
1672 get_one(ctx, &n);
1673 vars = mem_alloc_array_mayfail(mem_alloc_mayfail, frame_t *, 0, 0, n, sizeof(frame_t), &ctx->err);
1674 if (unlikely(!vars))
1675 return false;
1676 for (i = 0; i < n; i++) {
1677 get_one(ctx, &vars[i]);
1678 }
1679
1680 escape_label = alloc_escape_label(ctx);
1681 if (unlikely(!escape_label)) {
1682 mem_free(vars);
1683 return false;
1684 }
1685
1686 if (unlikely(!gen_test_variables(ctx, vars, n, escape_label))) {
1687 mem_free(vars);
1688 return false;
1689 }
1690 mem_free(vars);
1691
1692 continue;
1693 }
1694 case OPCODE_CHECKPOINT: {
1695 frame_t n_vars;
1696
1697 g(clear_flag_cache(ctx));
1698
1699 if (SIZEOF_IP_T == 2) {
1700 slot_1 = get_code(ctx);
1701 } else if (SIZEOF_IP_T == 4) {
1702 slot_1 = get_uint32(ctx);
1703 } else {
1704 not_reached();
1705 continue;
1706 }
1707
1708 if (unlikely(!(slot_1 + 1)))
1709 return false;
1710 while (slot_1 >= ctx->n_entries) {
1711 void *err_entries;
1712 struct cg_entry e;
1713 if (unlikely(!ctx->entries)) {
1714 if (unlikely(!array_init_mayfail(struct cg_entry, &ctx->entries, &ctx->n_entries, &ctx->err)))
1715 return false;
1716 }
1717 memset(&e, 0, sizeof(struct cg_entry));
1718 if (unlikely(!array_add_mayfail(struct cg_entry, &ctx->entries, &ctx->n_entries, e, &err_entries, &ctx->err))) {
1719 ctx->entries = err_entries;
1720 return false;
1721 }
1722 }
1723
1724 get_one(ctx, &n_vars);
1725
1726 escape_label = 0; /* avoid warning */
1727 if (likely(slot_1 != 0)) {
1728 escape_label = alloc_escape_label(ctx);
1729 if (unlikely(!escape_label))
1730 return false;
1731 }
1732
1733 if (n_vars || !slot_1) {
1734 frame_t i;
1735 uint32_t entry_label, nonflat_label;
1736 struct cg_entry *ce = &ctx->entries[slot_1];
1737
1738 if (unlikely(!array_init_mayfail(frame_t, &ce->variables, &ce->n_variables, &ctx->err)))
1739 return false;
1740 for (i = 0; i < n_vars; i++) {
1741 frame_t v;
1742 get_one(ctx, &v);
1743 if (unlikely(!array_add_mayfail(frame_t, &ce->variables, &ce->n_variables, v, NULL, &ctx->err)))
1744 return false;
1745 }
1746 if (!slot_1) {
1747 g(gen_test_variables(ctx, ce->variables, ce->n_variables, ctx->escape_nospill_label));
1748 }
1749 entry_label = alloc_label(ctx);
1750 if (unlikely(!entry_label))
1751 return false;
1752 gen_label(entry_label);
1753 ce->entry_label = entry_label;
1754
1755 nonflat_label = alloc_escape_label_for_ip(ctx, ctx->current_position);
1756 if (unlikely(!nonflat_label))
1757 return false;
1758 ce->nonflat_label = nonflat_label;
1759
1760 if (unlikely(!slot_1))
1761 g(gen_timestamp_test(ctx, ctx->escape_nospill_label));
1762 else
1763 g(gen_timestamp_test(ctx, escape_label));
1764 } else {
1765 g(gen_timestamp_test(ctx, escape_label));
1766
1767 gen_insn(INSN_ENTRY, 0, 0, 0);
1768 gen_four(slot_1);
1769 }
1770 continue;
1771 }
1772 case OPCODE_JMP: {
1773 int32_t x = get_jump_offset(ctx);
1774 g(gen_jump(ctx, x, OP_SIZE_NATIVE, COND_ALWAYS, -1U, -1U));
1775 continue;
1776 }
1777 case OPCODE_JMP_BACK_16: {
1778 int32_t x = get_code(ctx);
1779 g(gen_jump(ctx, -x - (int)(2 * sizeof(code_t)), OP_SIZE_NATIVE, COND_ALWAYS, -1U, -1U));
1780 continue;
1781 }
1782 case OPCODE_JMP_FALSE: {
1783 int32_t offs_false;
1784 get_one(ctx, &slot_1);
1785 offs_false = get_jump_offset(ctx);
1786 get_jump_offset(ctx);
1787 escape_label = alloc_escape_label(ctx);
1788 if (unlikely(!escape_label))
1789 return false;
1790 g(gen_test_1_cached(ctx, slot_1, escape_label));
1791 flag_set(ctx, slot_1, false);
1792 g(gen_cond_jump(ctx, slot_1, offs_false));
1793 continue;
1794 }
1795 case OPCODE_LABEL: {
1796 g(clear_flag_cache(ctx));
1797 continue;
1798 }
1799 #define init_args \
1800 do { \
1801 if (ctx->args != NULL) \
1802 mem_free(ctx->args); \
1803 g(array_init_mayfail(struct code_arg, &ctx->args, &ctx->args_l, &ctx->err));\
1804 } while (0)
1805 #define load_args \
1806 do { \
1807 init_args; \
1808 for (i_arg = 0; i_arg < n_args; i_arg++) { \
1809 struct code_arg a; \
1810 get_two(ctx, &a.slot, &a.flags); \
1811 a.type = 0; \
1812 g(array_add_mayfail(struct code_arg, &ctx->args, &ctx->args_l, a, NULL, &ctx->err));\
1813 } \
1814 } while (0)
1815 case OPCODE_LOAD_FN:
1816 get_two(ctx, &n_args, &slot_r);
1817 get_one(ctx, &fn_idx);
1818 load_args;
1819 g(gen_load_fn_or_curry(ctx, fn_idx, NO_FRAME_T, slot_r, 0));
1820 continue;
1821 case OPCODE_CURRY:
1822 get_two(ctx, &n_args, &slot_r);
1823 get_two(ctx, &slot_1, &flags);
1824 load_args;
1825 g(gen_load_fn_or_curry(ctx, NO_FRAME_T, slot_1, slot_r, flags));
1826 continue;
1827 case OPCODE_CALL:
1828 case OPCODE_CALL_STRICT:
1829 case OPCODE_CALL_SPARK:
1830 case OPCODE_CALL_LAZY:
1831 case OPCODE_CALL_CACHE:
1832 case OPCODE_CALL_SAVE: {
1833 get_two(ctx, &n_args, &n_ret);
1834 get_one(ctx, &fn_idx);
1835 jump_over_arguments_and_return:
1836 load_args;
1837 ctx->return_values = ctx->current_position;
1838 for (i_arg = 0; i_arg < n_ret; i_arg++) {
1839 #if ARG_MODE_N >= 3
1840 get_uint32(ctx);
1841 #else
1842 get_code(ctx);
1843 #endif
1844 get_code(ctx);
1845 }
1846 if (unlikely(profiling))
1847 goto unconditional_escape;
1848 if (code == OPCODE_CALL || code == OPCODE_CALL_STRICT) {
1849 g(gen_call(ctx, code, fn_idx));
1850 continue;
1851 }
1852 /*if (code == OPCODE_CALL_INDIRECT || code == OPCODE_CALL_INDIRECT_STRICT) {
1853 if (unlikely(!gen_call_indirect(ctx, code, slot_1, flags)))
1854 return false;
1855 continue;
1856 }*/
1857 goto unconditional_escape;
1858 }
1859 case OPCODE_CALL_INDIRECT:
1860 case OPCODE_CALL_INDIRECT_STRICT:
1861 case OPCODE_CALL_INDIRECT_SPARK:
1862 case OPCODE_CALL_INDIRECT_LAZY:
1863 case OPCODE_CALL_INDIRECT_CACHE:
1864 case OPCODE_CALL_INDIRECT_SAVE: {
1865 fn_idx = 0; /* avoid warning */
1866 get_two(ctx, &n_args, &n_ret);
1867 get_two(ctx, &slot_1, &flags);
1868 goto jump_over_arguments_and_return;
1869 }
1870 case OPCODE_RETURN: {
1871 n_args = da(ctx->fn,function)->n_return_values;
1872 load_args;
1873 if (unlikely(profiling))
1874 goto unconditional_escape;
1875 g(gen_return(ctx));
1876 continue;
1877 }
1878 case OPCODE_STRUCTURED: {
1879 init_args;
1880 get_two(ctx, &slot_1, &slot_2);
1881 do {
1882 struct code_arg a;
1883 get_two(ctx, &flags, &slot_r);
1884 get_one(ctx, &opt);
1885 a.slot = slot_r;
1886 a.flags = flags;
1887 a.type = opt;
1888 g(array_add_mayfail(struct code_arg, &ctx->args, &ctx->args_l, a, NULL, &ctx->err));
1889 } while (!(flags & OPCODE_STRUCTURED_FLAG_END));
1890 g(gen_structured(ctx, slot_1, slot_2));
1891 continue;
1892 }
1893 case OPCODE_RECORD_CREATE: {
1894 init_args;
1895 get_two(ctx, &slot_r, &n_args);
1896 for (i_arg = 0; i_arg < n_args; i_arg++) {
1897 struct code_arg a;
1898 get_two(ctx, &slot_1, &flags);
1899 a.slot = slot_1;
1900 a.flags = flags;
1901 a.type = 0;
1902 g(array_add_mayfail(struct code_arg, &ctx->args, &ctx->args_l, a, NULL, &ctx->err));
1903 }
1904 g(gen_record_create(ctx, slot_r));
1905 continue;
1906 }
1907 case OPCODE_RECORD_LOAD: {
1908 get_two(ctx, &slot_1, &opt);
1909 get_two(ctx, &slot_r, &flags);
1910 g(gen_record_load(ctx, slot_1, slot_r, opt, flags));
1911 continue;
1912 }
1913 case OPCODE_OPTION_CREATE_EMPTY_FLAT: {
1914 get_two(ctx, &slot_r, &opt);
1915 g(gen_option_create_empty_flat(ctx, opt, slot_r));
1916 continue;
1917 }
1918 case OPCODE_OPTION_CREATE_EMPTY: {
1919 get_two(ctx, &slot_r, &opt);
1920 g(gen_option_create_empty(ctx, opt, slot_r));
1921 continue;
1922 }
1923 case OPCODE_OPTION_CREATE: {
1924 get_two(ctx, &slot_r, &opt);
1925 get_two(ctx, &slot_1, &flags);
1926 g(gen_option_create(ctx, opt, slot_1, slot_r, flags));
1927 continue;
1928 }
1929 case OPCODE_OPTION_LOAD: {
1930 get_two(ctx, &slot_1, &opt);
1931 get_two(ctx, &slot_r, &flags);
1932 g(gen_option_load(ctx, slot_1, slot_r, opt, flags));
1933 continue;
1934 }
1935 case OPCODE_OPTION_TEST_FLAT: {
1936 get_two(ctx, &slot_1, &opt);
1937 get_one(ctx, &slot_r);
1938 g(gen_option_test_flat(ctx, slot_1, opt, slot_r));
1939 continue;
1940 }
1941 case OPCODE_OPTION_TEST: {
1942 get_two(ctx, &slot_1, &opt);
1943 get_one(ctx, &slot_r);
1944 g(gen_option_test(ctx, slot_1, opt, slot_r));
1945 continue;
1946 }
1947 case OPCODE_OPTION_ORD_FLAT: {
1948 get_two(ctx, &slot_1, &slot_r);
1949 g(gen_option_ord(ctx, slot_1, slot_r, true));
1950 continue;
1951 }
1952 case OPCODE_OPTION_ORD: {
1953 get_two(ctx, &slot_1, &slot_r);
1954 g(gen_option_ord(ctx, slot_1, slot_r, false));
1955 continue;
1956 }
1957 case OPCODE_ARRAY_CREATE: {
1958 init_args;
1959 get_two(ctx, &slot_r, &n_args);
1960 for (i_arg = 0; i_arg < n_args; i_arg++) {
1961 struct code_arg a;
1962 get_two(ctx, &slot_1, &flags);
1963 a.slot = slot_1;
1964 a.flags = flags;
1965 a.type = 0;
1966 g(array_add_mayfail(struct code_arg, &ctx->args, &ctx->args_l, a, NULL, &ctx->err));
1967 }
1968 g(gen_array_create(ctx, slot_r));
1969 continue;
1970 }
1971 case OPCODE_ARRAY_CREATE_EMPTY_FLAT: {
1972 get_two(ctx, &slot_r, &flags);
1973 g(gen_array_create_empty_flat(ctx, slot_r, flags));
1974 continue;
1975 }
1976 case OPCODE_ARRAY_CREATE_EMPTY: {
1977 get_one(ctx, &slot_r);
1978 g(gen_array_create_empty(ctx, slot_r));
1979 continue;
1980 }
1981 case OPCODE_ARRAY_FILL: {
1982 get_two(ctx, &slot_1, &flags);
1983 get_two(ctx, &slot_2, &slot_r);
1984 g(gen_array_fill(ctx, slot_1, flags, slot_2, slot_r));
1985 continue;
1986 }
1987 case OPCODE_ARRAY_STRING: {
1988 frame_t i;
1989 get_two(ctx, &slot_r, &i);
1990 g(gen_array_string(ctx, type_get_fixed(0, true)->tag, cast_ptr(uint8_t *, ctx->current_position), i, slot_r));
1991 ctx->current_position += (i + 1) >> 1;
1992 continue;
1993 }
1994 case OPCODE_ARRAY_UNICODE: {
1995 frame_t i;
1996 get_two(ctx, &slot_r, &i);
1997 g(gen_array_string(ctx, type_get_int(2)->tag, cast_ptr(uint8_t *, ctx->current_position), i, slot_r));
1998 ctx->current_position += i * 2;
1999 continue;
2000 }
2001 case OPCODE_ARRAY_LOAD: {
2002 get_two(ctx, &slot_1, &slot_2);
2003 get_two(ctx, &slot_r, &flags);
2004 g(gen_array_load(ctx, slot_1, slot_2, slot_r, flags));
2005 continue;
2006 }
2007 case OPCODE_ARRAY_LEN: {
2008 get_two(ctx, &slot_1, &slot_r);
2009 get_one(ctx, &flags);
2010 g(gen_array_len(ctx, slot_1, NO_FRAME_T, slot_r, false, 0));
2011 continue;
2012 }
2013 case OPCODE_ARRAY_LEN_GREATER_THAN: {
2014 get_two(ctx, &slot_1, &slot_2);
2015 get_two(ctx, &slot_r, &flags);
2016 escape_label = alloc_escape_label(ctx);
2017 if (unlikely(!escape_label))
2018 return false;
2019 if (flags & OPCODE_FLAG_FUSED) {
2020 g(gen_fused_binary(ctx, MODE_ARRAY_LEN_GT, 0, 0, escape_label, slot_1, slot_2, slot_r, &failed));
2021 if (unlikely(!failed))
2022 continue;
2023 }
2024 g(gen_array_len(ctx, slot_1, slot_2, slot_r, false, 0));
2025 continue;
2026 }
2027 case OPCODE_ARRAY_SUB: {
2028 get_two(ctx, &slot_1, &slot_2);
2029 get_two(ctx, &slot_3, &slot_r);
2030 get_one(ctx, &flags);
2031 g(gen_array_sub(ctx, slot_1, slot_2, slot_3, slot_r, flags));
2032 continue;
2033 }
2034 case OPCODE_ARRAY_SKIP: {
2035 get_two(ctx, &slot_1, &slot_2);
2036 get_two(ctx, &slot_r, &flags);
2037 g(gen_array_skip(ctx, slot_1, slot_2, slot_r, flags));
2038 continue;
2039 }
2040 case OPCODE_ARRAY_APPEND: {
2041 get_two(ctx, &slot_r, &flags);
2042 get_two(ctx, &slot_1, &slot_2);
2043 g(gen_array_append(ctx, slot_1, slot_2, slot_r, flags));
2044 continue;
2045 }
2046 case OPCODE_ARRAY_APPEND_ONE_FLAT: {
2047 get_two(ctx, &slot_r, &flags);
2048 get_two(ctx, &slot_1, &slot_2);
2049 g(gen_array_append_one_flat(ctx, slot_1, slot_2, slot_r, flags));
2050 continue;
2051 }
2052 case OPCODE_ARRAY_APPEND_ONE: {
2053 get_two(ctx, &slot_r, &flags);
2054 get_two(ctx, &slot_1, &slot_2);
2055 g(gen_array_append_one(ctx, slot_1, slot_2, slot_r, flags));
2056 continue;
2057 }
2058 case OPCODE_ARRAY_FLATTEN: {
2059 get_two(ctx, &slot_r, &flags);
2060 get_one(ctx, &slot_1);
2061 goto unconditional_escape;
2062 }
2063 case OPCODE_IO: {
2064 get_two(ctx, &flags, &slot_1);
2065 get_two(ctx, &slot_2, &slot_3);
2066 g(gen_io(ctx, flags, slot_1, slot_2, slot_3));
2067 continue;
2068 }
2069 case OPCODE_INTERNAL_FUNCTION:
2070 case OPCODE_EXIT_THREAD:
2071 case OPCODE_UNREACHABLE: {
2072 goto unconditional_escape;
2073 }
2074 default: {
2075 #if 1
2076 /*if (getenv("DUMP") && !strcmp(da(ctx->fn,function)->function_name, getenv("DUMP")))*/
2077 warning("gen_function: %s: unknown opcode %04x, %s", da(ctx->fn,function)->function_name, *ctx->instr_start, decode_opcode(*ctx->instr_start, false));
2078 #endif
2079 return false;
2080 }
2081 }
2082 }
2083
2084 return true;
2085 }
2086
2087 static bool attr_w gen_entries(struct codegen_context *ctx)
2088 {
2089 size_t i;
2090 for (i = 0; i < ctx->n_entries; i++) {
2091 struct cg_entry *ce = &ctx->entries[i];
2092 if (ce->entry_label) {
2093 gen_insn(INSN_ENTRY, 0, 0, 0);
2094 gen_four(i);
2095
2096 g(gen_test_variables(ctx, ce->variables, ce->n_variables, ce->nonflat_label));
2097
2098 gen_insn(INSN_JMP, 0, 0, 0);
2099 gen_four(ce->entry_label);
2100 }
2101 }
2102 return true;
2103 }
2104
2105 static bool attr_w gen_epilogues(struct codegen_context *ctx)
2106 {
2107 frame_t v;
2108 ip_t ip;
2109 uint32_t escape_label, nospill_label;
2110 escape_label = alloc_label(ctx);
2111 if (unlikely(!escape_label))
2112 return false;
2113 nospill_label = alloc_label(ctx);
2114 if (unlikely(!nospill_label))
2115 return false;
2116 #if defined(ARCH_PARISC)
2117 if (ctx->call_label) {
2118 gen_label(ctx->call_label);
2119 g(gen_call_millicode(ctx));
2120 }
2121 #endif
2122 if (ctx->reload_label) {
2123 gen_label(ctx->reload_label);
2124 g(gen_mov(ctx, i_size(OP_SIZE_ADDRESS), R_FRAME, R_RET0));
2125 g(gen_escape_arg(ctx, (ip_t)-1, nospill_label));
2126 }
2127 gen_label(ctx->escape_nospill_label);
2128 g(gen_escape_arg(ctx, 0, nospill_label));
2129 for (ip = 0; ip < da(ctx->fn,function)->code_size; ip++) {
2130 struct cg_exit *ce = ctx->code_exits[ip];
2131 if (ce && (ce->undo_label || ce->escape_label)) {
2132 if (ce->undo_label) {
2133 size_t i;
2134 gen_label(ce->undo_label);
2135 gen_insn(ce->undo_opcode, ce->undo_op_size, ce->undo_aux, ce->undo_writes_flags);
2136 for (i = 0; i < ce->undo_parameters_len; i++)
2137 gen_one(ce->undo_parameters[i]);
2138 }
2139 if (ce->escape_label) {
2140 gen_label(ce->escape_label);
2141 }
2142 g(gen_escape_arg(ctx, ip, escape_label));
2143 }
2144 }
2145 gen_label(escape_label);
2146 for (v = MIN_USEABLE_SLOT; v < function_n_variables(ctx->fn); v++) {
2147 if (slot_is_register(ctx, v)) {
2148 g(spill(ctx, v));
2149 }
2150 }
2151 gen_label(nospill_label);
2152 g(gen_escape(ctx));
2153 return true;
2154 }
2155
2156 static bool attr_w cgen_entry(struct codegen_context *ctx)
2157 {
2158 uint32_t entry_id = cget_four(ctx);
2159 ajla_assert_lo(entry_id < ctx->n_entries, (file_line, "cgen_entry: invalid entry %lx", (unsigned long)entry_id));
2160 ctx->entries[entry_id].entry_to_pos = ctx->mcode_size;
2161 return true;
2162 }
2163
2164 static bool attr_w cgen_label(struct codegen_context *ctx)
2165 {
2166 uint32_t label_id = cget_four(ctx);
2167 ctx->label_to_pos[label_id] = ctx->mcode_size;
2168 return true;
2169 }
2170
2171 static bool attr_w attr_unused cgen_trap(struct codegen_context *ctx, uint32_t label)
2172 {
2173 struct trap_record tr;
2174 tr.source_ip = ctx->mcode_size;
2175 tr.destination_ip = label;
2176 if (unlikely(!array_add_mayfail(struct trap_record, &ctx->trap_records, &ctx->trap_records_size, tr, NULL, &ctx->err)))
2177 return false;
2178 return true;
2179 }
2180
2181 static bool attr_w add_relocation(struct codegen_context *ctx, unsigned length, int offset, bool *known)
2182 {
2183 struct relocation rel;
2184 rel.label_id = cget_four(ctx);
2185 rel.length = length;
2186 rel.position = ctx->mcode_size;
2187 rel.jmp_instr = ctx->code_position - 8 - offset - ctx->code;
2188 if (unlikely(!array_add_mayfail(struct relocation, &ctx->reloc, &ctx->reloc_size, rel, NULL, &ctx->err)))
2189 return false;
2190 if (known)
2191 *known = ctx->label_to_pos[rel.label_id] != (size_t)-1;
2192 return true;
2193 }
2194
2195
2196 #if defined(ARCH_ALPHA)
2197 #include "c2-alpha.inc"
2198 #elif defined(ARCH_ARM32)
2199 #include "c2-arm.inc"
2200 #elif defined(ARCH_ARM64)
2201 #include "c2-arm64.inc"
2202 #elif defined(ARCH_IA64)
2203 #include "c2-ia64.inc"
2204 #elif defined(ARCH_LOONGARCH64)
2205 #include "c2-loong.inc"
2206 #elif defined(ARCH_MIPS)
2207 #include "c2-mips.inc"
2208 #elif defined(ARCH_PARISC)
2209 #include "c2-hppa.inc"
2210 #elif defined(ARCH_POWER)
2211 #include "c2-power.inc"
2212 #elif defined(ARCH_S390)
2213 #include "c2-s390.inc"
2214 #elif defined(ARCH_SPARC)
2215 #include "c2-sparc.inc"
2216 #elif defined(ARCH_RISCV64)
2217 #include "c2-riscv.inc"
2218 #elif defined(ARCH_X86)
2219 #include "c2-x86.inc"
2220 #endif
2221
2222
2223 static bool attr_w gen_mcode(struct codegen_context *ctx)
2224 {
2225 ctx->code_position = ctx->code;
2226
2227 while (ctx->code_position != ctx->code + ctx->code_size) {
2228 uint32_t insn;
2229 ajla_assert_lo(ctx->code_position < ctx->code + ctx->code_size, (file_line, "gen_mcode: ran out of code"));
2230 #ifdef DEBUG_INSNS
2231 insn = cget_four(ctx);
2232 debug("line: %u/%u", insn >> 24, insn & 0x00FFFFFFU);
2233 #endif
2234 insn = cget_four(ctx);
2235 g(cgen_insn(ctx, insn));
2236 }
2237
2238 return true;
2239 }
2240
2241 #define RELOCS_RETRY -1
2242 #define RELOCS_FAIL 0
2243 #define RELOCS_OK 1
2244
2245 static int8_t resolve_relocs(struct codegen_context *ctx)
2246 {
2247 size_t i;
2248 int8_t status = RELOCS_OK;
2249 for (i = 0; i < ctx->reloc_size; i++) {
2250 struct relocation *reloc = &ctx->reloc[i];
2251 if (!resolve_relocation(ctx, reloc)) {
2252 uint8_t *jmp_instr;
2253 uint32_t insn;
2254 uint32_t new_length;
2255 status = RELOCS_RETRY;
2256 if (unlikely(reloc->length + zero >= JMP_LIMIT))
2257 return RELOCS_FAIL;
2258 new_length = reloc->length + 1;
2259 jmp_instr = ctx->code + reloc->jmp_instr;
2260 insn = (uint32_t)jmp_instr[0] +
2261 ((uint32_t)jmp_instr[1] << 8) +
2262 ((uint32_t)jmp_instr[2] << 16) +
2263 ((uint32_t)jmp_instr[3] << 24);
2264 insn &= ~INSN_JUMP_SIZE;
2265 insn |= (uint32_t)new_length << INSN_JUMP_SIZE_SHIFT;
2266 jmp_instr[0] = insn;
2267 jmp_instr[1] = insn >> 8;
2268 jmp_instr[2] = insn >> 16;
2269 jmp_instr[3] = insn >> 24;
2270 }
2271 }
2272 return status;
2273 }
2274
2275 static void resolve_traps(struct codegen_context *ctx)
2276 {
2277 size_t i;
2278 for (i = 0; i < ctx->trap_records_size; i++) {
2279 struct trap_record *tr = &ctx->trap_records[i];
2280 tr->destination_ip = ctx->label_to_pos[tr->destination_ip];
2281 }
2282 }
2283
2284
2285 static bool attr_w codegen_map(struct codegen_context *ctx)
2286 {
2287 void *ptr;
2288 frame_t i;
2289 array_finish(uint8_t, &ctx->mcode, &ctx->mcode_size);
2290 ptr = os_code_map(ctx->mcode, ctx->mcode_size, &ctx->err);
2291 ctx->mcode = NULL;
2292 if (unlikely(!ptr)) {
2293 return false;
2294 }
2295 for (i = 0; i < ctx->n_entries; i++) {
2296 char *entry = cast_ptr(char *, ptr) + ctx->entries[i].entry_to_pos;
2297 da(ctx->codegen,codegen)->unoptimized_code[i] = entry;
2298 da(ctx->codegen,codegen)->n_entries++;
2299 }
2300 da(ctx->codegen,codegen)->unoptimized_code_base = ptr;
2301 da(ctx->codegen,codegen)->unoptimized_code_size = ctx->mcode_size;
2302
2303 return true;
2304 }
2305
2306
2307 void *codegen_fn(frame_s *fp, const code_t *ip, union internal_arg ia[])
2308 {
2309 struct codegen_context ctx_;
2310 struct codegen_context *ctx = &ctx_;
2311 frame_t i;
2312 int8_t rr;
2313 struct data *codegen;
2314 uint32_t l;
2315
2316 init_ctx(ctx);
2317 ctx->fn = ia[0].ptr;
2318
2319 #ifdef DEBUG_ENV
2320 if (getenv("CG") && strcmp(da(ctx->fn,function)->function_name, getenv("CG")))
2321 goto fail;
2322 #endif
2323
2324 ctx->local_directory = mem_alloc_array_mayfail(mem_calloc_mayfail, struct data **, 0, 0, da(ctx->fn,function)->local_directory_size, sizeof(struct data *), &ctx->err);
2325 if (unlikely(!ctx->local_directory))
2326 goto fail;
2327
2328 if (0) for (i = 0; i < da(ctx->fn,function)->local_directory_size; i++) {
2329 struct data *callee;
2330 pointer_t *ptr;
2331 ptr = da(ctx->fn,function)->local_directory[i];
2332 pointer_follow(ptr, false, callee, PF_SPARK, NULL, 0,
2333 SUBMIT_EX(ex_);
2334 goto next_one,
2335 goto next_one;
2336 );
2337 ctx->local_directory[i] = callee;
2338 next_one:;
2339 }
2340 for (i = 0; i < da(ctx->fn,function)->local_directory_size; i++) {
2341 struct data *callee;
2342 pointer_t *ptr;
2343 if (ctx->local_directory[i])
2344 continue;
2345 ptr = da(ctx->fn,function)->local_directory[i];
2346 pointer_follow(ptr, false, callee, PF_WAIT, fp, ip,
2347 done_ctx(ctx);
2348 return ex_,
2349 goto fail
2350 );
2351 ctx->local_directory[i] = callee;
2352 /*debug("processing call: %s -> %s", da(ctx->fn,function)->function_name, da(callee,function)->function_name);*/
2353 }
2354
2355 if (da(ctx->fn,function)->module_designator) {
2356 struct function_descriptor *sfd = save_find_function_descriptor(da(ctx->fn,function)->module_designator, da(ctx->fn,function)->function_designator);
2357 if (sfd && sfd->unoptimized_code_size) {
2358 codegen = data_alloc_flexible(codegen, unoptimized_code, sfd->n_entries, &ctx->err);
2359 if (unlikely(!codegen))
2360 goto fail;
2361 da(codegen,codegen)->unoptimized_code_base = sfd->unoptimized_code_base;
2362 da(codegen,codegen)->unoptimized_code_size = sfd->unoptimized_code_size;
2363 da(codegen,codegen)->function = ctx->fn;
2364 da(codegen,codegen)->is_saved = true;
2365 da(codegen,codegen)->n_entries = sfd->n_entries;
2366 da(codegen,codegen)->offsets = NULL;
2367 for (i = 0; i < sfd->n_entries; i++) {
2368 da(codegen,codegen)->unoptimized_code[i] = cast_ptr(char *, da(codegen,codegen)->unoptimized_code_base) + sfd->entries[i];
2369 /*debug("%s: %p + %lx -> %p", da(ctx->fn,function)->function_name, da(codegen,codegen)->unoptimized_code_base, sfd->entries[i], da(codegen,codegen)->unoptimized_code[i]);*/
2370 }
2371 #ifdef HAVE_CODEGEN_TRAPS
2372 da(codegen,codegen)->trap_records = sfd->trap_records;
2373 da(codegen,codegen)->trap_records_size = sfd->trap_records_size;
2374 data_trap_insert(codegen);
2375 #endif
2376 goto have_codegen;
2377 }
2378 }
2379
2380 /*debug("trying: %s", da(ctx->fn,function)->function_name);*/
2381 if (unlikely(!array_init_mayfail(uint8_t, &ctx->code, &ctx->code_size, &ctx->err)))
2382 goto fail;
2383
2384 ctx->code_labels = mem_alloc_array_mayfail(mem_calloc_mayfail, uint32_t *, 0, 0, da(ctx->fn,function)->code_size, sizeof(uint32_t), &ctx->err);
2385 if (unlikely(!ctx->code_labels))
2386 goto fail;
2387
2388 ctx->code_exits = mem_alloc_array_mayfail(mem_calloc_mayfail, struct cg_exit **, 0, 0, da(ctx->fn,function)->code_size, sizeof(struct cg_exit *), &ctx->err);
2389 if (unlikely(!ctx->code_exits))
2390 goto fail;
2391
2392 ctx->flag_cache = mem_alloc_array_mayfail(mem_calloc_mayfail, uint8_t *, 0, 0, function_n_variables(ctx->fn), sizeof(int8_t), &ctx->err);
2393 if (unlikely(!ctx->flag_cache))
2394 goto fail;
2395
2396 ctx->registers = mem_alloc_array_mayfail(mem_alloc_mayfail, short *, 0, 0, function_n_variables(ctx->fn), sizeof(short), &ctx->err);
2397 if (unlikely(!ctx->registers))
2398 goto fail;
2399
2400 if (unlikely(!array_init_mayfail(frame_t, &ctx->need_spill, &ctx->need_spill_l, &ctx->err)))
2401 goto fail;
2402
2403 if (unlikely(!gen_registers(ctx)))
2404 goto fail;
2405
2406 if (unlikely(!gen_function(ctx)))
2407 goto fail;
2408
2409 if (unlikely(!gen_entries(ctx)))
2410 goto fail;
2411
2412 if (unlikely(!gen_epilogues(ctx)))
2413 goto fail;
2414
2415 if (unlikely(!(ctx->label_id + 1)))
2416 goto fail;
2417 if (unlikely(!(ctx->label_to_pos = mem_alloc_array_mayfail(mem_alloc_mayfail, size_t *, 0, 0, ctx->label_id + 1, sizeof(size_t), &ctx->err))))
2418 goto fail;
2419
2420 again:
2421 for (l = 0; l < ctx->label_id + 1; l++)
2422 ctx->label_to_pos[l] = (size_t)-1;
2423
2424 if (unlikely(!array_init_mayfail(uint8_t, &ctx->mcode, &ctx->mcode_size, &ctx->err)))
2425 goto fail;
2426
2427 if (unlikely(!array_init_mayfail(struct relocation, &ctx->reloc, &ctx->reloc_size, &ctx->err)))
2428 goto fail;
2429
2430 if (unlikely(!array_init_mayfail(struct trap_record, &ctx->trap_records, &ctx->trap_records_size, &ctx->err)))
2431 goto fail;
2432
2433 #ifdef ARCH_CONTEXT
2434 init_arch_context(ctx);
2435 #endif
2436
2437 if (unlikely(!gen_mcode(ctx)))
2438 goto fail;
2439
2440 rr = resolve_relocs(ctx);
2441 if (unlikely(rr == RELOCS_FAIL)) {
2442 /*debug("relocation fail: %s", da(ctx->fn,function)->function_name);*/
2443 goto fail;
2444 }
2445 if (rr == RELOCS_RETRY) {
2446 mem_free(ctx->mcode);
2447 ctx->mcode = NULL;
2448 mem_free(ctx->reloc);
2449 ctx->reloc = NULL;
2450 mem_free(ctx->trap_records);
2451 ctx->trap_records = NULL;
2452 goto again;
2453 }
2454
2455 resolve_traps(ctx);
2456
2457 #ifdef DEBUG_ENV
2458 if ((getenv("DUMP") && !strcmp(getenv("DUMP"), da(ctx->fn,function)->function_name)) || getenv("DUMP_ALL")) {
2459 char *hex;
2460 size_t hexl;
2461 size_t i;
2462 handle_t h;
2463
2464 mutex_lock(&dump_mutex);
2465 str_init(&hex, &hexl);
2466 str_add_string(&hex, &hexl, "_");
2467 str_add_unsigned(&hex, &hexl, dump_seq++, 10);
2468 str_add_string(&hex, &hexl, "_");
2469 str_add_string(&hex, &hexl, da(ctx->fn,function)->function_name);
2470 str_add_string(&hex, &hexl, ":");
2471 for (i = 0; i < hexl; i++)
2472 if (hex[i] == '/')
2473 hex[i] = '_';
2474 for (i = 0; i < ctx->mcode_size; i++) {
2475 uint8_t a = ctx->mcode[i];
2476 if (!(i & 0xff))
2477 str_add_string(&hex, &hexl, "\n .byte 0x");
2478 else
2479 str_add_string(&hex, &hexl, ",0x");
2480 if (a < 16)
2481 str_add_char(&hex, &hexl, '0');
2482 str_add_unsigned(&hex, &hexl, a, 16);
2483 }
2484 str_add_string(&hex, &hexl, "\n");
2485 h = os_open(os_cwd, "dump.s", O_WRONLY | O_APPEND, 0600, NULL);
2486 os_write_all(h, hex, hexl, NULL);
2487 os_close(h);
2488 mem_free(hex);
2489 mutex_unlock(&dump_mutex);
2490 }
2491 #endif
2492
2493 ctx->codegen = data_alloc_flexible(codegen, unoptimized_code, ctx->n_entries, &ctx->err);
2494 if (unlikely(!ctx->codegen))
2495 goto fail;
2496 da(ctx->codegen,codegen)->function = ctx->fn;
2497 da(ctx->codegen,codegen)->is_saved = false;
2498 da(ctx->codegen,codegen)->n_entries = 0;
2499 da(ctx->codegen,codegen)->offsets = NULL;
2500
2501 if (unlikely(!codegen_map(ctx)))
2502 goto fail;
2503
2504 codegen = ctx->codegen;
2505 ctx->codegen = NULL;
2506
2507 #ifdef HAVE_CODEGEN_TRAPS
2508 da(codegen,codegen)->trap_records = ctx->trap_records;
2509 da(codegen,codegen)->trap_records_size = ctx->trap_records_size;
2510 ctx->trap_records = NULL;
2511 data_trap_insert(codegen);
2512 #endif
2513
2514 have_codegen:
2515 done_ctx(ctx);
2516 return function_return(fp, pointer_data(codegen));
2517
2518 fail:
2519 /*debug("FAILED: %s", da(ctx->fn,function)->function_name);*/
2520 done_ctx(ctx);
2521 return function_return(fp, pointer_thunk(thunk_alloc_exception_error(error_ajla(EC_SYNC, AJLA_ERROR_NOT_SUPPORTED), NULL, NULL, NULL pass_file_line)));
2522 }
2523
2524 void codegen_free(struct data *codegen)
2525 {
2526 if (unlikely(da(codegen,codegen)->offsets != NULL))
2527 mem_free(da(codegen,codegen)->offsets);
2528 if (likely(da(codegen,codegen)->is_saved))
2529 return;
2530 #ifdef HAVE_CODEGEN_TRAPS
2531 mem_free(da(codegen,codegen)->trap_records);
2532 #endif
2533 os_code_unmap(da(codegen,codegen)->unoptimized_code_base, da(codegen,codegen)->unoptimized_code_size);
2534 }
2535
2536 #if defined(ARCH_IA64)
2537 static uintptr_t ia64_stub[2];
2538 #endif
2539 #if defined(ARCH_PARISC32) && defined(ARCH_PARISC_USE_STUBS)
2540 static uintptr_t parisc_stub[2];
2541 #endif
2542 #if defined(ARCH_PARISC64) && defined(ARCH_PARISC_USE_STUBS)
2543 static uintptr_t parisc_stub[4];
2544 #endif
2545 #if defined(ARCH_POWER) && defined(AIX_CALL)
2546 static uintptr_t ppc_stub[3];
2547 #endif
2548
2549 void name(codegen_init)(void)
2550 {
2551 struct codegen_context ctx_;
2552 struct codegen_context *ctx = &ctx_;
2553 void *ptr;
2554
2555 #if (defined(ARCH_X86_64) || defined(ARCH_X86_X32)) && !defined(ARCH_X86_WIN_ABI)
2556 #if defined(HAVE_SYSCALL) && defined(HAVE_ASM_PRCTL_H) && defined(HAVE_SYS_SYSCALL_H)
2557 if (!dll) {
2558 int r;
2559 EINTR_LOOP(r, syscall(SYS_arch_prctl, ARCH_SET_GS, &cg_upcall_vector));
2560 if (!r)
2561 upcall_register = R_GS;
2562 }
2563 #elif defined(HAVE_AMD64_SET_GSBASE) && defined(HAVE_X86_SYSARCH_H)
2564 if (!dll) {
2565 int r;
2566 EINTR_LOOP(r, amd64_set_gsbase(&cg_upcall_vector));
2567 if (!r)
2568 upcall_register = R_GS;
2569 }
2570 #elif defined(HAVE_SYSARCH) && defined(HAVE_X86_SYSARCH_H) && defined(X86_64_SET_GSBASE)
2571 if (!dll) {
2572 int r;
2573 void *ptr = &cg_upcall_vector;
2574 EINTR_LOOP(r, sysarch(X86_64_SET_GSBASE, &ptr));
2575 if (!r)
2576 upcall_register = R_GS;
2577 }
2578 #endif
2579 #endif
2580
2581 init_ctx(ctx);
2582 ctx->fn = NULL;
2583
2584 array_init(uint8_t, &ctx->code, &ctx->code_size);
2585
2586 if (unlikely(!gen_entry(ctx)))
2587 goto fail;
2588
2589 array_init(uint8_t, &ctx->mcode, &ctx->mcode_size);
2590
2591 #ifdef ARCH_CONTEXT
2592 init_arch_context(ctx);
2593 #endif
2594
2595 if (unlikely(!gen_mcode(ctx)))
2596 goto fail;
2597
2598 array_finish(uint8_t, &ctx->mcode, &ctx->mcode_size);
2599 ptr = os_code_map(ctx->mcode, ctx->mcode_size, NULL);
2600 codegen_ptr = ptr;
2601 codegen_size = ctx->mcode_size;
2602 ctx->mcode = NULL;
2603 #if defined(ARCH_IA64)
2604 ia64_stub[0] = ptr_to_num(ptr);
2605 ia64_stub[1] = 0;
2606 codegen_entry = cast_ptr(codegen_type, ia64_stub);
2607 #elif defined(ARCH_PARISC32) && defined(ARCH_PARISC_USE_STUBS)
2608 parisc_stub[0] = ptr_to_num(ptr);
2609 parisc_stub[1] = 0;
2610 codegen_entry = cast_ptr(codegen_type, cast_ptr(char *, parisc_stub) + 2);
2611 #elif defined(ARCH_PARISC64) && defined(ARCH_PARISC_USE_STUBS)
2612 parisc_stub[0] = 0;
2613 parisc_stub[1] = 0;
2614 parisc_stub[2] = ptr_to_num(ptr);
2615 parisc_stub[3] = 0;
2616 codegen_entry = cast_ptr(codegen_type, parisc_stub);
2617 #elif defined(ARCH_POWER) && defined(AIX_CALL)
2618 ppc_stub[0] = ptr_to_num(ptr);
2619 ppc_stub[1] = 0;
2620 ppc_stub[2] = 0;
2621 codegen_entry = cast_ptr(codegen_type, ppc_stub);
2622 #else
2623 codegen_entry = ptr;
2624 #endif
2625 done_ctx(ctx);
2626
2627 #ifdef DEBUG_ENV
2628 mutex_init(&dump_mutex);
2629 if (getenv("DUMP") || getenv("DUMP_ALL")) {
2630 size_t i;
2631 char *hex;
2632 size_t hexl;
2633 str_init(&hex, &hexl);
2634 #if defined(ARCH_RISCV64)
2635 str_add_string(&hex, &hexl, " .attribute arch, \"rv64i2p1_m2p0_a2p1_f2p2_d2p2_c2p0_zicsr2p0_zifencei2p0_zba1p0_zbb1p0_zbc1p0_zbs1p0\"\n");
2636 #endif
2637 for (i = 0; i < codegen_size; i++) {
2638 uint8_t a = cast_ptr(uint8_t *, codegen_ptr)[i];
2639 str_add_string(&hex, &hexl, " .byte 0x");
2640 if (a < 16)
2641 str_add_char(&hex, &hexl, '0');
2642 str_add_unsigned(&hex, &hexl, a, 16);
2643 str_add_char(&hex, &hexl, '\n');
2644 }
2645 os_write_atomic(".", "dump.s", hex, hexl, NULL);
2646 mem_free(hex);
2647 }
2648 #endif
2649
2650 return;
2651
2652 fail:
2653 fatal("couldn't compile global entry");
2654 }
2655
2656 void name(codegen_done)(void)
2657 {
2658 os_code_unmap(codegen_ptr, codegen_size);
2659 #ifdef DEBUG_ENV
2660 mutex_done(&dump_mutex);
2661 #endif
2662 }
2663
2664 #else
2665
2666 void name(codegen_init)(void)
2667 {
2668 }
2669
2670 void name(codegen_done)(void)
2671 {
2672 }
2673
2674 #endif
2675
2676 #endif
Ajla - a purely functional programming language