?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
codegen: improve floating point comparisons on loongarch, mips, parisc
[ajla.git] / codegen.c
blobac2b7d675fdfd82eecb73e9bfcdbe1d5bb816233
1 /*
2 * Copyright (C) 2024 Mikulas Patocka
3 *
4 * This file is part of Ajla.
5 *
6 * Ajla is free software: you can redistribute it and/or modify it under the
7 * terms of the GNU General Public License as published by the Free Software
8 * Foundation, either version 3 of the License, or (at your option) any later
9 * version.
10 *
11 * Ajla is distributed in the hope that it will be useful, but WITHOUT ANY
12 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
13 * A PARTICULAR PURPOSE. See the GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License along with
16 * Ajla. If not, see <https://www.gnu.org/licenses/>.
17 */
18
19 #include "ajla.h"
20
21 #ifndef FILE_OMIT
22
23 #include "data.h"
24 #include "os.h"
25 #include "os_util.h"
26 #include "util.h"
27 #include "ipfn.h"
28 #include "ipret.h"
29 #include "funct.h"
30 #include "thread.h"
31 #include "task.h"
32 #include "save.h"
33
34 #include "codegen.h"
35
36 #ifdef HAVE_CODEGEN
37
38 #define flag_cache_chicken 0
39 #define must_be_flat_chicken 0
40 #define ra_chicken 0
41
42 #define INLINE_BITMAP_SLOTS 16
43 #define INLINE_COPY_SIZE 64
44
45 /*#define DEBUG_INSNS*/
46 /*#define DEBUG_GARBAGE*/
47
48 #if (defined(ARCH_X86_64) || defined(ARCH_X86_X32)) && !defined(ARCH_X86_WIN_ABI) && defined(HAVE_SYSCALL) && defined(HAVE_ASM_PRCTL_H) && defined(HAVE_SYS_SYSCALL_H)
49 #include <asm/prctl.h>
50 #include <sys/syscall.h>
51 #include <unistd.h>
52 #endif
53
54 code_return_t (*codegen_entry)(frame_s *, struct cg_upcall_vector_s *, tick_stamp_t, void *);
55 static void *codegen_ptr;
56 static size_t codegen_size;
57
58 #ifdef DEBUG_ENV
59 static mutex_t dump_mutex;
60 static uint64_t dump_seq = 0;
61 #endif
62
63 /*
64 * insn:
65 * opcode - 16 bits
66 * op_size - 3 bits
67 * aux - 7 bits
68 * writes flags - 2 bit
69 * jmp size - 2 bits
70 */
71
72 #define INSN_OPCODE 0x0000ffffUL
73 #define INSN_OP_SIZE 0x00070000UL
74 #define INSN_AUX 0x03f80000UL
75 #define INSN_WRITES_FLAGS 0x0c000000UL
76 #define INSN_JUMP_SIZE 0x30000000UL
77
78 #define INSN_OPCODE_SHIFT 0
79 #define INSN_OP_SIZE_SHIFT 16
80 #define INSN_AUX_SHIFT 19
81 #define INSN_WRITES_FLAGS_SHIFT 26
82 #define INSN_JUMP_SIZE_SHIFT 28
83
84 #define insn_opcode(insn) (((insn) >> INSN_OPCODE_SHIFT) & (INSN_OPCODE >> INSN_OPCODE_SHIFT))
85 #define insn_op_size(insn) (((insn) >> INSN_OP_SIZE_SHIFT) & (INSN_OP_SIZE >> INSN_OP_SIZE_SHIFT))
86 #define insn_aux(insn) (((insn) >> INSN_AUX_SHIFT) & (INSN_AUX >> INSN_AUX_SHIFT))
87 #define insn_writes_flags(insn) (((insn) >> INSN_WRITES_FLAGS_SHIFT) & (INSN_WRITES_FLAGS >> INSN_WRITES_FLAGS_SHIFT))
88 #define insn_jump_size(insn) (((insn) >> INSN_JUMP_SIZE_SHIFT) & (INSN_JUMP_SIZE >> INSN_JUMP_SIZE_SHIFT))
89
90 #define ALU_ADD 0x00
91 #define ALU_OR 0x01
92 #define ALU_ADC 0x02
93 #define ALU_SBB 0x03
94 #define ALU_AND 0x04
95 #define ALU_SUB 0x05
96 #define ALU_XOR 0x06
97 #define ALU_ORN 0x08
98 #define ALU_ANDN 0x09
99 #define ALU_XORN 0x0a
100 #define ALU_MUL 0x10
101 #define ALU_UMULH 0x11
102 #define ALU_SMULH 0x12
103 #define ALU_UDIV 0x13
104 #define ALU_SDIV 0x14
105 #define ALU_UREM 0x15
106 #define ALU_SREM 0x16
107 #define ALU_SAVE 0x17
108 #define ALU_EXTBL 0x18
109 #define ALU_EXTWL 0x19
110 #define ALU_EXTLL 0x1a
111 #define ALU_EXTLH 0x1b
112 #define ALU_INSBL 0x1c
113 #define ALU_MSKBL 0x1d
114 #define ALU_ZAP 0x20
115 #define ALU_ZAPNOT 0x21
116
117 #define ALU1_NOT 0x00
118 #define ALU1_NEG 0x01
119 #define ALU1_NGC 0x02
120 #define ALU1_INC 0x03
121 #define ALU1_DEC 0x04
122 #define ALU1_BSWAP 0x05
123 #define ALU1_BSWAP16 0x06
124 #define ALU1_BREV 0x07
125 #define ALU1_BSF 0x08
126 #define ALU1_BSR 0x09
127 #define ALU1_LZCNT 0x0a
128 #define ALU1_POPCNT 0x0b
129
130 #define FP_ALU_ADD 0
131 #define FP_ALU_SUB 1
132 #define FP_ALU_MUL 2
133 #define FP_ALU_DIV 3
134 #define FP_ALU1_NEG 0
135 #define FP_ALU1_SQRT 1
136 #define FP_ALU1_ROUND 2
137 #define FP_ALU1_FLOOR 3
138 #define FP_ALU1_CEIL 4
139 #define FP_ALU1_TRUNC 5
140 #define FP_ALU1_VCNT8 6
141 #define FP_ALU1_VPADDL 7
142 #define FP_ALU1_ADDV 8
143
144 #define COND_O 0x0
145 #define COND_NO 0x1
146 #define COND_B 0x2
147 #define COND_AE 0x3
148 #define COND_E 0x4
149 #define COND_NE 0x5
150 #define COND_BE 0x6
151 #define COND_A 0x7
152 #define COND_S 0x8
153 #define COND_NS 0x9
154 #define COND_P 0xa
155 #define COND_NP 0xb
156 #define COND_L 0xc
157 #define COND_GE 0xd
158 #define COND_LE 0xe
159 #define COND_G 0xf
160 #define COND_BLBC 0x10
161 #define COND_BLBS 0x11
162 #define COND_ALWAYS 0x12
163
164 #define COND_FP 0x20
165 #define FP_COND_P (COND_FP | COND_P)
166 #define FP_COND_NP (COND_FP | COND_NP)
167 #define FP_COND_E (COND_FP | COND_E)
168 #define FP_COND_NE (COND_FP | COND_NE)
169 #define FP_COND_A (COND_FP | COND_A)
170 #define FP_COND_BE (COND_FP | COND_BE)
171 #define FP_COND_B (COND_FP | COND_B)
172 #define FP_COND_AE (COND_FP | COND_AE)
173
174 #define ROT_ROL 0x0
175 #define ROT_ROR 0x1
176 #define ROT_RCL 0x2
177 #define ROT_RCR 0x3
178 #define ROT_SHL 0x4
179 #define ROT_SHR 0x5
180 #define ROT_SAR 0x7
181 #define ROT_SAL 0x8
182
183 #define BTX_BT 0x0
184 #define BTX_BTS 0x1
185 #define BTX_BTR 0x2
186 #define BTX_BTC 0x3
187 #define BTX_BTEXT 0x4
188
189 #define OP_SIZE_1 0
190 #define OP_SIZE_2 1
191 #define OP_SIZE_4 2
192 #define OP_SIZE_8 3
193 #define OP_SIZE_16 4
194 #define OP_SIZE_10 7
195
196 #define MOV_MASK_0_16 0x0
197 #define MOV_MASK_16_32 0x1
198 #define MOV_MASK_32_48 0x2
199 #define MOV_MASK_48_64 0x3
200 #define MOV_MASK_0_8 0x4
201 #define MOV_MASK_32_64 0x5
202 #define MOV_MASK_52_64 0x6
203
204 #define JMP_SHORTEST 0x0000
205 #define JMP_SHORT 0x0001
206 #define JMP_LONG 0x0002
207 #define JMP_EXTRA_LONG 0x0003
208
209 enum {
210 INSN_ENTRY,
211 INSN_LABEL,
212 INSN_RET,
213 INSN_RET_IMM,
214 INSN_ARM_PUSH,
215 INSN_ARM_POP,
216 INSN_S390_PUSH,
217 INSN_S390_POP,
218 INSN_IA64_ALLOC,
219 INSN_IA64_DEALLOC,
220 INSN_PUSH,
221 INSN_POP,
222 INSN_CALL,
223 INSN_CALL_INDIRECT,
224 INSN_MOV,
225 INSN_MOVSX,
226 INSN_MOV_U,
227 INSN_MOV_LR,
228 INSN_CMP,
229 INSN_CMP_DEST_REG,
230 INSN_CMN,
231 INSN_TEST,
232 INSN_TEST_DEST_REG,
233 INSN_ALU,
234 INSN_ALU_PARTIAL,
235 INSN_ALU_FLAGS,
236 INSN_ALU_FLAGS_PARTIAL,
237 INSN_ALU_TRAP,
238 INSN_ALU_FLAGS_TRAP,
239 INSN_ALU1,
240 INSN_ALU1_PARTIAL,
241 INSN_ALU1_FLAGS,
242 INSN_ALU1_FLAGS_PARTIAL,
243 INSN_ALU1_TRAP,
244 INSN_LEA3,
245 INSN_ROT,
246 INSN_ROT_PARTIAL,
247 INSN_BT,
248 INSN_BTX,
249 INSN_MUL_L,
250 INSN_DIV_L,
251 INSN_MADD,
252 INSN_CBW,
253 INSN_CBW_PARTIAL,
254 INSN_CWD,
255 INSN_CWD_PARTIAL,
256 INSN_SET_COND,
257 INSN_SET_COND_PARTIAL,
258 INSN_CMOV,
259 INSN_CMOV_XCC,
260 INSN_MOVR,
261 INSN_CSEL_SEL,
262 INSN_CSEL_INC,
263 INSN_CSEL_INV,
264 INSN_CSEL_NEG,
265 INSN_STP,
266 INSN_LDP,
267 INSN_MOV_MASK,
268 INSN_MEMCPY,
269 INSN_MEMSET,
270 INSN_FP_CMP,
271 INSN_FP_CMP_DEST_REG,
272 INSN_FP_CMP_DEST_REG_TRAP,
273 INSN_FP_CMP_UNORDERED_DEST_REG,
274 INSN_FP_CMP_COND,
275 INSN_FP_TEST_REG,
276 INSN_FP_TO_INT_FLAGS,
277 INSN_FP_ALU,
278 INSN_FP_ALU1,
279 INSN_FP_TO_INT32,
280 INSN_FP_TO_INT64,
281 INSN_FP_TO_INT64_TRAP,
282 INSN_FP_FROM_INT32,
283 INSN_FP_FROM_INT64,
284 INSN_FP_INT64_TO_INT32_TRAP,
285 INSN_FP_CVT,
286 INSN_X87_FLD,
287 INSN_X87_FILD,
288 INSN_X87_FSTP,
289 INSN_X87_FISTP,
290 INSN_X87_FISTTP,
291 INSN_X87_FCOMP,
292 INSN_X87_FCOMPP,
293 INSN_X87_FCOMIP,
294 INSN_X87_ALU,
295 INSN_X87_ALUP,
296 INSN_X87_FCHS,
297 INSN_X87_FSQRT,
298 INSN_X87_FRNDINT,
299 INSN_X87_FNSTSW,
300 INSN_X87_FLDCW,
301 INSN_JMP,
302 INSN_JMP_COND,
303 INSN_JMP_COND_LOGICAL,
304 INSN_JMP_REG,
305 INSN_JMP_REG_BIT,
306 INSN_JMP_2REGS,
307 INSN_JMP_FP_TEST,
308 INSN_JMP_INDIRECT,
309 INSN_MB,
310 INSN_CALL_MILLICODE,
311 };
312
313 #define ARG_REGS_MAX 0xc0
314 #define ARG_SHIFTED_REGISTER 0xc0
315 #define ARG_SHIFT_AMOUNT 0x3f
316 #define ARG_SHIFT_MODE 0xc0
317 #define ARG_SHIFT_LSL 0x00
318 #define ARG_SHIFT_LSR 0x40
319 #define ARG_SHIFT_ASR 0x80
320 #define ARG_SHIFT_ROR 0xc0
321 #define ARG_EXTENDED_REGISTER 0xc1
322 #define ARG_EXTEND_SHIFT 0x07
323 #define ARG_EXTEND_MODE 0x38
324 #define ARG_EXTEND_UXTB 0x00
325 #define ARG_EXTEND_UXTH 0x08
326 #define ARG_EXTEND_UXTW 0x10
327 #define ARG_EXTEND_UXTX 0x18
328 #define ARG_EXTEND_SXTB 0x20
329 #define ARG_EXTEND_SXTH 0x28
330 #define ARG_EXTEND_SXTW 0x30
331 #define ARG_EXTEND_SXTX 0x38
332 #define ARG_ADDRESS_0 0xd0
333 #define ARG_ADDRESS_1 0xd1
334 #define ARG_ADDRESS_1_2 0xd2
335 #define ARG_ADDRESS_1_4 0xd3
336 #define ARG_ADDRESS_1_8 0xd4
337 #define ARG_ADDRESS_1_PRE_I 0xd5
338 #define ARG_ADDRESS_1_POST_I 0xd6
339 #define ARG_ADDRESS_2 0xd7
340 #define ARG_ADDRESS_2_2 0xd8
341 #define ARG_ADDRESS_2_4 0xd9
342 #define ARG_ADDRESS_2_8 0xda
343 #define ARG_ADDRESS_2_UXTW 0xdb
344 #define ARG_ADDRESS_2_SXTW 0xdc
345 #define ARG_IMM 0xe0
346
347 #define ARG_IS_ADDRESS(a) ((a) >= ARG_ADDRESS_0 && (a) <= ARG_ADDRESS_2_SXTW)
348
349 #ifdef POINTER_COMPRESSION
350 #define OP_SIZE_SLOT OP_SIZE_4
351 #else
352 #define OP_SIZE_SLOT OP_SIZE_ADDRESS
353 #endif
354
355 #define OP_SIZE_BITMAP (bitmap_64bit ? OP_SIZE_8 : OP_SIZE_4)
356
357 #define OP_SIZE_INT log_2(sizeof(int_default_t))
358
359 #define check_insn(insn) \
360 do { \
361 /*if ((insn_opcode(insn) == INSN_ALU || insn_opcode(insn) == INSN_ALU1) && insn_op_size(insn) != OP_SIZE_NATIVE) internal(file_line, "invalid insn %08x", (unsigned)(insn));*/\
362 /*if (insn == 0x001a000e) internal(file_line, "invalid insn %08x", insn);*/\
363 } while (0)
364
365 #ifdef DEBUG_INSNS
366 #define gen_line() gen_four(__LINE__)
367 #else
368 #define gen_line() do { } while (0)
369 #endif
370
371 #ifdef ARCH_IA64
372 #define ARCH_CONTEXT struct { \
373 uint64_t insns[3]; \
374 uint8_t insn_units[3]; \
375 bool insn_stops[3]; \
376 uint64_t wr_mask[4]; \
377 }
378 #endif
379
380 #define gen_insn(opcode, op_size, aux, writes_flags) \
381 do { \
382 uint32_t dword = \
383 (uint32_t)(opcode) << INSN_OPCODE_SHIFT | \
384 (uint32_t)(op_size) << INSN_OP_SIZE_SHIFT | \
385 (uint32_t)(aux) << INSN_AUX_SHIFT | \
386 (uint32_t)(writes_flags) << INSN_WRITES_FLAGS_SHIFT; \
387 check_insn(dword); \
388 gen_line(); \
389 gen_four(dword); \
390 } while (0)
391
392 static size_t arg_size(uint8_t arg)
393 {
394 if (arg < ARG_REGS_MAX)
395 return 1;
396 if (arg >= ARG_SHIFTED_REGISTER && arg <= ARG_EXTENDED_REGISTER)
397 return 3;
398 if (arg == ARG_ADDRESS_0)
399 return 9;
400 if (arg >= ARG_ADDRESS_1 && arg <= ARG_ADDRESS_1_POST_I)
401 return 10;
402 if (arg >= ARG_ADDRESS_2 && arg <= ARG_ADDRESS_2_SXTW)
403 return 11;
404 if (arg == ARG_IMM)
405 return 9;
406 internal(file_line, "arg_size: invalid argument %02x", arg);
407 return 0;
408 }
409
410 struct relocation {
411 uint32_t label_id;
412 uint8_t length;
413 size_t position;
414 size_t jmp_instr;
415 };
416
417 struct code_arg {
418 frame_t slot;
419 frame_t flags;
420 frame_t type;
421 };
422
423 struct cg_entry {
424 size_t entry_to_pos;
425 frame_t *variables;
426 size_t n_variables;
427 uint32_t entry_label;
428 uint32_t nonflat_label;
429 };
430
431 struct cg_exit {
432 uint32_t undo_label;
433 uint8_t undo_opcode;
434 uint8_t undo_op_size;
435 uint8_t undo_aux;
436 uint8_t undo_writes_flags;
437 uint8_t undo_parameters[35];
438 uint8_t undo_parameters_len;
439 uint32_t escape_label;
440 };
441
442 struct codegen_context {
443 struct data *fn;
444 struct data **local_directory;
445
446 const code_t *instr_start;
447 const code_t *current_position;
448 uchar_efficient_t arg_mode;
449
450 uint32_t label_id;
451 struct cg_entry *entries;
452 frame_t n_entries;
453
454 uint8_t *code;
455 size_t code_size;
456
457 uint8_t *code_position;
458
459 uint32_t *code_labels;
460 struct cg_exit **code_exits;
461 uint32_t escape_nospill_label;
462 uint32_t call_label;
463 uint32_t reload_label;
464
465 uint8_t *mcode;
466 size_t mcode_size;
467
468 size_t *label_to_pos;
469 struct relocation *reloc;
470 size_t reloc_size;
471
472 struct trap_record *trap_records;
473 size_t trap_records_size;
474
475 struct code_arg *args;
476 size_t args_l;
477 const code_t *return_values;
478
479 int8_t *flag_cache;
480 short *registers;
481 frame_t *need_spill;
482 size_t need_spill_l;
483
484 unsigned base_reg;
485 bool offset_reg;
486 int64_t offset_imm;
487
488 bool const_reg;
489 int64_t const_imm;
490
491 struct data *codegen;
492
493 int upcall_args;
494 frame_t *var_aux;
495
496 ajla_error_t err;
497
498 #ifdef ARCH_CONTEXT
499 ARCH_CONTEXT a;
500 #endif
501 };
502
503 static void init_ctx(struct codegen_context *ctx)
504 {
505 ctx->local_directory = NULL;
506 ctx->label_id = 0;
507 ctx->entries = NULL;
508 ctx->n_entries = 0;
509 ctx->code = NULL;
510 ctx->code_labels = NULL;
511 ctx->code_exits = NULL;
512 ctx->escape_nospill_label = 0;
513 ctx->call_label = 0;
514 ctx->reload_label = 0;
515 ctx->mcode = NULL;
516 ctx->label_to_pos = NULL;
517 ctx->reloc = NULL;
518 ctx->trap_records = NULL;
519 ctx->args = NULL;
520 ctx->flag_cache = NULL;
521 ctx->registers = NULL;
522 ctx->need_spill = NULL;
523 ctx->codegen = NULL;
524 ctx->upcall_args = -1;
525 ctx->var_aux = NULL;
526 }
527
528 static void done_ctx(struct codegen_context *ctx)
529 {
530 if (ctx->local_directory)
531 mem_free(ctx->local_directory);
532 if (ctx->entries) {
533 size_t i;
534 for (i = 0; i < ctx->n_entries; i++) {
535 struct cg_entry *ce = &ctx->entries[i];
536 if (ce->variables)
537 mem_free(ce->variables);
538 }
539 mem_free(ctx->entries);
540 }
541 if (ctx->code)
542 mem_free(ctx->code);
543 if (ctx->code_labels)
544 mem_free(ctx->code_labels);
545 if (ctx->code_exits) {
546 ip_t ip;
547 ip_t cs = da(ctx->fn,function)->code_size;
548 for (ip = 0; ip < cs; ip++) {
549 if (ctx->code_exits[ip])
550 mem_free(ctx->code_exits[ip]);
551 }
552 mem_free(ctx->code_exits);
553 }
554 if (ctx->mcode)
555 mem_free(ctx->mcode);
556 if (ctx->label_to_pos)
557 mem_free(ctx->label_to_pos);
558 if (ctx->reloc)
559 mem_free(ctx->reloc);
560 if (ctx->trap_records)
561 mem_free(ctx->trap_records);
562 if (ctx->args)
563 mem_free(ctx->args);
564 if (ctx->flag_cache)
565 mem_free(ctx->flag_cache);
566 if (ctx->registers)
567 mem_free(ctx->registers);
568 if (ctx->need_spill)
569 mem_free(ctx->need_spill);
570 if (ctx->codegen)
571 data_free(ctx->codegen);
572 if (ctx->var_aux)
573 mem_free(ctx->var_aux);
574 }
575
576
577 static inline code_t get_code(struct codegen_context *ctx)
578 {
579 ajla_assert(ctx->current_position < da(ctx->fn,function)->code + da(ctx->fn,function)->code_size, (file_line, "get_code: ran out of code"));
580 return *ctx->current_position++;
581 }
582
583 static inline uint32_t get_uint32(struct codegen_context *ctx)
584 {
585 uint32_t a1 = get_code(ctx);
586 uint32_t a2 = get_code(ctx);
587 #if !CODE_ENDIAN
588 return a1 + (a2 << 16);
589 #else
590 return a2 + (a1 << 16);
591 #endif
592 }
593
594 static int32_t get_jump_offset(struct codegen_context *ctx)
595 {
596 if (SIZEOF_IP_T == 2) {
597 return (int32_t)(int16_t)get_code(ctx);
598 } else if (SIZEOF_IP_T == 4) {
599 return (int32_t)get_uint32(ctx);
600 } else {
601 not_reached();
602 return -1;
603 }
604 }
605
606 static inline void get_one(struct codegen_context *ctx, frame_t *v)
607 {
608 if (!ctx->arg_mode) {
609 code_t c = get_code(ctx);
610 ajla_assert(!(c & ~0xff), (file_line, "get_one: high byte is not cleared: %u", (unsigned)c));
611 *v = c & 0xff;
612 } else if (ctx->arg_mode == 1) {
613 *v = get_code(ctx);
614 #if ARG_MODE_N >= 2
615 } else if (ctx->arg_mode == 2) {
616 *v = get_uint32(ctx);
617 #endif
618 } else {
619 internal(file_line, "get_one: invalid arg mode %u", ctx->arg_mode);
620 }
621 }
622
623 static inline void get_two(struct codegen_context *ctx, frame_t *v1, frame_t *v2)
624 {
625 if (!ctx->arg_mode) {
626 code_t c = get_code(ctx);
627 *v1 = c & 0xff;
628 *v2 = c >> 8;
629 } else if (ctx->arg_mode == 1) {
630 *v1 = get_code(ctx);
631 *v2 = get_code(ctx);
632 #if ARG_MODE_N >= 2
633 } else if (ctx->arg_mode == 2) {
634 *v1 = get_uint32(ctx);
635 *v2 = get_uint32(ctx);
636 #endif
637 } else {
638 internal(file_line, "get_two: invalid arg mode %u", ctx->arg_mode);
639 }
640 }
641
642
643 static uint32_t alloc_label(struct codegen_context *ctx)
644 {
645 return ++ctx->label_id;
646 }
647
648 static struct cg_exit *alloc_cg_exit_for_ip(struct codegen_context *ctx, const code_t *code)
649 {
650 ip_t ip = code - da(ctx->fn,function)->code;
651 struct cg_exit *ce = ctx->code_exits[ip];
652 if (!ce) {
653 ce = mem_calloc_mayfail(struct cg_exit *, sizeof(struct cg_exit), &ctx->err);
654 if (unlikely(!ce))
655 return NULL;
656 ctx->code_exits[ip] = ce;
657 }
658 return ce;
659 }
660
661 static struct cg_exit *alloc_undo_label(struct codegen_context *ctx)
662 {
663 struct cg_exit *ce = alloc_cg_exit_for_ip(ctx, ctx->instr_start);
664 if (unlikely(!ce))
665 return NULL;
666 if (unlikely(ce->undo_label != 0))
667 internal(file_line, "alloc_cg_exit: undo label already allocated");
668 ce->undo_label = alloc_label(ctx);
669 if (unlikely(!ce->undo_label))
670 return NULL;
671 return ce;
672 }
673
674 static uint32_t alloc_escape_label_for_ip(struct codegen_context *ctx, const code_t *code)
675 {
676 struct cg_exit *ce = alloc_cg_exit_for_ip(ctx, code);
677 if (!ce)
678 return 0;
679 if (!ce->escape_label)
680 ce->escape_label = alloc_label(ctx);
681 return ce->escape_label;
682 }
683
684 static uint32_t alloc_escape_label(struct codegen_context *ctx)
685 {
686 return alloc_escape_label_for_ip(ctx, ctx->instr_start);
687 }
688
689 static uint32_t attr_unused alloc_call_label(struct codegen_context *ctx)
690 {
691 if (!ctx->call_label) {
692 ctx->call_label = alloc_label(ctx);
693 }
694 return ctx->call_label;
695 }
696
697 static uint32_t alloc_reload_label(struct codegen_context *ctx)
698 {
699 if (!ctx->reload_label) {
700 ctx->reload_label = alloc_label(ctx);
701 }
702 return ctx->reload_label;
703 }
704
705 static size_t attr_unused mark_params(struct codegen_context *ctx)
706 {
707 return ctx->code_size;
708 }
709
710 static void attr_unused copy_params(struct codegen_context *ctx, struct cg_exit *ce, size_t mark)
711 {
712 if (ctx->code_size - mark > n_array_elements(ce->undo_parameters))
713 internal(file_line, "undo_parameters is too small: %"PRIuMAX" > %"PRIuMAX"", (uintmax_t)(ctx->code_size - mark), (uintmax_t)n_array_elements(ce->undo_parameters));
714 memcpy(ce->undo_parameters, ctx->code + mark, ctx->code_size - mark);
715 ce->undo_parameters_len = ctx->code_size - mark;
716 ctx->code_size = mark;
717 }
718
719 #define g(call) \
720 do { \
721 if (unlikely(!call)) \
722 return false; \
723 } while (0)
724
725 #define gen_one(byte) \
726 do { \
727 /*debug("gen %d: %02x", __LINE__, (uint8_t)(byte))*/; \
728 if (unlikely(!array_add_mayfail(uint8_t, &ctx->code, &ctx->code_size, byte, NULL, &ctx->err)))\
729 return false; \
730 } while (0)
731
732 #if defined(C_LITTLE_ENDIAN)
733 #define gen_two(word) \
734 do { \
735 uint16_t word_ = (word); \
736 /*debug("gen %d: %04x", __LINE__, (uint16_t)(word_));*/ \
737 if (unlikely(!array_add_multiple_mayfail(uint8_t, &ctx->code, &ctx->code_size, cast_ptr(uint8_t *, &word_), 2, NULL, &ctx->err)))\
738 return false; \
739 } while (0)
740 #define gen_four(dword) \
741 do { \
742 uint32_t dword_ = (dword); \
743 /*debug("gen %d: %08x", __LINE__, (uint32_t)(dword_));*/ \
744 if (unlikely(!array_add_multiple_mayfail(uint8_t, &ctx->code, &ctx->code_size, cast_ptr(uint8_t *, &dword_), 4, NULL, &ctx->err)))\
745 return false; \
746 } while (0)
747 #define gen_eight(qword) \
748 do { \
749 uint64_t qword_ = (qword); \
750 /*debug("gen %d: %016lx", __LINE__, (uint64_t)(qword_));*/ \
751 if (unlikely(!array_add_multiple_mayfail(uint8_t, &ctx->code, &ctx->code_size, cast_ptr(uint8_t *, &qword_), 8, NULL, &ctx->err)))\
752 return false; \
753 } while (0)
754 #else
755 #define gen_two(word) \
756 do { \
757 uint16_t word_ = (word); \
758 gen_one(word_ & 0xffU); \
759 gen_one(word_ >> 8); \
760 } while (0)
761 #define gen_four(dword) \
762 do { \
763 uint32_t dword_ = (dword); \
764 gen_two(dword_ & 0xffffU); \
765 gen_two(dword_ >> 15 >> 1); \
766 } while (0)
767 #define gen_eight(qword) \
768 do { \
769 uint64_t qword_ = (qword); \
770 gen_four(qword_ & 0xffffffffUL); \
771 gen_four(qword_ >> 15 >> 15 >> 2); \
772 } while (0)
773 #endif
774
775 #define gen_label(label_id) \
776 do { \
777 gen_insn(INSN_LABEL, 0, 0, 0); \
778 gen_four(label_id); \
779 } while (0)
780
781
782 static uint8_t attr_unused cget_one(struct codegen_context *ctx)
783 {
784 ajla_assert(ctx->code_position < ctx->code + ctx->code_size, (file_line, "cget_one: ran out of code"));
785 return *ctx->code_position++;
786 }
787
788 static uint16_t attr_unused cget_two(struct codegen_context *ctx)
789 {
790 #if defined(C_LITTLE_ENDIAN)
791 uint16_t r;
792 ajla_assert(ctx->code_position < ctx->code + ctx->code_size, (file_line, "cget_two: ran out of code"));
793 memcpy(&r, ctx->code_position, 2);
794 ctx->code_position += 2;
795 return r;
796 #else
797 uint16_t r = cget_one(ctx);
798 r |= cget_one(ctx) << 8;
799 return r;
800 #endif
801 }
802
803 static uint32_t cget_four(struct codegen_context *ctx)
804 {
805 #if defined(C_LITTLE_ENDIAN)
806 uint32_t r;
807 ajla_assert(ctx->code_position < ctx->code + ctx->code_size, (file_line, "cget_four: ran out of code"));
808 memcpy(&r, ctx->code_position, 4);
809 ctx->code_position += 4;
810 return r;
811 #else
812 uint32_t r = cget_two(ctx);
813 r |= (uint32_t)cget_two(ctx) << 16;
814 return r;
815 #endif
816 }
817
818 static uint64_t attr_unused cget_eight(struct codegen_context *ctx)
819 {
820 #if defined(C_LITTLE_ENDIAN)
821 uint64_t r;
822 ajla_assert(ctx->code_position < ctx->code + ctx->code_size, (file_line, "cget_eight: ran out of code"));
823 memcpy(&r, ctx->code_position, 8);
824 ctx->code_position += 8;
825 return r;
826 #else
827 uint64_t r = cget_four(ctx);
828 r |= (uint64_t)cget_four(ctx) << 32;
829 return r;
830 #endif
831 }
832
833 static int64_t get_imm(uint8_t *ptr)
834 {
835 #if defined(C_LITTLE_ENDIAN)
836 int64_t r;
837 memcpy(&r, ptr, 8);
838 return r;
839 #else
840 int64_t r;
841 r = (uint64_t)ptr[0] |
842 ((uint64_t)ptr[1] << 8) |
843 ((uint64_t)ptr[2] << 16) |
844 ((uint64_t)ptr[3] << 24) |
845 ((uint64_t)ptr[4] << 32) |
846 ((uint64_t)ptr[5] << 40) |
847 ((uint64_t)ptr[6] << 48) |
848 ((uint64_t)ptr[7] << 56);
849 return r;
850 #endif
851 }
852
853 #define cgen_one(byte) \
854 do { \
855 if (unlikely(!array_add_mayfail(uint8_t, &ctx->mcode, &ctx->mcode_size, byte, NULL, &ctx->err)))\
856 return false; \
857 } while (0)
858
859 #if defined(C_LITTLE_ENDIAN) || 1
860 #define cgen_two(word) \
861 do { \
862 uint16_t word_ = (word); \
863 if (unlikely(!array_add_multiple_mayfail(uint8_t, &ctx->mcode, &ctx->mcode_size, cast_ptr(uint8_t *, &word_), 2, NULL, &ctx->err)))\
864 return false; \
865 } while (0)
866 #define cgen_four(dword) \
867 do { \
868 uint32_t dword_ = (dword); \
869 /*if (dword_ == 0x1ee02000) internal(file_line, "invalid instruction");*/\
870 if (unlikely(!array_add_multiple_mayfail(uint8_t, &ctx->mcode, &ctx->mcode_size, cast_ptr(uint8_t *, &dword_), 4, NULL, &ctx->err)))\
871 return false; \
872 } while (0)
873 #define cgen_eight(qword) \
874 do { \
875 uint64_t qword_ = (qword); \
876 if (unlikely(!array_add_multiple_mayfail(uint8_t, &ctx->mcode, &ctx->mcode_size, cast_ptr(uint8_t *, &qword_), 8, NULL, &ctx->err)))\
877 return false; \
878 } while (0)
879 #else
880 #define cgen_two(word) \
881 do { \
882 cgen_one((word) & 0xff); \
883 cgen_one((word) >> 8); \
884 } while (0)
885 #define cgen_four(dword) \
886 do { \
887 cgen_two((dword) & 0xffff); \
888 cgen_two((dword) >> 15 >> 1); \
889 } while (0)
890 #define cgen_eight(qword) \
891 do { \
892 cgen_four((qword) & 0xffffffff); \
893 cgen_four((qword) >> 15 >> 15 >> 2); \
894 } while (0)
895 #endif
896
897
898 #define IMM_PURPOSE_LDR_OFFSET 1
899 #define IMM_PURPOSE_LDR_SX_OFFSET 2
900 #define IMM_PURPOSE_STR_OFFSET 3
901 #define IMM_PURPOSE_LDP_STP_OFFSET 4
902 #define IMM_PURPOSE_VLDR_VSTR_OFFSET 5
903 #define IMM_PURPOSE_MVI_CLI_OFFSET 6
904 #define IMM_PURPOSE_STORE_VALUE 7
905 #define IMM_PURPOSE_ADD 8
906 #define IMM_PURPOSE_SUB 9
907 #define IMM_PURPOSE_CMP 10
908 #define IMM_PURPOSE_CMP_LOGICAL 11
909 #define IMM_PURPOSE_AND 12
910 #define IMM_PURPOSE_OR 13
911 #define IMM_PURPOSE_XOR 14
912 #define IMM_PURPOSE_ANDN 15
913 #define IMM_PURPOSE_TEST 16
914 #define IMM_PURPOSE_JMP_2REGS 17
915 #define IMM_PURPOSE_MUL 18
916 #define IMM_PURPOSE_CMOV 19
917 #define IMM_PURPOSE_MOVR 20
918 #define IMM_PURPOSE_BITWISE 21
919
920
921 static bool attr_w gen_upcall_end(struct codegen_context *ctx, unsigned args);
922
923 #define gen_address_offset() \
924 do { \
925 if (likely(!ctx->offset_reg)) { \
926 gen_one(ARG_ADDRESS_1); \
927 gen_one(ctx->base_reg); \
928 gen_eight(ctx->offset_imm); \
929 } else { \
930 gen_one(ARG_ADDRESS_2); \
931 gen_one(ctx->base_reg); \
932 gen_one(R_OFFSET_IMM); \
933 gen_eight(0); \
934 } \
935 } while (0)
936
937 #define gen_imm_offset() \
938 do { \
939 if (likely(!ctx->const_reg)) { \
940 gen_one(ARG_IMM); \
941 gen_eight(ctx->const_imm); \
942 } else { \
943 gen_one(R_CONST_IMM); \
944 } \
945 } while (0)
946
947 #define is_imm() (!ctx->const_reg)
948
949
950 #if defined(ARCH_ALPHA)
951 #include "c1-alpha.inc"
952 #elif defined(ARCH_ARM32)
953 #include "c1-arm.inc"
954 #elif defined(ARCH_ARM64)
955 #include "c1-arm64.inc"
956 #elif defined(ARCH_IA64)
957 #include "c1-ia64.inc"
958 #elif defined(ARCH_LOONGARCH64)
959 #include "c1-loong.inc"
960 #elif defined(ARCH_MIPS)
961 #include "c1-mips.inc"
962 #elif defined(ARCH_PARISC)
963 #include "c1-hppa.inc"
964 #elif defined(ARCH_POWER)
965 #include "c1-power.inc"
966 #elif defined(ARCH_S390)
967 #include "c1-s390.inc"
968 #elif defined(ARCH_SPARC)
969 #include "c1-sparc.inc"
970 #elif defined(ARCH_RISCV64)
971 #include "c1-riscv.inc"
972 #elif defined(ARCH_X86)
973 #include "c1-x86.inc"
974 #endif
975
976
977 #ifndef ARCH_SUPPORTS_TRAPS
978 #define ARCH_SUPPORTS_TRAPS 0
979 #define ARCH_TRAP_BEFORE 0
980 #endif
981
982
983 #include "cg-util.inc"
984
985 #include "cg-frame.inc"
986
987 #include "cg-flags.inc"
988
989 #include "cg-flcch.inc"
990
991 #include "cg-ptr.inc"
992
993 #include "cg-alu.inc"
994
995 #include "cg-ops.inc"
996
997
998 #ifndef n_regs_saved
999 #define n_regs_saved n_array_elements(regs_saved)
1000 #endif
1001
1002 #ifndef n_regs_volatile
1003 #define n_regs_volatile n_array_elements(regs_volatile)
1004 #endif
1005
1006 #ifndef n_fp_saved
1007 #define n_fp_saved n_array_elements(fp_saved)
1008 #endif
1009
1010 #ifndef n_fp_volatile
1011 #define n_fp_volatile n_array_elements(fp_volatile)
1012 #endif
1013
1014 #ifndef n_vector_volatile
1015 #define n_vector_volatile n_array_elements(vector_volatile)
1016 #endif
1017
1018 static bool attr_w gen_registers(struct codegen_context *ctx)
1019 {
1020 frame_t v;
1021 size_t index_saved = 0;
1022 size_t index_volatile = 0;
1023 size_t index_fp_saved = 0;
1024 size_t index_fp_volatile = 0;
1025 size_t attr_unused index_vector_volatile = 0;
1026 #ifdef ARCH_S390
1027 bool uses_x = false;
1028 for (v = MIN_USEABLE_SLOT; v < function_n_variables(ctx->fn); v++) {
1029 const struct type *t = get_type_of_local(ctx, v);
1030 if (t && TYPE_TAG_IS_REAL(t->tag) && TYPE_TAG_IDX_REAL(t->tag) == 4) {
1031 uses_x = true;
1032 break;
1033 }
1034 }
1035 #endif
1036 /*for (v = function_n_variables(ctx->fn) - 1; v >= MIN_USEABLE_SLOT; v--)*/
1037 for (v = MIN_USEABLE_SLOT; v < function_n_variables(ctx->fn); v++) {
1038 const struct type *t;
1039 ctx->registers[v] = -1;
1040 if (ra_chicken)
1041 continue;
1042 t = get_type_of_local(ctx, v);
1043 if (unlikely(!t))
1044 continue;
1045 if (!da(ctx->fn,function)->local_variables_flags[v].must_be_flat)
1046 continue;
1047 if (!ARCH_HAS_BWX && t->size < 1U << OP_SIZE_4)
1048 continue;
1049 if (TYPE_TAG_IS_FIXED(t->tag) || TYPE_TAG_IS_INT(t->tag) || t->tag == TYPE_TAG_flat_option) {
1050 if (!is_power_of_2(t->size) || t->size > 1U << OP_SIZE_NATIVE)
1051 continue;
1052 if (index_saved < n_regs_saved + zero
1053 #if defined(ARCH_PARISC) || defined(ARCH_SPARC)
1054 && t->size <= 1U << OP_SIZE_ADDRESS
1055 #endif
1056 ) {
1057 ctx->registers[v] = regs_saved[index_saved++];
1058 } else if (index_volatile < n_regs_volatile + zero) {
1059 ctx->registers[v] = regs_volatile[index_volatile++];
1060 } else {
1061 continue;
1062 }
1063 } else if (TYPE_TAG_IS_REAL(t->tag)) {
1064 unsigned real_type = TYPE_TAG_IDX_REAL(t->tag);
1065 if ((SUPPORTED_FP >> real_type) & 1) {
1066 #ifdef ARCH_POWER
1067 if (real_type == 4) {
1068 if (index_vector_volatile < n_vector_volatile + zero) {
1069 ctx->registers[v] = vector_volatile[index_vector_volatile++];
1070 goto success;
1071 }
1072 continue;
1073 }
1074 #endif
1075 #ifdef ARCH_S390
1076 if (real_type == 4) {
1077 if (!(index_fp_saved & 1) && index_fp_saved + 1 < n_fp_saved + zero) {
1078 ctx->registers[v] = fp_saved[index_fp_saved++];
1079 index_fp_saved++;
1080 goto success;
1081 }
1082 if (index_fp_saved & 1 && index_fp_saved + 2 < n_fp_saved + zero) {
1083 index_fp_saved++;
1084 ctx->registers[v] = fp_saved[index_fp_saved++];
1085 index_fp_saved++;
1086 goto success;
1087 }
1088 if (!(index_fp_volatile & 1) && index_fp_volatile + 1 < n_fp_volatile + zero) {
1089 ctx->registers[v] = fp_volatile[index_fp_volatile++];
1090 index_fp_volatile++;
1091 goto success;
1092 }
1093 if (index_fp_volatile & 1 && index_fp_volatile + 2 < n_fp_volatile + zero) {
1094 index_fp_volatile++;
1095 ctx->registers[v] = fp_volatile[index_fp_volatile++];
1096 index_fp_volatile++;
1097 goto success;
1098 }
1099 continue;
1100 }
1101 #endif
1102 if (index_fp_saved < n_fp_saved + zero) {
1103 ctx->registers[v] = fp_saved[index_fp_saved++];
1104 } else if (index_fp_volatile < n_fp_volatile + zero) {
1105 ctx->registers[v] = fp_volatile[index_fp_volatile++];
1106 } else {
1107 continue;
1108 }
1109 } else {
1110 continue;
1111 }
1112 } else {
1113 continue;
1114 }
1115 goto success;
1116 success:
1117 if (!reg_is_saved(ctx->registers[v])) {
1118 if (unlikely(!array_add_mayfail(frame_t, &ctx->need_spill, &ctx->need_spill_l, v, NULL, &ctx->err)))
1119 return false;
1120 }
1121 }
1122
1123 return true;
1124 }
1125
1126 static bool attr_w gen_fused_binary(struct codegen_context *ctx, unsigned mode, unsigned op_size, unsigned op, uint32_t escape_label, frame_t slot_1, frame_t slot_2, frame_t slot_r, bool *failed)
1127 {
1128 const code_t *backup = ctx->current_position;
1129 code_t code;
1130 frame_t slot_dr, slot_test;
1131 int32_t offs_false;
1132
1133 *failed = false;
1134
1135 next_code:
1136 code = get_code(ctx);
1137 ctx->arg_mode = code / OPCODE_MODE_MULT;
1138 code %= OPCODE_MODE_MULT;
1139 ajla_assert_lo(ctx->arg_mode < ARG_MODE_N, (file_line, "gen_fused_binary: invalid opcode %04x", (unsigned)*ctx->instr_start));
1140
1141 if (code == OPCODE_DEREFERENCE) {
1142 get_one(ctx, &slot_dr);
1143 if (unlikely(!flag_is_clear(ctx, slot_dr))) {
1144 internal(file_line, "gen_fused_binary: flag not cleared for destination slot");
1145 }
1146 goto next_code;
1147 }
1148 if (unlikely(code != OPCODE_JMP_FALSE))
1149 internal(file_line, "gen_fused_binary: binary operation is not followed by jmp false: %x, %s", code, decode_opcode(code, true));
1150 get_one(ctx, &slot_test);
1151 if (unlikely(slot_test != slot_r))
1152 internal(file_line, "gen_fused_binary: the result of the binary operation and the tested variable do not match");
1153 offs_false = get_jump_offset(ctx);
1154 get_jump_offset(ctx);
1155
1156 if (mode != MODE_REAL)
1157 g(gen_alu_jmp(ctx, mode, op_size, op, slot_1, slot_2, offs_false, failed));
1158 else
1159 g(gen_fp_alu_jmp(ctx, op_size, op, escape_label, slot_1, slot_2, offs_false, failed));
1160
1161 if (*failed)
1162 ctx->current_position = backup;
1163
1164 return true;
1165 }
1166
1167 static bool attr_w gen_function(struct codegen_context *ctx)
1168 {
1169 ctx->current_position = da(ctx->fn,function)->code;
1170
1171 ctx->escape_nospill_label = alloc_label(ctx);
1172 if (unlikely(!ctx->escape_nospill_label))
1173 return false;
1174
1175 while (ctx->current_position != da(ctx->fn,function)->code + da(ctx->fn,function)->code_size) {
1176 ip_t ip;
1177 code_t code;
1178 unsigned op, type;
1179 frame_t slot_1, slot_2, slot_3, slot_r, flags, fn_idx, opt;
1180 arg_t n_args, n_ret, i_arg;
1181 uint32_t label_id;
1182 uint32_t escape_label;
1183 bool failed;
1184
1185 ajla_assert_lo(ctx->current_position < da(ctx->fn,function)->code + da(ctx->fn,function)->code_size, (file_line, "gen_function: ran out of code in %s", da(ctx->fn,function)->function_name));
1186
1187 ctx->instr_start = ctx->current_position;
1188
1189 /*debug("%s: %04x, %s", da(ctx->fn,function)->function_name, *ctx->instr_start, decode_opcode(*ctx->instr_start, true));*/
1190
1191 ip = ctx->instr_start - da(ctx->fn,function)->code;
1192 if (likely(!ctx->code_labels[ip])) {
1193 ctx->code_labels[ip] = alloc_label(ctx);
1194 if (unlikely(!ctx->code_labels[ip]))
1195 return false;
1196 }
1197 gen_label(ctx->code_labels[ip]);
1198
1199 code = get_code(ctx);
1200 ctx->arg_mode = code / OPCODE_MODE_MULT;
1201 code %= OPCODE_MODE_MULT;
1202 ajla_assert_lo(ctx->arg_mode < ARG_MODE_N, (file_line, "gen_function: invalid opcode %04x", (unsigned)*ctx->instr_start));
1203
1204 if (code >= OPCODE_FIXED_OP + uzero && code < OPCODE_INT_OP) {
1205 code -= OPCODE_FIXED_OP;
1206 op = (code / OPCODE_FIXED_OP_MULT) % OPCODE_FIXED_TYPE_MULT;
1207 type = code / OPCODE_FIXED_TYPE_MULT;
1208 if (op < OPCODE_FIXED_OP_UNARY) {
1209 get_two(ctx, &slot_1, &slot_2);
1210 get_two(ctx, &slot_r, &flags);
1211 escape_label = alloc_escape_label(ctx);
1212 if (unlikely(!escape_label))
1213 return false;
1214 g(gen_test_2_cached(ctx, slot_1, slot_2, escape_label));
1215 flag_set(ctx, slot_1, false);
1216 flag_set(ctx, slot_2, false);
1217 flag_set(ctx, slot_r, false);
1218 if (flags & OPCODE_FLAG_FUSED) {
1219 g(gen_fused_binary(ctx, MODE_FIXED, type, op, escape_label, slot_1, slot_2, slot_r, &failed));
1220 if (unlikely(!failed))
1221 continue;
1222 }
1223 g(gen_alu(ctx, MODE_FIXED, type, op, escape_label, slot_1, slot_2, slot_r));
1224 continue;
1225 } else if (op < OPCODE_FIXED_OP_N) {
1226 get_two(ctx, &slot_1, &slot_r);
1227 get_one(ctx, &flags);
1228 escape_label = alloc_escape_label(ctx);
1229 if (unlikely(!escape_label))
1230 return false;
1231 g(gen_test_1_cached(ctx, slot_1, escape_label));
1232 flag_set(ctx, slot_1, false);
1233 flag_set(ctx, slot_r, false);
1234 g(gen_alu1(ctx, MODE_FIXED, type, op, escape_label, slot_1, slot_r));
1235 continue;
1236 } else if (op == OPCODE_FIXED_OP_ldc) {
1237 unsigned i;
1238 get_one(ctx, &slot_r);
1239 g(gen_constant(ctx, false, type, false, slot_r));
1240 for (i = 0; i < 1U << type; i += 2)
1241 get_code(ctx);
1242 flag_set(ctx, slot_r, false);
1243 continue;
1244 } else if (op == OPCODE_FIXED_OP_ldc16) {
1245 get_one(ctx, &slot_r);
1246 g(gen_constant(ctx, false, type, true, slot_r));
1247 get_code(ctx);
1248 flag_set(ctx, slot_r, false);
1249 continue;
1250 } else if (op == OPCODE_FIXED_OP_move || op == OPCODE_FIXED_OP_copy) {
1251 get_two(ctx, &slot_1, &slot_r);
1252 escape_label = alloc_escape_label(ctx);
1253 if (unlikely(!escape_label))
1254 return false;
1255 g(gen_test_1_cached(ctx, slot_1, escape_label));
1256 flag_set(ctx, slot_1, false);
1257 flag_set(ctx, slot_r, false);
1258 g(gen_copy(ctx, type, slot_1, slot_r));
1259 continue;
1260 } else {
1261 internal(file_line, "gen_function: bad fixed code %04x", *ctx->instr_start);
1262 }
1263 } else if (code >= OPCODE_INT_OP && code < OPCODE_REAL_OP) {
1264 code -= OPCODE_INT_OP;
1265 op = (code / OPCODE_INT_OP_MULT) % OPCODE_INT_TYPE_MULT;
1266 type = code / OPCODE_INT_TYPE_MULT;
1267 if (op < OPCODE_INT_OP_UNARY) {
1268 get_two(ctx, &slot_1, &slot_2);
1269 get_two(ctx, &slot_r, &flags);
1270 escape_label = alloc_escape_label(ctx);
1271 if (unlikely(!escape_label))
1272 return false;
1273 g(gen_test_2_cached(ctx, slot_1, slot_2, escape_label));
1274 flag_set(ctx, slot_1, false);
1275 flag_set(ctx, slot_2, false);
1276 flag_set(ctx, slot_r, false);
1277 if (flags & OPCODE_FLAG_FUSED) {
1278 g(gen_fused_binary(ctx, MODE_INT, type, op, escape_label, slot_1, slot_2, slot_r, &failed));
1279 if (unlikely(!failed))
1280 continue;
1281 }
1282 g(gen_alu(ctx, MODE_INT, type, op, escape_label, slot_1, slot_2, slot_r));
1283 continue;
1284 } else if (op < OPCODE_INT_OP_N) {
1285 get_two(ctx, &slot_1, &slot_r);
1286 get_one(ctx, &flags);
1287 if ((op == OPCODE_INT_OP_to_int || op == OPCODE_INT_OP_from_int) && slot_1 == slot_r)
1288 continue;
1289 escape_label = alloc_escape_label(ctx);
1290 if (unlikely(!escape_label))
1291 return false;
1292 g(gen_test_1_cached(ctx, slot_1, escape_label));
1293 flag_set(ctx, slot_1, false);
1294 flag_set(ctx, slot_r, false);
1295 g(gen_alu1(ctx, MODE_INT, type, op, escape_label, slot_1, slot_r));
1296 continue;
1297 } else if (op == OPCODE_INT_OP_ldc) {
1298 unsigned i;
1299 get_one(ctx, &slot_r);
1300 g(gen_constant(ctx, false, type, false, slot_r));
1301 for (i = 0; i < 1U << type; i += 2)
1302 get_code(ctx);
1303 flag_set(ctx, slot_r, false);
1304 continue;
1305 } else if (op == OPCODE_INT_OP_ldc16) {
1306 get_one(ctx, &slot_r);
1307 g(gen_constant(ctx, false, type, true, slot_r));
1308 get_code(ctx);
1309 flag_set(ctx, slot_r, false);
1310 continue;
1311 } else if (op == OPCODE_INT_OP_move || op == OPCODE_INT_OP_copy) {
1312 get_two(ctx, &slot_1, &slot_r);
1313 escape_label = alloc_escape_label(ctx);
1314 if (unlikely(!escape_label))
1315 return false;
1316 g(gen_test_1_cached(ctx, slot_1, escape_label));
1317 flag_set(ctx, slot_1, false);
1318 flag_set(ctx, slot_r, false);
1319 g(gen_copy(ctx, type, slot_1, slot_r));
1320 continue;
1321 } else {
1322 internal(file_line, "gen_function: bad integer code %04x", *ctx->instr_start);
1323 }
1324 } else if (code >= OPCODE_REAL_OP && code < OPCODE_BOOL_OP) {
1325 code -= OPCODE_REAL_OP;
1326 op = (code / OPCODE_REAL_OP_MULT) % OPCODE_REAL_TYPE_MULT;
1327 type = code / OPCODE_REAL_TYPE_MULT;
1328 if (op < OPCODE_REAL_OP_UNARY) {
1329 get_two(ctx, &slot_1, &slot_2);
1330 get_two(ctx, &slot_r, &flags);
1331 escape_label = alloc_escape_label(ctx);
1332 if (unlikely(!escape_label))
1333 return false;
1334 g(gen_test_2_cached(ctx, slot_1, slot_2, escape_label));
1335 flag_set(ctx, slot_1, false);
1336 flag_set(ctx, slot_2, false);
1337 flag_set(ctx, slot_r, false);
1338 if (flags & OPCODE_FLAG_FUSED) {
1339 g(gen_fused_binary(ctx, MODE_REAL, type, op, escape_label, slot_1, slot_2, slot_r, &failed));
1340 if (unlikely(!failed))
1341 continue;
1342 }
1343 g(gen_fp_alu(ctx, type, op, escape_label, slot_1, slot_2, slot_r));
1344 continue;
1345 } else if (op < OPCODE_REAL_OP_N) {
1346 get_two(ctx, &slot_1, &slot_r);
1347 get_one(ctx, &flags);
1348 escape_label = alloc_escape_label(ctx);
1349 if (unlikely(!escape_label))
1350 return false;
1351 g(gen_test_1_cached(ctx, slot_1, escape_label));
1352 flag_set(ctx, slot_1, false);
1353 flag_set(ctx, slot_r, false);
1354 g(gen_fp_alu1(ctx, type, op, escape_label, slot_1, slot_r));
1355 continue;
1356 } else if (op == OPCODE_REAL_OP_ldc) {
1357 const struct type *t;
1358 unsigned i;
1359 get_one(ctx, &slot_r);
1360 t = type_get_real(type);
1361 g(gen_real_constant(ctx, t, slot_r));
1362 for (i = 0; i < t->size; i += 2)
1363 get_code(ctx);
1364 flag_set(ctx, slot_r, false);
1365 continue;
1366 } else if (op == OPCODE_REAL_OP_move || op == OPCODE_REAL_OP_copy) {
1367 get_two(ctx, &slot_1, &slot_r);
1368 escape_label = alloc_escape_label(ctx);
1369 if (unlikely(!escape_label))
1370 return false;
1371 g(gen_test_1_cached(ctx, slot_1, escape_label));
1372 flag_set(ctx, slot_1, false);
1373 flag_set(ctx, slot_r, false);
1374 g(gen_memcpy_slots(ctx, slot_r, slot_1));
1375 continue;
1376 } else {
1377 internal(file_line, "gen_function: bad real code %04x", *ctx->instr_start);
1378 }
1379 } else if (code >= OPCODE_BOOL_OP && code < OPCODE_EXTRA) {
1380 code -= OPCODE_BOOL_OP;
1381 op = (code / OPCODE_BOOL_OP_MULT) % OPCODE_BOOL_TYPE_MULT;
1382 type = log_2(sizeof(ajla_flat_option_t));
1383 if (op < OPCODE_BOOL_OP_UNARY) {
1384 get_two(ctx, &slot_1, &slot_2);
1385 get_two(ctx, &slot_r, &flags);
1386 escape_label = alloc_escape_label(ctx);
1387 if (unlikely(!escape_label))
1388 return false;
1389 g(gen_test_2_cached(ctx, slot_1, slot_2, escape_label));
1390 flag_set(ctx, slot_1, false);
1391 flag_set(ctx, slot_2, false);
1392 flag_set(ctx, slot_r, false);
1393 if (flags & OPCODE_FLAG_FUSED) {
1394 g(gen_fused_binary(ctx, MODE_BOOL, type, op, escape_label, slot_1, slot_2, slot_r, &failed));
1395 if (unlikely(!failed))
1396 continue;
1397 }
1398 g(gen_alu(ctx, MODE_BOOL, type, op, escape_label, slot_1, slot_2, slot_r));
1399 continue;
1400 } else if (op < OPCODE_BOOL_OP_N) {
1401 get_two(ctx, &slot_1, &slot_r);
1402 get_one(ctx, &flags);
1403 escape_label = alloc_escape_label(ctx);
1404 if (unlikely(!escape_label))
1405 return false;
1406 g(gen_test_1_cached(ctx, slot_1, escape_label));
1407 flag_set(ctx, slot_1, false);
1408 flag_set(ctx, slot_r, false);
1409 g(gen_alu1(ctx, MODE_BOOL, type, op, escape_label, slot_1, slot_r));
1410 continue;
1411 } else if (op == OPCODE_BOOL_OP_move || op == OPCODE_BOOL_OP_copy) {
1412 get_two(ctx, &slot_1, &slot_r);
1413 escape_label = alloc_escape_label(ctx);
1414 if (unlikely(!escape_label))
1415 return false;
1416 g(gen_test_1_cached(ctx, slot_1, escape_label));
1417 flag_set(ctx, slot_1, false);
1418 flag_set(ctx, slot_r, false);
1419 g(gen_copy(ctx, type, slot_1, slot_r));
1420 continue;
1421 } else {
1422 internal(file_line, "gen_function: bad boolean code %04x", *ctx->instr_start);
1423 }
1424 } else switch (code) {
1425 case OPCODE_INT_LDC_LONG: {
1426 uint32_t words, w;
1427 get_one(ctx, &slot_r);
1428 words = get_uint32(ctx);
1429 for (w = 0; w < words; w++)
1430 get_code(ctx);
1431 unconditional_escape:
1432 escape_label = alloc_escape_label(ctx);
1433 if (unlikely(!escape_label))
1434 return false;
1435 gen_insn(INSN_JMP, 0, 0, 0);
1436 gen_four(escape_label);
1437 continue;
1438 }
1439 case OPCODE_IS_EXCEPTION: {
1440 get_two(ctx, &slot_1, &slot_r);
1441 get_one(ctx, &flags);
1442 g(gen_is_exception(ctx, slot_1, slot_r));
1443 continue;
1444 }
1445 case OPCODE_EXCEPTION_CLASS:
1446 case OPCODE_EXCEPTION_TYPE:
1447 case OPCODE_EXCEPTION_AUX: {
1448 get_two(ctx, &slot_1, &slot_r);
1449 get_one(ctx, &flags);
1450 goto unconditional_escape;
1451 }
1452 case OPCODE_SYSTEM_PROPERTY: {
1453 get_two(ctx, &slot_1, &slot_r);
1454 get_one(ctx, &flags);
1455 g(gen_system_property(ctx, slot_1, slot_r));
1456 continue;
1457 }
1458 case OPCODE_FLAT_MOVE:
1459 case OPCODE_FLAT_COPY: {
1460 get_two(ctx, &slot_1, &slot_r);
1461 g(gen_flat_move_copy(ctx, slot_1, slot_r));
1462 continue;
1463 }
1464 case OPCODE_REF_MOVE:
1465 case OPCODE_REF_MOVE_CLEAR:
1466 case OPCODE_REF_COPY: {
1467 get_two(ctx, &slot_1, &slot_r);
1468 g(gen_ref_move_copy(ctx, code, slot_1, slot_r));
1469 continue;
1470 }
1471 case OPCODE_BOX_MOVE_CLEAR:
1472 case OPCODE_BOX_COPY: {
1473 get_two(ctx, &slot_1, &slot_r);
1474 g(gen_box_move_copy(ctx, code, slot_1, slot_r));
1475 continue;
1476 }
1477 case OPCODE_TAKE_BORROWED:
1478 get_one(ctx, &slot_1);
1479 if (!da(ctx->fn,function)->local_variables_flags[slot_1].may_be_borrowed)
1480 continue;
1481 if (unlikely(!(label_id = alloc_label(ctx))))
1482 return false;
1483 if (flag_is_set(ctx, slot_1))
1484 goto take_borrowed_done;
1485 if (flag_is_clear(ctx, slot_1)) {
1486 g(gen_set_1(ctx, R_FRAME, slot_1, 0, true));
1487 goto do_take_borrowed;
1488 }
1489 g(gen_test_1(ctx, R_FRAME, slot_1, 0, label_id, false, TEST_SET));
1490 do_take_borrowed:
1491 g(gen_upcall_start(ctx, 1));
1492 g(gen_frame_load(ctx, OP_SIZE_SLOT, garbage, slot_1, 0, R_ARG0));
1493 g(gen_upcall_argument(ctx, 0));
1494 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_pointer_reference_owned), 1));
1495 flag_set(ctx, slot_1, true);
1496 take_borrowed_done:
1497 gen_label(label_id);
1498 continue;
1499 case OPCODE_DEREFERENCE:
1500 case OPCODE_DEREFERENCE_CLEAR: {
1501 bool need_bit_test;
1502 /*const struct type *type;*/
1503 get_one(ctx, &slot_1);
1504 if (flag_is_clear(ctx, slot_1))
1505 goto skip_dereference;
1506 /*type = get_type_of_local(ctx, slot_1);*/
1507 /*need_bit_test = 1 || TYPE_IS_FLAT(type) || da(ctx->fn,function)->local_variables[slot_1].may_be_borrowed;*/
1508 need_bit_test = !flag_is_set(ctx, slot_1);
1509 if (need_bit_test) {
1510 if (unlikely(!(label_id = alloc_label(ctx))))
1511 return false;
1512 g(gen_test_1(ctx, R_FRAME, slot_1, 0, label_id, true, TEST_CLEAR));
1513 } else {
1514 g(gen_set_1(ctx, R_FRAME, slot_1, 0, false));
1515 label_id = 0; /* avoid warning */
1516 }
1517 g(gen_upcall_start(ctx, 1));
1518 g(gen_frame_load(ctx, OP_SIZE_SLOT, garbage, slot_1, 0, R_ARG0));
1519 g(gen_upcall_argument(ctx, 0));
1520 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_pointer_dereference), 1));
1521 if (need_bit_test)
1522 gen_label(label_id);
1523 skip_dereference:
1524 if (code == OPCODE_DEREFERENCE_CLEAR)
1525 g(gen_frame_clear(ctx, OP_SIZE_SLOT, slot_1));
1526 flag_set(ctx, slot_1, false);
1527 continue;
1528 }
1529 case OPCODE_EVAL: {
1530 get_one(ctx, &slot_1);
1531 g(gen_eval(ctx, slot_1));
1532 continue;
1533 }
1534 case OPCODE_ESCAPE_NONFLAT: {
1535 frame_t n, i;
1536 frame_t *vars;
1537
1538 get_one(ctx, &n);
1539 vars = mem_alloc_array_mayfail(mem_alloc_mayfail, frame_t *, 0, 0, n, sizeof(frame_t), &ctx->err);
1540 if (unlikely(!vars))
1541 return false;
1542 for (i = 0; i < n; i++) {
1543 get_one(ctx, &vars[i]);
1544 }
1545
1546 escape_label = alloc_escape_label(ctx);
1547 if (unlikely(!escape_label)) {
1548 mem_free(vars);
1549 return false;
1550 }
1551
1552 if (unlikely(!gen_test_multiple(ctx, vars, n, escape_label))) {
1553 mem_free(vars);
1554 return false;
1555 }
1556 mem_free(vars);
1557
1558 continue;
1559 }
1560 case OPCODE_CHECKPOINT: {
1561 frame_t n_vars;
1562
1563 g(clear_flag_cache(ctx));
1564
1565 if (SIZEOF_IP_T == 2) {
1566 slot_1 = get_code(ctx);
1567 } else if (SIZEOF_IP_T == 4) {
1568 slot_1 = get_uint32(ctx);
1569 } else {
1570 not_reached();
1571 continue;
1572 }
1573
1574 if (unlikely(!(slot_1 + 1)))
1575 return false;
1576 while (slot_1 >= ctx->n_entries) {
1577 void *err_entries;
1578 struct cg_entry e;
1579 if (unlikely(!ctx->entries)) {
1580 if (unlikely(!array_init_mayfail(struct cg_entry, &ctx->entries, &ctx->n_entries, &ctx->err)))
1581 return false;
1582 }
1583 memset(&e, 0, sizeof(struct cg_entry));
1584 if (unlikely(!array_add_mayfail(struct cg_entry, &ctx->entries, &ctx->n_entries, e, &err_entries, &ctx->err))) {
1585 ctx->entries = err_entries;
1586 return false;
1587 }
1588 }
1589
1590 get_one(ctx, &n_vars);
1591
1592 escape_label = 0; /* avoid warning */
1593 if (likely(slot_1 != 0)) {
1594 escape_label = alloc_escape_label(ctx);
1595 if (unlikely(!escape_label))
1596 return false;
1597 }
1598
1599 if (n_vars || !slot_1) {
1600 frame_t i;
1601 uint32_t entry_label, nonflat_label;
1602 struct cg_entry *ce = &ctx->entries[slot_1];
1603
1604 if (unlikely(!array_init_mayfail(frame_t, &ce->variables, &ce->n_variables, &ctx->err)))
1605 return false;
1606 for (i = 0; i < n_vars; i++) {
1607 frame_t v;
1608 get_one(ctx, &v);
1609 if (unlikely(!array_add_mayfail(frame_t, &ce->variables, &ce->n_variables, v, NULL, &ctx->err)))
1610 return false;
1611 }
1612 if (!slot_1) {
1613 g(gen_test_multiple(ctx, ce->variables, ce->n_variables, ctx->escape_nospill_label));
1614 }
1615 entry_label = alloc_label(ctx);
1616 if (unlikely(!entry_label))
1617 return false;
1618 gen_label(entry_label);
1619 ce->entry_label = entry_label;
1620
1621 nonflat_label = alloc_escape_label_for_ip(ctx, ctx->current_position);
1622 if (unlikely(!nonflat_label))
1623 return false;
1624 ce->nonflat_label = nonflat_label;
1625
1626 if (unlikely(!slot_1))
1627 g(gen_timestamp_test(ctx, ctx->escape_nospill_label));
1628 else
1629 g(gen_timestamp_test(ctx, escape_label));
1630 } else {
1631 g(gen_timestamp_test(ctx, escape_label));
1632
1633 gen_insn(INSN_ENTRY, 0, 0, 0);
1634 gen_four(slot_1);
1635 }
1636 continue;
1637 }
1638 case OPCODE_JMP: {
1639 int32_t x = get_jump_offset(ctx);
1640 g(gen_jump(ctx, x, OP_SIZE_NATIVE, COND_ALWAYS, -1U, -1U));
1641 continue;
1642 }
1643 case OPCODE_JMP_BACK_16: {
1644 int32_t x = get_code(ctx);
1645 g(gen_jump(ctx, -x - (int)(2 * sizeof(code_t)), OP_SIZE_NATIVE, COND_ALWAYS, -1U, -1U));
1646 continue;
1647 }
1648 case OPCODE_JMP_FALSE: {
1649 int32_t offs_false;
1650 get_one(ctx, &slot_1);
1651 offs_false = get_jump_offset(ctx);
1652 get_jump_offset(ctx);
1653 escape_label = alloc_escape_label(ctx);
1654 if (unlikely(!escape_label))
1655 return false;
1656 g(gen_test_1_cached(ctx, slot_1, escape_label));
1657 flag_set(ctx, slot_1, false);
1658 g(gen_cond_jump(ctx, slot_1, offs_false));
1659 continue;
1660 }
1661 case OPCODE_LABEL: {
1662 g(clear_flag_cache(ctx));
1663 continue;
1664 }
1665 #define init_args \
1666 do { \
1667 if (ctx->args != NULL) \
1668 mem_free(ctx->args); \
1669 g(array_init_mayfail(struct code_arg, &ctx->args, &ctx->args_l, &ctx->err));\
1670 } while (0)
1671 #define load_args \
1672 do { \
1673 init_args; \
1674 for (i_arg = 0; i_arg < n_args; i_arg++) { \
1675 struct code_arg a; \
1676 get_two(ctx, &a.slot, &a.flags); \
1677 a.type = 0; \
1678 g(array_add_mayfail(struct code_arg, &ctx->args, &ctx->args_l, a, NULL, &ctx->err));\
1679 } \
1680 } while (0)
1681 case OPCODE_LOAD_FN:
1682 get_two(ctx, &n_args, &slot_r);
1683 get_one(ctx, &fn_idx);
1684 load_args;
1685 g(gen_load_fn_or_curry(ctx, fn_idx, NO_FRAME_T, slot_r, 0));
1686 continue;
1687 case OPCODE_CURRY:
1688 get_two(ctx, &n_args, &slot_r);
1689 get_two(ctx, &slot_1, &flags);
1690 load_args;
1691 g(gen_load_fn_or_curry(ctx, NO_FRAME_T, slot_1, slot_r, flags));
1692 continue;
1693 case OPCODE_CALL:
1694 case OPCODE_CALL_STRICT:
1695 case OPCODE_CALL_SPARK:
1696 case OPCODE_CALL_LAZY:
1697 case OPCODE_CALL_CACHE:
1698 case OPCODE_CALL_SAVE: {
1699 get_two(ctx, &n_args, &n_ret);
1700 get_one(ctx, &fn_idx);
1701 jump_over_arguments_and_return:
1702 load_args;
1703 ctx->return_values = ctx->current_position;
1704 for (i_arg = 0; i_arg < n_ret; i_arg++) {
1705 #if ARG_MODE_N >= 3
1706 get_uint32(ctx);
1707 #else
1708 get_code(ctx);
1709 #endif
1710 get_code(ctx);
1711 }
1712 if (unlikely(profiling))
1713 goto unconditional_escape;
1714 if (code == OPCODE_CALL || code == OPCODE_CALL_STRICT) {
1715 g(gen_call(ctx, code, fn_idx));
1716 continue;
1717 }
1718 /*if (code == OPCODE_CALL_INDIRECT || code == OPCODE_CALL_INDIRECT_STRICT) {
1719 if (unlikely(!gen_call_indirect(ctx, code, slot_1, flags)))
1720 return false;
1721 continue;
1722 }*/
1723 goto unconditional_escape;
1724 }
1725 case OPCODE_CALL_INDIRECT:
1726 case OPCODE_CALL_INDIRECT_STRICT:
1727 case OPCODE_CALL_INDIRECT_SPARK:
1728 case OPCODE_CALL_INDIRECT_LAZY:
1729 case OPCODE_CALL_INDIRECT_CACHE:
1730 case OPCODE_CALL_INDIRECT_SAVE: {
1731 fn_idx = 0; /* avoid warning */
1732 get_two(ctx, &n_args, &n_ret);
1733 get_two(ctx, &slot_1, &flags);
1734 goto jump_over_arguments_and_return;
1735 }
1736 case OPCODE_RETURN: {
1737 n_args = da(ctx->fn,function)->n_return_values;
1738 load_args;
1739 if (unlikely(profiling))
1740 goto unconditional_escape;
1741 g(gen_return(ctx));
1742 continue;
1743 }
1744 case OPCODE_STRUCTURED: {
1745 init_args;
1746 get_two(ctx, &slot_1, &slot_2);
1747 do {
1748 struct code_arg a;
1749 get_two(ctx, &flags, &slot_r);
1750 get_one(ctx, &opt);
1751 a.slot = slot_r;
1752 a.flags = flags;
1753 a.type = opt;
1754 g(array_add_mayfail(struct code_arg, &ctx->args, &ctx->args_l, a, NULL, &ctx->err));
1755 } while (!(flags & OPCODE_STRUCTURED_FLAG_END));
1756 g(gen_structured(ctx, slot_1, slot_2));
1757 continue;
1758 }
1759 case OPCODE_RECORD_CREATE: {
1760 init_args;
1761 get_two(ctx, &slot_r, &n_args);
1762 for (i_arg = 0; i_arg < n_args; i_arg++) {
1763 struct code_arg a;
1764 get_two(ctx, &slot_1, &flags);
1765 a.slot = slot_1;
1766 a.flags = flags;
1767 a.type = 0;
1768 g(array_add_mayfail(struct code_arg, &ctx->args, &ctx->args_l, a, NULL, &ctx->err));
1769 }
1770 g(gen_record_create(ctx, slot_r));
1771 continue;
1772 }
1773 case OPCODE_RECORD_LOAD: {
1774 get_two(ctx, &slot_1, &opt);
1775 get_two(ctx, &slot_r, &flags);
1776 g(gen_record_load(ctx, slot_1, slot_r, opt, flags));
1777 continue;
1778 }
1779 case OPCODE_OPTION_CREATE_EMPTY_FLAT: {
1780 get_two(ctx, &slot_r, &opt);
1781 g(gen_option_create_empty_flat(ctx, opt, slot_r));
1782 continue;
1783 }
1784 case OPCODE_OPTION_CREATE_EMPTY: {
1785 get_two(ctx, &slot_r, &opt);
1786 g(gen_option_create_empty(ctx, opt, slot_r));
1787 continue;
1788 }
1789 case OPCODE_OPTION_CREATE: {
1790 get_two(ctx, &slot_r, &opt);
1791 get_two(ctx, &slot_1, &flags);
1792 g(gen_option_create(ctx, opt, slot_1, slot_r, flags));
1793 continue;
1794 }
1795 case OPCODE_OPTION_LOAD: {
1796 get_two(ctx, &slot_1, &opt);
1797 get_two(ctx, &slot_r, &flags);
1798 g(gen_option_load(ctx, slot_1, slot_r, opt, flags));
1799 continue;
1800 }
1801 case OPCODE_OPTION_TEST_FLAT: {
1802 get_two(ctx, &slot_1, &opt);
1803 get_one(ctx, &slot_r);
1804 g(gen_option_test_flat(ctx, slot_1, opt, slot_r));
1805 continue;
1806 }
1807 case OPCODE_OPTION_TEST: {
1808 get_two(ctx, &slot_1, &opt);
1809 get_one(ctx, &slot_r);
1810 g(gen_option_test(ctx, slot_1, opt, slot_r));
1811 continue;
1812 }
1813 case OPCODE_OPTION_ORD_FLAT: {
1814 get_two(ctx, &slot_1, &slot_r);
1815 g(gen_option_ord(ctx, slot_1, slot_r, true));
1816 continue;
1817 }
1818 case OPCODE_OPTION_ORD: {
1819 get_two(ctx, &slot_1, &slot_r);
1820 g(gen_option_ord(ctx, slot_1, slot_r, false));
1821 continue;
1822 }
1823 case OPCODE_ARRAY_CREATE: {
1824 init_args;
1825 get_two(ctx, &slot_r, &n_args);
1826 for (i_arg = 0; i_arg < n_args; i_arg++) {
1827 struct code_arg a;
1828 get_two(ctx, &slot_1, &flags);
1829 a.slot = slot_1;
1830 a.flags = flags;
1831 a.type = 0;
1832 g(array_add_mayfail(struct code_arg, &ctx->args, &ctx->args_l, a, NULL, &ctx->err));
1833 }
1834 g(gen_array_create(ctx, slot_r));
1835 continue;
1836 }
1837 case OPCODE_ARRAY_CREATE_EMPTY_FLAT: {
1838 get_two(ctx, &slot_r, &flags);
1839 g(gen_array_create_empty_flat(ctx, slot_r, flags));
1840 continue;
1841 }
1842 case OPCODE_ARRAY_CREATE_EMPTY: {
1843 get_one(ctx, &slot_r);
1844 g(gen_array_create_empty(ctx, slot_r));
1845 continue;
1846 }
1847 case OPCODE_ARRAY_FILL: {
1848 get_two(ctx, &slot_1, &flags);
1849 get_two(ctx, &slot_2, &slot_r);
1850 g(gen_array_fill(ctx, slot_1, flags, slot_2, slot_r));
1851 continue;
1852 }
1853 case OPCODE_ARRAY_STRING: {
1854 frame_t i;
1855 get_two(ctx, &slot_r, &i);
1856 g(gen_array_string(ctx, type_get_fixed(0, true)->tag, cast_ptr(uint8_t *, ctx->current_position), i, slot_r));
1857 ctx->current_position += (i + 1) >> 1;
1858 continue;
1859 }
1860 case OPCODE_ARRAY_UNICODE: {
1861 frame_t i;
1862 get_two(ctx, &slot_r, &i);
1863 g(gen_array_string(ctx, type_get_int(2)->tag, cast_ptr(uint8_t *, ctx->current_position), i, slot_r));
1864 ctx->current_position += i * 2;
1865 continue;
1866 }
1867 case OPCODE_ARRAY_LOAD: {
1868 get_two(ctx, &slot_1, &slot_2);
1869 get_two(ctx, &slot_r, &flags);
1870 g(gen_array_load(ctx, slot_1, slot_2, slot_r, flags));
1871 continue;
1872 }
1873 case OPCODE_ARRAY_LEN: {
1874 get_two(ctx, &slot_1, &slot_r);
1875 get_one(ctx, &flags);
1876 g(gen_array_len(ctx, slot_1, NO_FRAME_T, slot_r));
1877 continue;
1878 }
1879 case OPCODE_ARRAY_LEN_GREATER_THAN: {
1880 get_two(ctx, &slot_1, &slot_2);
1881 get_two(ctx, &slot_r, &flags);
1882 g(gen_array_len(ctx, slot_1, slot_2, slot_r));
1883 continue;
1884 }
1885 case OPCODE_ARRAY_SUB: {
1886 get_two(ctx, &slot_1, &slot_2);
1887 get_two(ctx, &slot_3, &slot_r);
1888 get_one(ctx, &flags);
1889 g(gen_array_sub(ctx, slot_1, slot_2, slot_3, slot_r, flags));
1890 continue;
1891 }
1892 case OPCODE_ARRAY_SKIP: {
1893 get_two(ctx, &slot_1, &slot_2);
1894 get_two(ctx, &slot_r, &flags);
1895 g(gen_array_skip(ctx, slot_1, slot_2, slot_r, flags));
1896 continue;
1897 }
1898 case OPCODE_ARRAY_APPEND: {
1899 get_two(ctx, &slot_r, &flags);
1900 get_two(ctx, &slot_1, &slot_2);
1901 g(gen_array_append(ctx, slot_1, slot_2, slot_r, flags));
1902 continue;
1903 }
1904 case OPCODE_ARRAY_APPEND_ONE_FLAT: {
1905 get_two(ctx, &slot_r, &flags);
1906 get_two(ctx, &slot_1, &slot_2);
1907 g(gen_array_append_one_flat(ctx, slot_1, slot_2, slot_r, flags));
1908 continue;
1909 }
1910 case OPCODE_ARRAY_APPEND_ONE: {
1911 get_two(ctx, &slot_r, &flags);
1912 get_two(ctx, &slot_1, &slot_2);
1913 g(gen_array_append_one(ctx, slot_1, slot_2, slot_r, flags));
1914 continue;
1915 }
1916 case OPCODE_ARRAY_FLATTEN: {
1917 get_two(ctx, &slot_r, &flags);
1918 get_one(ctx, &slot_1);
1919 goto unconditional_escape;
1920 }
1921 case OPCODE_IO: {
1922 get_two(ctx, &flags, &slot_1);
1923 get_two(ctx, &slot_2, &slot_3);
1924 g(gen_io(ctx, flags, slot_1, slot_2, slot_3));
1925 continue;
1926 }
1927 case OPCODE_INTERNAL_FUNCTION:
1928 case OPCODE_EXIT_THREAD:
1929 case OPCODE_UNREACHABLE: {
1930 goto unconditional_escape;
1931 }
1932 default: {
1933 #if 1
1934 /*if (getenv("DUMP") && !strcmp(da(ctx->fn,function)->function_name, getenv("DUMP")))*/
1935 warning("gen_function: %s: unknown opcode %04x, %s", da(ctx->fn,function)->function_name, *ctx->instr_start, decode_opcode(*ctx->instr_start, false));
1936 #endif
1937 return false;
1938 }
1939 }
1940 }
1941
1942 return true;
1943 }
1944
1945 static bool attr_w gen_entries(struct codegen_context *ctx)
1946 {
1947 size_t i;
1948 for (i = 0; i < ctx->n_entries; i++) {
1949 struct cg_entry *ce = &ctx->entries[i];
1950 if (ce->entry_label) {
1951 gen_insn(INSN_ENTRY, 0, 0, 0);
1952 gen_four(i);
1953
1954 g(gen_test_multiple(ctx, ce->variables, ce->n_variables, ce->nonflat_label));
1955
1956 gen_insn(INSN_JMP, 0, 0, 0);
1957 gen_four(ce->entry_label);
1958 }
1959 }
1960 return true;
1961 }
1962
1963 static bool attr_w gen_epilogues(struct codegen_context *ctx)
1964 {
1965 frame_t v;
1966 ip_t ip;
1967 uint32_t escape_label, nospill_label;
1968 escape_label = alloc_label(ctx);
1969 if (unlikely(!escape_label))
1970 return false;
1971 nospill_label = alloc_label(ctx);
1972 if (unlikely(!nospill_label))
1973 return false;
1974 #if defined(ARCH_PARISC)
1975 if (ctx->call_label) {
1976 gen_label(ctx->call_label);
1977 g(gen_call_millicode(ctx));
1978 }
1979 #endif
1980 if (ctx->reload_label) {
1981 gen_label(ctx->reload_label);
1982 g(gen_mov(ctx, i_size(OP_SIZE_ADDRESS), R_FRAME, R_RET0));
1983 g(gen_escape_arg(ctx, (ip_t)-1, escape_label));
1984 }
1985 gen_label(ctx->escape_nospill_label);
1986 g(gen_escape_arg(ctx, 0, nospill_label));
1987 for (ip = 0; ip < da(ctx->fn,function)->code_size; ip++) {
1988 struct cg_exit *ce = ctx->code_exits[ip];
1989 if (ce && (ce->undo_label || ce->escape_label)) {
1990 if (ce->undo_label) {
1991 size_t i;
1992 gen_label(ce->undo_label);
1993 gen_insn(ce->undo_opcode, ce->undo_op_size, ce->undo_aux, ce->undo_writes_flags);
1994 for (i = 0; i < ce->undo_parameters_len; i++)
1995 gen_one(ce->undo_parameters[i]);
1996 }
1997 if (ce->escape_label) {
1998 gen_label(ce->escape_label);
1999 }
2000 g(gen_escape_arg(ctx, ip, escape_label));
2001 }
2002 }
2003 gen_label(escape_label);
2004 for (v = MIN_USEABLE_SLOT; v < function_n_variables(ctx->fn); v++) {
2005 if (ctx->registers[v] >= 0) {
2006 g(spill(ctx, v));
2007 }
2008 }
2009 gen_label(nospill_label);
2010 g(gen_escape(ctx));
2011 return true;
2012 }
2013
2014 static bool attr_w cgen_entry(struct codegen_context *ctx)
2015 {
2016 uint32_t entry_id = cget_four(ctx);
2017 ajla_assert_lo(entry_id < ctx->n_entries, (file_line, "cgen_entry: invalid entry %lx", (unsigned long)entry_id));
2018 ctx->entries[entry_id].entry_to_pos = ctx->mcode_size;
2019 return true;
2020 }
2021
2022 static bool attr_w cgen_label(struct codegen_context *ctx)
2023 {
2024 uint32_t label_id = cget_four(ctx);
2025 ctx->label_to_pos[label_id] = ctx->mcode_size;
2026 return true;
2027 }
2028
2029 static bool attr_w attr_unused cgen_trap(struct codegen_context *ctx, uint32_t label)
2030 {
2031 struct trap_record tr;
2032 tr.source_ip = ctx->mcode_size;
2033 tr.destination_ip = label;
2034 if (unlikely(!array_add_mayfail(struct trap_record, &ctx->trap_records, &ctx->trap_records_size, tr, NULL, &ctx->err)))
2035 return false;
2036 return true;
2037 }
2038
2039 static bool attr_w add_relocation(struct codegen_context *ctx, unsigned length, int offset, bool *known)
2040 {
2041 struct relocation rel;
2042 rel.label_id = cget_four(ctx);
2043 rel.length = length;
2044 rel.position = ctx->mcode_size;
2045 rel.jmp_instr = ctx->code_position - 8 - offset - ctx->code;
2046 if (unlikely(!array_add_mayfail(struct relocation, &ctx->reloc, &ctx->reloc_size, rel, NULL, &ctx->err)))
2047 return false;
2048 if (known)
2049 *known = ctx->label_to_pos[rel.label_id] != (size_t)-1;
2050 return true;
2051 }
2052
2053
2054 #if defined(ARCH_ALPHA)
2055 #include "c2-alpha.inc"
2056 #elif defined(ARCH_ARM32)
2057 #include "c2-arm.inc"
2058 #elif defined(ARCH_ARM64)
2059 #include "c2-arm64.inc"
2060 #elif defined(ARCH_IA64)
2061 #include "c2-ia64.inc"
2062 #elif defined(ARCH_LOONGARCH64)
2063 #include "c2-loong.inc"
2064 #elif defined(ARCH_MIPS)
2065 #include "c2-mips.inc"
2066 #elif defined(ARCH_PARISC)
2067 #include "c2-hppa.inc"
2068 #elif defined(ARCH_POWER)
2069 #include "c2-power.inc"
2070 #elif defined(ARCH_S390)
2071 #include "c2-s390.inc"
2072 #elif defined(ARCH_SPARC)
2073 #include "c2-sparc.inc"
2074 #elif defined(ARCH_RISCV64)
2075 #include "c2-riscv.inc"
2076 #elif defined(ARCH_X86)
2077 #include "c2-x86.inc"
2078 #endif
2079
2080
2081 static bool attr_w gen_mcode(struct codegen_context *ctx)
2082 {
2083 ctx->code_position = ctx->code;
2084
2085 while (ctx->code_position != ctx->code + ctx->code_size) {
2086 uint32_t insn;
2087 ajla_assert_lo(ctx->code_position < ctx->code + ctx->code_size, (file_line, "gen_mcode: ran out of code"));
2088 #ifdef DEBUG_INSNS
2089 insn = cget_four(ctx);
2090 debug("line: %u", insn);
2091 #endif
2092 insn = cget_four(ctx);
2093 g(cgen_insn(ctx, insn));
2094 }
2095
2096 return true;
2097 }
2098
2099 #define RELOCS_RETRY -1
2100 #define RELOCS_FAIL 0
2101 #define RELOCS_OK 1
2102
2103 static int8_t resolve_relocs(struct codegen_context *ctx)
2104 {
2105 size_t i;
2106 int8_t status = RELOCS_OK;
2107 for (i = 0; i < ctx->reloc_size; i++) {
2108 struct relocation *reloc = &ctx->reloc[i];
2109 if (!resolve_relocation(ctx, reloc)) {
2110 uint8_t *jmp_instr;
2111 uint32_t insn;
2112 uint32_t new_length;
2113 status = RELOCS_RETRY;
2114 if (unlikely(reloc->length + zero >= JMP_LIMIT))
2115 return RELOCS_FAIL;
2116 new_length = reloc->length + 1;
2117 jmp_instr = ctx->code + reloc->jmp_instr;
2118 insn = (uint32_t)jmp_instr[0] +
2119 ((uint32_t)jmp_instr[1] << 8) +
2120 ((uint32_t)jmp_instr[2] << 16) +
2121 ((uint32_t)jmp_instr[3] << 24);
2122 insn &= ~INSN_JUMP_SIZE;
2123 insn |= (uint32_t)new_length << INSN_JUMP_SIZE_SHIFT;
2124 jmp_instr[0] = insn;
2125 jmp_instr[1] = insn >> 8;
2126 jmp_instr[2] = insn >> 16;
2127 jmp_instr[3] = insn >> 24;
2128 }
2129 }
2130 return status;
2131 }
2132
2133 static void resolve_traps(struct codegen_context *ctx)
2134 {
2135 size_t i;
2136 for (i = 0; i < ctx->trap_records_size; i++) {
2137 struct trap_record *tr = &ctx->trap_records[i];
2138 tr->destination_ip = ctx->label_to_pos[tr->destination_ip];
2139 }
2140 }
2141
2142
2143 static bool attr_w codegen_map(struct codegen_context *ctx)
2144 {
2145 void *ptr;
2146 frame_t i;
2147 array_finish(uint8_t, &ctx->mcode, &ctx->mcode_size);
2148 ptr = os_code_map(ctx->mcode, ctx->mcode_size, &ctx->err);
2149 ctx->mcode = NULL;
2150 if (unlikely(!ptr)) {
2151 return false;
2152 }
2153 for (i = 0; i < ctx->n_entries; i++) {
2154 char *entry = cast_ptr(char *, ptr) + ctx->entries[i].entry_to_pos;
2155 da(ctx->codegen,codegen)->unoptimized_code[i] = entry;
2156 da(ctx->codegen,codegen)->n_entries++;
2157 }
2158 da(ctx->codegen,codegen)->unoptimized_code_base = ptr;
2159 da(ctx->codegen,codegen)->unoptimized_code_size = ctx->mcode_size;
2160
2161 return true;
2162 }
2163
2164
2165 void *codegen_fn(frame_s *fp, const code_t *ip, union internal_arg ia[])
2166 {
2167 struct codegen_context ctx_;
2168 struct codegen_context *ctx = &ctx_;
2169 frame_t i;
2170 int8_t rr;
2171 struct data *codegen;
2172 uint32_t l;
2173
2174 init_ctx(ctx);
2175 ctx->fn = ia[0].ptr;
2176
2177 #ifdef DEBUG_ENV
2178 if (getenv("CG") && strcmp(da(ctx->fn,function)->function_name, getenv("CG")))
2179 goto fail;
2180 #endif
2181
2182 ctx->local_directory = mem_alloc_array_mayfail(mem_calloc_mayfail, struct data **, 0, 0, da(ctx->fn,function)->local_directory_size, sizeof(struct data *), &ctx->err);
2183 if (unlikely(!ctx->local_directory))
2184 goto fail;
2185
2186 if (0) for (i = 0; i < da(ctx->fn,function)->local_directory_size; i++) {
2187 struct data *callee;
2188 pointer_t *ptr;
2189 ptr = da(ctx->fn,function)->local_directory[i];
2190 pointer_follow(ptr, false, callee, PF_SPARK, NULL, 0,
2191 SUBMIT_EX(ex_);
2192 goto next_one,
2193 goto next_one;
2194 );
2195 ctx->local_directory[i] = callee;
2196 next_one:;
2197 }
2198 for (i = 0; i < da(ctx->fn,function)->local_directory_size; i++) {
2199 struct data *callee;
2200 pointer_t *ptr;
2201 if (ctx->local_directory[i])
2202 continue;
2203 ptr = da(ctx->fn,function)->local_directory[i];
2204 pointer_follow(ptr, false, callee, PF_WAIT, fp, ip,
2205 done_ctx(ctx);
2206 return ex_,
2207 goto fail
2208 );
2209 ctx->local_directory[i] = callee;
2210 /*debug("processing call: %s -> %s", da(ctx->fn,function)->function_name, da(callee,function)->function_name);*/
2211 }
2212
2213 if (da(ctx->fn,function)->module_designator) {
2214 struct function_descriptor *sfd = save_find_function_descriptor(da(ctx->fn,function)->module_designator, da(ctx->fn,function)->function_designator);
2215 if (sfd && sfd->unoptimized_code_size) {
2216 codegen = data_alloc_flexible(codegen, unoptimized_code, sfd->n_entries, &ctx->err);
2217 if (unlikely(!codegen))
2218 goto fail;
2219 da(codegen,codegen)->unoptimized_code_base = sfd->unoptimized_code_base;
2220 da(codegen,codegen)->unoptimized_code_size = sfd->unoptimized_code_size;
2221 da(codegen,codegen)->function = ctx->fn;
2222 da(codegen,codegen)->is_saved = true;
2223 da(codegen,codegen)->n_entries = sfd->n_entries;
2224 da(codegen,codegen)->offsets = NULL;
2225 for (i = 0; i < sfd->n_entries; i++) {
2226 da(codegen,codegen)->unoptimized_code[i] = cast_ptr(char *, da(codegen,codegen)->unoptimized_code_base) + sfd->entries[i];
2227 /*debug("%s: %p + %lx -> %p", da(ctx->fn,function)->function_name, da(codegen,codegen)->unoptimized_code_base, sfd->entries[i], da(codegen,codegen)->unoptimized_code[i]);*/
2228 }
2229 #ifdef HAVE_CODEGEN_TRAPS
2230 da(codegen,codegen)->trap_records = sfd->trap_records;
2231 da(codegen,codegen)->trap_records_size = sfd->trap_records_size;
2232 data_trap_insert(codegen);
2233 #endif
2234 goto have_codegen;
2235 }
2236 }
2237
2238 /*debug("trying: %s", da(ctx->fn,function)->function_name);*/
2239 if (unlikely(!array_init_mayfail(uint8_t, &ctx->code, &ctx->code_size, &ctx->err)))
2240 goto fail;
2241
2242 ctx->code_labels = mem_alloc_array_mayfail(mem_calloc_mayfail, uint32_t *, 0, 0, da(ctx->fn,function)->code_size, sizeof(uint32_t), &ctx->err);
2243 if (unlikely(!ctx->code_labels))
2244 goto fail;
2245
2246 ctx->code_exits = mem_alloc_array_mayfail(mem_calloc_mayfail, struct cg_exit **, 0, 0, da(ctx->fn,function)->code_size, sizeof(struct cg_exit *), &ctx->err);
2247 if (unlikely(!ctx->code_exits))
2248 goto fail;
2249
2250 ctx->flag_cache = mem_alloc_array_mayfail(mem_calloc_mayfail, int8_t *, 0, 0, function_n_variables(ctx->fn), sizeof(int8_t), &ctx->err);
2251 if (unlikely(!ctx->flag_cache))
2252 goto fail;
2253
2254 ctx->registers = mem_alloc_array_mayfail(mem_alloc_mayfail, short *, 0, 0, function_n_variables(ctx->fn), sizeof(short), &ctx->err);
2255 if (unlikely(!ctx->registers))
2256 goto fail;
2257
2258 if (unlikely(!array_init_mayfail(frame_t, &ctx->need_spill, &ctx->need_spill_l, &ctx->err)))
2259 goto fail;
2260
2261 if (unlikely(!gen_registers(ctx)))
2262 goto fail;
2263
2264 if (unlikely(!gen_function(ctx)))
2265 goto fail;
2266
2267 if (unlikely(!gen_entries(ctx)))
2268 goto fail;
2269
2270 if (unlikely(!gen_epilogues(ctx)))
2271 goto fail;
2272
2273 if (unlikely(!(ctx->label_id + 1)))
2274 goto fail;
2275 if (unlikely(!(ctx->label_to_pos = mem_alloc_array_mayfail(mem_alloc_mayfail, size_t *, 0, 0, ctx->label_id + 1, sizeof(size_t), &ctx->err))))
2276 goto fail;
2277
2278 again:
2279 for (l = 0; l < ctx->label_id + 1; l++)
2280 ctx->label_to_pos[l] = (size_t)-1;
2281
2282 if (unlikely(!array_init_mayfail(uint8_t, &ctx->mcode, &ctx->mcode_size, &ctx->err)))
2283 goto fail;
2284
2285 if (unlikely(!array_init_mayfail(struct relocation, &ctx->reloc, &ctx->reloc_size, &ctx->err)))
2286 goto fail;
2287
2288 if (unlikely(!array_init_mayfail(struct trap_record, &ctx->trap_records, &ctx->trap_records_size, &ctx->err)))
2289 goto fail;
2290
2291 #ifdef ARCH_CONTEXT
2292 init_arch_context(ctx);
2293 #endif
2294
2295 if (unlikely(!gen_mcode(ctx)))
2296 goto fail;
2297
2298 rr = resolve_relocs(ctx);
2299 if (unlikely(rr == RELOCS_FAIL)) {
2300 /*debug("relocation fail: %s", da(ctx->fn,function)->function_name);*/
2301 goto fail;
2302 }
2303 if (rr == RELOCS_RETRY) {
2304 mem_free(ctx->mcode);
2305 ctx->mcode = NULL;
2306 mem_free(ctx->reloc);
2307 ctx->reloc = NULL;
2308 mem_free(ctx->trap_records);
2309 ctx->trap_records = NULL;
2310 goto again;
2311 }
2312
2313 resolve_traps(ctx);
2314
2315 #ifdef DEBUG_ENV
2316 if ((getenv("DUMP") && !strcmp(getenv("DUMP"), da(ctx->fn,function)->function_name)) || getenv("DUMP_ALL")) {
2317 char *hex;
2318 size_t hexl;
2319 size_t i;
2320 handle_t h;
2321
2322 mutex_lock(&dump_mutex);
2323 str_init(&hex, &hexl);
2324 str_add_string(&hex, &hexl, "_");
2325 str_add_unsigned(&hex, &hexl, dump_seq++, 10);
2326 str_add_string(&hex, &hexl, "_");
2327 str_add_string(&hex, &hexl, da(ctx->fn,function)->function_name);
2328 str_add_string(&hex, &hexl, ":");
2329 for (i = 0; i < hexl; i++)
2330 if (hex[i] == '/')
2331 hex[i] = '_';
2332 for (i = 0; i < ctx->mcode_size; i++) {
2333 uint8_t a = ctx->mcode[i];
2334 if (!(i & 0xff))
2335 str_add_string(&hex, &hexl, "\n .byte 0x");
2336 else
2337 str_add_string(&hex, &hexl, ",0x");
2338 if (a < 16)
2339 str_add_char(&hex, &hexl, '0');
2340 str_add_unsigned(&hex, &hexl, a, 16);
2341 }
2342 str_add_string(&hex, &hexl, "\n");
2343 h = os_open(os_cwd, "dump.s", O_WRONLY | O_APPEND, 0600, NULL);
2344 os_write_all(h, hex, hexl, NULL);
2345 os_close(h);
2346 mem_free(hex);
2347 mutex_unlock(&dump_mutex);
2348 }
2349 #endif
2350
2351 ctx->codegen = data_alloc_flexible(codegen, unoptimized_code, ctx->n_entries, &ctx->err);
2352 if (unlikely(!ctx->codegen))
2353 goto fail;
2354 da(ctx->codegen,codegen)->function = ctx->fn;
2355 da(ctx->codegen,codegen)->is_saved = false;
2356 da(ctx->codegen,codegen)->n_entries = 0;
2357 da(ctx->codegen,codegen)->offsets = NULL;
2358
2359 if (unlikely(!codegen_map(ctx)))
2360 goto fail;
2361
2362 codegen = ctx->codegen;
2363 ctx->codegen = NULL;
2364
2365 #ifdef HAVE_CODEGEN_TRAPS
2366 da(codegen,codegen)->trap_records = ctx->trap_records;
2367 da(codegen,codegen)->trap_records_size = ctx->trap_records_size;
2368 ctx->trap_records = NULL;
2369 data_trap_insert(codegen);
2370 #endif
2371
2372 have_codegen:
2373 done_ctx(ctx);
2374 return function_return(fp, pointer_data(codegen));
2375
2376 fail:
2377 /*debug("FAILED: %s", da(ctx->fn,function)->function_name);*/
2378 done_ctx(ctx);
2379 return function_return(fp, pointer_thunk(thunk_alloc_exception_error(error_ajla(EC_SYNC, AJLA_ERROR_NOT_SUPPORTED), NULL, NULL, NULL pass_file_line)));
2380 }
2381
2382 void codegen_free(struct data *codegen)
2383 {
2384 if (unlikely(da(codegen,codegen)->offsets != NULL))
2385 mem_free(da(codegen,codegen)->offsets);
2386 if (likely(da(codegen,codegen)->is_saved))
2387 return;
2388 #ifdef HAVE_CODEGEN_TRAPS
2389 mem_free(da(codegen,codegen)->trap_records);
2390 #endif
2391 os_code_unmap(da(codegen,codegen)->unoptimized_code_base, da(codegen,codegen)->unoptimized_code_size);
2392 }
2393
2394 #if defined(ARCH_IA64)
2395 static uintptr_t ia64_stub[2];
2396 #endif
2397 #if defined(ARCH_PARISC32) && defined(ARCH_PARISC_USE_STUBS)
2398 static uintptr_t parisc_stub[2];
2399 #endif
2400 #if defined(ARCH_PARISC64) && defined(ARCH_PARISC_USE_STUBS)
2401 static uintptr_t parisc_stub[4];
2402 #endif
2403 #if defined(ARCH_POWER) && defined(AIX_CALL)
2404 static uintptr_t ppc_stub[3];
2405 #endif
2406
2407 void name(codegen_init)(void)
2408 {
2409 struct codegen_context ctx_;
2410 struct codegen_context *ctx = &ctx_;
2411 void *ptr;
2412
2413 #if (defined(ARCH_X86_64) || defined(ARCH_X86_X32)) && !defined(ARCH_X86_WIN_ABI) && defined(HAVE_SYSCALL) && defined(HAVE_ASM_PRCTL_H) && defined(HAVE_SYS_SYSCALL_H)
2414 if (!dll) {
2415 int r;
2416 EINTR_LOOP(r, syscall(SYS_arch_prctl, ARCH_SET_GS, &cg_upcall_vector));
2417 if (!r)
2418 upcall_register = R_GS;
2419 }
2420 #endif
2421
2422 init_ctx(ctx);
2423 ctx->fn = NULL;
2424
2425 array_init(uint8_t, &ctx->code, &ctx->code_size);
2426
2427 if (unlikely(!gen_entry(ctx)))
2428 goto fail;
2429
2430 array_init(uint8_t, &ctx->mcode, &ctx->mcode_size);
2431
2432 #ifdef ARCH_CONTEXT
2433 init_arch_context(ctx);
2434 #endif
2435
2436 if (unlikely(!gen_mcode(ctx)))
2437 goto fail;
2438
2439 array_finish(uint8_t, &ctx->mcode, &ctx->mcode_size);
2440 ptr = os_code_map(ctx->mcode, ctx->mcode_size, NULL);
2441 codegen_ptr = ptr;
2442 codegen_size = ctx->mcode_size;
2443 ctx->mcode = NULL;
2444 #if defined(ARCH_IA64)
2445 ia64_stub[0] = ptr_to_num(ptr);
2446 ia64_stub[1] = 0;
2447 codegen_entry = cast_ptr(codegen_type, ia64_stub);
2448 #elif defined(ARCH_PARISC32) && defined(ARCH_PARISC_USE_STUBS)
2449 parisc_stub[0] = ptr_to_num(ptr);
2450 parisc_stub[1] = 0;
2451 codegen_entry = cast_ptr(codegen_type, cast_ptr(char *, parisc_stub) + 2);
2452 #elif defined(ARCH_PARISC64) && defined(ARCH_PARISC_USE_STUBS)
2453 parisc_stub[0] = 0;
2454 parisc_stub[1] = 0;
2455 parisc_stub[2] = ptr_to_num(ptr);
2456 parisc_stub[3] = 0;
2457 codegen_entry = cast_ptr(codegen_type, parisc_stub);
2458 #elif defined(ARCH_POWER) && defined(AIX_CALL)
2459 ppc_stub[0] = ptr_to_num(ptr);
2460 ppc_stub[1] = 0;
2461 ppc_stub[2] = 0;
2462 codegen_entry = cast_ptr(codegen_type, ppc_stub);
2463 #else
2464 codegen_entry = ptr;
2465 #endif
2466 done_ctx(ctx);
2467
2468 #ifdef DEBUG_ENV
2469 mutex_init(&dump_mutex);
2470 if (getenv("DUMP") || getenv("DUMP_ALL")) {
2471 size_t i;
2472 char *hex;
2473 size_t hexl;
2474 str_init(&hex, &hexl);
2475 #if defined(ARCH_RISCV64)
2476 str_add_string(&hex, &hexl, " .attribute arch, \"rv64i2p1_m2p0_a2p1_f2p2_d2p2_c2p0_zicsr2p0_zifencei2p0_zba1p0_zbb1p0_zbc1p0_zbs1p0\"\n");
2477 #endif
2478 for (i = 0; i < codegen_size; i++) {
2479 uint8_t a = cast_ptr(uint8_t *, codegen_ptr)[i];
2480 str_add_string(&hex, &hexl, " .byte 0x");
2481 if (a < 16)
2482 str_add_char(&hex, &hexl, '0');
2483 str_add_unsigned(&hex, &hexl, a, 16);
2484 str_add_char(&hex, &hexl, '\n');
2485 }
2486 os_write_atomic(".", "dump.s", hex, hexl, NULL);
2487 mem_free(hex);
2488 }
2489 #endif
2490
2491 return;
2492
2493 fail:
2494 fatal("couldn't compile global entry");
2495 }
2496
2497 void name(codegen_done)(void)
2498 {
2499 os_code_unmap(codegen_ptr, codegen_size);
2500 #ifdef DEBUG_ENV
2501 mutex_done(&dump_mutex);
2502 #endif
2503 }
2504
2505 #else
2506
2507 void name(codegen_init)(void)
2508 {
2509 }
2510
2511 void name(codegen_done)(void)
2512 {
2513 }
2514
2515 #endif
2516
2517 #endif
Ajla - a purely functional programming language