?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
x86: R_SCRATCH_4 may alias R_SAVED_2; make R_SI and R_DI allocatable
[ajla.git] / codegen.c
blob21d3065a365b1c6be82a964736391069681fb44f
1 /*
2 * Copyright (C) 2024 Mikulas Patocka
3 *
4 * This file is part of Ajla.
5 *
6 * Ajla is free software: you can redistribute it and/or modify it under the
7 * terms of the GNU General Public License as published by the Free Software
8 * Foundation, either version 3 of the License, or (at your option) any later
9 * version.
10 *
11 * Ajla is distributed in the hope that it will be useful, but WITHOUT ANY
12 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
13 * A PARTICULAR PURPOSE. See the GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License along with
16 * Ajla. If not, see <https://www.gnu.org/licenses/>.
17 */
18
19 #include "ajla.h"
20
21 #ifndef FILE_OMIT
22
23 #include "data.h"
24 #include "os.h"
25 #include "os_util.h"
26 #include "util.h"
27 #include "ipfn.h"
28 #include "ipret.h"
29 #include "funct.h"
30 #include "thread.h"
31 #include "task.h"
32 #include "save.h"
33
34 #include "codegen.h"
35
36 #ifdef HAVE_CODEGEN
37
38 #define flag_cache_chicken 0
39 #define must_be_flat_chicken 0
40 #define ra_chicken 0
41
42 #define INLINE_BITMAP_SLOTS 16
43 #define INLINE_COPY_SIZE 64
44
45 /*#define DEBUG_INSNS*/
46
47 code_return_t (*codegen_entry)(frame_s *, struct cg_upcall_vector_s *, tick_stamp_t, void *);
48 static void *codegen_ptr;
49 static size_t codegen_size;
50
51 /*
52 * insn:
53 * opcode - 16 bits
54 * op_size - 3 bits
55 * aux - 7 bits
56 * writes flags - 2 bit
57 * jmp size - 2 bits
58 */
59
60 #define INSN_OPCODE 0x0000ffffUL
61 #define INSN_OP_SIZE 0x00070000UL
62 #define INSN_AUX 0x03f80000UL
63 #define INSN_WRITES_FLAGS 0x0c000000UL
64 #define INSN_JUMP_SIZE 0x30000000UL
65
66 #define INSN_OPCODE_SHIFT 0
67 #define INSN_OP_SIZE_SHIFT 16
68 #define INSN_AUX_SHIFT 19
69 #define INSN_WRITES_FLAGS_SHIFT 26
70 #define INSN_JUMP_SIZE_SHIFT 28
71
72 #define insn_opcode(insn) (((insn) >> INSN_OPCODE_SHIFT) & (INSN_OPCODE >> INSN_OPCODE_SHIFT))
73 #define insn_op_size(insn) (((insn) >> INSN_OP_SIZE_SHIFT) & (INSN_OP_SIZE >> INSN_OP_SIZE_SHIFT))
74 #define insn_aux(insn) (((insn) >> INSN_AUX_SHIFT) & (INSN_AUX >> INSN_AUX_SHIFT))
75 #define insn_writes_flags(insn) (((insn) >> INSN_WRITES_FLAGS_SHIFT) & (INSN_WRITES_FLAGS >> INSN_WRITES_FLAGS_SHIFT))
76 #define insn_jump_size(insn) (((insn) >> INSN_JUMP_SIZE_SHIFT) & (INSN_JUMP_SIZE >> INSN_JUMP_SIZE_SHIFT))
77
78 #define ALU_ADD 0x00
79 #define ALU_OR 0x01
80 #define ALU_ADC 0x02
81 #define ALU_SBB 0x03
82 #define ALU_AND 0x04
83 #define ALU_SUB 0x05
84 #define ALU_XOR 0x06
85 #define ALU_ORN 0x08
86 #define ALU_ANDN 0x09
87 #define ALU_XORN 0x0a
88 #define ALU_MUL 0x10
89 #define ALU_UMULH 0x11
90 #define ALU_SMULH 0x12
91 #define ALU_UDIV 0x13
92 #define ALU_SDIV 0x14
93 #define ALU_UREM 0x15
94 #define ALU_SREM 0x16
95 #define ALU_SAVE 0x17
96 #define ALU_EXTBL 0x18
97 #define ALU_EXTWL 0x19
98 #define ALU_EXTLL 0x1a
99 #define ALU_EXTLH 0x1b
100 #define ALU_INSBL 0x1c
101 #define ALU_MSKBL 0x1d
102 #define ALU_ZAP 0x20
103 #define ALU_ZAPNOT 0x21
104
105 #define ALU1_NOT 0x00
106 #define ALU1_NEG 0x01
107 #define ALU1_NGC 0x02
108 #define ALU1_INC 0x03
109 #define ALU1_DEC 0x04
110 #define ALU1_BSWAP 0x05
111 #define ALU1_BSWAP16 0x06
112 #define ALU1_BREV 0x07
113 #define ALU1_BSF 0x08
114 #define ALU1_BSR 0x09
115 #define ALU1_LZCNT 0x0a
116 #define ALU1_POPCNT 0x0b
117
118 #define FP_ALU_ADD 0
119 #define FP_ALU_SUB 1
120 #define FP_ALU_MUL 2
121 #define FP_ALU_DIV 3
122 #define FP_ALU1_NEG 0
123 #define FP_ALU1_SQRT 1
124 #define FP_ALU1_ROUND 2
125 #define FP_ALU1_FLOOR 3
126 #define FP_ALU1_CEIL 4
127 #define FP_ALU1_TRUNC 5
128 #define FP_ALU1_VCNT8 6
129 #define FP_ALU1_VPADDL 7
130 #define FP_ALU1_ADDV 8
131
132 #define COND_O 0x0
133 #define COND_NO 0x1
134 #define COND_B 0x2
135 #define COND_AE 0x3
136 #define COND_E 0x4
137 #define COND_NE 0x5
138 #define COND_BE 0x6
139 #define COND_A 0x7
140 #define COND_S 0x8
141 #define COND_NS 0x9
142 #define COND_P 0xa
143 #define COND_NP 0xb
144 #define COND_L 0xc
145 #define COND_GE 0xd
146 #define COND_LE 0xe
147 #define COND_G 0xf
148 #define COND_BLBC 0x10
149 #define COND_BLBS 0x11
150 #define COND_ALWAYS 0x12
151
152 #define COND_FP 0x20
153 #define FP_COND_P (COND_FP | COND_P)
154 #define FP_COND_NP (COND_FP | COND_NP)
155 #define FP_COND_E (COND_FP | COND_E)
156 #define FP_COND_NE (COND_FP | COND_NE)
157 #define FP_COND_A (COND_FP | COND_A)
158 #define FP_COND_BE (COND_FP | COND_BE)
159 #define FP_COND_B (COND_FP | COND_B)
160 #define FP_COND_AE (COND_FP | COND_AE)
161
162 #define ROT_ROL 0x0
163 #define ROT_ROR 0x1
164 #define ROT_RCL 0x2
165 #define ROT_RCR 0x3
166 #define ROT_SHL 0x4
167 #define ROT_SHR 0x5
168 #define ROT_SAR 0x7
169 #define ROT_SAL 0x8
170
171 #define BTX_BT 0x0
172 #define BTX_BTS 0x1
173 #define BTX_BTR 0x2
174 #define BTX_BTC 0x3
175 #define BTX_BTEXT 0x4
176
177 #define OP_SIZE_1 0
178 #define OP_SIZE_2 1
179 #define OP_SIZE_4 2
180 #define OP_SIZE_8 3
181 #define OP_SIZE_16 4
182 #define OP_SIZE_10 7
183
184 #define MOV_MASK_0_16 0x0
185 #define MOV_MASK_16_32 0x1
186 #define MOV_MASK_32_48 0x2
187 #define MOV_MASK_48_64 0x3
188 #define MOV_MASK_0_8 0x4
189 #define MOV_MASK_32_64 0x5
190 #define MOV_MASK_52_64 0x6
191
192 #define JMP_SHORTEST 0x0000
193 #define JMP_SHORT 0x0001
194 #define JMP_LONG 0x0002
195 #define JMP_EXTRA_LONG 0x0003
196
197 enum {
198 INSN_ENTRY,
199 INSN_LABEL,
200 INSN_RET,
201 INSN_RET_IMM,
202 INSN_ARM_PUSH,
203 INSN_ARM_POP,
204 INSN_S390_PUSH,
205 INSN_S390_POP,
206 INSN_IA64_ALLOC,
207 INSN_IA64_DEALLOC,
208 INSN_PUSH,
209 INSN_POP,
210 INSN_CALL,
211 INSN_CALL_INDIRECT,
212 INSN_MOV,
213 INSN_MOVSX,
214 INSN_MOV_U,
215 INSN_MOV_LR,
216 INSN_CMP,
217 INSN_CMP_DEST_REG,
218 INSN_CMN,
219 INSN_TEST,
220 INSN_TEST_DEST_REG,
221 INSN_ALU,
222 INSN_ALU_PARTIAL,
223 INSN_ALU_FLAGS,
224 INSN_ALU_FLAGS_PARTIAL,
225 INSN_ALU_TRAP,
226 INSN_ALU_FLAGS_TRAP,
227 INSN_ALU1,
228 INSN_ALU1_PARTIAL,
229 INSN_ALU1_FLAGS,
230 INSN_ALU1_FLAGS_PARTIAL,
231 INSN_ALU1_TRAP,
232 INSN_LEA3,
233 INSN_ROT,
234 INSN_ROT_PARTIAL,
235 INSN_BT,
236 INSN_BTX,
237 INSN_MUL_L,
238 INSN_DIV_L,
239 INSN_MADD,
240 INSN_CBW,
241 INSN_CBW_PARTIAL,
242 INSN_CWD,
243 INSN_CWD_PARTIAL,
244 INSN_SET_COND,
245 INSN_SET_COND_PARTIAL,
246 INSN_CMOV,
247 INSN_CMOV_XCC,
248 INSN_MOVR,
249 INSN_CSEL_SEL,
250 INSN_CSEL_INC,
251 INSN_CSEL_INV,
252 INSN_CSEL_NEG,
253 INSN_STP,
254 INSN_LDP,
255 INSN_MOV_MASK,
256 INSN_MEMCPY,
257 INSN_MEMSET,
258 INSN_FP_CMP,
259 INSN_FP_CMP_DEST_REG,
260 INSN_FP_CMP_DEST_REG_TRAP,
261 INSN_FP_CMP_UNORDERED_DEST_REG,
262 INSN_FP_CMP_COND,
263 INSN_FP_TEST_REG,
264 INSN_FP_TO_INT_FLAGS,
265 INSN_FP_ALU,
266 INSN_FP_ALU1,
267 INSN_FP_TO_INT32,
268 INSN_FP_TO_INT64,
269 INSN_FP_TO_INT64_TRAP,
270 INSN_FP_FROM_INT32,
271 INSN_FP_FROM_INT64,
272 INSN_FP_INT64_TO_INT32_TRAP,
273 INSN_FP_CVT,
274 INSN_X87_FLD,
275 INSN_X87_FILD,
276 INSN_X87_FSTP,
277 INSN_X87_FISTP,
278 INSN_X87_FISTTP,
279 INSN_X87_FCOMP,
280 INSN_X87_FCOMPP,
281 INSN_X87_FCOMIP,
282 INSN_X87_ALU,
283 INSN_X87_ALUP,
284 INSN_X87_FCHS,
285 INSN_X87_FSQRT,
286 INSN_X87_FRNDINT,
287 INSN_X87_FNSTSW,
288 INSN_X87_FLDCW,
289 INSN_JMP,
290 INSN_JMP_COND,
291 INSN_JMP_COND_LOGICAL,
292 INSN_JMP_REG,
293 INSN_JMP_REG_BIT,
294 INSN_JMP_2REGS,
295 INSN_JMP_FP_TEST,
296 INSN_JMP_INDIRECT,
297 INSN_MB,
298 INSN_CALL_MILLICODE,
299 };
300
301 #define ARG_REGS_MAX 0xc0
302 #define ARG_SHIFTED_REGISTER 0xc0
303 #define ARG_SHIFT_AMOUNT 0x3f
304 #define ARG_SHIFT_MODE 0xc0
305 #define ARG_SHIFT_LSL 0x00
306 #define ARG_SHIFT_LSR 0x40
307 #define ARG_SHIFT_ASR 0x80
308 #define ARG_SHIFT_ROR 0xc0
309 #define ARG_EXTENDED_REGISTER 0xc1
310 #define ARG_EXTEND_SHIFT 0x07
311 #define ARG_EXTEND_MODE 0x38
312 #define ARG_EXTEND_UXTB 0x00
313 #define ARG_EXTEND_UXTH 0x08
314 #define ARG_EXTEND_UXTW 0x10
315 #define ARG_EXTEND_UXTX 0x18
316 #define ARG_EXTEND_SXTB 0x20
317 #define ARG_EXTEND_SXTH 0x28
318 #define ARG_EXTEND_SXTW 0x30
319 #define ARG_EXTEND_SXTX 0x38
320 #define ARG_ADDRESS_0 0xd0
321 #define ARG_ADDRESS_1 0xd1
322 #define ARG_ADDRESS_1_2 0xd2
323 #define ARG_ADDRESS_1_4 0xd3
324 #define ARG_ADDRESS_1_8 0xd4
325 #define ARG_ADDRESS_1_PRE_I 0xd5
326 #define ARG_ADDRESS_1_POST_I 0xd6
327 #define ARG_ADDRESS_2 0xd7
328 #define ARG_ADDRESS_2_2 0xd8
329 #define ARG_ADDRESS_2_4 0xd9
330 #define ARG_ADDRESS_2_8 0xda
331 #define ARG_ADDRESS_2_UXTW 0xdb
332 #define ARG_ADDRESS_2_SXTW 0xdc
333 #define ARG_IMM 0xe0
334
335 #define ARG_IS_ADDRESS(a) ((a) >= ARG_ADDRESS_0 && (a) <= ARG_ADDRESS_2_SXTW)
336
337 #ifdef POINTER_COMPRESSION
338 #define OP_SIZE_SLOT OP_SIZE_4
339 #else
340 #define OP_SIZE_SLOT OP_SIZE_ADDRESS
341 #endif
342
343 #define OP_SIZE_BITMAP (bitmap_64bit ? OP_SIZE_8 : OP_SIZE_4)
344
345 #define OP_SIZE_INT log_2(sizeof(int_default_t))
346
347 #define check_insn(insn) \
348 do { \
349 /*if ((insn_opcode(insn) == INSN_ALU || insn_opcode(insn) == INSN_ALU1) && insn_op_size(insn) != OP_SIZE_NATIVE) internal(file_line, "invalid insn %08x", (unsigned)(insn));*/\
350 /*if (insn == 0x001a000e) internal(file_line, "invalid insn %08x", insn);*/\
351 } while (0)
352
353 #ifdef DEBUG_INSNS
354 #define gen_line() gen_four(__LINE__)
355 #else
356 #define gen_line() do { } while (0)
357 #endif
358
359 #ifdef ARCH_IA64
360 #define ARCH_CONTEXT struct { \
361 uint64_t insns[3]; \
362 uint8_t insn_units[3]; \
363 bool insn_stops[3]; \
364 uint64_t wr_mask[4]; \
365 }
366 #endif
367
368 #define gen_insn(opcode, op_size, aux, writes_flags) \
369 do { \
370 uint32_t dword = \
371 (uint32_t)(opcode) << INSN_OPCODE_SHIFT | \
372 (uint32_t)(op_size) << INSN_OP_SIZE_SHIFT | \
373 (uint32_t)(aux) << INSN_AUX_SHIFT | \
374 (uint32_t)(writes_flags) << INSN_WRITES_FLAGS_SHIFT; \
375 check_insn(dword); \
376 gen_line(); \
377 gen_four(dword); \
378 } while (0)
379
380 static size_t arg_size(uint8_t arg)
381 {
382 if (arg < ARG_REGS_MAX)
383 return 1;
384 if (arg >= ARG_SHIFTED_REGISTER && arg <= ARG_EXTENDED_REGISTER)
385 return 3;
386 if (arg == ARG_ADDRESS_0)
387 return 9;
388 if (arg >= ARG_ADDRESS_1 && arg <= ARG_ADDRESS_1_POST_I)
389 return 10;
390 if (arg >= ARG_ADDRESS_2 && arg <= ARG_ADDRESS_2_SXTW)
391 return 11;
392 if (arg == ARG_IMM)
393 return 9;
394 internal(file_line, "arg_size: invalid argument %02x", arg);
395 return 0;
396 }
397
398 struct relocation {
399 uint32_t label_id;
400 uint8_t length;
401 size_t position;
402 size_t jmp_instr;
403 };
404
405 struct code_arg {
406 frame_t slot;
407 frame_t flags;
408 frame_t type;
409 };
410
411 struct cg_entry {
412 size_t entry_to_pos;
413 frame_t *variables;
414 size_t n_variables;
415 uint32_t entry_label;
416 uint32_t nonflat_label;
417 };
418
419 struct codegen_context {
420 struct data *fn;
421 struct data **local_directory;
422
423 const code_t *instr_start;
424 const code_t *current_position;
425 uchar_efficient_t arg_mode;
426
427 uint32_t label_id;
428 struct cg_entry *entries;
429 frame_t n_entries;
430
431 uint8_t *code;
432 size_t code_size;
433
434 uint8_t *code_position;
435
436 uint32_t *code_labels;
437 uint32_t *escape_labels;
438 uint32_t escape_nospill_label;
439 uint32_t call_label;
440 uint32_t reload_label;
441
442 uint8_t *mcode;
443 size_t mcode_size;
444
445 size_t *label_to_pos;
446 struct relocation *reloc;
447 size_t reloc_size;
448
449 struct trap_record *trap_records;
450 size_t trap_records_size;
451
452 struct code_arg *args;
453 size_t args_l;
454 const code_t *return_values;
455
456 int8_t *flag_cache;
457 short *registers;
458 frame_t *need_spill;
459 size_t need_spill_l;
460
461 unsigned base_reg;
462 bool offset_reg;
463 int64_t offset_imm;
464
465 bool const_reg;
466 int64_t const_imm;
467
468 struct data *codegen;
469
470 int upcall_args;
471 frame_t *var_aux;
472
473 ajla_error_t err;
474
475 #ifdef ARCH_CONTEXT
476 ARCH_CONTEXT a;
477 #endif
478 };
479
480 static void init_ctx(struct codegen_context *ctx)
481 {
482 ctx->local_directory = NULL;
483 ctx->label_id = 0;
484 ctx->entries = NULL;
485 ctx->n_entries = 0;
486 ctx->code = NULL;
487 ctx->code_labels = NULL;
488 ctx->escape_labels = NULL;
489 ctx->escape_nospill_label = 0;
490 ctx->call_label = 0;
491 ctx->reload_label = 0;
492 ctx->mcode = NULL;
493 ctx->label_to_pos = NULL;
494 ctx->reloc = NULL;
495 ctx->trap_records = NULL;
496 ctx->args = NULL;
497 ctx->flag_cache = NULL;
498 ctx->registers = NULL;
499 ctx->need_spill = NULL;
500 ctx->codegen = NULL;
501 ctx->upcall_args = -1;
502 ctx->var_aux = NULL;
503 }
504
505 static void done_ctx(struct codegen_context *ctx)
506 {
507 if (ctx->local_directory)
508 mem_free(ctx->local_directory);
509 if (ctx->entries) {
510 size_t i;
511 for (i = 0; i < ctx->n_entries; i++) {
512 struct cg_entry *ce = &ctx->entries[i];
513 if (ce->variables)
514 mem_free(ce->variables);
515 }
516 mem_free(ctx->entries);
517 }
518 if (ctx->code)
519 mem_free(ctx->code);
520 if (ctx->code_labels)
521 mem_free(ctx->code_labels);
522 if (ctx->escape_labels)
523 mem_free(ctx->escape_labels);
524 if (ctx->mcode)
525 mem_free(ctx->mcode);
526 if (ctx->label_to_pos)
527 mem_free(ctx->label_to_pos);
528 if (ctx->reloc)
529 mem_free(ctx->reloc);
530 if (ctx->trap_records)
531 mem_free(ctx->trap_records);
532 if (ctx->args)
533 mem_free(ctx->args);
534 if (ctx->flag_cache)
535 mem_free(ctx->flag_cache);
536 if (ctx->registers)
537 mem_free(ctx->registers);
538 if (ctx->need_spill)
539 mem_free(ctx->need_spill);
540 if (ctx->codegen)
541 data_free(ctx->codegen);
542 if (ctx->var_aux)
543 mem_free(ctx->var_aux);
544 }
545
546
547 static inline code_t get_code(struct codegen_context *ctx)
548 {
549 ajla_assert(ctx->current_position < da(ctx->fn,function)->code + da(ctx->fn,function)->code_size, (file_line, "get_code: ran out of code"));
550 return *ctx->current_position++;
551 }
552
553 static inline uint32_t get_uint32(struct codegen_context *ctx)
554 {
555 uint32_t a1 = get_code(ctx);
556 uint32_t a2 = get_code(ctx);
557 #if !CODE_ENDIAN
558 return a1 + (a2 << 16);
559 #else
560 return a2 + (a1 << 16);
561 #endif
562 }
563
564 static int32_t get_jump_offset(struct codegen_context *ctx)
565 {
566 if (SIZEOF_IP_T == 2) {
567 return (int32_t)(int16_t)get_code(ctx);
568 } else if (SIZEOF_IP_T == 4) {
569 return (int32_t)get_uint32(ctx);
570 } else {
571 not_reached();
572 return -1;
573 }
574 }
575
576 static inline void get_one(struct codegen_context *ctx, frame_t *v)
577 {
578 if (!ctx->arg_mode) {
579 code_t c = get_code(ctx);
580 ajla_assert(!(c & ~0xff), (file_line, "get_one: high byte is not cleared: %u", (unsigned)c));
581 *v = c & 0xff;
582 } else if (ctx->arg_mode == 1) {
583 *v = get_code(ctx);
584 #if ARG_MODE_N >= 2
585 } else if (ctx->arg_mode == 2) {
586 *v = get_uint32(ctx);
587 #endif
588 } else {
589 internal(file_line, "get_one: invalid arg mode %u", ctx->arg_mode);
590 }
591 }
592
593 static inline void get_two(struct codegen_context *ctx, frame_t *v1, frame_t *v2)
594 {
595 if (!ctx->arg_mode) {
596 code_t c = get_code(ctx);
597 *v1 = c & 0xff;
598 *v2 = c >> 8;
599 } else if (ctx->arg_mode == 1) {
600 *v1 = get_code(ctx);
601 *v2 = get_code(ctx);
602 #if ARG_MODE_N >= 2
603 } else if (ctx->arg_mode == 2) {
604 *v1 = get_uint32(ctx);
605 *v2 = get_uint32(ctx);
606 #endif
607 } else {
608 internal(file_line, "get_two: invalid arg mode %u", ctx->arg_mode);
609 }
610 }
611
612
613 static uint32_t alloc_label(struct codegen_context *ctx)
614 {
615 return ++ctx->label_id;
616 }
617
618 static uint32_t alloc_escape_label_for_ip(struct codegen_context *ctx, const code_t *code)
619 {
620 ip_t ip = code - da(ctx->fn,function)->code;
621 if (!ctx->escape_labels[ip])
622 ctx->escape_labels[ip] = alloc_label(ctx);
623 return ctx->escape_labels[ip];
624 }
625
626 static uint32_t alloc_escape_label(struct codegen_context *ctx)
627 {
628 return alloc_escape_label_for_ip(ctx, ctx->instr_start);
629 }
630
631 static uint32_t attr_unused alloc_call_label(struct codegen_context *ctx)
632 {
633 if (!ctx->call_label) {
634 ctx->call_label = alloc_label(ctx);
635 }
636 return ctx->call_label;
637 }
638
639 static uint32_t alloc_reload_label(struct codegen_context *ctx)
640 {
641 if (!ctx->reload_label) {
642 ctx->reload_label = alloc_label(ctx);
643 }
644 return ctx->reload_label;
645 }
646
647 #define g(call) \
648 do { \
649 if (unlikely(!call)) \
650 return false; \
651 } while (0)
652
653 #define gen_one(byte) \
654 do { \
655 /*debug("gen %d: %02x", __LINE__, (uint8_t)(byte))*/; \
656 if (unlikely(!array_add_mayfail(uint8_t, &ctx->code, &ctx->code_size, byte, NULL, &ctx->err)))\
657 return false; \
658 } while (0)
659
660 #if defined(C_LITTLE_ENDIAN)
661 #define gen_two(word) \
662 do { \
663 uint16_t word_ = (word); \
664 /*debug("gen %d: %04x", __LINE__, (uint16_t)(word_));*/ \
665 if (unlikely(!array_add_multiple_mayfail(uint8_t, &ctx->code, &ctx->code_size, cast_ptr(uint8_t *, &word_), 2, NULL, &ctx->err)))\
666 return false; \
667 } while (0)
668 #define gen_four(dword) \
669 do { \
670 uint32_t dword_ = (dword); \
671 /*debug("gen %d: %08x", __LINE__, (uint32_t)(dword_));*/ \
672 if (unlikely(!array_add_multiple_mayfail(uint8_t, &ctx->code, &ctx->code_size, cast_ptr(uint8_t *, &dword_), 4, NULL, &ctx->err)))\
673 return false; \
674 } while (0)
675 #define gen_eight(qword) \
676 do { \
677 uint64_t qword_ = (qword); \
678 /*debug("gen %d: %016lx", __LINE__, (uint64_t)(qword_));*/ \
679 if (unlikely(!array_add_multiple_mayfail(uint8_t, &ctx->code, &ctx->code_size, cast_ptr(uint8_t *, &qword_), 8, NULL, &ctx->err)))\
680 return false; \
681 } while (0)
682 #else
683 #define gen_two(word) \
684 do { \
685 uint16_t word_ = (word); \
686 gen_one(word_ & 0xffU); \
687 gen_one(word_ >> 8); \
688 } while (0)
689 #define gen_four(dword) \
690 do { \
691 uint32_t dword_ = (dword); \
692 gen_two(dword_ & 0xffffU); \
693 gen_two(dword_ >> 15 >> 1); \
694 } while (0)
695 #define gen_eight(qword) \
696 do { \
697 uint64_t qword_ = (qword); \
698 gen_four(qword_ & 0xffffffffUL); \
699 gen_four(qword_ >> 15 >> 15 >> 2); \
700 } while (0)
701 #endif
702
703 #define gen_label(label_id) \
704 do { \
705 gen_insn(INSN_LABEL, 0, 0, 0); \
706 gen_four(label_id); \
707 } while (0)
708
709
710 static uint8_t attr_unused cget_one(struct codegen_context *ctx)
711 {
712 ajla_assert(ctx->code_position < ctx->code + ctx->code_size, (file_line, "cget_one: ran out of code"));
713 return *ctx->code_position++;
714 }
715
716 static uint16_t attr_unused cget_two(struct codegen_context *ctx)
717 {
718 #if defined(C_LITTLE_ENDIAN)
719 uint16_t r;
720 ajla_assert(ctx->code_position < ctx->code + ctx->code_size, (file_line, "cget_two: ran out of code"));
721 memcpy(&r, ctx->code_position, 2);
722 ctx->code_position += 2;
723 return r;
724 #else
725 uint16_t r = cget_one(ctx);
726 r |= cget_one(ctx) << 8;
727 return r;
728 #endif
729 }
730
731 static uint32_t cget_four(struct codegen_context *ctx)
732 {
733 #if defined(C_LITTLE_ENDIAN)
734 uint32_t r;
735 ajla_assert(ctx->code_position < ctx->code + ctx->code_size, (file_line, "cget_four: ran out of code"));
736 memcpy(&r, ctx->code_position, 4);
737 ctx->code_position += 4;
738 return r;
739 #else
740 uint32_t r = cget_two(ctx);
741 r |= (uint32_t)cget_two(ctx) << 16;
742 return r;
743 #endif
744 }
745
746 static uint64_t attr_unused cget_eight(struct codegen_context *ctx)
747 {
748 #if defined(C_LITTLE_ENDIAN)
749 uint64_t r;
750 ajla_assert(ctx->code_position < ctx->code + ctx->code_size, (file_line, "cget_eight: ran out of code"));
751 memcpy(&r, ctx->code_position, 8);
752 ctx->code_position += 8;
753 return r;
754 #else
755 uint64_t r = cget_four(ctx);
756 r |= (uint64_t)cget_four(ctx) << 32;
757 return r;
758 #endif
759 }
760
761 static int64_t get_imm(uint8_t *ptr)
762 {
763 #if defined(C_LITTLE_ENDIAN)
764 int64_t r;
765 memcpy(&r, ptr, 8);
766 return r;
767 #else
768 int64_t r;
769 r = (uint64_t)ptr[0] |
770 ((uint64_t)ptr[1] << 8) |
771 ((uint64_t)ptr[2] << 16) |
772 ((uint64_t)ptr[3] << 24) |
773 ((uint64_t)ptr[4] << 32) |
774 ((uint64_t)ptr[5] << 40) |
775 ((uint64_t)ptr[6] << 48) |
776 ((uint64_t)ptr[7] << 56);
777 return r;
778 #endif
779 }
780
781 #define cgen_one(byte) \
782 do { \
783 if (unlikely(!array_add_mayfail(uint8_t, &ctx->mcode, &ctx->mcode_size, byte, NULL, &ctx->err)))\
784 return false; \
785 } while (0)
786
787 #if defined(C_LITTLE_ENDIAN) || 1
788 #define cgen_two(word) \
789 do { \
790 uint16_t word_ = (word); \
791 if (unlikely(!array_add_multiple_mayfail(uint8_t, &ctx->mcode, &ctx->mcode_size, cast_ptr(uint8_t *, &word_), 2, NULL, &ctx->err)))\
792 return false; \
793 } while (0)
794 #define cgen_four(dword) \
795 do { \
796 uint32_t dword_ = (dword); \
797 /*if (dword_ == 0x1ee02000) internal(file_line, "invalid instruction");*/\
798 if (unlikely(!array_add_multiple_mayfail(uint8_t, &ctx->mcode, &ctx->mcode_size, cast_ptr(uint8_t *, &dword_), 4, NULL, &ctx->err)))\
799 return false; \
800 } while (0)
801 #define cgen_eight(qword) \
802 do { \
803 uint64_t qword_ = (qword); \
804 if (unlikely(!array_add_multiple_mayfail(uint8_t, &ctx->mcode, &ctx->mcode_size, cast_ptr(uint8_t *, &qword_), 8, NULL, &ctx->err)))\
805 return false; \
806 } while (0)
807 #else
808 #define cgen_two(word) \
809 do { \
810 cgen_one((word) & 0xff); \
811 cgen_one((word) >> 8); \
812 } while (0)
813 #define cgen_four(dword) \
814 do { \
815 cgen_two((dword) & 0xffff); \
816 cgen_two((dword) >> 15 >> 1); \
817 } while (0)
818 #define cgen_eight(qword) \
819 do { \
820 cgen_four((qword) & 0xffffffff); \
821 cgen_four((qword) >> 15 >> 15 >> 2); \
822 } while (0)
823 #endif
824
825
826 #define IMM_PURPOSE_LDR_OFFSET 1
827 #define IMM_PURPOSE_LDR_SX_OFFSET 2
828 #define IMM_PURPOSE_STR_OFFSET 3
829 #define IMM_PURPOSE_LDP_STP_OFFSET 4
830 #define IMM_PURPOSE_VLDR_VSTR_OFFSET 5
831 #define IMM_PURPOSE_MVI_CLI_OFFSET 6
832 #define IMM_PURPOSE_STORE_VALUE 7
833 #define IMM_PURPOSE_ADD 8
834 #define IMM_PURPOSE_SUB 9
835 #define IMM_PURPOSE_CMP 10
836 #define IMM_PURPOSE_CMP_LOGICAL 11
837 #define IMM_PURPOSE_AND 12
838 #define IMM_PURPOSE_OR 13
839 #define IMM_PURPOSE_XOR 14
840 #define IMM_PURPOSE_ANDN 15
841 #define IMM_PURPOSE_TEST 16
842 #define IMM_PURPOSE_JMP_2REGS 17
843 #define IMM_PURPOSE_MUL 18
844 #define IMM_PURPOSE_CMOV 19
845 #define IMM_PURPOSE_MOVR 20
846 #define IMM_PURPOSE_BITWISE 21
847
848
849 static bool attr_w gen_extend(struct codegen_context *ctx, unsigned op_size, bool sx, unsigned dest, unsigned src);
850 static bool attr_w gen_upcall_end(struct codegen_context *ctx, unsigned args);
851
852 #define gen_address_offset() \
853 do { \
854 if (likely(!ctx->offset_reg)) { \
855 gen_one(ARG_ADDRESS_1); \
856 gen_one(ctx->base_reg); \
857 gen_eight(ctx->offset_imm); \
858 } else { \
859 gen_one(ARG_ADDRESS_2); \
860 gen_one(ctx->base_reg); \
861 gen_one(R_OFFSET_IMM); \
862 gen_eight(0); \
863 } \
864 } while (0)
865
866 #define gen_imm_offset() \
867 do { \
868 if (likely(!ctx->const_reg)) { \
869 gen_one(ARG_IMM); \
870 gen_eight(ctx->const_imm); \
871 } else { \
872 gen_one(R_CONST_IMM); \
873 } \
874 } while (0)
875
876 #define is_imm() (!ctx->const_reg)
877
878
879 #if defined(ARCH_ALPHA)
880 #include "c1-alpha.inc"
881 #elif defined(ARCH_ARM32)
882 #include "c1-arm.inc"
883 #elif defined(ARCH_ARM64)
884 #include "c1-arm64.inc"
885 #elif defined(ARCH_IA64)
886 #include "c1-ia64.inc"
887 #elif defined(ARCH_LOONGARCH64)
888 #include "c1-loong.inc"
889 #elif defined(ARCH_MIPS)
890 #include "c1-mips.inc"
891 #elif defined(ARCH_PARISC)
892 #include "c1-hppa.inc"
893 #elif defined(ARCH_POWER)
894 #include "c1-power.inc"
895 #elif defined(ARCH_S390)
896 #include "c1-s390.inc"
897 #elif defined(ARCH_SPARC)
898 #include "c1-sparc.inc"
899 #elif defined(ARCH_RISCV64)
900 #include "c1-riscv.inc"
901 #elif defined(ARCH_X86)
902 #include "c1-x86.inc"
903 #endif
904
905
906 #ifndef ARCH_SUPPORTS_TRAPS
907 #define ARCH_SUPPORTS_TRAPS 0
908 #endif
909
910
911 #if !defined(POINTER_COMPRESSION)
912 #define gen_pointer_compression(base) do { } while (0)
913 #define gen_address_offset_compressed() gen_address_offset()
914 #elif defined(ARCH_X86)
915 #define gen_pointer_compression(base) do { } while (0)
916 #define gen_address_offset_compressed() \
917 do { \
918 if (likely(!ctx->offset_reg)) { \
919 gen_one(ARG_ADDRESS_1 + POINTER_COMPRESSION); \
920 gen_one(ctx->base_reg); \
921 gen_eight(ctx->offset_imm); \
922 } else { \
923 gen_one(ARG_ADDRESS_2 + POINTER_COMPRESSION); \
924 gen_one(R_OFFSET_IMM); \
925 gen_one(ctx->base_reg); \
926 gen_eight(0); \
927 } \
928 } while (0)
929 #else
930 #define gen_pointer_compression(base) \
931 do { \
932 if (ARCH_PREFERS_SX(OP_SIZE_4)) { \
933 g(gen_extend(ctx, OP_SIZE_4, false, base, base));\
934 \
935 gen_insn(INSN_ROT + ARCH_PARTIAL_ALU(OP_SIZE_ADDRESS), OP_SIZE_ADDRESS, ROT_SHL, ROT_WRITES_FLAGS(ROT_SHL));\
936 gen_one(base); \
937 gen_one(base); \
938 gen_one(ARG_IMM); \
939 gen_eight(POINTER_COMPRESSION); \
940 } else { \
941 gen_insn(INSN_ROT + ARCH_PARTIAL_ALU(OP_SIZE_ADDRESS), OP_SIZE_ADDRESS, ROT_SHL, ROT_WRITES_FLAGS(ROT_SHL));\
942 gen_one(base); \
943 gen_one(base); \
944 gen_one(ARG_IMM); \
945 gen_eight(POINTER_COMPRESSION); \
946 } \
947 } while (0)
948 #define gen_address_offset_compressed() gen_address_offset()
949 #endif
950
951
952 #if defined(C_LITTLE_ENDIAN)
953 #define lo_word(size) (0)
954 #define hi_word(size) ((size_t)1 << (size))
955 #elif defined(C_BIG_ENDIAN)
956 #define lo_word(size) ((size_t)1 << (size))
957 #define hi_word(size) (0)
958 #else
959 endian not defined
960 #endif
961
962
963 static const struct type *get_type_of_local(struct codegen_context *ctx, frame_t pos)
964 {
965 const struct type *t;
966 const struct data *function = ctx->fn;
967 t = da(function,function)->local_variables[pos].type;
968 if (t)
969 TYPE_TAG_VALIDATE(t->tag);
970 return t;
971 }
972
973 static bool attr_w gen_sanitize_returned_pointer(struct codegen_context attr_unused *ctx, unsigned attr_unused reg)
974 {
975 #if defined(ARCH_X86_X32)
976 gen_insn(INSN_MOV, OP_SIZE_ADDRESS, 0, 0);
977 gen_one(reg);
978 gen_one(reg);
979 #endif
980 return true;
981 }
982
983 static bool attr_w gen_3address_alu(struct codegen_context *ctx, unsigned size, unsigned alu, unsigned dest, unsigned src1, unsigned src2)
984 {
985 if (unlikely(dest == src2) && (alu == ALU_ADD || alu == ALU_OR || alu == ALU_AND || alu == ALU_XOR || alu == ALU_MUL || alu == ALU_UMULH || alu == ALU_SMULH)) {
986 unsigned swap = src1;
987 src1 = src2;
988 src2 = swap;
989 }
990 if (unlikely(dest == src2)) {
991 internal(file_line, "gen_3address_alu: invalid registers: %u, %u, %x, %x, %x", size, alu, dest, src1, src2);
992 }
993 if (!ARCH_IS_3ADDRESS && dest != src1
994 #if defined(ARCH_X86)
995 && alu != ALU_ADD
996 #endif
997 ) {
998 gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
999 gen_one(dest);
1000 gen_one(src1);
1001
1002 gen_insn(INSN_ALU + ARCH_PARTIAL_ALU(size), size, alu, ALU_WRITES_FLAGS(alu, false));
1003 gen_one(dest);
1004 gen_one(dest);
1005 gen_one(src2);
1006
1007 return true;
1008 }
1009 gen_insn(INSN_ALU + ARCH_PARTIAL_ALU(size), size, alu, ALU_WRITES_FLAGS(alu, false));
1010 gen_one(dest);
1011 gen_one(src1);
1012 gen_one(src2);
1013 return true;
1014 }
1015
1016 static bool attr_w gen_3address_alu_imm(struct codegen_context *ctx, unsigned size, unsigned alu, unsigned dest, unsigned src, int64_t imm)
1017 {
1018 unsigned purpose =
1019 alu == ALU_ADD ? IMM_PURPOSE_ADD :
1020 alu == ALU_SUB ? IMM_PURPOSE_SUB :
1021 alu == ALU_MUL ? IMM_PURPOSE_MUL :
1022 alu == ALU_UMULH ? IMM_PURPOSE_MUL :
1023 alu == ALU_SMULH ? IMM_PURPOSE_MUL :
1024 alu == ALU_ANDN ? IMM_PURPOSE_ANDN :
1025 alu == ALU_AND ? IMM_PURPOSE_AND :
1026 alu == ALU_OR ? IMM_PURPOSE_OR :
1027 alu == ALU_XOR ? IMM_PURPOSE_XOR :
1028 alu == ALU_EXTBL ? IMM_PURPOSE_OR :
1029 alu == ALU_EXTWL ? IMM_PURPOSE_OR :
1030 alu == ALU_EXTLL ? IMM_PURPOSE_OR :
1031 alu == ALU_EXTLH ? IMM_PURPOSE_OR :
1032 alu == ALU_INSBL ? IMM_PURPOSE_OR :
1033 alu == ALU_MSKBL ? IMM_PURPOSE_OR :
1034 alu == ALU_ZAP ? IMM_PURPOSE_ANDN :
1035 alu == ALU_ZAPNOT ? IMM_PURPOSE_AND :
1036 -1U;
1037 if (unlikely(purpose == -1U))
1038 internal(file_line, "gen_3address_alu_imm: invalid parameters: size %u, alu %u, dest %u, src %u, imm %"PRIxMAX"", size, alu, dest, src, (uintmax_t)imm);
1039 if (
1040 dest != src
1041 #if !defined(ARCH_S390)
1042 && !ARCH_IS_3ADDRESS
1043 #endif
1044 #if defined(ARCH_X86)
1045 && alu != ALU_ADD
1046 #endif
1047 ) {
1048 gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
1049 gen_one(dest);
1050 gen_one(src);
1051
1052 g(gen_imm(ctx, imm, purpose, i_size(OP_SIZE_ADDRESS)));
1053 gen_insn(INSN_ALU + ARCH_PARTIAL_ALU(size), size, alu, ALU_WRITES_FLAGS(alu, is_imm()));
1054 gen_one(dest);
1055 gen_one(dest);
1056 gen_imm_offset();
1057
1058 return true;
1059 }
1060 g(gen_imm(ctx, imm, purpose, i_size(OP_SIZE_ADDRESS)));
1061 gen_insn(INSN_ALU + ARCH_PARTIAL_ALU(size), size, alu, ALU_WRITES_FLAGS(alu, is_imm()));
1062 gen_one(dest);
1063 gen_one(src);
1064 gen_imm_offset();
1065
1066 return true;
1067 }
1068
1069 static bool attr_w attr_unused gen_3address_rot(struct codegen_context *ctx, unsigned size, unsigned alu, unsigned dest, unsigned src1, unsigned src2)
1070 {
1071 if (unlikely(dest == src2))
1072 internal(file_line, "gen_3address_rot: invalid registers: %u, %u, %x, %x, %x", size, alu, dest, src1, src2);
1073 if (!ARCH_IS_3ADDRESS && dest != src1) {
1074 gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
1075 gen_one(dest);
1076 gen_one(src1);
1077
1078 gen_insn(INSN_ROT + ARCH_PARTIAL_ALU(size), size, alu, ROT_WRITES_FLAGS(alu));
1079 gen_one(dest);
1080 gen_one(dest);
1081 gen_one(src2);
1082
1083 return true;
1084 }
1085 gen_insn(INSN_ROT + ARCH_PARTIAL_ALU(size), size, alu, ROT_WRITES_FLAGS(alu));
1086 gen_one(dest);
1087 gen_one(src1);
1088 gen_one(src2);
1089
1090 return true;
1091 }
1092
1093 static bool attr_w gen_3address_rot_imm(struct codegen_context *ctx, unsigned size, unsigned alu, unsigned dest, unsigned src, int64_t imm, unsigned writes_flags)
1094 {
1095 if (!ARCH_IS_3ADDRESS && dest != src) {
1096 gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
1097 gen_one(dest);
1098 gen_one(src);
1099
1100 gen_insn(INSN_ROT + ARCH_PARTIAL_ALU(size), size, alu, ROT_WRITES_FLAGS(alu) | writes_flags);
1101 gen_one(dest);
1102 gen_one(dest);
1103 gen_one(ARG_IMM);
1104 gen_eight(imm);
1105
1106 return true;
1107 }
1108 gen_insn(INSN_ROT + ARCH_PARTIAL_ALU(size), size, alu, ROT_WRITES_FLAGS(alu) | writes_flags);
1109 gen_one(dest);
1110 gen_one(src);
1111 gen_one(ARG_IMM);
1112 gen_eight(imm);
1113 return true;
1114 }
1115
1116 static bool attr_w attr_unused gen_load_two(struct codegen_context *ctx, unsigned dest, unsigned src, int64_t offset)
1117 {
1118 if (!ARCH_HAS_BWX) {
1119 if (!(offset & 7)) {
1120 g(gen_address(ctx, src, offset, IMM_PURPOSE_LDR_OFFSET, OP_SIZE_8));
1121 gen_insn(INSN_MOV_U, OP_SIZE_NATIVE, 0, 0);
1122 gen_one(dest);
1123 gen_address_offset();
1124
1125 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_EXTWL, dest, dest, src));
1126 } else {
1127 g(gen_imm(ctx, offset, IMM_PURPOSE_ADD, i_size(OP_SIZE_ADDRESS)));
1128 gen_insn(INSN_ALU, i_size(OP_SIZE_ADDRESS), ALU_ADD, ALU_WRITES_FLAGS(ALU_ADD, is_imm()));
1129 gen_one(R_OFFSET_IMM);
1130 gen_one(src);
1131 gen_imm_offset();
1132
1133 gen_insn(INSN_MOV_U, OP_SIZE_NATIVE, 0, 0);
1134 gen_one(dest);
1135 gen_one(ARG_ADDRESS_1);
1136 gen_one(R_OFFSET_IMM);
1137 gen_eight(0);
1138
1139 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_EXTWL, dest, dest, R_OFFSET_IMM));
1140 }
1141 #if defined(ARCH_S390)
1142 } else if (!cpu_test_feature(CPU_FEATURE_extended_imm)) {
1143 g(gen_address(ctx, src, offset, IMM_PURPOSE_LDR_SX_OFFSET, OP_SIZE_2));
1144 gen_insn(INSN_MOVSX, OP_SIZE_2, 0, 0);
1145 gen_one(dest);
1146 gen_address_offset();
1147
1148 g(gen_3address_alu_imm(ctx, OP_SIZE_NATIVE, ALU_AND, dest, dest, 0xffff));
1149 #endif
1150 } else {
1151 g(gen_address(ctx, src, offset, IMM_PURPOSE_LDR_OFFSET, OP_SIZE_2));
1152 gen_insn(INSN_MOV, OP_SIZE_2, 0, 0);
1153 gen_one(dest);
1154 gen_address_offset();
1155 }
1156 return true;
1157 }
1158
1159 static bool attr_w gen_load_code_32(struct codegen_context *ctx, unsigned dest, unsigned src, int64_t offset)
1160 {
1161 #if ARG_MODE_N == 3 && defined(ARCH_ALPHA) && !(defined(C_BIG_ENDIAN) ^ CODE_ENDIAN)
1162 if (!ARCH_HAS_BWX && UNALIGNED_TRAP) {
1163 if (offset & 7) {
1164 g(gen_3address_alu_imm(ctx, OP_SIZE_NATIVE, ALU_ADD, R_OFFSET_IMM, src, offset));
1165 src = R_OFFSET_IMM;
1166 offset = 0;
1167 }
1168 g(gen_address(ctx, src, offset, IMM_PURPOSE_LDR_OFFSET, OP_SIZE_8));
1169 gen_insn(INSN_MOV_U, OP_SIZE_NATIVE, 0, 0);
1170 gen_one(dest);
1171 gen_address_offset();
1172
1173 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_EXTLL, dest, dest, src));
1174
1175 g(gen_address(ctx, src, offset + 3, IMM_PURPOSE_LDR_OFFSET, OP_SIZE_8));
1176 gen_insn(INSN_MOV_U, OP_SIZE_NATIVE, 0, 0);
1177 gen_one(R_CONST_IMM);
1178 gen_address_offset();
1179
1180 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_EXTLH, R_CONST_IMM, R_CONST_IMM, src));
1181
1182 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_OR, dest, dest, R_CONST_IMM));
1183
1184 return true;
1185 }
1186 #endif
1187 #if ARG_MODE_N == 3 && defined(ARCH_MIPS) && !(defined(C_BIG_ENDIAN) ^ CODE_ENDIAN)
1188 if (!MIPS_R6 && UNALIGNED_TRAP) {
1189 g(gen_address(ctx, src, offset, IMM_PURPOSE_LDR_OFFSET, OP_SIZE_4));
1190 gen_insn(INSN_MOV_LR, OP_SIZE_4, !CODE_ENDIAN, 0);
1191 gen_one(dest);
1192 gen_one(dest);
1193 gen_address_offset();
1194
1195 g(gen_address(ctx, src, offset + 3, IMM_PURPOSE_LDR_OFFSET, OP_SIZE_4));
1196 gen_insn(INSN_MOV_LR, OP_SIZE_4, CODE_ENDIAN, 0);
1197 gen_one(dest);
1198 gen_one(dest);
1199 gen_address_offset();
1200
1201 return true;
1202 }
1203 #endif
1204 #if ARG_MODE_N == 3
1205 #if !(defined(C_BIG_ENDIAN) ^ CODE_ENDIAN)
1206 if (UNALIGNED_TRAP)
1207 #endif
1208 {
1209 g(gen_load_two(ctx, dest, src, offset));
1210 g(gen_load_two(ctx, R_CONST_IMM, src, offset + 2));
1211 #if CODE_ENDIAN
1212 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SHL, dest, dest, 16, false));
1213 #else
1214 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SHL, R_CONST_IMM, R_CONST_IMM, 16, false));
1215 #endif
1216 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_OR, dest, dest, R_CONST_IMM));
1217 return true;
1218 }
1219 #endif
1220 g(gen_address(ctx, src, offset, IMM_PURPOSE_LDR_OFFSET, ARG_MODE_N - 1));
1221 gen_insn(INSN_MOV, ARG_MODE_N - 1, 0, 0);
1222 gen_one(dest);
1223 gen_address_offset();
1224 return true;
1225 }
1226
1227 static bool attr_w gen_cmp_dest_reg(struct codegen_context *ctx, unsigned attr_unused size, unsigned reg1, unsigned reg2, unsigned reg_dest, int64_t imm, unsigned cond)
1228 {
1229 unsigned neg_result = false;
1230
1231 if (reg2 == (unsigned)-1)
1232 g(gen_imm(ctx, imm, IMM_PURPOSE_CMP, i_size(size)));
1233 #if defined(ARCH_ALPHA)
1234 if (cond == COND_NE) {
1235 gen_insn(INSN_CMP_DEST_REG, i_size(size), COND_E, 0);
1236 gen_one(reg_dest);
1237 gen_one(reg1);
1238 if (reg2 == (unsigned)-1)
1239 gen_imm_offset();
1240 else
1241 gen_one(reg2);
1242 neg_result = true;
1243 goto done;
1244 }
1245 #endif
1246 #if defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS) || defined(ARCH_RISCV64)
1247 if (cond == COND_E || cond == COND_NE) {
1248 gen_insn(INSN_ALU, i_size(size), ALU_XOR, ALU_WRITES_FLAGS(ALU_XOR, reg2 == (unsigned)-1 ? is_imm() : false));
1249 gen_one(reg_dest);
1250 gen_one(reg1);
1251 if (reg2 == (unsigned)-1)
1252 gen_imm_offset();
1253 else
1254 gen_one(reg2);
1255
1256 if (cond == COND_E) {
1257 g(gen_imm(ctx, 1, IMM_PURPOSE_CMP, i_size(size)));
1258 gen_insn(INSN_CMP_DEST_REG, i_size(size), COND_B, 0);
1259 gen_one(reg_dest);
1260 gen_one(reg_dest);
1261 gen_imm_offset();
1262 } else {
1263 gen_insn(INSN_CMP_DEST_REG, i_size(size), COND_B, 0);
1264 gen_one(reg_dest);
1265 gen_one(ARG_IMM);
1266 gen_eight(0);
1267 gen_one(reg_dest);
1268 }
1269 goto done;
1270 }
1271 if (cond == COND_GE || cond == COND_LE || cond == COND_AE || cond == COND_BE) {
1272 cond ^= 1;
1273 neg_result = true;
1274 }
1275 #endif
1276 #if defined(ARCH_IA64)
1277 gen_insn(INSN_CMP_DEST_REG, i_size(size), cond, 0);
1278 gen_one(R_CMP_RESULT);
1279 gen_one(reg1);
1280 if (reg2 == (unsigned)-1)
1281 gen_imm_offset();
1282 else
1283 gen_one(reg2);
1284
1285 if (reg_dest != R_CMP_RESULT) {
1286 gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
1287 gen_one(reg_dest);
1288 gen_one(R_CMP_RESULT);
1289 }
1290
1291 goto done;
1292 #endif
1293 gen_insn(INSN_CMP_DEST_REG, i_size(size), cond, 0);
1294 gen_one(reg_dest);
1295 gen_one(reg1);
1296 if (reg2 == (unsigned)-1)
1297 gen_imm_offset();
1298 else
1299 gen_one(reg2);
1300
1301 goto done;
1302 done:
1303 if (neg_result)
1304 g(gen_3address_alu_imm(ctx, i_size(size), ALU_XOR, reg_dest, reg_dest, 1));
1305
1306 return true;
1307 }
1308
1309 static bool attr_w gen_cmp_test_jmp(struct codegen_context *ctx, unsigned insn, unsigned op_size, unsigned reg1, unsigned reg2, unsigned cond, uint32_t label)
1310 {
1311 bool arch_use_flags = ARCH_HAS_FLAGS;
1312 #if defined(ARCH_ARM64)
1313 if (insn == INSN_TEST && reg1 == reg2 && (cond == COND_E || cond == COND_NE))
1314 arch_use_flags = false;
1315 #endif
1316 #if defined(ARCH_SPARC)
1317 if (insn == INSN_TEST && reg1 == reg2)
1318 arch_use_flags = false;
1319 #endif
1320 if (arch_use_flags) {
1321 if (COND_IS_LOGICAL(cond)) {
1322 gen_insn(insn, op_size, 0, 2);
1323 gen_one(reg1);
1324 gen_one(reg2);
1325
1326 gen_insn(INSN_JMP_COND_LOGICAL, op_size, cond, 0);
1327 gen_four(label);
1328
1329 return true;
1330 }
1331
1332 gen_insn(insn, op_size, 0, 1);
1333 gen_one(reg1);
1334 gen_one(reg2);
1335
1336 #if defined(ARCH_POWER) || defined(ARCH_S390)
1337 if (insn == INSN_TEST) {
1338 if (cond == COND_S)
1339 cond = COND_L;
1340 if (cond == COND_NS)
1341 cond = COND_GE;
1342 }
1343 #endif
1344 gen_insn(INSN_JMP_COND, op_size, cond, 0);
1345 gen_four(label);
1346 } else {
1347 if (insn == INSN_CMP) {
1348 #if defined(ARCH_LOONGARCH64) || defined(ARCH_PARISC) || defined(ARCH_RISCV64)
1349 gen_insn(INSN_JMP_2REGS, op_size, cond, 0);
1350 gen_one(reg1);
1351 gen_one(reg2);
1352 gen_four(label);
1353 return true;
1354 #else
1355 #ifdef R_CMP_RESULT
1356 unsigned jmp_cond = COND_NE;
1357 #if defined(ARCH_MIPS)
1358 if (cond == COND_E || cond == COND_NE) {
1359 gen_insn(INSN_JMP_2REGS, op_size, cond, 0);
1360 gen_one(reg1);
1361 gen_one(reg2);
1362 gen_four(label);
1363 return true;
1364 }
1365 if (cond == COND_AE || cond == COND_BE || cond == COND_LE || cond == COND_GE) {
1366 cond ^= 1;
1367 jmp_cond ^= 1;
1368 }
1369 #endif
1370 #if defined(ARCH_ALPHA)
1371 if (cond == COND_NE) {
1372 g(gen_3address_alu(ctx, op_size, ALU_XOR, R_CMP_RESULT, reg1, reg2));
1373 } else
1374 #endif
1375 {
1376 gen_insn(INSN_CMP_DEST_REG, op_size, cond, 0);
1377 gen_one(R_CMP_RESULT);
1378 gen_one(reg1);
1379 gen_one(reg2);
1380 }
1381
1382 gen_insn(INSN_JMP_REG, OP_SIZE_NATIVE, jmp_cond, 0);
1383 gen_one(R_CMP_RESULT);
1384 gen_four(label);
1385 #else
1386 internal(file_line, "gen_cmp_test_jmp: R_CMP_RESULT not defined");
1387 #endif
1388 #endif
1389 } else if (insn == INSN_TEST) {
1390 if (reg1 != reg2) {
1391 internal(file_line, "gen_cmp_test_jmp: INSN_TEST with two distinct registers is unsupported");
1392 }
1393 #if defined(ARCH_IA64)
1394 if (cond == COND_S)
1395 cond = COND_L;
1396 if (cond == COND_NS)
1397 cond = COND_GE;
1398 g(gen_imm(ctx, 0, IMM_PURPOSE_CMP, OP_SIZE_NATIVE));
1399 gen_insn(INSN_CMP_DEST_REG, OP_SIZE_NATIVE, cond, 0);
1400 gen_one(R_CMP_RESULT);
1401 gen_one(reg1);
1402 gen_imm_offset();
1403
1404 reg1 = R_CMP_RESULT;
1405 cond = COND_NE;
1406 #endif
1407 gen_insn(INSN_JMP_REG, OP_SIZE_NATIVE, cond, 0);
1408 gen_one(reg1);
1409 gen_four(label);
1410 }
1411 }
1412 return true;
1413 }
1414
1415 static bool attr_w gen_cmp_test_imm_jmp(struct codegen_context *ctx, unsigned insn, unsigned attr_unused op_size, unsigned reg1, int64_t value, unsigned cond, uint32_t label)
1416 {
1417 if (insn == INSN_TEST && (cond == COND_E || cond == COND_NE) && is_power_of_2((uint64_t)value)) {
1418 #ifdef HAVE_BUILTIN_CTZ
1419 unsigned attr_unused bit = __builtin_ctzll(value);
1420 #else
1421 unsigned attr_unused bit = 0;
1422 uint64_t v = value;
1423 while ((v = v >> 1))
1424 bit++;
1425 #endif
1426 #if defined(ARCH_ALPHA) || defined(ARCH_PARISC)
1427 if (value == 1 && (cond == COND_E || cond == COND_NE)) {
1428 gen_insn(INSN_JMP_REG, OP_SIZE_NATIVE, cond == COND_E ? COND_BLBC : COND_BLBS, 0);
1429 gen_one(reg1);
1430 gen_four(label);
1431 return true;
1432 }
1433 #endif
1434 #if defined(ARCH_ARM64) || defined(ARCH_PARISC)
1435 gen_insn(INSN_JMP_REG_BIT, OP_SIZE_NATIVE, bit | ((cond == COND_NE) << 6), 0);
1436 gen_one(reg1);
1437 gen_four(label);
1438
1439 return true;
1440 #endif
1441 #if defined(ARCH_POWER)
1442 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SHL, R_CONST_IMM, reg1, (8U << OP_SIZE_NATIVE) - 1 - bit, true));
1443
1444 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, cond == COND_E ? COND_GE : COND_L, 0);
1445 gen_four(label);
1446
1447 return true;
1448 #endif
1449 #if defined(ARCH_IA64)
1450 gen_insn(INSN_TEST_DEST_REG, OP_SIZE_NATIVE, bit | ((cond == COND_NE) << 6), 0);
1451 gen_one(R_CMP_RESULT);
1452 gen_one(reg1);
1453
1454 gen_insn(INSN_JMP_REG, OP_SIZE_NATIVE, COND_NE, 0);
1455 gen_one(R_CMP_RESULT);
1456 gen_four(label);
1457
1458 return true;
1459 #endif
1460 #if defined(R_CMP_RESULT)
1461 if (!is_direct_const(1ULL << bit, IMM_PURPOSE_AND, OP_SIZE_NATIVE) && ARCH_HAS_BTX(BTX_BTEXT, OP_SIZE_NATIVE, true)) {
1462 gen_insn(INSN_BTX, OP_SIZE_NATIVE, BTX_BTEXT, 0);
1463 gen_one(R_CMP_RESULT);
1464 gen_one(reg1);
1465 gen_one(ARG_IMM);
1466 gen_eight(bit);
1467
1468 gen_insn(INSN_JMP_REG, OP_SIZE_NATIVE, cond, 0);
1469 gen_one(R_CMP_RESULT);
1470 gen_four(label);
1471
1472 return true;
1473 }
1474 #endif
1475 }
1476 #if ARCH_HAS_FLAGS
1477 if (unlikely(insn == INSN_CMP) && COND_IS_LOGICAL(cond)) {
1478 g(gen_imm(ctx, value, IMM_PURPOSE_CMP_LOGICAL, op_size));
1479 gen_insn(insn, op_size, 0, 2);
1480 gen_one(reg1);
1481 gen_imm_offset();
1482
1483 gen_insn(INSN_JMP_COND_LOGICAL, op_size, cond, 0);
1484 gen_four(label);
1485
1486 return true;
1487 }
1488 g(gen_imm(ctx, value, insn == INSN_CMP ? IMM_PURPOSE_CMP : IMM_PURPOSE_TEST, op_size));
1489 gen_insn(insn, op_size, 0, 1);
1490 gen_one(reg1);
1491 gen_imm_offset();
1492
1493 gen_insn(INSN_JMP_COND, op_size, cond, 0);
1494 gen_four(label);
1495 #else
1496 if (insn == INSN_CMP) {
1497 #if defined(ARCH_LOONGARCH64) || defined(ARCH_PARISC) || defined(ARCH_RISCV64)
1498 g(gen_imm(ctx, value, IMM_PURPOSE_JMP_2REGS, op_size));
1499 #if defined(ARCH_PARISC)
1500 gen_insn(INSN_JMP_2REGS, op_size, cond, 0);
1501 #else
1502 gen_insn(INSN_JMP_2REGS, i_size(op_size), cond, 0);
1503 #endif
1504 gen_one(reg1);
1505 gen_imm_offset();
1506 gen_four(label);
1507 return true;
1508 #else
1509 unsigned final_cond = COND_NE;
1510 #if defined(ARCH_ALPHA)
1511 if (cond == COND_AE || cond == COND_A || cond == COND_GE || cond == COND_G) {
1512 g(gen_load_constant(ctx, R_CONST_IMM, value));
1513 gen_insn(INSN_CMP_DEST_REG, OP_SIZE_NATIVE, cond, 0);
1514 gen_one(R_CMP_RESULT);
1515 gen_one(reg1);
1516 gen_one(R_CONST_IMM);
1517 } else if (cond == COND_NE) {
1518 g(gen_3address_alu_imm(ctx, OP_SIZE_NATIVE, ALU_XOR, R_CMP_RESULT, reg1, value));
1519 } else
1520 #endif
1521 #if defined(ARCH_MIPS)
1522 if (cond == COND_E || cond == COND_NE) {
1523 g(gen_load_constant(ctx, R_CONST_IMM, value));
1524 gen_insn(INSN_JMP_2REGS, OP_SIZE_NATIVE, cond, 0);
1525 gen_one(reg1);
1526 gen_one(R_CONST_IMM);
1527 gen_four(label);
1528 return true;
1529 }
1530 if (cond == COND_AE || cond == COND_BE || cond == COND_LE || cond == COND_GE) {
1531 cond ^= 1;
1532 final_cond ^= 1;
1533 }
1534 if (cond == COND_A || cond == COND_G) {
1535 g(gen_load_constant(ctx, R_CONST_IMM, value));
1536 gen_insn(INSN_CMP_DEST_REG, OP_SIZE_NATIVE, cond, 0);
1537 gen_one(R_CMP_RESULT);
1538 gen_one(reg1);
1539 gen_one(R_CONST_IMM);
1540 } else
1541 #endif
1542 {
1543 g(gen_imm(ctx, value, IMM_PURPOSE_CMP, OP_SIZE_NATIVE));
1544 gen_insn(INSN_CMP_DEST_REG, OP_SIZE_NATIVE, cond, 0);
1545 gen_one(R_CMP_RESULT);
1546 gen_one(reg1);
1547 gen_imm_offset();
1548 }
1549
1550 gen_insn(INSN_JMP_REG, OP_SIZE_NATIVE, final_cond, 0);
1551 gen_one(R_CMP_RESULT);
1552 gen_four(label);
1553 #endif
1554 } else if (insn == INSN_TEST) {
1555 #if defined(ARCH_IA64)
1556 internal(file_line, "gen_cmp_test_imm_jmp: value %"PRIxMAX" not supported", (uintmax_t)value);
1557 #endif
1558 g(gen_3address_alu_imm(ctx, OP_SIZE_NATIVE, ALU_AND, R_CMP_RESULT, reg1, value));
1559
1560 gen_insn(INSN_JMP_REG, OP_SIZE_NATIVE, cond, 0);
1561 gen_one(R_CMP_RESULT);
1562 gen_four(label);
1563 } else {
1564 internal(file_line, "gen_cmp_test_imm_jmp: invalid insn");
1565 }
1566 #endif
1567 return true;
1568 }
1569
1570 static bool attr_w gen_jmp_on_zero(struct codegen_context *ctx, unsigned attr_unused op_size, unsigned reg, unsigned cond, uint32_t label)
1571 {
1572 #if defined(ARCH_ALPHA) || defined(ARCH_ARM64) || defined(ARCH_LOONGARCH64) || defined(ARCH_RISCV64)
1573 if (1)
1574 #elif defined(ARCH_SPARC)
1575 if (SPARC_9)
1576 #else
1577 if (0)
1578 #endif
1579 {
1580 gen_insn(INSN_JMP_REG, i_size(op_size), cond, 0);
1581 gen_one(reg);
1582 gen_four(label);
1583
1584 return true;
1585 }
1586 g(gen_cmp_test_jmp(ctx, INSN_TEST, i_size(op_size), reg, reg, cond, label));
1587
1588 return true;
1589 }
1590
1591 static bool attr_w gen_jmp_if_negative(struct codegen_context *ctx, unsigned reg, uint32_t label)
1592 {
1593 #if defined(ARCH_ARM64) || defined(ARCH_PARISC)
1594 gen_insn(INSN_JMP_REG_BIT, OP_SIZE_NATIVE, (INT_DEFAULT_BITS - 1) | ((uint32_t)1 << 6), 0);
1595 gen_one(reg);
1596 gen_four(label);
1597 #else
1598 g(gen_jmp_on_zero(ctx, OP_SIZE_INT, reg, COND_S, label));
1599 #endif
1600 return true;
1601 }
1602
1603
1604 static bool attr_w gen_frame_load_raw(struct codegen_context *ctx, unsigned size, bool sx, frame_t slot, int64_t offset, unsigned reg);
1605 static bool attr_w gen_frame_store_raw(struct codegen_context *ctx, unsigned size, frame_t slot, int64_t offset, unsigned reg);
1606
1607 static bool attr_w spill(struct codegen_context *ctx, frame_t v)
1608 {
1609 const struct type *t = get_type_of_local(ctx, v);
1610 g(gen_frame_store_raw(ctx, log_2(t->size), v, 0, ctx->registers[v]));
1611 return true;
1612 }
1613
1614 static bool attr_w unspill(struct codegen_context *ctx, frame_t v)
1615 {
1616 const struct type *t = get_type_of_local(ctx, v);
1617 g(gen_frame_load_raw(ctx, log_2(t->size), false, v, 0, ctx->registers[v]));
1618 return true;
1619 }
1620
1621 static bool attr_w gen_upcall_start(struct codegen_context *ctx, unsigned args)
1622 {
1623 size_t i;
1624 ajla_assert_lo(ctx->upcall_args == -1, (file_line, "gen_upcall_start: gen_upcall_end not called"));
1625 ctx->upcall_args = (int)args;
1626
1627 for (i = 0; i < ctx->need_spill_l; i++)
1628 g(spill(ctx, ctx->need_spill[i]));
1629
1630 /*gen_insn(INSN_PUSH, OP_SIZE_8, 0, 0);
1631 gen_one(R_R8);
1632 gen_insn(INSN_PUSH, OP_SIZE_8, 0, 0);
1633 gen_one(R_R9);*/
1634
1635 return true;
1636 }
1637
1638 static bool attr_w gen_upcall_end(struct codegen_context *ctx, unsigned args)
1639 {
1640 size_t i;
1641 ajla_assert_lo(ctx->upcall_args == (int)args, (file_line, "gen_upcall_end: gen_upcall_start mismatch: %d", ctx->upcall_args));
1642 ctx->upcall_args = -1;
1643
1644 for (i = 0; i < ctx->need_spill_l; i++)
1645 g(unspill(ctx, ctx->need_spill[i]));
1646
1647
1648 /*gen_insn(INSN_POP, OP_SIZE_8, 0, 0);
1649 gen_one(R_R9);
1650 gen_insn(INSN_POP, OP_SIZE_8, 0, 0);
1651 gen_one(R_R8);*/
1652
1653 return true;
1654 }
1655
1656
1657 #define frame_offs(x) ((ssize_t)offsetof(struct frame_struct, x) - (ssize_t)frame_offset)
1658
1659 static bool attr_w gen_set_1(struct codegen_context *ctx, unsigned base, frame_t slot_1, int64_t offset, bool val)
1660 {
1661 #ifdef HAVE_BITWISE_FRAME
1662 int bit = slot_1 & ((1 << (OP_SIZE_BITMAP + 3)) - 1);
1663 offset += slot_1 >> (OP_SIZE_BITMAP + 3) << OP_SIZE_BITMAP;
1664 #if defined(ARCH_X86)
1665 if (OP_SIZE_BITMAP == OP_SIZE_4) {
1666 g(gen_address(ctx, base, offset, IMM_PURPOSE_STR_OFFSET, OP_SIZE_BITMAP));
1667 if (val) {
1668 g(gen_imm(ctx, (int32_t)((uint32_t)1 << bit), IMM_PURPOSE_OR, OP_SIZE_BITMAP));
1669 gen_insn(INSN_ALU, OP_SIZE_BITMAP, ALU_OR, ALU_WRITES_FLAGS(ALU_OR, true));
1670 } else {
1671 g(gen_imm(ctx, ~(int32_t)((uint32_t)1 << bit), IMM_PURPOSE_AND, OP_SIZE_BITMAP));
1672 gen_insn(INSN_ALU, OP_SIZE_BITMAP, ALU_AND, ALU_WRITES_FLAGS(ALU_AND, true));
1673 }
1674 gen_address_offset();
1675 gen_address_offset();
1676 gen_imm_offset();
1677 } else {
1678 g(gen_address(ctx, base, offset, IMM_PURPOSE_STR_OFFSET, OP_SIZE_BITMAP));
1679 g(gen_imm(ctx, bit, IMM_PURPOSE_BITWISE, OP_SIZE_BITMAP));
1680 gen_insn(INSN_BTX, OP_SIZE_BITMAP, val ? BTX_BTS : BTX_BTR, 1);
1681 gen_address_offset();
1682 gen_address_offset();
1683 gen_imm_offset();
1684 }
1685 #else
1686 g(gen_address(ctx, base, offset, ARCH_PREFERS_SX(OP_SIZE_BITMAP) ? IMM_PURPOSE_LDR_SX_OFFSET : IMM_PURPOSE_LDR_OFFSET, OP_SIZE_BITMAP));
1687 gen_insn(ARCH_PREFERS_SX(OP_SIZE_BITMAP) ? INSN_MOVSX : INSN_MOV, OP_SIZE_BITMAP, 0, 0);
1688 gen_one(R_SCRATCH_NA_1);
1689 gen_address_offset();
1690
1691 if (!is_direct_const(!val ? ~(1ULL << bit) : 1ULL << bit, !val ? IMM_PURPOSE_AND : IMM_PURPOSE_OR, OP_SIZE_NATIVE) && ARCH_HAS_BTX(!val ? BTX_BTR : BTX_BTS, OP_SIZE_NATIVE, true)) {
1692 g(gen_imm(ctx, bit, IMM_PURPOSE_BITWISE, OP_SIZE_NATIVE));
1693 gen_insn(INSN_BTX, OP_SIZE_NATIVE, !val ? BTX_BTR : BTX_BTS, 0);
1694 gen_one(R_SCRATCH_NA_1);
1695 gen_one(R_SCRATCH_NA_1);
1696 gen_imm_offset();
1697 } else if (!val) {
1698 g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_BITMAP), ALU_AND, R_SCRATCH_NA_1, R_SCRATCH_NA_1, ~((uintptr_t)1 << bit)));
1699 } else {
1700 g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_BITMAP), val ? ALU_OR : ALU_ANDN, R_SCRATCH_NA_1, R_SCRATCH_NA_1, (uintptr_t)1 << bit));
1701 }
1702
1703 g(gen_address(ctx, base, offset, IMM_PURPOSE_STR_OFFSET, OP_SIZE_BITMAP));
1704 gen_insn(INSN_MOV, OP_SIZE_BITMAP, 0, 0);
1705 gen_address_offset();
1706 gen_one(R_SCRATCH_NA_1);
1707 #endif
1708 #else
1709 #if !defined(ARCH_X86)
1710 if (!ARCH_HAS_BWX) {
1711 g(gen_address(ctx, base, offset + (slot_1 & ~7), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_8));
1712 gen_insn(INSN_MOV, OP_SIZE_8, 0, 0);
1713 gen_one(R_SCRATCH_NA_1);
1714 gen_address_offset();
1715
1716 if (!val) {
1717 g(gen_3address_alu_imm(ctx, OP_SIZE_8, ALU_MSKBL, R_SCRATCH_NA_1, R_SCRATCH_NA_1, slot_1 & 7));
1718 } else {
1719 g(gen_3address_alu_imm(ctx, OP_SIZE_8, ALU_OR, R_SCRATCH_NA_1, R_SCRATCH_NA_1, 1ULL << ((slot_1 & 7) * 8)));
1720 }
1721
1722 g(gen_address(ctx, base, offset + (slot_1 & ~7), IMM_PURPOSE_STR_OFFSET, OP_SIZE_8));
1723 gen_insn(INSN_MOV, OP_SIZE_8, 0, 0);
1724 gen_address_offset();
1725 gen_one(R_SCRATCH_NA_1);
1726
1727 return true;
1728 }
1729 #endif
1730 g(gen_address(ctx, base, offset + slot_1, IMM_PURPOSE_MVI_CLI_OFFSET, OP_SIZE_1));
1731 g(gen_imm(ctx, val, IMM_PURPOSE_STORE_VALUE, OP_SIZE_1));
1732 gen_insn(INSN_MOV, OP_SIZE_1, 0, 0);
1733 gen_address_offset();
1734 gen_imm_offset();
1735 #endif
1736 return true;
1737 }
1738
1739 static bool attr_w gen_set_1_variable(struct codegen_context *ctx, unsigned slot_reg, int64_t offset, bool val)
1740 {
1741 #ifdef HAVE_BITWISE_FRAME
1742 #if defined(ARCH_X86)
1743 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_STR_OFFSET, OP_SIZE_BITMAP));
1744 gen_insn(INSN_BTX, OP_SIZE_BITMAP, val ? BTX_BTS : BTX_BTR, 1);
1745 gen_address_offset();
1746 gen_address_offset();
1747 gen_one(slot_reg);
1748 #else
1749 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SHR, R_SCRATCH_NA_1, slot_reg, OP_SIZE_BITMAP + 3, false));
1750
1751 if (ARCH_HAS_SHIFTED_ADD(OP_SIZE_BITMAP)) {
1752 gen_insn(INSN_ALU, i_size(OP_SIZE_ADDRESS), ALU_ADD, ALU_WRITES_FLAGS(ALU_ADD, false));
1753 gen_one(R_SCRATCH_NA_1);
1754 gen_one(R_FRAME);
1755 gen_one(ARG_SHIFTED_REGISTER);
1756 gen_one(ARG_SHIFT_LSL | OP_SIZE_BITMAP);
1757 gen_one(R_SCRATCH_NA_1);
1758 } else {
1759 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SHL, R_SCRATCH_NA_1, R_SCRATCH_NA_1, OP_SIZE_BITMAP, false));
1760
1761 g(gen_3address_alu(ctx, i_size(OP_SIZE_ADDRESS), ALU_ADD, R_SCRATCH_NA_1, R_SCRATCH_NA_1, R_FRAME));
1762 }
1763 if (ARCH_HAS_BTX(!val ? BTX_BTR : BTX_BTS, OP_SIZE_BITMAP, false)) {
1764 g(gen_address(ctx, R_SCRATCH_NA_1, offset, ARCH_PREFERS_SX(OP_SIZE_BITMAP) ? IMM_PURPOSE_LDR_SX_OFFSET : IMM_PURPOSE_LDR_OFFSET, OP_SIZE_BITMAP));
1765 gen_insn(ARCH_PREFERS_SX(OP_SIZE_BITMAP) ? INSN_MOVSX : INSN_MOV, OP_SIZE_BITMAP, 0, 0);
1766 gen_one(R_SCRATCH_NA_3);
1767 gen_address_offset();
1768
1769 gen_insn(INSN_BTX, OP_SIZE_BITMAP, !val ? BTX_BTR : BTX_BTS, 0);
1770 gen_one(R_SCRATCH_NA_3);
1771 gen_one(R_SCRATCH_NA_3);
1772 gen_one(slot_reg);
1773
1774 goto save_it;
1775 }
1776 if (ARCH_SHIFT_SIZE > OP_SIZE_BITMAP) {
1777 g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_BITMAP), ALU_AND, R_SCRATCH_NA_3, slot_reg, (1U << (OP_SIZE_BITMAP + 3)) - 1));
1778
1779 g(gen_load_constant(ctx, R_SCRATCH_NA_2, 1));
1780
1781 g(gen_3address_rot(ctx, i_size(OP_SIZE_BITMAP), ROT_SHL, R_SCRATCH_NA_2, R_SCRATCH_NA_2, R_SCRATCH_NA_3));
1782 } else {
1783 g(gen_load_constant(ctx, R_SCRATCH_NA_2, 1));
1784
1785 g(gen_3address_rot(ctx, OP_SIZE_BITMAP, ROT_SHL, R_SCRATCH_NA_2, R_SCRATCH_NA_2, slot_reg));
1786 }
1787 g(gen_address(ctx, R_SCRATCH_NA_1, offset, ARCH_PREFERS_SX(OP_SIZE_BITMAP) ? IMM_PURPOSE_LDR_SX_OFFSET : IMM_PURPOSE_LDR_OFFSET, OP_SIZE_BITMAP));
1788 gen_insn(ARCH_PREFERS_SX(OP_SIZE_BITMAP) ? INSN_MOVSX : INSN_MOV, OP_SIZE_BITMAP, 0, 0);
1789 gen_one(R_SCRATCH_NA_3);
1790 gen_address_offset();
1791
1792 if (!val && !ARCH_HAS_ANDN) {
1793 gen_insn(INSN_ALU1, i_size(OP_SIZE_BITMAP), ALU1_NOT, ALU1_WRITES_FLAGS(ALU1_NOT));
1794 gen_one(R_SCRATCH_2);
1795 gen_one(R_SCRATCH_2);
1796
1797 g(gen_3address_alu(ctx, i_size(OP_SIZE_BITMAP), ALU_AND, R_SCRATCH_NA_3, R_SCRATCH_NA_3, R_SCRATCH_NA_2));
1798 } else {
1799 g(gen_3address_alu(ctx, i_size(OP_SIZE_BITMAP), val ? ALU_OR : ALU_ANDN, R_SCRATCH_NA_3, R_SCRATCH_NA_3, R_SCRATCH_NA_2));
1800 }
1801
1802 goto save_it;
1803 save_it:
1804 g(gen_address(ctx, R_SCRATCH_NA_1, offset, IMM_PURPOSE_STR_OFFSET, OP_SIZE_BITMAP));
1805 gen_insn(INSN_MOV, OP_SIZE_BITMAP, 0, 0);
1806 gen_address_offset();
1807 gen_one(R_SCRATCH_NA_3);
1808 #endif
1809 #else
1810 #if defined(ARCH_X86)
1811 g(gen_imm(ctx, val, IMM_PURPOSE_STORE_VALUE, OP_SIZE_1));
1812 gen_insn(INSN_MOV, OP_SIZE_1, 0, 0);
1813 gen_one(ARG_ADDRESS_2);
1814 gen_one(R_FRAME);
1815 gen_one(slot_reg);
1816 gen_eight(offset);
1817 gen_imm_offset();
1818 #else
1819 g(gen_3address_alu(ctx, i_size(OP_SIZE_ADDRESS), ALU_ADD, R_SCRATCH_NA_1, R_FRAME, slot_reg));
1820 if (!ARCH_HAS_BWX) {
1821 g(gen_address(ctx, R_SCRATCH_NA_1, offset, IMM_PURPOSE_LDR_OFFSET, OP_SIZE_8));
1822 gen_insn(INSN_MOV_U, OP_SIZE_8, 0, 0);
1823 gen_one(R_SCRATCH_NA_2);
1824 gen_address_offset();
1825 if (!val) {
1826 g(gen_3address_alu(ctx, OP_SIZE_8, ALU_MSKBL, R_SCRATCH_NA_2, R_SCRATCH_NA_2, R_SCRATCH_NA_1));
1827 } else {
1828 g(gen_load_constant(ctx, R_SCRATCH_NA_3, 1));
1829
1830 g(gen_3address_alu(ctx, OP_SIZE_8, ALU_INSBL, R_SCRATCH_NA_3, R_SCRATCH_NA_3, R_SCRATCH_NA_1));
1831
1832 g(gen_3address_alu(ctx, OP_SIZE_8, ALU_OR, R_SCRATCH_NA_2, R_SCRATCH_NA_2, R_SCRATCH_NA_3));
1833 }
1834 g(gen_address(ctx, R_SCRATCH_NA_1, offset, IMM_PURPOSE_STR_OFFSET, OP_SIZE_8));
1835 gen_insn(INSN_MOV_U, OP_SIZE_8, 0, 0);
1836 gen_address_offset();
1837 gen_one(R_SCRATCH_NA_2);
1838
1839 return true;
1840 }
1841
1842 g(gen_address(ctx, R_SCRATCH_NA_1, offset, IMM_PURPOSE_MVI_CLI_OFFSET, OP_SIZE_1));
1843 g(gen_imm(ctx, val, IMM_PURPOSE_STORE_VALUE, OP_SIZE_1));
1844 gen_insn(INSN_MOV, OP_SIZE_1, 0, 0);
1845 gen_address_offset();
1846 gen_imm_offset();
1847 #endif
1848 #endif
1849 return true;
1850 }
1851
1852 #define TEST 0
1853 #define TEST_CLEAR 1
1854 #define TEST_SET 2
1855
1856 static bool attr_w gen_test_1(struct codegen_context *ctx, unsigned base, frame_t slot_1, int64_t offset, uint32_t label, bool jz, uint8_t test)
1857 {
1858 #ifdef HAVE_BITWISE_FRAME
1859 int bit = slot_1 & ((1 << (OP_SIZE_BITMAP + 3)) - 1);
1860 offset += slot_1 >> (OP_SIZE_BITMAP + 3) << OP_SIZE_BITMAP;
1861 #if defined(ARCH_X86)
1862 if (test == TEST) {
1863 if (OP_SIZE_BITMAP == OP_SIZE_4) {
1864 g(gen_address(ctx, base, offset, ARCH_PREFERS_SX(OP_SIZE_BITMAP) ? IMM_PURPOSE_LDR_SX_OFFSET : IMM_PURPOSE_LDR_OFFSET, OP_SIZE_BITMAP));
1865 g(gen_imm(ctx, (int32_t)((uint32_t)1 << bit), IMM_PURPOSE_TEST, OP_SIZE_BITMAP));
1866 gen_insn(INSN_TEST, OP_SIZE_BITMAP, 0, 1);
1867 gen_address_offset();
1868 gen_imm_offset();
1869
1870 gen_insn(INSN_JMP_COND, OP_SIZE_BITMAP, jz ? COND_E : COND_NE, 0);
1871 gen_four(label);
1872
1873 return true;
1874 }
1875 g(gen_address(ctx, base, offset, test == TEST ? IMM_PURPOSE_LDR_OFFSET : IMM_PURPOSE_STR_OFFSET, OP_SIZE_BITMAP));
1876 g(gen_imm(ctx, bit, IMM_PURPOSE_BITWISE, OP_SIZE_BITMAP));
1877 gen_insn(INSN_BT, OP_SIZE_BITMAP, 0, 1);
1878 gen_address_offset();
1879 gen_imm_offset();
1880 } else {
1881 g(gen_address(ctx, base, offset, test == TEST ? IMM_PURPOSE_LDR_OFFSET : IMM_PURPOSE_STR_OFFSET, OP_SIZE_BITMAP));
1882 g(gen_imm(ctx, bit, IMM_PURPOSE_BITWISE, OP_SIZE_BITMAP));
1883 gen_insn(INSN_BTX, OP_SIZE_BITMAP, test == TEST_CLEAR ? BTX_BTR : BTX_BTS, 1);
1884 gen_address_offset();
1885 gen_address_offset();
1886 gen_imm_offset();
1887 }
1888
1889 gen_insn(INSN_JMP_COND, OP_SIZE_1, jz ? COND_AE : COND_B, 0);
1890 gen_four(label);
1891 #else
1892 g(gen_address(ctx, base, offset, ARCH_PREFERS_SX(OP_SIZE_BITMAP) ? IMM_PURPOSE_LDR_SX_OFFSET : IMM_PURPOSE_LDR_OFFSET, OP_SIZE_BITMAP));
1893 gen_insn(ARCH_PREFERS_SX(OP_SIZE_BITMAP) ? INSN_MOVSX : INSN_MOV, OP_SIZE_BITMAP, 0, 0);
1894 gen_one(R_SCRATCH_NA_1);
1895 gen_address_offset();
1896
1897 if (jz ? test == TEST_SET : test == TEST_CLEAR) {
1898 if (!is_direct_const(test == TEST_CLEAR ? ~(1ULL << bit) : 1ULL << bit, test == TEST_CLEAR ? IMM_PURPOSE_AND : IMM_PURPOSE_OR, OP_SIZE_NATIVE) && ARCH_HAS_BTX(test == TEST_CLEAR ? BTX_BTR : BTX_BTS, OP_SIZE_NATIVE, true)) {
1899 g(gen_imm(ctx, bit, IMM_PURPOSE_BITWISE, OP_SIZE_NATIVE));
1900 gen_insn(INSN_BTX, OP_SIZE_NATIVE, test == TEST_CLEAR ? BTX_BTR : BTX_BTS, 0);
1901 gen_one(R_SCRATCH_NA_2);
1902 gen_one(R_SCRATCH_NA_1);
1903 gen_imm_offset();
1904 } else if (test == TEST_CLEAR) {
1905 g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_BITMAP), ALU_AND, R_SCRATCH_NA_2, R_SCRATCH_NA_1, ~((uintptr_t)1 << bit)));
1906 } else {
1907 g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_BITMAP), test == TEST_SET ? ALU_OR : ALU_ANDN, R_SCRATCH_NA_2, R_SCRATCH_NA_1, (uintptr_t)1 << bit));
1908 }
1909
1910 g(gen_address(ctx, base, offset, IMM_PURPOSE_STR_OFFSET, OP_SIZE_BITMAP));
1911 gen_insn(INSN_MOV, OP_SIZE_BITMAP, 0, 0);
1912 gen_address_offset();
1913 gen_one(R_SCRATCH_NA_2);
1914 }
1915 #if defined(ARCH_ARM) || defined(ARCH_IA64) || defined(ARCH_LOONGARCH64) || defined(ARCH_PARISC) || defined(ARCH_POWER) || defined(ARCH_S390)
1916 g(gen_cmp_test_imm_jmp(ctx, INSN_TEST, i_size(OP_SIZE_BITMAP), R_SCRATCH_NA_1, (uintptr_t)1 << bit, !jz ? COND_NE : COND_E, label));
1917 #else
1918 g(gen_3address_rot_imm(ctx, i_size(OP_SIZE_BITMAP), ROT_SHL, R_SCRATCH_NA_3, R_SCRATCH_NA_1, (1U << (i_size(OP_SIZE_BITMAP) + 3)) - 1 - bit, false));
1919
1920 gen_insn(INSN_JMP_REG, i_size(OP_SIZE_BITMAP), !jz ? COND_S : COND_NS, 0);
1921 gen_one(R_SCRATCH_NA_3);
1922 gen_four(label);
1923 #endif
1924 if (!jz ? test == TEST_SET : test == TEST_CLEAR) {
1925 if (!is_direct_const(test == TEST_CLEAR ? ~(1ULL << bit) : 1ULL << bit, test == TEST_CLEAR ? IMM_PURPOSE_XOR : IMM_PURPOSE_OR, OP_SIZE_NATIVE) && ARCH_HAS_BTX(test == TEST_CLEAR ? BTX_BTR : BTX_BTS, OP_SIZE_NATIVE, true)) {
1926 g(gen_imm(ctx, bit, IMM_PURPOSE_BITWISE, OP_SIZE_NATIVE));
1927 gen_insn(INSN_BTX, OP_SIZE_NATIVE, test == TEST_CLEAR ? BTX_BTR : BTX_BTS, 0);
1928 gen_one(R_SCRATCH_NA_1);
1929 gen_one(R_SCRATCH_NA_1);
1930 gen_imm_offset();
1931 } else {
1932 #if defined(ARCH_S390)
1933 if (test == TEST_CLEAR)
1934 g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_BITMAP), ALU_AND, R_SCRATCH_NA_1, R_SCRATCH_NA_1, ~((uintptr_t)1 << bit)));
1935 else
1936 #endif
1937 g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_BITMAP), test == TEST_SET ? ALU_OR : ALU_XOR, R_SCRATCH_NA_1, R_SCRATCH_NA_1, (uintptr_t)1 << bit));
1938 }
1939 g(gen_address(ctx, base, offset, IMM_PURPOSE_STR_OFFSET, OP_SIZE_BITMAP));
1940 gen_insn(INSN_MOV, OP_SIZE_BITMAP, 0, 0);
1941 gen_address_offset();
1942 gen_one(R_SCRATCH_NA_1);
1943 }
1944 #endif
1945 #else
1946 #if defined(ARCH_X86) || defined(ARCH_S390)
1947 g(gen_address(ctx, base, offset + slot_1, IMM_PURPOSE_MVI_CLI_OFFSET, OP_SIZE_1));
1948 gen_insn(INSN_CMP, OP_SIZE_1, 0, 2);
1949 gen_address_offset();
1950 gen_one(ARG_IMM);
1951 gen_eight(0);
1952
1953 if (jz ? test == TEST_SET : test == TEST_CLEAR) {
1954 g(gen_set_1(ctx, base, slot_1, offset, test == TEST_SET));
1955 }
1956
1957 gen_insn(INSN_JMP_COND, OP_SIZE_1, jz ? COND_E : COND_NE, 0);
1958 gen_four(label);
1959
1960 if (!jz ? test == TEST_SET : test == TEST_CLEAR) {
1961 g(gen_set_1(ctx, base, slot_1, offset, test == TEST_SET));
1962 }
1963 #else
1964 if (!ARCH_HAS_BWX) {
1965 g(gen_address(ctx, base, offset + (slot_1 & ~7), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_8));
1966 gen_insn(INSN_MOV, OP_SIZE_8, 0, 0);
1967 gen_one(R_SCRATCH_NA_2);
1968 gen_address_offset();
1969
1970 g(gen_3address_alu_imm(ctx, OP_SIZE_8, ALU_EXTBL, R_SCRATCH_NA_2, R_SCRATCH_NA_2, slot_1 & 7));
1971 } else {
1972 g(gen_address(ctx, base, offset + slot_1, IMM_PURPOSE_LDR_OFFSET, OP_SIZE_1));
1973 gen_insn(INSN_MOV, OP_SIZE_1, 0, 0);
1974 gen_one(R_SCRATCH_NA_2);
1975 gen_address_offset();
1976 }
1977
1978 if (jz ? test == TEST_SET : test == TEST_CLEAR) {
1979 g(gen_set_1(ctx, base, slot_1, offset, test == TEST_SET));
1980 }
1981
1982 g(gen_jmp_on_zero(ctx, OP_SIZE_1, R_SCRATCH_NA_2, jz ? COND_E : COND_NE, label));
1983
1984 if (!jz ? test == TEST_SET : test == TEST_CLEAR) {
1985 g(gen_set_1(ctx, base, slot_1, offset, test == TEST_SET));
1986 }
1987 #endif
1988 #endif
1989 return true;
1990 }
1991
1992 static bool attr_w gen_test_2(struct codegen_context *ctx, frame_t slot_1, frame_t slot_2, uint32_t label)
1993 {
1994 unsigned attr_unused bit1, bit2;
1995 frame_t attr_unused addr1, addr2;
1996 if (unlikely(slot_1 == slot_2)) {
1997 g(gen_test_1(ctx, R_FRAME, slot_1, 0, label, false, TEST));
1998 return true;
1999 }
2000 #ifdef HAVE_BITWISE_FRAME
2001 addr1 = slot_1 >> (OP_SIZE_BITMAP + 3) << OP_SIZE_BITMAP;
2002 addr2 = slot_2 >> (OP_SIZE_BITMAP + 3) << OP_SIZE_BITMAP;
2003 if (addr1 != addr2)
2004 goto dont_optimize;
2005 bit1 = slot_1 & ((1 << (OP_SIZE_BITMAP + 3)) - 1);
2006 bit2 = slot_2 & ((1 << (OP_SIZE_BITMAP + 3)) - 1);
2007 #if defined(ARCH_X86)
2008 g(gen_address(ctx, R_FRAME, addr1, IMM_PURPOSE_LDR_OFFSET, OP_SIZE_BITMAP));
2009 if (OP_SIZE_BITMAP == OP_SIZE_4) {
2010 g(gen_imm(ctx, (int32_t)(((uintptr_t)1 << bit1) | ((uintptr_t)1 << bit2)), IMM_PURPOSE_TEST, OP_SIZE_BITMAP));
2011 } else {
2012 g(gen_imm(ctx, ((uintptr_t)1 << bit1) | ((uintptr_t)1 << bit2), IMM_PURPOSE_TEST, OP_SIZE_BITMAP));
2013 }
2014 gen_insn(INSN_TEST, OP_SIZE_BITMAP, 0, 1);
2015 gen_address_offset();
2016 gen_imm_offset();
2017
2018 gen_insn(INSN_JMP_COND, OP_SIZE_BITMAP, COND_NE, 0);
2019 gen_four(label);
2020
2021 return true;
2022 #else
2023 g(gen_address(ctx, R_FRAME, addr1, ARCH_PREFERS_SX(OP_SIZE_BITMAP) ? IMM_PURPOSE_LDR_SX_OFFSET : IMM_PURPOSE_LDR_OFFSET, OP_SIZE_BITMAP));
2024 gen_insn(ARCH_PREFERS_SX(OP_SIZE_BITMAP) ? INSN_MOVSX : INSN_MOV, OP_SIZE_BITMAP, 0, 0);
2025 gen_one(R_SCRATCH_NA_1);
2026 gen_address_offset();
2027
2028 if (is_direct_const(1ULL << bit1 | 1ULL << bit2, IMM_PURPOSE_TEST, OP_SIZE_BITMAP)) {
2029 g(gen_cmp_test_imm_jmp(ctx, INSN_TEST, i_size(OP_SIZE_BITMAP), R_SCRATCH_NA_1, 1ULL << bit1 | 1ULL << bit2, COND_NE, label));
2030 return true;
2031 }
2032 #if defined(ARCH_ARM) || defined(ARCH_IA64) || defined(ARCH_PARISC) || defined(ARCH_S390)
2033 g(gen_cmp_test_imm_jmp(ctx, INSN_TEST, i_size(OP_SIZE_BITMAP), R_SCRATCH_NA_1, (uintptr_t)1 << bit1, COND_NE, label));
2034 g(gen_cmp_test_imm_jmp(ctx, INSN_TEST, i_size(OP_SIZE_BITMAP), R_SCRATCH_NA_1, (uintptr_t)1 << bit2, COND_NE, label));
2035
2036 return true;
2037 #endif
2038 if (ARCH_HAS_BTX(BTX_BTEXT, OP_SIZE_NATIVE, true)) {
2039 gen_insn(INSN_BTX, OP_SIZE_NATIVE, BTX_BTEXT, 0);
2040 gen_one(R_SCRATCH_NA_2);
2041 gen_one(R_SCRATCH_NA_1);
2042 gen_one(ARG_IMM);
2043 gen_eight(bit1);
2044
2045 gen_insn(INSN_BTX, OP_SIZE_NATIVE, BTX_BTEXT, 0);
2046 gen_one(R_SCRATCH_NA_1);
2047 gen_one(R_SCRATCH_NA_1);
2048 gen_one(ARG_IMM);
2049 gen_eight(bit2);
2050
2051 g(gen_3address_alu(ctx, i_size(OP_SIZE_NATIVE), ALU_OR, R_SCRATCH_NA_1, R_SCRATCH_NA_1, R_SCRATCH_NA_2));
2052
2053 gen_insn(INSN_JMP_REG, i_size(OP_SIZE_NATIVE), COND_NE, 0);
2054 gen_one(R_SCRATCH_NA_1);
2055 gen_four(label);
2056
2057 return true;
2058 }
2059 g(gen_3address_rot_imm(ctx, i_size(OP_SIZE_BITMAP), ROT_SHL, R_SCRATCH_NA_2, R_SCRATCH_NA_1, (1U << (i_size(OP_SIZE_BITMAP) + 3)) - 1 - bit1, false));
2060 g(gen_3address_rot_imm(ctx, i_size(OP_SIZE_BITMAP), ROT_SHL, R_SCRATCH_NA_1, R_SCRATCH_NA_1, (1U << (i_size(OP_SIZE_BITMAP) + 3)) - 1 - bit2, false));
2061 #if defined(ARCH_POWER)
2062 gen_insn(INSN_ALU, i_size(OP_SIZE_BITMAP), ALU_OR, 1);
2063 gen_one(R_SCRATCH_NA_1);
2064 gen_one(R_SCRATCH_NA_1);
2065 gen_one(R_SCRATCH_NA_2);
2066
2067 gen_insn(INSN_JMP_COND, i_size(OP_SIZE_BITMAP), COND_L, 0);
2068 gen_four(label);
2069 #else
2070 g(gen_3address_alu(ctx, i_size(OP_SIZE_BITMAP), ALU_OR, R_SCRATCH_NA_1, R_SCRATCH_NA_1, R_SCRATCH_NA_2));
2071
2072 gen_insn(INSN_JMP_REG, i_size(OP_SIZE_BITMAP), COND_S, 0);
2073 gen_one(R_SCRATCH_NA_1);
2074 gen_four(label);
2075 #endif
2076 return true;
2077 #endif
2078 dont_optimize:
2079 g(gen_test_1(ctx, R_FRAME, slot_1, 0, label, false, TEST));
2080 g(gen_test_1(ctx, R_FRAME, slot_2, 0, label, false, TEST));
2081 #else
2082 #if defined(ARCH_X86)
2083 g(gen_address(ctx, R_FRAME, slot_1, IMM_PURPOSE_LDR_OFFSET, OP_SIZE_1));
2084 gen_insn(INSN_MOV, OP_SIZE_1, 0, 0);
2085 gen_one(R_SCRATCH_1);
2086 gen_address_offset();
2087
2088 g(gen_address(ctx, R_FRAME, slot_2, IMM_PURPOSE_LDR_OFFSET, OP_SIZE_1));
2089 gen_insn(INSN_ALU_PARTIAL, OP_SIZE_1, ALU_OR, 1);
2090 gen_one(R_SCRATCH_1);
2091 gen_one(R_SCRATCH_1);
2092 gen_address_offset();
2093
2094 gen_insn(INSN_JMP_COND, OP_SIZE_1, COND_NE, 0);
2095 gen_four(label);
2096 #else
2097 if (!ARCH_HAS_BWX || !ARCH_HAS_FLAGS
2098 #if defined(ARCH_S390)
2099 || 1
2100 #endif
2101 ) {
2102 if (!ARCH_HAS_BWX && (slot_1 & ~7) == (slot_2 & ~7)) {
2103 g(gen_address(ctx, R_FRAME, slot_1 & ~7, IMM_PURPOSE_LDR_OFFSET, OP_SIZE_8));
2104 gen_insn(INSN_MOV, OP_SIZE_8, 0, 0);
2105 gen_one(R_SCRATCH_1);
2106 gen_address_offset();
2107
2108 g(gen_3address_alu_imm(ctx, OP_SIZE_NATIVE, ALU_ZAPNOT, R_SCRATCH_1, R_SCRATCH_1, (1U << (slot_1 & 7)) | (1U << (slot_2 & 7))));
2109
2110 g(gen_jmp_on_zero(ctx, OP_SIZE_8, R_SCRATCH_1, COND_NE, label));
2111 } else {
2112 g(gen_test_1(ctx, R_FRAME, slot_1, 0, label, false, TEST));
2113 g(gen_test_1(ctx, R_FRAME, slot_2, 0, label, false, TEST));
2114 }
2115 } else {
2116 g(gen_address(ctx, R_FRAME, slot_1, IMM_PURPOSE_LDR_OFFSET, OP_SIZE_1));
2117 gen_insn(INSN_MOV, OP_SIZE_1, 0, 0);
2118 gen_one(R_SCRATCH_1);
2119 gen_address_offset();
2120
2121 g(gen_address(ctx, R_FRAME, slot_2, IMM_PURPOSE_LDR_OFFSET, OP_SIZE_1));
2122 gen_insn(INSN_MOV, OP_SIZE_1, 0, 0);
2123 gen_one(R_SCRATCH_2);
2124 gen_address_offset();
2125 #if defined(ARCH_ARM) || defined(ARCH_SPARC)
2126 gen_insn(INSN_CMN, OP_SIZE_NATIVE, 0, 1);
2127 gen_one(R_SCRATCH_1);
2128 gen_one(R_SCRATCH_2);
2129 #else
2130 gen_insn(INSN_ALU, OP_SIZE_NATIVE, ALU_OR, 1);
2131 gen_one(R_SCRATCH_1);
2132 gen_one(R_SCRATCH_1);
2133 gen_one(R_SCRATCH_2);
2134 #endif
2135 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_NE, 0);
2136 gen_four(label);
2137 }
2138 #endif
2139 #endif
2140 return true;
2141 }
2142
2143 static int frame_t_compare(const void *p1, const void *p2)
2144 {
2145 if (*(const frame_t *)p1 < *(const frame_t *)p2)
2146 return -1;
2147 if (likely(*(const frame_t *)p1 > *(const frame_t *)p2))
2148 return 1;
2149 return 0;
2150 }
2151
2152 static bool attr_w gen_test_multiple(struct codegen_context *ctx, frame_t *variables, size_t n_variables, uint32_t label)
2153 {
2154 size_t i;
2155 size_t attr_unused pos;
2156 qsort(variables, n_variables, sizeof(frame_t), frame_t_compare);
2157
2158 #if 1
2159 for (i = 0; i < n_variables; i++) {
2160 frame_t v = variables[i];
2161 if (ctx->registers[v] >= 0) {
2162 g(unspill(ctx, v));
2163 }
2164 }
2165 #else
2166 {
2167 frame_t v;
2168 for (v = MIN_USEABLE_SLOT; v < function_n_variables(ctx->fn); v++) {
2169 if (ctx->registers[v] >= 0)
2170 g(unspill(ctx, v));
2171 }
2172 }
2173 #endif
2174
2175 if (!n_variables)
2176 goto tested;
2177 if (n_variables == 1) {
2178 g(gen_test_1(ctx, R_FRAME, variables[0], 0, label, false, TEST));
2179 goto tested;
2180 }
2181 if (n_variables == 2) {
2182 g(gen_test_2(ctx, variables[0], variables[1], label));
2183 goto tested;
2184 }
2185 #if defined(HAVE_BITWISE_FRAME)
2186 pos = 0;
2187 while (pos < n_variables) {
2188 frame_t addr = variables[pos] >> (OP_SIZE_BITMAP + 3) << OP_SIZE_BITMAP;
2189 unsigned bit = variables[pos] & ((1 << (OP_SIZE_BITMAP + 3)) - 1);
2190 uintptr_t mask = (uintptr_t)1 << bit;
2191 unsigned n_bits = 1;
2192 pos++;
2193 while (pos < n_variables) {
2194 frame_t addr2 = variables[pos] >> (OP_SIZE_BITMAP + 3) << OP_SIZE_BITMAP;
2195 unsigned bit2 = variables[pos] & ((1 << (OP_SIZE_BITMAP + 3)) - 1);
2196 uintptr_t mask2 = (uintptr_t)1 << bit2;
2197 if (addr != addr2)
2198 break;
2199 #if defined(ARCH_S390)
2200 if (!is_direct_const(mask | mask2, IMM_PURPOSE_TEST, OP_SIZE_BITMAP))
2201 break;
2202 #endif
2203 mask |= mask2;
2204 n_bits++;
2205 pos++;
2206 }
2207 if (n_bits == 1) {
2208 g(gen_test_1(ctx, R_FRAME, variables[pos - 1], 0, label, false, TEST));
2209 continue;
2210 } else if (n_bits == 2) {
2211 g(gen_test_2(ctx, variables[pos - 2], variables[pos - 1], label));
2212 continue;
2213 }
2214 #if defined(ARCH_X86)
2215 g(gen_address(ctx, R_FRAME, addr, IMM_PURPOSE_LDR_OFFSET, OP_SIZE_BITMAP));
2216 if (OP_SIZE_BITMAP == OP_SIZE_4) {
2217 g(gen_imm(ctx, (int32_t)mask, IMM_PURPOSE_TEST, OP_SIZE_BITMAP));
2218 } else {
2219 g(gen_imm(ctx, mask, IMM_PURPOSE_TEST, OP_SIZE_BITMAP));
2220 }
2221 gen_insn(INSN_TEST, OP_SIZE_BITMAP, 0, 1);
2222 gen_address_offset();
2223 gen_imm_offset();
2224
2225 gen_insn(INSN_JMP_COND, OP_SIZE_BITMAP, COND_NE, 0);
2226 gen_four(label);
2227 #else
2228 g(gen_address(ctx, R_FRAME, addr, ARCH_PREFERS_SX(OP_SIZE_BITMAP) ? IMM_PURPOSE_LDR_SX_OFFSET : IMM_PURPOSE_LDR_OFFSET, OP_SIZE_BITMAP));
2229 gen_insn(ARCH_PREFERS_SX(OP_SIZE_BITMAP) ? INSN_MOVSX : INSN_MOV, OP_SIZE_BITMAP, 0, 0);
2230 gen_one(R_SCRATCH_NA_1);
2231 gen_address_offset();
2232
2233 g(gen_cmp_test_imm_jmp(ctx, INSN_TEST, i_size(OP_SIZE_BITMAP), R_SCRATCH_NA_1, mask, COND_NE, label));
2234 #endif
2235 }
2236 goto tested;
2237 #elif !defined(HAVE_BITWISE_FRAME)
2238 #if defined(ARCH_X86)
2239 g(gen_address(ctx, R_FRAME, variables[0], IMM_PURPOSE_LDR_OFFSET, OP_SIZE_1));
2240 gen_insn(INSN_MOV, OP_SIZE_1, 0, 0);
2241 gen_one(R_SCRATCH_1);
2242 gen_address_offset();
2243
2244 for (i = 1; i < n_variables; i++) {
2245 g(gen_address(ctx, R_FRAME, variables[i], IMM_PURPOSE_LDR_OFFSET, OP_SIZE_1));
2246 gen_insn(INSN_ALU_PARTIAL, OP_SIZE_1, ALU_OR, 1);
2247 gen_one(R_SCRATCH_1);
2248 gen_one(R_SCRATCH_1);
2249 gen_address_offset();
2250 }
2251
2252 gen_insn(INSN_JMP_COND, OP_SIZE_1, COND_NE, 0);
2253 gen_four(label);
2254
2255 goto tested;
2256 #endif
2257 if (!ARCH_HAS_BWX) {
2258 pos = 0;
2259 while (pos < n_variables) {
2260 frame_t addr = variables[pos] & ~7;
2261 unsigned bit = variables[pos] & 7;
2262 unsigned mask = 1U << bit;
2263 pos++;
2264 while (pos < n_variables) {
2265 frame_t addr2 = variables[pos] & ~7;
2266 unsigned bit2 = variables[pos] & 7;
2267 unsigned mask2 = 1U << bit2;
2268
2269 if (addr != addr2)
2270 break;
2271
2272 mask |= mask2;
2273 pos++;
2274 }
2275 g(gen_address(ctx, R_FRAME, addr, IMM_PURPOSE_LDR_OFFSET, OP_SIZE_8));
2276 gen_insn(INSN_MOV, OP_SIZE_8, 0, 0);
2277 gen_one(R_SCRATCH_1);
2278 gen_address_offset();
2279
2280 g(gen_3address_alu_imm(ctx, OP_SIZE_NATIVE, ALU_ZAPNOT, R_SCRATCH_1, R_SCRATCH_1, mask));
2281
2282 g(gen_jmp_on_zero(ctx, OP_SIZE_8, R_SCRATCH_1, COND_NE, label));
2283 }
2284 goto tested;
2285 }
2286 #endif
2287 for (i = 0; i < n_variables; i++) {
2288 g(gen_test_1(ctx, R_FRAME, variables[i], 0, label, false, TEST));
2289 }
2290 goto tested;
2291 tested:
2292 return true;
2293 }
2294
2295 static bool attr_w clear_flag_cache(struct codegen_context *ctx)
2296 {
2297 memset(ctx->flag_cache, 0, function_n_variables(ctx->fn) * sizeof(int8_t));
2298 return true;
2299 }
2300
2301 /*#define clear_flag_cache(ctx) \
2302 (debug("clearing flag cache @ %d", __LINE__), memset((ctx)->flag_cache, 0, function_n_variables(ctx->fn) * sizeof(int8_t)), true)*/
2303
2304 static inline void flag_set(struct codegen_context *ctx, frame_t slot, bool val)
2305 {
2306 if (val && !must_be_flat_chicken && da(ctx->fn,function)->local_variables_flags[slot].must_be_flat) {
2307 internal(file_line, "flag_set: %s: setting slot %lu as not flat", da(ctx->fn,function)->function_name, (unsigned long)slot);
2308 }
2309 ctx->flag_cache[slot] = !val ? -1 : 1;
2310 }
2311
2312 static inline void flag_set_unknown(struct codegen_context *ctx, frame_t slot)
2313 {
2314 ctx->flag_cache[slot] = 0;
2315 }
2316
2317 static inline bool flag_must_be_flat(struct codegen_context *ctx, frame_t slot)
2318 {
2319 if (!must_be_flat_chicken && da(ctx->fn,function)->local_variables_flags[slot].must_be_flat) {
2320 if (ctx->flag_cache[slot] == 1)
2321 internal(file_line, "flag_must_be_flat: %s: must_be_flat slot %lu is not flat", da(ctx->fn,function)->function_name, (unsigned long)slot);
2322 return true;
2323 }
2324 return false;
2325 }
2326
2327 static inline bool flag_is_clear(struct codegen_context *ctx, frame_t slot)
2328 {
2329 if (!flag_cache_chicken && ctx->flag_cache[slot] == -1)
2330 return true;
2331 if (flag_must_be_flat(ctx, slot))
2332 return true;
2333 return false;
2334 }
2335
2336 static inline bool flag_is_set(struct codegen_context *ctx, frame_t slot)
2337 {
2338 if (!flag_cache_chicken && ctx->flag_cache[slot] == 1)
2339 return true;
2340 return false;
2341 }
2342
2343 static bool attr_w gen_test_1_cached(struct codegen_context *ctx, frame_t slot_1, uint32_t label)
2344 {
2345 if (flag_is_clear(ctx, slot_1))
2346 return true;
2347 return gen_test_1(ctx, R_FRAME, slot_1, 0, label, false, TEST);
2348 }
2349
2350 static bool attr_w gen_test_2_cached(struct codegen_context *ctx, frame_t slot_1, frame_t slot_2, uint32_t label)
2351 {
2352 if (flag_is_clear(ctx, slot_1))
2353 return gen_test_1_cached(ctx, slot_2, label);
2354 if (flag_is_clear(ctx, slot_2))
2355 return gen_test_1_cached(ctx, slot_1, label);
2356 return gen_test_2(ctx, slot_1, slot_2, label);
2357 }
2358
2359 static bool attr_w gen_test_1_jz_cached(struct codegen_context *ctx, frame_t slot_1, uint32_t label)
2360 {
2361 const struct type *type = get_type_of_local(ctx, slot_1);
2362 if (!TYPE_IS_FLAT(type) && !da(ctx->fn,function)->local_variables_flags[slot_1].may_be_borrowed)
2363 return true;
2364 if (flag_is_set(ctx, slot_1))
2365 return true;
2366 return gen_test_1(ctx, R_FRAME, slot_1, 0, label, true, TEST);
2367 }
2368
2369 static bool attr_w gen_frame_address(struct codegen_context *ctx, frame_t slot, int64_t offset, unsigned reg)
2370 {
2371 offset += (size_t)slot * slot_size;
2372 g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_ADDRESS), ALU_ADD, reg, R_FRAME, offset));
2373 return true;
2374 }
2375
2376 static bool attr_w gen_frame_load_raw(struct codegen_context *ctx, unsigned size, bool sx, frame_t slot, int64_t offset, unsigned reg)
2377 {
2378 if (likely(!reg_is_fp(reg)))
2379 sx |= ARCH_PREFERS_SX(size);
2380 offset += (size_t)slot * slot_size;
2381 if (!ARCH_HAS_BWX && size < OP_SIZE_4) {
2382 g(gen_address(ctx, R_FRAME, offset, reg_is_fp(reg) ? IMM_PURPOSE_VLDR_VSTR_OFFSET : IMM_PURPOSE_LDR_SX_OFFSET, OP_SIZE_4));
2383 gen_insn(INSN_MOVSX, OP_SIZE_4, 0, 0);
2384 gen_one(reg);
2385 gen_address_offset();
2386
2387 g(gen_extend(ctx, size, sx, reg, reg));
2388
2389 return true;
2390 }
2391 #if defined(ARCH_ALPHA)
2392 if (size < OP_SIZE_4) {
2393 g(gen_address(ctx, R_FRAME, offset, reg_is_fp(reg) ? IMM_PURPOSE_VLDR_VSTR_OFFSET : IMM_PURPOSE_LDR_OFFSET, size));
2394 gen_insn(INSN_MOV, size, 0, 0);
2395 gen_one(reg);
2396 gen_address_offset();
2397
2398 if (sx)
2399 g(gen_extend(ctx, size, sx, reg, reg));
2400
2401 return true;
2402 }
2403 #endif
2404 #if defined(ARCH_MIPS)
2405 if (reg_is_fp(reg) && size == OP_SIZE_8 && !MIPS_HAS_LS_DOUBLE) {
2406 #if defined(C_LITTLE_ENDIAN)
2407 g(gen_frame_load_raw(ctx, OP_SIZE_4, false, 0, offset, reg));
2408 g(gen_frame_load_raw(ctx, OP_SIZE_4, false, 0, offset + 4, reg + 1));
2409 #else
2410 g(gen_frame_load_raw(ctx, OP_SIZE_4, false, 0, offset, reg + 1));
2411 g(gen_frame_load_raw(ctx, OP_SIZE_4, false, 0, offset + 4, reg));
2412 #endif
2413 return true;
2414 }
2415 #endif
2416 #if defined(ARCH_IA64) || defined(ARCH_PARISC)
2417 if (sx) {
2418 g(gen_address(ctx, R_FRAME, offset, reg_is_fp(reg) ? IMM_PURPOSE_VLDR_VSTR_OFFSET : sx ? IMM_PURPOSE_LDR_SX_OFFSET : IMM_PURPOSE_LDR_OFFSET, size));
2419 gen_insn(INSN_MOV, size, 0, 0);
2420 gen_one(reg);
2421 gen_address_offset();
2422
2423 g(gen_extend(ctx, size, sx, reg, reg));
2424
2425 return true;
2426 }
2427 #endif
2428 #if defined(ARCH_POWER)
2429 if (size == OP_SIZE_1 && sx) {
2430 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDR_OFFSET, size));
2431 gen_insn(INSN_MOV, size, 0, 0);
2432 gen_one(reg);
2433 gen_address_offset();
2434
2435 g(gen_extend(ctx, size, sx, reg, reg));
2436
2437 return true;
2438 }
2439 #endif
2440 #if defined(ARCH_S390)
2441 if (size == OP_SIZE_1 && !cpu_test_feature(CPU_FEATURE_long_displacement)) {
2442 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDR_OFFSET, size));
2443 gen_insn(INSN_MOV_MASK, OP_SIZE_NATIVE, MOV_MASK_0_8, 0);
2444 gen_one(reg);
2445 gen_one(reg);
2446 gen_address_offset();
2447
2448 g(gen_extend(ctx, size, sx, reg, reg));
2449
2450 return true;
2451 }
2452 if (size == OP_SIZE_16 && reg_is_fp(reg)) {
2453 g(gen_frame_load_raw(ctx, OP_SIZE_8, false, 0, offset, reg));
2454 g(gen_frame_load_raw(ctx, OP_SIZE_8, false, 0, offset + 8, reg + 2));
2455
2456 return true;
2457 }
2458 #endif
2459 g(gen_address(ctx, R_FRAME, offset, reg_is_fp(reg) ? IMM_PURPOSE_VLDR_VSTR_OFFSET : sx ? IMM_PURPOSE_LDR_SX_OFFSET : IMM_PURPOSE_LDR_OFFSET, size));
2460 gen_insn(unlikely(sx) ? INSN_MOVSX : INSN_MOV, size, 0, 0);
2461 gen_one(reg);
2462 gen_address_offset();
2463
2464 return true;
2465 }
2466
2467 static bool attr_w gen_frame_load(struct codegen_context *ctx, unsigned size, bool sx, frame_t slot, int64_t offset, unsigned reg)
2468 {
2469 ajla_assert_lo(slot >= MIN_USEABLE_SLOT && slot < function_n_variables(ctx->fn), (file_line, "gen_frame_load: invalid slot: %lu >= %lu", (unsigned long)slot, (unsigned long)function_n_variables(ctx->fn)));
2470 if (ctx->registers[slot] >= 0) {
2471 if (unlikely(offset != 0))
2472 internal(file_line, "gen_frame_load: offset is non-zero: %"PRIdMAX"", (intmax_t)offset);
2473 if (sx && !ARCH_PREFERS_SX(size)) {
2474 g(gen_extend(ctx, size, true, reg, ctx->registers[slot]));
2475 return true;
2476 }
2477 if (reg != (unsigned)ctx->registers[slot]) {
2478 gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
2479 gen_one(reg);
2480 gen_one(ctx->registers[slot]);
2481 }
2482 return true;
2483 }
2484
2485 return gen_frame_load_raw(ctx, size, sx, slot, offset, reg);
2486 }
2487
2488 #if defined(ARCH_X86)
2489 static bool attr_w gen_frame_load_x87(struct codegen_context *ctx, unsigned insn, unsigned size, unsigned alu, frame_t slot)
2490 {
2491 g(gen_address(ctx, R_FRAME, (size_t)slot * slot_size, IMM_PURPOSE_LDR_OFFSET, size));
2492 gen_insn(insn, size, alu, 0);
2493 gen_address_offset();
2494 return true;
2495 }
2496
2497 static bool attr_w gen_frame_store_x87(struct codegen_context *ctx, unsigned insn, unsigned size, frame_t slot)
2498 {
2499 g(gen_address(ctx, R_FRAME, (size_t)slot * slot_size, IMM_PURPOSE_STR_OFFSET, size));
2500 gen_insn(insn, size, 0, 0);
2501 gen_address_offset();
2502 return true;
2503 }
2504 #endif
2505
2506 static bool attr_w gen_frame_load_op(struct codegen_context *ctx, unsigned size, bool attr_unused sx, unsigned alu, unsigned writes_flags, frame_t slot, int64_t offset, unsigned reg)
2507 {
2508 ajla_assert_lo(slot >= MIN_USEABLE_SLOT && slot < function_n_variables(ctx->fn), (file_line, "gen_frame_load_op: invalid slot: %lu >= %lu", (unsigned long)slot, (unsigned long)function_n_variables(ctx->fn)));
2509 if (ctx->registers[slot] >= 0) {
2510 gen_insn(INSN_ALU + ARCH_PARTIAL_ALU(i_size(size)), i_size(size), alu, ALU_WRITES_FLAGS(alu, false) | writes_flags);
2511 gen_one(reg);
2512 gen_one(reg);
2513 gen_one(ctx->registers[slot]);
2514 return true;
2515 }
2516 #if defined(ARCH_X86) || defined(ARCH_S390)
2517 #if defined(ARCH_S390)
2518 if (size >= OP_SIZE_4)
2519 #endif
2520 {
2521 offset += (size_t)slot * slot_size;
2522 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDR_OFFSET, size));
2523 gen_insn(INSN_ALU + ARCH_PARTIAL_ALU(size), size, alu, (alu == ALU_MUL ? ALU_WRITES_FLAGS(alu, false) : 1) | writes_flags);
2524 gen_one(reg);
2525 gen_one(reg);
2526 gen_address_offset();
2527 return true;
2528 }
2529 #endif
2530 #if !defined(ARCH_X86)
2531 g(gen_frame_load(ctx, size, sx, slot, offset, R_SCRATCH_NA_1));
2532 gen_insn(INSN_ALU + ARCH_PARTIAL_ALU(i_size(size)), i_size(size), alu, ALU_WRITES_FLAGS(alu, false) | writes_flags);
2533 gen_one(reg);
2534 gen_one(reg);
2535 gen_one(R_SCRATCH_NA_1);
2536 return true;
2537 #endif
2538 }
2539
2540 static bool attr_w attr_unused gen_frame_load_op1(struct codegen_context *ctx, unsigned size, unsigned alu, unsigned writes_flags, frame_t slot, int64_t offset, unsigned reg)
2541 {
2542 ajla_assert_lo(slot >= MIN_USEABLE_SLOT && slot < function_n_variables(ctx->fn), (file_line, "gen_frame_load_op1: invalid slot: %lu >= %lu", (unsigned long)slot, (unsigned long)function_n_variables(ctx->fn)));
2543 if (ctx->registers[slot] >= 0) {
2544 gen_insn(INSN_ALU1 + ARCH_PARTIAL_ALU(size), size, alu, ALU1_WRITES_FLAGS(alu) | writes_flags);
2545 gen_one(reg);
2546 gen_one(ctx->registers[slot]);
2547 return true;
2548 }
2549 #if defined(ARCH_X86)
2550 offset += (size_t)slot * slot_size;
2551 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDR_OFFSET, size));
2552 gen_insn(INSN_ALU1 + ARCH_PARTIAL_ALU(size), size, alu, ALU1_WRITES_FLAGS(alu) | writes_flags);
2553 gen_one(reg);
2554 gen_address_offset();
2555 return true;
2556 #endif
2557 #if !defined(ARCH_X86)
2558 g(gen_frame_load(ctx, size, false, slot, offset, reg));
2559 gen_insn(INSN_ALU1 + ARCH_PARTIAL_ALU(size), size, alu, ALU1_WRITES_FLAGS(alu) | writes_flags);
2560 gen_one(reg);
2561 gen_one(reg);
2562 return true;
2563 #endif
2564 }
2565
2566 #if ARCH_HAS_FLAGS
2567 static bool attr_w gen_frame_load_cmp(struct codegen_context *ctx, unsigned size, bool logical, bool attr_unused sx, bool swap, frame_t slot, int64_t offset, unsigned reg)
2568 {
2569 if (ctx->registers[slot] >= 0) {
2570 #if defined(ARCH_X86)
2571 gen_insn(INSN_CMP, size, 0, 1 + logical);
2572 #else
2573 gen_insn(INSN_CMP, maximum(size, OP_SIZE_4), 0, 1 + logical);
2574 #endif
2575 if (!swap) {
2576 gen_one(reg);
2577 gen_one(ctx->registers[slot]);
2578 } else {
2579 gen_one(ctx->registers[slot]);
2580 gen_one(reg);
2581 }
2582 return true;
2583 }
2584 #if defined(ARCH_S390) || defined(ARCH_X86)
2585 #if defined(ARCH_S390)
2586 if (size < OP_SIZE_4)
2587 goto no_load_op;
2588 #endif
2589 offset += (size_t)slot * slot_size;
2590 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDR_OFFSET, size));
2591 gen_insn(INSN_CMP, size, 0, 1 + logical);
2592 if (!swap) {
2593 gen_one(reg);
2594 gen_address_offset();
2595 } else {
2596 gen_address_offset();
2597 gen_one(reg);
2598 }
2599 return true;
2600 #endif
2601 #if defined(R_SCRATCH_NA_1)
2602 goto no_load_op;
2603 no_load_op:
2604 g(gen_frame_load(ctx, size, sx, slot, offset, R_SCRATCH_NA_1));
2605 gen_insn(INSN_CMP, maximum(size, OP_SIZE_4), 0, 1 + logical);
2606 if (!swap) {
2607 gen_one(reg);
2608 gen_one(R_SCRATCH_NA_1);
2609 } else {
2610 gen_one(R_SCRATCH_NA_1);
2611 gen_one(reg);
2612 }
2613 return true;
2614 #endif
2615 }
2616
2617 static bool attr_w gen_frame_load_cmp_imm(struct codegen_context *ctx, unsigned size, bool logical, bool attr_unused sx, frame_t slot, int64_t offset, int64_t value)
2618 {
2619 if (ctx->registers[slot] >= 0) {
2620 g(gen_imm(ctx, value, IMM_PURPOSE_CMP, size));
2621 gen_insn(INSN_CMP, i_size(size), 0, 1 + logical);
2622 gen_one(ctx->registers[slot]);
2623 gen_imm_offset();
2624 return true;
2625 }
2626 #if defined(ARCH_S390) || defined(ARCH_X86)
2627 #if defined(ARCH_S390)
2628 if (size != OP_SIZE_1 || !logical)
2629 goto no_load_op;
2630 #endif
2631 offset += (size_t)slot * slot_size;
2632 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_MVI_CLI_OFFSET, size));
2633 g(gen_imm(ctx, value, IMM_PURPOSE_CMP, size));
2634 gen_insn(INSN_CMP, size, 0, 1 + logical);
2635 gen_address_offset();
2636 gen_imm_offset();
2637 return true;
2638 #endif
2639 #if defined(R_SCRATCH_NA_1)
2640 goto no_load_op;
2641 no_load_op:
2642 g(gen_frame_load(ctx, size, sx, slot, offset, R_SCRATCH_NA_1));
2643 g(gen_imm(ctx, value, IMM_PURPOSE_CMP, size));
2644 gen_insn(INSN_CMP, i_size(size), 0, 1 + logical);
2645 gen_one(R_SCRATCH_NA_1);
2646 gen_imm_offset();
2647 return true;
2648 #endif
2649 }
2650 #endif
2651
2652 static bool attr_w gen_frame_load_2(struct codegen_context *ctx, unsigned size, frame_t slot, int64_t offset, unsigned reg1, unsigned reg2)
2653 {
2654 #if defined(ARCH_ARM64)
2655 offset += (size_t)slot * slot_size;
2656 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDP_STP_OFFSET, size));
2657 gen_insn(INSN_LDP, size, 0, 0);
2658 gen_one(reg1);
2659 gen_one(reg2);
2660 gen_address_offset();
2661 return true;
2662 #endif
2663 #if defined(ARCH_ARM32)
2664 if (likely(!(reg1 & 1)) && likely(reg2 == reg1 + 1) && likely(cpu_test_feature(CPU_FEATURE_armv6)))
2665 #elif defined(ARCH_SPARC32)
2666 if (likely(!(reg2 & 1)) && likely(reg1 == reg2 + 1))
2667 #elif defined(ARCH_S390)
2668 if (likely(reg1 == reg2 + 1))
2669 #else
2670 if (0)
2671 #endif
2672 {
2673 offset += (size_t)slot * slot_size;
2674 if (UNALIGNED_TRAP) {
2675 if (unlikely((offset & ((2U << size) - 1)) != 0)) {
2676 offset -= (size_t)slot * slot_size;
2677 goto skip_ldd;
2678 }
2679 }
2680 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDP_STP_OFFSET, size));
2681 gen_insn(INSN_LDP, size, 0, 0);
2682 gen_one(reg1);
2683 gen_one(reg2);
2684 gen_address_offset();
2685 return true;
2686 }
2687 goto skip_ldd;
2688 skip_ldd:
2689 g(gen_frame_load(ctx, size, false, slot, offset + lo_word(size), reg1));
2690 g(gen_frame_load(ctx, size, false, slot, offset + hi_word(size), reg2));
2691 return true;
2692 }
2693
2694 static bool attr_w gen_frame_store_raw(struct codegen_context *ctx, unsigned size, frame_t slot, int64_t offset, unsigned reg)
2695 {
2696 offset += (size_t)slot * slot_size;
2697 if (!ARCH_HAS_BWX)
2698 size = maximum(OP_SIZE_4, size);
2699 #if defined(ARCH_MIPS)
2700 if (reg_is_fp(reg) && size == OP_SIZE_8 && !MIPS_HAS_LS_DOUBLE) {
2701 #if defined(C_LITTLE_ENDIAN)
2702 g(gen_frame_store_raw(ctx, OP_SIZE_4, 0, offset, reg));
2703 g(gen_frame_store_raw(ctx, OP_SIZE_4, 0, offset + 4, reg + 1));
2704 #else
2705 g(gen_frame_store_raw(ctx, OP_SIZE_4, 0, offset, reg + 1));
2706 g(gen_frame_store_raw(ctx, OP_SIZE_4, 0, offset + 4, reg));
2707 #endif
2708 return true;
2709 }
2710 #endif
2711 #if defined(ARCH_S390)
2712 if (size == OP_SIZE_16 && reg_is_fp(reg)) {
2713 g(gen_frame_store_raw(ctx, OP_SIZE_8, 0, offset, reg));
2714 g(gen_frame_store_raw(ctx, OP_SIZE_8, 0, offset + 8, reg + 2));
2715 return true;
2716 }
2717 #endif
2718 g(gen_address(ctx, R_FRAME, offset, reg_is_fp(reg) ? IMM_PURPOSE_VLDR_VSTR_OFFSET : IMM_PURPOSE_STR_OFFSET, size));
2719 gen_insn(INSN_MOV, size, 0, 0);
2720 gen_address_offset();
2721 gen_one(reg);
2722 return true;
2723 }
2724
2725 static bool attr_w gen_frame_store(struct codegen_context *ctx, unsigned size, frame_t slot, int64_t offset, unsigned reg)
2726 {
2727 ajla_assert_lo(slot >= MIN_USEABLE_SLOT && slot < function_n_variables(ctx->fn), (file_line, "gen_frame_store: invalid slot: %lu >= %lu", (unsigned long)slot, (unsigned long)function_n_variables(ctx->fn)));
2728 if (ctx->registers[slot] >= 0) {
2729 if (unlikely(offset != 0))
2730 internal(file_line, "gen_frame_store: offset is non-zero: %"PRIdMAX"", (intmax_t)offset);
2731 if (reg != (unsigned)ctx->registers[slot]) {
2732 gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
2733 gen_one(ctx->registers[slot]);
2734 gen_one(reg);
2735 }
2736 return true;
2737 }
2738 return gen_frame_store_raw(ctx, size, slot, offset, reg);
2739 }
2740
2741 static bool attr_w gen_frame_store_2(struct codegen_context *ctx, unsigned size, frame_t slot, int64_t offset, unsigned reg1, unsigned reg2)
2742 {
2743 #if defined(ARCH_ARM64)
2744 offset += (size_t)slot * slot_size;
2745 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDP_STP_OFFSET, size));
2746 gen_insn(INSN_STP, size, 0, 0);
2747 gen_address_offset();
2748 gen_one(reg1);
2749 gen_one(reg2);
2750 return true;
2751 #endif
2752 #if defined(ARCH_ARM32)
2753 if (likely(!(reg1 & 1)) && likely(reg2 == reg1 + 1) && likely(cpu_test_feature(CPU_FEATURE_armv6)))
2754 #elif defined(ARCH_SPARC32)
2755 if (likely(!(reg2 & 1)) && likely(reg1 == reg2 + 1))
2756 #elif defined(ARCH_S390)
2757 if (likely(reg1 == reg2 + 1))
2758 #else
2759 if (0)
2760 #endif
2761 {
2762 offset += (size_t)slot * slot_size;
2763 if (UNALIGNED_TRAP) {
2764 if (unlikely((offset & ((2U << size) - 1)) != 0)) {
2765 offset -= (size_t)slot * slot_size;
2766 goto skip_ldd;
2767 }
2768 }
2769 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDP_STP_OFFSET, size));
2770 gen_insn(INSN_STP, size, 0, 0);
2771 gen_address_offset();
2772 gen_one(reg1);
2773 gen_one(reg2);
2774 return true;
2775 }
2776 goto skip_ldd;
2777 skip_ldd:
2778 g(gen_frame_store(ctx, size, slot, offset + lo_word(size), reg1));
2779 g(gen_frame_store(ctx, size, slot, offset + hi_word(size), reg2));
2780 return true;
2781 }
2782
2783 static bool attr_w gen_frame_store_imm_raw(struct codegen_context *ctx, unsigned size, frame_t slot, int64_t offset, int64_t imm)
2784 {
2785 offset += (size_t)slot * slot_size;
2786 if (!ARCH_HAS_BWX)
2787 size = maximum(OP_SIZE_4, size);
2788 g(gen_address(ctx, R_FRAME, offset, size == OP_SIZE_1 ? IMM_PURPOSE_MVI_CLI_OFFSET : IMM_PURPOSE_STR_OFFSET, size));
2789 g(gen_imm(ctx, imm, IMM_PURPOSE_STORE_VALUE, size));
2790 gen_insn(INSN_MOV, size, 0, 0);
2791 gen_address_offset();
2792 gen_imm_offset();
2793 return true;
2794 }
2795
2796 static bool attr_w gen_frame_store_imm(struct codegen_context *ctx, unsigned size, frame_t slot, int64_t offset, int64_t imm)
2797 {
2798 ajla_assert_lo(slot >= MIN_USEABLE_SLOT && slot < function_n_variables(ctx->fn), (file_line, "gen_frame_store_imm: invalid slot: %lu >= %lu", (unsigned long)slot, (unsigned long)function_n_variables(ctx->fn)));
2799 if (ctx->registers[slot] >= 0) {
2800 if (unlikely(offset != 0))
2801 internal(file_line, "gen_frame_store_imm: offset is non-zero: %"PRIdMAX"", (intmax_t)offset);
2802 g(gen_load_constant(ctx, ctx->registers[slot], imm));
2803 return true;
2804 }
2805 return gen_frame_store_imm_raw(ctx, size, slot, offset, imm);
2806 }
2807
2808 static bool attr_w gen_frame_clear_raw(struct codegen_context *ctx, unsigned size, frame_t slot)
2809 {
2810 g(gen_frame_store_imm_raw(ctx, size, slot, 0, 0));
2811 return true;
2812 }
2813
2814 static bool attr_w gen_frame_clear(struct codegen_context *ctx, unsigned size, frame_t slot)
2815 {
2816 g(gen_frame_store_imm(ctx, size, slot, 0, 0));
2817 return true;
2818 }
2819
2820 #if defined(POINTER_COMPRESSION)
2821 #define POINTER_THUNK_BIT 0
2822 #elif defined(POINTER_IGNORE_START)
2823 #define POINTER_THUNK_BIT POINTER_IGNORE_TOP_BIT
2824 #elif defined(POINTER_TAG)
2825 #define POINTER_THUNK_BIT POINTER_TAG_BIT
2826 #else
2827 unsupported pointer mode
2828 #endif
2829
2830 static bool attr_w gen_ptr_is_thunk(struct codegen_context *ctx, unsigned reg, bool jnz, uint32_t label)
2831 {
2832 #if defined(ARCH_X86)
2833 if (POINTER_THUNK_BIT < 8
2834 #if defined(ARCH_X86_32)
2835 && reg < 4
2836 #endif
2837 ) {
2838 g(gen_cmp_test_imm_jmp(ctx, INSN_TEST, OP_SIZE_1, reg, (uint64_t)1 << POINTER_THUNK_BIT, jnz ? COND_NE : COND_E, label));
2839 } else
2840 #endif
2841 {
2842 g(gen_cmp_test_imm_jmp(ctx, INSN_TEST, OP_SIZE_SLOT, reg, (uint64_t)1 << POINTER_THUNK_BIT, jnz ? COND_NE : COND_E, label));
2843 }
2844 return true;
2845 }
2846
2847 static bool attr_w gen_barrier(struct codegen_context *ctx)
2848 {
2849 if (ARCH_NEEDS_BARRIER)
2850 gen_insn(INSN_MB, 0, 0, 0);
2851 return true;
2852 }
2853
2854 static bool attr_w gen_compare_refcount(struct codegen_context *ctx, unsigned ptr, unsigned val, unsigned cond, uint32_t label)
2855 {
2856 unsigned op_size = log_2(sizeof(refcount_int_t));
2857 #if defined(ARCH_X86)
2858 bool logical = COND_IS_LOGICAL(cond);
2859 g(gen_address(ctx, ptr, offsetof(struct data, refcount_), IMM_PURPOSE_LDR_OFFSET, op_size));
2860 g(gen_imm(ctx, val, IMM_PURPOSE_CMP, op_size));
2861 gen_insn(INSN_CMP, op_size, 0, 1 + logical);
2862 gen_address_offset();
2863 gen_imm_offset();
2864
2865 gen_insn(!logical ? INSN_JMP_COND : INSN_JMP_COND_LOGICAL, op_size, cond, 0);
2866 gen_four(label);
2867 #else
2868 g(gen_address(ctx, ptr, offsetof(struct data, refcount_), IMM_PURPOSE_LDR_OFFSET, op_size));
2869 gen_insn(INSN_MOV, op_size, 0, 0);
2870 gen_one(R_SCRATCH_2);
2871 gen_address_offset();
2872
2873 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, op_size, R_SCRATCH_2, val, cond, label));
2874 #endif
2875 return true;
2876 }
2877
2878 static bool attr_w gen_compare_ptr_tag(struct codegen_context *ctx, unsigned reg, unsigned tag, unsigned cond, uint32_t label, unsigned tmp_reg)
2879 {
2880 #if defined(DATA_TAG_AT_ALLOC)
2881 g(gen_3address_rot_imm(ctx, OP_SIZE_ADDRESS, ROT_SHR, tmp_reg, reg, POINTER_IGNORE_START, false));
2882 #elif defined(REFCOUNT_TAG)
2883 #if REFCOUNT_STEP == 256 && defined(C_LITTLE_ENDIAN) && !defined(ARCH_ALPHA)
2884 #if defined(ARCH_X86)
2885 g(gen_imm(ctx, tag, IMM_PURPOSE_CMP, OP_SIZE_4));
2886 gen_insn(INSN_CMP, OP_SIZE_1, 0, 1);
2887 gen_one(ARG_ADDRESS_1);
2888 gen_one(reg);
2889 gen_eight(offsetof(struct data, refcount_));
2890 gen_imm_offset();
2891
2892 gen_insn(INSN_JMP_COND, OP_SIZE_1, cond, 0);
2893 gen_four(label);
2894 return true;
2895 #else
2896 gen_insn(INSN_MOV, OP_SIZE_1, 0, 0);
2897 gen_one(tmp_reg);
2898 gen_one(ARG_ADDRESS_1);
2899 gen_one(reg);
2900 gen_eight(offsetof(struct data, refcount_));
2901 #endif
2902 #else
2903 gen_insn(INSN_MOV, log_2(sizeof(refcount_int_t)), 0, 0);
2904 gen_one(tmp_reg);
2905 gen_one(ARG_ADDRESS_1);
2906 gen_one(reg);
2907 gen_eight(offsetof(struct data, refcount_));
2908
2909 g(gen_3address_alu_imm(ctx, log_2(sizeof(refcount_int_t)), ALU_AND, tmp_reg, tmp_reg, REFCOUNT_STEP - 1));
2910 #endif
2911 #else
2912 #if defined(ARCH_S390)
2913 if (sizeof(tag_t) == 1 && !cpu_test_feature(CPU_FEATURE_long_displacement)) {
2914 g(gen_address(ctx, reg, offsetof(struct data, tag), IMM_PURPOSE_LDR_OFFSET, log_2(sizeof(tag_t))));
2915 gen_insn(INSN_MOV_MASK, OP_SIZE_NATIVE, MOV_MASK_0_8, 0);
2916 gen_one(tmp_reg);
2917 gen_one(tmp_reg);
2918 gen_address_offset();
2919
2920 g(gen_extend(ctx, log_2(sizeof(tag_t)), false, tmp_reg, tmp_reg));
2921 } else
2922 #endif
2923 {
2924 g(gen_address(ctx, reg, offsetof(struct data, tag), IMM_PURPOSE_LDR_OFFSET, log_2(sizeof(tag_t))));
2925 gen_insn(INSN_MOV, log_2(sizeof(tag_t)), 0, 0);
2926 gen_one(tmp_reg);
2927 gen_address_offset();
2928 }
2929 #endif
2930 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, i_size(OP_SIZE_4), tmp_reg, tag, cond, label));
2931 return true;
2932 }
2933
2934 static bool attr_w gen_compare_da_tag(struct codegen_context *ctx, unsigned reg, unsigned tag, unsigned cond, uint32_t label, unsigned tmp_reg)
2935 {
2936 #if defined(POINTER_COMPRESSION)
2937 #if defined(ARCH_X86) && POINTER_COMPRESSION <= 3 && defined(REFCOUNT_TAG) && REFCOUNT_STEP == 256 && defined(C_LITTLE_ENDIAN)
2938 g(gen_imm(ctx, tag, IMM_PURPOSE_CMP, log_2(sizeof(tag_t))));
2939 gen_insn(INSN_CMP, log_2(sizeof(tag_t)), 0, 0);
2940 gen_one(ARG_ADDRESS_1 + POINTER_COMPRESSION);
2941 gen_one(reg);
2942 gen_eight(offsetof(struct data, refcount_));
2943 gen_imm_offset();
2944
2945 gen_insn(INSN_JMP_COND, OP_SIZE_4, cond, 0);
2946 gen_four(label);
2947
2948 return true;
2949 #endif
2950 if (ARCH_PREFERS_SX(OP_SIZE_4)) {
2951 g(gen_extend(ctx, OP_SIZE_4, false, tmp_reg, reg));
2952
2953 g(gen_3address_rot_imm(ctx, OP_SIZE_ADDRESS, ROT_SHL, tmp_reg, tmp_reg, POINTER_COMPRESSION, false));
2954 } else {
2955 g(gen_3address_rot_imm(ctx, OP_SIZE_ADDRESS, ROT_SHL, tmp_reg, reg, POINTER_COMPRESSION, false));
2956 }
2957 g(gen_compare_ptr_tag(ctx, tmp_reg, tag, cond, label, tmp_reg));
2958 return true;
2959 #endif
2960 g(gen_compare_ptr_tag(ctx, reg, tag, cond, label, tmp_reg));
2961 return true;
2962 }
2963
2964 static bool attr_w gen_compare_tag_and_refcount(struct codegen_context *ctx, unsigned reg, unsigned tag, uint32_t label, unsigned attr_unused tmp_reg)
2965 {
2966 #if defined(REFCOUNT_TAG)
2967 g(gen_compare_refcount(ctx, reg, tag, COND_NE, label));
2968 #else
2969 g(gen_compare_ptr_tag(ctx, reg, tag, COND_NE, label, tmp_reg));
2970 g(gen_compare_refcount(ctx, reg, REFCOUNT_STEP, COND_AE, label));
2971 #endif
2972 return true;
2973 }
2974
2975 static bool attr_w gen_decompress_pointer(struct codegen_context *ctx, unsigned reg, int64_t offset)
2976 {
2977 #ifdef POINTER_COMPRESSION
2978 #if defined(ARCH_X86) && POINTER_COMPRESSION <= 3
2979 if (offset) {
2980 g(gen_imm(ctx, offset, IMM_PURPOSE_ADD, i_size(OP_SIZE_ADDRESS)));
2981 gen_insn(INSN_ALU, i_size(OP_SIZE_ADDRESS), ALU_ADD, ALU_WRITES_FLAGS(ALU_ADD, is_imm()));
2982 gen_one(reg);
2983 gen_one(ARG_SHIFTED_REGISTER);
2984 gen_one(ARG_SHIFT_LSL | POINTER_COMPRESSION);
2985 gen_one(reg);
2986 gen_imm_offset();
2987 return true;
2988 }
2989 #endif
2990 if (ARCH_PREFERS_SX(OP_SIZE_4))
2991 g(gen_extend(ctx, OP_SIZE_4, false, reg, reg));
2992 g(gen_3address_rot_imm(ctx, OP_SIZE_ADDRESS, ROT_SHL, reg, reg, POINTER_COMPRESSION, false));
2993 #endif
2994 if (offset)
2995 g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_ADDRESS), ALU_ADD, reg, reg, offset));
2996 return true;
2997 }
2998
2999 static bool attr_w gen_compress_pointer(struct codegen_context attr_unused *ctx, unsigned attr_unused reg)
3000 {
3001 #ifdef POINTER_COMPRESSION
3002 g(gen_3address_rot_imm(ctx, OP_SIZE_ADDRESS, ROT_SHR, reg, reg, POINTER_COMPRESSION, false));
3003 #endif
3004 return true;
3005 }
3006
3007 static bool attr_w gen_frame_get_pointer(struct codegen_context *ctx, frame_t slot, bool deref, unsigned dest)
3008 {
3009 if (!deref) {
3010 g(gen_upcall_start(ctx, 1));
3011 g(gen_frame_load(ctx, OP_SIZE_SLOT, false, slot, 0, R_ARG0));
3012 g(gen_upcall_argument(ctx, 0));
3013 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_pointer_reference_owned), 1));
3014 g(gen_frame_load(ctx, OP_SIZE_SLOT, false, slot, 0, dest));
3015 } else if (!da(ctx->fn,function)->local_variables_flags[slot].may_be_borrowed) {
3016 g(gen_frame_load(ctx, OP_SIZE_SLOT, false, slot, 0, dest));
3017 g(gen_set_1(ctx, R_FRAME, slot, 0, false));
3018 flag_set(ctx, slot, false);
3019 } else {
3020 uint32_t skip_label;
3021 skip_label = alloc_label(ctx);
3022 if (unlikely(!skip_label))
3023 return false;
3024 if (flag_is_set(ctx, slot)) {
3025 g(gen_set_1(ctx, R_FRAME, slot, 0, false));
3026 goto move_it;
3027 }
3028 if (flag_is_clear(ctx, slot))
3029 goto do_reference;
3030 g(gen_test_1(ctx, R_FRAME, slot, 0, skip_label, false, TEST_CLEAR));
3031 do_reference:
3032 g(gen_upcall_start(ctx, 1));
3033 g(gen_frame_load(ctx, OP_SIZE_SLOT, false, slot, 0, R_ARG0));
3034 g(gen_upcall_argument(ctx, 0));
3035 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_pointer_reference_owned), 1));
3036 move_it:
3037 gen_label(skip_label);
3038 g(gen_frame_load(ctx, OP_SIZE_SLOT, false, slot, 0, dest));
3039 g(gen_frame_clear(ctx, OP_SIZE_SLOT, slot));
3040 flag_set(ctx, slot, false);
3041 }
3042 return true;
3043 }
3044
3045 static bool attr_w gen_frame_set_pointer(struct codegen_context *ctx, frame_t slot, unsigned src)
3046 {
3047 g(gen_set_1(ctx, R_FRAME, slot, 0, true));
3048 flag_set(ctx, slot, true);
3049 g(gen_frame_store(ctx, OP_SIZE_SLOT, slot, 0, src));
3050 return true;
3051 }
3052
3053 static bool attr_w gen_alu_upcall(struct codegen_context *ctx, size_t upcall, frame_t slot_1, frame_t slot_2, frame_t slot_r, uint32_t label_ovf)
3054 {
3055 if (ctx->registers[slot_1] >= 0)
3056 g(spill(ctx, slot_1));
3057 if (slot_2 != NO_FRAME_T && ctx->registers[slot_2] >= 0)
3058 g(spill(ctx, slot_2));
3059 g(gen_upcall_start(ctx, slot_2 != NO_FRAME_T ? 3 : 2));
3060 g(gen_frame_address(ctx, slot_1, 0, R_ARG0));
3061 g(gen_upcall_argument(ctx, 0));
3062 if (slot_2 != NO_FRAME_T) {
3063 g(gen_frame_address(ctx, slot_2, 0, R_ARG1));
3064 g(gen_upcall_argument(ctx, 1));
3065 g(gen_frame_address(ctx, slot_r, 0, R_ARG2));
3066 g(gen_upcall_argument(ctx, 2));
3067 g(gen_upcall(ctx, upcall, 3));
3068 } else {
3069 g(gen_frame_address(ctx, slot_r, 0, R_ARG1));
3070 g(gen_upcall_argument(ctx, 1));
3071 g(gen_upcall(ctx, upcall, 2));
3072 }
3073 if (label_ovf)
3074 g(gen_jmp_on_zero(ctx, OP_SIZE_1, R_RET0, COND_E, label_ovf));
3075 if (ctx->registers[slot_r] >= 0)
3076 g(unspill(ctx, slot_r));
3077 return true;
3078 }
3079
3080 static bool attr_w gen_alu_typed_upcall(struct codegen_context *ctx, size_t upcall, unsigned op_size, frame_t slot_1, frame_t slot_2, frame_t slot_r, uint32_t label_ovf)
3081 {
3082 upcall += op_size * sizeof(void (*)(void));
3083 return gen_alu_upcall(ctx, upcall, slot_1, slot_2, slot_r, label_ovf);
3084 }
3085
3086 #if defined(ARCH_X86)
3087 static bool attr_w gen_frame_set_cond(struct codegen_context *ctx, unsigned attr_unused size, bool attr_unused logical, unsigned cond, frame_t slot)
3088 {
3089 size_t offset;
3090 if (ctx->registers[slot] >= 0) {
3091 if (sizeof(ajla_flat_option_t) > 1) {
3092 gen_insn(INSN_MOV, OP_SIZE_4, 0, 0);
3093 gen_one(ctx->registers[slot]);
3094 gen_one(ARG_IMM);
3095 gen_eight(0);
3096 }
3097 gen_insn(INSN_SET_COND_PARTIAL, OP_SIZE_1, cond, 0);
3098 gen_one(ctx->registers[slot]);
3099 gen_one(ctx->registers[slot]);
3100 }
3101 offset = (size_t)slot * slot_size;
3102 if (sizeof(ajla_flat_option_t) > 1) {
3103 gen_insn(INSN_MOV, OP_SIZE_4, 0, 0);
3104 gen_one(R_SCRATCH_1);
3105 gen_one(ARG_IMM);
3106 gen_eight(0);
3107
3108 gen_insn(INSN_SET_COND, OP_SIZE_1, cond, 0);
3109 gen_one(R_SCRATCH_1);
3110
3111 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot, 0, R_SCRATCH_1));
3112 } else {
3113 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_STR_OFFSET, OP_SIZE_1));
3114 gen_insn(INSN_SET_COND, OP_SIZE_1, cond, 0);
3115 gen_address_offset();
3116 }
3117 return true;
3118 }
3119 #elif defined(ARCH_ARM64)
3120 static bool attr_w gen_frame_set_cond(struct codegen_context *ctx, unsigned attr_unused size, bool attr_unused logical, unsigned cond, frame_t slot)
3121 {
3122 if (ctx->registers[slot] >= 0) {
3123 gen_insn(INSN_SET_COND, OP_SIZE_4, cond, 0);
3124 gen_one(ctx->registers[slot]);
3125 } else {
3126 gen_insn(INSN_SET_COND, OP_SIZE_4, cond, 0);
3127 gen_one(R_SCRATCH_1);
3128 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot, 0, R_SCRATCH_1));
3129 }
3130 return true;
3131 }
3132 #elif ARCH_HAS_FLAGS
3133 static bool attr_w gen_frame_set_cond(struct codegen_context *ctx, unsigned size, bool logical, unsigned cond, frame_t slot)
3134 {
3135 #if defined(ARCH_POWER)
3136 if (!cpu_test_feature(CPU_FEATURE_v203))
3137 #elif defined(ARCH_S390)
3138 if (!cpu_test_feature(CPU_FEATURE_misc_45))
3139 #elif defined(ARCH_SPARC32)
3140 if (!SPARC_9)
3141 #else
3142 if (0)
3143 #endif
3144 {
3145 uint32_t label;
3146 g(gen_load_constant(ctx, R_SCRATCH_1, 1));
3147 label = alloc_label(ctx);
3148 if (unlikely(!label))
3149 return false;
3150 gen_insn(!logical ? INSN_JMP_COND : INSN_JMP_COND_LOGICAL, i_size(size), cond, 0);
3151 gen_four(label);
3152 g(gen_load_constant(ctx, R_SCRATCH_1, 0));
3153 gen_label(label);
3154 goto do_store;
3155 }
3156 g(gen_load_constant(ctx, R_SCRATCH_1, 1));
3157 g(gen_imm(ctx, 0, IMM_PURPOSE_CMOV, OP_SIZE_NATIVE));
3158 if (cond & COND_FP) {
3159 gen_insn(INSN_CMOV, OP_SIZE_NATIVE, cond ^ 1, 0);
3160 } else {
3161 #if defined(ARCH_S390)
3162 gen_insn(logical ? INSN_CMOV_XCC : INSN_CMOV, OP_SIZE_NATIVE, cond ^ 1, 0);
3163 #else
3164 gen_insn(size == OP_SIZE_8 ? INSN_CMOV_XCC : INSN_CMOV, OP_SIZE_NATIVE, cond ^ 1, 0);
3165 #endif
3166 }
3167 gen_one(R_SCRATCH_1);
3168 gen_one(R_SCRATCH_1);
3169 gen_imm_offset();
3170 do_store:
3171 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot, 0, R_SCRATCH_1));
3172 return true;
3173 }
3174 #endif
3175
3176 static bool attr_w attr_unused gen_frame_cmp_imm_set_cond_reg(struct codegen_context *ctx, unsigned size, unsigned reg, int64_t imm, unsigned cond, frame_t slot_r)
3177 {
3178 g(gen_cmp_dest_reg(ctx, size, reg, (unsigned)-1, reg, imm, cond));
3179 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, reg));
3180
3181 return true;
3182 }
3183
3184 static bool attr_w gen_frame_load_cmp_set_cond(struct codegen_context *ctx, unsigned size, bool sx, frame_t slot, int64_t offset, unsigned reg, unsigned cond, frame_t slot_r)
3185 {
3186 #if ARCH_HAS_FLAGS
3187 bool logical = COND_IS_LOGICAL(cond);
3188 g(gen_frame_load_cmp(ctx, size, logical, sx, false, slot, offset, reg));
3189 g(gen_frame_set_cond(ctx, size, logical, cond, slot_r));
3190 #else
3191 g(gen_frame_load(ctx, size, sx, slot, offset, R_SCRATCH_NA_1));
3192
3193 g(gen_cmp_dest_reg(ctx, size, reg, R_SCRATCH_NA_1, R_SCRATCH_NA_1, 0, cond));
3194
3195 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, R_SCRATCH_NA_1));
3196 #endif
3197 return true;
3198 }
3199
3200 static bool attr_w gen_frame_load_cmp_imm_set_cond(struct codegen_context *ctx, unsigned size, bool sx, frame_t slot, int64_t offset, int64_t value, unsigned cond, frame_t slot_r)
3201 {
3202 #if ARCH_HAS_FLAGS
3203 bool logical = COND_IS_LOGICAL(cond);
3204 #if defined(ARCH_S390)
3205 if (cond == COND_E)
3206 logical = true;
3207 #endif
3208 g(gen_frame_load_cmp_imm(ctx, size, logical, sx, slot, offset, value));
3209 g(gen_frame_set_cond(ctx, size, false, cond, slot_r));
3210 #else
3211 g(gen_frame_load(ctx, size, sx, slot, offset, R_SCRATCH_NA_1));
3212 g(gen_frame_cmp_imm_set_cond_reg(ctx, size, R_SCRATCH_NA_1, value, cond, slot_r));
3213 #endif
3214 return true;
3215 }
3216
3217 #if defined(ARCH_X86)
3218 static bool attr_w gen_cmov(struct codegen_context *ctx, unsigned op_size, unsigned cond, unsigned reg, uint32_t *label)
3219 {
3220 if (unlikely(op_size < OP_SIZE_4))
3221 internal(file_line, "gen_cmov: unsupported operand size");
3222 if (likely(cpu_test_feature(CPU_FEATURE_cmov))) {
3223 gen_insn(INSN_CMOV, op_size, cond, 0);
3224 gen_one(reg);
3225 gen_one(reg);
3226 *label = 0;
3227 } else {
3228 *label = alloc_label(ctx);
3229 if (unlikely(!*label))
3230 return false;
3231 gen_insn(INSN_JMP_COND, op_size, cond ^ 1, 0);
3232 gen_four(*label);
3233 gen_insn(INSN_MOV, op_size, 0, 0);
3234 gen_one(reg);
3235 }
3236 return true;
3237 }
3238 #endif
3239
3240 static bool attr_w gen_extend(struct codegen_context *ctx, unsigned op_size, bool sx, unsigned dest, unsigned src)
3241 {
3242 unsigned attr_unused shift;
3243 if (unlikely(op_size == OP_SIZE_NATIVE)) {
3244 if (dest != src) {
3245 gen_insn(INSN_MOV, op_size, 0, 0);
3246 gen_one(dest);
3247 gen_one(src);
3248 return true;
3249 }
3250 return true;
3251 }
3252 #if defined(ARCH_IA64) || defined(ARCH_LOONGARCH64) || defined(ARCH_PARISC) || defined(ARCH_X86)
3253 gen_insn(sx ? INSN_MOVSX : INSN_MOV, op_size, 0, 0);
3254 gen_one(dest);
3255 gen_one(src);
3256 return true;
3257 #endif
3258 #if defined(ARCH_POWER)
3259 if (!sx || op_size == OP_SIZE_2 || cpu_test_feature(CPU_FEATURE_ppc)) {
3260 gen_insn(sx ? INSN_MOVSX : INSN_MOV, op_size, 0, 0);
3261 gen_one(dest);
3262 gen_one(src);
3263 return true;
3264 }
3265 #endif
3266 if (OP_SIZE_NATIVE == OP_SIZE_4) {
3267 shift = op_size == OP_SIZE_1 ? 24 : 16;
3268 } else if (OP_SIZE_NATIVE == OP_SIZE_8) {
3269 shift = op_size == OP_SIZE_1 ? 56 : op_size == OP_SIZE_2 ? 48 : 32;
3270 } else {
3271 internal(file_line, "gen_extend: invalid OP_SIZE_NATIVE");
3272 }
3273 #if defined(ARCH_ALPHA)
3274 if (!sx) {
3275 g(gen_3address_alu_imm(ctx, OP_SIZE_NATIVE, ALU_ZAPNOT, dest, src, op_size == OP_SIZE_1 ? 0x1 : op_size == OP_SIZE_2 ? 0x3 : 0xf));
3276 return true;
3277 } else if (op_size == OP_SIZE_4 || ARCH_HAS_BWX) {
3278 gen_insn(INSN_MOVSX, op_size, 0, 0);
3279 gen_one(dest);
3280 gen_one(src);
3281 return true;
3282 }
3283 #endif
3284 #if defined(ARCH_MIPS)
3285 if (sx && shift == 32) {
3286 gen_insn(INSN_ROT + ARCH_PARTIAL_ALU(OP_SIZE_4), OP_SIZE_4, ROT_SHL, ROT_WRITES_FLAGS(ROT_SHL));
3287 gen_one(dest);
3288 gen_one(src);
3289 gen_one(ARG_IMM);
3290 gen_eight(0);
3291 return true;
3292 }
3293 if (sx && MIPS_HAS_ROT) {
3294 gen_insn(INSN_MOVSX, op_size, 0, 0);
3295 gen_one(dest);
3296 gen_one(src);
3297 return true;
3298 }
3299 #endif
3300 #if defined(ARCH_S390)
3301 if (((op_size == OP_SIZE_1 || op_size == OP_SIZE_2) && cpu_test_feature(CPU_FEATURE_extended_imm)) || op_size == OP_SIZE_4) {
3302 gen_insn(!sx ? INSN_MOV : INSN_MOVSX, op_size, 0, 0);
3303 gen_one(dest);
3304 gen_one(src);
3305 return true;
3306 }
3307 #endif
3308 #if defined(ARCH_SPARC)
3309 if (shift == 32) {
3310 gen_insn(INSN_ROT + ARCH_PARTIAL_ALU(OP_SIZE_4), OP_SIZE_4, sx ? ROT_SAR : ROT_SHR, ROT_WRITES_FLAGS(sx ? ROT_SAR : ROT_SHR));
3311 gen_one(dest);
3312 gen_one(src);
3313 gen_one(ARG_IMM);
3314 gen_eight(0);
3315 return true;
3316 }
3317 #endif
3318 #if defined(ARCH_RISCV64)
3319 if (sx && (op_size == OP_SIZE_4 || likely(cpu_test_feature(CPU_FEATURE_zbb)))) {
3320 gen_insn(INSN_MOVSX, op_size, 0, 0);
3321 gen_one(dest);
3322 gen_one(src);
3323 return true;
3324 }
3325 if (!sx && ((op_size == OP_SIZE_1) ||
3326 (op_size == OP_SIZE_2 && likely(cpu_test_feature(CPU_FEATURE_zbb))) ||
3327 (op_size == OP_SIZE_4 && likely(cpu_test_feature(CPU_FEATURE_zba))))) {
3328 gen_insn(INSN_MOV, op_size, 0, 0);
3329 gen_one(dest);
3330 gen_one(src);
3331 return true;
3332 }
3333 #endif
3334 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SHL, dest, src, shift, false));
3335 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, sx ? ROT_SAR : ROT_SHR, dest, dest, shift, false));
3336
3337 return true;
3338 }
3339
3340 static bool attr_w gen_cmp_extended(struct codegen_context *ctx, unsigned cmp_op_size, unsigned sub_op_size, unsigned reg, unsigned attr_unused tmp_reg, uint32_t label_ovf)
3341 {
3342 if (unlikely(sub_op_size >= cmp_op_size))
3343 return true;
3344 #if defined(ARCH_ARM64)
3345 gen_insn(INSN_CMP, cmp_op_size, 0, 1);
3346 gen_one(reg);
3347 gen_one(ARG_EXTENDED_REGISTER);
3348 gen_one(sub_op_size == OP_SIZE_1 ? ARG_EXTEND_SXTB : sub_op_size == OP_SIZE_2 ? ARG_EXTEND_SXTH : ARG_EXTEND_SXTW);
3349 gen_one(reg);
3350
3351 gen_insn(INSN_JMP_COND, cmp_op_size, COND_NE, 0);
3352 gen_four(label_ovf);
3353 #else
3354 g(gen_extend(ctx, sub_op_size, true, tmp_reg, reg));
3355
3356 g(gen_cmp_test_jmp(ctx, INSN_CMP, cmp_op_size, reg, tmp_reg, COND_NE, label_ovf));
3357 #endif
3358 return true;
3359 }
3360
3361 static bool attr_w gen_lea3(struct codegen_context *ctx, unsigned dest, unsigned base, unsigned shifted, unsigned shift, int64_t offset)
3362 {
3363 #if defined(ARCH_X86)
3364 gen_insn(INSN_LEA3, i_size(OP_SIZE_ADDRESS), shift, 0);
3365 gen_one(dest);
3366 gen_one(base);
3367 gen_one(shifted);
3368 gen_one(ARG_IMM);
3369 gen_eight(likely(imm_is_32bit(offset)) ? offset : 0);
3370
3371 if (unlikely(!imm_is_32bit(offset)))
3372 g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_ADDRESS), ALU_ADD, dest, dest, offset));
3373
3374 return true;
3375 #endif
3376 if (ARCH_HAS_SHIFTED_ADD(shift)) {
3377 gen_insn(INSN_ALU, i_size(OP_SIZE_ADDRESS), ALU_ADD, ALU_WRITES_FLAGS(ALU_ADD, false));
3378 gen_one(dest);
3379 gen_one(base);
3380 gen_one(ARG_SHIFTED_REGISTER);
3381 gen_one(ARG_SHIFT_LSL | shift);
3382 gen_one(shifted);
3383
3384 if (offset) {
3385 g(gen_imm(ctx, offset, IMM_PURPOSE_ADD, i_size(OP_SIZE_ADDRESS)));
3386 gen_insn(INSN_ALU, i_size(OP_SIZE_ADDRESS), ALU_ADD, ALU_WRITES_FLAGS(ALU_ADD, is_imm()));
3387 gen_one(dest);
3388 gen_one(dest);
3389 gen_imm_offset();
3390 }
3391
3392 return true;
3393 }
3394
3395 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SHL, dest, shifted, shift, false));
3396
3397 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_ADD, dest, dest, base));
3398
3399 if (offset)
3400 g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_ADDRESS), ALU_ADD, dest, dest, offset));
3401 return true;
3402 }
3403
3404 static bool attr_w gen_memcpy_raw(struct codegen_context *ctx, unsigned dest_base, int64_t dest_offset, unsigned src_base, int64_t src_offset, size_t size, size_t attr_unused align)
3405 {
3406 if (!size)
3407 return true;
3408 if (!ARCH_HAS_BWX) {
3409 if (align < 4 || (size & 3))
3410 goto call_memcpy;
3411 }
3412 #if defined(ARCH_S390)
3413 if (size <= 0x10) {
3414 if (!(size & 3) || cpu_test_feature(CPU_FEATURE_extended_imm))
3415 goto do_explicit_copy;
3416 }
3417 if (size <= 0x100 && dest_offset >= 0 && dest_offset < 0x1000 && src_offset >= 0 && src_offset < 0x1000) {
3418 gen_insn(INSN_MEMCPY, 0, 0, 0);
3419 gen_one(ARG_ADDRESS_1);
3420 gen_one(dest_base);
3421 gen_eight(dest_offset);
3422 gen_one(ARG_ADDRESS_1);
3423 gen_one(src_base);
3424 gen_eight(src_offset);
3425 gen_one(ARG_IMM);
3426 gen_eight(size);
3427 return true;
3428 }
3429 goto call_memcpy;
3430 do_explicit_copy:
3431 #endif
3432 if (size <= INLINE_COPY_SIZE) {
3433 while (size) {
3434 unsigned this_step;
3435 unsigned this_op_size;
3436 #if defined(ARCH_ARM)
3437 if (size >= 2U << OP_SIZE_NATIVE
3438 #if defined(ARCH_ARM32)
3439 && align >= 1U << OP_SIZE_NATIVE
3440 #endif
3441 ) {
3442 g(gen_address(ctx, src_base, src_offset, IMM_PURPOSE_LDP_STP_OFFSET, OP_SIZE_NATIVE));
3443 gen_insn(INSN_LDP, OP_SIZE_NATIVE, 0, 0);
3444 gen_one(R_SCRATCH_NA_1);
3445 gen_one(R_SCRATCH_NA_2);
3446 gen_address_offset();
3447
3448 g(gen_address(ctx, dest_base, dest_offset, IMM_PURPOSE_LDP_STP_OFFSET, OP_SIZE_NATIVE));
3449 gen_insn(INSN_STP, OP_SIZE_NATIVE, 0, 0);
3450 gen_address_offset();
3451 gen_one(R_SCRATCH_NA_1);
3452 gen_one(R_SCRATCH_NA_2);
3453
3454 size -= 2U << OP_SIZE_NATIVE;
3455 src_offset += 2U << OP_SIZE_NATIVE;
3456 dest_offset += 2U << OP_SIZE_NATIVE;
3457
3458 continue;
3459 }
3460 #endif
3461 if (size >= 8 && OP_SIZE_NATIVE >= OP_SIZE_8)
3462 this_step = 8;
3463 else if (size >= 4)
3464 this_step = 4;
3465 else if (size >= 2)
3466 this_step = 2;
3467 else
3468 this_step = 1;
3469 if (UNALIGNED_TRAP)
3470 this_step = minimum(this_step, align);
3471 this_op_size = log_2(this_step);
3472
3473 g(gen_address(ctx, src_base, src_offset, ARCH_PREFERS_SX(this_op_size) ? IMM_PURPOSE_LDR_SX_OFFSET : IMM_PURPOSE_LDR_OFFSET, this_op_size));
3474 gen_insn(ARCH_PREFERS_SX(this_op_size) ? INSN_MOVSX : INSN_MOV, this_op_size, 0, 0);
3475 gen_one(R_SCRATCH_1);
3476 gen_address_offset();
3477
3478 g(gen_address(ctx, dest_base, dest_offset, IMM_PURPOSE_STR_OFFSET, this_op_size));
3479 gen_insn(INSN_MOV, this_op_size, 0, 0);
3480 gen_address_offset();
3481 gen_one(R_SCRATCH_1);
3482
3483 size -= this_step;
3484 src_offset += this_step;
3485 dest_offset += this_step;
3486 }
3487 return true;
3488 }
3489
3490 call_memcpy:
3491 g(gen_upcall_start(ctx, 3));
3492 if (unlikely(R_ARG0 == src_base)) {
3493 if (unlikely(R_ARG1 == dest_base))
3494 internal(file_line, "gen_memcpy_raw: swapped registers: %u, %u", src_base, dest_base);
3495 g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_ADDRESS), ALU_ADD, R_ARG1, src_base, src_offset));
3496 g(gen_upcall_argument(ctx, 1));
3497 }
3498
3499 g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_ADDRESS), ALU_ADD, R_ARG0, dest_base, dest_offset));
3500 g(gen_upcall_argument(ctx, 0));
3501
3502 if (R_ARG0 != src_base) {
3503 g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_ADDRESS), ALU_ADD, R_ARG1, src_base, src_offset));
3504 g(gen_upcall_argument(ctx, 1));
3505 }
3506
3507 #if (defined(ARCH_X86_64) || defined(ARCH_X86_X32)) && !defined(ARCH_X86_WIN_ABI)
3508 if (cpu_test_feature(CPU_FEATURE_erms)) {
3509 g(gen_load_constant(ctx, R_CX, size));
3510
3511 gen_insn(INSN_MEMCPY, 0, 0, 0);
3512 gen_one(ARG_ADDRESS_1_POST_I);
3513 gen_one(R_DI);
3514 gen_eight(0);
3515 gen_one(ARG_ADDRESS_1_POST_I);
3516 gen_one(R_SI);
3517 gen_eight(0);
3518 gen_one(R_CX);
3519 g(gen_upcall_end(ctx, 3));
3520 return true;
3521 }
3522 #endif
3523
3524 g(gen_load_constant(ctx, R_ARG2, size));
3525 g(gen_upcall_argument(ctx, 2));
3526
3527 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, mem_copy), 3));
3528
3529 return true;
3530 }
3531
3532 static bool attr_w gen_memcpy_to_slot(struct codegen_context *ctx, frame_t dest_slot, unsigned src_base, int64_t src_offset)
3533 {
3534 const struct type *t = get_type_of_local(ctx, dest_slot);
3535 unsigned size = log_2(t->size);
3536 short dest_reg = ctx->registers[dest_slot];
3537 if (dest_reg >= 0) {
3538 if (ARCH_PREFERS_SX(size) && !reg_is_fp(dest_reg)) {
3539 g(gen_address(ctx, src_base, src_offset, IMM_PURPOSE_LDR_SX_OFFSET, size));
3540 gen_insn(INSN_MOVSX, size, 0, 0);
3541 } else {
3542 g(gen_address(ctx, src_base, src_offset, reg_is_fp(dest_reg) ? IMM_PURPOSE_VLDR_VSTR_OFFSET : IMM_PURPOSE_LDR_OFFSET, size));
3543 gen_insn(INSN_MOV, size, 0, 0);
3544 }
3545 gen_one(dest_reg);
3546 gen_address_offset();
3547 return true;
3548 }
3549 g(gen_memcpy_raw(ctx, R_FRAME, (size_t)dest_slot * slot_size, src_base, src_offset, t->size, t->align));
3550 return true;
3551 }
3552
3553 static bool attr_w gen_memcpy_from_slot(struct codegen_context *ctx, unsigned dest_base, int64_t dest_offset, frame_t src_slot)
3554 {
3555 const struct type *t = get_type_of_local(ctx, src_slot);
3556 unsigned size = log_2(t->size);
3557 short src_reg = ctx->registers[src_slot];
3558 if (src_reg >= 0) {
3559 g(gen_address(ctx, dest_base, dest_offset, reg_is_fp(src_reg) ? IMM_PURPOSE_VLDR_VSTR_OFFSET : IMM_PURPOSE_STR_OFFSET, size));
3560 gen_insn(INSN_MOV, size, 0, 0);
3561 gen_address_offset();
3562 gen_one(src_reg);
3563 return true;
3564 }
3565 g(gen_memcpy_raw(ctx, dest_base, dest_offset, R_FRAME, (size_t)src_slot * slot_size, t->size, t->align));
3566 return true;
3567 }
3568
3569 static bool attr_w gen_memcpy_slots(struct codegen_context *ctx, frame_t dest_slot, frame_t src_slot)
3570 {
3571 const struct type *t = get_type_of_local(ctx, src_slot);
3572 unsigned size = log_2(t->size);
3573 short dest_reg = ctx->registers[dest_slot];
3574 short src_reg = ctx->registers[src_slot];
3575 if (dest_reg >= 0 && src_reg >= 0) {
3576 gen_insn(INSN_MOV, reg_is_fp(src_reg) ? size : OP_SIZE_NATIVE, 0, 0);
3577 gen_one(dest_reg);
3578 gen_one(src_reg);
3579 return true;
3580 }
3581 if (dest_reg >= 0) {
3582 g(gen_frame_load(ctx, size, false, src_slot, 0, dest_reg));
3583 return true;
3584 }
3585 if (src_reg >= 0) {
3586 g(gen_frame_store(ctx, size, dest_slot, 0, src_reg));
3587 return true;
3588 }
3589 g(gen_memcpy_raw(ctx, R_FRAME, (size_t)dest_slot * slot_size, R_FRAME, (size_t)src_slot * slot_size, t->size, maximum(slot_size, t->align)));
3590 return true;
3591 }
3592
3593 static bool attr_w gen_clear_bitmap(struct codegen_context *ctx, unsigned additional_offset, unsigned dest_base, int64_t dest_offset, frame_t bitmap_slots)
3594 {
3595 if (bitmap_slots <= INLINE_BITMAP_SLOTS) {
3596 bool attr_unused scratch_2_zeroed = false;
3597 size_t bitmap_length = (size_t)bitmap_slots * slot_size;
3598 size_t clear_offset = 0;
3599 additional_offset += (unsigned)dest_offset;
3600 #if defined(ARCH_X86)
3601 gen_insn(INSN_ALU, OP_SIZE_4, ALU_XOR, ALU_WRITES_FLAGS(ALU_XOR, false));
3602 gen_one(R_SCRATCH_1);
3603 gen_one(R_SCRATCH_1);
3604 gen_one(R_SCRATCH_1);
3605 #endif
3606 #if defined(ARCH_ARM32) || defined(ARCH_S390)
3607 g(gen_load_constant(ctx, R_SCRATCH_1, 0));
3608 #endif
3609 while (clear_offset < bitmap_length) {
3610 size_t len = bitmap_length - clear_offset;
3611 if (len > frame_align)
3612 len = frame_align;
3613 if (additional_offset)
3614 len = minimum(len, additional_offset & -additional_offset);
3615 #if defined(ARCH_ARM32) || defined(ARCH_S390)
3616 len = minimum(len, 2U << OP_SIZE_NATIVE);
3617 if (len == 2U << OP_SIZE_NATIVE) {
3618 if (!scratch_2_zeroed) {
3619 g(gen_load_constant(ctx, R_SCRATCH_2, 0));
3620 scratch_2_zeroed = true;
3621 }
3622 g(gen_address(ctx, dest_base, dest_offset + clear_offset, IMM_PURPOSE_LDP_STP_OFFSET, OP_SIZE_NATIVE));
3623 gen_insn(INSN_STP, OP_SIZE_NATIVE, 0, 0);
3624 gen_address_offset();
3625 gen_one(R_SCRATCH_1);
3626 gen_one(R_SCRATCH_2);
3627 goto next_loop;
3628 }
3629 #elif defined(ARCH_ARM64)
3630 len = minimum(len, 1U << OP_SIZE_16);
3631 if (len == 1U << OP_SIZE_16) {
3632 g(gen_address(ctx, dest_base, dest_offset + clear_offset, IMM_PURPOSE_LDP_STP_OFFSET, OP_SIZE_8));
3633 g(gen_imm(ctx, 0, IMM_PURPOSE_STORE_VALUE, OP_SIZE_8));
3634 gen_insn(INSN_STP, OP_SIZE_NATIVE, 0, 0);
3635 gen_address_offset();
3636 gen_imm_offset();
3637 gen_imm_offset();
3638 goto next_loop;
3639 }
3640 #elif defined(ARCH_X86)
3641 len = minimum(len, 1U << OP_SIZE_16);
3642 if (len == 1U << OP_SIZE_16 && cpu_test_feature(CPU_FEATURE_sse)) {
3643 if (!scratch_2_zeroed) {
3644 gen_insn(INSN_ALU, OP_SIZE_16, ALU_XOR, 0);
3645 gen_one(R_XMM0);
3646 gen_one(R_XMM0);
3647 gen_one(R_XMM0);
3648 scratch_2_zeroed = true;
3649 }
3650 g(gen_address(ctx, dest_base, dest_offset + clear_offset, IMM_PURPOSE_VLDR_VSTR_OFFSET, OP_SIZE_16));
3651 gen_insn(INSN_MOV, OP_SIZE_16, 0, 0);
3652 gen_address_offset();
3653 gen_one(R_XMM0);
3654 goto next_loop;
3655 }
3656 #endif
3657 len = minimum(len, 1U << OP_SIZE_NATIVE);
3658 len = (size_t)1 << high_bit(len);
3659 #if defined(ARCH_X86) || defined(ARCH_ARM32) || defined(ARCH_S390)
3660 g(gen_address(ctx, dest_base, dest_offset + clear_offset, IMM_PURPOSE_STR_OFFSET, log_2(len)));
3661 gen_insn(INSN_MOV, log_2(len), 0, 0);
3662 gen_address_offset();
3663 gen_one(R_SCRATCH_1);
3664 #else
3665 g(gen_address(ctx, dest_base, dest_offset + clear_offset, IMM_PURPOSE_STR_OFFSET, log_2(len)));
3666 g(gen_imm(ctx, 0, IMM_PURPOSE_STORE_VALUE, log_2(len)));
3667 gen_insn(INSN_MOV, log_2(len), 0, 0);
3668 gen_address_offset();
3669 gen_imm_offset();
3670 #endif
3671 goto next_loop;
3672 next_loop:
3673 clear_offset += len;
3674 additional_offset += len;
3675 }
3676 return true;
3677 }
3678 #if (defined(ARCH_X86_64) || defined(ARCH_X86_X32)) && !defined(ARCH_X86_WIN_ABI)
3679 if (cpu_test_feature(CPU_FEATURE_erms)) {
3680 gen_insn(INSN_PUSH, OP_SIZE_8, 0, 0);
3681 gen_one(R_DI);
3682
3683 g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_ADDRESS), ALU_ADD, R_DI, dest_base, dest_offset));
3684
3685 g(gen_load_constant(ctx, R_CX, (size_t)bitmap_slots * slot_size));
3686
3687 gen_insn(INSN_ALU, OP_SIZE_4, ALU_XOR, ALU_WRITES_FLAGS(ALU_XOR, false));
3688 gen_one(R_AX);
3689 gen_one(R_AX);
3690 gen_one(R_AX);
3691
3692 gen_insn(INSN_MEMSET, 0, 0, 0);
3693 gen_one(ARG_ADDRESS_1_POST_I);
3694 gen_one(R_DI);
3695 gen_eight(0);
3696 gen_one(R_CX);
3697 gen_one(R_AX);
3698
3699 gen_insn(INSN_POP, OP_SIZE_8, 0, 0);
3700 gen_one(R_DI);
3701
3702 return true;
3703 }
3704 #endif
3705 g(gen_upcall_start(ctx, 2));
3706
3707 g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_ADDRESS), ALU_ADD, R_ARG0, dest_base, dest_offset));
3708 g(gen_upcall_argument(ctx, 0));
3709
3710 g(gen_load_constant(ctx, R_ARG1, (size_t)bitmap_slots * slot_size));
3711 g(gen_upcall_argument(ctx, 1));
3712
3713 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, mem_clear), 2));
3714
3715 return true;
3716 }
3717
3718 static bool attr_w load_function_offset(struct codegen_context *ctx, unsigned dest, size_t fn_offset)
3719 {
3720 g(gen_frame_load_raw(ctx, OP_SIZE_ADDRESS, false, 0, frame_offs(function), dest));
3721
3722 g(gen_address(ctx, dest, fn_offset, IMM_PURPOSE_LDR_OFFSET, OP_SIZE_ADDRESS));
3723 gen_insn(INSN_MOV, OP_SIZE_ADDRESS, 0, 0);
3724 gen_one(dest);
3725 gen_address_offset();
3726
3727 return true;
3728 }
3729
3730 #define MODE_FIXED 0
3731 #define MODE_INT 1
3732 #define MODE_BOOL 2
3733
3734 static bool attr_w gen_alu(struct codegen_context *ctx, unsigned mode, unsigned op_size, unsigned op, uint32_t label_ovf, frame_t slot_1, frame_t slot_2, frame_t slot_r)
3735 {
3736 unsigned alu;
3737 bool sgn, mod;
3738 switch (mode) {
3739 case MODE_FIXED: switch (op) {
3740 case OPCODE_FIXED_OP_add: alu = ALU_ADD; goto do_alu;
3741 case OPCODE_FIXED_OP_subtract: alu = ALU_SUB; goto do_alu;
3742 case OPCODE_FIXED_OP_multiply: goto do_multiply;
3743 case OPCODE_FIXED_OP_divide:
3744 case OPCODE_FIXED_OP_divide_alt1: sgn = true; mod = false; goto do_divide;
3745 case OPCODE_FIXED_OP_udivide:
3746 case OPCODE_FIXED_OP_udivide_alt1: sgn = false; mod = false; goto do_divide;
3747 case OPCODE_FIXED_OP_modulo:
3748 case OPCODE_FIXED_OP_modulo_alt1: sgn = true; mod = true; goto do_divide;
3749 case OPCODE_FIXED_OP_umodulo:
3750 case OPCODE_FIXED_OP_umodulo_alt1: sgn = false; mod = true; goto do_divide;
3751 case OPCODE_FIXED_OP_power: return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_binary_power_int8_t), op_size, slot_1, slot_2, slot_r, 0);
3752 case OPCODE_FIXED_OP_and: alu = ALU_AND; goto do_alu;
3753 case OPCODE_FIXED_OP_or: alu = ALU_OR; goto do_alu;
3754 case OPCODE_FIXED_OP_xor: alu = ALU_XOR; goto do_alu;
3755 case OPCODE_FIXED_OP_shl: alu = ROT_SHL; goto do_shift;
3756 case OPCODE_FIXED_OP_shr: alu = ROT_SAR; goto do_shift;
3757 case OPCODE_FIXED_OP_ushr: alu = ROT_SHR; goto do_shift;
3758 case OPCODE_FIXED_OP_rol: alu = ROT_ROL; goto do_shift;
3759 case OPCODE_FIXED_OP_ror: alu = ROT_ROR; goto do_shift;
3760 case OPCODE_FIXED_OP_bts: alu = BTX_BTS; goto do_bt;
3761 case OPCODE_FIXED_OP_btr: alu = BTX_BTR; goto do_bt;
3762 case OPCODE_FIXED_OP_btc: alu = BTX_BTC; goto do_bt;
3763 case OPCODE_FIXED_OP_equal: alu = COND_E; goto do_compare;
3764 case OPCODE_FIXED_OP_not_equal: alu = COND_NE; goto do_compare;
3765 case OPCODE_FIXED_OP_less: alu = COND_L; goto do_compare;
3766 case OPCODE_FIXED_OP_less_equal: alu = COND_LE; goto do_compare;
3767 case OPCODE_FIXED_OP_uless: alu = COND_B; goto do_compare;
3768 case OPCODE_FIXED_OP_uless_equal: alu = COND_BE; goto do_compare;
3769 case OPCODE_FIXED_OP_bt: alu = BTX_BT; goto do_bt;
3770 default: internal(file_line, "gen_alu: unsupported fixed operation %u", op);
3771 }
3772 case MODE_INT: switch (op) {
3773 case OPCODE_INT_OP_add: alu = ALU_ADD; goto do_alu;
3774 case OPCODE_INT_OP_subtract: alu = ALU_SUB; goto do_alu;
3775 case OPCODE_INT_OP_multiply: goto do_multiply;
3776 case OPCODE_INT_OP_divide:
3777 case OPCODE_INT_OP_divide_alt1: sgn = true; mod = false; goto do_divide;
3778 case OPCODE_INT_OP_modulo:
3779 case OPCODE_INT_OP_modulo_alt1: sgn = true; mod = true; goto do_divide;
3780 case OPCODE_INT_OP_power: return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, INT_binary_power_int8_t), op_size, slot_1, slot_2, slot_r, label_ovf);
3781 case OPCODE_INT_OP_and: alu = ALU_AND; mode = MODE_FIXED; goto do_alu;
3782 case OPCODE_INT_OP_or: alu = ALU_OR; mode = MODE_FIXED; goto do_alu;
3783 case OPCODE_INT_OP_xor: alu = ALU_XOR; mode = MODE_FIXED; goto do_alu;
3784 case OPCODE_INT_OP_shl: alu = ROT_SHL; goto do_shift;
3785 case OPCODE_INT_OP_shr: alu = ROT_SAR; goto do_shift;
3786 case OPCODE_INT_OP_bts: alu = BTX_BTS; goto do_bt;
3787 case OPCODE_INT_OP_btr: alu = BTX_BTR; goto do_bt;
3788 case OPCODE_INT_OP_btc: alu = BTX_BTC; goto do_bt;
3789 case OPCODE_INT_OP_equal: alu = COND_E; goto do_compare;
3790 case OPCODE_INT_OP_not_equal: alu = COND_NE; goto do_compare;
3791 case OPCODE_INT_OP_less: alu = COND_L; goto do_compare;
3792 case OPCODE_INT_OP_less_equal: alu = COND_LE; goto do_compare;
3793 case OPCODE_INT_OP_bt: alu = BTX_BT; goto do_bt;
3794 default: internal(file_line, "gen_alu: unsupported int operation %u", op);
3795 }
3796 case MODE_BOOL: switch (op) {
3797 case OPCODE_BOOL_OP_and: alu = ALU_AND; mode = MODE_FIXED; goto do_alu;
3798 case OPCODE_BOOL_OP_or: alu = ALU_OR; mode = MODE_FIXED; goto do_alu;
3799 case OPCODE_BOOL_OP_equal: alu = COND_E; goto do_compare;
3800 case OPCODE_BOOL_OP_not_equal: alu = ALU_XOR; mode = MODE_FIXED; goto do_alu;
3801 case OPCODE_BOOL_OP_less: alu = COND_L; goto do_compare;
3802 case OPCODE_BOOL_OP_less_equal: alu = COND_LE; goto do_compare;
3803 default: internal(file_line, "gen_alu: unsupported bool operation %u", op);
3804 }
3805 }
3806 internal(file_line, "gen_alu: unsupported mode %u", mode);
3807
3808 /*******
3809 * ALU *
3810 *******/
3811 do_alu: {
3812 size_t attr_unused offset;
3813 uint8_t attr_unused long_imm;
3814 unsigned first_flags;
3815 unsigned second_flags;
3816 unsigned second_alu;
3817 unsigned attr_unused op_size_flags;
3818 if (unlikely(op_size > OP_SIZE_NATIVE)) {
3819 #if !defined(ARCH_X86) && !defined(ARCH_ARM) && !defined(ARCH_PARISC) && !defined(ARCH_POWER) && !defined(ARCH_SPARC32)
3820 if (mode == MODE_FIXED) {
3821 if (alu == ALU_ADD) {
3822 g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_add_,TYPE_INT_MAX)), slot_1, slot_2, slot_r, 0));
3823 return true;
3824 } else if (alu == ALU_SUB) {
3825 g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_subtract_,TYPE_INT_MAX)), slot_1, slot_2, slot_r, 0));
3826 return true;
3827 }
3828 } else if (mode == MODE_INT) {
3829 if (alu == ALU_ADD) {
3830 g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(INT_binary_add_,TYPE_INT_MAX)), slot_1, slot_2, slot_r, label_ovf));
3831 return true;
3832 } else if (alu == ALU_SUB) {
3833 g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(INT_binary_subtract_,TYPE_INT_MAX)), slot_1, slot_2, slot_r, label_ovf));
3834 return true;
3835 }
3836 }
3837 #endif
3838 first_flags = alu == ALU_ADD || alu == ALU_SUB ? 2 : 0;
3839 second_flags = mode == MODE_INT ? 1 : 0;
3840 second_alu = alu == ALU_ADD ? ALU_ADC : alu == ALU_SUB ? ALU_SBB : alu;
3841 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
3842 #if defined(ARCH_X86)
3843 g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, false, alu, first_flags, slot_2, lo_word(OP_SIZE_NATIVE), R_SCRATCH_1));
3844 g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, false, second_alu, second_flags, slot_2, hi_word(OP_SIZE_NATIVE), R_SCRATCH_2));
3845 #else
3846 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_2, 0, R_SCRATCH_3, R_SCRATCH_4));
3847 gen_insn(INSN_ALU, OP_SIZE_NATIVE, alu, first_flags | ALU_WRITES_FLAGS(alu, false));
3848 gen_one(R_SCRATCH_1);
3849 gen_one(R_SCRATCH_1);
3850 gen_one(R_SCRATCH_3);
3851 #if defined(ARCH_PARISC)
3852 if (mode == MODE_INT) {
3853 gen_insn(INSN_ALU_FLAGS_TRAP, OP_SIZE_NATIVE, second_alu, ALU_WRITES_FLAGS(second_alu, false));
3854 gen_one(R_SCRATCH_2);
3855 gen_one(R_SCRATCH_2);
3856 gen_one(R_SCRATCH_4);
3857 gen_four(label_ovf);
3858 } else
3859 #endif
3860 {
3861 gen_insn(first_flags ? INSN_ALU_FLAGS : INSN_ALU, OP_SIZE_NATIVE, second_alu, second_flags | ALU_WRITES_FLAGS(second_alu, false));
3862 gen_one(R_SCRATCH_2);
3863 gen_one(R_SCRATCH_2);
3864 gen_one(R_SCRATCH_4);
3865 }
3866 #endif
3867 #if !defined(ARCH_PARISC)
3868 if (mode == MODE_INT) {
3869 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_O, 0);
3870 gen_four(label_ovf);
3871 }
3872 #endif
3873 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_1, R_SCRATCH_2));
3874 return true;
3875 }
3876
3877 #if defined(ARCH_X86)
3878 if (1)
3879 #elif defined(ARCH_S390)
3880 if (op_size >= OP_SIZE_4)
3881 #else
3882 if (0)
3883 #endif
3884 {
3885 g(gen_frame_load(ctx, op_size, false, slot_1, 0, R_SCRATCH_1));
3886 g(gen_frame_load_op(ctx, op_size, false, alu, mode == MODE_INT, slot_2, 0, R_SCRATCH_1));
3887 goto check_ovf_store;
3888 }
3889 op_size_flags = !ARCH_HAS_FLAGS && !ARCH_SUPPORTS_TRAPS ? OP_SIZE_NATIVE : OP_SIZE_4;
3890 #if defined(ARCH_POWER)
3891 op_size_flags = OP_SIZE_NATIVE;
3892 #endif
3893 g(gen_frame_load(ctx, op_size, mode == MODE_INT && (op_size < op_size_flags || ARCH_SUPPORTS_TRAPS), slot_1, 0, R_SCRATCH_1));
3894 g(gen_frame_load(ctx, op_size, mode == MODE_INT && (op_size < op_size_flags || ARCH_SUPPORTS_TRAPS), slot_2, 0, R_SCRATCH_2));
3895 #if !ARCH_HAS_FLAGS
3896 if (mode == MODE_INT && op_size >= OP_SIZE_4) {
3897 if (ARCH_SUPPORTS_TRAPS) {
3898 gen_insn(INSN_ALU_TRAP, op_size, alu, ALU_WRITES_FLAGS(alu, false));
3899 gen_one(R_SCRATCH_1);
3900 gen_one(R_SCRATCH_1);
3901 gen_one(R_SCRATCH_2);
3902 gen_four(label_ovf);
3903 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
3904 return true;
3905 }
3906 if (op_size >= OP_SIZE_NATIVE) {
3907 g(gen_3address_alu(ctx, i_size(op_size), alu, R_SCRATCH_3, R_SCRATCH_1, R_SCRATCH_2));
3908 #if defined(ARCH_IA64)
3909 g(gen_3address_alu(ctx, i_size(op_size), ALU_XOR, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_2));
3910 g(gen_3address_alu(ctx, i_size(op_size), ALU_XOR, R_SCRATCH_2, R_SCRATCH_2, R_SCRATCH_3));
3911 if (alu == ALU_ADD) {
3912 gen_insn(INSN_ALU, i_size(op_size), ALU_ANDN, ALU_WRITES_FLAGS(ALU_ANDN, false));
3913 gen_one(R_SCRATCH_1);
3914 gen_one(R_SCRATCH_2);
3915 gen_one(R_SCRATCH_1);
3916 } else {
3917 gen_insn(INSN_ALU, i_size(op_size), ALU_ANDN, ALU_WRITES_FLAGS(ALU_ANDN, false));
3918 gen_one(R_SCRATCH_1);
3919 gen_one(R_SCRATCH_1);
3920 gen_one(R_SCRATCH_2);
3921 }
3922 g(gen_cmp_test_jmp(ctx, INSN_TEST, i_size(op_size), R_SCRATCH_1, R_SCRATCH_1, COND_S, label_ovf));
3923 #else
3924 gen_insn(INSN_CMP_DEST_REG, i_size(op_size), COND_L, 0);
3925 gen_one(R_SCRATCH_1);
3926 if (alu == ALU_ADD) {
3927 gen_one(R_SCRATCH_3);
3928 gen_one(R_SCRATCH_1);
3929 } else {
3930 gen_one(R_SCRATCH_1);
3931 gen_one(R_SCRATCH_3);
3932 }
3933
3934 g(gen_imm(ctx, 0, IMM_PURPOSE_CMP, i_size(op_size)));
3935 gen_insn(INSN_CMP_DEST_REG, i_size(op_size), COND_L, 0);
3936 gen_one(R_SCRATCH_2);
3937 gen_one(R_SCRATCH_2);
3938 gen_imm_offset();
3939
3940 g(gen_cmp_test_jmp(ctx, INSN_CMP, i_size(op_size), R_SCRATCH_1, R_SCRATCH_2, COND_NE, label_ovf));
3941 #endif
3942 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_3));
3943 return true;
3944 }
3945 }
3946 #endif
3947 gen_insn(INSN_ALU + ARCH_PARTIAL_ALU(i_size(op_size)), i_size(op_size), alu, (mode == MODE_INT && op_size >= op_size_flags) | ALU_WRITES_FLAGS(alu, false));
3948 gen_one(R_SCRATCH_1);
3949 gen_one(R_SCRATCH_1);
3950 gen_one(R_SCRATCH_2);
3951
3952 if (mode == MODE_INT && unlikely(op_size < op_size_flags)) {
3953 g(gen_cmp_extended(ctx, op_size_flags, op_size, R_SCRATCH_1, R_SCRATCH_2, label_ovf));
3954 } else
3955 check_ovf_store:
3956 if (mode == MODE_INT) {
3957 gen_insn(INSN_JMP_COND, op_size, COND_O, 0);
3958 gen_four(label_ovf);
3959 }
3960 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
3961 return true;
3962 }
3963
3964 /************
3965 * MULTIPLY *
3966 ************/
3967 do_multiply: {
3968 size_t attr_unused offset;
3969 uint8_t attr_unused long_imm;
3970 if (unlikely(op_size > OP_SIZE_NATIVE) || unlikely(!ARCH_HAS_MUL)) {
3971 if (mode == MODE_INT) {
3972 g(gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, INT_binary_multiply_int8_t), op_size, slot_1, slot_2, slot_r, label_ovf));
3973 return true;
3974 }
3975 #if defined(ARCH_X86)
3976 g(gen_frame_load(ctx, OP_SIZE_NATIVE, false, slot_1, hi_word(OP_SIZE_NATIVE), R_CX));
3977 g(gen_frame_load(ctx, OP_SIZE_NATIVE, false, slot_2, hi_word(OP_SIZE_NATIVE), R_AX));
3978 g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, false, ALU_MUL, true, slot_2, lo_word(OP_SIZE_NATIVE), R_CX));
3979 g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, false, ALU_MUL, true, slot_1, lo_word(OP_SIZE_NATIVE), R_AX));
3980 gen_insn(INSN_ALU, OP_SIZE_NATIVE, ALU_ADD, ALU_WRITES_FLAGS(ALU_ADD, false));
3981 gen_one(R_CX);
3982 gen_one(R_CX);
3983 gen_one(R_AX);
3984 g(gen_frame_load(ctx, OP_SIZE_NATIVE, false, slot_2, lo_word(OP_SIZE_NATIVE), R_AX));
3985
3986 offset = (size_t)slot_1 * slot_size + lo_word(OP_SIZE_NATIVE);
3987 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
3988 gen_insn(INSN_MUL_L, OP_SIZE_NATIVE, 0, 1);
3989 gen_one(R_AX);
3990 gen_one(R_DX);
3991 gen_one(R_AX);
3992 gen_address_offset();
3993
3994 gen_insn(INSN_ALU, OP_SIZE_NATIVE, ALU_ADD, ALU_WRITES_FLAGS(ALU_ADD, false));
3995 gen_one(R_DX);
3996 gen_one(R_DX);
3997 gen_one(R_CX);
3998
3999 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_AX, R_DX));
4000
4001 return true;
4002 #elif defined(ARCH_ARM32)
4003 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
4004 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_2, 0, R_SCRATCH_3, R_SCRATCH_4));
4005
4006 gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
4007 gen_one(R_SCRATCH_NA_1);
4008 gen_one(R_SCRATCH_1);
4009
4010 gen_insn(INSN_ALU, OP_SIZE_NATIVE, ALU_MUL, ALU_WRITES_FLAGS(ALU_MUL, false));
4011 gen_one(R_SCRATCH_4);
4012 gen_one(R_SCRATCH_1);
4013 gen_one(R_SCRATCH_4);
4014
4015 gen_insn(INSN_MADD, OP_SIZE_NATIVE, 0, 0);
4016 gen_one(R_SCRATCH_2);
4017 gen_one(R_SCRATCH_3);
4018 gen_one(R_SCRATCH_2);
4019 gen_one(R_SCRATCH_4);
4020
4021 gen_insn(INSN_MUL_L, OP_SIZE_NATIVE, 0, 0);
4022 gen_one(R_SCRATCH_1);
4023 gen_one(R_SCRATCH_4);
4024 gen_one(R_SCRATCH_NA_1);
4025 gen_one(R_SCRATCH_3);
4026
4027 gen_insn(INSN_ALU, OP_SIZE_NATIVE, ALU_ADD, ALU_WRITES_FLAGS(ALU_ADD, false));
4028 gen_one(R_SCRATCH_2);
4029 gen_one(R_SCRATCH_2);
4030 gen_one(R_SCRATCH_4);
4031
4032 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_1, R_SCRATCH_2));
4033
4034 return true;
4035 #elif defined(ARCH_ARM64)
4036 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
4037 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_2, 0, R_SCRATCH_3, R_SCRATCH_4));
4038
4039 gen_insn(INSN_ALU, OP_SIZE_NATIVE, ALU_UMULH, ALU_WRITES_FLAGS(ALU_UMULH, false));
4040 gen_one(R_SCRATCH_NA_1);
4041 gen_one(R_SCRATCH_1);
4042 gen_one(R_SCRATCH_3);
4043
4044 gen_insn(INSN_MADD, OP_SIZE_NATIVE, 0, 0);
4045 gen_one(R_SCRATCH_NA_1);
4046 gen_one(R_SCRATCH_2);
4047 gen_one(R_SCRATCH_3);
4048 gen_one(R_SCRATCH_NA_1);
4049
4050 gen_insn(INSN_MADD, OP_SIZE_NATIVE, 0, 0);
4051 gen_one(R_SCRATCH_2);
4052 gen_one(R_SCRATCH_1);
4053 gen_one(R_SCRATCH_4);
4054 gen_one(R_SCRATCH_NA_1);
4055
4056 gen_insn(INSN_ALU, OP_SIZE_NATIVE, ALU_MUL, ALU_WRITES_FLAGS(ALU_MUL, false));
4057 gen_one(R_SCRATCH_1);
4058 gen_one(R_SCRATCH_1);
4059 gen_one(R_SCRATCH_3);
4060
4061 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_1, R_SCRATCH_2));
4062
4063 return true;
4064 #else
4065 g(gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_binary_multiply_int8_t), op_size, slot_1, slot_2, slot_r, 0));
4066 return true;
4067 #endif
4068 }
4069
4070 #if defined(ARCH_X86)
4071 g(gen_frame_load(ctx, op_size, false, slot_1, 0, R_SCRATCH_1));
4072 g(gen_frame_load_op(ctx, op_size, false, ALU_MUL, mode == MODE_INT, slot_2, 0, R_SCRATCH_1));
4073 if (mode == MODE_INT) {
4074 gen_insn(INSN_JMP_COND, op_size, COND_O, 0);
4075 gen_four(label_ovf);
4076 }
4077 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
4078 return true;
4079 #endif
4080 #if defined(ARCH_ALPHA)
4081 if (mode == MODE_INT && op_size >= OP_SIZE_4 && ARCH_SUPPORTS_TRAPS) {
4082 g(gen_frame_load(ctx, op_size, true, slot_1, 0, R_SCRATCH_1));
4083 g(gen_frame_load(ctx, op_size, true, slot_2, 0, R_SCRATCH_2));
4084
4085 gen_insn(INSN_ALU_TRAP, op_size, ALU_MUL, ALU_WRITES_FLAGS(ALU_MUL, false));
4086 gen_one(R_SCRATCH_1);
4087 gen_one(R_SCRATCH_1);
4088 gen_one(R_SCRATCH_2);
4089 gen_four(label_ovf);
4090 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
4091
4092 return true;
4093 }
4094 #endif
4095 #if defined(ARCH_ARM32)
4096 if (mode == MODE_INT && op_size == OP_SIZE_4) {
4097 g(gen_frame_load(ctx, op_size, false, slot_1, 0, R_SCRATCH_1));
4098 g(gen_frame_load(ctx, op_size, false, slot_2, 0, R_SCRATCH_2));
4099
4100 gen_insn(INSN_MUL_L, OP_SIZE_NATIVE, 0, 0);
4101 gen_one(R_SCRATCH_3);
4102 gen_one(R_SCRATCH_4);
4103 gen_one(R_SCRATCH_1);
4104 gen_one(R_SCRATCH_2);
4105
4106 gen_insn(INSN_CMP, OP_SIZE_NATIVE, 0, 1);
4107 gen_one(R_SCRATCH_4);
4108 gen_one(ARG_SHIFTED_REGISTER);
4109 gen_one(ARG_SHIFT_ASR | 0x1f);
4110 gen_one(R_SCRATCH_3);
4111
4112 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_NE, 0);
4113 gen_four(label_ovf);
4114
4115 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_3));
4116
4117 return true;
4118 }
4119 #endif
4120 #if defined(ARCH_ARM64)
4121 if (mode == MODE_INT && op_size == OP_SIZE_4) {
4122 g(gen_frame_load(ctx, op_size, op_size < OP_SIZE_4, slot_1, 0, R_SCRATCH_1));
4123 g(gen_frame_load(ctx, op_size, op_size < OP_SIZE_4, slot_2, 0, R_SCRATCH_2));
4124 gen_insn(INSN_ALU, OP_SIZE_8, ALU_MUL, ALU_WRITES_FLAGS(ALU_MUL, false));
4125 gen_one(R_SCRATCH_1);
4126 gen_one(ARG_EXTENDED_REGISTER);
4127 gen_one(ARG_EXTEND_SXTW);
4128 gen_one(R_SCRATCH_1);
4129 gen_one(ARG_EXTENDED_REGISTER);
4130 gen_one(ARG_EXTEND_SXTW);
4131 gen_one(R_SCRATCH_2);
4132
4133 gen_insn(INSN_CMP, OP_SIZE_8, 0, 1);
4134 gen_one(R_SCRATCH_1);
4135 gen_one(ARG_EXTENDED_REGISTER);
4136 gen_one(ARG_EXTEND_SXTW);
4137 gen_one(R_SCRATCH_1);
4138
4139 gen_insn(INSN_JMP_COND, OP_SIZE_8, COND_NE, 0);
4140 gen_four(label_ovf);
4141
4142 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
4143
4144 return true;
4145 }
4146 if (mode == MODE_INT && op_size == OP_SIZE_8) {
4147 g(gen_frame_load(ctx, op_size, op_size < OP_SIZE_4, slot_1, 0, R_SCRATCH_1));
4148 g(gen_frame_load(ctx, op_size, op_size < OP_SIZE_4, slot_2, 0, R_SCRATCH_2));
4149 gen_insn(INSN_ALU, OP_SIZE_8, ALU_SMULH, ALU_WRITES_FLAGS(ALU_SMULH, false));
4150 gen_one(R_SCRATCH_3);
4151 gen_one(R_SCRATCH_1);
4152 gen_one(R_SCRATCH_2);
4153
4154 gen_insn(INSN_ALU, OP_SIZE_8, ALU_MUL, ALU_WRITES_FLAGS(ALU_MUL, false));
4155 gen_one(R_SCRATCH_1);
4156 gen_one(R_SCRATCH_1);
4157 gen_one(R_SCRATCH_2);
4158
4159 gen_insn(INSN_CMP, OP_SIZE_8, 0, 1);
4160 gen_one(R_SCRATCH_3);
4161 gen_one(ARG_SHIFTED_REGISTER);
4162 gen_one(ARG_SHIFT_ASR | 0x3f);
4163 gen_one(R_SCRATCH_1);
4164
4165 gen_insn(INSN_JMP_COND, OP_SIZE_8, COND_NE, 0);
4166 gen_four(label_ovf);
4167
4168 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
4169
4170 return true;
4171 }
4172 #endif
4173 #if defined(ARCH_POWER)
4174 if (mode == MODE_INT && op_size == OP_SIZE_NATIVE) {
4175 g(gen_frame_load(ctx, op_size, true, slot_1, 0, R_SCRATCH_1));
4176 g(gen_frame_load(ctx, op_size, true, slot_2, 0, R_SCRATCH_2));
4177
4178 gen_insn(INSN_ALU, op_size, ALU_MUL, 1);
4179 gen_one(R_SCRATCH_1);
4180 gen_one(R_SCRATCH_1);
4181 gen_one(R_SCRATCH_2);
4182
4183 gen_insn(INSN_JMP_COND, op_size, COND_O, 0);
4184 gen_four(label_ovf);
4185
4186 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
4187
4188 return true;
4189 }
4190 #endif
4191 #if defined(ARCH_LOONGARCH64) || (defined(ARCH_MIPS) && MIPS_R6) || defined(ARCH_RISCV64)
4192 if (mode == MODE_INT && op_size == OP_SIZE_NATIVE) {
4193 g(gen_frame_load(ctx, op_size, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1));
4194 g(gen_frame_load(ctx, op_size, OP_SIZE_NATIVE, slot_2, 0, R_SCRATCH_2));
4195
4196 gen_insn(INSN_ALU, OP_SIZE_NATIVE, ALU_SMULH, ALU_WRITES_FLAGS(ALU_SMULH, false));
4197 gen_one(R_SCRATCH_3);
4198 gen_one(R_SCRATCH_1);
4199 gen_one(R_SCRATCH_2);
4200
4201 gen_insn(INSN_ALU, OP_SIZE_NATIVE, ALU_MUL, ALU_WRITES_FLAGS(ALU_MUL, false));
4202 gen_one(R_SCRATCH_1);
4203 gen_one(R_SCRATCH_1);
4204 gen_one(R_SCRATCH_2);
4205
4206 gen_insn(INSN_ROT, OP_SIZE_NATIVE, ROT_SAR, ROT_WRITES_FLAGS(ROT_SAR));
4207 gen_one(R_SCRATCH_4);
4208 gen_one(R_SCRATCH_1);
4209 gen_one(ARG_IMM);
4210 gen_eight(0x3f);
4211
4212 g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, R_SCRATCH_3, R_SCRATCH_4, COND_NE, label_ovf));
4213
4214 g(gen_frame_store(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_1));
4215
4216 return true;
4217 }
4218 #endif
4219 #if defined(ARCH_S390)
4220 if (mode == MODE_INT && op_size >= OP_SIZE_4 && likely(cpu_test_feature(CPU_FEATURE_misc_insn_ext_2))) {
4221 g(gen_frame_load(ctx, op_size, true, slot_1, 0, R_SCRATCH_1));
4222 g(gen_frame_load_op(ctx, op_size, true, ALU_MUL, 1, slot_2, 0, R_SCRATCH_1));
4223
4224 gen_insn(INSN_JMP_COND, op_size, COND_O, 0);
4225 gen_four(label_ovf);
4226
4227 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
4228 return true;
4229 }
4230 #endif
4231 #if (defined(ARCH_MIPS) && !MIPS_R6) || defined(ARCH_S390)
4232 #if defined(ARCH_MIPS)
4233 if (mode == MODE_INT && op_size >= OP_SIZE_4)
4234 #endif
4235 #if defined(ARCH_S390)
4236 if (mode == MODE_INT && op_size == OP_SIZE_4)
4237 #endif
4238 {
4239 g(gen_frame_load(ctx, op_size, true, slot_1, 0, R_SCRATCH_1));
4240 g(gen_frame_load(ctx, op_size, true, slot_2, 0, R_SCRATCH_3));
4241
4242 gen_insn(INSN_MUL_L, op_size, 0, 0);
4243 gen_one(R_SCRATCH_1);
4244 gen_one(R_SCRATCH_2);
4245 gen_one(R_SCRATCH_1);
4246 gen_one(R_SCRATCH_3);
4247
4248 g(gen_3address_rot_imm(ctx, op_size, ROT_SAR, R_SCRATCH_4, R_SCRATCH_1, (1U << (op_size + 3)) - 1, false));
4249
4250 g(gen_cmp_test_jmp(ctx, INSN_CMP, op_size, R_SCRATCH_2, R_SCRATCH_4, COND_NE, label_ovf));
4251
4252 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
4253 return true;
4254 }
4255 #endif
4256 if (mode == MODE_INT && op_size == OP_SIZE_NATIVE) {
4257 g(gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, INT_binary_multiply_int8_t), op_size, slot_1, slot_2, slot_r, label_ovf));
4258 return true;
4259 }
4260
4261 g(gen_frame_load(ctx, op_size, true, slot_1, 0, R_SCRATCH_1));
4262 if (op_size < OP_SIZE_NATIVE && mode == MODE_INT) {
4263 g(gen_frame_load(ctx, op_size, true, slot_2, 0, R_SCRATCH_2));
4264
4265 gen_insn(INSN_ALU, OP_SIZE_NATIVE, ALU_MUL, ALU_WRITES_FLAGS(ALU_MUL, false));
4266 gen_one(R_SCRATCH_1);
4267 gen_one(R_SCRATCH_1);
4268 gen_one(R_SCRATCH_2);
4269 } else {
4270 g(gen_frame_load_op(ctx, op_size, true, ALU_MUL, 0, slot_2, 0, R_SCRATCH_1));
4271 }
4272
4273 if (mode == MODE_INT) {
4274 g(gen_cmp_extended(ctx, OP_SIZE_NATIVE, op_size, R_SCRATCH_1, R_SCRATCH_2, label_ovf));
4275 }
4276
4277 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
4278
4279 return true;
4280 }
4281
4282 /**********
4283 * DIVIDE *
4284 **********/
4285 do_divide: {
4286 uint32_t attr_unused label_skip = 0; /* avoid warning */
4287 uint32_t attr_unused label_skip2 = 0; /* avoid warning */
4288 uint32_t attr_unused label_end = 0; /* avoid warning */
4289 uint32_t attr_unused label_div_0 = 0; /* avoid warning */
4290 unsigned attr_unused divide_alu = 0; /* avoid warning */
4291 bool attr_unused have_mod = false;
4292 bool attr_unused force_sx = false;
4293 unsigned attr_unused div_op_size = i_size(op_size);
4294 if (unlikely(op_size > OP_SIZE_NATIVE) || unlikely(!ARCH_HAS_DIV)
4295 #if defined(ARCH_S390)
4296 || !(Z || (op_size <= OP_SIZE_4 && sgn))
4297 #endif
4298 ) {
4299 size_t upcall;
4300 if (mode == MODE_INT) {
4301 upcall = !mod ? offsetof(struct cg_upcall_vector_s, INT_binary_divide_int8_t) : offsetof(struct cg_upcall_vector_s, INT_binary_modulo_int8_t);
4302 } else if (sgn) {
4303 upcall = !mod ? offsetof(struct cg_upcall_vector_s, FIXED_binary_divide_int8_t) : offsetof(struct cg_upcall_vector_s, FIXED_binary_modulo_int8_t);
4304 } else {
4305 upcall = !mod ? offsetof(struct cg_upcall_vector_s, FIXED_binary_udivide_int8_t) : offsetof(struct cg_upcall_vector_s, FIXED_binary_umodulo_int8_t);
4306 }
4307 g(gen_alu_typed_upcall(ctx, upcall, op_size, slot_1, slot_2, slot_r, mode == MODE_INT ? label_ovf : 0));
4308 return true;
4309 }
4310 #if defined(ARCH_X86) || defined(ARCH_S390)
4311 if (mode == MODE_FIXED) {
4312 label_skip = alloc_label(ctx);
4313 if (unlikely(!label_skip))
4314 return false;
4315 label_end = alloc_label(ctx);
4316 if (unlikely(!label_end))
4317 return false;
4318 if (sgn) {
4319 label_skip2 = alloc_label(ctx);
4320 if (unlikely(!label_skip2))
4321 return false;
4322 }
4323 }
4324 #if defined(ARCH_X86)
4325 if (R_SCRATCH_1 != R_AX || R_SCRATCH_2 != R_DX || R_SCRATCH_3 != R_CX)
4326 internal(file_line, "gen_alu: bad scratch registers");
4327 #endif
4328 g(gen_frame_load(ctx, op_size, sgn, slot_1, 0, R_SCRATCH_1));
4329 g(gen_frame_load(ctx, op_size, sgn, slot_2, 0, R_SCRATCH_3));
4330
4331 g(gen_jmp_on_zero(ctx, i_size(op_size), R_SCRATCH_3, COND_E, mode == MODE_INT ? label_ovf : label_skip));
4332
4333 if (sgn) {
4334 uint64_t val;
4335 uint32_t label_not_minus_1;
4336 label_not_minus_1 = alloc_label(ctx);
4337 if (unlikely(!label_not_minus_1))
4338 return false;
4339
4340 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, op_size, R_SCRATCH_3, -1, COND_NE, label_not_minus_1));
4341
4342 val = -(uint64_t)0x80 << (((1 << op_size) - 1) * 8);
4343 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, op_size, R_SCRATCH_1, val, COND_E, mode == MODE_INT ? label_ovf : label_skip2));
4344
4345 gen_label(label_not_minus_1);
4346 }
4347
4348 #if defined(ARCH_X86)
4349 if (op_size >= OP_SIZE_2) {
4350 if (sgn) {
4351 gen_insn(INSN_CWD + ARCH_PARTIAL_ALU(op_size), op_size, 0, 0);
4352 gen_one(R_SCRATCH_2);
4353 gen_one(R_SCRATCH_1);
4354 if (op_size == OP_SIZE_2)
4355 gen_one(R_SCRATCH_2);
4356 } else {
4357 gen_insn(INSN_ALU, OP_SIZE_4, ALU_XOR, 1);
4358 gen_one(R_SCRATCH_2);
4359 gen_one(R_SCRATCH_2);
4360 gen_one(R_SCRATCH_2);
4361 }
4362 }
4363 gen_insn(INSN_DIV_L, op_size, sgn, 1);
4364 gen_one(R_SCRATCH_1);
4365 gen_one(i_size(op_size) == OP_SIZE_1 ? R_SCRATCH_1 : R_SCRATCH_2);
4366 gen_one(R_SCRATCH_1);
4367 gen_one(i_size(op_size) == OP_SIZE_1 ? R_SCRATCH_1 : R_SCRATCH_2);
4368 gen_one(R_SCRATCH_3);
4369 #else
4370 if (!sgn && op_size < OP_SIZE_4) {
4371 g(gen_extend(ctx, op_size, false, R_SCRATCH_1, R_SCRATCH_1));
4372 g(gen_extend(ctx, op_size, false, R_SCRATCH_3, R_SCRATCH_3));
4373 }
4374 if (!sgn) {
4375 g(gen_load_constant(ctx, R_SCRATCH_2, 0));
4376 } else if (op_size <= OP_SIZE_4) {
4377 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_2, R_SCRATCH_1, (1U << (OP_SIZE_NATIVE + 3)) - 1, false));
4378 }
4379 gen_insn(INSN_DIV_L, i_size(op_size), sgn, 1);
4380 gen_one(R_SCRATCH_2);
4381 gen_one(R_SCRATCH_1);
4382 gen_one(R_SCRATCH_2);
4383 gen_one(R_SCRATCH_1);
4384 gen_one(R_SCRATCH_3);
4385 #endif
4386 if (mod && i_size(op_size) == OP_SIZE_1) {
4387 gen_insn(INSN_ROT_PARTIAL, OP_SIZE_2, ROT_SHR, 1);
4388 gen_one(R_SCRATCH_1);
4389 gen_one(R_SCRATCH_1);
4390 gen_one(ARG_IMM);
4391 gen_eight(8);
4392 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
4393 } else if (mod) {
4394 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_2));
4395 } else {
4396 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
4397 }
4398 if (mode == MODE_FIXED) {
4399 gen_insn(INSN_JMP, 0, 0, 0);
4400 gen_four(label_end);
4401
4402 if (sgn) {
4403 gen_label(label_skip2);
4404
4405 if (mod)
4406 g(gen_frame_clear(ctx, op_size, slot_r));
4407 else
4408 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
4409
4410 gen_insn(INSN_JMP, 0, 0, 0);
4411 gen_four(label_end);
4412 }
4413
4414 gen_label(label_skip);
4415 if (!mod)
4416 g(gen_frame_clear(ctx, op_size, slot_r));
4417 else
4418 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
4419 gen_label(label_end);
4420 }
4421 return true;
4422 #else
4423 #if defined(ARCH_MIPS)
4424 have_mod = true;
4425 div_op_size = maximum(op_size, OP_SIZE_4);
4426 if (op_size == OP_SIZE_4)
4427 force_sx = true;
4428 #endif
4429 #if defined(ARCH_POWER)
4430 have_mod = cpu_test_feature(CPU_FEATURE_v30);
4431 div_op_size = maximum(op_size, OP_SIZE_4);
4432 #endif
4433 #if defined(ARCH_LOONGARCH64) || defined(ARCH_RISCV64)
4434 have_mod = true;
4435 div_op_size = maximum(op_size, OP_SIZE_4);
4436 #endif
4437 label_end = alloc_label(ctx);
4438 if (unlikely(!label_end))
4439 return false;
4440
4441 g(gen_frame_load(ctx, op_size, (sgn && op_size < i_size(op_size)) || force_sx, slot_1, 0, R_SCRATCH_1));
4442 g(gen_frame_load(ctx, op_size, (sgn && op_size < i_size(op_size)) || force_sx, slot_2, 0, R_SCRATCH_2));
4443
4444 if (ARCH_PREFERS_SX(op_size) && !sgn && op_size < i_size(op_size)) {
4445 g(gen_extend(ctx, op_size, false, R_SCRATCH_1, R_SCRATCH_1));
4446 g(gen_extend(ctx, op_size, false, R_SCRATCH_2, R_SCRATCH_2));
4447 }
4448
4449 if (mode == MODE_INT) {
4450 g(gen_jmp_on_zero(ctx, i_size(op_size), R_SCRATCH_2, COND_E, label_ovf));
4451 if (sgn) {
4452 uint64_t val;
4453 uint32_t label_not_minus_1;
4454 label_not_minus_1 = alloc_label(ctx);
4455 if (unlikely(!label_not_minus_1))
4456 return false;
4457
4458 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, i_size(op_size), R_SCRATCH_2, -1, COND_NE, label_not_minus_1));
4459
4460 val = 0xFFFFFFFFFFFFFF80ULL << (((1 << op_size) - 1) * 8);
4461 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, i_size(op_size), R_SCRATCH_1, val, COND_E, label_ovf));
4462
4463 gen_label(label_not_minus_1);
4464 }
4465 } else {
4466 #if !(defined(ARCH_ARM) && ARM_ASM_DIV_NO_TRAP)
4467 if (!mod) {
4468 g(gen_load_constant(ctx, R_SCRATCH_3, 0));
4469 } else {
4470 gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
4471 gen_one(R_SCRATCH_3);
4472 gen_one(R_SCRATCH_1);
4473 }
4474 g(gen_jmp_on_zero(ctx, i_size(op_size), R_SCRATCH_2, COND_E, label_end));
4475 if (sgn) {
4476 uint64_t val;
4477 uint32_t label_not_minus_1;
4478 label_not_minus_1 = alloc_label(ctx);
4479 if (unlikely(!label_not_minus_1))
4480 return false;
4481
4482 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, i_size(op_size), R_SCRATCH_2, -1, COND_NE, label_not_minus_1));
4483
4484 if (!mod) {
4485 gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
4486 gen_one(R_SCRATCH_3);
4487 gen_one(R_SCRATCH_1);
4488 } else {
4489 g(gen_load_constant(ctx, R_SCRATCH_3, 0));
4490 }
4491
4492 val = 0xFFFFFFFFFFFFFF80ULL << (((1 << op_size) - 1) * 8);
4493 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, i_size(op_size), R_SCRATCH_1, val, COND_E, label_end));
4494
4495 gen_label(label_not_minus_1);
4496 }
4497 #endif
4498 }
4499 if (mod && have_mod) {
4500 g(gen_3address_alu(ctx, div_op_size, sgn ? ALU_SREM : ALU_UREM, R_SCRATCH_3, R_SCRATCH_1, R_SCRATCH_2));
4501 } else {
4502 g(gen_3address_alu(ctx, div_op_size, sgn ? ALU_SDIV : ALU_UDIV, R_SCRATCH_3, R_SCRATCH_1, R_SCRATCH_2));
4503 }
4504
4505 if (mod && !have_mod) {
4506 #if defined(ARCH_ARM)
4507 gen_insn(INSN_MADD, i_size(op_size), 1, 0);
4508 gen_one(R_SCRATCH_3);
4509 gen_one(R_SCRATCH_3);
4510 gen_one(R_SCRATCH_2);
4511 gen_one(R_SCRATCH_1);
4512 #else
4513 g(gen_3address_alu(ctx, i_size(op_size), ALU_MUL, R_SCRATCH_2, R_SCRATCH_2, R_SCRATCH_3));
4514
4515 g(gen_3address_alu(ctx, i_size(op_size), ALU_SUB, R_SCRATCH_3, R_SCRATCH_1, R_SCRATCH_2));
4516 #endif
4517 }
4518
4519 gen_label(label_end);
4520 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_3));
4521 return true;
4522 #endif
4523 }
4524 /*********
4525 * SHIFT *
4526 *********/
4527 do_shift: {
4528 bool sx;
4529 bool must_mask;
4530 unsigned op_s;
4531 if (unlikely(op_size > OP_SIZE_NATIVE)) {
4532 size_t upcall;
4533 if (mode == MODE_FIXED) {
4534 switch (alu) {
4535 case ROT_SHL: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_shl_,TYPE_INT_MAX));
4536 break;
4537 case ROT_SAR: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_shr_,TYPE_INT_MAX));
4538 break;
4539 case ROT_SHR: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_ushr_,TYPE_INT_MAX));
4540 break;
4541 case ROT_ROL: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_rol_,TYPE_INT_MAX));
4542 break;
4543 case ROT_ROR: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_ror_,TYPE_INT_MAX));
4544 break;
4545 default: internal(file_line, "do_alu: invalid shift %u", alu);
4546 }
4547 } else {
4548 switch (alu) {
4549 case ROT_SHL: upcall = offsetof(struct cg_upcall_vector_s, cat(INT_binary_shl_,TYPE_INT_MAX));
4550 break;
4551 case ROT_SAR: upcall = offsetof(struct cg_upcall_vector_s, cat(INT_binary_shr_,TYPE_INT_MAX));
4552 break;
4553 default: internal(file_line, "do_alu: invalid shift %u", alu);
4554 }
4555 }
4556 g(gen_alu_upcall(ctx, upcall, slot_1, slot_2, slot_r, mode == MODE_INT ? label_ovf : 0));
4557 return true;
4558 }
4559 op_s = i_size_rot(op_size);
4560 #if defined(ARCH_X86)
4561 if (mode == MODE_INT && alu == ROT_SHL && op_size < OP_SIZE_NATIVE)
4562 op_s = op_size + 1;
4563 #endif
4564 must_mask = op_size < ARCH_SHIFT_SIZE;
4565 sx = (alu == ROT_SAR && op_size < op_s) || (alu == ROT_SHL && op_size < OP_SIZE_NATIVE && mode == MODE_INT);
4566 #if defined(ARCH_MIPS)
4567 sx |= op_size == OP_SIZE_4;
4568 #endif
4569 g(gen_frame_load(ctx, op_size, sx, slot_1, 0, R_SCRATCH_1));
4570 g(gen_frame_load(ctx, op_size, false, slot_2, 0, R_SCRATCH_3));
4571 if (ARCH_PREFERS_SX(op_size) && !sx && op_size < op_s)
4572 g(gen_extend(ctx, op_size, false, R_SCRATCH_1, R_SCRATCH_1));
4573
4574 if (mode == MODE_INT) {
4575 int64_t imm = (1U << (op_size + 3)) - 1;
4576 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, maximum(op_size, OP_SIZE_4), R_SCRATCH_3, imm, COND_A, label_ovf));
4577 } else {
4578 #if defined(ARCH_ARM)
4579 if (alu == ROT_ROL) {
4580 gen_insn(INSN_ALU1, OP_SIZE_4, ALU1_NEG, ALU1_WRITES_FLAGS(ALU1_NEG));
4581 gen_one(R_SCRATCH_3);
4582 gen_one(R_SCRATCH_3);
4583 alu = ROT_ROR;
4584 }
4585 #endif
4586 #if defined(ARCH_LOONGARCH64)
4587 if (alu == ROT_ROL && op_size >= OP_SIZE_4) {
4588 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_NEG, ALU1_WRITES_FLAGS(ALU1_NEG));
4589 gen_one(R_SCRATCH_3);
4590 gen_one(R_SCRATCH_3);
4591 alu = ROT_ROR;
4592 }
4593 #endif
4594 #if defined(ARCH_MIPS)
4595 if (MIPS_HAS_ROT && alu == ROT_ROL && op_size >= OP_SIZE_4) {
4596 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_NEG, ALU1_WRITES_FLAGS(ALU1_NEG));
4597 gen_one(R_SCRATCH_3);
4598 gen_one(R_SCRATCH_3);
4599 alu = ROT_ROR;
4600 }
4601 #endif
4602 #if defined(ARCH_POWER)
4603 if (alu == ROT_ROR && op_size >= OP_SIZE_4) {
4604 gen_insn(INSN_ALU1 + ARCH_PARTIAL_ALU(OP_SIZE_NATIVE), OP_SIZE_NATIVE, ALU1_NEG, ALU1_WRITES_FLAGS(ALU1_NEG));
4605 gen_one(R_SCRATCH_3);
4606 gen_one(R_SCRATCH_3);
4607 alu = ROT_ROL;
4608 }
4609 #endif
4610 #if defined(ARCH_S390)
4611 if (Z && alu == ROT_ROR && op_size >= OP_SIZE_4) {
4612 gen_insn(INSN_ALU1 + ARCH_PARTIAL_ALU(OP_SIZE_4), OP_SIZE_4, ALU1_NEG, ALU1_WRITES_FLAGS(ALU1_NEG));
4613 gen_one(R_SCRATCH_3);
4614 gen_one(R_SCRATCH_3);
4615 alu = ROT_ROL;
4616 }
4617 #endif
4618 if (must_mask) {
4619 g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_4), ALU_AND, R_SCRATCH_3, R_SCRATCH_3, (1U << (op_size + 3)) - 1));
4620 }
4621 }
4622
4623 #if defined(ARCH_X86)
4624 if (mode == MODE_INT && alu == ROT_SHL && op_size == OP_SIZE_NATIVE) {
4625 gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
4626 gen_one(R_SCRATCH_2);
4627 gen_one(R_SCRATCH_1);
4628 }
4629
4630 g(gen_3address_rot(ctx, op_s, alu, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_3));
4631
4632 if (mode == MODE_INT && alu == ROT_SHL) {
4633 if (op_size < OP_SIZE_NATIVE) {
4634 gen_insn(INSN_MOVSX, op_size, 0, 0);
4635 gen_one(R_SCRATCH_3);
4636 gen_one(R_SCRATCH_1);
4637
4638 g(gen_cmp_test_jmp(ctx, INSN_CMP, op_s, R_SCRATCH_1, R_SCRATCH_3, COND_NE, label_ovf));
4639 } else {
4640 gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
4641 gen_one(R_SCRATCH_4);
4642 gen_one(R_SCRATCH_1);
4643
4644 g(gen_3address_rot(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_4, R_SCRATCH_4, R_SCRATCH_3));
4645
4646 g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, R_SCRATCH_2, R_SCRATCH_4, COND_NE, label_ovf));
4647 }
4648 }
4649 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
4650 return true;
4651 #endif
4652 #if defined(ARCH_ARM)
4653 if (op_size <= OP_SIZE_2 && alu == ROT_ROR) {
4654 gen_insn(INSN_ALU, OP_SIZE_4, ALU_OR, ALU_WRITES_FLAGS(ALU_OR, false));
4655 gen_one(R_SCRATCH_1);
4656 gen_one(R_SCRATCH_1);
4657 gen_one(ARG_SHIFTED_REGISTER);
4658 gen_one(ARG_SHIFT_LSL | (1U << (op_size + 3)));
4659 gen_one(R_SCRATCH_1);
4660 if (op_size == OP_SIZE_1)
4661 alu = ROT_SHR;
4662 }
4663 goto do_generic_shift;
4664 #endif
4665 #if defined(ARCH_LOONGARCH64)
4666 if (alu == ROT_ROR && op_size >= OP_SIZE_4)
4667 goto do_generic_shift;
4668 #endif
4669 #if defined(ARCH_MIPS)
4670 if (MIPS_HAS_ROT && alu == ROT_ROR && op_size >= OP_SIZE_4)
4671 goto do_generic_shift;
4672 #endif
4673 #if defined(ARCH_POWER)
4674 if (alu == ROT_ROL && op_size >= OP_SIZE_4)
4675 goto do_generic_shift;
4676 #endif
4677 #if defined(ARCH_RISCV64)
4678 if ((alu == ROT_ROL || alu == ROT_ROR) && likely(cpu_test_feature(CPU_FEATURE_zbb))) {
4679 if (likely(op_size >= OP_SIZE_4)) {
4680 goto do_generic_shift;
4681 }
4682 }
4683 #endif
4684 #if defined(ARCH_S390)
4685 if (Z && alu == ROT_ROL && op_size >= OP_SIZE_4)
4686 goto do_generic_shift;
4687 #endif
4688 if (alu == ROT_ROL || alu == ROT_ROR) {
4689 g(gen_3address_rot(ctx, op_s, alu == ROT_ROL ? ROT_SHL : ROT_SHR, R_SCRATCH_2, R_SCRATCH_1, R_SCRATCH_3));
4690 gen_insn(INSN_ALU1 + ARCH_PARTIAL_ALU(i_size(OP_SIZE_4)), i_size(OP_SIZE_4), ALU1_NEG, ALU1_WRITES_FLAGS(ALU1_NEG));
4691 gen_one(R_SCRATCH_3);
4692 gen_one(R_SCRATCH_3);
4693 if (must_mask) {
4694 g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_4), ALU_AND, R_SCRATCH_3, R_SCRATCH_3, (1U << (op_size + 3)) - 1));
4695 }
4696 g(gen_3address_rot(ctx, op_s, alu == ROT_ROL ? ROT_SHR : ROT_SHL, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_3));
4697 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_OR, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_2));
4698 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
4699 return true;
4700 }
4701
4702 goto do_generic_shift;
4703 do_generic_shift:
4704 if (mode == MODE_INT && alu == ROT_SHL) {
4705 #if defined(ARCH_S390)
4706 if (op_size >= OP_SIZE_4) {
4707 g(gen_3address_rot(ctx, op_size, ROT_SAL, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_3));
4708
4709 gen_insn(INSN_JMP_COND, op_size, COND_O, 0);
4710 gen_four(label_ovf);
4711 } else
4712 #endif
4713 if (op_size <= OP_SIZE_NATIVE - 1) {
4714 g(gen_3address_rot(ctx, OP_SIZE_NATIVE, alu, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_3));
4715
4716 g(gen_cmp_extended(ctx, OP_SIZE_NATIVE, op_size, R_SCRATCH_1, R_SCRATCH_2, label_ovf));
4717 } else {
4718 g(gen_3address_rot(ctx, OP_SIZE_NATIVE, alu, R_SCRATCH_2, R_SCRATCH_1, R_SCRATCH_3));
4719
4720 g(gen_3address_rot(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_4, R_SCRATCH_2, R_SCRATCH_3));
4721
4722 g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, R_SCRATCH_1, R_SCRATCH_4, COND_NE, label_ovf));
4723
4724 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_2));
4725
4726 return true;
4727 }
4728 } else {
4729 g(gen_3address_rot(ctx, op_s, alu, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_3));
4730 }
4731
4732 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
4733 return true;
4734 }
4735 /******
4736 * BT *
4737 ******/
4738 do_bt: {
4739 unsigned attr_unused op_s;
4740 bool need_mask;
4741 if (unlikely(op_size > OP_SIZE_NATIVE)) {
4742 size_t upcall;
4743 if (mode == MODE_FIXED) {
4744 switch (alu) {
4745 case BTX_BTS: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_bts_,TYPE_INT_MAX));
4746 break;
4747 case BTX_BTR: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_btr_,TYPE_INT_MAX));
4748 break;
4749 case BTX_BTC: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_btc_,TYPE_INT_MAX));
4750 break;
4751 case BTX_BT: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_bt_,TYPE_INT_MAX));
4752 break;
4753 default: internal(file_line, "do_alu: invalid bit test %u", alu);
4754 }
4755 } else {
4756 switch (alu) {
4757 case BTX_BTS: upcall = offsetof(struct cg_upcall_vector_s, cat(INT_binary_bts_,TYPE_INT_MAX));
4758 break;
4759 case BTX_BTR: upcall = offsetof(struct cg_upcall_vector_s, cat(INT_binary_btr_,TYPE_INT_MAX));
4760 break;
4761 case BTX_BTC: upcall = offsetof(struct cg_upcall_vector_s, cat(INT_binary_btc_,TYPE_INT_MAX));
4762 break;
4763 case BTX_BT: upcall = offsetof(struct cg_upcall_vector_s, cat(INT_binary_bt_,TYPE_INT_MAX));
4764 break;
4765 default: internal(file_line, "do_alu: invalid bit test %u", alu);
4766 }
4767 }
4768 g(gen_alu_upcall(ctx, upcall, slot_1, slot_2, slot_r, label_ovf));
4769 return true;
4770 }
4771 op_s = minimum(OP_SIZE_NATIVE, ARCH_SHIFT_SIZE);
4772 op_s = maximum(op_s, op_size);
4773 g(gen_frame_load(ctx, op_size, false, slot_1, 0, R_SCRATCH_1));
4774 g(gen_frame_load(ctx, op_size, false, slot_2, 0, R_SCRATCH_2));
4775 if (mode == MODE_INT) {
4776 int64_t imm = (1U << (op_size + 3)) - 1;
4777 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, maximum(op_size, OP_SIZE_4), R_SCRATCH_2, imm, alu == BTX_BT ? COND_A : COND_AE, label_ovf));
4778 }
4779 if (alu != BTX_BT) {
4780 if (!ARCH_HAS_BTX(alu, OP_SIZE_NATIVE, false))
4781 goto do_generic_bt;
4782 need_mask = !ARCH_HAS_BTX(alu, op_size, false);
4783 } else {
4784 #if defined(ARCH_X86)
4785 need_mask = op_size < OP_SIZE_2;
4786 #else
4787 if (!ARCH_HAS_BTX(BTX_BTEXT, OP_SIZE_NATIVE, false))
4788 goto do_generic_bt;
4789 need_mask = !ARCH_HAS_BTX(BTX_BTEXT, op_size, false);
4790 #endif
4791 }
4792 if (need_mask) {
4793 g(gen_3address_alu_imm(ctx, OP_SIZE_4, ALU_AND, R_SCRATCH_2, R_SCRATCH_2, (1U << (op_size + 3)) - 1));
4794 }
4795 if (alu == BTX_BT) {
4796 #if defined(ARCH_X86)
4797 gen_insn(INSN_BT, maximum(op_size, OP_SIZE_2), 0, 1);
4798 gen_one(R_SCRATCH_1);
4799 gen_one(R_SCRATCH_2);
4800
4801 g(gen_frame_set_cond(ctx, maximum(op_size, OP_SIZE_2), false, COND_B, slot_r));
4802 #else
4803 gen_insn(INSN_BTX, need_mask ? OP_SIZE_NATIVE : op_size, BTX_BTEXT, 0);
4804 gen_one(R_SCRATCH_1);
4805 gen_one(R_SCRATCH_1);
4806 gen_one(R_SCRATCH_2);
4807
4808 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, R_SCRATCH_1));
4809 #endif
4810 } else {
4811 #if defined(ARCH_X86)
4812 gen_insn(INSN_BTX, maximum(op_size, OP_SIZE_2), alu, 1);
4813 #else
4814 gen_insn(INSN_BTX, need_mask ? OP_SIZE_NATIVE : op_size, alu, 0);
4815 #endif
4816 gen_one(R_SCRATCH_1);
4817 gen_one(R_SCRATCH_1);
4818 gen_one(R_SCRATCH_2);
4819
4820 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
4821 }
4822 return true;
4823
4824 goto do_generic_bt;
4825 do_generic_bt:
4826 if (mode == MODE_FIXED && op_size < ARCH_SHIFT_SIZE) {
4827 g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_4), ALU_AND, R_SCRATCH_2, R_SCRATCH_2, (1U << (op_size + 3)) - 1));
4828 }
4829 g(gen_load_constant(ctx, R_SCRATCH_3, 1));
4830
4831 g(gen_3address_rot(ctx, op_s, ROT_SHL, R_SCRATCH_3, R_SCRATCH_3, R_SCRATCH_2));
4832
4833 switch (alu) {
4834 case BTX_BT:
4835 #if ARCH_HAS_FLAGS
4836 #if defined(ARCH_S390) || defined(ARCH_POWER)
4837 gen_insn(INSN_ALU + ARCH_PARTIAL_ALU(i_size(op_size)), i_size(op_size), ALU_AND, 1);
4838 gen_one(R_SCRATCH_1);
4839 gen_one(R_SCRATCH_1);
4840 gen_one(R_SCRATCH_3);
4841 #else
4842 gen_insn(INSN_TEST, i_size(op_size), 0, 1);
4843 gen_one(R_SCRATCH_1);
4844 gen_one(R_SCRATCH_3);
4845 #endif
4846 g(gen_frame_set_cond(ctx, maximum(op_size, OP_SIZE_4), false, COND_NE, slot_r));
4847 #else
4848 g(gen_3address_alu(ctx, i_size(op_size), ALU_AND, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_3));
4849
4850 g(gen_frame_cmp_imm_set_cond_reg(ctx, i_size(op_size), R_SCRATCH_1, 0, COND_NE, slot_r));
4851 #endif
4852 return true;
4853 case BTX_BTS:
4854 g(gen_3address_alu(ctx, i_size(op_size), ALU_OR, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_3));
4855 break;
4856 case BTX_BTR:
4857 if (!ARCH_HAS_ANDN) {
4858 g(gen_3address_alu_imm(ctx, i_size(op_size), ALU_XOR, R_SCRATCH_3, R_SCRATCH_3, -1));
4859
4860 g(gen_3address_alu(ctx, i_size(op_size), ALU_AND, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_3));
4861 break;
4862 }
4863 g(gen_3address_alu(ctx, i_size(op_size), ALU_ANDN, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_3));
4864 break;
4865 case BTX_BTC:
4866 g(gen_3address_alu(ctx, i_size(op_size), ALU_XOR, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_3));
4867 break;
4868 default:
4869 internal(file_line, "gen_alu: unsupported bit test %u", alu);
4870 }
4871
4872 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
4873
4874 return true;
4875 }
4876 /***********
4877 * COMPARE *
4878 ***********/
4879 do_compare: {
4880 if (unlikely(op_size > OP_SIZE_NATIVE)) {
4881 size_t attr_unused upcall;
4882 switch (alu) {
4883 case COND_E:
4884 case COND_NE:
4885 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
4886 g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, false, ALU_XOR, 0, slot_2, lo_word(OP_SIZE_NATIVE), R_SCRATCH_1));
4887 g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, false, ALU_XOR, 0, slot_2, hi_word(OP_SIZE_NATIVE), R_SCRATCH_2));
4888 #if defined(ARCH_ARM64)
4889 gen_insn(INSN_ALU, OP_SIZE_NATIVE, ALU_OR, ALU_WRITES_FLAGS(ALU_OR, false));
4890 gen_one(R_SCRATCH_1);
4891 gen_one(R_SCRATCH_1);
4892 gen_one(R_SCRATCH_2);
4893
4894 gen_insn(INSN_CMP, OP_SIZE_NATIVE, 0, 1);
4895 gen_one(R_SCRATCH_1);
4896 gen_one(ARG_IMM);
4897 gen_eight(0);
4898 #else
4899 gen_insn(INSN_ALU, OP_SIZE_NATIVE, ALU_OR, ARCH_HAS_FLAGS);
4900 gen_one(R_SCRATCH_1);
4901 gen_one(R_SCRATCH_1);
4902 gen_one(R_SCRATCH_2);
4903 #endif
4904 #if ARCH_HAS_FLAGS
4905 g(gen_frame_set_cond(ctx, OP_SIZE_NATIVE, false, alu, slot_r));
4906 #else
4907 g(gen_frame_cmp_imm_set_cond_reg(ctx, OP_SIZE_NATIVE, R_SCRATCH_1, 0, alu, slot_r));
4908 #endif
4909 return true;
4910 #if defined(ARCH_X86) || defined(ARCH_ARM)
4911 case COND_L:
4912 case COND_B:
4913 g(gen_frame_load(ctx, OP_SIZE_NATIVE, false, slot_2, lo_word(OP_SIZE_NATIVE), R_SCRATCH_2));
4914 g(gen_frame_load(ctx, OP_SIZE_NATIVE, false, slot_1, hi_word(OP_SIZE_NATIVE), R_SCRATCH_1));
4915 g(gen_frame_load_cmp(ctx, OP_SIZE_NATIVE, false, false, true, slot_1, lo_word(OP_SIZE_NATIVE), R_SCRATCH_2));
4916 g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, false, ALU_SBB, true, slot_2, hi_word(OP_SIZE_NATIVE), R_SCRATCH_1));
4917 g(gen_frame_set_cond(ctx, OP_SIZE_NATIVE, false, alu, slot_r));
4918 return true;
4919 case COND_LE:
4920 case COND_BE:
4921 g(gen_frame_load(ctx, OP_SIZE_NATIVE, false, slot_1, lo_word(OP_SIZE_NATIVE), R_SCRATCH_2));
4922 g(gen_frame_load(ctx, OP_SIZE_NATIVE, false, slot_2, hi_word(OP_SIZE_NATIVE), R_SCRATCH_1));
4923 g(gen_frame_load_cmp(ctx, OP_SIZE_NATIVE, false, false, true, slot_2, lo_word(OP_SIZE_NATIVE), R_SCRATCH_2));
4924 g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, false, ALU_SBB, true, slot_1, hi_word(OP_SIZE_NATIVE), R_SCRATCH_1));
4925 g(gen_frame_set_cond(ctx, OP_SIZE_NATIVE, false, alu == COND_LE ? COND_GE : COND_AE, slot_r));
4926 return true;
4927 #else
4928 case COND_L: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_less_,TYPE_INT_MAX)); goto do_upcall;
4929 case COND_B: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_uless_,TYPE_INT_MAX)); goto do_upcall;
4930 case COND_LE: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_less_equal_,TYPE_INT_MAX)); goto do_upcall;
4931 case COND_BE: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_uless_equal_,TYPE_INT_MAX)); goto do_upcall;
4932 do_upcall: g(gen_alu_upcall(ctx, upcall, slot_1, slot_2, slot_r, 0));
4933 return true;
4934 #endif
4935 default:
4936 internal(file_line, "gen_alu: unsupported condition %u", alu);
4937 }
4938 return false;
4939 }
4940 #if defined(ARCH_X86)
4941 g(gen_frame_load(ctx, op_size, false, slot_1, 0, R_SCRATCH_1));
4942 g(gen_frame_load_cmp_set_cond(ctx, op_size, false, slot_2, 0, R_SCRATCH_1, alu, slot_r));
4943 #else
4944 g(gen_frame_load(ctx, op_size, alu == COND_L || alu == COND_LE, slot_1, 0, R_SCRATCH_1));
4945 g(gen_frame_load_cmp_set_cond(ctx, op_size, alu == COND_L || alu == COND_LE, slot_2, 0, R_SCRATCH_1, alu, slot_r));
4946 #endif
4947 return true;
4948 }
4949 }
4950
4951 static bool attr_w gen_alu1(struct codegen_context *ctx, unsigned mode, unsigned op_size, unsigned op, uint32_t label_ovf, frame_t slot_1, frame_t slot_r)
4952 {
4953 unsigned alu;
4954 switch (mode) {
4955 case MODE_FIXED: switch (op) {
4956 case OPCODE_FIXED_OP_not: alu = ALU1_NOT; goto do_alu;
4957 case OPCODE_FIXED_OP_neg: alu = ALU1_NEG; goto do_alu;
4958 case OPCODE_FIXED_OP_inc: alu = ALU1_INC; goto do_alu;
4959 case OPCODE_FIXED_OP_dec: alu = ALU1_DEC; goto do_alu;
4960 case OPCODE_FIXED_OP_bswap:
4961 case OPCODE_FIXED_OP_bswap_alt1: alu = ALU1_BSWAP; goto do_bswap;
4962 case OPCODE_FIXED_OP_brev:
4963 case OPCODE_FIXED_OP_brev_alt1: alu = ALU1_BREV; goto do_brev;
4964 case OPCODE_FIXED_OP_bsf:
4965 case OPCODE_FIXED_OP_bsf_alt1: alu = ALU1_BSF; goto do_bsf_bsr_popcnt;
4966 case OPCODE_FIXED_OP_bsr:
4967 case OPCODE_FIXED_OP_bsr_alt1: alu = ALU1_BSR; goto do_bsf_bsr_popcnt;
4968 case OPCODE_FIXED_OP_popcnt:
4969 case OPCODE_FIXED_OP_popcnt_alt1: alu = ALU1_POPCNT; goto do_bsf_bsr_popcnt;
4970 case OPCODE_FIXED_OP_to_int: goto do_fixed_conv;
4971 case OPCODE_FIXED_OP_from_int: goto do_fixed_conv;
4972 case OPCODE_FIXED_OP_uto_int: goto conv_uto_int;
4973 case OPCODE_FIXED_OP_ufrom_int: goto conv_ufrom_int;
4974 default: internal(file_line, "gen_alu1: unsupported fixed operation %u", op);
4975 }
4976 case MODE_INT: switch (op) {
4977 case OPCODE_INT_OP_not: alu = ALU1_NOT; mode = MODE_FIXED; goto do_alu;
4978 case OPCODE_INT_OP_neg: alu = ALU1_NEG; goto do_alu;
4979 case OPCODE_INT_OP_inc: alu = ALU1_INC; goto do_alu;
4980 case OPCODE_INT_OP_dec: alu = ALU1_DEC; goto do_alu;
4981 case OPCODE_INT_OP_bsf: alu = ALU1_BSF; goto do_bsf_bsr_popcnt;
4982 case OPCODE_INT_OP_bsr: alu = ALU1_BSR; goto do_bsf_bsr_popcnt;
4983 case OPCODE_INT_OP_popcnt:
4984 case OPCODE_INT_OP_popcnt_alt1: alu = ALU1_POPCNT; goto do_bsf_bsr_popcnt;
4985 case OPCODE_INT_OP_to_int: goto do_conv;
4986 case OPCODE_INT_OP_from_int: goto do_conv;
4987 default: internal(file_line, "gen_alu1: unsupported int operation %u", op);
4988 }
4989 case MODE_BOOL: switch (op) {
4990 case OPCODE_BOOL_OP_not: goto do_bool_not;
4991 default: internal(file_line, "gen_alu1: unsupported bool operation %u", op);
4992 }
4993 }
4994 internal(file_line, "gen_alu1: unsupported mode %u", mode);
4995
4996 /*******
4997 * ALU *
4998 *******/
4999 do_alu: {
5000 if (op_size > OP_SIZE_NATIVE) {
5001 #if !defined(ARCH_X86) && !defined(ARCH_ARM) && !defined(ARCH_POWER)
5002 if (alu == ALU1_NEG) {
5003 if (mode == MODE_FIXED)
5004 g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(FIXED_unary_neg_,TYPE_INT_MAX)), slot_1, NO_FRAME_T, slot_r, 0));
5005 else
5006 g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(INT_unary_neg_,TYPE_INT_MAX)), slot_1, NO_FRAME_T, slot_r, label_ovf));
5007 return true;
5008 }
5009 if (alu == ALU1_DEC) {
5010 if (mode == MODE_FIXED)
5011 g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(FIXED_unary_dec_,TYPE_INT_MAX)), slot_1, NO_FRAME_T, slot_r, 0));
5012 else
5013 g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(INT_unary_dec_,TYPE_INT_MAX)), slot_1, NO_FRAME_T, slot_r, label_ovf));
5014 return true;
5015 }
5016 if (alu == ALU1_INC) {
5017 if (mode == MODE_FIXED)
5018 g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(FIXED_unary_inc_,TYPE_INT_MAX)), slot_1, NO_FRAME_T, slot_r, 0));
5019 else
5020 g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(INT_unary_inc_,TYPE_INT_MAX)), slot_1, NO_FRAME_T, slot_r, label_ovf));
5021 return true;
5022 }
5023 #endif
5024 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
5025 #if defined(ARCH_S390)
5026 if (alu == ALU1_NOT) {
5027 g(gen_load_constant(ctx, R_SCRATCH_3, -1));
5028
5029 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_XOR, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_3));
5030 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_XOR, R_SCRATCH_2, R_SCRATCH_2, R_SCRATCH_3));
5031
5032 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_1, R_SCRATCH_2));
5033 return true;
5034 }
5035 #endif
5036 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, alu, (alu == ALU1_INC || alu == ALU1_DEC || alu == ALU1_NEG ? 2 : 0) | ALU1_WRITES_FLAGS(alu));
5037 gen_one(R_SCRATCH_1);
5038 gen_one(R_SCRATCH_1);
5039 if (alu == ALU1_NOT) {
5040 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_NOT, ALU1_WRITES_FLAGS(ALU1_NOT));
5041 gen_one(R_SCRATCH_2);
5042 gen_one(R_SCRATCH_2);
5043 } else if (alu == ALU1_INC || alu == ALU1_DEC) {
5044 g(gen_imm(ctx, 0, alu == ALU1_INC ? IMM_PURPOSE_ADD : IMM_PURPOSE_SUB, OP_SIZE_NATIVE));
5045 gen_insn(INSN_ALU, OP_SIZE_NATIVE, alu == ALU1_INC ? ALU_ADC : ALU_SBB, (mode == MODE_INT) | ALU_WRITES_FLAGS(alu == ALU1_INC ? ALU_ADC : ALU_SBB, is_imm()));
5046 gen_one(R_SCRATCH_2);
5047 gen_one(R_SCRATCH_2);
5048 gen_imm_offset();
5049 } else {
5050 #if defined(ARCH_X86)
5051 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_NOT, ALU1_WRITES_FLAGS(ALU1_NOT));
5052 gen_one(R_SCRATCH_2);
5053 gen_one(R_SCRATCH_2);
5054
5055 g(gen_imm(ctx, -1, IMM_PURPOSE_SUB, OP_SIZE_NATIVE));
5056 gen_insn(INSN_ALU, OP_SIZE_NATIVE, ALU_SBB, ALU_WRITES_FLAGS(ALU_SBB, is_imm()));
5057 gen_one(R_SCRATCH_2);
5058 gen_one(R_SCRATCH_2);
5059 gen_imm_offset();
5060 #else
5061 gen_insn(INSN_ALU1_FLAGS, OP_SIZE_NATIVE, ALU1_NGC, (mode == MODE_INT) | ALU1_WRITES_FLAGS(ALU1_NGC));
5062 gen_one(R_SCRATCH_2);
5063 gen_one(R_SCRATCH_2);
5064 #endif
5065 }
5066 if (mode == MODE_INT) {
5067 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_O, 0);
5068 gen_four(label_ovf);
5069 }
5070 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_1, R_SCRATCH_2));
5071 return true;
5072 }
5073 g(gen_frame_load(ctx, op_size, mode == MODE_INT && op_size >= OP_SIZE_4 && ARCH_SUPPORTS_TRAPS, slot_1, 0, R_SCRATCH_1));
5074 #if defined(ARCH_S390)
5075 if (alu == ALU1_NOT) {
5076 g(gen_3address_alu_imm(ctx, i_size(op_size), ALU_XOR, R_SCRATCH_1, R_SCRATCH_1, -1));
5077
5078 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
5079 return true;
5080 }
5081 #endif
5082 #if defined(ARCH_X86)
5083 gen_insn(INSN_ALU1 + ARCH_PARTIAL_ALU(op_size), op_size, alu, (mode == MODE_INT) | ALU1_WRITES_FLAGS(alu));
5084 gen_one(R_SCRATCH_1);
5085 gen_one(R_SCRATCH_1);
5086 #else
5087 if (mode == MODE_INT) {
5088 bool arch_use_flags = ARCH_HAS_FLAGS;
5089 #if defined(ARCH_POWER)
5090 arch_use_flags = false;
5091 if (op_size == OP_SIZE_NATIVE) {
5092 gen_insn(INSN_ALU1, i_size(op_size), alu, ALU1_WRITES_FLAGS(alu));
5093 gen_one(R_SCRATCH_2);
5094 gen_one(R_SCRATCH_1);
5095 if (alu == ALU1_NEG) {
5096 gen_insn(INSN_ALU, i_size(op_size), ALU_AND, 1);
5097 gen_one(R_CG_SCRATCH);
5098 gen_one(R_SCRATCH_2);
5099 gen_one(R_SCRATCH_1);
5100 } else if (alu == ALU1_INC) {
5101 gen_insn(INSN_ALU, i_size(op_size), ALU_ANDN, 1);
5102 gen_one(R_CG_SCRATCH);
5103 gen_one(R_SCRATCH_2);
5104 gen_one(R_SCRATCH_1);
5105 } else if (alu == ALU1_DEC) {
5106 gen_insn(INSN_ALU, i_size(op_size), ALU_ANDN, 1);
5107 gen_one(R_CG_SCRATCH);
5108 gen_one(R_SCRATCH_1);
5109 gen_one(R_SCRATCH_2);
5110 }
5111 gen_insn(INSN_JMP_COND, op_size, COND_L, 0);
5112 gen_four(label_ovf);
5113
5114 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_2));
5115
5116 return true;
5117 }
5118 #endif
5119 if (!arch_use_flags && !ARCH_SUPPORTS_TRAPS && ARCH_IS_3ADDRESS && ARCH_HAS_ANDN && op_size == OP_SIZE_NATIVE) {
5120 gen_insn(INSN_ALU1 + ARCH_PARTIAL_ALU(i_size(op_size)), i_size(op_size), alu, ALU1_WRITES_FLAGS(alu));
5121 gen_one(R_SCRATCH_2);
5122 gen_one(R_SCRATCH_1);
5123
5124 if (alu == ALU1_NEG) {
5125 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_AND, R_SCRATCH_3, R_SCRATCH_2, R_SCRATCH_1));
5126 } else if (alu == ALU1_INC) {
5127 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_ANDN, R_SCRATCH_3, R_SCRATCH_2, R_SCRATCH_1));
5128 } else if (alu == ALU1_DEC) {
5129 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_ANDN, R_SCRATCH_3, R_SCRATCH_1, R_SCRATCH_2));
5130 }
5131 g(gen_jmp_on_zero(ctx, OP_SIZE_NATIVE, R_SCRATCH_3, COND_S, label_ovf));
5132
5133 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_2));
5134
5135 return true;
5136 }
5137 if (op_size <= OP_SIZE_2 || (!arch_use_flags && !ARCH_SUPPORTS_TRAPS)) {
5138 int64_t imm = ((alu != ALU1_INC && ARCH_PREFERS_SX(op_size) ? -0x80ULL : 0x80ULL) << (((1 << op_size) - 1) * 8)) - (alu == ALU1_INC);
5139
5140 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, R_SCRATCH_1, imm, COND_E, label_ovf));
5141
5142 mode = MODE_FIXED;
5143 }
5144 }
5145 #if !ARCH_HAS_FLAGS
5146 if (mode == MODE_INT) {
5147 gen_insn(INSN_ALU1_TRAP, op_size, alu, ALU1_WRITES_FLAGS(alu));
5148 gen_one(R_SCRATCH_1);
5149 gen_one(R_SCRATCH_1);
5150 gen_four(label_ovf);
5151 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
5152 return true;
5153 }
5154 #endif
5155 gen_insn(INSN_ALU1 + ARCH_PARTIAL_ALU(i_size(op_size)), i_size(op_size), alu, (mode == MODE_INT) | ALU1_WRITES_FLAGS(alu));
5156 gen_one(R_SCRATCH_1);
5157 gen_one(R_SCRATCH_1);
5158 #endif
5159 if (mode == MODE_INT) {
5160 gen_insn(INSN_JMP_COND, maximum(OP_SIZE_4, op_size), COND_O, 0);
5161 gen_four(label_ovf);
5162 }
5163 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
5164 return true;
5165 }
5166
5167 /*******
5168 * NOT *
5169 *******/
5170 do_bool_not: {
5171 g(gen_frame_load(ctx, op_size, false, slot_1, 0, R_SCRATCH_1));
5172
5173 g(gen_3address_alu_imm(ctx, i_size(op_size), ALU_XOR, R_SCRATCH_1, R_SCRATCH_1, 1));
5174
5175 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
5176 return true;
5177 }
5178
5179 /*********
5180 * BSWAP *
5181 *********/
5182 do_bswap: {
5183 bool attr_unused sx = false;
5184 #if defined(ARCH_X86) || defined(ARCH_ARM) || defined(ARCH_IA64) || defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS) || defined(ARCH_RISCV64) || defined(ARCH_S390)
5185 #if defined(ARCH_ARM32)
5186 if (unlikely(!cpu_test_feature(CPU_FEATURE_armv6)))
5187 goto do_generic_bswap;
5188 #endif
5189 #if defined(ARCH_MIPS)
5190 if (unlikely(!MIPS_HAS_ROT))
5191 goto do_generic_bswap;
5192 sx = op_size == OP_SIZE_4;
5193 #endif
5194 #if defined(ARCH_RISCV64)
5195 if (unlikely(!cpu_test_feature(CPU_FEATURE_zbb)))
5196 goto do_generic_bswap;
5197 #endif
5198 #if defined(ARCH_S390)
5199 if (op_size == OP_SIZE_2)
5200 goto do_generic_bswap;
5201 #endif
5202 #if defined(ARCH_X86)
5203 if (op_size >= OP_SIZE_4 && !cpu_test_feature(CPU_FEATURE_bswap))
5204 goto do_generic_bswap;
5205 #endif
5206 if (op_size > OP_SIZE_NATIVE)
5207 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
5208 else
5209 g(gen_frame_load(ctx, op_size, sx, slot_1, 0, R_SCRATCH_1));
5210
5211 if (op_size == OP_SIZE_1) {
5212 #if defined(ARCH_IA64) || defined(ARCH_RISCV64)
5213 } else if (op_size == OP_SIZE_2 || op_size == OP_SIZE_4) {
5214 gen_insn(INSN_ALU1 + ARCH_PARTIAL_ALU(OP_SIZE_NATIVE), OP_SIZE_NATIVE, ALU1_BSWAP, ALU1_WRITES_FLAGS(ALU1_BSWAP));
5215 gen_one(R_SCRATCH_1);
5216 gen_one(R_SCRATCH_1);
5217
5218 gen_insn(INSN_ROT, OP_SIZE_NATIVE, ROT_SAR, ROT_WRITES_FLAGS(ROT_SAR));
5219 gen_one(R_SCRATCH_1);
5220 gen_one(R_SCRATCH_1);
5221 gen_one(ARG_IMM);
5222 gen_eight(op_size == OP_SIZE_2 ? 48 : 32);
5223 #endif
5224 } else if (op_size == OP_SIZE_2) {
5225 #if defined(ARCH_X86)
5226 gen_insn(INSN_ROT_PARTIAL, OP_SIZE_2, ROT_ROR, 1);
5227 gen_one(R_SCRATCH_1);
5228 gen_one(R_SCRATCH_1);
5229 gen_one(ARG_IMM);
5230 gen_eight(8);
5231 #else
5232 gen_insn(INSN_ALU1, OP_SIZE_4, ALU1_BSWAP16, ALU1_WRITES_FLAGS(ALU1_BSWAP16));
5233 gen_one(R_SCRATCH_1);
5234 gen_one(R_SCRATCH_1);
5235 #endif
5236 } else {
5237 gen_insn(INSN_ALU1 + ARCH_PARTIAL_ALU(minimum(op_size, OP_SIZE_NATIVE)), minimum(op_size, OP_SIZE_NATIVE), ALU1_BSWAP, ALU1_WRITES_FLAGS(ALU1_BSWAP));
5238 gen_one(R_SCRATCH_1);
5239 gen_one(R_SCRATCH_1);
5240 }
5241 if (op_size > OP_SIZE_NATIVE) {
5242 gen_insn(INSN_ALU1 + ARCH_PARTIAL_ALU(OP_SIZE_NATIVE), OP_SIZE_NATIVE, ALU1_BSWAP, ALU1_WRITES_FLAGS(ALU1_BSWAP));
5243 gen_one(R_SCRATCH_2);
5244 gen_one(R_SCRATCH_2);
5245 }
5246
5247 if (op_size > OP_SIZE_NATIVE)
5248 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_2, R_SCRATCH_1));
5249 else
5250 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
5251 return true;
5252 #endif
5253 goto do_generic_bswap;
5254 do_generic_bswap:
5255 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_unary_bswap_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, 0);
5256 }
5257 /********
5258 * BREV *
5259 ********/
5260 do_brev: {
5261 #if defined(ARCH_ARM) || defined(ARCH_LOONGARCH64) || (defined(ARCH_MIPS) && MIPS_R6)
5262 #if defined(ARCH_ARM32)
5263 if (unlikely(!cpu_test_feature(CPU_FEATURE_armv6t2)))
5264 goto do_generic_brev;
5265 #endif
5266 if (op_size > OP_SIZE_NATIVE)
5267 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
5268 else
5269 g(gen_frame_load(ctx, op_size, false, slot_1, 0, R_SCRATCH_1));
5270
5271 gen_insn(INSN_ALU1, minimum(maximum(OP_SIZE_4, op_size), OP_SIZE_NATIVE), ALU1_BREV, ALU1_WRITES_FLAGS(ALU1_BREV));
5272 gen_one(R_SCRATCH_1);
5273 gen_one(R_SCRATCH_1);
5274 if (op_size <= OP_SIZE_2) {
5275 gen_insn(INSN_ROT + ARCH_PARTIAL_ALU(OP_SIZE_4), OP_SIZE_4, ROT_SHR, ROT_WRITES_FLAGS(ROT_SHR));
5276 gen_one(R_SCRATCH_1);
5277 gen_one(R_SCRATCH_1);
5278 gen_one(ARG_IMM);
5279 gen_eight(op_size == OP_SIZE_1 ? 24 : 16);
5280 }
5281 if (op_size > OP_SIZE_NATIVE) {
5282 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_BREV, ALU1_WRITES_FLAGS(ALU1_BREV));
5283 gen_one(R_SCRATCH_2);
5284 gen_one(R_SCRATCH_2);
5285 }
5286
5287 if (op_size > OP_SIZE_NATIVE)
5288 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_2, R_SCRATCH_1));
5289 else
5290 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
5291 return true;
5292 #endif
5293 goto do_generic_brev;
5294 do_generic_brev:
5295 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_unary_brev_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, 0);
5296 }
5297 /******************
5298 * BSF/BSR/POPCNT *
5299 ******************/
5300 do_bsf_bsr_popcnt: {
5301 if (op_size > OP_SIZE_NATIVE)
5302 goto do_generic_bsf_bsr_popcnt;
5303 #if defined(ARCH_X86)
5304 if (alu == ALU1_POPCNT && unlikely(!cpu_test_feature(CPU_FEATURE_popcnt)))
5305 goto do_generic_bsf_bsr_popcnt;
5306 if (op_size == OP_SIZE_1 || ((alu == ALU1_BSR || alu == ALU1_POPCNT) && mode == MODE_INT)) {
5307 g(gen_frame_load(ctx, op_size, false, slot_1, 0, R_SCRATCH_1));
5308 if ((alu == ALU1_BSR || alu == ALU1_POPCNT) && mode == MODE_INT) {
5309 g(gen_cmp_test_jmp(ctx, INSN_TEST, op_size, R_SCRATCH_1, R_SCRATCH_1, alu == ALU1_BSR ? COND_LE : COND_S, label_ovf));
5310 }
5311 gen_insn(INSN_ALU1 + ARCH_PARTIAL_ALU(op_size), maximum(op_size, OP_SIZE_2), alu, 1);
5312 gen_one(R_SCRATCH_1);
5313 gen_one(R_SCRATCH_1);
5314 if ((alu == ALU1_BSR || alu == ALU1_POPCNT) && mode == MODE_INT)
5315 goto x86_bsf_bsr_popcnt_finish;
5316 } else {
5317 g(gen_frame_load_op1(ctx, op_size, alu, 1, slot_1, 0, R_SCRATCH_1));
5318 }
5319 if (alu == ALU1_POPCNT)
5320 goto x86_bsf_bsr_popcnt_finish;
5321 if (mode == MODE_FIXED) {
5322 uint32_t cmov_label;
5323 gen_insn(INSN_MOV, maximum(op_size, OP_SIZE_4), 0, 0);
5324 gen_one(R_SCRATCH_2);
5325 gen_one(ARG_IMM);
5326 gen_eight(-1);
5327 g(gen_cmov(ctx, maximum(op_size, OP_SIZE_4), COND_E, R_SCRATCH_1, &cmov_label));
5328 gen_one(R_SCRATCH_2);
5329 if (cmov_label)
5330 gen_label(cmov_label);
5331
5332 } else {
5333 gen_insn(INSN_JMP_COND, maximum(op_size, OP_SIZE_2), COND_E, 0);
5334 gen_four(label_ovf);
5335 }
5336 x86_bsf_bsr_popcnt_finish:
5337 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
5338 return true;
5339 #endif
5340 #if defined(ARCH_ARM)
5341 #if defined(ARCH_ARM32)
5342 if (alu == ALU1_BSR && unlikely(!cpu_test_feature(CPU_FEATURE_armv6)))
5343 goto do_generic_bsf_bsr_popcnt;
5344 if (alu == ALU1_BSF && unlikely(!cpu_test_feature(CPU_FEATURE_armv6t2)))
5345 goto do_generic_bsf_bsr_popcnt;
5346 #endif
5347 if (alu == ALU1_POPCNT && unlikely(!cpu_test_feature(CPU_FEATURE_neon)))
5348 goto do_generic_bsf_bsr_popcnt;
5349 g(gen_frame_load(ctx, op_size, mode == MODE_INT, slot_1, 0, R_SCRATCH_1));
5350 if (mode == MODE_INT) {
5351 g(gen_cmp_test_jmp(ctx, INSN_TEST, i_size(op_size), R_SCRATCH_1, R_SCRATCH_1, alu == ALU1_BSR ? COND_LE : alu == ALU1_BSF ? COND_E : COND_S, label_ovf));
5352 }
5353
5354 if (alu == ALU1_POPCNT) {
5355 gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
5356 gen_one(FR_SCRATCH_1);
5357 gen_one(R_SCRATCH_1);
5358 gen_insn(INSN_FP_ALU1, OP_SIZE_NATIVE, FP_ALU1_VCNT8, 0);
5359 gen_one(FR_SCRATCH_1);
5360 gen_one(FR_SCRATCH_1);
5361 #if defined(ARCH_ARM32)
5362 if (op_size > OP_SIZE_1) {
5363 gen_insn(INSN_FP_ALU1, OP_SIZE_1, FP_ALU1_VPADDL, 0);
5364 gen_one(FR_SCRATCH_1);
5365 gen_one(FR_SCRATCH_1);
5366 }
5367 if (op_size > OP_SIZE_2) {
5368 gen_insn(INSN_FP_ALU1, OP_SIZE_2, FP_ALU1_VPADDL, 0);
5369 gen_one(FR_SCRATCH_1);
5370 gen_one(FR_SCRATCH_1);
5371 }
5372 #else
5373 if (op_size > OP_SIZE_1) {
5374 gen_insn(INSN_FP_ALU1, OP_SIZE_1, FP_ALU1_ADDV, 0);
5375 gen_one(FR_SCRATCH_1);
5376 gen_one(FR_SCRATCH_1);
5377 }
5378 #endif
5379 g(gen_frame_store(ctx, op_size, slot_r, 0, FR_SCRATCH_1));
5380 return true;
5381 }
5382
5383 if (mode == MODE_FIXED && alu == ALU1_BSF) {
5384 gen_insn(INSN_TEST, i_size(op_size), 0, 1);
5385 gen_one(R_SCRATCH_1);
5386 gen_one(R_SCRATCH_1);
5387 }
5388
5389 if (alu == ALU1_BSF) {
5390 gen_insn(INSN_ALU1, i_size(op_size), ALU1_BREV, ALU1_WRITES_FLAGS(ALU1_BREV));
5391 gen_one(R_SCRATCH_1);
5392 gen_one(R_SCRATCH_1);
5393 }
5394
5395 gen_insn(INSN_ALU1, i_size(op_size), ALU1_LZCNT, ALU1_WRITES_FLAGS(ALU1_LZCNT));
5396 gen_one(R_SCRATCH_1);
5397 gen_one(R_SCRATCH_1);
5398
5399 if (alu == ALU1_BSR) {
5400 g(gen_load_constant(ctx, R_SCRATCH_2, op_size == OP_SIZE_8 ? 63 : 31));
5401 gen_insn(INSN_ALU, i_size(op_size), ALU_SUB, ALU_WRITES_FLAGS(ALU_SUB, false));
5402 gen_one(R_SCRATCH_1);
5403 gen_one(R_SCRATCH_2);
5404 gen_one(R_SCRATCH_1);
5405 }
5406
5407 if (mode == MODE_FIXED && alu == ALU1_BSF) {
5408 #if defined(ARCH_ARM32)
5409 g(gen_imm(ctx, -1, IMM_PURPOSE_CMOV, OP_SIZE_NATIVE));
5410 gen_insn(INSN_CMOV, OP_SIZE_NATIVE, COND_E, 0);
5411 gen_one(R_SCRATCH_1);
5412 gen_one(R_SCRATCH_1);
5413 gen_imm_offset();
5414 #else
5415 gen_insn(INSN_CSEL_INV, i_size(op_size), COND_NE, 0);
5416 gen_one(R_SCRATCH_1);
5417 gen_one(ARG_IMM);
5418 gen_eight(0);
5419 gen_one(R_SCRATCH_1);
5420 #endif
5421 }
5422
5423 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
5424 return true;
5425 #endif
5426 #if defined(ARCH_ALPHA)
5427 if (likely(cpu_test_feature(CPU_FEATURE_cix))) {
5428 if (mode == MODE_INT) {
5429 g(gen_frame_load(ctx, op_size, true, slot_1, 0, R_SCRATCH_1));
5430 g(gen_cmp_test_jmp(ctx, INSN_TEST, OP_SIZE_NATIVE, R_SCRATCH_1, R_SCRATCH_1, alu == ALU1_BSR ? COND_LE : alu == ALU1_BSF ? COND_E : COND_S, label_ovf));
5431 } else {
5432 g(gen_frame_load(ctx, op_size, false, slot_1, 0, R_SCRATCH_1));
5433 if (ARCH_PREFERS_SX(op_size))
5434 g(gen_extend(ctx, op_size, false, R_SCRATCH_1, R_SCRATCH_1));
5435 }
5436 if (alu == ALU1_POPCNT) {
5437 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_POPCNT, ALU1_WRITES_FLAGS(ALU1_POPCNT));
5438 gen_one(R_SCRATCH_2);
5439 gen_one(R_SCRATCH_1);
5440 }
5441 if (alu == ALU1_BSF) {
5442 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_BSF, ALU1_WRITES_FLAGS(ALU1_BSF));
5443 gen_one(R_SCRATCH_2);
5444 gen_one(R_SCRATCH_1);
5445
5446 if (mode == MODE_FIXED) {
5447 g(gen_imm(ctx, -1, IMM_PURPOSE_MOVR, OP_SIZE_INT));
5448 gen_insn(INSN_MOVR, OP_SIZE_NATIVE, COND_E, 0);
5449 gen_one(R_SCRATCH_2);
5450 gen_one(R_SCRATCH_2);
5451 gen_one(R_SCRATCH_1);
5452 gen_imm_offset();
5453 }
5454 }
5455 if (alu == ALU1_BSR) {
5456 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_LZCNT, ALU1_WRITES_FLAGS(ALU1_LZCNT));
5457 gen_one(R_SCRATCH_2);
5458 gen_one(R_SCRATCH_1);
5459
5460 g(gen_load_constant(ctx, R_SCRATCH_3, OP_SIZE_NATIVE == OP_SIZE_8 ? 63 : 31));
5461
5462 gen_insn(INSN_ALU, OP_SIZE_NATIVE, ALU_SUB, ALU_WRITES_FLAGS(ALU_SUB, false));
5463 gen_one(R_SCRATCH_2);
5464 gen_one(R_SCRATCH_3);
5465 gen_one(R_SCRATCH_2);
5466 }
5467 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_2));
5468 return true;
5469 }
5470 #endif
5471 #if defined(ARCH_MIPS)
5472 if (MIPS_HAS_CLZ && alu != ALU1_POPCNT) {
5473 if (mode == MODE_INT) {
5474 g(gen_frame_load(ctx, op_size, true, slot_1, 0, R_SCRATCH_1));
5475 g(gen_cmp_test_jmp(ctx, INSN_TEST, OP_SIZE_NATIVE, R_SCRATCH_1, R_SCRATCH_1, alu == ALU1_BSR ? COND_LE : alu == ALU1_BSF ? COND_E : COND_S, label_ovf));
5476 } else {
5477 g(gen_frame_load(ctx, op_size, false, slot_1, 0, R_SCRATCH_1));
5478 }
5479 if (alu == ALU1_BSF) {
5480 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_NEG, ALU1_WRITES_FLAGS(ALU1_NEG));
5481 gen_one(R_SCRATCH_2);
5482 gen_one(R_SCRATCH_1);
5483
5484 gen_insn(INSN_ALU, OP_SIZE_NATIVE, ALU_AND, ALU_WRITES_FLAGS(ALU_AND, false));
5485 gen_one(R_SCRATCH_1);
5486 gen_one(R_SCRATCH_1);
5487 gen_one(R_SCRATCH_2);
5488 }
5489 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_LZCNT, ALU1_WRITES_FLAGS(ALU1_LZCNT));
5490 gen_one(R_SCRATCH_2);
5491 gen_one(R_SCRATCH_1);
5492
5493 g(gen_load_constant(ctx, R_SCRATCH_3, OP_SIZE_NATIVE == OP_SIZE_8 ? 63 : 31));
5494
5495 gen_insn(INSN_ALU, OP_SIZE_NATIVE, ALU_SUB, ALU_WRITES_FLAGS(ALU_SUB, false));
5496 gen_one(R_SCRATCH_2);
5497 gen_one(R_SCRATCH_3);
5498 gen_one(R_SCRATCH_2);
5499
5500 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_2));
5501 return true;
5502 }
5503 #endif
5504 #if defined(ARCH_POWER)
5505 if (alu == ALU1_BSF && (unlikely(!cpu_test_feature(CPU_FEATURE_v203)) || unlikely(!cpu_test_feature(CPU_FEATURE_v30))))
5506 goto do_generic_bsf_bsr_popcnt;
5507 if (alu == ALU1_POPCNT && unlikely(!cpu_test_feature(CPU_FEATURE_v206)))
5508 goto do_generic_bsf_bsr_popcnt;
5509 g(gen_frame_load(ctx, op_size, mode == MODE_INT, slot_1, 0, R_SCRATCH_1));
5510 if (mode == MODE_INT) {
5511 g(gen_cmp_test_jmp(ctx, INSN_TEST, i_size(op_size), R_SCRATCH_1, R_SCRATCH_1, alu == ALU1_BSR ? COND_LE : alu == ALU1_BSF ? COND_E : COND_S, label_ovf));
5512 }
5513 if (alu == ALU1_POPCNT) {
5514 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_POPCNT, ALU1_WRITES_FLAGS(ALU1_POPCNT));
5515 gen_one(R_SCRATCH_2);
5516 gen_one(R_SCRATCH_1);
5517 }
5518 if (alu == ALU1_BSF) {
5519 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_BSF, ALU1_WRITES_FLAGS(ALU1_BSF));
5520 gen_one(R_SCRATCH_2);
5521 gen_one(R_SCRATCH_1);
5522
5523 if (mode == MODE_FIXED) {
5524 gen_insn(INSN_ALU, i_size(op_size), ALU_AND, 1);
5525 gen_one(R_SCRATCH_3);
5526 gen_one(R_SCRATCH_1);
5527 gen_one(R_SCRATCH_1);
5528
5529 g(gen_imm(ctx, -1, IMM_PURPOSE_CMOV, OP_SIZE_NATIVE));
5530 gen_insn(INSN_CMOV, OP_SIZE_NATIVE, COND_E, 0);
5531 gen_one(R_SCRATCH_2);
5532 gen_one(R_SCRATCH_2);
5533 gen_imm_offset();
5534 }
5535 }
5536 if (alu == ALU1_BSR) {
5537 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_LZCNT, ALU1_WRITES_FLAGS(ALU1_LZCNT));
5538 gen_one(R_SCRATCH_2);
5539 gen_one(R_SCRATCH_1);
5540
5541 g(gen_load_constant(ctx, R_SCRATCH_3, OP_SIZE_NATIVE == OP_SIZE_8 ? 63 : 31));
5542
5543 gen_insn(INSN_ALU, OP_SIZE_NATIVE, ALU_SUB, ALU_WRITES_FLAGS(ALU_SUB, false));
5544 gen_one(R_SCRATCH_2);
5545 gen_one(R_SCRATCH_3);
5546 gen_one(R_SCRATCH_2);
5547 }
5548 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_2));
5549 return true;
5550 #endif
5551 #if defined(ARCH_LOONGARCH64) || defined(ARCH_RISCV64)
5552 #if defined(ARCH_LOONGARCH64)
5553 if (alu == ALU1_POPCNT)
5554 goto do_generic_bsf_bsr_popcnt;
5555 #endif
5556 #if defined(ARCH_RISCV64)
5557 if (unlikely(!cpu_test_feature(CPU_FEATURE_zbb)))
5558 goto do_generic_bsf_bsr_popcnt;
5559 #endif
5560 g(gen_frame_load(ctx, op_size, true, slot_1, 0, R_SCRATCH_1));
5561 if (mode == MODE_INT) {
5562 g(gen_cmp_test_jmp(ctx, INSN_TEST, OP_SIZE_NATIVE, R_SCRATCH_1, R_SCRATCH_1, alu == ALU1_BSR ? COND_LE : alu == ALU1_BSF ? COND_E : COND_S, label_ovf));
5563 } else {
5564 if (op_size < OP_SIZE_4)
5565 g(gen_extend(ctx, op_size, false, R_SCRATCH_1, R_SCRATCH_1));
5566 }
5567 if (alu == ALU1_POPCNT) {
5568 gen_insn(INSN_ALU1, maximum(OP_SIZE_4, op_size), ALU1_POPCNT, ALU1_WRITES_FLAGS(ALU1_POPCNT));
5569 gen_one(R_SCRATCH_2);
5570 gen_one(R_SCRATCH_1);
5571 }
5572 if (alu == ALU1_BSF) {
5573 gen_insn(INSN_ALU1, maximum(OP_SIZE_4, op_size), ALU1_BSF, ALU1_WRITES_FLAGS(ALU1_BSF));
5574 gen_one(R_SCRATCH_2);
5575 gen_one(R_SCRATCH_1);
5576
5577 if (mode == MODE_FIXED) {
5578 g(gen_imm(ctx, 1, IMM_PURPOSE_CMP, OP_SIZE_NATIVE));
5579 gen_insn(INSN_CMP_DEST_REG, OP_SIZE_NATIVE, COND_B, 0);
5580 gen_one(R_SCRATCH_3);
5581 gen_one(R_SCRATCH_1);
5582 gen_imm_offset();
5583
5584 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_NEG, ALU1_WRITES_FLAGS(ALU1_NEG));
5585 gen_one(R_SCRATCH_3);
5586 gen_one(R_SCRATCH_3);
5587
5588 gen_insn(INSN_ALU, OP_SIZE_NATIVE, ALU_OR, ALU_WRITES_FLAGS(ALU_OR, false));
5589 gen_one(R_SCRATCH_2);
5590 gen_one(R_SCRATCH_2);
5591 gen_one(R_SCRATCH_3);
5592 }
5593 }
5594 if (alu == ALU1_BSR) {
5595 gen_insn(INSN_ALU1, maximum(OP_SIZE_4, op_size), ALU1_LZCNT, ALU1_WRITES_FLAGS(ALU1_LZCNT));
5596 gen_one(R_SCRATCH_2);
5597 gen_one(R_SCRATCH_1);
5598
5599 g(gen_load_constant(ctx, R_SCRATCH_3, op_size <= OP_SIZE_4 ? 31 : 63));
5600
5601 gen_insn(INSN_ALU, OP_SIZE_NATIVE, ALU_SUB, ALU_WRITES_FLAGS(ALU_SUB, false));
5602 gen_one(R_SCRATCH_2);
5603 gen_one(R_SCRATCH_3);
5604 gen_one(R_SCRATCH_2);
5605 }
5606 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_2));
5607 return true;
5608 #endif
5609 #if defined(ARCH_IA64) || defined(ARCH_S390) || defined(ARCH_SPARC)
5610 if (alu == ALU1_BSF && !ARCH_HAS_ANDN)
5611 goto do_generic_bsf_bsr_popcnt;
5612 #if defined(ARCH_S390)
5613 if (!cpu_test_feature(CPU_FEATURE_misc_45) || !cpu_test_feature(CPU_FEATURE_misc_insn_ext_3))
5614 goto do_generic_bsf_bsr_popcnt;
5615 #endif
5616 #if defined(ARCH_SPARC)
5617 if (!SPARC_9)
5618 goto do_generic_bsf_bsr_popcnt;
5619 #endif
5620 g(gen_frame_load(ctx, op_size, mode == MODE_INT, slot_1, 0, R_SCRATCH_1));
5621 if (mode == MODE_INT) {
5622 g(gen_cmp_test_jmp(ctx, INSN_TEST, maximum(op_size, OP_SIZE_4), R_SCRATCH_1, R_SCRATCH_1, alu == ALU1_BSR ? COND_LE : alu == ALU1_BSF ? COND_E : COND_S, label_ovf));
5623 } else {
5624 if (ARCH_PREFERS_SX(op_size) && alu == ALU1_POPCNT && op_size < OP_SIZE_NATIVE)
5625 g(gen_extend(ctx, op_size, false, R_SCRATCH_1, R_SCRATCH_1));
5626 }
5627 if (alu == ALU1_POPCNT) {
5628 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_POPCNT, ALU1_WRITES_FLAGS(ALU1_POPCNT));
5629 gen_one(R_SCRATCH_1);
5630 gen_one(R_SCRATCH_1);
5631 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
5632 return true;
5633 }
5634 if (alu == ALU1_BSF) {
5635 g(gen_3address_alu_imm(ctx, OP_SIZE_NATIVE, ALU_SUB, R_SCRATCH_2, R_SCRATCH_1, 1));
5636
5637 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_ANDN, R_SCRATCH_2, R_SCRATCH_2, R_SCRATCH_1));
5638
5639 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_POPCNT, ALU1_WRITES_FLAGS(ALU1_POPCNT));
5640 gen_one(R_SCRATCH_2);
5641 gen_one(R_SCRATCH_2);
5642
5643 if (mode == MODE_FIXED) {
5644 unsigned attr_unused test_reg = R_SCRATCH_1;
5645 #if defined(ARCH_S390)
5646 g(gen_imm(ctx, 0, COND_IS_LOGICAL(COND_E) ? IMM_PURPOSE_CMP_LOGICAL : IMM_PURPOSE_CMP, OP_SIZE_NATIVE));
5647 gen_insn(INSN_CMP, OP_SIZE_NATIVE, 0, 1 + COND_IS_LOGICAL(COND_E));
5648 gen_one(R_SCRATCH_1);
5649 gen_imm_offset();
5650
5651 g(gen_imm(ctx, -1, IMM_PURPOSE_CMOV, OP_SIZE_NATIVE));
5652 gen_insn(INSN_CMOV, OP_SIZE_NATIVE, COND_E, 0);
5653 gen_one(R_SCRATCH_2);
5654 gen_one(R_SCRATCH_2);
5655 gen_imm_offset();
5656 #else
5657 #if defined(ARCH_IA64)
5658 g(gen_cmp_dest_reg(ctx, OP_SIZE_NATIVE, R_SCRATCH_1, (unsigned)-1, R_CMP_RESULT, 0, COND_NE));
5659 test_reg = R_CMP_RESULT;
5660 #endif
5661 g(gen_imm(ctx, -1, IMM_PURPOSE_MOVR, OP_SIZE_NATIVE));
5662 gen_insn(INSN_MOVR, OP_SIZE_NATIVE, COND_E, 0);
5663 gen_one(R_SCRATCH_2);
5664 gen_one(R_SCRATCH_2);
5665 gen_one(test_reg);
5666 gen_imm_offset();
5667 #endif
5668 }
5669
5670 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_2));
5671 return true;
5672 }
5673 #endif
5674 do_generic_bsf_bsr_popcnt:
5675 if (alu == ALU1_BSF) {
5676 if (mode == MODE_FIXED)
5677 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_unary_bsf_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, 0);
5678 else
5679 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, INT_unary_bsf_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, label_ovf);
5680 }
5681 if (alu == ALU1_BSR) {
5682 if (mode == MODE_FIXED)
5683 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_unary_bsr_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, 0);
5684 else
5685 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, INT_unary_bsr_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, label_ovf);
5686 }
5687 if (alu == ALU1_POPCNT) {
5688 if (mode == MODE_FIXED)
5689 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_unary_popcnt_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, 0);
5690 else
5691 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, INT_unary_popcnt_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, label_ovf);
5692 }
5693 }
5694 /**************
5695 * CONVERSION *
5696 **************/
5697 do_fixed_conv:
5698 do_conv: {
5699 unsigned src_op_size, dest_op_size;
5700 const struct type *src_type, *dest_type;
5701 src_type = get_type_of_local(ctx, slot_1);
5702 dest_type = get_type_of_local(ctx, slot_r);
5703
5704 if (TYPE_TAG_IS_FIXED(src_type->tag)) {
5705 src_op_size = TYPE_TAG_IDX_FIXED(src_type->tag) >> 1;
5706 } else {
5707 src_op_size = TYPE_TAG_IDX_INT(src_type->tag);
5708 }
5709
5710 if (TYPE_TAG_IS_FIXED(dest_type->tag)) {
5711 dest_op_size = TYPE_TAG_IDX_FIXED(dest_type->tag) >> 1;
5712 } else {
5713 dest_op_size = TYPE_TAG_IDX_INT(dest_type->tag);
5714 }
5715
5716 if (src_op_size <= OP_SIZE_NATIVE) {
5717 g(gen_frame_load(ctx, src_op_size, true, slot_1, 0, R_SCRATCH_1));
5718 } else {
5719 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
5720 }
5721
5722 if (dest_op_size >= src_op_size) {
5723 if (dest_op_size <= OP_SIZE_NATIVE) {
5724 g(gen_frame_store(ctx, dest_op_size, slot_r, 0, R_SCRATCH_1));
5725 } else {
5726 if (src_op_size <= OP_SIZE_NATIVE) {
5727 #if defined(ARCH_X86)
5728 if (R_SCRATCH_1 != R_AX || R_SCRATCH_2 != R_DX)
5729 internal(file_line, "gen_alu1: bad scratch registers");
5730 gen_insn(INSN_CWD, OP_SIZE_NATIVE, 0, 0);
5731 gen_one(R_DX);
5732 gen_one(R_AX);
5733 #else
5734 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_2, R_SCRATCH_1, (1U << (OP_SIZE_NATIVE + 3)) - 1, false));
5735 #endif
5736 }
5737 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_1, R_SCRATCH_2));
5738 }
5739 return true;
5740 } else {
5741 if (src_op_size > OP_SIZE_NATIVE) {
5742 #if defined(ARCH_ARM)
5743 gen_insn(INSN_CMP, OP_SIZE_NATIVE, 0, 1);
5744 gen_one(R_SCRATCH_2);
5745 gen_one(ARG_SHIFTED_REGISTER);
5746 gen_one(ARG_SHIFT_ASR | ((1U << (OP_SIZE_NATIVE + 3)) - 1));
5747 gen_one(R_SCRATCH_1);
5748
5749 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_NE, 0);
5750 gen_four(label_ovf);
5751 #else
5752 gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
5753 gen_one(R_SCRATCH_3);
5754 gen_one(R_SCRATCH_1);
5755
5756 gen_insn(INSN_ROT, OP_SIZE_NATIVE, ROT_SAR, ROT_WRITES_FLAGS(ROT_SAR));
5757 gen_one(R_SCRATCH_3);
5758 gen_one(R_SCRATCH_3);
5759 gen_one(ARG_IMM);
5760 gen_eight((1U << (OP_SIZE_NATIVE + 3)) - 1);
5761
5762 g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, R_SCRATCH_2, R_SCRATCH_3, COND_NE, label_ovf));
5763 #endif
5764
5765 src_op_size = OP_SIZE_NATIVE;
5766 }
5767 if (src_op_size > dest_op_size) {
5768 g(gen_cmp_extended(ctx, OP_SIZE_NATIVE, dest_op_size, R_SCRATCH_1, R_SCRATCH_3, label_ovf));
5769 }
5770 g(gen_frame_store(ctx, dest_op_size, slot_r, 0, R_SCRATCH_1));
5771 return true;
5772 }
5773 }
5774
5775 conv_uto_int: {
5776 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_uto_int_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, label_ovf);
5777 }
5778
5779 conv_ufrom_int: {
5780 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_ufrom_int_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, label_ovf);
5781 }
5782 }
5783
5784 static bool attr_w gen_constant(struct codegen_context *ctx, unsigned op_size, bool shrt, frame_t slot_r)
5785 {
5786 uintbig_t c;
5787 if (shrt) {
5788 c = (int16_t)get_unaligned_16(ctx->current_position);
5789 } else switch (op_size) {
5790 #define fx(n, type, utype, sz, bits) \
5791 case n: \
5792 c = (type)cat(get_unaligned_,bits)(ctx->current_position);\
5793 break;
5794 for_all_fixed(fx);
5795 #undef fx
5796 default:
5797 internal(file_line, "gen_constant: invalid type %u", op_size);
5798 }
5799 if (op_size > OP_SIZE_NATIVE) {
5800 g(gen_frame_store_imm(ctx, OP_SIZE_NATIVE, slot_r, lo_word(OP_SIZE_NATIVE), c));
5801 g(gen_frame_store_imm(ctx, OP_SIZE_NATIVE, slot_r, hi_word(OP_SIZE_NATIVE), c >> 1 >> ((1U << (OP_SIZE_NATIVE + 3)) - 1)));
5802 return true;
5803 } else {
5804 g(gen_frame_store_imm(ctx, op_size, slot_r, 0, c));
5805 }
5806 return true;
5807 }
5808
5809 static bool attr_w gen_real_constant(struct codegen_context *ctx, const struct type *t, frame_t slot_r)
5810 {
5811 int64_t offset;
5812 if (is_power_of_2(t->size) && t->size <= sizeof(uintbig_t))
5813 return gen_constant(ctx, log_2(t->size), false, slot_r);
5814
5815 g(load_function_offset(ctx, R_SCRATCH_3, offsetof(struct data, u_.function.code)));
5816
5817 offset = (ctx->current_position - da(ctx->fn,function)->code) * sizeof(code_t);
5818
5819 g(gen_memcpy_to_slot(ctx, slot_r, R_SCRATCH_3, offset));
5820 return true;
5821 }
5822
5823 static bool attr_w gen_copy(struct codegen_context *ctx, unsigned op_size, frame_t slot_1, frame_t slot_r)
5824 {
5825 if (unlikely(op_size > OP_SIZE_NATIVE)) {
5826 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
5827 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_1, R_SCRATCH_2));
5828 return true;
5829 } else {
5830 g(gen_frame_load(ctx, op_size, false, slot_1, 0, R_SCRATCH_1));
5831 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
5832 return true;
5833 }
5834 }
5835
5836 static unsigned real_type_to_op_size(unsigned real_type)
5837 {
5838 switch (real_type) {
5839 case 0: return OP_SIZE_2;
5840 case 1: return OP_SIZE_4;
5841 case 2: return OP_SIZE_8;
5842 case 3: return OP_SIZE_10;
5843 case 4: return OP_SIZE_16;
5844 default:
5845 internal(file_line, "real_type_to_op_size: invalid type %u", real_type);
5846 return 0;
5847 }
5848 }
5849
5850 static bool attr_w gen_fp_alu(struct codegen_context *ctx, unsigned real_type, unsigned op, uint32_t label_ovf, frame_t slot_1, frame_t slot_2, frame_t slot_r)
5851 {
5852 unsigned attr_unused fp_alu;
5853 size_t upc;
5854 unsigned attr_unused op_size = real_type_to_op_size(real_type);
5855 switch (op) {
5856 case OPCODE_REAL_OP_add:
5857 case OPCODE_REAL_OP_add_alt1:
5858 case OPCODE_REAL_OP_add_alt2: fp_alu = FP_ALU_ADD; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_add_real16_t); label_ovf = 0; goto do_alu;
5859 case OPCODE_REAL_OP_subtract:
5860 case OPCODE_REAL_OP_subtract_alt1:
5861 case OPCODE_REAL_OP_subtract_alt2: fp_alu = FP_ALU_SUB; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_subtract_real16_t); label_ovf = 0; goto do_alu;
5862 case OPCODE_REAL_OP_multiply:
5863 case OPCODE_REAL_OP_multiply_alt1:
5864 case OPCODE_REAL_OP_multiply_alt2: fp_alu = FP_ALU_MUL; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_multiply_real16_t); label_ovf = 0; goto do_alu;
5865 case OPCODE_REAL_OP_divide:
5866 case OPCODE_REAL_OP_divide_alt1:
5867 case OPCODE_REAL_OP_divide_alt2: fp_alu = FP_ALU_DIV; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_divide_real16_t); label_ovf = 0; goto do_alu;
5868 case OPCODE_REAL_OP_modulo:
5869 case OPCODE_REAL_OP_power:
5870 case OPCODE_REAL_OP_ldexp:
5871 case OPCODE_REAL_OP_atan2: upc = offsetof(struct cg_upcall_vector_s, REAL_binary_modulo_real16_t) + (op - OPCODE_REAL_OP_modulo) * TYPE_REAL_N * sizeof(void (*)(void)); goto do_upcall;
5872 case OPCODE_REAL_OP_equal:
5873 case OPCODE_REAL_OP_equal_alt1:
5874 case OPCODE_REAL_OP_equal_alt2: fp_alu = FP_COND_E; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_equal_real16_t); goto do_cmp;
5875 case OPCODE_REAL_OP_not_equal:
5876 case OPCODE_REAL_OP_not_equal_alt1:
5877 case OPCODE_REAL_OP_not_equal_alt2: fp_alu = FP_COND_NE; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_not_equal_real16_t); goto do_cmp;
5878 case OPCODE_REAL_OP_less:
5879 case OPCODE_REAL_OP_less_alt1:
5880 case OPCODE_REAL_OP_less_alt2: fp_alu = FP_COND_B; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_less_real16_t); goto do_cmp;
5881 case OPCODE_REAL_OP_less_equal:
5882 case OPCODE_REAL_OP_less_equal_alt1:
5883 case OPCODE_REAL_OP_less_equal_alt2: fp_alu = FP_COND_BE; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_less_equal_real16_t); goto do_cmp;
5884 default: internal(file_line, "gen_fp_alu: unsupported operation %u", op);
5885 }
5886
5887 do_alu:
5888 if ((SUPPORTED_FP >> real_type) & 1) {
5889 #if defined(ARCH_IA64)
5890 if (unlikely(fp_alu == FP_ALU_DIV))
5891 goto do_upcall;
5892 #endif
5893 #if defined(ARCH_X86)
5894 if (1)
5895 #elif defined(ARCH_S390)
5896 if ((op_size <= OP_SIZE_8 && (size_t)slot_2 * slot_size < 4096) || ctx->registers[slot_2] >= 0)
5897 #else
5898 if (ctx->registers[slot_2] >= 0)
5899 #endif
5900 {
5901 g(gen_frame_load(ctx, op_size, false, slot_1, 0, FR_SCRATCH_1));
5902 if (ctx->registers[slot_2] >= 0) {
5903 gen_insn(INSN_FP_ALU, op_size, fp_alu, 0);
5904 gen_one(FR_SCRATCH_1);
5905 gen_one(FR_SCRATCH_1);
5906 gen_one(ctx->registers[slot_2]);
5907 } else {
5908 g(gen_address(ctx, R_FRAME, (size_t)slot_2 * slot_size, IMM_PURPOSE_VLDR_VSTR_OFFSET, op_size));
5909 gen_insn(INSN_FP_ALU, op_size, fp_alu, 0);
5910 gen_one(FR_SCRATCH_1);
5911 gen_one(FR_SCRATCH_1);
5912 gen_address_offset();
5913 }
5914 g(gen_frame_store(ctx, op_size, slot_r, 0, FR_SCRATCH_1));
5915 return true;
5916 }
5917 #if defined(ARCH_ALPHA)
5918 g(gen_frame_load(ctx, op_size, false, slot_1, 0, FR_SCRATCH_1));
5919 g(gen_frame_load(ctx, op_size, false, slot_2, 0, FR_SCRATCH_2));
5920 gen_insn(INSN_FP_ALU, op_size, fp_alu, 0);
5921 gen_one(FR_SCRATCH_3);
5922 gen_one(FR_SCRATCH_1);
5923 gen_one(FR_SCRATCH_2);
5924 g(gen_frame_store(ctx, op_size, slot_r, 0, FR_SCRATCH_3));
5925 #else
5926 g(gen_frame_load(ctx, op_size, false, slot_1, 0, FR_SCRATCH_1));
5927 g(gen_frame_load(ctx, op_size, false, slot_2, 0, FR_SCRATCH_2));
5928 gen_insn(INSN_FP_ALU, op_size, fp_alu, 0);
5929 gen_one(FR_SCRATCH_1);
5930 gen_one(FR_SCRATCH_1);
5931 gen_one(FR_SCRATCH_2);
5932 g(gen_frame_store(ctx, op_size, slot_r, 0, FR_SCRATCH_1));
5933 #endif
5934 return true;
5935 }
5936 #ifdef SUPPORTED_FP_X87
5937 if ((SUPPORTED_FP_X87 >> real_type) & 1) {
5938 if (real_type != 3) {
5939 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_2));
5940 g(gen_frame_load_x87(ctx, INSN_X87_ALU, op_size, fp_alu, slot_1));
5941 } else {
5942 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
5943 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_2));
5944 gen_insn(INSN_X87_ALUP, op_size, fp_alu, 0);
5945 gen_one(R_ST1);
5946 }
5947 g(gen_frame_store_x87(ctx, INSN_X87_FSTP, op_size, slot_r));
5948 return true;
5949 }
5950 #endif
5951 #ifdef SUPPORTED_FP_HALF_CVT
5952 if ((SUPPORTED_FP_HALF_CVT >> real_type) & 1) {
5953 g(gen_frame_load(ctx, op_size, false, slot_1, 0, FR_SCRATCH_1));
5954 g(gen_frame_load(ctx, op_size, false, slot_2, 0, FR_SCRATCH_2));
5955 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
5956 gen_one(FR_SCRATCH_1);
5957 gen_one(FR_SCRATCH_1);
5958 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
5959 gen_one(FR_SCRATCH_2);
5960 gen_one(FR_SCRATCH_2);
5961 gen_insn(INSN_FP_ALU, OP_SIZE_4, fp_alu, 0);
5962 gen_one(FR_SCRATCH_1);
5963 gen_one(FR_SCRATCH_1);
5964 gen_one(FR_SCRATCH_2);
5965 gen_insn(INSN_FP_CVT, OP_SIZE_4, op_size, 0);
5966 gen_one(FR_SCRATCH_1);
5967 gen_one(FR_SCRATCH_1);
5968 g(gen_frame_store(ctx, op_size, slot_r, 0, FR_SCRATCH_1));
5969 return true;
5970 }
5971 #endif
5972 goto do_upcall;
5973
5974 do_cmp:
5975 if ((SUPPORTED_FP >> real_type) & 1
5976 #if defined(ARCH_ALPHA)
5977 && ARCH_SUPPORTS_TRAPS
5978 #endif
5979 ) {
5980 g(gen_frame_load(ctx, op_size, false, slot_1, 0, FR_SCRATCH_1));
5981 g(gen_frame_load(ctx, op_size, false, slot_2, 0, FR_SCRATCH_2));
5982 #if defined(ARCH_ALPHA)
5983 gen_insn(INSN_FP_CMP_DEST_REG_TRAP, op_size, fp_alu == FP_COND_NE ? FP_COND_E : fp_alu, 0);
5984 gen_one(FR_SCRATCH_3);
5985 gen_one(FR_SCRATCH_1);
5986 gen_one(FR_SCRATCH_2);
5987 gen_four(label_ovf);
5988
5989 if (!cpu_test_feature(CPU_FEATURE_fix)) {
5990 g(gen_frame_store(ctx, OP_SIZE_4, slot_r, 0, FR_SCRATCH_3));
5991 g(gen_frame_load(ctx, OP_SIZE_4, false, slot_r, 0, R_SCRATCH_1));
5992 } else {
5993 gen_insn(INSN_MOV, OP_SIZE_4, 0, 0);
5994 gen_one(R_SCRATCH_1);
5995 gen_one(FR_SCRATCH_3);
5996 }
5997
5998 if (fp_alu == FP_COND_NE) {
5999 g(gen_imm(ctx, 0, IMM_PURPOSE_CMP, OP_SIZE_NATIVE));
6000 gen_insn(INSN_CMP_DEST_REG, OP_SIZE_NATIVE, COND_E, 0);
6001 gen_one(R_SCRATCH_1);
6002 gen_one(R_SCRATCH_1);
6003 gen_imm_offset();
6004 } else {
6005 gen_insn(INSN_ROT, OP_SIZE_NATIVE, ROT_SHR, ROT_WRITES_FLAGS(ROT_SHR));
6006 gen_one(R_SCRATCH_1);
6007 gen_one(R_SCRATCH_1);
6008 gen_one(ARG_IMM);
6009 gen_eight(30);
6010 }
6011
6012 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, R_SCRATCH_1));
6013
6014 return true;
6015 #elif defined(ARCH_IA64)
6016 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_P, 0);
6017 gen_one(R_CMP_RESULT);
6018 gen_one(FR_SCRATCH_1);
6019 gen_one(FR_SCRATCH_2);
6020
6021 gen_insn(INSN_JMP_REG, OP_SIZE_NATIVE, COND_NE, 0);
6022 gen_one(R_CMP_RESULT);
6023 gen_four(label_ovf);
6024
6025 gen_insn(INSN_FP_CMP_DEST_REG, op_size, fp_alu, 0);
6026 gen_one(R_CMP_RESULT);
6027 gen_one(FR_SCRATCH_1);
6028 gen_one(FR_SCRATCH_2);
6029
6030 gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
6031 gen_one(R_SCRATCH_1);
6032 gen_one(R_CMP_RESULT);
6033
6034 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, R_SCRATCH_1));
6035
6036 return true;
6037 #elif defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS) || defined(ARCH_PARISC)
6038 gen_insn(INSN_FP_CMP_COND, op_size, FP_COND_P, 1);
6039 gen_one(FR_SCRATCH_1);
6040 gen_one(FR_SCRATCH_2);
6041
6042 gen_insn(INSN_JMP_FP_TEST, 0, FP_COND_P, 0);
6043 gen_four(label_ovf);
6044
6045 gen_insn(INSN_FP_CMP_COND, op_size, fp_alu, 1);
6046 gen_one(FR_SCRATCH_1);
6047 gen_one(FR_SCRATCH_2);
6048
6049 gen_insn(INSN_FP_TEST_REG, OP_SIZE_NATIVE, fp_alu, 0);
6050 gen_one(R_SCRATCH_1);
6051
6052 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, R_SCRATCH_1));
6053
6054 return true;
6055 #elif defined(ARCH_RISCV64)
6056 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_E, 0);
6057 gen_one(R_SCRATCH_1);
6058 gen_one(FR_SCRATCH_1);
6059 gen_one(FR_SCRATCH_1);
6060
6061 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_E, 0);
6062 gen_one(R_SCRATCH_2);
6063 gen_one(FR_SCRATCH_2);
6064 gen_one(FR_SCRATCH_2);
6065
6066 gen_insn(INSN_ALU, OP_SIZE_NATIVE, ALU_AND, ALU_WRITES_FLAGS(ALU_AND, false));
6067 gen_one(R_SCRATCH_1);
6068 gen_one(R_SCRATCH_1);
6069 gen_one(R_SCRATCH_2);
6070
6071 g(gen_jmp_on_zero(ctx, OP_SIZE_NATIVE, R_SCRATCH_1, COND_E, label_ovf));
6072
6073 gen_insn(INSN_FP_CMP_DEST_REG, op_size, fp_alu == FP_COND_NE ? FP_COND_E : fp_alu, 0);
6074 gen_one(R_SCRATCH_1);
6075 gen_one(FR_SCRATCH_1);
6076 gen_one(FR_SCRATCH_2);
6077
6078 if (fp_alu == FP_COND_NE) {
6079 g(gen_imm(ctx, 1, IMM_PURPOSE_XOR, OP_SIZE_NATIVE));
6080 gen_insn(INSN_ALU, OP_SIZE_NATIVE, ALU_XOR, ALU_WRITES_FLAGS(ALU_AND, false));
6081 gen_one(R_SCRATCH_1);
6082 gen_one(R_SCRATCH_1);
6083 gen_imm_offset();
6084 }
6085
6086 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, R_SCRATCH_1));
6087 return true;
6088 #else
6089 gen_insn(INSN_FP_CMP, op_size, 0, 1);
6090 gen_one(FR_SCRATCH_1);
6091 gen_one(FR_SCRATCH_2);
6092 #if defined(ARCH_ARM32)
6093 gen_insn(INSN_FP_TO_INT_FLAGS, 0, 0, 1);
6094 #endif
6095 gen_insn(INSN_JMP_COND, op_size, FP_COND_P, 0);
6096 gen_four(label_ovf);
6097 g(gen_frame_set_cond(ctx, op_size, false, fp_alu, slot_r));
6098 return true;
6099 #endif
6100 }
6101 #ifdef SUPPORTED_FP_X87
6102 if ((SUPPORTED_FP_X87 >> real_type) & 1) {
6103 if (likely(cpu_test_feature(CPU_FEATURE_cmov))) {
6104 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_2));
6105 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
6106 gen_insn(INSN_X87_FCOMIP, op_size, 0, 0);
6107 gen_one(R_ST1);
6108 gen_insn(INSN_X87_FSTP, op_size, 0, 0);
6109 gen_one(R_ST0);
6110 gen_insn(INSN_JMP_COND, op_size, COND_P, 0);
6111 gen_four(label_ovf);
6112 g(gen_frame_set_cond(ctx, op_size, false, fp_alu & 0xf, slot_r));
6113 return true;
6114 }
6115
6116 if (real_type != 3) {
6117 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
6118 g(gen_frame_load_x87(ctx, INSN_X87_FCOMP, op_size, 0, slot_2));
6119 } else {
6120 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_2));
6121 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
6122 gen_insn(INSN_X87_FCOMPP, op_size, 0, 0);
6123 }
6124
6125 gen_insn(INSN_X87_FNSTSW, 0, 0, 0);
6126 gen_one(R_AX);
6127 gen_one(R_AX);
6128
6129 gen_insn(INSN_TEST, OP_SIZE_2, 0, 1);
6130 gen_one(R_AX);
6131 gen_one(ARG_IMM);
6132 gen_eight(0x0400);
6133
6134 gen_insn(INSN_JMP_COND, OP_SIZE_2, COND_NE, 0);
6135 gen_four(label_ovf);
6136
6137 switch (fp_alu) {
6138 case FP_COND_E:
6139 gen_insn(INSN_TEST, OP_SIZE_2, 0, 1);
6140 gen_one(R_AX);
6141 gen_one(ARG_IMM);
6142 gen_eight(0x4000);
6143 g(gen_frame_set_cond(ctx, OP_SIZE_2, false, COND_NE, slot_r));
6144 break;
6145 case FP_COND_NE:
6146 gen_insn(INSN_TEST, OP_SIZE_2, 0, 1);
6147 gen_one(R_AX);
6148 gen_one(ARG_IMM);
6149 gen_eight(0x4000);
6150 g(gen_frame_set_cond(ctx, OP_SIZE_2, false, COND_E, slot_r));
6151 break;
6152 case FP_COND_B:
6153 gen_insn(INSN_TEST, OP_SIZE_2, 0, 1);
6154 gen_one(R_AX);
6155 gen_one(ARG_IMM);
6156 gen_eight(0x0100);
6157 g(gen_frame_set_cond(ctx, OP_SIZE_2, false, COND_NE, slot_r));
6158 break;
6159 case FP_COND_BE:
6160 gen_insn(INSN_TEST, OP_SIZE_2, 0, 1);
6161 gen_one(R_AX);
6162 gen_one(ARG_IMM);
6163 gen_eight(0x4100);
6164 g(gen_frame_set_cond(ctx, OP_SIZE_2, false, COND_NE, slot_r));
6165 break;
6166 default:
6167 internal(file_line, "gen_fp_alu: invalid condition %u", fp_alu);
6168 }
6169 return true;
6170 }
6171 #endif
6172 #ifdef SUPPORTED_FP_HALF_CVT
6173 if ((SUPPORTED_FP_HALF_CVT >> real_type) & 1) {
6174 g(gen_frame_load(ctx, op_size, false, slot_1, 0, FR_SCRATCH_1));
6175 g(gen_frame_load(ctx, op_size, false, slot_2, 0, FR_SCRATCH_2));
6176 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
6177 gen_one(FR_SCRATCH_1);
6178 gen_one(FR_SCRATCH_1);
6179 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
6180 gen_one(FR_SCRATCH_2);
6181 gen_one(FR_SCRATCH_2);
6182 gen_insn(INSN_FP_CMP, OP_SIZE_4, 0, 1);
6183 gen_one(FR_SCRATCH_1);
6184 gen_one(FR_SCRATCH_2);
6185 #if defined(ARCH_ARM32)
6186 gen_insn(INSN_FP_TO_INT_FLAGS, 0, 0, 1);
6187 #endif
6188 gen_insn(INSN_JMP_COND, op_size, FP_COND_P, 0);
6189 gen_four(label_ovf);
6190 g(gen_frame_set_cond(ctx, op_size, false, fp_alu, slot_r));
6191 return true;
6192 }
6193 #endif
6194
6195 do_upcall:
6196 return gen_alu_typed_upcall(ctx, upc, real_type, slot_1, slot_2, slot_r, label_ovf);
6197 }
6198
6199 #define OP_IS_ROUND(alu) ((alu) == FP_ALU1_ROUND || (alu) == FP_ALU1_FLOOR || (alu) == FP_ALU1_CEIL || (alu) == FP_ALU1_TRUNC)
6200
6201 static bool attr_w gen_fp_alu1(struct codegen_context *ctx, unsigned real_type, unsigned op, uint32_t label_ovf, frame_t slot_1, frame_t slot_r)
6202 {
6203 unsigned attr_unused fp_alu;
6204 size_t upc;
6205 unsigned attr_unused op_size = real_type_to_op_size(real_type);
6206 switch (op) {
6207 case OPCODE_REAL_OP_neg:
6208 case OPCODE_REAL_OP_neg_alt1:
6209 case OPCODE_REAL_OP_neg_alt2: fp_alu = FP_ALU1_NEG; upc = offsetof(struct cg_upcall_vector_s, REAL_unary_neg_real16_t); label_ovf = 0; goto do_alu;
6210 case OPCODE_REAL_OP_sqrt:
6211 case OPCODE_REAL_OP_sqrt_alt1:
6212 case OPCODE_REAL_OP_sqrt_alt2: fp_alu = FP_ALU1_SQRT; upc = offsetof(struct cg_upcall_vector_s, REAL_unary_sqrt_real16_t); label_ovf = 0; goto do_alu;
6213 case OPCODE_REAL_OP_round: fp_alu = FP_ALU1_ROUND; upc = offsetof(struct cg_upcall_vector_s, REAL_unary_round_real16_t); label_ovf = 0; goto do_alu;
6214 case OPCODE_REAL_OP_floor: fp_alu = FP_ALU1_FLOOR; upc = offsetof(struct cg_upcall_vector_s, REAL_unary_floor_real16_t); label_ovf = 0; goto do_alu;
6215 case OPCODE_REAL_OP_ceil: fp_alu = FP_ALU1_CEIL; upc = offsetof(struct cg_upcall_vector_s, REAL_unary_ceil_real16_t); label_ovf = 0; goto do_alu;
6216 case OPCODE_REAL_OP_trunc: fp_alu = FP_ALU1_TRUNC; upc = offsetof(struct cg_upcall_vector_s, REAL_unary_trunc_real16_t); label_ovf = 0; goto do_alu;
6217 case OPCODE_REAL_OP_to_int:
6218 case OPCODE_REAL_OP_to_int_alt1:
6219 case OPCODE_REAL_OP_to_int_alt2: upc = offsetof(struct cg_upcall_vector_s, REAL_unary_to_int_real16_t); goto do_to_int;
6220 case OPCODE_REAL_OP_from_int:
6221 case OPCODE_REAL_OP_from_int_alt1:
6222 case OPCODE_REAL_OP_from_int_alt2: upc = offsetof(struct cg_upcall_vector_s, REAL_unary_from_int_real16_t); label_ovf = 0; goto do_from_int;
6223 case OPCODE_REAL_OP_is_exception:
6224 case OPCODE_REAL_OP_is_exception_alt1:
6225 case OPCODE_REAL_OP_is_exception_alt2: upc = offsetof(struct cg_upcall_vector_s, REAL_unary_is_exception_real16_t); label_ovf = 0; goto do_is_exception;
6226 default: upc = offsetof(struct cg_upcall_vector_s, REAL_unary_cbrt_real16_t) + (op - OPCODE_REAL_OP_cbrt) * TYPE_REAL_N * sizeof(void (*)(void)); label_ovf = 0; goto do_upcall;
6227 }
6228
6229 do_alu:
6230 if ((SUPPORTED_FP >> real_type) & 1 && (
6231 #if defined(ARCH_ALPHA)
6232 fp_alu == FP_ALU1_NEG ||
6233 (fp_alu == FP_ALU1_SQRT && cpu_test_feature(CPU_FEATURE_fix)) ||
6234 #elif defined(ARCH_ARM32)
6235 fp_alu == FP_ALU1_NEG ||
6236 fp_alu == FP_ALU1_SQRT ||
6237 #elif defined(ARCH_ARM64)
6238 true ||
6239 #elif defined(ARCH_IA64)
6240 fp_alu == FP_ALU1_NEG ||
6241 #elif defined(ARCH_LOONGARCH64)
6242 fp_alu == FP_ALU1_NEG ||
6243 fp_alu == FP_ALU1_SQRT ||
6244 fp_alu == FP_ALU1_ROUND ||
6245 #elif defined(ARCH_MIPS)
6246 fp_alu == FP_ALU1_NEG ||
6247 (fp_alu == FP_ALU1_SQRT && MIPS_HAS_SQRT) ||
6248 #elif defined(ARCH_PARISC)
6249 (fp_alu == FP_ALU1_NEG && PA_20) ||
6250 fp_alu == FP_ALU1_SQRT ||
6251 #elif defined(ARCH_POWER)
6252 fp_alu == FP_ALU1_NEG ||
6253 (fp_alu == FP_ALU1_SQRT && cpu_test_feature(CPU_FEATURE_p2) && real_type != 4) ||
6254 #elif defined(ARCH_S390)
6255 true ||
6256 #elif defined(ARCH_SPARC)
6257 fp_alu == FP_ALU1_NEG ||
6258 fp_alu == FP_ALU1_SQRT ||
6259 #elif defined(ARCH_RISCV64)
6260 fp_alu == FP_ALU1_NEG ||
6261 fp_alu == FP_ALU1_SQRT ||
6262 #elif defined(ARCH_X86)
6263 fp_alu == FP_ALU1_SQRT ||
6264 (OP_IS_ROUND(fp_alu) && cpu_test_feature(CPU_FEATURE_sse41)) ||
6265 #endif
6266 false)) {
6267 #if defined(ARCH_S390)
6268 if (op_size <= OP_SIZE_8 && (size_t)slot_1 * slot_size < 4096 && fp_alu == FP_ALU1_SQRT) {
6269 if (ctx->registers[slot_1] >= 0) {
6270 gen_insn(INSN_FP_ALU1, op_size, fp_alu, 0);
6271 gen_one(FR_SCRATCH_1);
6272 gen_one(ctx->registers[slot_1]);
6273 g(gen_frame_store(ctx, op_size, slot_r, 0, FR_SCRATCH_1));
6274 } else {
6275 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size, IMM_PURPOSE_VLDR_VSTR_OFFSET, op_size));
6276 gen_insn(INSN_FP_ALU1, op_size, fp_alu, 0);
6277 gen_one(FR_SCRATCH_1);
6278 gen_address_offset();
6279 g(gen_frame_store(ctx, op_size, slot_r, 0, FR_SCRATCH_1));
6280 }
6281 return true;
6282 }
6283 #endif
6284 g(gen_frame_load(ctx, op_size, false, slot_1, 0, FR_SCRATCH_1));
6285 gen_insn(INSN_FP_ALU1, op_size, fp_alu, 0);
6286 gen_one(FR_SCRATCH_2);
6287 gen_one(FR_SCRATCH_1);
6288 g(gen_frame_store(ctx, op_size, slot_r, 0, FR_SCRATCH_2));
6289 return true;
6290 }
6291 #ifdef SUPPORTED_FP_X87
6292 if ((SUPPORTED_FP_X87 >> real_type) & 1) {
6293 if (fp_alu == FP_ALU1_NEG) {
6294 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
6295 gen_insn(INSN_X87_FCHS, op_size, 0, 0);
6296 g(gen_frame_store_x87(ctx, INSN_X87_FSTP, op_size, slot_r));
6297 return true;
6298 } else if (fp_alu == FP_ALU1_SQRT) {
6299 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
6300 gen_insn(INSN_X87_FSQRT, op_size, 0, 0);
6301 g(gen_frame_store_x87(ctx, INSN_X87_FSTP, op_size, slot_r));
6302 return true;
6303 } else if (fp_alu == FP_ALU1_ROUND) {
6304 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
6305 gen_insn(INSN_X87_FRNDINT, op_size, 0, 0);
6306 g(gen_frame_store_x87(ctx, INSN_X87_FSTP, op_size, slot_r));
6307 return true;
6308 }
6309 }
6310 #endif
6311 #ifdef SUPPORTED_FP_HALF_CVT
6312 if ((SUPPORTED_FP_HALF_CVT >> real_type) & 1 && (
6313 #if defined(ARCH_ARM32)
6314 fp_alu == FP_ALU1_NEG ||
6315 fp_alu == FP_ALU1_SQRT ||
6316 #elif defined(ARCH_ARM64)
6317 true ||
6318 #elif defined(ARCH_X86)
6319 fp_alu == FP_ALU1_SQRT ||
6320 (OP_IS_ROUND(fp_alu) && cpu_test_feature(CPU_FEATURE_sse41)) ||
6321 #endif
6322 false)) {
6323 g(gen_frame_load(ctx, op_size, false, slot_1, 0, FR_SCRATCH_1));
6324 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
6325 gen_one(FR_SCRATCH_1);
6326 gen_one(FR_SCRATCH_1);
6327 gen_insn(INSN_FP_ALU1, OP_SIZE_4, fp_alu, 0);
6328 gen_one(FR_SCRATCH_1);
6329 gen_one(FR_SCRATCH_1);
6330 gen_insn(INSN_FP_CVT, OP_SIZE_4, op_size, 0);
6331 gen_one(FR_SCRATCH_1);
6332 gen_one(FR_SCRATCH_1);
6333 g(gen_frame_store(ctx, op_size, slot_r, 0, FR_SCRATCH_1));
6334 return true;
6335 }
6336 #endif
6337 goto do_upcall;
6338
6339 do_to_int:
6340 if ((SUPPORTED_FP >> real_type) & 1
6341 #if defined(ARCH_ALPHA)
6342 && ARCH_SUPPORTS_TRAPS
6343 #endif
6344 #if defined(ARCH_MIPS)
6345 && MIPS_HAS_TRUNC
6346 #endif
6347 ) {
6348 g(gen_frame_load(ctx, op_size, false, slot_1, 0, FR_SCRATCH_1));
6349 goto do_cvt_to_int;
6350 do_cvt_to_int:;
6351 #if defined(ARCH_X86)
6352 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_TO_INT32 : INSN_FP_TO_INT64, op_size, 0, 0);
6353 gen_one(R_SCRATCH_1);
6354 gen_one(FR_SCRATCH_1);
6355
6356 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, OP_SIZE_INT, R_SCRATCH_1, sign_bit(uint_default_t), COND_E, label_ovf));
6357
6358 g(gen_frame_store(ctx, OP_SIZE_INT, slot_r, 0, R_SCRATCH_1));
6359 return true;
6360 #endif
6361 #if defined(ARCH_ARM) || defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS)
6362 #if defined(ARCH_ARM)
6363 gen_insn(INSN_FP_CMP, op_size, 0, 1);
6364 gen_one(FR_SCRATCH_1);
6365 gen_one(FR_SCRATCH_1);
6366 #if defined(ARCH_ARM32)
6367 gen_insn(INSN_FP_TO_INT_FLAGS, 0, 0, 1);
6368 #endif
6369 gen_insn(INSN_JMP_COND, op_size, FP_COND_P, 0);
6370 gen_four(label_ovf);
6371 #else
6372 gen_insn(INSN_FP_CMP_COND, op_size, FP_COND_P, 1);
6373 gen_one(FR_SCRATCH_1);
6374 gen_one(FR_SCRATCH_1);
6375
6376 gen_insn(INSN_JMP_FP_TEST, 0, FP_COND_P, 0);
6377 gen_four(label_ovf);
6378 #endif
6379 #if defined(ARCH_ARM32) || defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS)
6380 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_TO_INT32 : INSN_FP_TO_INT64, op_size, 0, 0);
6381 gen_one(FR_SCRATCH_1);
6382 gen_one(FR_SCRATCH_1);
6383
6384 gen_insn(INSN_MOV, OP_SIZE_INT, 0, 0);
6385 gen_one(R_SCRATCH_1);
6386 gen_one(FR_SCRATCH_1);
6387 #else
6388 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_TO_INT32 : INSN_FP_TO_INT64, op_size, 0, 0);
6389 gen_one(R_SCRATCH_1);
6390 gen_one(FR_SCRATCH_1);
6391 #endif
6392 g(gen_imm(ctx, (int_default_t)(sign_bit(uint_default_t) + 1), IMM_PURPOSE_ADD, OP_SIZE_INT));
6393 gen_insn(INSN_ALU, OP_SIZE_INT, ALU_ADD, ALU_WRITES_FLAGS(ALU_ADD, is_imm()));
6394 gen_one(R_SCRATCH_2);
6395 gen_one(R_SCRATCH_1);
6396 gen_imm_offset();
6397
6398 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, OP_SIZE_INT, R_SCRATCH_2, 1, COND_BE, label_ovf));
6399
6400 g(gen_frame_store(ctx, OP_SIZE_INT, slot_r, 0, R_SCRATCH_1));
6401 return true;
6402 #endif
6403 #if defined(ARCH_IA64)
6404 gen_insn(INSN_FP_TO_INT64, op_size, 0, 0);
6405 gen_one(FR_SCRATCH_1);
6406 gen_one(FR_SCRATCH_1);
6407
6408 gen_insn(INSN_MOV, OP_SIZE_8, 0, 0);
6409 gen_one(R_SCRATCH_1);
6410 gen_one(FR_SCRATCH_1);
6411
6412 if (OP_SIZE_INT == OP_SIZE_4) {
6413 g(gen_extend(ctx, OP_SIZE_4, true, R_SCRATCH_2, R_SCRATCH_1));
6414 g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, R_SCRATCH_1, R_SCRATCH_2, COND_NE, label_ovf));
6415 } else {
6416 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, R_SCRATCH_1, sign_bit(uint64_t), COND_E, label_ovf));
6417 }
6418
6419 g(gen_frame_store(ctx, OP_SIZE_INT, slot_r, 0, R_SCRATCH_1));
6420 return true;
6421 #endif
6422 #if defined(ARCH_PARISC) || defined(ARCH_POWER) || defined(ARCH_SPARC)
6423 #if defined(ARCH_POWER)
6424 if (!cpu_test_feature(CPU_FEATURE_ppc))
6425 goto do_upcall;
6426 if (OP_SIZE_INT == OP_SIZE_4)
6427 goto do_upcall;
6428 #endif
6429 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_TO_INT32 : INSN_FP_TO_INT64, op_size, 0, 0);
6430 gen_one(FR_SCRATCH_1);
6431 gen_one(FR_SCRATCH_1);
6432
6433 g(gen_frame_store(ctx, OP_SIZE_INT, slot_r, 0, FR_SCRATCH_1));
6434 g(gen_frame_load(ctx, OP_SIZE_INT, false, slot_r, 0, R_SCRATCH_1));
6435
6436 g(gen_imm(ctx, sign_bit(uint_default_t) + 1, IMM_PURPOSE_ADD, OP_SIZE_INT));
6437 gen_insn(INSN_ALU, i_size(OP_SIZE_INT), ALU_ADD, ALU_WRITES_FLAGS(ALU_ADD, is_imm()));
6438 gen_one(R_SCRATCH_2);
6439 gen_one(R_SCRATCH_1);
6440 gen_imm_offset();
6441
6442 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, OP_SIZE_INT, R_SCRATCH_2, 1, COND_BE, label_ovf));
6443
6444 return true;
6445 #endif
6446 #if defined(ARCH_ALPHA)
6447 gen_insn(INSN_FP_TO_INT64_TRAP, op_size, 0, 0);
6448 gen_one(FR_SCRATCH_2);
6449 gen_one(FR_SCRATCH_1);
6450 gen_four(label_ovf);
6451
6452 if (OP_SIZE_INT == OP_SIZE_4) {
6453 gen_insn(INSN_FP_INT64_TO_INT32_TRAP, 0, 0, 0);
6454 gen_one(FR_SCRATCH_3);
6455 gen_one(FR_SCRATCH_2);
6456 gen_four(label_ovf);
6457 g(gen_frame_store(ctx, OP_SIZE_INT, slot_r, 0, FR_SCRATCH_3));
6458 } else {
6459 g(gen_frame_store(ctx, OP_SIZE_INT, slot_r, 0, FR_SCRATCH_2));
6460 }
6461 return true;
6462 #endif
6463 #if defined(ARCH_S390)
6464 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_TO_INT32 : INSN_FP_TO_INT64, op_size, 0, 1);
6465 gen_one(R_SCRATCH_1);
6466 gen_one(FR_SCRATCH_1);
6467
6468 gen_insn(INSN_JMP_COND, op_size, FP_COND_P, 0);
6469 gen_four(label_ovf);
6470
6471 g(gen_frame_store(ctx, OP_SIZE_INT, slot_r, 0, R_SCRATCH_1));
6472 return true;
6473 #endif
6474 #if defined(ARCH_RISCV64)
6475 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_TO_INT32 : INSN_FP_TO_INT64, op_size, 0, 0);
6476 gen_one(R_SCRATCH_1);
6477 gen_one(FR_SCRATCH_1);
6478
6479 g(gen_load_constant(ctx, R_SCRATCH_2, sign_bit(int_default_t)));
6480
6481 g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, R_SCRATCH_1, R_SCRATCH_2, COND_E, label_ovf));
6482
6483 g(gen_imm(ctx, -1, IMM_PURPOSE_XOR, i_size(size)));
6484 gen_insn(INSN_ALU, OP_SIZE_NATIVE, ALU_XOR, ALU_WRITES_FLAGS(ALU_XOR, is_imm()));
6485 gen_one(R_SCRATCH_2);
6486 gen_one(R_SCRATCH_2);
6487 gen_imm_offset();
6488
6489 g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, R_SCRATCH_1, R_SCRATCH_2, COND_E, label_ovf));
6490
6491 g(gen_frame_store(ctx, OP_SIZE_INT, slot_r, 0, R_SCRATCH_1));
6492 return true;
6493 #endif
6494 }
6495 #ifdef SUPPORTED_FP_X87
6496 if ((SUPPORTED_FP_X87 >> real_type) & 1) {
6497 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
6498
6499 if (likely(cpu_test_feature(CPU_FEATURE_sse3))) {
6500 g(gen_frame_store_x87(ctx, INSN_X87_FISTTP, OP_SIZE_INT, slot_r));
6501 } else {
6502 gen_insn(INSN_PUSH, OP_SIZE_NATIVE, 0, 0);
6503 gen_one(ARG_IMM);
6504 gen_eight(0x0f7f);
6505
6506 gen_insn(INSN_X87_FLDCW, 0, 0, 0);
6507 gen_one(ARG_ADDRESS_1);
6508 gen_one(R_SP);
6509 gen_eight(0);
6510
6511 g(gen_frame_store_x87(ctx, INSN_X87_FISTP, OP_SIZE_INT, slot_r));
6512
6513 gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
6514 gen_one(ARG_ADDRESS_1);
6515 gen_one(R_SP);
6516 gen_eight(0);
6517 gen_one(ARG_IMM);
6518 gen_eight(0x037f);
6519
6520 gen_insn(INSN_X87_FLDCW, 0, 0, 0);
6521 gen_one(ARG_ADDRESS_1);
6522 gen_one(R_SP);
6523 gen_eight(0);
6524
6525 gen_insn(INSN_ALU, i_size(OP_SIZE_ADDRESS), ALU_ADD, ALU_WRITES_FLAGS(ALU_ADD, true));
6526 gen_one(R_SP);
6527 gen_one(R_SP);
6528 gen_one(ARG_IMM);
6529 gen_eight(1 << OP_SIZE_NATIVE);
6530 }
6531 if (ctx->registers[slot_r] >= 0)
6532 g(unspill(ctx, slot_r));
6533 g(gen_frame_load(ctx, OP_SIZE_INT, false, slot_r, 0, R_SCRATCH_1));
6534
6535 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, OP_SIZE_INT, R_SCRATCH_1, sign_bit(int_default_t), COND_E, label_ovf));
6536
6537 return true;
6538 }
6539 #endif
6540 #ifdef SUPPORTED_FP_HALF_CVT
6541 if ((SUPPORTED_FP_HALF_CVT >> real_type) & 1) {
6542 g(gen_frame_load(ctx, op_size, false, slot_1, 0, FR_SCRATCH_1));
6543 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
6544 gen_one(FR_SCRATCH_1);
6545 gen_one(FR_SCRATCH_1);
6546 real_type = 1;
6547 op_size = real_type_to_op_size(real_type);
6548 goto do_cvt_to_int;
6549 }
6550 #endif
6551 goto do_upcall;
6552
6553 do_from_int:
6554 if ((SUPPORTED_FP >> real_type) & 1) {
6555 #if defined(ARCH_ALPHA) || defined(ARCH_ARM32) || defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS) || defined(ARCH_PARISC) || defined(ARCH_POWER) || defined(ARCH_SPARC)
6556 int int_op_size = OP_SIZE_INT;
6557 #if defined(ARCH_POWER)
6558 if (int_op_size == OP_SIZE_4)
6559 goto do_upcall;
6560 if (op_size == OP_SIZE_4 && !cpu_test_feature(CPU_FEATURE_v206))
6561 goto do_upcall;
6562 if (op_size == OP_SIZE_8 && !cpu_test_feature(CPU_FEATURE_ppc))
6563 goto do_upcall;
6564 #endif
6565 g(gen_frame_load(ctx, int_op_size, false, slot_1, 0, FR_SCRATCH_1));
6566 #if defined(ARCH_ALPHA)
6567 if (OP_SIZE_INT == OP_SIZE_4) {
6568 gen_insn(INSN_MOVSX, OP_SIZE_4, 0, 0);
6569 gen_one(FR_SCRATCH_1);
6570 gen_one(FR_SCRATCH_1);
6571
6572 int_op_size = OP_SIZE_8;
6573 }
6574 #endif
6575 gen_insn(int_op_size == OP_SIZE_4 ? INSN_FP_FROM_INT32 : INSN_FP_FROM_INT64, op_size, 0, 0);
6576 gen_one(FR_SCRATCH_2);
6577 gen_one(FR_SCRATCH_1);
6578
6579 g(gen_frame_store(ctx, op_size, slot_r, 0, FR_SCRATCH_2));
6580 return true;
6581 #elif defined(ARCH_IA64)
6582 g(gen_frame_load(ctx, OP_SIZE_INT, true, slot_1, 0, R_SCRATCH_1));
6583
6584 gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
6585 gen_one(FR_SCRATCH_1);
6586 gen_one(R_SCRATCH_1);
6587
6588 gen_insn(INSN_FP_FROM_INT64, op_size, 0, 0);
6589 gen_one(FR_SCRATCH_1);
6590 gen_one(FR_SCRATCH_1);
6591
6592 g(gen_frame_store(ctx, op_size, slot_r, 0, FR_SCRATCH_1));
6593 return true;
6594 #else
6595 g(gen_frame_load(ctx, OP_SIZE_INT, false, slot_1, 0, R_SCRATCH_1));
6596
6597 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_FROM_INT32 : INSN_FP_FROM_INT64, op_size, 0, 0);
6598 gen_one(FR_SCRATCH_1);
6599 gen_one(R_SCRATCH_1);
6600
6601 g(gen_frame_store(ctx, op_size, slot_r, 0, FR_SCRATCH_1));
6602 return true;
6603 #endif
6604 }
6605 #ifdef SUPPORTED_FP_X87
6606 if ((SUPPORTED_FP_X87 >> real_type) & 1) {
6607 if (ctx->registers[slot_1] >= 0)
6608 g(spill(ctx, slot_1));
6609 g(gen_frame_load_x87(ctx, INSN_X87_FILD, OP_SIZE_INT, 0, slot_1));
6610 g(gen_frame_store_x87(ctx, INSN_X87_FSTP, op_size, slot_r));
6611 return true;
6612 }
6613 #endif
6614 #ifdef SUPPORTED_FP_HALF_CVT
6615 if ((SUPPORTED_FP_HALF_CVT >> real_type) & 1) {
6616 #if defined(ARCH_ARM32)
6617 g(gen_frame_load(ctx, OP_SIZE_INT, false, slot_1, 0, FR_SCRATCH_1));
6618
6619 gen_insn(INSN_FP_FROM_INT32, OP_SIZE_4, 0, 0);
6620 gen_one(FR_SCRATCH_1);
6621 gen_one(FR_SCRATCH_1);
6622 #else
6623 g(gen_frame_load(ctx, OP_SIZE_INT, false, slot_1, 0, R_SCRATCH_1));
6624 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_FROM_INT32 : INSN_FP_FROM_INT64, OP_SIZE_4, 0, 0);
6625 gen_one(FR_SCRATCH_1);
6626 gen_one(R_SCRATCH_1);
6627 #endif
6628 gen_insn(INSN_FP_CVT, OP_SIZE_4, op_size, 0);
6629 gen_one(FR_SCRATCH_1);
6630 gen_one(FR_SCRATCH_1);
6631 g(gen_frame_store(ctx, op_size, slot_r, 0, FR_SCRATCH_1));
6632 return true;
6633 }
6634 #endif
6635 goto do_upcall;
6636
6637 do_is_exception:
6638 if ((SUPPORTED_FP >> real_type) & 1) {
6639 g(gen_frame_load(ctx, op_size, false, slot_1, 0, FR_SCRATCH_1));
6640 #if defined(ARCH_ALPHA)
6641 gen_insn(INSN_FP_CMP_UNORDERED_DEST_REG, op_size, 0, 0);
6642 gen_one(FR_SCRATCH_2);
6643 gen_one(FR_SCRATCH_1);
6644 gen_one(FR_SCRATCH_1);
6645
6646 if (!cpu_test_feature(CPU_FEATURE_fix)) {
6647 g(gen_frame_store(ctx, OP_SIZE_4, slot_r, 0, FR_SCRATCH_2));
6648 g(gen_frame_load(ctx, OP_SIZE_4, false, slot_r, 0, R_SCRATCH_1));
6649 } else {
6650 gen_insn(INSN_MOV, OP_SIZE_4, 0, 0);
6651 gen_one(R_SCRATCH_1);
6652 gen_one(FR_SCRATCH_2);
6653 }
6654
6655 gen_insn(INSN_ROT, OP_SIZE_NATIVE, ROT_SHR, ROT_WRITES_FLAGS(ROT_SHR));
6656 gen_one(R_SCRATCH_1);
6657 gen_one(R_SCRATCH_1);
6658 gen_one(ARG_IMM);
6659 gen_eight(30);
6660
6661 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, R_SCRATCH_1));
6662
6663 return true;
6664 #elif defined(ARCH_IA64)
6665 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_P, 0);
6666 gen_one(R_CMP_RESULT);
6667 gen_one(FR_SCRATCH_1);
6668 gen_one(FR_SCRATCH_1);
6669
6670 gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
6671 gen_one(R_SCRATCH_1);
6672 gen_one(R_CMP_RESULT);
6673
6674 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, R_SCRATCH_1));
6675 #elif defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS) || defined(ARCH_PARISC)
6676 gen_insn(INSN_FP_CMP_COND, op_size, FP_COND_P, 1);
6677 gen_one(FR_SCRATCH_1);
6678 gen_one(FR_SCRATCH_1);
6679
6680 gen_insn(INSN_FP_TEST_REG, OP_SIZE_NATIVE, FP_COND_P, 0);
6681 gen_one(R_SCRATCH_1);
6682
6683 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, R_SCRATCH_1));
6684 #elif defined(ARCH_RISCV64)
6685 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_E, 0);
6686 gen_one(R_SCRATCH_1);
6687 gen_one(FR_SCRATCH_1);
6688 gen_one(FR_SCRATCH_1);
6689
6690 g(gen_imm(ctx, 1, IMM_PURPOSE_XOR, OP_SIZE_NATIVE));
6691 gen_insn(INSN_ALU, OP_SIZE_NATIVE, ALU_XOR, ALU_WRITES_FLAGS(ALU_XOR, is_imm()));
6692 gen_one(R_SCRATCH_1);
6693 gen_one(R_SCRATCH_1);
6694 gen_imm_offset();
6695
6696 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, R_SCRATCH_1));
6697 #else
6698 gen_insn(INSN_FP_CMP, op_size, 0, 1);
6699 gen_one(FR_SCRATCH_1);
6700 gen_one(FR_SCRATCH_1);
6701 #if defined(ARCH_ARM32)
6702 gen_insn(INSN_FP_TO_INT_FLAGS, 0, 0, 1);
6703 #endif
6704 g(gen_frame_set_cond(ctx, op_size, false, FP_COND_P, slot_r));
6705 #endif
6706 return true;
6707 }
6708 #ifdef SUPPORTED_FP_X87
6709 if ((SUPPORTED_FP_X87 >> real_type) & 1) {
6710 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
6711 if (likely(cpu_test_feature(CPU_FEATURE_cmov))) {
6712 gen_insn(INSN_X87_FCOMIP, op_size, 0, 0);
6713 gen_one(R_ST0);
6714
6715 g(gen_frame_set_cond(ctx, op_size, false, COND_P, slot_r));
6716 return true;
6717 }
6718
6719 gen_insn(INSN_X87_FCOMP, op_size, 0, 0);
6720 gen_one(R_ST0);
6721
6722 gen_insn(INSN_X87_FNSTSW, 0, 0, 0);
6723 gen_one(R_AX);
6724 gen_one(R_AX);
6725
6726 gen_insn(INSN_TEST, OP_SIZE_2, 0, 1);
6727 gen_one(R_AX);
6728 gen_one(ARG_IMM);
6729 gen_eight(0x0400);
6730
6731 g(gen_frame_set_cond(ctx, op_size, false, COND_NE, slot_r));
6732
6733 return true;
6734 }
6735 #endif
6736 #ifdef SUPPORTED_FP_HALF_CVT
6737 if ((SUPPORTED_FP_HALF_CVT >> real_type) & 1) {
6738 g(gen_frame_load(ctx, op_size, false, slot_1, 0, FR_SCRATCH_1));
6739 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
6740 gen_one(FR_SCRATCH_1);
6741 gen_one(FR_SCRATCH_1);
6742 gen_insn(INSN_FP_CMP, OP_SIZE_4, 0, 1);
6743 gen_one(FR_SCRATCH_1);
6744 gen_one(FR_SCRATCH_1);
6745 #if defined(ARCH_ARM32)
6746 gen_insn(INSN_FP_TO_INT_FLAGS, 0, 0, 1);
6747 #endif
6748 g(gen_frame_set_cond(ctx, op_size, false, FP_COND_P, slot_r));
6749 return true;
6750 }
6751 #endif
6752
6753 do_upcall:
6754 g(gen_alu_typed_upcall(ctx, upc, real_type, slot_1, NO_FRAME_T, slot_r, label_ovf));
6755 return true;
6756 }
6757
6758 static bool attr_w gen_is_exception(struct codegen_context *ctx, frame_t slot_1, frame_t slot_r)
6759 {
6760 uint32_t no_ex_label, escape_label;
6761 const struct type *type = get_type_of_local(ctx, slot_1);
6762
6763 no_ex_label = alloc_label(ctx);
6764 if (unlikely(!no_ex_label))
6765 return false;
6766 escape_label = alloc_escape_label(ctx);
6767 if (unlikely(!escape_label))
6768 return false;
6769
6770 if (TYPE_IS_FLAT(type))
6771 g(gen_test_1_jz_cached(ctx, slot_1, no_ex_label));
6772
6773 g(gen_frame_load(ctx, OP_SIZE_SLOT, false, slot_1, 0, R_SCRATCH_1));
6774 g(gen_ptr_is_thunk(ctx, R_SCRATCH_1, true, escape_label));
6775 g(gen_barrier(ctx));
6776
6777 if (!TYPE_IS_FLAT(type)) {
6778 g(gen_compare_da_tag(ctx, R_SCRATCH_1, DATA_TAG_flat, COND_E, escape_label, R_SCRATCH_1));
6779 }
6780
6781 gen_label(no_ex_label);
6782 g(gen_frame_clear(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r));
6783
6784 flag_set(ctx, slot_r, false);
6785
6786 return true;
6787 }
6788
6789 static bool attr_w gen_system_property(struct codegen_context *ctx, frame_t slot_1, frame_t slot_r)
6790 {
6791 uint32_t escape_label;
6792
6793 escape_label = alloc_escape_label(ctx);
6794 if (unlikely(!escape_label))
6795 return false;
6796
6797 g(gen_test_1_cached(ctx, slot_1, escape_label));
6798
6799 g(gen_upcall_start(ctx, 1));
6800
6801 g(gen_frame_load(ctx, OP_SIZE_INT, false, slot_1, 0, R_ARG0));
6802 g(gen_upcall_argument(ctx, 0));
6803
6804 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, ipret_system_property), 1));
6805
6806 g(gen_frame_store(ctx, OP_SIZE_INT, slot_r, 0, R_RET0));
6807
6808 flag_set(ctx, slot_1, false);
6809 flag_set(ctx, slot_r, false);
6810
6811 return true;
6812 }
6813
6814 static bool attr_w gen_flat_move_copy(struct codegen_context *ctx, frame_t slot_1, frame_t slot_r)
6815 {
6816 uint32_t escape_label;
6817
6818 escape_label = alloc_escape_label(ctx);
6819 if (unlikely(!escape_label))
6820 return false;
6821
6822 g(gen_test_1_cached(ctx, slot_1, escape_label));
6823
6824 g(gen_memcpy_slots(ctx, slot_r, slot_1));
6825
6826 flag_set(ctx, slot_1, false);
6827 flag_set(ctx, slot_r, false);
6828
6829 return false;
6830 }
6831
6832 static bool attr_w gen_ref_move_copy(struct codegen_context *ctx, code_t code, frame_t slot_1, frame_t slot_r)
6833 {
6834 g(gen_frame_load(ctx, OP_SIZE_SLOT, false, slot_1, 0, R_SCRATCH_1));
6835 g(gen_frame_store(ctx, OP_SIZE_SLOT, slot_r, 0, R_SCRATCH_1));
6836 g(gen_set_1(ctx, R_FRAME, slot_r, 0, true));
6837 flag_set(ctx, slot_r, true);
6838 if (code == OPCODE_REF_COPY) {
6839 g(gen_upcall_start(ctx, 1));
6840 gen_insn(INSN_MOV, i_size(OP_SIZE_ADDRESS), 0, 0);
6841 gen_one(R_ARG0);
6842 gen_one(R_SCRATCH_1);
6843 g(gen_upcall_argument(ctx, 0));
6844 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_pointer_reference_owned), 1));
6845 } else if (code == OPCODE_REF_MOVE && !da(ctx->fn,function)->local_variables_flags[slot_1].may_be_borrowed) {
6846 g(gen_set_1(ctx, R_FRAME, slot_1, 0, false));
6847 flag_set(ctx, slot_1, false);
6848 } else {
6849 uint32_t label_id;
6850 if (unlikely(!(label_id = alloc_label(ctx))))
6851 return false;
6852 if (flag_is_set(ctx, slot_1)) {
6853 g(gen_set_1(ctx, R_FRAME, slot_1, 0, false));
6854 goto move_it;
6855 }
6856 if (flag_is_clear(ctx, slot_1))
6857 goto do_reference;
6858 g(gen_test_1(ctx, R_FRAME, slot_1, 0, label_id, false, TEST_CLEAR));
6859 do_reference:
6860 g(gen_upcall_start(ctx, 1));
6861 gen_insn(INSN_MOV, i_size(OP_SIZE_ADDRESS), 0, 0);
6862 gen_one(R_ARG0);
6863 gen_one(R_SCRATCH_1);
6864 g(gen_upcall_argument(ctx, 0));
6865 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_pointer_reference_owned), 1));
6866 move_it:
6867 gen_label(label_id);
6868 if (code == OPCODE_REF_MOVE_CLEAR)
6869 g(gen_frame_clear(ctx, OP_SIZE_SLOT, slot_1));
6870 flag_set(ctx, slot_1, false);
6871 }
6872 return true;
6873 }
6874
6875 static bool attr_w gen_box_move_copy(struct codegen_context *ctx, code_t code, frame_t slot_1, frame_t slot_r)
6876 {
6877 if (flag_must_be_flat(ctx, slot_r)) {
6878 uint32_t escape_label = alloc_escape_label(ctx);
6879 if (unlikely(!escape_label))
6880 return false;
6881 gen_insn(INSN_JMP, 0, 0, 0);
6882 gen_four(escape_label);
6883 return true;
6884 }
6885
6886 if (ctx->registers[slot_1] >= 0)
6887 g(spill(ctx, slot_1));
6888
6889 g(gen_upcall_start(ctx, 3));
6890
6891 gen_insn(INSN_MOV, i_size(OP_SIZE_ADDRESS), 0, 0);
6892 gen_one(R_ARG0);
6893 gen_one(R_FRAME);
6894 g(gen_upcall_argument(ctx, 0));
6895
6896 g(gen_load_constant(ctx, R_ARG1, slot_1));
6897 g(gen_upcall_argument(ctx, 1));
6898
6899 g(gen_load_constant(ctx, R_ARG2, code == OPCODE_BOX_MOVE_CLEAR));
6900 g(gen_upcall_argument(ctx, 2));
6901
6902 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_ipret_copy_variable_to_pointer), 3));
6903
6904 if (code == OPCODE_BOX_MOVE_CLEAR) {
6905 g(gen_frame_clear(ctx, OP_SIZE_SLOT, slot_1));
6906 flag_set(ctx, slot_1, false);
6907 }
6908
6909 g(gen_frame_set_pointer(ctx, slot_r, R_RET0));
6910
6911 return true;
6912 }
6913
6914 static bool attr_w gen_eval(struct codegen_context *ctx, frame_t slot_1)
6915 {
6916 uint32_t escape_label, skip_label;
6917
6918 escape_label = alloc_escape_label(ctx);
6919 if (unlikely(!escape_label))
6920 return false;
6921
6922 skip_label = alloc_label(ctx);
6923 if (unlikely(!skip_label))
6924 return false;
6925
6926 g(gen_test_1_jz_cached(ctx, slot_1, skip_label));
6927
6928 g(gen_frame_load(ctx, OP_SIZE_SLOT, false, slot_1, 0, R_SCRATCH_1));
6929 g(gen_ptr_is_thunk(ctx, R_SCRATCH_1, true, escape_label));
6930
6931 gen_label(skip_label);
6932
6933 return true;
6934 }
6935
6936 static bool attr_w gen_jump(struct codegen_context *ctx, int32_t jmp_offset, unsigned cond)
6937 {
6938 ip_t ip = (ctx->current_position - da(ctx->fn,function)->code) + (jmp_offset / (int)sizeof(code_t));
6939 if (likely(!ctx->code_labels[ip])) {
6940 ctx->code_labels[ip] = alloc_label(ctx);
6941 if (unlikely(!ctx->code_labels[ip]))
6942 return false;
6943 }
6944 if (!cond) {
6945 gen_insn(INSN_JMP, 0, 0, 0);
6946 gen_four(ctx->code_labels[ip]);
6947 } else if (cond == 1) {
6948 #if defined(ARCH_S390)
6949 gen_insn(INSN_JMP_COND_LOGICAL, maximum(OP_SIZE_4, log_2(sizeof(ajla_flat_option_t))), COND_E, 0);
6950 #else
6951 gen_insn(COND_IS_LOGICAL(COND_E) ? INSN_JMP_COND_LOGICAL : INSN_JMP_COND, maximum(OP_SIZE_4, log_2(sizeof(ajla_flat_option_t))), COND_E, 0);
6952 #endif
6953 gen_four(ctx->code_labels[ip]);
6954 } else if (cond == 2) {
6955 g(gen_jmp_on_zero(ctx, OP_SIZE_NATIVE, R_SCRATCH_1, COND_E, ctx->code_labels[ip]));
6956 } else {
6957 internal(file_line, "gen_jump: invalid condition %u", cond);
6958 }
6959 return true;
6960 }
6961
6962 static bool attr_w gen_cond_jump(struct codegen_context *ctx, frame_t slot, int32_t jmp_offset)
6963 {
6964 unsigned size = log_2(sizeof(ajla_flat_option_t));
6965 size_t attr_unused offset;
6966 if (ctx->registers[slot] >= 0) {
6967 goto no_load_op;
6968 }
6969 #if defined(ARCH_S390) || defined(ARCH_X86)
6970 offset = (size_t)slot * slot_size;
6971 #if defined(ARCH_S390)
6972 if (size != OP_SIZE_1)
6973 goto no_load_op;
6974 #endif
6975 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_MVI_CLI_OFFSET, size));
6976 gen_insn(INSN_CMP, size, 0, 2);
6977 gen_address_offset();
6978 gen_one(ARG_IMM);
6979 gen_eight(0);
6980
6981 g(gen_jump(ctx, jmp_offset, 1));
6982 return true;
6983 #endif
6984 goto no_load_op;
6985 no_load_op:
6986 g(gen_frame_load(ctx, size, false, slot, 0, R_SCRATCH_1));
6987 g(gen_jump(ctx, jmp_offset, 2));
6988 return true;
6989 }
6990
6991 static bool attr_w gen_load_fn_or_curry(struct codegen_context *ctx, frame_t fn_idx, frame_t slot_fn, frame_t slot_r, unsigned flags)
6992 {
6993 bool curry = fn_idx == NO_FRAME_T;
6994 uint32_t escape_label;
6995 arg_t i;
6996
6997 escape_label = alloc_escape_label(ctx);
6998 if (unlikely(!escape_label))
6999 return false;
7000
7001 g(gen_upcall_start(ctx, 1));
7002
7003 g(gen_load_constant(ctx, R_ARG0, ctx->args_l));
7004 g(gen_upcall_argument(ctx, 0));
7005
7006 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_data_alloc_function_reference_mayfail), 1));
7007 g(gen_sanitize_returned_pointer(ctx, R_RET0));
7008 g(gen_jmp_on_zero(ctx, OP_SIZE_ADDRESS, R_RET0, COND_E, escape_label));
7009
7010 gen_insn(INSN_MOV, i_size(OP_SIZE_ADDRESS), 0, 0);
7011 gen_one(R_SAVED_1);
7012 gen_one(R_RET0);
7013
7014 if (!curry) {
7015 g(load_function_offset(ctx, R_SCRATCH_1, offsetof(struct data, u_.function.local_directory[fn_idx])));
7016
7017 g(gen_address(ctx, R_SAVED_1, offsetof(struct data, u_.function_reference.u.direct), IMM_PURPOSE_STR_OFFSET, OP_SIZE_ADDRESS));
7018 gen_insn(INSN_MOV, OP_SIZE_ADDRESS, 0, 0);
7019 gen_address_offset();
7020 gen_one(R_SCRATCH_1);
7021
7022 g(gen_address(ctx, R_SAVED_1, offsetof(struct data, u_.function_reference.is_indirect), IMM_PURPOSE_STR_OFFSET, log_2(sizeof(bool))));
7023 g(gen_imm(ctx, 0, IMM_PURPOSE_STORE_VALUE, log_2(sizeof(uchar_efficient_t))));
7024 gen_insn(INSN_MOV, log_2(sizeof(uchar_efficient_t)), 0, 0);
7025 gen_address_offset();
7026 gen_imm_offset();
7027 } else {
7028 g(gen_frame_get_pointer(ctx, slot_fn, (flags & OPCODE_FLAG_FREE_ARGUMENT) != 0, R_SCRATCH_1));
7029
7030 g(gen_address(ctx, R_SAVED_1, offsetof(struct data, u_.function_reference.u.indirect), IMM_PURPOSE_STR_OFFSET, OP_SIZE_ADDRESS));
7031 gen_insn(INSN_MOV, OP_SIZE_ADDRESS, 0, 0);
7032 gen_address_offset();
7033 gen_one(R_SCRATCH_1);
7034
7035 g(gen_address(ctx, R_SAVED_1, offsetof(struct data, u_.function_reference.is_indirect), IMM_PURPOSE_STR_OFFSET, log_2(sizeof(bool))));
7036 g(gen_imm(ctx, 1, IMM_PURPOSE_STORE_VALUE, log_2(sizeof(uchar_efficient_t))));
7037 gen_insn(INSN_MOV, log_2(sizeof(uchar_efficient_t)), 0, 0);
7038 gen_address_offset();
7039 gen_imm_offset();
7040 }
7041
7042 for (i = 0; i < ctx->args_l; i++) {
7043 uintptr_t arg_offset_tag = offsetof(struct data, u_.function_reference.arguments[i].tag);
7044 uintptr_t arg_offset_ptr = offsetof(struct data, u_.function_reference.arguments[i].u.ptr);
7045 uintptr_t arg_offset_slot = offsetof(struct data, u_.function_reference.arguments[i].u.slot);
7046 frame_t arg_slot = ctx->args[i].slot;
7047 const struct type *t = get_type_of_local(ctx, arg_slot);
7048 uint32_t skip_flat_label, set_ptr_label, next_arg_label;
7049 skip_flat_label = alloc_label(ctx);
7050 if (unlikely(!skip_flat_label))
7051 return false;
7052 set_ptr_label = alloc_label(ctx);
7053 if (unlikely(!set_ptr_label))
7054 return false;
7055 next_arg_label = alloc_label(ctx);
7056 if (unlikely(!next_arg_label))
7057 return false;
7058 if (TYPE_IS_FLAT(t)) {
7059 g(gen_test_1_cached(ctx, arg_slot, skip_flat_label));
7060 if (t->size <= slot_size && TYPE_TAG_IS_BUILTIN(t->tag)) {
7061 unsigned copy_size = OP_SIZE_SLOT;
7062 if (is_power_of_2(t->size))
7063 copy_size = log_2(t->size);
7064 if (!ARCH_HAS_BWX)
7065 copy_size = maximum(copy_size, OP_SIZE_4);
7066 g(gen_address(ctx, R_SAVED_1, arg_offset_tag, IMM_PURPOSE_STR_OFFSET, log_2(sizeof(type_tag_t))));
7067 g(gen_imm(ctx, t->tag, IMM_PURPOSE_STORE_VALUE, log_2(sizeof(type_tag_t))));
7068 gen_insn(INSN_MOV, log_2(sizeof(type_tag_t)), 0, 0);
7069 gen_address_offset();
7070 gen_imm_offset();
7071
7072 if (ctx->registers[arg_slot] >= 0) {
7073 g(gen_address(ctx, R_SAVED_1, arg_offset_slot, IMM_PURPOSE_STR_OFFSET, copy_size));
7074 gen_insn(INSN_MOV, copy_size, 0, 0);
7075 gen_address_offset();
7076 gen_one(ctx->registers[arg_slot]);
7077 goto copied;
7078 }
7079 #if defined(ARCH_S390)
7080 if (copy_size == OP_SIZE_1 && !cpu_test_feature(CPU_FEATURE_long_displacement)) {
7081 g(gen_address(ctx, R_FRAME, (size_t)arg_slot * slot_size, IMM_PURPOSE_LDR_OFFSET, copy_size));
7082 gen_insn(INSN_MOV_MASK, OP_SIZE_NATIVE, MOV_MASK_0_8, 0);
7083 gen_one(R_SCRATCH_1);
7084 gen_one(R_SCRATCH_1);
7085 gen_address_offset();
7086 } else
7087 #endif
7088 {
7089 g(gen_address(ctx, R_FRAME, (size_t)arg_slot * slot_size, ARCH_PREFERS_SX(copy_size) ? IMM_PURPOSE_LDR_SX_OFFSET : IMM_PURPOSE_LDR_OFFSET, copy_size));
7090 gen_insn(ARCH_PREFERS_SX(copy_size) ? INSN_MOVSX : INSN_MOV, copy_size, 0, 0);
7091 gen_one(R_SCRATCH_1);
7092 gen_address_offset();
7093 }
7094
7095 g(gen_address(ctx, R_SAVED_1, arg_offset_slot, IMM_PURPOSE_STR_OFFSET, copy_size));
7096 gen_insn(INSN_MOV, copy_size, 0, 0);
7097 gen_address_offset();
7098 gen_one(R_SCRATCH_1);
7099 copied:
7100 gen_insn(INSN_JMP, 0, 0, 0);
7101 gen_four(next_arg_label);
7102 } else {
7103 if (ctx->registers[arg_slot] >= 0)
7104 g(spill(ctx, arg_slot));
7105
7106 g(gen_upcall_start(ctx, 3));
7107
7108 gen_insn(INSN_MOV, i_size(OP_SIZE_ADDRESS), 0, 0);
7109 gen_one(R_ARG0);
7110 gen_one(R_FRAME);
7111 g(gen_upcall_argument(ctx, 0));
7112
7113 g(gen_load_constant(ctx, R_ARG1, arg_slot));
7114 g(gen_upcall_argument(ctx, 1));
7115
7116 g(gen_imm(ctx, (size_t)arg_slot * slot_size, IMM_PURPOSE_ADD, i_size(OP_SIZE_ADDRESS)));
7117 gen_insn(INSN_ALU, i_size(OP_SIZE_ADDRESS), ALU_ADD, ALU_WRITES_FLAGS(ALU_ADD, is_imm()));
7118 gen_one(R_ARG2);
7119 gen_one(R_FRAME);
7120 gen_imm_offset();
7121 g(gen_upcall_argument(ctx, 2));
7122
7123 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_flat_to_data), 3));
7124
7125 gen_insn(INSN_JMP, 0, 0, 0);
7126 gen_four(set_ptr_label);
7127 }
7128 }
7129
7130 gen_label(skip_flat_label);
7131 g(gen_frame_get_pointer(ctx, arg_slot, (ctx->args[i].flags & OPCODE_FLAG_FREE_ARGUMENT) != 0, R_RET0));
7132
7133 gen_label(set_ptr_label);
7134 g(gen_address(ctx, R_SAVED_1, arg_offset_tag, IMM_PURPOSE_STR_OFFSET, log_2(sizeof(type_tag_t))));
7135 g(gen_imm(ctx, TYPE_TAG_unknown, IMM_PURPOSE_STORE_VALUE, log_2(sizeof(type_tag_t))));
7136 gen_insn(INSN_MOV, log_2(sizeof(type_tag_t)), 0, 0);
7137 gen_address_offset();
7138 gen_imm_offset();
7139
7140 g(gen_address(ctx, R_SAVED_1, arg_offset_ptr, IMM_PURPOSE_STR_OFFSET, OP_SIZE_SLOT));
7141 gen_insn(INSN_MOV, OP_SIZE_SLOT, 0, 0);
7142 gen_address_offset();
7143 gen_one(R_RET0);
7144
7145 gen_label(next_arg_label);
7146 }
7147
7148 g(gen_compress_pointer(ctx, R_SAVED_1));
7149 g(gen_frame_set_pointer(ctx, slot_r, R_SAVED_1));
7150
7151 return true;
7152 }
7153
7154 static bool attr_w gen_call(struct codegen_context *ctx, code_t code, frame_t fn_idx)
7155 {
7156 struct data *new_fn = ctx->local_directory[fn_idx];
7157 frame_t required_slots = da(new_fn,function)->frame_slots;
7158 frame_t bitmap_slots = da(new_fn,function)->n_bitmap_slots;
7159 frame_t v;
7160 uint32_t escape_label;
7161 int64_t new_fp_offset;
7162 uchar_efficient_t call_mode;
7163 arg_t i;
7164 bool arch_use_flags = ARCH_HAS_FLAGS;
7165 #if defined(ARCH_POWER)
7166 arch_use_flags = false;
7167 #endif
7168
7169 escape_label = alloc_escape_label(ctx);
7170 if (unlikely(!escape_label))
7171 return false;
7172
7173 for (v = MIN_USEABLE_SLOT; v < function_n_variables(ctx->fn); v++) {
7174 if (ctx->registers[v] >= 0) {
7175 g(spill(ctx, v));
7176 }
7177 }
7178
7179 g(gen_frame_load_raw(ctx, log_2(sizeof(stack_size_t)), false, 0, frame_offs(available_slots), R_SCRATCH_1));
7180 g(gen_imm(ctx, required_slots, IMM_PURPOSE_SUB, i_size(log_2(sizeof(stack_size_t)))));
7181 gen_insn(INSN_ALU + ARCH_PARTIAL_ALU(i_size(log_2(sizeof(stack_size_t)))), i_size(log_2(sizeof(stack_size_t))), ALU_SUB, arch_use_flags);
7182 gen_one(R_SCRATCH_1);
7183 gen_one(R_SCRATCH_1);
7184 gen_imm_offset();
7185
7186 if (arch_use_flags) {
7187 gen_insn(COND_IS_LOGICAL(COND_B) ? INSN_JMP_COND_LOGICAL : INSN_JMP_COND, log_2(sizeof(stack_size_t)), COND_B, 0);
7188 gen_four(escape_label);
7189 } else {
7190 g(gen_cmp_test_jmp(ctx, INSN_TEST, OP_SIZE_NATIVE, R_SCRATCH_1, R_SCRATCH_1, COND_S, escape_label));
7191 }
7192
7193 new_fp_offset = -(ssize_t)(required_slots * slot_size);
7194
7195 g(gen_frame_store_raw(ctx, log_2(sizeof(stack_size_t)), 0, new_fp_offset + frame_offs(available_slots), R_SCRATCH_1));
7196 g(gen_frame_store_imm_raw(ctx, log_2(sizeof(ip_t)), 0, new_fp_offset + frame_offs(previous_ip), ctx->return_values - da(ctx->fn,function)->code));
7197 g(gen_frame_load_raw(ctx, log_2(sizeof(timestamp_t)), false, 0, frame_offs(timestamp), R_SCRATCH_1));
7198 g(gen_frame_store_raw(ctx, log_2(sizeof(timestamp_t)), 0, new_fp_offset + frame_offs(timestamp), R_SCRATCH_1));
7199 call_mode = code == OPCODE_CALL ? CALL_MODE_NORMAL : code == OPCODE_CALL_STRICT ? CALL_MODE_STRICT : CALL_MODE_SPARK;
7200 g(gen_frame_store_imm_raw(ctx, log_2(sizeof(uchar_efficient_t)), 0, new_fp_offset + frame_offs(mode), call_mode));
7201
7202 g(gen_clear_bitmap(ctx, frame_offset, R_FRAME, new_fp_offset, bitmap_slots));
7203
7204 for (i = 0; i < ctx->args_l; i++) {
7205 const struct code_arg *src_arg = &ctx->args[i];
7206 const struct local_arg *dest_arg = &da(new_fn,function)->args[i];
7207 const struct type *t = get_type_of_local(ctx, src_arg->slot);
7208 uint32_t non_flat_label, thunk_label, incr_ref_label, next_arg_label;
7209 non_flat_label = alloc_label(ctx);
7210 if (unlikely(!non_flat_label))
7211 return false;
7212 thunk_label = alloc_label(ctx);
7213 if (unlikely(!thunk_label))
7214 return false;
7215 incr_ref_label = alloc_label(ctx);
7216 if (unlikely(!incr_ref_label))
7217 return false;
7218 next_arg_label = alloc_label(ctx);
7219 if (unlikely(!next_arg_label))
7220 return false;
7221 if (TYPE_IS_FLAT(t)) {
7222 g(gen_test_1_cached(ctx, src_arg->slot, non_flat_label));
7223 if (dest_arg->may_be_flat) {
7224 g(gen_memcpy_from_slot(ctx, R_FRAME, new_fp_offset + (size_t)dest_arg->slot * slot_size, src_arg->slot));
7225 } else {
7226 g(gen_upcall_start(ctx, 3));
7227
7228 gen_insn(INSN_MOV, i_size(OP_SIZE_ADDRESS), 0, 0);
7229 gen_one(R_ARG0);
7230 gen_one(R_FRAME);
7231 g(gen_upcall_argument(ctx, 0));
7232
7233 g(gen_load_constant(ctx, R_ARG1, src_arg->slot));
7234 g(gen_upcall_argument(ctx, 1));
7235
7236 g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_ADDRESS), ALU_ADD, R_ARG2, R_FRAME, (size_t)src_arg->slot * slot_size));
7237 g(gen_upcall_argument(ctx, 2));
7238
7239 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_flat_to_data), 3));
7240
7241 g(gen_frame_store_raw(ctx, OP_SIZE_SLOT, dest_arg->slot, new_fp_offset, R_RET0));
7242
7243 g(gen_set_1(ctx, R_FRAME, dest_arg->slot, new_fp_offset, true));
7244 }
7245
7246 if (flag_is_clear(ctx, src_arg->slot))
7247 goto skip_ref_argument;
7248
7249 gen_insn(INSN_JMP, 0, 0, 0);
7250 gen_four(next_arg_label);
7251 }
7252 gen_label(non_flat_label);
7253
7254 if (dest_arg->may_be_borrowed && src_arg->flags & OPCODE_CALL_MAY_LEND) {
7255 g(gen_frame_load(ctx, OP_SIZE_SLOT, false, src_arg->slot, 0, R_SCRATCH_1));
7256 g(gen_ptr_is_thunk(ctx, R_SCRATCH_1, true, thunk_label));
7257 g(gen_frame_store_raw(ctx, OP_SIZE_SLOT, dest_arg->slot, new_fp_offset, R_SCRATCH_1));
7258 gen_insn(INSN_JMP, 0, 0, 0);
7259 gen_four(next_arg_label);
7260 } else if (dest_arg->may_be_borrowed && src_arg->flags & OPCODE_CALL_MAY_GIVE) {
7261 g(gen_test_1_cached(ctx, src_arg->slot, thunk_label));
7262 g(gen_frame_load(ctx, OP_SIZE_SLOT, false, src_arg->slot, 0, R_SCRATCH_1));
7263 g(gen_ptr_is_thunk(ctx, R_SCRATCH_1, true, thunk_label));
7264 g(gen_frame_store_raw(ctx, OP_SIZE_SLOT, dest_arg->slot, new_fp_offset, R_SCRATCH_1));
7265 g(gen_frame_clear_raw(ctx, OP_SIZE_SLOT, src_arg->slot));
7266 gen_insn(INSN_JMP, 0, 0, 0);
7267 gen_four(next_arg_label);
7268 }
7269
7270 gen_label(thunk_label);
7271 g(gen_set_1(ctx, R_FRAME, dest_arg->slot, new_fp_offset, true));
7272 g(gen_frame_load(ctx, OP_SIZE_SLOT, false, src_arg->slot, 0, R_SCRATCH_1));
7273 g(gen_frame_store_raw(ctx, OP_SIZE_SLOT, dest_arg->slot, new_fp_offset, R_SCRATCH_1));
7274 if (src_arg->flags & OPCODE_FLAG_FREE_ARGUMENT) {
7275 g(gen_frame_clear_raw(ctx, OP_SIZE_SLOT, src_arg->slot));
7276 if (flag_is_set(ctx, src_arg->slot)) {
7277 g(gen_set_1(ctx, R_FRAME, src_arg->slot, 0, false));
7278 flag_set(ctx, src_arg->slot, false);
7279 goto skip_ref_argument;
7280 }
7281 if (flag_is_clear(ctx, src_arg->slot))
7282 goto do_reference;
7283 g(gen_test_1(ctx, R_FRAME, src_arg->slot, 0, incr_ref_label, true, TEST_CLEAR));
7284 gen_insn(INSN_JMP, 0, 0, 0);
7285 gen_four(next_arg_label);
7286 }
7287 do_reference:
7288 gen_label(incr_ref_label);
7289
7290 g(gen_upcall_start(ctx, 1));
7291
7292 gen_insn(INSN_MOV, i_size(OP_SIZE_ADDRESS), 0, 0);
7293 gen_one(R_ARG0);
7294 gen_one(R_SCRATCH_1);
7295 g(gen_upcall_argument(ctx, 0));
7296
7297 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_pointer_reference_owned), 1));
7298
7299 skip_ref_argument:
7300 gen_label(next_arg_label);
7301 }
7302
7303 g(load_function_offset(ctx, R_SCRATCH_1, offsetof(struct data, u_.function.local_directory[fn_idx])));
7304
7305 g(gen_address(ctx, R_SCRATCH_1, 0, IMM_PURPOSE_STR_OFFSET, OP_SIZE_SLOT));
7306 gen_insn(ARCH_PREFERS_SX(OP_SIZE_SLOT) ? INSN_MOVSX : INSN_MOV, OP_SIZE_SLOT, 0, 0);
7307 gen_one(R_SCRATCH_1);
7308 gen_address_offset();
7309
7310 g(gen_decompress_pointer(ctx, R_SCRATCH_1, 0));
7311
7312 g(gen_frame_store_raw(ctx, OP_SIZE_ADDRESS, 0, frame_offs(function) + new_fp_offset, R_SCRATCH_1));
7313
7314 #if !defined(ARCH_X86) && !defined(ARCH_PARISC)
7315 g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_ADDRESS), ALU_SUB, R_FRAME, R_FRAME, -new_fp_offset));
7316 #else
7317 g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_ADDRESS), ALU_ADD, R_FRAME, R_FRAME, new_fp_offset));
7318 #endif
7319
7320 g(gen_address(ctx, R_SCRATCH_1, offsetof(struct data, u_.function.codegen), ARCH_PREFERS_SX(OP_SIZE_SLOT) ? IMM_PURPOSE_LDR_SX_OFFSET : IMM_PURPOSE_LDR_OFFSET, OP_SIZE_SLOT));
7321 gen_insn(ARCH_PREFERS_SX(OP_SIZE_SLOT) ? INSN_MOVSX : INSN_MOV, OP_SIZE_SLOT, 0, 0);
7322 gen_one(R_SCRATCH_1);
7323 gen_address_offset();
7324
7325 g(gen_ptr_is_thunk(ctx, R_SCRATCH_1, true, ctx->escape_nospill_label));
7326 g(gen_barrier(ctx));
7327
7328 gen_pointer_compression(R_SCRATCH_1);
7329 #if (defined(ARCH_X86) && !defined(ARCH_X86_X32)) || defined(ARCH_ARM32)
7330 g(gen_address(ctx, R_SCRATCH_1, offsetof(struct data, u_.codegen.unoptimized_code_base), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_ADDRESS));
7331 gen_insn(INSN_JMP_INDIRECT, 0, 0, 0);
7332 gen_address_offset_compressed();
7333 #else
7334 g(gen_address(ctx, R_SCRATCH_1, offsetof(struct data, u_.codegen.unoptimized_code_base), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_ADDRESS));
7335 gen_insn(INSN_MOV, OP_SIZE_ADDRESS, 0, 0);
7336 gen_one(R_SCRATCH_1);
7337 gen_address_offset_compressed();
7338
7339 gen_insn(INSN_JMP_INDIRECT, 0, 0, 0);
7340 gen_one(R_SCRATCH_1);
7341 #endif
7342 g(clear_flag_cache(ctx));
7343
7344 return true;
7345 }
7346
7347 static bool attr_w gen_return(struct codegen_context *ctx)
7348 {
7349 int64_t new_fp_offset;
7350 uint32_t escape_label;
7351 arg_t i;
7352 int64_t retval_offset;
7353
7354 escape_label = alloc_escape_label(ctx);
7355 if (unlikely(!escape_label))
7356 return false;
7357
7358 new_fp_offset = (size_t)da(ctx->fn,function)->frame_slots * slot_size;
7359
7360 g(gen_frame_load_raw(ctx, OP_SIZE_ADDRESS, false, 0, new_fp_offset + frame_offs(function), R_SCRATCH_2));
7361
7362 g(gen_jmp_on_zero(ctx, OP_SIZE_ADDRESS, R_SCRATCH_2, COND_E, escape_label));
7363
7364 g(gen_address(ctx, R_SCRATCH_2, offsetof(struct data, u_.function.codegen), ARCH_PREFERS_SX(OP_SIZE_SLOT) ? IMM_PURPOSE_LDR_SX_OFFSET : IMM_PURPOSE_LDR_OFFSET, OP_SIZE_SLOT));
7365 gen_insn(ARCH_PREFERS_SX(OP_SIZE_SLOT) ? INSN_MOVSX : INSN_MOV, OP_SIZE_SLOT, 0, 0);
7366 gen_one(R_SCRATCH_1);
7367 gen_address_offset();
7368
7369 g(gen_ptr_is_thunk(ctx, R_SCRATCH_1, true, escape_label));
7370 g(gen_barrier(ctx));
7371
7372 g(gen_frame_load_raw(ctx, log_2(sizeof(timestamp_t)), false, 0, frame_offs(timestamp), R_SCRATCH_1));
7373 g(gen_frame_store_raw(ctx, log_2(sizeof(timestamp_t)), 0, new_fp_offset + frame_offs(timestamp), R_SCRATCH_1));
7374
7375 g(gen_frame_load_raw(ctx, log_2(sizeof(ip_t)), false, 0, frame_offs(previous_ip), R_SCRATCH_1));
7376
7377 g(gen_address(ctx, R_SCRATCH_2, offsetof(struct data, u_.function.code), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_ADDRESS));
7378 gen_insn(INSN_MOV, OP_SIZE_ADDRESS, 0, 0);
7379 gen_one(R_SCRATCH_2);
7380 gen_address_offset();
7381
7382 g(gen_lea3(ctx, R_SAVED_1, R_SCRATCH_2, R_SCRATCH_1, log_2(sizeof(code_t)), 0));
7383
7384 retval_offset = 0;
7385 for (i = 0; i < ctx->args_l; i++) {
7386 const struct code_arg *src_arg = &ctx->args[i];
7387 const struct type *t = get_type_of_local(ctx, src_arg->slot);
7388 uint32_t copy_ptr_label, load_write_ptr_label, write_ptr_label, next_arg_label;
7389
7390 copy_ptr_label = alloc_label(ctx);
7391 if (unlikely(!copy_ptr_label))
7392 return false;
7393
7394 load_write_ptr_label = alloc_label(ctx);
7395 if (unlikely(!load_write_ptr_label))
7396 return false;
7397
7398 write_ptr_label = alloc_label(ctx);
7399 if (unlikely(!write_ptr_label))
7400 return false;
7401
7402 next_arg_label = alloc_label(ctx);
7403 if (unlikely(!next_arg_label))
7404 return false;
7405
7406 g(gen_load_code_32(ctx, R_SAVED_2, R_SAVED_1, retval_offset));
7407
7408 if (TYPE_IS_FLAT(t)) {
7409 uint32_t flat_to_data_label;
7410 g(gen_test_1_cached(ctx, src_arg->slot, copy_ptr_label));
7411
7412 flat_to_data_label = alloc_label(ctx);
7413 if (unlikely(!flat_to_data_label))
7414 return false;
7415
7416 #if defined(ARCH_X86)
7417 g(gen_address(ctx, R_SAVED_1, retval_offset + 2 + 2 * (ARG_MODE_N >= 3), IMM_PURPOSE_LDR_OFFSET, log_2(sizeof(code_t))));
7418 g(gen_imm(ctx, OPCODE_MAY_RETURN_FLAT, IMM_PURPOSE_TEST, log_2(sizeof(code_t))));
7419 gen_insn(INSN_TEST, log_2(sizeof(code_t)), 0, 1);
7420 gen_address_offset();
7421 gen_imm_offset();
7422
7423 gen_insn(INSN_JMP_COND, log_2(sizeof(code_t)), COND_E, 0);
7424 gen_four(flat_to_data_label);
7425 #else
7426 g(gen_load_two(ctx, R_SCRATCH_1, R_SAVED_1, retval_offset + 2 + 2 * (ARG_MODE_N >= 3)));
7427
7428 g(gen_cmp_test_imm_jmp(ctx, INSN_TEST, OP_SIZE_NATIVE, R_SCRATCH_1, OPCODE_MAY_RETURN_FLAT, COND_E, flat_to_data_label));
7429 #endif
7430 #if defined(ARCH_X86)
7431 if (is_power_of_2(t->size) && t->size <= 2U << OP_SIZE_NATIVE) {
7432 if (t->size == 2U << OP_SIZE_NATIVE) {
7433 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, src_arg->slot, 0, R_SCRATCH_1, R_SCRATCH_2));
7434
7435 gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
7436 gen_one(ARG_ADDRESS_2 + OP_SIZE_SLOT);
7437 gen_one(R_FRAME);
7438 gen_one(R_SAVED_2);
7439 gen_eight(new_fp_offset + lo_word(OP_SIZE_NATIVE));
7440 gen_one(R_SCRATCH_1);
7441
7442 gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
7443 gen_one(ARG_ADDRESS_2 + OP_SIZE_SLOT);
7444 gen_one(R_FRAME);
7445 gen_one(R_SAVED_2);
7446 gen_eight(new_fp_offset + hi_word(OP_SIZE_NATIVE));
7447 gen_one(R_SCRATCH_2);
7448 } else {
7449 g(gen_frame_load(ctx, log_2(t->size), false, src_arg->slot, 0, R_SCRATCH_1));
7450
7451 gen_insn(INSN_MOV, log_2(t->size), 0, 0);
7452 gen_one(ARG_ADDRESS_2 + OP_SIZE_SLOT);
7453 gen_one(R_FRAME);
7454 gen_one(R_SAVED_2);
7455 gen_eight(new_fp_offset);
7456 gen_one(R_SCRATCH_1);
7457 }
7458 } else
7459 #endif
7460 {
7461 g(gen_lea3(ctx, R_SCRATCH_2, R_FRAME, R_SAVED_2, OP_SIZE_SLOT, new_fp_offset));
7462
7463 g(gen_memcpy_from_slot(ctx, R_SCRATCH_2, 0, src_arg->slot));
7464 }
7465
7466 gen_insn(INSN_JMP, 0, 0, 0);
7467 gen_four(next_arg_label);
7468
7469 gen_label(flat_to_data_label);
7470
7471 if (ctx->registers[src_arg->slot] >= 0)
7472 g(spill(ctx, src_arg->slot));
7473
7474 g(gen_upcall_start(ctx, 3));
7475
7476 gen_insn(INSN_MOV, i_size(OP_SIZE_ADDRESS), 0, 0);
7477 gen_one(R_ARG0);
7478 gen_one(R_FRAME);
7479 g(gen_upcall_argument(ctx, 0));
7480
7481 g(gen_load_constant(ctx, R_ARG1, src_arg->slot));
7482 g(gen_upcall_argument(ctx, 1));
7483
7484 g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_ADDRESS), ALU_ADD, R_ARG2, R_FRAME, (size_t)src_arg->slot * slot_size));
7485 g(gen_upcall_argument(ctx, 2));
7486
7487 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_flat_to_data), 3));
7488
7489 if (flag_is_clear(ctx, src_arg->slot))
7490 goto skip_ref_argument;
7491
7492 gen_insn(INSN_JMP, 0, 0, 0);
7493 gen_four(write_ptr_label);
7494 }
7495
7496 gen_label(copy_ptr_label);
7497
7498 if (unlikely(!(src_arg->flags & OPCODE_FLAG_FREE_ARGUMENT))) {
7499 g(gen_upcall_start(ctx, 1));
7500 g(gen_frame_load(ctx, OP_SIZE_SLOT, false, src_arg->slot, 0, R_ARG0));
7501 g(gen_upcall_argument(ctx, 0));
7502 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_pointer_reference_owned), 1));
7503 } else if (da(ctx->fn,function)->local_variables_flags[src_arg->slot].may_be_borrowed) {
7504 g(gen_test_1_cached(ctx, src_arg->slot, load_write_ptr_label));
7505 g(gen_upcall_start(ctx, 1));
7506 g(gen_frame_load(ctx, OP_SIZE_SLOT, false, src_arg->slot, 0, R_ARG0));
7507 g(gen_upcall_argument(ctx, 0));
7508 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_pointer_reference_owned), 1));
7509 }
7510
7511 gen_label(load_write_ptr_label);
7512
7513 g(gen_frame_load(ctx, OP_SIZE_SLOT, false, src_arg->slot, 0, R_RET0));
7514
7515 skip_ref_argument:
7516 gen_label(write_ptr_label);
7517
7518 #if defined(ARCH_X86)
7519 gen_insn(INSN_MOV, OP_SIZE_SLOT, 0, 0);
7520 gen_one(ARG_ADDRESS_2 + OP_SIZE_SLOT);
7521 gen_one(R_FRAME);
7522 gen_one(R_SAVED_2);
7523 gen_eight(new_fp_offset);
7524 gen_one(R_RET0);
7525 goto scaled_store_done;
7526 #endif
7527 if (ARCH_HAS_SHIFTED_ADD(OP_SIZE_SLOT)) {
7528 gen_insn(INSN_ALU, i_size(OP_SIZE_ADDRESS), ALU_ADD, ALU_WRITES_FLAGS(ALU_ADD, false));
7529 gen_one(R_SCRATCH_3);
7530 gen_one(R_FRAME);
7531 gen_one(ARG_SHIFTED_REGISTER);
7532 gen_one(ARG_SHIFT_LSL | OP_SIZE_SLOT);
7533 gen_one(R_SAVED_2);
7534
7535 g(gen_address(ctx, R_SCRATCH_3, new_fp_offset, IMM_PURPOSE_STR_OFFSET, OP_SIZE_SLOT));
7536 gen_insn(INSN_MOV, OP_SIZE_SLOT, 0, 0);
7537 gen_address_offset();
7538 gen_one(R_RET0);
7539 goto scaled_store_done;
7540 }
7541
7542 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SHL, R_SCRATCH_3, R_SAVED_2, OP_SIZE_SLOT, false));
7543
7544 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_ADD, R_SCRATCH_3, R_SCRATCH_3, R_FRAME));
7545
7546 g(gen_address(ctx, R_SCRATCH_3, new_fp_offset, IMM_PURPOSE_STR_OFFSET, OP_SIZE_SLOT));
7547 gen_insn(INSN_MOV, OP_SIZE_SLOT, 0, 0);
7548 gen_address_offset();
7549 gen_one(R_RET0);
7550
7551 scaled_store_done:
7552 g(gen_set_1_variable(ctx, R_SAVED_2, new_fp_offset, true));
7553
7554 gen_label(next_arg_label);
7555
7556 retval_offset += 4 + 2 * (ARG_MODE_N >= 3);
7557 }
7558
7559 g(gen_frame_load_raw(ctx, OP_SIZE_ADDRESS, false, 0, new_fp_offset + frame_offs(function), R_SCRATCH_1));
7560
7561 g(gen_address(ctx, R_SCRATCH_1, offsetof(struct data, u_.function.codegen), ARCH_PREFERS_SX(OP_SIZE_SLOT) ? IMM_PURPOSE_LDR_SX_OFFSET : IMM_PURPOSE_LDR_OFFSET, OP_SIZE_SLOT));
7562 gen_insn(ARCH_PREFERS_SX(OP_SIZE_SLOT) ? INSN_MOVSX : INSN_MOV, OP_SIZE_SLOT, 0, 0);
7563 gen_one(R_SCRATCH_1);
7564 gen_address_offset();
7565
7566 g(gen_decompress_pointer(ctx, R_SCRATCH_1, 0));
7567
7568 g(gen_load_code_32(ctx, R_SCRATCH_2, R_SAVED_1, retval_offset + 2));
7569
7570 g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_ADDRESS), ALU_ADD, R_FRAME, R_FRAME, new_fp_offset));
7571
7572 #if defined(ARCH_X86) && !defined(ARCH_X86_X32)
7573 gen_insn(INSN_JMP_INDIRECT, 0, 0, 0);
7574 gen_one(ARG_ADDRESS_2 + OP_SIZE_ADDRESS);
7575 gen_one(R_SCRATCH_1);
7576 gen_one(R_SCRATCH_2);
7577 gen_eight(offsetof(struct data, u_.codegen.unoptimized_code));
7578
7579 goto scaled_jmp_done;
7580 #endif
7581 #if defined(ARCH_X86)
7582 gen_insn(INSN_MOV, OP_SIZE_ADDRESS, 0, 0);
7583 gen_one(R_SCRATCH_1);
7584 gen_one(ARG_ADDRESS_2 + OP_SIZE_ADDRESS);
7585 gen_one(R_SCRATCH_1);
7586 gen_one(R_SCRATCH_2);
7587 gen_eight(offsetof(struct data, u_.codegen.unoptimized_code));
7588
7589 gen_insn(INSN_JMP_INDIRECT, 0, 0, 0);
7590 gen_one(R_SCRATCH_1);
7591
7592 goto scaled_jmp_done;
7593 #endif
7594 #if defined(ARCH_ARM32)
7595 gen_insn(INSN_ALU, i_size(OP_SIZE_ADDRESS), ALU_ADD, ALU_WRITES_FLAGS(ALU_ADD, false));
7596 gen_one(R_SCRATCH_1);
7597 gen_one(R_SCRATCH_1);
7598 gen_one(ARG_SHIFTED_REGISTER);
7599 gen_one(ARG_SHIFT_LSL | OP_SIZE_ADDRESS);
7600 gen_one(R_SCRATCH_2);
7601
7602 g(gen_address(ctx, R_SCRATCH_1, offsetof(struct data, u_.codegen.unoptimized_code), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_ADDRESS));
7603 gen_insn(INSN_JMP_INDIRECT, 0, 0, 0);
7604 gen_address_offset();
7605
7606 goto scaled_jmp_done;
7607 #endif
7608 if (ARCH_HAS_SHIFTED_ADD(OP_SIZE_ADDRESS)) {
7609 gen_insn(INSN_ALU, i_size(OP_SIZE_ADDRESS), ALU_ADD, ALU_WRITES_FLAGS(ALU_ADD, false));
7610 gen_one(R_SCRATCH_1);
7611 gen_one(R_SCRATCH_1);
7612 gen_one(ARG_SHIFTED_REGISTER);
7613 gen_one(ARG_SHIFT_LSL | OP_SIZE_ADDRESS);
7614 gen_one(R_SCRATCH_2);
7615
7616 g(gen_address(ctx, R_SCRATCH_1, offsetof(struct data, u_.codegen.unoptimized_code), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_ADDRESS));
7617 gen_insn(INSN_MOV, OP_SIZE_ADDRESS, 0, 0);
7618 gen_one(R_SCRATCH_1);
7619 gen_address_offset();
7620
7621 gen_insn(INSN_JMP_INDIRECT, 0, 0, 0);
7622 gen_one(R_SCRATCH_1);
7623
7624 goto scaled_jmp_done;
7625 }
7626
7627 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SHL, R_SCRATCH_2, R_SCRATCH_2, OP_SIZE_ADDRESS, false));
7628
7629 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_ADD, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_2));
7630
7631 g(gen_address(ctx, R_SCRATCH_1, offsetof(struct data, u_.codegen.unoptimized_code), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_ADDRESS));
7632 gen_insn(INSN_MOV, OP_SIZE_ADDRESS, 0, 0);
7633 gen_one(R_SCRATCH_1);
7634 gen_address_offset();
7635
7636 gen_insn(INSN_JMP_INDIRECT, 0, 0, 0);
7637 gen_one(R_SCRATCH_1);
7638
7639 goto scaled_jmp_done;
7640 scaled_jmp_done:
7641 return true;
7642 }
7643
7644 static bool attr_w gen_scaled_array_address(struct codegen_context *ctx, size_t element_size, unsigned reg_dst, unsigned reg_src, unsigned reg_index, int64_t offset_src);
7645 static bool attr_w gen_check_array_len(struct codegen_context *ctx, unsigned reg_array, bool allocated, unsigned reg_len, unsigned cond, uint32_t escape_label);
7646
7647 static bool attr_w gen_structured(struct codegen_context *ctx, frame_t slot_struct, frame_t slot_elem)
7648 {
7649 uint32_t escape_label;
7650 const struct type *struct_type, *elem_type;
7651 size_t i;
7652
7653 escape_label = alloc_escape_label(ctx);
7654 if (unlikely(!escape_label))
7655 return false;
7656
7657 struct_type = get_type_of_local(ctx, slot_struct);
7658 elem_type = get_type_of_local(ctx, slot_elem);
7659
7660 if (TYPE_IS_FLAT(struct_type) && struct_type->tag != TYPE_TAG_flat_option) {
7661 if (!TYPE_IS_FLAT(elem_type)) {
7662 goto struct_zero;
7663 } else {
7664 g(gen_test_1_cached(ctx, slot_struct, escape_label));
7665 flag_set(ctx, slot_struct, false);
7666 }
7667 } else {
7668 struct_zero:
7669 g(gen_test_1_jz_cached(ctx, slot_struct, escape_label));
7670 struct_type = NULL;
7671 }
7672
7673 g(gen_frame_address(ctx, slot_struct, 0, R_SAVED_1));
7674
7675 for (i = 0; i < ctx->args_l; i++) {
7676 frame_t param_slot = ctx->args[i].slot;
7677 if (struct_type) {
7678 switch (ctx->args[i].flags & OPCODE_STRUCTURED_MASK) {
7679 case OPCODE_STRUCTURED_RECORD: {
7680 struct flat_record_definition_entry *e;
7681 ajla_assert_lo(struct_type->tag == TYPE_TAG_flat_record, (file_line, "gen_structured: invalid tag %u, expected %u", struct_type->tag, TYPE_TAG_flat_record));
7682 e = &type_def(struct_type,flat_record)->entries[param_slot];
7683
7684 g(gen_imm(ctx, e->flat_offset, IMM_PURPOSE_ADD, i_size(OP_SIZE_ADDRESS)));
7685 gen_insn(INSN_ALU, i_size(OP_SIZE_ADDRESS), ALU_ADD, ALU_WRITES_FLAGS(ALU_ADD, is_imm()));
7686 gen_one(R_SAVED_1);
7687 gen_one(R_SAVED_1);
7688 gen_imm_offset();
7689
7690 struct_type = e->subtype;
7691 break;
7692 }
7693 case OPCODE_STRUCTURED_ARRAY: {
7694 ajla_assert_lo(struct_type->tag == TYPE_TAG_flat_array, (file_line, "gen_structured: invalid tag %u, expected %u", struct_type->tag, TYPE_TAG_flat_array));
7695 g(gen_test_1_cached(ctx, param_slot, escape_label));
7696 flag_set(ctx, param_slot, false);
7697 g(gen_frame_load(ctx, OP_SIZE_INT, false, param_slot, 0, R_SCRATCH_1));
7698
7699 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, OP_SIZE_INT, R_SCRATCH_1, type_def(struct_type,flat_array)->n_elements, COND_AE, escape_label));
7700
7701 g(gen_scaled_array_address(ctx, type_def(struct_type,flat_array)->base->size, R_SAVED_1, R_SAVED_1, R_SCRATCH_1, 0));
7702
7703 struct_type = type_def(struct_type,flat_array)->base;
7704 break;
7705 }
7706 default:
7707 internal(file_line, "gen_structured: invalid structured flags %x", (unsigned)ctx->args[i].flags);
7708 }
7709 } else {
7710 gen_insn(ARCH_PREFERS_SX(OP_SIZE_SLOT) ? INSN_MOVSX : INSN_MOV, OP_SIZE_SLOT, 0, 0);
7711 gen_one(R_SCRATCH_1);
7712 gen_one(ARG_ADDRESS_1);
7713 gen_one(R_SAVED_1);
7714 gen_eight(0);
7715
7716 g(gen_ptr_is_thunk(ctx, R_SCRATCH_1, true, escape_label));
7717 g(gen_barrier(ctx));
7718
7719 g(gen_decompress_pointer(ctx, R_SCRATCH_1, 0));
7720
7721 g(gen_compare_refcount(ctx, R_SCRATCH_1, REFCOUNT_STEP, COND_AE, escape_label));
7722
7723 switch (ctx->args[i].flags & OPCODE_STRUCTURED_MASK) {
7724 case OPCODE_STRUCTURED_RECORD: {
7725 const struct type *rec_type, *e_type;
7726 rec_type = da_type(ctx->fn, ctx->args[i].type);
7727 TYPE_TAG_VALIDATE(rec_type->tag);
7728 if (unlikely(rec_type->tag == TYPE_TAG_flat_record))
7729 rec_type = type_def(rec_type,flat_record)->base;
7730 e_type = type_def(rec_type,record)->types[param_slot];
7731 if (!TYPE_IS_FLAT(e_type) || (e_type->tag == TYPE_TAG_flat_option && !(ctx->args[i].flags & OPCODE_STRUCTURED_FLAG_END))) {
7732 g(gen_test_1(ctx, R_SCRATCH_1, param_slot, data_record_offset, escape_label, true, TEST));
7733 } else {
7734 g(gen_test_1(ctx, R_SCRATCH_1, param_slot, data_record_offset, escape_label, false, TEST));
7735 struct_type = e_type;
7736 }
7737 g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_ADDRESS), ALU_ADD, R_SAVED_1, R_SCRATCH_1, data_record_offset + (size_t)param_slot * slot_size));
7738 break;
7739 }
7740 case OPCODE_STRUCTURED_OPTION: {
7741 unsigned op_size = log_2(sizeof(ajla_option_t));
7742 #if defined(ARCH_X86)
7743 g(gen_address(ctx, R_SCRATCH_1, offsetof(struct data, u_.option.option), IMM_PURPOSE_LDR_OFFSET, op_size));
7744 g(gen_imm(ctx, param_slot, IMM_PURPOSE_CMP, op_size));
7745 gen_insn(INSN_CMP, op_size, 0, 1);
7746 gen_address_offset();
7747 gen_imm_offset();
7748
7749 gen_insn(INSN_JMP_COND, op_size, COND_NE, 0);
7750 gen_four(escape_label);
7751 #else
7752 g(gen_address(ctx, R_SCRATCH_1, offsetof(struct data, u_.option.option), ARCH_PREFERS_SX(op_size) ? IMM_PURPOSE_LDR_SX_OFFSET : IMM_PURPOSE_LDR_OFFSET, op_size));
7753 gen_insn(ARCH_PREFERS_SX(op_size) ? INSN_MOVSX : INSN_MOV, op_size, 0, 0);
7754 gen_one(R_SCRATCH_2);
7755 gen_address_offset();
7756
7757 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, i_size(op_size), R_SCRATCH_2, param_slot, COND_NE, escape_label));
7758 #endif
7759 g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_ADDRESS), ALU_ADD, R_SAVED_1, R_SCRATCH_1, offsetof(struct data, u_.option.pointer)));
7760 break;
7761 }
7762 case OPCODE_STRUCTURED_ARRAY: {
7763 const struct type *e_type = da_type(ctx->fn, ctx->args[i].type);
7764
7765 g(gen_test_1_cached(ctx, param_slot, escape_label));
7766 flag_set(ctx, param_slot, false);
7767
7768 g(gen_frame_load(ctx, OP_SIZE_INT, false, param_slot, 0, R_SCRATCH_2));
7769
7770 g(gen_check_array_len(ctx, R_SCRATCH_1, false, R_SCRATCH_2, COND_AE, escape_label));
7771
7772 if (!TYPE_IS_FLAT(e_type) || (e_type->tag == TYPE_TAG_flat_option && !(ctx->args[i].flags & OPCODE_STRUCTURED_FLAG_END))) {
7773 g(gen_compare_ptr_tag(ctx, R_SCRATCH_1, DATA_TAG_array_pointers, COND_NE, escape_label, R_SCRATCH_3));
7774
7775 g(gen_address(ctx, R_SCRATCH_1, offsetof(struct data, u_.array_pointers.pointer), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_ADDRESS));
7776 gen_insn(INSN_MOV, OP_SIZE_ADDRESS, 0, 0);
7777 gen_one(R_SCRATCH_1);
7778 gen_address_offset();
7779
7780 g(gen_scaled_array_address(ctx, slot_size, R_SAVED_1, R_SCRATCH_1, R_SCRATCH_2, 0));
7781 } else {
7782 g(gen_compare_ptr_tag(ctx, R_SCRATCH_1, DATA_TAG_array_flat, COND_NE, escape_label, R_SCRATCH_3));
7783
7784 g(gen_scaled_array_address(ctx, e_type->size, R_SAVED_1, R_SCRATCH_1, R_SCRATCH_2, data_array_offset));
7785
7786 struct_type = e_type;
7787 }
7788 break;
7789 }
7790 default: {
7791 internal(file_line, "gen_structured: invalid structured flags %x", (unsigned)ctx->args[i].flags);
7792 }
7793 }
7794 }
7795 }
7796
7797 if (struct_type) {
7798 g(gen_test_1_cached(ctx, slot_elem, escape_label));
7799 flag_set(ctx, slot_elem, false);
7800 g(gen_memcpy_from_slot(ctx, R_SAVED_1, 0, slot_elem));
7801 } else {
7802 uint32_t skip_deref_label;
7803 skip_deref_label = alloc_label(ctx);
7804 if (unlikely(!skip_deref_label))
7805 return false;
7806
7807 if (TYPE_IS_FLAT(elem_type))
7808 g(gen_test_1_jz_cached(ctx, slot_elem, escape_label));
7809
7810 gen_insn(ARCH_PREFERS_SX(OP_SIZE_SLOT) ? INSN_MOVSX : INSN_MOV, OP_SIZE_SLOT, 0, 0);
7811 gen_one(R_SCRATCH_1);
7812 gen_one(ARG_ADDRESS_1);
7813 gen_one(R_SAVED_1);
7814 gen_eight(0);
7815
7816 g(gen_jmp_on_zero(ctx, OP_SIZE_SLOT, R_SCRATCH_1, COND_E, skip_deref_label));
7817
7818 g(gen_upcall_start(ctx, 1));
7819 gen_insn(INSN_MOV, i_size(OP_SIZE_ADDRESS), 0, 0);
7820 gen_one(R_ARG0);
7821 gen_one(R_SCRATCH_1);
7822 g(gen_upcall_argument(ctx, 0));
7823 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_pointer_dereference), 1));
7824
7825 gen_label(skip_deref_label);
7826
7827 g(gen_frame_get_pointer(ctx, slot_elem, (ctx->args[i - 1].flags & OPCODE_STRUCTURED_FREE_VARIABLE) != 0, R_SCRATCH_1));
7828
7829 gen_insn(INSN_MOV, OP_SIZE_SLOT, 0, 0);
7830 gen_one(ARG_ADDRESS_1);
7831 gen_one(R_SAVED_1);
7832 gen_eight(0);
7833 gen_one(R_SCRATCH_1);
7834 }
7835
7836 return true;
7837 }
7838
7839 static bool attr_w gen_record_create(struct codegen_context *ctx, frame_t slot_r)
7840 {
7841 const struct type *t;
7842 const struct record_definition *def;
7843 uint32_t escape_label;
7844 arg_t i, ii;
7845
7846 escape_label = alloc_escape_label(ctx);
7847 if (unlikely(!escape_label))
7848 return false;
7849
7850 t = get_type_of_local(ctx, slot_r);
7851 if (t->tag == TYPE_TAG_flat_record) {
7852 const struct flat_record_definition *flat_def;
7853 const struct type *flat_type = t;
7854 t = type_def(t,flat_record)->base;
7855 def = type_def(t,record);
7856 flat_def = type_def(flat_type,flat_record);
7857 for (i = 0; i < ctx->args_l; i++) {
7858 frame_t var_slot = ctx->args[i].slot;
7859 g(gen_test_1_cached(ctx, var_slot, escape_label));
7860 flag_set(ctx, var_slot, false);
7861 }
7862 for (i = 0, ii = 0; i < ctx->args_l; i++, ii++) {
7863 frame_t var_slot, flat_offset, record_slot;
7864 while (unlikely(record_definition_is_elided(def, ii)))
7865 ii++;
7866 var_slot = ctx->args[i].slot;
7867 record_slot = record_definition_slot(def, ii);
7868 flat_offset = flat_def->entries[record_slot].flat_offset;
7869 g(gen_memcpy_from_slot(ctx, R_FRAME, (size_t)slot_r * slot_size + flat_offset, var_slot));
7870 }
7871 return true;
7872 }
7873
7874 def = type_def(t,record);
7875
7876 g(gen_upcall_start(ctx, 2));
7877
7878 gen_insn(INSN_MOV, i_size(OP_SIZE_ADDRESS), 0, 0);
7879 gen_one(R_ARG0);
7880 gen_one(R_FRAME);
7881 g(gen_upcall_argument(ctx, 0));
7882
7883 g(gen_load_constant(ctx, R_ARG1, slot_r));
7884 g(gen_upcall_argument(ctx, 1));
7885
7886 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_data_alloc_record_mayfail), 2));
7887 g(gen_sanitize_returned_pointer(ctx, R_RET0));
7888 g(gen_jmp_on_zero(ctx, OP_SIZE_ADDRESS, R_RET0, COND_E, escape_label));
7889
7890 gen_insn(INSN_MOV, i_size(OP_SIZE_ADDRESS), 0, 0);
7891 gen_one(R_SAVED_1);
7892 gen_one(R_RET0);
7893
7894 g(gen_clear_bitmap(ctx, 0, R_SAVED_1, data_record_offset, bitmap_slots(def->n_slots)));
7895
7896 for (i = 0, ii = 0; i < ctx->args_l; i++, ii++) {
7897 frame_t var_slot, var_flags, record_slot;
7898 const struct type *var_type, *record_type;
7899 uint32_t skip_flat_label, set_ptr_label, next_arg_label;
7900
7901 skip_flat_label = alloc_label(ctx);
7902 if (unlikely(!skip_flat_label))
7903 return false;
7904 set_ptr_label = alloc_label(ctx);
7905 if (unlikely(!set_ptr_label))
7906 return false;
7907 next_arg_label = alloc_label(ctx);
7908 if (unlikely(!next_arg_label))
7909 return false;
7910
7911 while (unlikely(record_definition_is_elided(def, ii)))
7912 ii++;
7913 var_slot = ctx->args[i].slot;
7914 var_type = get_type_of_local(ctx, var_slot);
7915 var_flags = ctx->args[i].flags;
7916 record_slot = record_definition_slot(def, ii);
7917 record_type = def->types[record_slot];
7918 if (TYPE_IS_FLAT(var_type)) {
7919 g(gen_test_1_cached(ctx, var_slot, skip_flat_label));
7920 if (TYPE_IS_FLAT(record_type)) {
7921 g(gen_memcpy_from_slot(ctx, R_SAVED_1, data_record_offset + (size_t)record_slot * slot_size, var_slot));
7922
7923 gen_insn(INSN_JMP, 0, 0, 0);
7924 gen_four(next_arg_label);
7925 } else {
7926 if (ctx->registers[var_slot] >= 0)
7927 g(spill(ctx, var_slot));
7928
7929 g(gen_upcall_start(ctx, 3));
7930
7931 gen_insn(INSN_MOV, i_size(OP_SIZE_ADDRESS), 0, 0);
7932 gen_one(R_ARG0);
7933 gen_one(R_FRAME);
7934 g(gen_upcall_argument(ctx, 0));
7935
7936 g(gen_load_constant(ctx, R_ARG1, var_slot));
7937 g(gen_upcall_argument(ctx, 1));
7938
7939 g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_ADDRESS), ALU_ADD, R_ARG2, R_FRAME, (size_t)var_slot * slot_size));
7940 g(gen_upcall_argument(ctx, 2));
7941
7942 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_flat_to_data), 3));
7943
7944 gen_insn(INSN_JMP, 0, 0, 0);
7945 gen_four(set_ptr_label);
7946 }
7947 }
7948
7949 gen_label(skip_flat_label);
7950 g(gen_frame_get_pointer(ctx, var_slot, (var_flags & OPCODE_FLAG_FREE_ARGUMENT) != 0, R_RET0));
7951
7952 gen_label(set_ptr_label);
7953 g(gen_address(ctx, R_SAVED_1, data_record_offset + (size_t)record_slot * slot_size, IMM_PURPOSE_STR_OFFSET, OP_SIZE_SLOT));
7954 gen_insn(INSN_MOV, OP_SIZE_SLOT, 0, 0);
7955 gen_address_offset();
7956 gen_one(R_RET0);
7957
7958 g(gen_set_1(ctx, R_SAVED_1, record_slot, data_record_offset, true));
7959
7960 gen_label(next_arg_label);
7961 }
7962
7963 g(gen_compress_pointer(ctx, R_SAVED_1));
7964 g(gen_frame_set_pointer(ctx, slot_r, R_SAVED_1));
7965
7966 return true;
7967 }
7968
7969 static bool attr_w gen_record_load(struct codegen_context *ctx, frame_t slot_1, frame_t slot_r, frame_t rec_slot, frame_t flags)
7970 {
7971 const struct type *rec_type, *entry_type;
7972 uint32_t escape_label;
7973
7974 rec_type = get_type_of_local(ctx, slot_1);
7975 if (unlikely(rec_type->tag == TYPE_TAG_unknown)) {
7976 ajla_assert_lo(!*da(ctx->fn,function)->function_name, (file_line, "gen_record_load: function %s has record without definition", da(ctx->fn,function)->function_name));
7977 return false;
7978 }
7979
7980 escape_label = alloc_escape_label(ctx);
7981 if (unlikely(!escape_label))
7982 return false;
7983
7984 /*debug("gen_record_load: %s: %u, %u", da(ctx->fn,function)->function_name, TYPE_TAG_unknown, rec_type->tag);*/
7985 if (TYPE_IS_FLAT(rec_type)) {
7986 const struct flat_record_definition_entry *ft = &type_def(rec_type,flat_record)->entries[rec_slot];
7987 g(gen_test_1_cached(ctx, slot_1, escape_label));
7988 g(gen_memcpy_to_slot(ctx, slot_r, R_FRAME, (size_t)slot_1 * slot_size + ft->flat_offset));
7989 flag_set(ctx, slot_1, false);
7990 flag_set(ctx, slot_r, false);
7991 return true;
7992 }
7993 entry_type = type_def(rec_type,record)->types[rec_slot];
7994
7995 g(gen_frame_load(ctx, OP_SIZE_SLOT, false, slot_1, 0, R_SCRATCH_2));
7996 g(gen_ptr_is_thunk(ctx, R_SCRATCH_2, true, escape_label));
7997 g(gen_barrier(ctx));
7998
7999 g(gen_decompress_pointer(ctx, R_SCRATCH_2, 0));
8000
8001 if (TYPE_IS_FLAT(entry_type)) {
8002 g(gen_test_1(ctx, R_SCRATCH_2, rec_slot, data_record_offset, escape_label, false, TEST));
8003 g(gen_memcpy_to_slot(ctx, slot_r, R_SCRATCH_2, (size_t)rec_slot * slot_size + data_record_offset));
8004 flag_set(ctx, slot_r, false);
8005 return true;
8006 }
8007
8008 if (flag_must_be_flat(ctx, slot_r)) {
8009 gen_insn(INSN_JMP, 0, 0, 0);
8010 gen_four(escape_label);
8011 return true;
8012 }
8013
8014 g(gen_test_1(ctx, R_SCRATCH_2, rec_slot, data_record_offset, escape_label, true, TEST));
8015
8016 g(gen_address(ctx, R_SCRATCH_2, (size_t)rec_slot * slot_size + data_record_offset, ARCH_PREFERS_SX(OP_SIZE_SLOT) ? IMM_PURPOSE_LDR_SX_OFFSET : IMM_PURPOSE_LDR_OFFSET, OP_SIZE_SLOT));
8017 gen_insn(ARCH_PREFERS_SX(OP_SIZE_SLOT) ? INSN_MOVSX : INSN_MOV, OP_SIZE_SLOT, 0, 0);
8018 gen_one(R_SCRATCH_1);
8019 gen_address_offset();
8020
8021 g(gen_ptr_is_thunk(ctx, R_SCRATCH_1, true, escape_label));
8022
8023 if (flags & OPCODE_STRUCT_MAY_BORROW) {
8024 g(gen_frame_store(ctx, OP_SIZE_SLOT, slot_r, 0, R_SCRATCH_1));
8025 flag_set(ctx, slot_r, false);
8026 } else {
8027 g(gen_frame_set_pointer(ctx, slot_r, R_SCRATCH_1));
8028
8029 g(gen_upcall_start(ctx, 1));
8030 gen_insn(INSN_MOV, i_size(OP_SIZE_ADDRESS), 0, 0);
8031 gen_one(R_ARG0);
8032 gen_one(R_SCRATCH_1);
8033 g(gen_upcall_argument(ctx, 0));
8034 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_pointer_reference_owned), 1));
8035 }
8036 return true;
8037 }
8038
8039 static bool attr_w gen_option_create_empty_flat(struct codegen_context *ctx, ajla_flat_option_t opt, frame_t slot_r)
8040 {
8041 g(gen_frame_store_imm(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, opt));
8042 flag_set(ctx, slot_r, false);
8043 return true;
8044 }
8045
8046 static bool attr_w gen_option_create_empty(struct codegen_context *ctx, ajla_option_t opt, frame_t slot_r)
8047 {
8048 unsigned option_size = log_2(sizeof(ajla_option_t));
8049 uint32_t escape_label;
8050
8051 escape_label = alloc_escape_label(ctx);
8052 if (unlikely(!escape_label))
8053 return false;
8054
8055 if (flag_must_be_flat(ctx, slot_r)) {
8056 gen_insn(INSN_JMP, 0, 0, 0);
8057 gen_four(escape_label);
8058 return true;
8059 }
8060
8061 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_data_alloc_option_mayfail), 0));
8062 g(gen_sanitize_returned_pointer(ctx, R_RET0));
8063 g(gen_jmp_on_zero(ctx, OP_SIZE_ADDRESS, R_RET0, COND_E, escape_label));
8064
8065 g(gen_address(ctx, R_RET0, offsetof(struct data, u_.option.option), IMM_PURPOSE_STR_OFFSET, option_size));
8066 g(gen_imm(ctx, opt, IMM_PURPOSE_STORE_VALUE, option_size));
8067 gen_insn(INSN_MOV, option_size, 0, 0);
8068 gen_address_offset();
8069 gen_imm_offset();
8070
8071 g(gen_address(ctx, R_RET0, offsetof(struct data, u_.option.pointer), IMM_PURPOSE_STR_OFFSET, OP_SIZE_SLOT));
8072 g(gen_imm(ctx, 0, IMM_PURPOSE_STORE_VALUE, OP_SIZE_SLOT));
8073 gen_insn(INSN_MOV, OP_SIZE_SLOT, 0, 0);
8074 gen_address_offset();
8075 gen_imm_offset();
8076
8077 g(gen_compress_pointer(ctx, R_RET0));
8078 g(gen_frame_set_pointer(ctx, slot_r, R_RET0));
8079
8080 return true;
8081 }
8082
8083 static bool attr_w gen_option_create(struct codegen_context *ctx, ajla_option_t opt, frame_t slot_1, frame_t slot_r, frame_t flags)
8084 {
8085 unsigned option_size = log_2(sizeof(ajla_option_t));
8086 const struct type *type;
8087 uint32_t escape_label, get_pointer_label, got_pointer_label;
8088
8089 escape_label = alloc_escape_label(ctx);
8090 if (unlikely(!escape_label))
8091 return false;
8092
8093 if (flag_must_be_flat(ctx, slot_r)) {
8094 gen_insn(INSN_JMP, 0, 0, 0);
8095 gen_four(escape_label);
8096 return true;
8097 }
8098
8099 get_pointer_label = alloc_label(ctx);
8100 if (unlikely(!get_pointer_label))
8101 return false;
8102
8103 got_pointer_label = alloc_label(ctx);
8104 if (unlikely(!got_pointer_label))
8105 return false;
8106
8107 type = get_type_of_local(ctx, slot_1);
8108
8109 g(gen_upcall_start(ctx, 0));
8110 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_data_alloc_option_mayfail), 0));
8111 g(gen_sanitize_returned_pointer(ctx, R_RET0));
8112 g(gen_jmp_on_zero(ctx, OP_SIZE_ADDRESS, R_RET0, COND_E, escape_label));
8113
8114 gen_insn(INSN_MOV, i_size(OP_SIZE_ADDRESS), 0, 0);
8115 gen_one(R_SAVED_1);
8116 gen_one(R_RET0);
8117
8118 g(gen_address(ctx, R_RET0, offsetof(struct data, u_.option.option), IMM_PURPOSE_STR_OFFSET, option_size));
8119 g(gen_imm(ctx, opt, IMM_PURPOSE_STORE_VALUE, option_size));
8120 gen_insn(INSN_MOV, option_size, 0, 0);
8121 gen_address_offset();
8122 gen_imm_offset();
8123
8124 if (TYPE_IS_FLAT(type)) {
8125 g(gen_test_1_cached(ctx, slot_1, get_pointer_label));
8126
8127 if (ctx->registers[slot_1] >= 0)
8128 g(spill(ctx, slot_1));
8129
8130 g(gen_upcall_start(ctx, 3));
8131
8132 gen_insn(INSN_MOV, i_size(OP_SIZE_ADDRESS), 0, 0);
8133 gen_one(R_ARG0);
8134 gen_one(R_FRAME);
8135 g(gen_upcall_argument(ctx, 0));
8136
8137 g(gen_load_constant(ctx, R_ARG1, slot_1));
8138 g(gen_upcall_argument(ctx, 1));
8139
8140 g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_ADDRESS), ALU_ADD, R_ARG2, R_FRAME, (size_t)slot_1 * slot_size));
8141 g(gen_upcall_argument(ctx, 2));
8142
8143 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_flat_to_data), 3));
8144
8145 if (flag_is_clear(ctx, slot_1))
8146 goto skip_get_pointer_label;
8147
8148 gen_insn(INSN_JMP, 0, 0, 0);
8149 gen_four(got_pointer_label);
8150 }
8151
8152 gen_label(get_pointer_label);
8153 g(gen_frame_get_pointer(ctx, slot_1, (flags & OPCODE_FLAG_FREE_ARGUMENT) != 0, R_RET0));
8154
8155 skip_get_pointer_label:
8156 gen_label(got_pointer_label);
8157 g(gen_address(ctx, R_SAVED_1, offsetof(struct data, u_.option.pointer), IMM_PURPOSE_STR_OFFSET, OP_SIZE_SLOT));
8158 gen_insn(INSN_MOV, OP_SIZE_SLOT, 0, 0);
8159 gen_address_offset();
8160 gen_one(R_RET0);
8161
8162 g(gen_compress_pointer(ctx, R_SAVED_1));
8163 g(gen_frame_set_pointer(ctx, slot_r, R_SAVED_1));
8164
8165 return true;
8166 }
8167
8168 static bool attr_w gen_option_cmp(struct codegen_context *ctx, unsigned reg, frame_t opt, uint32_t label, frame_t slot_r)
8169 {
8170 unsigned op_size = log_2(sizeof(ajla_option_t));
8171 #if ARCH_HAS_FLAGS
8172 #if defined(ARCH_X86)
8173 g(gen_address(ctx, reg, offsetof(struct data, u_.option.option), IMM_PURPOSE_LDR_OFFSET, op_size));
8174 g(gen_imm(ctx, opt, IMM_PURPOSE_CMP, op_size));
8175 gen_insn(INSN_CMP, op_size, 0, 1);
8176 gen_address_offset();
8177 gen_imm_offset();
8178 #else
8179 g(gen_address(ctx, reg, offsetof(struct data, u_.option.option), ARCH_PREFERS_SX(op_size) ? IMM_PURPOSE_LDR_SX_OFFSET : IMM_PURPOSE_LDR_OFFSET, op_size));
8180 gen_insn(ARCH_PREFERS_SX(op_size) ? INSN_MOVSX : INSN_MOV, op_size, 0, 0);
8181 gen_one(R_SCRATCH_2);
8182 gen_address_offset();
8183
8184 g(gen_imm(ctx, opt, IMM_PURPOSE_CMP, op_size));
8185 gen_insn(INSN_CMP, op_size, 0, 1);
8186 gen_one(R_SCRATCH_2);
8187 gen_imm_offset();
8188 #endif
8189 if (label) {
8190 gen_insn(INSN_JMP_COND, op_size, COND_NE, 0);
8191 gen_four(label);
8192 } else {
8193 g(gen_frame_set_cond(ctx, op_size, false, COND_E, slot_r));
8194 }
8195 return true;
8196 #else
8197 g(gen_address(ctx, reg, offsetof(struct data, u_.option.option), ARCH_PREFERS_SX(op_size) ? IMM_PURPOSE_LDR_SX_OFFSET : IMM_PURPOSE_LDR_OFFSET, op_size));
8198 gen_insn(ARCH_PREFERS_SX(op_size) ? INSN_MOVSX : INSN_MOV, op_size, 0, 0);
8199 gen_one(R_SCRATCH_2);
8200 gen_address_offset();
8201
8202 g(gen_cmp_dest_reg(ctx, op_size, R_SCRATCH_2, (unsigned)-1, label ? R_CMP_RESULT : R_SCRATCH_2, opt, COND_E));
8203
8204 if (label) {
8205 gen_insn(INSN_JMP_REG, i_size(op_size), COND_E, 0);
8206 gen_one(R_CMP_RESULT);
8207 gen_four(label);
8208 } else {
8209 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, R_SCRATCH_2));
8210 }
8211 return true;
8212 #endif
8213 }
8214
8215 static bool attr_w gen_option_load(struct codegen_context *ctx, frame_t slot_1, frame_t slot_r, ajla_option_t opt, frame_t flags)
8216 {
8217 const struct type *type;
8218 uint32_t escape_label;
8219
8220 escape_label = alloc_escape_label(ctx);
8221 if (unlikely(!escape_label))
8222 return false;
8223
8224 if (flag_must_be_flat(ctx, slot_r)) {
8225 gen_insn(INSN_JMP, 0, 0, 0);
8226 gen_four(escape_label);
8227 return true;
8228 }
8229
8230 type = get_type_of_local(ctx, slot_1);
8231 if (TYPE_IS_FLAT(type)) {
8232 g(gen_test_1_jz_cached(ctx, slot_1, escape_label));
8233 }
8234
8235 g(gen_frame_load(ctx, OP_SIZE_SLOT, false, slot_1, 0, R_SCRATCH_1));
8236 g(gen_ptr_is_thunk(ctx, R_SCRATCH_1, true, escape_label));
8237 g(gen_barrier(ctx));
8238 g(gen_decompress_pointer(ctx, R_SCRATCH_1, 0));
8239 g(gen_option_cmp(ctx, R_SCRATCH_1, opt, escape_label, 0));
8240
8241 g(gen_address(ctx, R_SCRATCH_1, offsetof(struct data, u_.option.pointer), ARCH_PREFERS_SX(OP_SIZE_SLOT) ? IMM_PURPOSE_LDR_SX_OFFSET : IMM_PURPOSE_LDR_OFFSET, OP_SIZE_SLOT));
8242 gen_insn(ARCH_PREFERS_SX(OP_SIZE_SLOT) ? INSN_MOVSX : INSN_MOV, OP_SIZE_SLOT, 0, 0);
8243 gen_one(R_SCRATCH_1);
8244 gen_address_offset();
8245
8246 g(gen_ptr_is_thunk(ctx, R_SCRATCH_1, true, escape_label));
8247
8248 if (flags & OPCODE_STRUCT_MAY_BORROW) {
8249 g(gen_frame_store(ctx, OP_SIZE_SLOT, slot_r, 0, R_SCRATCH_1));
8250 flag_set(ctx, slot_r, false);
8251 } else {
8252 g(gen_frame_set_pointer(ctx, slot_r, R_SCRATCH_1));
8253
8254 g(gen_upcall_start(ctx, 1));
8255 gen_insn(INSN_MOV, i_size(OP_SIZE_ADDRESS), 0, 0);
8256 gen_one(R_ARG0);
8257 gen_one(R_SCRATCH_1);
8258 g(gen_upcall_argument(ctx, 0));
8259 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_pointer_reference_owned), 1));
8260 }
8261
8262 return true;
8263 }
8264
8265 static bool attr_w gen_option_test_flat(struct codegen_context *ctx, frame_t slot_1, frame_t opt, frame_t slot_r)
8266 {
8267 unsigned op_size = log_2(sizeof(ajla_flat_option_t));
8268 uint32_t escape_label;
8269
8270 escape_label = alloc_escape_label(ctx);
8271 if (unlikely(!escape_label))
8272 return false;
8273
8274 g(gen_test_1_cached(ctx, slot_1, escape_label));
8275
8276 flag_set(ctx, slot_1, false);
8277 flag_set(ctx, slot_r, false);
8278
8279 if (unlikely(opt != (ajla_flat_option_t)opt)) {
8280 g(gen_frame_clear(ctx, op_size, slot_r));
8281 return true;
8282 }
8283
8284 g(gen_frame_load_cmp_imm_set_cond(ctx, op_size, false, slot_1, 0, opt, COND_E, slot_r));
8285
8286 return true;
8287 }
8288
8289 static bool attr_w gen_option_test(struct codegen_context *ctx, frame_t slot_1, frame_t opt, frame_t slot_r)
8290 {
8291 uint32_t escape_label;
8292
8293 escape_label = alloc_escape_label(ctx);
8294 if (unlikely(!escape_label))
8295 return false;
8296
8297 g(gen_frame_load(ctx, OP_SIZE_SLOT, false, slot_1, 0, R_SCRATCH_1));
8298 g(gen_ptr_is_thunk(ctx, R_SCRATCH_1, true, escape_label));
8299 g(gen_barrier(ctx));
8300
8301 flag_set(ctx, slot_r, false);
8302
8303 if (unlikely(opt != (ajla_option_t)opt)) {
8304 g(gen_frame_clear(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r));
8305 return true;
8306 }
8307
8308 g(gen_decompress_pointer(ctx, R_SCRATCH_1, 0));
8309 g(gen_option_cmp(ctx, R_SCRATCH_1, opt, 0, slot_r));
8310
8311 return true;
8312 }
8313
8314 static bool attr_w gen_option_ord(struct codegen_context *ctx, frame_t slot_1, frame_t slot_r, bool flat)
8315 {
8316 unsigned op_size = log_2(sizeof(ajla_option_t));
8317 unsigned op_size_flat = log_2(sizeof(ajla_flat_option_t));
8318 uint32_t escape_label, ptr_label, store_label;
8319
8320 escape_label = alloc_escape_label(ctx);
8321 if (unlikely(!escape_label))
8322 return false;
8323
8324 ptr_label = alloc_label(ctx);
8325 if (unlikely(!ptr_label))
8326 return false;
8327
8328 store_label = alloc_label(ctx);
8329 if (unlikely(!store_label))
8330 return false;
8331
8332 if (flat) {
8333 g(gen_test_1_cached(ctx, slot_1, ptr_label));
8334
8335 g(gen_frame_load(ctx, op_size_flat, false, slot_1, 0, R_SCRATCH_1));
8336
8337 if (flag_is_clear(ctx, slot_1))
8338 goto skip_ptr_label;
8339
8340 gen_insn(INSN_JMP, 0, 0, 0);
8341 gen_four(store_label);
8342 }
8343
8344 gen_label(ptr_label);
8345 g(gen_frame_load(ctx, OP_SIZE_SLOT, false, slot_1, 0, R_SCRATCH_1));
8346 g(gen_ptr_is_thunk(ctx, R_SCRATCH_1, true, escape_label));
8347 g(gen_barrier(ctx));
8348
8349 g(gen_decompress_pointer(ctx, R_SCRATCH_1, 0));
8350
8351 g(gen_address(ctx, R_SCRATCH_1, offsetof(struct data, u_.option.option), ARCH_PREFERS_SX(op_size) ? IMM_PURPOSE_LDR_SX_OFFSET : IMM_PURPOSE_LDR_OFFSET, op_size));
8352 gen_insn(ARCH_PREFERS_SX(op_size) ? INSN_MOVSX : INSN_MOV, op_size, 0, 0);
8353 gen_one(R_SCRATCH_1);
8354 gen_address_offset();
8355
8356 skip_ptr_label:
8357 gen_label(store_label);
8358 g(gen_frame_store(ctx, OP_SIZE_INT, slot_r, 0, R_SCRATCH_1));
8359 flag_set(ctx, slot_r, false);
8360
8361 return true;
8362 }
8363
8364 static bool attr_w gen_array_create(struct codegen_context *ctx, frame_t slot_r)
8365 {
8366 size_t i;
8367 const struct type *type;
8368 uint32_t escape_label;
8369
8370 escape_label = alloc_escape_label(ctx);
8371 if (unlikely(!escape_label))
8372 return false;
8373
8374 ajla_assert_lo(ctx->args_l != 0, (file_line, "gen_array_create: zero entries"));
8375
8376 if (unlikely(ctx->args_l >= sign_bit(uint_default_t))) {
8377 gen_insn(INSN_JMP, 0, 0, 0);
8378 gen_four(escape_label);
8379 return true;
8380 }
8381
8382 type = get_type_of_local(ctx, ctx->args[0].slot);
8383 for (i = 1; i < ctx->args_l; i++) {
8384 const struct type *t = get_type_of_local(ctx, ctx->args[i].slot);
8385 if (unlikely(t != type))
8386 internal(file_line, "gen_array_create: types do not match: %u != %u", type->tag, t->tag);
8387 }
8388
8389 if (TYPE_IS_FLAT(type)) {
8390 int64_t offset;
8391 for (i = 0; i < ctx->args_l; i++) {
8392 g(gen_test_1_cached(ctx, ctx->args[i].slot, escape_label));
8393 flag_set(ctx, ctx->args[i].slot, false);
8394 }
8395
8396 g(gen_upcall_start(ctx, 3));
8397
8398 gen_insn(INSN_MOV, i_size(OP_SIZE_ADDRESS), 0, 0);
8399 gen_one(R_ARG0);
8400 gen_one(R_FRAME);
8401 g(gen_upcall_argument(ctx, 0));
8402
8403 g(gen_load_constant(ctx, R_ARG1, ctx->args[0].slot));
8404 g(gen_upcall_argument(ctx, 1));
8405
8406 g(gen_load_constant(ctx, R_ARG2, ctx->args_l));
8407 g(gen_upcall_argument(ctx, 2));
8408
8409 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_data_alloc_array_flat_slot_mayfail), 3));
8410 g(gen_sanitize_returned_pointer(ctx, R_RET0));
8411 g(gen_jmp_on_zero(ctx, OP_SIZE_ADDRESS, R_RET0, COND_E, escape_label));
8412
8413 gen_insn(INSN_MOV, i_size(OP_SIZE_ADDRESS), 0, 0);
8414 gen_one(R_SAVED_1);
8415 gen_one(R_RET0);
8416
8417 offset = data_array_offset;
8418 for (i = 0; i < ctx->args_l; i++) {
8419 g(gen_memcpy_from_slot(ctx, R_SAVED_1, offset, ctx->args[i].slot));
8420 offset += type->size;
8421 }
8422 } else {
8423 int64_t offset;
8424 g(gen_upcall_start(ctx, 2));
8425
8426 g(gen_load_constant(ctx, R_ARG0, ctx->args_l));
8427 g(gen_upcall_argument(ctx, 0));
8428
8429 g(gen_load_constant(ctx, R_ARG1, ctx->args_l));
8430 g(gen_upcall_argument(ctx, 1));
8431
8432 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_data_alloc_array_pointers_mayfail), 2));
8433 g(gen_sanitize_returned_pointer(ctx, R_RET0));
8434 g(gen_jmp_on_zero(ctx, OP_SIZE_ADDRESS, R_RET0, COND_E, escape_label));
8435
8436 gen_insn(INSN_MOV, i_size(OP_SIZE_ADDRESS), 0, 0);
8437 gen_one(R_SAVED_1);
8438 gen_one(R_RET0);
8439
8440 g(gen_address(ctx, R_RET0, offsetof(struct data, u_.array_pointers.pointer), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_ADDRESS));
8441 gen_insn(INSN_MOV, OP_SIZE_ADDRESS, 0, 0);
8442 gen_one(R_SAVED_2);
8443 gen_address_offset();
8444
8445 offset = 0;
8446 for (i = 0; i < ctx->args_l; i++) {
8447 g(gen_frame_get_pointer(ctx, ctx->args[i].slot, (ctx->args[i].flags & OPCODE_FLAG_FREE_ARGUMENT) != 0, R_SCRATCH_1));
8448 g(gen_address(ctx, R_SAVED_2, offset, IMM_PURPOSE_STR_OFFSET, OP_SIZE_SLOT));
8449 gen_insn(INSN_MOV, OP_SIZE_SLOT, 0, 0);
8450 gen_address_offset();
8451 gen_one(R_SCRATCH_1);
8452 offset += sizeof(pointer_t);
8453 }
8454 }
8455 g(gen_compress_pointer(ctx, R_SAVED_1));
8456 g(gen_frame_set_pointer(ctx, slot_r, R_SAVED_1));
8457 return true;
8458 }
8459
8460 static bool attr_w gen_array_create_empty_flat(struct codegen_context *ctx, frame_t slot_r, frame_t local_type)
8461 {
8462 uint32_t escape_label;
8463
8464 escape_label = alloc_escape_label(ctx);
8465 if (unlikely(!escape_label))
8466 return false;
8467
8468 g(gen_upcall_start(ctx, 3));
8469
8470 gen_insn(INSN_MOV, i_size(OP_SIZE_ADDRESS), 0, 0);
8471 gen_one(R_ARG0);
8472 gen_one(R_FRAME);
8473 g(gen_upcall_argument(ctx, 0));
8474
8475 g(gen_load_constant(ctx, R_ARG1, local_type));
8476 g(gen_upcall_argument(ctx, 1));
8477
8478 g(gen_load_constant(ctx, R_ARG2, 0));
8479 g(gen_upcall_argument(ctx, 2));
8480
8481 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_data_alloc_array_flat_types_ptr_mayfail), 3));
8482 g(gen_sanitize_returned_pointer(ctx, R_RET0));
8483 g(gen_jmp_on_zero(ctx, OP_SIZE_ADDRESS, R_RET0, COND_E, escape_label));
8484
8485 g(gen_compress_pointer(ctx, R_RET0));
8486 g(gen_frame_set_pointer(ctx, slot_r, R_RET0));
8487
8488 return true;
8489 }
8490
8491 static bool attr_w gen_array_create_empty(struct codegen_context *ctx, frame_t slot_r)
8492 {
8493 uint32_t escape_label;
8494
8495 escape_label = alloc_escape_label(ctx);
8496 if (unlikely(!escape_label))
8497 return false;
8498
8499 g(gen_upcall_start(ctx, 2));
8500
8501 g(gen_load_constant(ctx, R_ARG0, 0));
8502 g(gen_upcall_argument(ctx, 0));
8503
8504 g(gen_load_constant(ctx, R_ARG1, 0));
8505 g(gen_upcall_argument(ctx, 1));
8506
8507 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_data_alloc_array_pointers_mayfail), 2));
8508 g(gen_sanitize_returned_pointer(ctx, R_RET0));
8509 g(gen_jmp_on_zero(ctx, OP_SIZE_ADDRESS, R_RET0, COND_E, escape_label));
8510
8511 g(gen_compress_pointer(ctx, R_RET0));
8512 g(gen_frame_set_pointer(ctx, slot_r, R_RET0));
8513
8514 return true;
8515 }
8516
8517 static bool attr_w gen_array_fill(struct codegen_context *ctx, frame_t slot_1, frame_t flags, frame_t slot_2, frame_t slot_r)
8518 {
8519 const struct type *content_type, *array_type;
8520 uint32_t escape_label;
8521
8522 escape_label = alloc_escape_label(ctx);
8523 if (unlikely(!escape_label))
8524 return false;
8525
8526 g(gen_test_1_cached(ctx, slot_2, escape_label));
8527
8528 content_type = get_type_of_local(ctx, slot_1);
8529 array_type = get_type_of_local(ctx, slot_r);
8530
8531 if (TYPE_IS_FLAT(array_type)) {
8532 int64_t dest_offset;
8533 size_t i;
8534 const struct flat_array_definition *def = type_def(array_type,flat_array);
8535
8536 ajla_assert_lo(TYPE_IS_FLAT(content_type), (file_line, "gen_array_fill: array is flat but content is not"));
8537
8538 g(gen_test_1_cached(ctx, slot_1, escape_label));
8539
8540 dest_offset = (size_t)slot_r * slot_size;
8541 for (i = 0; i < def->n_elements; i++) {
8542 g(gen_memcpy_from_slot(ctx, R_FRAME, dest_offset, slot_1));
8543 dest_offset += def->base->size;
8544 }
8545 flag_set(ctx, slot_1, false);
8546 flag_set(ctx, slot_r, false);
8547
8548 return true;
8549 }
8550
8551 if (ctx->registers[slot_1] >= 0)
8552 g(spill(ctx, slot_1));
8553
8554 if (unlikely((flags & OPCODE_ARRAY_FILL_FLAG_SPARSE) != 0)) {
8555 uint32_t get_ptr_label, got_ptr_label;
8556
8557 get_ptr_label = alloc_label(ctx);
8558 if (unlikely(!get_ptr_label))
8559 return false;
8560
8561 got_ptr_label = alloc_label(ctx);
8562 if (unlikely(!got_ptr_label))
8563 return false;
8564
8565 if (TYPE_IS_FLAT(content_type)) {
8566 g(gen_test_1_cached(ctx, slot_1, get_ptr_label));
8567
8568 g(gen_upcall_start(ctx, 3));
8569
8570 gen_insn(INSN_MOV, i_size(OP_SIZE_ADDRESS), 0, 0);
8571 gen_one(R_ARG0);
8572 gen_one(R_FRAME);
8573 g(gen_upcall_argument(ctx, 0));
8574
8575 g(gen_load_constant(ctx, R_ARG1, slot_1));
8576 g(gen_upcall_argument(ctx, 1));
8577
8578 g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_ADDRESS), ALU_ADD, R_ARG2, R_FRAME, (size_t)slot_1 * slot_size));
8579 g(gen_upcall_argument(ctx, 2));
8580
8581 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_flat_to_data), 3));
8582
8583 gen_insn(ARCH_PREFERS_SX(i_size(OP_SIZE_SLOT)) ? INSN_MOVSX : INSN_MOV, i_size(OP_SIZE_SLOT), 0, 0);
8584 gen_one(R_SCRATCH_2);
8585 gen_one(R_RET0);
8586 g(gen_upcall_argument(ctx, 1));
8587
8588 gen_insn(INSN_JMP, 0, 0, 0);
8589 gen_four(got_ptr_label);
8590 }
8591
8592 gen_label(get_ptr_label);
8593
8594 g(gen_frame_get_pointer(ctx, slot_1, (flags & OPCODE_FLAG_FREE_ARGUMENT) != 0, R_SCRATCH_2));
8595 g(gen_upcall_argument(ctx, 1));
8596
8597 gen_label(got_ptr_label);
8598
8599 g(gen_frame_load(ctx, OP_SIZE_INT, true, slot_2, 0, R_SCRATCH_1));
8600 g(gen_jmp_if_negative(ctx, R_SCRATCH_1, escape_label));
8601
8602 g(gen_upcall_start(ctx, 2));
8603 gen_insn(INSN_MOV, i_size(OP_SIZE_ADDRESS), 0, 0);
8604 gen_one(R_ARG0);
8605 gen_one(R_SCRATCH_1);
8606 g(gen_upcall_argument(ctx, 0));
8607
8608 gen_insn(INSN_MOV, i_size(OP_SIZE_ADDRESS), 0, 0);
8609 gen_one(R_ARG1);
8610 gen_one(R_SCRATCH_2);
8611 g(gen_upcall_argument(ctx, 1));
8612
8613 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_array_create_sparse), 2));
8614 } else if (TYPE_IS_FLAT(content_type)) {
8615 g(gen_test_1_cached(ctx, slot_1, escape_label));
8616 flag_set(ctx, slot_1, false);
8617
8618 g(gen_frame_load(ctx, OP_SIZE_INT, true, slot_2, 0, R_SCRATCH_1));
8619 g(gen_jmp_if_negative(ctx, R_SCRATCH_1, escape_label));
8620
8621 g(gen_upcall_start(ctx, 3));
8622 gen_insn(INSN_MOV, i_size(OP_SIZE_ADDRESS), 0, 0);
8623 gen_one(R_ARG0);
8624 gen_one(R_FRAME);
8625 g(gen_upcall_argument(ctx, 0));
8626
8627 gen_insn(INSN_MOV, i_size(OP_SIZE_INT), 0, 0);
8628 gen_one(R_ARG1);
8629 gen_one(R_SCRATCH_1);
8630 g(gen_upcall_argument(ctx, 1));
8631
8632 g(gen_load_constant(ctx, R_ARG2, slot_1));
8633 g(gen_upcall_argument(ctx, 2));
8634
8635 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_array_create_flat), 3));
8636 } else {
8637 g(gen_frame_get_pointer(ctx, slot_1, (flags & OPCODE_FLAG_FREE_ARGUMENT) != 0, R_SCRATCH_1));
8638
8639 g(gen_upcall_start(ctx, 4));
8640
8641 gen_insn(INSN_MOV, i_size(OP_SIZE_ADDRESS), 0, 0);
8642 gen_one(R_ARG3);
8643 gen_one(R_SCRATCH_1);
8644 g(gen_upcall_argument(ctx, 3));
8645
8646 gen_insn(INSN_MOV, i_size(OP_SIZE_ADDRESS), 0, 0);
8647 gen_one(R_ARG0);
8648 gen_one(R_FRAME);
8649 g(gen_upcall_argument(ctx, 0));
8650
8651 g(gen_load_constant(ctx, R_ARG1, ctx->instr_start - da(ctx->fn,function)->code));
8652 g(gen_upcall_argument(ctx, 1));
8653
8654 g(gen_load_constant(ctx, R_ARG2, slot_2));
8655 g(gen_upcall_argument(ctx, 2));
8656
8657 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_array_create_pointers), 4));
8658 }
8659 g(gen_frame_set_pointer(ctx, slot_r, R_RET0));
8660
8661 return true;
8662 }
8663
8664 static bool attr_w gen_array_string(struct codegen_context *ctx, type_tag_t tag, uint8_t *string, frame_t len, frame_t slot_r)
8665 {
8666 uint32_t escape_label;
8667 int64_t offset;
8668 const struct type *type;
8669
8670 escape_label = alloc_escape_label(ctx);
8671 if (unlikely(!escape_label))
8672 return false;
8673
8674 g(gen_upcall_start(ctx, 2));
8675
8676 g(gen_load_constant(ctx, R_ARG0, tag));
8677 g(gen_upcall_argument(ctx, 0));
8678
8679 g(gen_load_constant(ctx, R_ARG1, len));
8680 g(gen_upcall_argument(ctx, 1));
8681
8682 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_data_alloc_array_flat_tag_mayfail), 2));
8683 g(gen_sanitize_returned_pointer(ctx, R_RET0));
8684 g(gen_jmp_on_zero(ctx, OP_SIZE_ADDRESS, R_RET0, COND_E, escape_label));
8685
8686 gen_insn(INSN_MOV, i_size(OP_SIZE_ADDRESS), 0, 0);
8687 gen_one(R_SAVED_1);
8688 gen_one(R_RET0);
8689
8690 g(gen_compress_pointer(ctx, R_RET0));
8691 g(gen_frame_set_pointer(ctx, slot_r, R_RET0));
8692
8693 g(load_function_offset(ctx, R_SCRATCH_3, offsetof(struct data, u_.function.code)));
8694
8695 offset = string - cast_ptr(uint8_t *, da(ctx->fn,function)->code);
8696 type = type_get_from_tag(tag);
8697 g(gen_memcpy_raw(ctx, R_SAVED_1, data_array_offset, R_SCRATCH_3, offset, (size_t)len * type->size, minimum(type->align, align_of(code_t))));
8698
8699 return true;
8700 }
8701
8702 static bool attr_w gen_scaled_array_address(struct codegen_context *ctx, size_t element_size, unsigned reg_dst, unsigned reg_src, unsigned reg_index, int64_t offset_src)
8703 {
8704 if (is_power_of_2(element_size)) {
8705 unsigned shift = log_2(element_size);
8706 #if defined(ARCH_X86)
8707 if (shift <= 3 && imm_is_32bit(offset_src)) {
8708 gen_insn(INSN_LEA3, i_size(OP_SIZE_ADDRESS), shift, 0);
8709 gen_one(reg_dst);
8710 gen_one(reg_src);
8711 gen_one(reg_index);
8712 gen_one(ARG_IMM);
8713 gen_eight(offset_src);
8714 return true;
8715 }
8716 #endif
8717 if (ARCH_HAS_SHIFTED_ADD(shift)) {
8718 gen_insn(INSN_ALU, i_size(OP_SIZE_ADDRESS), ALU_ADD, ALU_WRITES_FLAGS(ALU_ADD, false));
8719 gen_one(reg_dst);
8720 gen_one(reg_src);
8721 gen_one(ARG_SHIFTED_REGISTER);
8722 gen_one(ARG_SHIFT_LSL | shift);
8723 gen_one(reg_index);
8724
8725 goto add_offset;
8726 }
8727
8728 if (shift) {
8729 gen_insn(INSN_ROT + ARCH_PARTIAL_ALU(OP_SIZE_ADDRESS), OP_SIZE_ADDRESS, ROT_SHL, ROT_WRITES_FLAGS(ROT_SHL));
8730 gen_one(reg_index);
8731 gen_one(reg_index);
8732 gen_one(ARG_IMM);
8733 gen_eight(shift);
8734 }
8735 } else {
8736 if (ARCH_HAS_MUL) {
8737 g(gen_imm(ctx, element_size, IMM_PURPOSE_MUL, i_size(OP_SIZE_ADDRESS)));
8738 gen_insn(INSN_ALU, i_size(OP_SIZE_ADDRESS), ALU_MUL, ALU_WRITES_FLAGS(ALU_MUL, is_imm()));
8739 gen_one(reg_index);
8740 gen_one(reg_index);
8741 gen_imm_offset();
8742 } else {
8743 size_t e_size = element_size;
8744 unsigned sh = 0;
8745 bool first_match = true;
8746
8747 gen_insn(INSN_MOV, i_size(OP_SIZE_ADDRESS), 0, 0);
8748 gen_one(R_CONST_IMM);
8749 gen_one(reg_index);
8750
8751 if (!e_size)
8752 g(gen_load_constant(ctx, reg_index, 0));
8753
8754 while (e_size) {
8755 if (e_size & 1) {
8756 if (first_match) {
8757 if (sh)
8758 g(gen_3address_rot_imm(ctx, OP_SIZE_ADDRESS, ROT_SHL, reg_index, reg_index, sh, false));
8759 first_match = false;
8760 } else if (ARCH_HAS_SHIFTED_ADD(sh)) {
8761 gen_insn(INSN_ALU, i_size(OP_SIZE_ADDRESS), ALU_ADD, ALU_WRITES_FLAGS(ALU_ADD, false));
8762 gen_one(reg_index);
8763 gen_one(reg_index);
8764 gen_one(ARG_SHIFTED_REGISTER);
8765 gen_one(ARG_SHIFT_LSL | sh);
8766 gen_one(R_CONST_IMM);
8767 } else {
8768 if (sh) {
8769 g(gen_3address_rot_imm(ctx, OP_SIZE_ADDRESS, ROT_SHL, R_CONST_IMM, R_CONST_IMM, sh, false));
8770 sh = 0;
8771 }
8772 g(gen_3address_alu(ctx, i_size(OP_SIZE_ADDRESS), ALU_ADD, reg_index, reg_index, R_CONST_IMM));
8773 }
8774 }
8775 sh += 1;
8776 e_size >>= 1;
8777 }
8778 }
8779 }
8780 #if defined(ARCH_S390)
8781 if (offset_src && s390_inline_address(offset_src)) {
8782 gen_insn(INSN_LEA3, i_size(OP_SIZE_ADDRESS), 0, 0);
8783 gen_one(reg_dst);
8784 gen_one(reg_index);
8785 gen_one(reg_src);
8786 gen_one(ARG_IMM);
8787 gen_eight(offset_src);
8788 return true;
8789 }
8790 #endif
8791 g(gen_3address_alu(ctx, i_size(OP_SIZE_ADDRESS), ALU_ADD, reg_dst, reg_index, reg_src));
8792 goto add_offset;
8793
8794 add_offset:
8795 if (offset_src) {
8796 g(gen_imm(ctx, offset_src, IMM_PURPOSE_ADD, i_size(OP_SIZE_ADDRESS)));
8797 gen_insn(INSN_ALU, i_size(OP_SIZE_ADDRESS), ALU_ADD, ALU_WRITES_FLAGS(ALU_ADD, is_imm()));
8798 gen_one(reg_dst);
8799 gen_one(reg_dst);
8800 gen_imm_offset();
8801 }
8802 return true;
8803 }
8804
8805 static bool attr_w gen_scaled_array_load(struct codegen_context *ctx, unsigned reg_src, int64_t offset_src, frame_t slot_r)
8806 {
8807 const struct type *t = get_type_of_local(ctx, slot_r);
8808 #if defined(ARCH_X86)
8809 if (is_power_of_2(t->size)) {
8810 unsigned shift = log_2(t->size);
8811 if (shift <= 3 && shift <= OP_SIZE_NATIVE && imm_is_32bit(offset_src)) {
8812 short reg = ctx->registers[slot_r];
8813 gen_insn(INSN_MOV, shift, 0, 0);
8814 gen_one(reg >= 0 ? reg : R_SCRATCH_2);
8815 gen_one(ARG_ADDRESS_2 + shift);
8816 gen_one(reg_src);
8817 gen_one(R_SCRATCH_2);
8818 gen_eight(offset_src);
8819
8820 if (reg < 0) {
8821 g(gen_address(ctx, R_FRAME, (size_t)slot_r * slot_size, IMM_PURPOSE_STR_OFFSET, shift));
8822 gen_insn(INSN_MOV, shift, 0, 0);
8823 gen_address_offset();
8824 gen_one(R_SCRATCH_2);
8825 }
8826
8827 return true;
8828 }
8829 }
8830 #endif
8831 #if defined(ARCH_S390)
8832 if (t->size == 1 && s390_inline_address(offset_src) && cpu_test_feature(CPU_FEATURE_extended_imm)) {
8833 short reg = ctx->registers[slot_r];
8834 gen_insn(INSN_MOVSX, OP_SIZE_1, 0, 0);
8835 gen_one(reg >= 0 ? reg : R_SCRATCH_2);
8836 gen_one(ARG_ADDRESS_2);
8837 gen_one(reg_src);
8838 gen_one(R_SCRATCH_2);
8839 gen_eight(offset_src);
8840
8841 if (reg < 0) {
8842 g(gen_address(ctx, R_FRAME, (size_t)slot_r * slot_size, IMM_PURPOSE_STR_OFFSET, OP_SIZE_1));
8843 gen_insn(INSN_MOV, OP_SIZE_1, 0, 0);
8844 gen_address_offset();
8845 gen_one(R_SCRATCH_2);
8846 }
8847
8848 return true;
8849 }
8850 #endif
8851 g(gen_scaled_array_address(ctx, t->size, R_SCRATCH_2, reg_src, R_SCRATCH_2, 0));
8852
8853 g(gen_memcpy_to_slot(ctx, slot_r, R_SCRATCH_2, offset_src));
8854
8855 return true;
8856 }
8857
8858 static bool attr_w gen_scaled_array_store(struct codegen_context *ctx, unsigned reg_src, int64_t offset_src, frame_t slot_1)
8859 {
8860 const struct type *t = get_type_of_local(ctx, slot_1);
8861 #if defined(ARCH_X86)
8862 if (is_power_of_2(t->size)) {
8863 unsigned shift = log_2(t->size);
8864 if (shift <= 3 && shift <= OP_SIZE_NATIVE && imm_is_32bit(offset_src)) {
8865 short reg = ctx->registers[slot_1];
8866 if (reg < 0) {
8867 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size, IMM_PURPOSE_LDR_OFFSET, shift));
8868 gen_insn(INSN_MOV, shift, 0, 0);
8869 gen_one(R_SCRATCH_3);
8870 gen_address_offset();
8871 reg = R_SCRATCH_3;
8872 }
8873
8874 gen_insn(INSN_MOV, shift, 0, 0);
8875 gen_one(ARG_ADDRESS_2 + shift);
8876 gen_one(reg_src);
8877 gen_one(R_SCRATCH_2);
8878 gen_eight(offset_src);
8879 gen_one(reg);
8880
8881 return true;
8882 }
8883 }
8884 #endif
8885 #if defined(ARCH_S390)
8886 if (t->size == 1 && s390_inline_address(offset_src) && cpu_test_feature(CPU_FEATURE_extended_imm)) {
8887 short reg = ctx->registers[slot_1];
8888 if (reg < 0) {
8889 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size, IMM_PURPOSE_LDR_SX_OFFSET, OP_SIZE_1));
8890 gen_insn(INSN_MOVSX, OP_SIZE_1, 0, 0);
8891 gen_one(R_SCRATCH_3);
8892 gen_address_offset();
8893 reg = R_SCRATCH_3;
8894 }
8895
8896 gen_insn(INSN_MOV, OP_SIZE_1, 0, 0);
8897 gen_one(ARG_ADDRESS_2);
8898 gen_one(reg_src);
8899 gen_one(R_SCRATCH_2);
8900 gen_eight(offset_src);
8901 gen_one(reg);
8902
8903 return true;
8904 }
8905 #endif
8906 g(gen_scaled_array_address(ctx, t->size, R_SCRATCH_2, reg_src, R_SCRATCH_2, 0));
8907
8908 g(gen_memcpy_from_slot(ctx, R_SCRATCH_2, offset_src, slot_1));
8909
8910 return true;
8911 }
8912
8913 static bool attr_w gen_check_array_len(struct codegen_context *ctx, unsigned reg_array, bool allocated, unsigned reg_len, unsigned cond, uint32_t escape_label)
8914 {
8915 size_t offset = !allocated ? offsetof(struct data, u_.array_flat.n_used_entries) : offsetof(struct data, u_.array_flat.n_allocated_entries);
8916 #if defined(ARCH_X86)
8917 g(gen_address(ctx, reg_array, offset, IMM_PURPOSE_LDR_OFFSET, OP_SIZE_INT));
8918 gen_insn(INSN_CMP, OP_SIZE_INT, 0, 1);
8919 gen_one(reg_len);
8920 gen_address_offset();
8921
8922 gen_insn(INSN_JMP_COND, OP_SIZE_INT, cond, 0);
8923 gen_four(escape_label);
8924 #else
8925 g(gen_address(ctx, reg_array, offset, ARCH_PREFERS_SX(OP_SIZE_INT) ? IMM_PURPOSE_LDR_SX_OFFSET : IMM_PURPOSE_LDR_OFFSET, OP_SIZE_INT));
8926 gen_insn(ARCH_PREFERS_SX(OP_SIZE_INT) ? INSN_MOVSX : INSN_MOV, OP_SIZE_INT, 0, 0);
8927 gen_one(R_SCRATCH_3);
8928 gen_address_offset();
8929
8930 g(gen_cmp_test_jmp(ctx, INSN_CMP, i_size(OP_SIZE_INT), reg_len, R_SCRATCH_3, cond, escape_label));
8931 #endif
8932 return true;
8933 }
8934
8935 static bool attr_w gen_array_load(struct codegen_context *ctx, frame_t slot_1, frame_t slot_idx, frame_t slot_r, frame_t flags)
8936 {
8937 const struct type *t = get_type_of_local(ctx, slot_1);
8938 const struct type *tr = get_type_of_local(ctx, slot_r);
8939 uint32_t escape_label;
8940
8941 escape_label = alloc_escape_label(ctx);
8942 if (unlikely(!escape_label))
8943 return false;
8944
8945 if (unlikely(t->tag == TYPE_TAG_flat_array)) {
8946 const struct flat_array_definition *def = type_def(t,flat_array);
8947
8948 g(gen_test_2_cached(ctx, slot_1, slot_idx, escape_label));
8949
8950 flag_set(ctx, slot_1, false);
8951 flag_set(ctx, slot_idx, false);
8952
8953 g(gen_frame_load(ctx, OP_SIZE_INT, false, slot_idx, 0, R_SCRATCH_2));
8954
8955 if (!(flags & OPCODE_ARRAY_INDEX_IN_RANGE))
8956 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, OP_SIZE_INT, R_SCRATCH_2, def->n_elements, COND_AE, escape_label));
8957
8958 g(gen_scaled_array_load(ctx, R_FRAME, (size_t)slot_1 * slot_size, slot_r));
8959 return true;
8960 }
8961
8962 g(gen_frame_load(ctx, OP_SIZE_SLOT, false, slot_1, 0, R_SCRATCH_1));
8963 g(gen_ptr_is_thunk(ctx, R_SCRATCH_1, true, escape_label));
8964 g(gen_barrier(ctx));
8965 g(gen_decompress_pointer(ctx, R_SCRATCH_1, 0));
8966
8967 g(gen_test_1_cached(ctx, slot_idx, escape_label));
8968 flag_set(ctx, slot_idx, false);
8969 g(gen_frame_load(ctx, OP_SIZE_INT, false, slot_idx, 0, R_SCRATCH_2));
8970
8971 if (!(flags & OPCODE_ARRAY_INDEX_IN_RANGE))
8972 g(gen_check_array_len(ctx, R_SCRATCH_1, false, R_SCRATCH_2, COND_AE, escape_label));
8973
8974 if (TYPE_IS_FLAT(tr)) {
8975 uint32_t label;
8976 g(gen_compare_ptr_tag(ctx, R_SCRATCH_1, DATA_TAG_array_slice, COND_A, escape_label, R_SCRATCH_4));
8977 #if defined(ARCH_X86) || defined(ARCH_S390)
8978 #if defined(ARCH_X86)
8979 if (unlikely(!cpu_test_feature(CPU_FEATURE_cmov)))
8980 #else
8981 if (unlikely(!cpu_test_feature(CPU_FEATURE_misc_45)))
8982 #endif
8983 {
8984 g(gen_address(ctx, R_SCRATCH_1, offsetof(struct data, u_.array_slice.flat_data_minus_data_array_offset), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_ADDRESS));
8985 gen_insn(INSN_MOV, OP_SIZE_ADDRESS, 0, 0);
8986 gen_one(R_SCRATCH_3);
8987 gen_address_offset();
8988 goto no_cmov;
8989 }
8990 g(gen_address(ctx, R_SCRATCH_1, offsetof(struct data, u_.array_slice.flat_data_minus_data_array_offset), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_ADDRESS));
8991 gen_insn(INSN_CMOV, OP_SIZE_ADDRESS, COND_E, 0);
8992 gen_one(R_SCRATCH_1);
8993 gen_one(R_SCRATCH_1);
8994 gen_address_offset();
8995 #elif defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS) || defined(ARCH_RISCV64)
8996 g(gen_3address_alu_imm(ctx, OP_SIZE_NATIVE, ALU_XOR, R_SCRATCH_4, R_SCRATCH_4, DATA_TAG_array_slice));
8997
8998 label = alloc_label(ctx);
8999 if (unlikely(!label))
9000 return false;
9001
9002 gen_insn(INSN_JMP_REG, OP_SIZE_NATIVE, COND_NE, 0);
9003 gen_one(R_SCRATCH_4);
9004 gen_four(label);
9005
9006 g(gen_address(ctx, R_SCRATCH_1, offsetof(struct data, u_.array_slice.flat_data_minus_data_array_offset), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_ADDRESS));
9007 gen_insn(INSN_MOV, OP_SIZE_ADDRESS, 0, 0);
9008 gen_one(R_SCRATCH_1);
9009 gen_address_offset();
9010
9011 gen_label(label);
9012 #else
9013 g(gen_address(ctx, R_SCRATCH_1, offsetof(struct data, u_.array_slice.flat_data_minus_data_array_offset), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_ADDRESS));
9014 gen_insn(INSN_MOV, OP_SIZE_ADDRESS, 0, 0);
9015 gen_one(R_SCRATCH_3);
9016 gen_address_offset();
9017 #if ARCH_HAS_FLAGS
9018 #if defined(ARCH_POWER)
9019 if (!cpu_test_feature(CPU_FEATURE_v203))
9020 goto no_cmov;
9021 #endif
9022 #if defined(ARCH_SPARC)
9023 if (!SPARC_9)
9024 goto no_cmov;
9025 #endif
9026 gen_insn(INSN_CMOV, i_size(OP_SIZE_ADDRESS), COND_E, 0);
9027 gen_one(R_SCRATCH_1);
9028 gen_one(R_SCRATCH_1);
9029 gen_one(R_SCRATCH_3);
9030 #else
9031 g(gen_imm(ctx, DATA_TAG_array_slice, IMM_PURPOSE_CMP, OP_SIZE_NATIVE));
9032 gen_insn(INSN_CMP_DEST_REG, OP_SIZE_NATIVE, COND_E, 0);
9033 gen_one(R_CMP_RESULT);
9034 gen_one(R_SCRATCH_4);
9035 gen_imm_offset();
9036
9037 gen_insn(INSN_MOVR, OP_SIZE_NATIVE, COND_NE, 0);
9038 gen_one(R_SCRATCH_1);
9039 gen_one(R_SCRATCH_1);
9040 gen_one(R_CMP_RESULT);
9041 gen_one(R_SCRATCH_3);
9042 #endif
9043 #endif
9044 if (0) {
9045 goto no_cmov;
9046 no_cmov:
9047 label = alloc_label(ctx);
9048 if (unlikely(!label))
9049 return false;
9050 gen_insn(INSN_JMP_COND, OP_SIZE_4, COND_NE, 0);
9051 gen_four(label);
9052
9053 gen_insn(INSN_MOV, i_size(OP_SIZE_ADDRESS), 0, 0);
9054 gen_one(R_SCRATCH_1);
9055 gen_one(R_SCRATCH_3);
9056
9057 gen_label(label);
9058 }
9059 g(gen_scaled_array_load(ctx, R_SCRATCH_1, data_array_offset, slot_r));
9060 flag_set(ctx, slot_r, false);
9061 return true;
9062 } else {
9063 if (flag_must_be_flat(ctx, slot_r)) {
9064 gen_insn(INSN_JMP, 0, 0, 0);
9065 gen_four(escape_label);
9066 return true;
9067 }
9068
9069 g(gen_compare_ptr_tag(ctx, R_SCRATCH_1, DATA_TAG_array_pointers, COND_NE, escape_label, R_SCRATCH_3));
9070
9071 g(gen_address(ctx, R_SCRATCH_1, offsetof(struct data, u_.array_pointers.pointer), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_ADDRESS));
9072 gen_insn(INSN_MOV, OP_SIZE_ADDRESS, 0, 0);
9073 gen_one(R_SCRATCH_1);
9074 gen_address_offset();
9075
9076 #if defined(ARCH_X86) || defined(ARCH_ARM)
9077 gen_insn(INSN_MOV, OP_SIZE_SLOT, 0, 0);
9078 gen_one(R_SCRATCH_1);
9079 gen_one(ARG_ADDRESS_2 + OP_SIZE_SLOT);
9080 gen_one(R_SCRATCH_1);
9081 gen_one(R_SCRATCH_2);
9082 gen_eight(0);
9083
9084 goto scaled_load_done;
9085 #endif
9086 #if defined(ARCH_LOONGARCH64) || defined(ARCH_PARISC) || defined(ARCH_POWER) || defined(ARCH_S390) || defined(ARCH_SPARC)
9087 g(gen_3address_rot_imm(ctx, OP_SIZE_ADDRESS, ROT_SHL, R_SCRATCH_2, R_SCRATCH_2, OP_SIZE_SLOT, false));
9088
9089 gen_insn(INSN_MOV, OP_SIZE_SLOT, 0, 0);
9090 gen_one(R_SCRATCH_1);
9091 gen_one(ARG_ADDRESS_2);
9092 gen_one(R_SCRATCH_1);
9093 gen_one(R_SCRATCH_2);
9094 gen_eight(0);
9095
9096 goto scaled_load_done;
9097 #endif
9098 if (ARCH_HAS_SHIFTED_ADD(OP_SIZE_SLOT)) {
9099 gen_insn(INSN_ALU, i_size(OP_SIZE_ADDRESS), ALU_ADD, ALU_WRITES_FLAGS(ALU_ADD, false));
9100 gen_one(R_SCRATCH_2);
9101 gen_one(ARG_SHIFTED_REGISTER);
9102 gen_one(ARG_SHIFT_LSL | OP_SIZE_SLOT);
9103 gen_one(R_SCRATCH_2);
9104 gen_one(R_SCRATCH_1);
9105
9106 gen_insn(ARCH_PREFERS_SX(OP_SIZE_SLOT) ? INSN_MOVSX : INSN_MOV, OP_SIZE_SLOT, 0, 0);
9107 gen_one(R_SCRATCH_1);
9108 gen_one(ARG_ADDRESS_1);
9109 gen_one(R_SCRATCH_2);
9110 gen_eight(0);
9111
9112 goto scaled_load_done;
9113 }
9114
9115 g(gen_3address_rot_imm(ctx, OP_SIZE_ADDRESS, ROT_SHL, R_SCRATCH_2, R_SCRATCH_2, OP_SIZE_SLOT, false));
9116
9117 g(gen_3address_alu(ctx, OP_SIZE_ADDRESS, ALU_ADD, R_SCRATCH_2, R_SCRATCH_2, R_SCRATCH_1));
9118
9119 gen_insn(ARCH_PREFERS_SX(OP_SIZE_SLOT) ? INSN_MOVSX : INSN_MOV, OP_SIZE_SLOT, 0, 0);
9120 gen_one(R_SCRATCH_1);
9121 gen_one(ARG_ADDRESS_1);
9122 gen_one(R_SCRATCH_2);
9123 gen_eight(0);
9124 scaled_load_done:
9125 g(gen_ptr_is_thunk(ctx, R_SCRATCH_1, true, escape_label));
9126
9127 if (flags & OPCODE_STRUCT_MAY_BORROW) {
9128 g(gen_frame_store(ctx, OP_SIZE_SLOT, slot_r, 0, R_SCRATCH_1));
9129 flag_set(ctx, slot_r, false);
9130 } else {
9131 g(gen_frame_set_pointer(ctx, slot_r, R_SCRATCH_1));
9132
9133 g(gen_upcall_start(ctx, 1));
9134 gen_insn(INSN_MOV, i_size(OP_SIZE_ADDRESS), 0, 0);
9135 gen_one(R_ARG0);
9136 gen_one(R_SCRATCH_1);
9137
9138 g(gen_upcall_argument(ctx, 0));
9139 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_pointer_reference_owned), 1));
9140 }
9141 return true;
9142 }
9143 }
9144
9145 static bool attr_w gen_array_len(struct codegen_context *ctx, frame_t slot_1, frame_t slot_2, frame_t slot_r)
9146 {
9147 const struct type *t = get_type_of_local(ctx, slot_1);
9148 uint32_t escape_label;
9149
9150 escape_label = alloc_escape_label(ctx);
9151 if (unlikely(!escape_label))
9152 return false;
9153
9154 if (slot_2 != NO_FRAME_T) {
9155 g(gen_test_1_cached(ctx, slot_2, escape_label));
9156 flag_set(ctx, slot_2, false);
9157 }
9158
9159 if (unlikely(t->tag == TYPE_TAG_flat_array)) {
9160 if (slot_2 == NO_FRAME_T) {
9161 g(gen_frame_store_imm(ctx, OP_SIZE_INT, slot_r, 0, (unsigned)type_def(t,flat_array)->n_elements));
9162 } else {
9163 g(gen_frame_load_cmp_imm_set_cond(ctx, OP_SIZE_INT, false, slot_2, 0, type_def(t,flat_array)->n_elements, COND_G, slot_r));
9164 }
9165 flag_set(ctx, slot_r, false);
9166 } else {
9167 g(gen_frame_load(ctx, OP_SIZE_SLOT, false, slot_1, 0, R_SCRATCH_1));
9168 g(gen_ptr_is_thunk(ctx, R_SCRATCH_1, true, escape_label));
9169 g(gen_barrier(ctx));
9170
9171 if (offsetof(struct data, u_.array_flat.n_used_entries) != offsetof(struct data, u_.array_slice.n_entries) ||
9172 offsetof(struct data, u_.array_flat.n_used_entries) != offsetof(struct data, u_.array_pointers.n_used_entries)) {
9173 not_reached();
9174 return false;
9175 }
9176 if (DATA_TAG_array_flat != DATA_TAG_array_slice - 1 ||
9177 DATA_TAG_array_slice != DATA_TAG_array_pointers - 1 ||
9178 DATA_TAG_array_same < DATA_TAG_array_flat ||
9179 DATA_TAG_array_btree < DATA_TAG_array_flat ||
9180 DATA_TAG_array_incomplete < DATA_TAG_array_flat) {
9181 not_reached();
9182 return false;
9183 }
9184
9185 gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
9186 gen_one(R_SCRATCH_2);
9187 gen_one(R_SCRATCH_1);
9188
9189 g(gen_compare_da_tag(ctx, R_SCRATCH_1, DATA_TAG_array_pointers, COND_A, escape_label, R_SCRATCH_1));
9190
9191 gen_pointer_compression(R_SCRATCH_2);
9192 g(gen_address(ctx, R_SCRATCH_2, offsetof(struct data, u_.array_flat.n_used_entries), ARCH_PREFERS_SX(OP_SIZE_INT) ? IMM_PURPOSE_LDR_SX_OFFSET : IMM_PURPOSE_LDR_OFFSET, OP_SIZE_INT));
9193 gen_insn(ARCH_PREFERS_SX(OP_SIZE_INT) ? INSN_MOVSX : INSN_MOV, OP_SIZE_INT, 0, 0);
9194 gen_one(R_SCRATCH_1);
9195 gen_address_offset_compressed();
9196
9197 if (slot_2 == NO_FRAME_T) {
9198 g(gen_frame_store(ctx, OP_SIZE_INT, slot_r, 0, R_SCRATCH_1));
9199 } else {
9200 g(gen_frame_load_cmp_set_cond(ctx, OP_SIZE_INT, false, slot_2, 0, R_SCRATCH_1, COND_G, slot_r));
9201 }
9202 flag_set(ctx, slot_r, false);
9203 }
9204 return true;
9205 }
9206
9207 static bool attr_w gen_array_sub(struct codegen_context *ctx, frame_t slot_array, frame_t slot_from, frame_t slot_to, frame_t slot_r, frame_t flags)
9208 {
9209 const struct type *t = get_type_of_local(ctx, slot_array);
9210 uint32_t escape_label, upcall_label;
9211
9212 escape_label = alloc_escape_label(ctx);
9213 if (unlikely(!escape_label))
9214 return false;
9215
9216 upcall_label = alloc_label(ctx);
9217 if (unlikely(!upcall_label))
9218 return false;
9219
9220 if (unlikely(TYPE_IS_FLAT(t))) {
9221 g(gen_test_1_jz_cached(ctx, slot_array, escape_label));
9222 }
9223
9224 g(gen_test_2_cached(ctx, slot_from, slot_to, escape_label));
9225
9226 if (ctx->registers[slot_array] >= 0)
9227 g(spill(ctx, slot_array));
9228 if (ctx->registers[slot_from] >= 0)
9229 g(spill(ctx, slot_from));
9230 if (ctx->registers[slot_to] >= 0)
9231 g(spill(ctx, slot_to));
9232
9233 g(gen_upcall_start(ctx, 4));
9234
9235 g(gen_frame_load_raw(ctx, OP_SIZE_SLOT, false, slot_array, 0, R_ARG0));
9236 g(gen_upcall_argument(ctx, 0));
9237
9238 g(gen_frame_load_raw(ctx, OP_SIZE_INT, false, slot_from, 0, R_ARG1));
9239 g(gen_upcall_argument(ctx, 1));
9240
9241 g(gen_frame_load_raw(ctx, OP_SIZE_INT, false, slot_to, 0, R_ARG2));
9242 g(gen_upcall_argument(ctx, 2));
9243
9244 g(gen_load_constant(ctx, R_ARG3, (flags & OPCODE_FLAG_FREE_ARGUMENT) != 0));
9245 g(gen_upcall_argument(ctx, 3));
9246
9247 if ((flags & OPCODE_FLAG_FREE_ARGUMENT) != 0) {
9248 g(gen_test_1_cached(ctx, slot_array, upcall_label));
9249 g(gen_load_constant(ctx, R_ARG3, 0));
9250 g(gen_upcall_argument(ctx, 3));
9251 }
9252
9253 gen_label(upcall_label);
9254 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_array_sub), 4));
9255
9256 g(gen_jmp_on_zero(ctx, OP_SIZE_SLOT, R_RET0, COND_E, escape_label));
9257
9258 if (slot_array != slot_r) {
9259 if (flags & OPCODE_FLAG_FREE_ARGUMENT) {
9260 g(gen_set_1(ctx, R_FRAME, slot_array, 0, false));
9261 g(gen_frame_clear(ctx, OP_SIZE_SLOT, slot_array));
9262 flag_set(ctx, slot_array, false);
9263 }
9264 }
9265
9266 g(gen_frame_set_pointer(ctx, slot_r, R_RET0));
9267
9268 return true;
9269 }
9270
9271 static bool attr_w gen_array_skip(struct codegen_context *ctx, frame_t slot_array, frame_t slot_from, frame_t slot_r, frame_t flags)
9272 {
9273 const struct type *t = get_type_of_local(ctx, slot_array);
9274 uint32_t escape_label, upcall_label;
9275
9276 escape_label = alloc_escape_label(ctx);
9277 if (unlikely(!escape_label))
9278 return false;
9279
9280 upcall_label = alloc_label(ctx);
9281 if (unlikely(!upcall_label))
9282 return false;
9283
9284 if (unlikely(TYPE_IS_FLAT(t))) {
9285 g(gen_test_1_jz_cached(ctx, slot_array, escape_label));
9286 }
9287
9288 g(gen_test_1_cached(ctx, slot_from, escape_label));
9289
9290 if (ctx->registers[slot_array] >= 0)
9291 g(spill(ctx, slot_array));
9292 if (ctx->registers[slot_from] >= 0)
9293 g(spill(ctx, slot_from));
9294
9295 g(gen_upcall_start(ctx, 3));
9296
9297 g(gen_frame_load(ctx, OP_SIZE_SLOT, false, slot_array, 0, R_ARG0));
9298 g(gen_upcall_argument(ctx, 0));
9299
9300 g(gen_frame_load(ctx, OP_SIZE_INT, false, slot_from, 0, R_ARG1));
9301 g(gen_upcall_argument(ctx, 1));
9302
9303 g(gen_load_constant(ctx, R_ARG2, (flags & OPCODE_FLAG_FREE_ARGUMENT) != 0));
9304 g(gen_upcall_argument(ctx, 2));
9305
9306 if ((flags & OPCODE_FLAG_FREE_ARGUMENT) != 0) {
9307 g(gen_test_1_cached(ctx, slot_array, upcall_label));
9308 g(gen_load_constant(ctx, R_ARG2, 0));
9309 g(gen_upcall_argument(ctx, 2));
9310 }
9311
9312 gen_label(upcall_label);
9313 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_array_skip), 3));
9314
9315 g(gen_jmp_on_zero(ctx, OP_SIZE_SLOT, R_RET0, COND_E, escape_label));
9316
9317 if (slot_array != slot_r) {
9318 if (flags & OPCODE_FLAG_FREE_ARGUMENT) {
9319 g(gen_set_1(ctx, R_FRAME, slot_array, 0, false));
9320 g(gen_frame_clear(ctx, OP_SIZE_SLOT, slot_array));
9321 flag_set(ctx, slot_array, false);
9322 }
9323 }
9324
9325 g(gen_frame_set_pointer(ctx, slot_r, R_RET0));
9326
9327 return true;
9328 }
9329
9330 static bool attr_w gen_array_append(struct codegen_context *ctx, frame_t slot_1, frame_t slot_2, frame_t slot_r, frame_t flags)
9331 {
9332 uint32_t escape_label;
9333
9334 escape_label = alloc_escape_label(ctx);
9335 if (unlikely(!escape_label))
9336 return false;
9337
9338 if (unlikely(TYPE_IS_FLAT(get_type_of_local(ctx, slot_1))))
9339 g(gen_test_1_jz_cached(ctx, slot_1, escape_label));
9340 if (unlikely(TYPE_IS_FLAT(get_type_of_local(ctx, slot_2))))
9341 g(gen_test_1_jz_cached(ctx, slot_2, escape_label));
9342
9343 g(gen_frame_load(ctx, OP_SIZE_SLOT, false, slot_1, 0, R_SCRATCH_1));
9344 g(gen_ptr_is_thunk(ctx, R_SCRATCH_1, true, escape_label));
9345 g(gen_frame_load(ctx, OP_SIZE_SLOT, false, slot_2, 0, R_SCRATCH_2));
9346 g(gen_ptr_is_thunk(ctx, R_SCRATCH_2, true, escape_label));
9347 g(gen_barrier(ctx));
9348
9349 g(gen_compare_da_tag(ctx, R_SCRATCH_1, DATA_TAG_array_incomplete, COND_E, escape_label, R_SCRATCH_1));
9350 g(gen_compare_da_tag(ctx, R_SCRATCH_2, DATA_TAG_array_incomplete, COND_E, escape_label, R_SCRATCH_2));
9351
9352 g(gen_frame_get_pointer(ctx, slot_2, (flags & OPCODE_FLAG_FREE_ARGUMENT_2) != 0, R_SAVED_1));
9353 g(gen_frame_get_pointer(ctx, slot_1, (flags & OPCODE_FLAG_FREE_ARGUMENT) != 0, R_SCRATCH_1));
9354 g(gen_upcall_start(ctx, 2));
9355 gen_insn(ARCH_PREFERS_SX(i_size(OP_SIZE_SLOT)) ? INSN_MOVSX : INSN_MOV, i_size(OP_SIZE_SLOT), 0, 0);
9356 gen_one(R_ARG0);
9357 gen_one(R_SCRATCH_1);
9358 g(gen_upcall_argument(ctx, 0));
9359 gen_insn(ARCH_PREFERS_SX(i_size(OP_SIZE_SLOT)) ? INSN_MOVSX : INSN_MOV, i_size(OP_SIZE_SLOT), 0, 0);
9360 gen_one(R_ARG1);
9361 gen_one(R_SAVED_1);
9362 g(gen_upcall_argument(ctx, 1));
9363 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_array_join), 2));
9364 g(gen_frame_set_pointer(ctx, slot_r, R_RET0));
9365 return true;
9366 }
9367
9368 static bool attr_w gen_array_append_one_flat(struct codegen_context *ctx, frame_t slot_1, frame_t slot_2, frame_t slot_r, frame_t flags)
9369 {
9370 uint32_t escape_label;
9371
9372 escape_label = alloc_escape_label(ctx);
9373 if (unlikely(!escape_label))
9374 return false;
9375
9376 if (unlikely(!(flags & OPCODE_FLAG_FREE_ARGUMENT))) {
9377 gen_insn(INSN_JMP, 0, 0, 0);
9378 gen_four(escape_label);
9379 return true;
9380 }
9381
9382 g(gen_test_1_jz_cached(ctx, slot_1, escape_label));
9383 g(gen_test_1_cached(ctx, slot_2, escape_label));
9384 flag_set(ctx, slot_2, false);
9385
9386 g(gen_frame_load(ctx, OP_SIZE_SLOT, false, slot_1, 0, R_SAVED_1));
9387 g(gen_ptr_is_thunk(ctx, R_SAVED_1, true, escape_label));
9388 g(gen_barrier(ctx));
9389
9390 g(gen_decompress_pointer(ctx, R_SAVED_1, 0));
9391
9392 g(gen_compare_tag_and_refcount(ctx, R_SAVED_1, DATA_TAG_array_flat, escape_label, R_SCRATCH_1));
9393
9394 g(gen_address(ctx, R_SAVED_1, offsetof(struct data, u_.array_flat.n_used_entries), ARCH_PREFERS_SX(OP_SIZE_INT) ? IMM_PURPOSE_LDR_SX_OFFSET : IMM_PURPOSE_LDR_OFFSET, OP_SIZE_INT));
9395 gen_insn(ARCH_PREFERS_SX(OP_SIZE_INT) ? INSN_MOVSX : INSN_MOV, OP_SIZE_INT, 0, 0);
9396 gen_one(R_SCRATCH_2);
9397 gen_address_offset();
9398
9399 g(gen_check_array_len(ctx, R_SAVED_1, true, R_SCRATCH_2, COND_E, escape_label));
9400
9401 g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_INT), ALU_ADD, R_SCRATCH_1, R_SCRATCH_2, 1));
9402
9403 g(gen_address(ctx, R_SAVED_1, offsetof(struct data, u_.array_flat.n_used_entries), IMM_PURPOSE_STR_OFFSET, OP_SIZE_INT));
9404 gen_insn(INSN_MOV, OP_SIZE_INT, 0, 0);
9405 gen_address_offset();
9406 gen_one(R_SCRATCH_1);
9407
9408 g(gen_scaled_array_store(ctx, R_SAVED_1, data_array_offset, slot_2));
9409
9410 if (slot_1 != slot_r) {
9411 g(gen_frame_clear(ctx, OP_SIZE_SLOT, slot_1));
9412 g(gen_set_1(ctx, R_FRAME, slot_1, 0, false));
9413 flag_set(ctx, slot_1, false);
9414 g(gen_compress_pointer(ctx, R_SAVED_1));
9415 g(gen_frame_set_pointer(ctx, slot_r, R_SAVED_1));
9416 }
9417
9418 return true;
9419 }
9420
9421 static bool attr_w gen_array_append_one(struct codegen_context *ctx, frame_t slot_1, frame_t slot_2, frame_t slot_r, frame_t flags)
9422 {
9423 uint32_t escape_label;
9424
9425 escape_label = alloc_escape_label(ctx);
9426 if (unlikely(!escape_label))
9427 return false;
9428
9429 if (unlikely(!(flags & OPCODE_FLAG_FREE_ARGUMENT))) {
9430 gen_insn(INSN_JMP, 0, 0, 0);
9431 gen_four(escape_label);
9432 return true;
9433 }
9434
9435 g(gen_test_1_jz_cached(ctx, slot_1, escape_label));
9436
9437 g(gen_frame_load(ctx, OP_SIZE_SLOT, false, slot_1, 0, R_SAVED_1));
9438 g(gen_ptr_is_thunk(ctx, R_SAVED_1, true, escape_label));
9439 g(gen_barrier(ctx));
9440
9441 g(gen_decompress_pointer(ctx, R_SAVED_1, 0));
9442
9443 g(gen_compare_tag_and_refcount(ctx, R_SAVED_1, DATA_TAG_array_pointers, escape_label, R_SCRATCH_1));
9444
9445 g(gen_address(ctx, R_SAVED_1, offsetof(struct data, u_.array_pointers.n_used_entries), ARCH_PREFERS_SX(OP_SIZE_INT) ? IMM_PURPOSE_LDR_SX_OFFSET : IMM_PURPOSE_LDR_OFFSET, OP_SIZE_INT));
9446 gen_insn(ARCH_PREFERS_SX(OP_SIZE_INT) ? INSN_MOVSX : INSN_MOV, OP_SIZE_INT, 0, 0);
9447 gen_one(R_SAVED_2);
9448 gen_address_offset();
9449
9450 g(gen_check_array_len(ctx, R_SAVED_1, true, R_SAVED_2, COND_E, escape_label));
9451
9452 g(gen_frame_get_pointer(ctx, slot_2, (flags & OPCODE_FLAG_FREE_ARGUMENT_2) != 0, R_SCRATCH_2));
9453
9454 g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_INT), ALU_ADD, R_SCRATCH_1, R_SAVED_2, 1));
9455
9456 g(gen_address(ctx, R_SAVED_1, offsetof(struct data, u_.array_pointers.n_used_entries), IMM_PURPOSE_STR_OFFSET, OP_SIZE_INT));
9457 gen_insn(INSN_MOV, OP_SIZE_INT, 0, 0);
9458 gen_address_offset();
9459 gen_one(R_SCRATCH_1);
9460
9461 g(gen_address(ctx, R_SAVED_1, offsetof(struct data, u_.array_pointers.pointer), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_ADDRESS));
9462 gen_insn(INSN_MOV, OP_SIZE_ADDRESS, 0, 0);
9463 gen_one(R_SCRATCH_3);
9464 gen_address_offset();
9465
9466 g(gen_scaled_array_address(ctx, slot_size, R_SAVED_2, R_SCRATCH_3, R_SAVED_2, 0));
9467
9468 gen_insn(INSN_MOV, OP_SIZE_SLOT, 0, 0);
9469 gen_one(ARG_ADDRESS_1);
9470 gen_one(R_SAVED_2);
9471 gen_eight(0);
9472 gen_one(R_SCRATCH_2);
9473
9474 if (slot_1 != slot_r) {
9475 g(gen_frame_clear(ctx, OP_SIZE_SLOT, slot_1));
9476 g(gen_set_1(ctx, R_FRAME, slot_1, 0, false));
9477 flag_set(ctx, slot_1, false);
9478 g(gen_compress_pointer(ctx, R_SAVED_1));
9479 g(gen_frame_set_pointer(ctx, slot_r, R_SAVED_1));
9480 }
9481
9482 return true;
9483 }
9484
9485 static bool attr_w gen_io(struct codegen_context *ctx, frame_t code, frame_t slot_1, frame_t slot_2, frame_t slot_3)
9486 {
9487 uint32_t reload_label;
9488 frame_t i;
9489
9490 reload_label = alloc_reload_label(ctx);
9491 if (unlikely(!reload_label))
9492 return false;
9493
9494 if (ctx->var_aux) {
9495 mem_free(ctx->var_aux);
9496 ctx->var_aux = NULL;
9497 }
9498 ctx->var_aux = mem_alloc_array_mayfail(mem_alloc_mayfail, frame_t *, 0, 0, slot_1 + slot_2, sizeof(frame_t), &ctx->err);
9499 if (unlikely(!ctx->var_aux))
9500 return false;
9501
9502 for (i = 0; i < slot_1 + slot_2; i++)
9503 ctx->var_aux[i] = get_uint32(ctx);
9504 for (i = 0; i < slot_3; i++)
9505 get_uint32(ctx);
9506
9507 for (i = 0; i < slot_2; i++) {
9508 frame_t input_slot = ctx->var_aux[slot_1 + i];
9509 if (ctx->registers[input_slot] >= 0)
9510 g(spill(ctx, input_slot));
9511 }
9512
9513 /*gen_insn(INSN_JMP, 0, 0, 0); gen_four(alloc_escape_label(ctx));*/
9514
9515 g(gen_upcall_start(ctx, 3));
9516 gen_insn(INSN_MOV, i_size(OP_SIZE_ADDRESS), 0, 0);
9517 gen_one(R_ARG0);
9518 gen_one(R_FRAME);
9519 g(gen_upcall_argument(ctx, 0));
9520
9521 g(gen_load_constant(ctx, R_ARG1, ctx->instr_start - da(ctx->fn,function)->code));
9522 g(gen_upcall_argument(ctx, 1));
9523
9524 g(gen_load_constant(ctx, R_ARG2, ((uint32_t)code << 24) | ((uint32_t)slot_1 << 16) | ((uint32_t)slot_2 << 8) | slot_3));
9525 g(gen_upcall_argument(ctx, 2));
9526 /*debug("arg2: %08x", ((uint32_t)code << 24) | ((uint32_t)slot_1 << 16) | ((uint32_t)slot_2 << 8) | slot_3);*/
9527
9528 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_ipret_io), 3));
9529 g(gen_sanitize_returned_pointer(ctx, R_RET0));
9530 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, OP_SIZE_ADDRESS, R_RET0, ptr_to_num(POINTER_FOLLOW_THUNK_GO), COND_NE, reload_label));
9531
9532 for (i = 0; i < slot_1; i++) {
9533 frame_t output_slot = ctx->var_aux[i];
9534 if (ctx->registers[output_slot] >= 0)
9535 g(unspill(ctx, output_slot));
9536 }
9537 for (i = 0; i < slot_1; i++) {
9538 frame_t output_slot = ctx->var_aux[i];
9539 flag_set_unknown(ctx, output_slot);
9540 if (da(ctx->fn,function)->local_variables_flags[output_slot].must_be_flat) {
9541 uint32_t escape_label = alloc_escape_label_for_ip(ctx, ctx->current_position);
9542 if (unlikely(!escape_label)) {
9543 mem_free(ctx->var_aux);
9544 return false;
9545 }
9546 g(gen_test_1(ctx, R_FRAME, output_slot, 0, escape_label, false, TEST));
9547 }
9548 }
9549
9550 return true;
9551 }
9552
9553
9554 static bool attr_w gen_registers(struct codegen_context *ctx)
9555 {
9556 frame_t v;
9557 unsigned int_reg = 0;
9558 const uint8_t *av = reg_available;
9559 /*for (v = function_n_variables(ctx->fn) - 1; v >= MIN_USEABLE_SLOT; v--)*/
9560 for (v = MIN_USEABLE_SLOT; v < function_n_variables(ctx->fn); v++) {
9561 const struct type *t;
9562 ctx->registers[v] = -1;
9563 if (ra_chicken)
9564 continue;
9565 t = get_type_of_local(ctx, v);
9566 if (unlikely(!t))
9567 continue;
9568 if (!da(ctx->fn,function)->local_variables_flags[v].must_be_flat)
9569 continue;
9570 if (!ARCH_HAS_BWX && t->size < 1U << OP_SIZE_4)
9571 continue;
9572 if ((TYPE_TAG_IS_FIXED(t->tag) || TYPE_TAG_IS_INT(t->tag)) && is_power_of_2(t->size) && t->size <= 1U << OP_SIZE_NATIVE) {
9573 if (int_reg >= av[0])
9574 continue;
9575 ctx->registers[v] = av[1 + int_reg];
9576 int_reg++;
9577 if (!reg_is_saved(ctx->registers[v])) {
9578 if (unlikely(!array_add_mayfail(frame_t, &ctx->need_spill, &ctx->need_spill_l, v, NULL, &ctx->err)))
9579 return false;
9580 }
9581 continue;
9582 }
9583 }
9584
9585 return true;
9586 }
9587
9588 static bool attr_w gen_function(struct codegen_context *ctx)
9589 {
9590 ctx->current_position = da(ctx->fn,function)->code;
9591
9592 ctx->escape_nospill_label = alloc_label(ctx);
9593 if (unlikely(!ctx->escape_nospill_label))
9594 return false;
9595
9596 while (ctx->current_position != da(ctx->fn,function)->code + da(ctx->fn,function)->code_size) {
9597 ip_t ip;
9598 code_t code;
9599 unsigned op, type;
9600 frame_t slot_1, slot_2, slot_3, slot_r, flags, fn_idx, opt;
9601 arg_t n_args, n_ret, i_arg;
9602 uint32_t label_id;
9603 uint32_t escape_label;
9604
9605 ajla_assert_lo(ctx->current_position < da(ctx->fn,function)->code + da(ctx->fn,function)->code_size, (file_line, "gen_function: ran out of code in %s", da(ctx->fn,function)->function_name));
9606
9607 ctx->instr_start = ctx->current_position;
9608
9609 /*debug("%s: %04x, %s", da(ctx->fn,function)->function_name, *ctx->instr_start, decode_opcode(*ctx->instr_start, true));*/
9610
9611 ip = ctx->instr_start - da(ctx->fn,function)->code;
9612 if (likely(!ctx->code_labels[ip])) {
9613 ctx->code_labels[ip] = alloc_label(ctx);
9614 if (unlikely(!ctx->code_labels[ip]))
9615 return false;
9616 }
9617 gen_label(ctx->code_labels[ip]);
9618
9619 code = get_code(ctx);
9620 ctx->arg_mode = code / OPCODE_MODE_MULT;
9621 code %= OPCODE_MODE_MULT;
9622 ajla_assert_lo(ctx->arg_mode < ARG_MODE_N, (file_line, "gen_function: invalid opcode %04x", (unsigned)*ctx->instr_start));
9623
9624 if (code >= OPCODE_FIXED_OP + uzero && code < OPCODE_INT_OP) {
9625 code -= OPCODE_FIXED_OP;
9626 op = (code / OPCODE_FIXED_OP_MULT) % OPCODE_FIXED_TYPE_MULT;
9627 type = code / OPCODE_FIXED_TYPE_MULT;
9628 if (op < OPCODE_FIXED_OP_UNARY) {
9629 get_two(ctx, &slot_1, &slot_2);
9630 get_two(ctx, &slot_r, &flags);
9631 escape_label = alloc_escape_label(ctx);
9632 if (unlikely(!escape_label))
9633 return false;
9634 g(gen_test_2_cached(ctx, slot_1, slot_2, escape_label));
9635 g(gen_alu(ctx, MODE_FIXED, type, op, escape_label, slot_1, slot_2, slot_r));
9636 flag_set(ctx, slot_1, false);
9637 flag_set(ctx, slot_2, false);
9638 flag_set(ctx, slot_r, false);
9639 continue;
9640 } else if (op < OPCODE_FIXED_OP_N) {
9641 get_two(ctx, &slot_1, &slot_r);
9642 get_one(ctx, &flags);
9643 escape_label = alloc_escape_label(ctx);
9644 if (unlikely(!escape_label))
9645 return false;
9646 g(gen_test_1_cached(ctx, slot_1, escape_label));
9647 g(gen_alu1(ctx, MODE_FIXED, type, op, escape_label, slot_1, slot_r));
9648 flag_set(ctx, slot_1, false);
9649 flag_set(ctx, slot_r, false);
9650 continue;
9651 } else if (op == OPCODE_FIXED_OP_ldc) {
9652 unsigned i;
9653 get_one(ctx, &slot_r);
9654 g(gen_constant(ctx, type, false, slot_r));
9655 for (i = 0; i < 1U << type; i += 2)
9656 get_code(ctx);
9657 flag_set(ctx, slot_r, false);
9658 continue;
9659 } else if (op == OPCODE_FIXED_OP_ldc16) {
9660 get_one(ctx, &slot_r);
9661 g(gen_constant(ctx, type, true, slot_r));
9662 get_code(ctx);
9663 flag_set(ctx, slot_r, false);
9664 continue;
9665 } else if (op == OPCODE_FIXED_OP_move || op == OPCODE_FIXED_OP_copy) {
9666 get_two(ctx, &slot_1, &slot_r);
9667 escape_label = alloc_escape_label(ctx);
9668 if (unlikely(!escape_label))
9669 return false;
9670 g(gen_test_1_cached(ctx, slot_1, escape_label));
9671 g(gen_copy(ctx, type, slot_1, slot_r));
9672 flag_set(ctx, slot_1, false);
9673 flag_set(ctx, slot_r, false);
9674 continue;
9675 } else {
9676 internal(file_line, "gen_function: bad fixed code %04x", *ctx->instr_start);
9677 }
9678 } else if (code >= OPCODE_INT_OP && code < OPCODE_REAL_OP) {
9679 code -= OPCODE_INT_OP;
9680 op = (code / OPCODE_INT_OP_MULT) % OPCODE_INT_TYPE_MULT;
9681 type = code / OPCODE_INT_TYPE_MULT;
9682 if (op < OPCODE_INT_OP_UNARY) {
9683 get_two(ctx, &slot_1, &slot_2);
9684 get_two(ctx, &slot_r, &flags);
9685 escape_label = alloc_escape_label(ctx);
9686 if (unlikely(!escape_label))
9687 return false;
9688 g(gen_test_2_cached(ctx, slot_1, slot_2, escape_label));
9689 g(gen_alu(ctx, MODE_INT, type, op, escape_label, slot_1, slot_2, slot_r));
9690 flag_set(ctx, slot_1, false);
9691 flag_set(ctx, slot_2, false);
9692 flag_set(ctx, slot_r, false);
9693 continue;
9694 } else if (op < OPCODE_INT_OP_N) {
9695 get_two(ctx, &slot_1, &slot_r);
9696 get_one(ctx, &flags);
9697 if ((op == OPCODE_INT_OP_to_int || op == OPCODE_INT_OP_from_int) && slot_1 == slot_r)
9698 continue;
9699 escape_label = alloc_escape_label(ctx);
9700 if (unlikely(!escape_label))
9701 return false;
9702 g(gen_test_1_cached(ctx, slot_1, escape_label));
9703 g(gen_alu1(ctx, MODE_INT, type, op, escape_label, slot_1, slot_r));
9704 flag_set(ctx, slot_1, false);
9705 flag_set(ctx, slot_r, false);
9706 continue;
9707 } else if (op == OPCODE_INT_OP_ldc) {
9708 unsigned i;
9709 get_one(ctx, &slot_r);
9710 g(gen_constant(ctx, type, false, slot_r));
9711 for (i = 0; i < 1U << type; i += 2)
9712 get_code(ctx);
9713 flag_set(ctx, slot_r, false);
9714 continue;
9715 } else if (op == OPCODE_INT_OP_ldc16) {
9716 get_one(ctx, &slot_r);
9717 g(gen_constant(ctx, type, true, slot_r));
9718 get_code(ctx);
9719 flag_set(ctx, slot_r, false);
9720 continue;
9721 } else if (op == OPCODE_INT_OP_move || op == OPCODE_INT_OP_copy) {
9722 get_two(ctx, &slot_1, &slot_r);
9723 escape_label = alloc_escape_label(ctx);
9724 if (unlikely(!escape_label))
9725 return false;
9726 g(gen_test_1_cached(ctx, slot_1, escape_label));
9727 g(gen_copy(ctx, type, slot_1, slot_r));
9728 flag_set(ctx, slot_1, false);
9729 flag_set(ctx, slot_r, false);
9730 continue;
9731 } else {
9732 internal(file_line, "gen_function: bad integer code %04x", *ctx->instr_start);
9733 }
9734 } else if (code >= OPCODE_REAL_OP && code < OPCODE_BOOL_OP) {
9735 code -= OPCODE_REAL_OP;
9736 op = (code / OPCODE_REAL_OP_MULT) % OPCODE_REAL_TYPE_MULT;
9737 type = code / OPCODE_REAL_TYPE_MULT;
9738 if (op < OPCODE_REAL_OP_UNARY) {
9739 get_two(ctx, &slot_1, &slot_2);
9740 get_two(ctx, &slot_r, &flags);
9741 escape_label = alloc_escape_label(ctx);
9742 if (unlikely(!escape_label))
9743 return false;
9744 g(gen_test_2_cached(ctx, slot_1, slot_2, escape_label));
9745 g(gen_fp_alu(ctx, type, op, escape_label, slot_1, slot_2, slot_r));
9746 flag_set(ctx, slot_1, false);
9747 flag_set(ctx, slot_2, false);
9748 flag_set(ctx, slot_r, false);
9749 continue;
9750 } else if (op < OPCODE_REAL_OP_N) {
9751 get_two(ctx, &slot_1, &slot_r);
9752 get_one(ctx, &flags);
9753 escape_label = alloc_escape_label(ctx);
9754 if (unlikely(!escape_label))
9755 return false;
9756 g(gen_test_1_cached(ctx, slot_1, escape_label));
9757 g(gen_fp_alu1(ctx, type, op, escape_label, slot_1, slot_r));
9758 flag_set(ctx, slot_1, false);
9759 flag_set(ctx, slot_r, false);
9760 continue;
9761 } else if (op == OPCODE_REAL_OP_ldc) {
9762 const struct type *t;
9763 unsigned i;
9764 get_one(ctx, &slot_r);
9765 t = type_get_real(type);
9766 g(gen_real_constant(ctx, t, slot_r));
9767 for (i = 0; i < t->size; i += 2)
9768 get_code(ctx);
9769 flag_set(ctx, slot_r, false);
9770 continue;
9771 } else if (op == OPCODE_REAL_OP_move || op == OPCODE_REAL_OP_copy) {
9772 get_two(ctx, &slot_1, &slot_r);
9773 escape_label = alloc_escape_label(ctx);
9774 if (unlikely(!escape_label))
9775 return false;
9776 g(gen_test_1_cached(ctx, slot_1, escape_label));
9777 g(gen_memcpy_slots(ctx, slot_r, slot_1));
9778 flag_set(ctx, slot_1, false);
9779 flag_set(ctx, slot_r, false);
9780 continue;
9781 } else {
9782 internal(file_line, "gen_function: bad real code %04x", *ctx->instr_start);
9783 }
9784 } else if (code >= OPCODE_BOOL_OP && code < OPCODE_EXTRA) {
9785 code -= OPCODE_BOOL_OP;
9786 op = (code / OPCODE_BOOL_OP_MULT) % OPCODE_BOOL_TYPE_MULT;
9787 type = log_2(sizeof(ajla_flat_option_t));
9788 if (op < OPCODE_BOOL_OP_UNARY) {
9789 get_two(ctx, &slot_1, &slot_2);
9790 get_two(ctx, &slot_r, &flags);
9791 escape_label = alloc_escape_label(ctx);
9792 if (unlikely(!escape_label))
9793 return false;
9794 g(gen_test_2_cached(ctx, slot_1, slot_2, escape_label));
9795 g(gen_alu(ctx, MODE_BOOL, type, op, escape_label, slot_1, slot_2, slot_r));
9796 flag_set(ctx, slot_1, false);
9797 flag_set(ctx, slot_2, false);
9798 flag_set(ctx, slot_r, false);
9799 continue;
9800 } else if (op < OPCODE_BOOL_OP_N) {
9801 get_two(ctx, &slot_1, &slot_r);
9802 get_one(ctx, &flags);
9803 escape_label = alloc_escape_label(ctx);
9804 if (unlikely(!escape_label))
9805 return false;
9806 g(gen_test_1_cached(ctx, slot_1, escape_label));
9807 g(gen_alu1(ctx, MODE_BOOL, type, op, escape_label, slot_1, slot_r));
9808 flag_set(ctx, slot_1, false);
9809 flag_set(ctx, slot_r, false);
9810 continue;
9811 } else if (op == OPCODE_BOOL_OP_move || op == OPCODE_BOOL_OP_copy) {
9812 get_two(ctx, &slot_1, &slot_r);
9813 escape_label = alloc_escape_label(ctx);
9814 if (unlikely(!escape_label))
9815 return false;
9816 g(gen_test_1_cached(ctx, slot_1, escape_label));
9817 g(gen_copy(ctx, type, slot_1, slot_r));
9818 flag_set(ctx, slot_1, false);
9819 flag_set(ctx, slot_r, false);
9820 continue;
9821 } else {
9822 internal(file_line, "gen_function: bad boolean code %04x", *ctx->instr_start);
9823 }
9824 } else switch (code) {
9825 case OPCODE_INT_LDC_LONG: {
9826 uint32_t words, w;
9827 get_one(ctx, &slot_r);
9828 words = get_uint32(ctx);
9829 for (w = 0; w < words; w++)
9830 get_code(ctx);
9831 unconditional_escape:
9832 escape_label = alloc_escape_label(ctx);
9833 if (unlikely(!escape_label))
9834 return false;
9835 gen_insn(INSN_JMP, 0, 0, 0);
9836 gen_four(escape_label);
9837 continue;
9838 }
9839 case OPCODE_IS_EXCEPTION: {
9840 get_two(ctx, &slot_1, &slot_r);
9841 get_one(ctx, &flags);
9842 g(gen_is_exception(ctx, slot_1, slot_r));
9843 continue;
9844 }
9845 case OPCODE_EXCEPTION_CLASS:
9846 case OPCODE_EXCEPTION_TYPE:
9847 case OPCODE_EXCEPTION_AUX: {
9848 get_two(ctx, &slot_1, &slot_r);
9849 get_one(ctx, &flags);
9850 goto unconditional_escape;
9851 }
9852 case OPCODE_SYSTEM_PROPERTY: {
9853 get_two(ctx, &slot_1, &slot_r);
9854 get_one(ctx, &flags);
9855 g(gen_system_property(ctx, slot_1, slot_r));
9856 continue;
9857 }
9858 case OPCODE_FLAT_MOVE:
9859 case OPCODE_FLAT_COPY: {
9860 get_two(ctx, &slot_1, &slot_r);
9861 g(gen_flat_move_copy(ctx, slot_1, slot_r));
9862 continue;
9863 }
9864 case OPCODE_REF_MOVE:
9865 case OPCODE_REF_MOVE_CLEAR:
9866 case OPCODE_REF_COPY: {
9867 get_two(ctx, &slot_1, &slot_r);
9868 g(gen_ref_move_copy(ctx, code, slot_1, slot_r));
9869 continue;
9870 }
9871 case OPCODE_BOX_MOVE_CLEAR:
9872 case OPCODE_BOX_COPY: {
9873 get_two(ctx, &slot_1, &slot_r);
9874 g(gen_box_move_copy(ctx, code, slot_1, slot_r));
9875 continue;
9876 }
9877 case OPCODE_TAKE_BORROWED:
9878 get_one(ctx, &slot_1);
9879 if (!da(ctx->fn,function)->local_variables_flags[slot_1].may_be_borrowed)
9880 continue;
9881 if (unlikely(!(label_id = alloc_label(ctx))))
9882 return false;
9883 if (flag_is_set(ctx, slot_1))
9884 goto take_borrowed_done;
9885 if (flag_is_clear(ctx, slot_1)) {
9886 g(gen_set_1(ctx, R_FRAME, slot_1, 0, true));
9887 goto do_take_borrowed;
9888 }
9889 g(gen_test_1(ctx, R_FRAME, slot_1, 0, label_id, false, TEST_SET));
9890 do_take_borrowed:
9891 g(gen_upcall_start(ctx, 1));
9892 g(gen_frame_load(ctx, OP_SIZE_SLOT, false, slot_1, 0, R_ARG0));
9893 g(gen_upcall_argument(ctx, 0));
9894 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_pointer_reference_owned), 1));
9895 flag_set(ctx, slot_1, true);
9896 take_borrowed_done:
9897 gen_label(label_id);
9898 continue;
9899 case OPCODE_DEREFERENCE:
9900 case OPCODE_DEREFERENCE_CLEAR: {
9901 bool need_bit_test;
9902 /*const struct type *type;*/
9903 get_one(ctx, &slot_1);
9904 if (flag_is_clear(ctx, slot_1))
9905 goto skip_dereference;
9906 /*type = get_type_of_local(ctx, slot_1);*/
9907 /*need_bit_test = 1 || TYPE_IS_FLAT(type) || da(ctx->fn,function)->local_variables[slot_1].may_be_borrowed;*/
9908 need_bit_test = !flag_is_set(ctx, slot_1);
9909 if (need_bit_test) {
9910 if (unlikely(!(label_id = alloc_label(ctx))))
9911 return false;
9912 g(gen_test_1(ctx, R_FRAME, slot_1, 0, label_id, true, TEST_CLEAR));
9913 } else {
9914 g(gen_set_1(ctx, R_FRAME, slot_1, 0, false));
9915 label_id = 0; /* avoid warning */
9916 }
9917 g(gen_upcall_start(ctx, 1));
9918 g(gen_frame_load(ctx, OP_SIZE_SLOT, false, slot_1, 0, R_ARG0));
9919 g(gen_upcall_argument(ctx, 0));
9920 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_pointer_dereference), 1));
9921 if (need_bit_test)
9922 gen_label(label_id);
9923 skip_dereference:
9924 if (code == OPCODE_DEREFERENCE_CLEAR)
9925 g(gen_frame_clear(ctx, OP_SIZE_SLOT, slot_1));
9926 flag_set(ctx, slot_1, false);
9927 continue;
9928 }
9929 case OPCODE_EVAL: {
9930 get_one(ctx, &slot_1);
9931 g(gen_eval(ctx, slot_1));
9932 continue;
9933 }
9934 case OPCODE_ESCAPE_NONFLAT: {
9935 frame_t n, i;
9936 frame_t *vars;
9937
9938 get_one(ctx, &n);
9939 vars = mem_alloc_array_mayfail(mem_alloc_mayfail, frame_t *, 0, 0, n, sizeof(frame_t), &ctx->err);
9940 if (unlikely(!vars))
9941 return false;
9942 for (i = 0; i < n; i++) {
9943 get_one(ctx, &vars[i]);
9944 }
9945
9946 escape_label = alloc_escape_label(ctx);
9947 if (unlikely(!escape_label)) {
9948 mem_free(vars);
9949 return false;
9950 }
9951
9952 if (unlikely(!gen_test_multiple(ctx, vars, n, escape_label))) {
9953 mem_free(vars);
9954 return false;
9955 }
9956 mem_free(vars);
9957
9958 continue;
9959 }
9960 case OPCODE_CHECKPOINT: {
9961 frame_t n_vars;
9962
9963 g(clear_flag_cache(ctx));
9964
9965 if (SIZEOF_IP_T == 2) {
9966 slot_1 = get_code(ctx);
9967 } else if (SIZEOF_IP_T == 4) {
9968 slot_1 = get_uint32(ctx);
9969 } else {
9970 not_reached();
9971 continue;
9972 }
9973
9974 if (unlikely(!(slot_1 + 1)))
9975 return false;
9976 while (slot_1 >= ctx->n_entries) {
9977 void *err_entries;
9978 struct cg_entry e;
9979 if (unlikely(!ctx->entries)) {
9980 if (unlikely(!array_init_mayfail(struct cg_entry, &ctx->entries, &ctx->n_entries, &ctx->err)))
9981 return false;
9982 }
9983 memset(&e, 0, sizeof(struct cg_entry));
9984 if (unlikely(!array_add_mayfail(struct cg_entry, &ctx->entries, &ctx->n_entries, e, &err_entries, &ctx->err))) {
9985 ctx->entries = err_entries;
9986 return false;
9987 }
9988 }
9989
9990 get_one(ctx, &n_vars);
9991
9992 escape_label = 0; /* avoid warning */
9993 if (likely(slot_1 != 0)) {
9994 escape_label = alloc_escape_label(ctx);
9995 if (unlikely(!escape_label))
9996 return false;
9997 }
9998
9999 if (n_vars || !slot_1) {
10000 frame_t i;
10001 uint32_t entry_label, nonflat_label;
10002 struct cg_entry *ce = &ctx->entries[slot_1];
10003
10004 if (unlikely(!array_init_mayfail(frame_t, &ce->variables, &ce->n_variables, &ctx->err)))
10005 return false;
10006 for (i = 0; i < n_vars; i++) {
10007 frame_t v;
10008 get_one(ctx, &v);
10009 if (unlikely(!array_add_mayfail(frame_t, &ce->variables, &ce->n_variables, v, NULL, &ctx->err)))
10010 return false;
10011 }
10012 if (!slot_1) {
10013 g(gen_test_multiple(ctx, ce->variables, ce->n_variables, ctx->escape_nospill_label));
10014 }
10015 entry_label = alloc_label(ctx);
10016 if (unlikely(!entry_label))
10017 return false;
10018 gen_label(entry_label);
10019 ce->entry_label = entry_label;
10020
10021 nonflat_label = alloc_escape_label_for_ip(ctx, ctx->current_position);
10022 if (unlikely(!nonflat_label))
10023 return false;
10024 ce->nonflat_label = nonflat_label;
10025
10026 if (unlikely(!slot_1))
10027 g(gen_timestamp_test(ctx, ctx->escape_nospill_label));
10028 else
10029 g(gen_timestamp_test(ctx, escape_label));
10030 } else {
10031 g(gen_timestamp_test(ctx, escape_label));
10032
10033 gen_insn(INSN_ENTRY, 0, 0, 0);
10034 gen_four(slot_1);
10035 }
10036 continue;
10037 }
10038 case OPCODE_JMP: {
10039 int32_t x = get_jump_offset(ctx);
10040 g(gen_jump(ctx, x, 0));
10041 continue;
10042 }
10043 case OPCODE_JMP_BACK_16: {
10044 int32_t x = get_code(ctx);
10045 g(gen_jump(ctx, -x - (int)(2 * sizeof(code_t)), 0));
10046 continue;
10047 }
10048 case OPCODE_JMP_FALSE: {
10049 int32_t offs_false;
10050 get_one(ctx, &slot_1);
10051 offs_false = get_jump_offset(ctx);
10052 get_jump_offset(ctx);
10053 escape_label = alloc_escape_label(ctx);
10054 if (unlikely(!escape_label))
10055 return false;
10056 g(gen_test_1_cached(ctx, slot_1, escape_label));
10057 g(gen_cond_jump(ctx, slot_1, offs_false));
10058 flag_set(ctx, slot_1, false);
10059 continue;
10060 }
10061 case OPCODE_LABEL: {
10062 g(clear_flag_cache(ctx));
10063 continue;
10064 }
10065 #define init_args \
10066 do { \
10067 if (ctx->args != NULL) \
10068 mem_free(ctx->args); \
10069 g(array_init_mayfail(struct code_arg, &ctx->args, &ctx->args_l, &ctx->err));\
10070 } while (0)
10071 #define load_args \
10072 do { \
10073 init_args; \
10074 for (i_arg = 0; i_arg < n_args; i_arg++) { \
10075 struct code_arg a; \
10076 get_two(ctx, &a.slot, &a.flags); \
10077 a.type = 0; \
10078 g(array_add_mayfail(struct code_arg, &ctx->args, &ctx->args_l, a, NULL, &ctx->err));\
10079 } \
10080 } while (0)
10081 case OPCODE_LOAD_FN:
10082 get_two(ctx, &n_args, &slot_r);
10083 get_one(ctx, &fn_idx);
10084 load_args;
10085 g(gen_load_fn_or_curry(ctx, fn_idx, NO_FRAME_T, slot_r, 0));
10086 continue;
10087 case OPCODE_CURRY:
10088 get_two(ctx, &n_args, &slot_r);
10089 get_two(ctx, &slot_1, &flags);
10090 load_args;
10091 g(gen_load_fn_or_curry(ctx, NO_FRAME_T, slot_1, slot_r, flags));
10092 continue;
10093 case OPCODE_CALL:
10094 case OPCODE_CALL_STRICT:
10095 case OPCODE_CALL_SPARK:
10096 case OPCODE_CALL_LAZY:
10097 case OPCODE_CALL_CACHE:
10098 case OPCODE_CALL_SAVE: {
10099 get_two(ctx, &n_args, &n_ret);
10100 get_one(ctx, &fn_idx);
10101 jump_over_arguments_and_return:
10102 load_args;
10103 ctx->return_values = ctx->current_position;
10104 for (i_arg = 0; i_arg < n_ret; i_arg++) {
10105 #if ARG_MODE_N >= 3
10106 get_uint32(ctx);
10107 #else
10108 get_code(ctx);
10109 #endif
10110 get_code(ctx);
10111 }
10112 if (unlikely(profiling))
10113 goto unconditional_escape;
10114 if (code == OPCODE_CALL || code == OPCODE_CALL_STRICT) {
10115 g(gen_call(ctx, code, fn_idx));
10116 continue;
10117 }
10118 /*if (code == OPCODE_CALL_INDIRECT || code == OPCODE_CALL_INDIRECT_STRICT) {
10119 if (unlikely(!gen_call_indirect(ctx, code, slot_1, flags)))
10120 return false;
10121 continue;
10122 }*/
10123 goto unconditional_escape;
10124 }
10125 case OPCODE_CALL_INDIRECT:
10126 case OPCODE_CALL_INDIRECT_STRICT:
10127 case OPCODE_CALL_INDIRECT_SPARK:
10128 case OPCODE_CALL_INDIRECT_LAZY:
10129 case OPCODE_CALL_INDIRECT_CACHE:
10130 case OPCODE_CALL_INDIRECT_SAVE: {
10131 fn_idx = 0; /* avoid warning */
10132 get_two(ctx, &n_args, &n_ret);
10133 get_two(ctx, &slot_1, &flags);
10134 goto jump_over_arguments_and_return;
10135 }
10136 case OPCODE_RETURN: {
10137 n_args = da(ctx->fn,function)->n_return_values;
10138 load_args;
10139 if (unlikely(profiling))
10140 goto unconditional_escape;
10141 g(gen_return(ctx));
10142 continue;
10143 }
10144 case OPCODE_STRUCTURED: {
10145 init_args;
10146 get_two(ctx, &slot_1, &slot_2);
10147 do {
10148 struct code_arg a;
10149 get_two(ctx, &flags, &slot_r);
10150 get_one(ctx, &opt);
10151 a.slot = slot_r;
10152 a.flags = flags;
10153 a.type = opt;
10154 g(array_add_mayfail(struct code_arg, &ctx->args, &ctx->args_l, a, NULL, &ctx->err));
10155 } while (!(flags & OPCODE_STRUCTURED_FLAG_END));
10156 g(gen_structured(ctx, slot_1, slot_2));
10157 continue;
10158 }
10159 case OPCODE_RECORD_CREATE: {
10160 init_args;
10161 get_two(ctx, &slot_r, &n_args);
10162 for (i_arg = 0; i_arg < n_args; i_arg++) {
10163 struct code_arg a;
10164 get_two(ctx, &slot_1, &flags);
10165 a.slot = slot_1;
10166 a.flags = flags;
10167 a.type = 0;
10168 g(array_add_mayfail(struct code_arg, &ctx->args, &ctx->args_l, a, NULL, &ctx->err));
10169 }
10170 g(gen_record_create(ctx, slot_r));
10171 continue;
10172 }
10173 case OPCODE_RECORD_LOAD: {
10174 get_two(ctx, &slot_1, &opt);
10175 get_two(ctx, &slot_r, &flags);
10176 g(gen_record_load(ctx, slot_1, slot_r, opt, flags));
10177 continue;
10178 }
10179 case OPCODE_OPTION_CREATE_EMPTY_FLAT: {
10180 get_two(ctx, &slot_r, &opt);
10181 g(gen_option_create_empty_flat(ctx, opt, slot_r));
10182 continue;
10183 }
10184 case OPCODE_OPTION_CREATE_EMPTY: {
10185 get_two(ctx, &slot_r, &opt);
10186 g(gen_option_create_empty(ctx, opt, slot_r));
10187 continue;
10188 }
10189 case OPCODE_OPTION_CREATE: {
10190 get_two(ctx, &slot_r, &opt);
10191 get_two(ctx, &slot_1, &flags);
10192 g(gen_option_create(ctx, opt, slot_1, slot_r, flags));
10193 continue;
10194 }
10195 case OPCODE_OPTION_LOAD: {
10196 get_two(ctx, &slot_1, &opt);
10197 get_two(ctx, &slot_r, &flags);
10198 g(gen_option_load(ctx, slot_1, slot_r, opt, flags));
10199 continue;
10200 }
10201 case OPCODE_OPTION_TEST_FLAT: {
10202 get_two(ctx, &slot_1, &opt);
10203 get_one(ctx, &slot_r);
10204 g(gen_option_test_flat(ctx, slot_1, opt, slot_r));
10205 continue;
10206 }
10207 case OPCODE_OPTION_TEST: {
10208 get_two(ctx, &slot_1, &opt);
10209 get_one(ctx, &slot_r);
10210 g(gen_option_test(ctx, slot_1, opt, slot_r));
10211 continue;
10212 }
10213 case OPCODE_OPTION_ORD_FLAT: {
10214 get_two(ctx, &slot_1, &slot_r);
10215 g(gen_option_ord(ctx, slot_1, slot_r, true));
10216 continue;
10217 }
10218 case OPCODE_OPTION_ORD: {
10219 get_two(ctx, &slot_1, &slot_r);
10220 g(gen_option_ord(ctx, slot_1, slot_r, false));
10221 continue;
10222 }
10223 case OPCODE_ARRAY_CREATE: {
10224 init_args;
10225 get_two(ctx, &slot_r, &n_args);
10226 for (i_arg = 0; i_arg < n_args; i_arg++) {
10227 struct code_arg a;
10228 get_two(ctx, &slot_1, &flags);
10229 a.slot = slot_1;
10230 a.flags = flags;
10231 a.type = 0;
10232 g(array_add_mayfail(struct code_arg, &ctx->args, &ctx->args_l, a, NULL, &ctx->err));
10233 }
10234 g(gen_array_create(ctx, slot_r));
10235 continue;
10236 }
10237 case OPCODE_ARRAY_CREATE_EMPTY_FLAT: {
10238 get_two(ctx, &slot_r, &flags);
10239 g(gen_array_create_empty_flat(ctx, slot_r, flags));
10240 continue;
10241 }
10242 case OPCODE_ARRAY_CREATE_EMPTY: {
10243 get_one(ctx, &slot_r);
10244 g(gen_array_create_empty(ctx, slot_r));
10245 continue;
10246 }
10247 case OPCODE_ARRAY_FILL: {
10248 get_two(ctx, &slot_1, &flags);
10249 get_two(ctx, &slot_2, &slot_r);
10250 g(gen_array_fill(ctx, slot_1, flags, slot_2, slot_r));
10251 continue;
10252 }
10253 case OPCODE_ARRAY_STRING: {
10254 frame_t i;
10255 get_two(ctx, &slot_r, &i);
10256 g(gen_array_string(ctx, type_get_fixed(0, true)->tag, cast_ptr(uint8_t *, ctx->current_position), i, slot_r));
10257 ctx->current_position += (i + 1) >> 1;
10258 continue;
10259 }
10260 case OPCODE_ARRAY_UNICODE: {
10261 frame_t i;
10262 get_two(ctx, &slot_r, &i);
10263 g(gen_array_string(ctx, type_get_int(2)->tag, cast_ptr(uint8_t *, ctx->current_position), i, slot_r));
10264 ctx->current_position += i * 2;
10265 continue;
10266 }
10267 case OPCODE_ARRAY_LOAD: {
10268 get_two(ctx, &slot_1, &slot_2);
10269 get_two(ctx, &slot_r, &flags);
10270 g(gen_array_load(ctx, slot_1, slot_2, slot_r, flags));
10271 continue;
10272 }
10273 case OPCODE_ARRAY_LEN: {
10274 get_two(ctx, &slot_1, &slot_r);
10275 get_one(ctx, &flags);
10276 g(gen_array_len(ctx, slot_1, NO_FRAME_T, slot_r));
10277 continue;
10278 }
10279 case OPCODE_ARRAY_LEN_GREATER_THAN: {
10280 get_two(ctx, &slot_1, &slot_2);
10281 get_two(ctx, &slot_r, &flags);
10282 g(gen_array_len(ctx, slot_1, slot_2, slot_r));
10283 continue;
10284 }
10285 case OPCODE_ARRAY_SUB: {
10286 get_two(ctx, &slot_1, &slot_2);
10287 get_two(ctx, &slot_3, &slot_r);
10288 get_one(ctx, &flags);
10289 g(gen_array_sub(ctx, slot_1, slot_2, slot_3, slot_r, flags));
10290 continue;
10291 }
10292 case OPCODE_ARRAY_SKIP: {
10293 get_two(ctx, &slot_1, &slot_2);
10294 get_two(ctx, &slot_r, &flags);
10295 g(gen_array_skip(ctx, slot_1, slot_2, slot_r, flags));
10296 continue;
10297 }
10298 case OPCODE_ARRAY_APPEND: {
10299 get_two(ctx, &slot_r, &flags);
10300 get_two(ctx, &slot_1, &slot_2);
10301 g(gen_array_append(ctx, slot_1, slot_2, slot_r, flags));
10302 continue;
10303 }
10304 case OPCODE_ARRAY_APPEND_ONE_FLAT: {
10305 get_two(ctx, &slot_r, &flags);
10306 get_two(ctx, &slot_1, &slot_2);
10307 g(gen_array_append_one_flat(ctx, slot_1, slot_2, slot_r, flags));
10308 continue;
10309 }
10310 case OPCODE_ARRAY_APPEND_ONE: {
10311 get_two(ctx, &slot_r, &flags);
10312 get_two(ctx, &slot_1, &slot_2);
10313 g(gen_array_append_one(ctx, slot_1, slot_2, slot_r, flags));
10314 continue;
10315 }
10316 case OPCODE_ARRAY_FLATTEN: {
10317 get_two(ctx, &slot_r, &flags);
10318 get_one(ctx, &slot_1);
10319 goto unconditional_escape;
10320 }
10321 case OPCODE_IO: {
10322 get_two(ctx, &flags, &slot_1);
10323 get_two(ctx, &slot_2, &slot_3);
10324 g(gen_io(ctx, flags, slot_1, slot_2, slot_3));
10325 continue;
10326 }
10327 case OPCODE_INTERNAL_FUNCTION:
10328 case OPCODE_EXIT_THREAD:
10329 case OPCODE_UNREACHABLE: {
10330 goto unconditional_escape;
10331 }
10332 default: {
10333 #if 1
10334 /*if (getenv("DUMP") && !strcmp(da(ctx->fn,function)->function_name, getenv("DUMP")))*/
10335 warning("gen_function: %s: unknown opcode %04x, %s", da(ctx->fn,function)->function_name, *ctx->instr_start, decode_opcode(*ctx->instr_start, false));
10336 #endif
10337 return false;
10338 }
10339 }
10340 }
10341
10342 return true;
10343 }
10344
10345 static bool attr_w gen_entries(struct codegen_context *ctx)
10346 {
10347 size_t i;
10348 for (i = 0; i < ctx->n_entries; i++) {
10349 struct cg_entry *ce = &ctx->entries[i];
10350 if (ce->entry_label) {
10351 gen_insn(INSN_ENTRY, 0, 0, 0);
10352 gen_four(i);
10353
10354 g(gen_test_multiple(ctx, ce->variables, ce->n_variables, ce->nonflat_label));
10355
10356 gen_insn(INSN_JMP, 0, 0, 0);
10357 gen_four(ce->entry_label);
10358 }
10359 }
10360 return true;
10361 }
10362
10363 static bool attr_w gen_epilogues(struct codegen_context *ctx)
10364 {
10365 frame_t v;
10366 ip_t ip;
10367 uint32_t escape_label, nospill_label;
10368 escape_label = alloc_label(ctx);
10369 if (unlikely(!escape_label))
10370 return false;
10371 nospill_label = alloc_label(ctx);
10372 if (unlikely(!nospill_label))
10373 return false;
10374 #if defined(ARCH_PARISC)
10375 if (ctx->call_label) {
10376 gen_label(ctx->call_label);
10377 g(gen_call_millicode(ctx));
10378 }
10379 #endif
10380 if (ctx->reload_label) {
10381 gen_label(ctx->reload_label);
10382 gen_insn(INSN_MOV, i_size(OP_SIZE_ADDRESS), 0, 0);
10383 gen_one(R_FRAME);
10384 gen_one(R_RET0);
10385 g(gen_escape_arg(ctx, (ip_t)-1, escape_label));
10386 }
10387 gen_label(ctx->escape_nospill_label);
10388 g(gen_escape_arg(ctx, 0, nospill_label));
10389 for (ip = 0; ip < da(ctx->fn,function)->code_size; ip++) {
10390 if (ctx->escape_labels[ip]) {
10391 gen_label(ctx->escape_labels[ip]);
10392 g(gen_escape_arg(ctx, ip, escape_label));
10393 }
10394 }
10395 gen_label(escape_label);
10396 for (v = MIN_USEABLE_SLOT; v < function_n_variables(ctx->fn); v++) {
10397 if (ctx->registers[v] >= 0) {
10398 g(spill(ctx, v));
10399 }
10400 }
10401 gen_label(nospill_label);
10402 g(gen_escape(ctx));
10403 return true;
10404 }
10405
10406 static bool attr_w cgen_entry(struct codegen_context *ctx)
10407 {
10408 uint32_t entry_id = cget_four(ctx);
10409 ajla_assert_lo(entry_id < ctx->n_entries, (file_line, "cgen_entry: invalid entry %lx", (unsigned long)entry_id));
10410 ctx->entries[entry_id].entry_to_pos = ctx->mcode_size;
10411 return true;
10412 }
10413
10414 static bool attr_w cgen_label(struct codegen_context *ctx)
10415 {
10416 uint32_t label_id = cget_four(ctx);
10417 ctx->label_to_pos[label_id] = ctx->mcode_size;
10418 return true;
10419 }
10420
10421 static bool attr_w attr_unused cgen_trap(struct codegen_context *ctx, uint32_t label)
10422 {
10423 struct trap_record tr;
10424 tr.source_ip = ctx->mcode_size;
10425 tr.destination_ip = label;
10426 if (unlikely(!array_add_mayfail(struct trap_record, &ctx->trap_records, &ctx->trap_records_size, tr, NULL, &ctx->err)))
10427 return false;
10428 return true;
10429 }
10430
10431 static bool attr_w add_relocation(struct codegen_context *ctx, unsigned length, int offset, bool *known)
10432 {
10433 struct relocation rel;
10434 rel.label_id = cget_four(ctx);
10435 rel.length = length;
10436 rel.position = ctx->mcode_size;
10437 rel.jmp_instr = ctx->code_position - 8 - offset - ctx->code;
10438 if (unlikely(!array_add_mayfail(struct relocation, &ctx->reloc, &ctx->reloc_size, rel, NULL, &ctx->err)))
10439 return false;
10440 if (known)
10441 *known = ctx->label_to_pos[rel.label_id] != (size_t)-1;
10442 return true;
10443 }
10444
10445
10446 #if defined(ARCH_ALPHA)
10447 #include "c2-alpha.inc"
10448 #elif defined(ARCH_ARM32)
10449 #include "c2-arm.inc"
10450 #elif defined(ARCH_ARM64)
10451 #include "c2-arm64.inc"
10452 #elif defined(ARCH_IA64)
10453 #include "c2-ia64.inc"
10454 #elif defined(ARCH_LOONGARCH64)
10455 #include "c2-loong.inc"
10456 #elif defined(ARCH_MIPS)
10457 #include "c2-mips.inc"
10458 #elif defined(ARCH_PARISC)
10459 #include "c2-hppa.inc"
10460 #elif defined(ARCH_POWER)
10461 #include "c2-power.inc"
10462 #elif defined(ARCH_S390)
10463 #include "c2-s390.inc"
10464 #elif defined(ARCH_SPARC)
10465 #include "c2-sparc.inc"
10466 #elif defined(ARCH_RISCV64)
10467 #include "c2-riscv.inc"
10468 #elif defined(ARCH_X86)
10469 #include "c2-x86.inc"
10470 #endif
10471
10472
10473 static bool attr_w gen_mcode(struct codegen_context *ctx)
10474 {
10475 ctx->code_position = ctx->code;
10476
10477 while (ctx->code_position != ctx->code + ctx->code_size) {
10478 uint32_t insn;
10479 ajla_assert_lo(ctx->code_position < ctx->code + ctx->code_size, (file_line, "gen_mcode: ran out of code"));
10480 #ifdef DEBUG_INSNS
10481 insn = cget_four(ctx);
10482 debug("line: %u", insn);
10483 #endif
10484 insn = cget_four(ctx);
10485 g(cgen_insn(ctx, insn));
10486 }
10487
10488 return true;
10489 }
10490
10491 #define RELOCS_RETRY -1
10492 #define RELOCS_FAIL 0
10493 #define RELOCS_OK 1
10494
10495 static int8_t resolve_relocs(struct codegen_context *ctx)
10496 {
10497 size_t i;
10498 int8_t status = RELOCS_OK;
10499 for (i = 0; i < ctx->reloc_size; i++) {
10500 struct relocation *reloc = &ctx->reloc[i];
10501 if (!resolve_relocation(ctx, reloc)) {
10502 uint8_t *jmp_instr;
10503 uint32_t insn;
10504 uint32_t new_length;
10505 status = RELOCS_RETRY;
10506 if (unlikely(reloc->length + zero >= JMP_LIMIT))
10507 return RELOCS_FAIL;
10508 new_length = reloc->length + 1;
10509 jmp_instr = ctx->code + reloc->jmp_instr;
10510 insn = (uint32_t)jmp_instr[0] +
10511 ((uint32_t)jmp_instr[1] << 8) +
10512 ((uint32_t)jmp_instr[2] << 16) +
10513 ((uint32_t)jmp_instr[3] << 24);
10514 insn &= ~INSN_JUMP_SIZE;
10515 insn |= (uint32_t)new_length << INSN_JUMP_SIZE_SHIFT;
10516 jmp_instr[0] = insn;
10517 jmp_instr[1] = insn >> 8;
10518 jmp_instr[2] = insn >> 16;
10519 jmp_instr[3] = insn >> 24;
10520 }
10521 }
10522 return status;
10523 }
10524
10525 static void resolve_traps(struct codegen_context *ctx)
10526 {
10527 size_t i;
10528 for (i = 0; i < ctx->trap_records_size; i++) {
10529 struct trap_record *tr = &ctx->trap_records[i];
10530 tr->destination_ip = ctx->label_to_pos[tr->destination_ip];
10531 }
10532 }
10533
10534
10535 static bool attr_w codegen_map(struct codegen_context *ctx)
10536 {
10537 void *ptr;
10538 frame_t i;
10539 array_finish(uint8_t, &ctx->mcode, &ctx->mcode_size);
10540 ptr = os_code_map(ctx->mcode, ctx->mcode_size, &ctx->err);
10541 ctx->mcode = NULL;
10542 if (unlikely(!ptr)) {
10543 return false;
10544 }
10545 for (i = 0; i < ctx->n_entries; i++) {
10546 char *entry = cast_ptr(char *, ptr) + ctx->entries[i].entry_to_pos;
10547 da(ctx->codegen,codegen)->unoptimized_code[i] = entry;
10548 da(ctx->codegen,codegen)->n_entries++;
10549 }
10550 da(ctx->codegen,codegen)->unoptimized_code_base = ptr;
10551 da(ctx->codegen,codegen)->unoptimized_code_size = ctx->mcode_size;
10552
10553 return true;
10554 }
10555
10556
10557 void *codegen_fn(frame_s *fp, const code_t *ip, union internal_arg ia[])
10558 {
10559 struct codegen_context ctx_;
10560 struct codegen_context *ctx = &ctx_;
10561 frame_t i;
10562 int8_t rr;
10563 struct data *codegen;
10564 uint32_t l;
10565
10566 init_ctx(ctx);
10567 ctx->fn = ia[0].ptr;
10568
10569 #ifdef DEBUG_ENV
10570 if (getenv("CG") && strcmp(da(ctx->fn,function)->function_name, getenv("CG")))
10571 goto fail;
10572 #endif
10573
10574 ctx->local_directory = mem_alloc_array_mayfail(mem_calloc_mayfail, struct data **, 0, 0, da(ctx->fn,function)->local_directory_size, sizeof(struct data *), &ctx->err);
10575 if (unlikely(!ctx->local_directory))
10576 goto fail;
10577
10578 if (0) for (i = 0; i < da(ctx->fn,function)->local_directory_size; i++) {
10579 struct data *callee;
10580 pointer_t *ptr;
10581 ptr = da(ctx->fn,function)->local_directory[i];
10582 pointer_follow(ptr, false, callee, PF_SPARK, NULL, 0,
10583 SUBMIT_EX(ex_);
10584 goto next_one,
10585 goto next_one;
10586 );
10587 ctx->local_directory[i] = callee;
10588 next_one:;
10589 }
10590 for (i = 0; i < da(ctx->fn,function)->local_directory_size; i++) {
10591 struct data *callee;
10592 pointer_t *ptr;
10593 if (ctx->local_directory[i])
10594 continue;
10595 ptr = da(ctx->fn,function)->local_directory[i];
10596 pointer_follow(ptr, false, callee, PF_WAIT, fp, ip,
10597 done_ctx(ctx);
10598 return ex_,
10599 goto fail
10600 );
10601 ctx->local_directory[i] = callee;
10602 /*debug("processing call: %s -> %s", da(ctx->fn,function)->function_name, da(callee,function)->function_name);*/
10603 }
10604
10605 if (da(ctx->fn,function)->module_designator) {
10606 struct function_descriptor *sfd = save_find_function_descriptor(da(ctx->fn,function)->module_designator, da(ctx->fn,function)->function_designator);
10607 if (sfd && sfd->unoptimized_code_size) {
10608 codegen = data_alloc_flexible(codegen, unoptimized_code, sfd->n_entries, &ctx->err);
10609 if (unlikely(!codegen))
10610 goto fail;
10611 da(codegen,codegen)->unoptimized_code_base = sfd->unoptimized_code_base;
10612 da(codegen,codegen)->unoptimized_code_size = sfd->unoptimized_code_size;
10613 da(codegen,codegen)->function = ctx->fn;
10614 da(codegen,codegen)->is_saved = true;
10615 da(codegen,codegen)->n_entries = sfd->n_entries;
10616 da(codegen,codegen)->offsets = NULL;
10617 for (i = 0; i < sfd->n_entries; i++) {
10618 da(codegen,codegen)->unoptimized_code[i] = cast_ptr(char *, da(codegen,codegen)->unoptimized_code_base) + sfd->entries[i];
10619 /*debug("%s: %p + %lx -> %p", da(ctx->fn,function)->function_name, da(codegen,codegen)->unoptimized_code_base, sfd->entries[i], da(codegen,codegen)->unoptimized_code[i]);*/
10620 }
10621 #ifdef HAVE_CODEGEN_TRAPS
10622 da(codegen,codegen)->trap_records = sfd->trap_records;
10623 da(codegen,codegen)->trap_records_size = sfd->trap_records_size;
10624 data_trap_insert(codegen);
10625 #endif
10626 goto have_codegen;
10627 }
10628 }
10629
10630 /*debug("trying: %s", da(ctx->fn,function)->function_name);*/
10631 if (unlikely(!array_init_mayfail(uint8_t, &ctx->code, &ctx->code_size, &ctx->err)))
10632 goto fail;
10633
10634 ctx->code_labels = mem_alloc_array_mayfail(mem_calloc_mayfail, uint32_t *, 0, 0, da(ctx->fn,function)->code_size, sizeof(uint32_t), &ctx->err);
10635 if (unlikely(!ctx->code_labels))
10636 goto fail;
10637
10638 ctx->escape_labels = mem_alloc_array_mayfail(mem_calloc_mayfail, uint32_t *, 0, 0, da(ctx->fn,function)->code_size, sizeof(uint32_t), &ctx->err);
10639 if (unlikely(!ctx->escape_labels))
10640 goto fail;
10641
10642 ctx->flag_cache = mem_alloc_array_mayfail(mem_calloc_mayfail, int8_t *, 0, 0, function_n_variables(ctx->fn), sizeof(int8_t), &ctx->err);
10643 if (unlikely(!ctx->flag_cache))
10644 goto fail;
10645
10646 ctx->registers = mem_alloc_array_mayfail(mem_alloc_mayfail, short *, 0, 0, function_n_variables(ctx->fn), sizeof(short), &ctx->err);
10647 if (unlikely(!ctx->registers))
10648 goto fail;
10649
10650 if (unlikely(!array_init_mayfail(frame_t, &ctx->need_spill, &ctx->need_spill_l, &ctx->err)))
10651 goto fail;
10652
10653 if (unlikely(!gen_registers(ctx)))
10654 goto fail;
10655
10656 if (unlikely(!gen_function(ctx)))
10657 goto fail;
10658
10659 if (unlikely(!gen_entries(ctx)))
10660 goto fail;
10661
10662 if (unlikely(!gen_epilogues(ctx)))
10663 goto fail;
10664
10665 if (unlikely(!(ctx->label_id + 1)))
10666 goto fail;
10667 if (unlikely(!(ctx->label_to_pos = mem_alloc_array_mayfail(mem_alloc_mayfail, size_t *, 0, 0, ctx->label_id + 1, sizeof(size_t), &ctx->err))))
10668 goto fail;
10669
10670 again:
10671 for (l = 0; l < ctx->label_id + 1; l++)
10672 ctx->label_to_pos[l] = (size_t)-1;
10673
10674 if (unlikely(!array_init_mayfail(uint8_t, &ctx->mcode, &ctx->mcode_size, &ctx->err)))
10675 goto fail;
10676
10677 if (unlikely(!array_init_mayfail(struct relocation, &ctx->reloc, &ctx->reloc_size, &ctx->err)))
10678 goto fail;
10679
10680 if (unlikely(!array_init_mayfail(struct trap_record, &ctx->trap_records, &ctx->trap_records_size, &ctx->err)))
10681 goto fail;
10682
10683 #ifdef ARCH_CONTEXT
10684 init_arch_context(ctx);
10685 #endif
10686
10687 if (unlikely(!gen_mcode(ctx)))
10688 goto fail;
10689
10690 rr = resolve_relocs(ctx);
10691 if (unlikely(rr == RELOCS_FAIL)) {
10692 /*debug("relocation fail: %s", da(ctx->fn,function)->function_name);*/
10693 goto fail;
10694 }
10695 if (rr == RELOCS_RETRY) {
10696 mem_free(ctx->mcode);
10697 ctx->mcode = NULL;
10698 mem_free(ctx->reloc);
10699 ctx->reloc = NULL;
10700 mem_free(ctx->trap_records);
10701 ctx->trap_records = NULL;
10702 goto again;
10703 }
10704
10705 resolve_traps(ctx);
10706
10707 #ifdef DEBUG_ENV
10708 /*debug("success: %"PRIuMAX" %s", (uintmax_t)ctx->mcode_size, da(ctx->fn,function)->function_name);*/
10709 if (getenv("DUMP") && !strcmp(getenv("DUMP"), da(ctx->fn,function)->function_name)) {
10710 char *hex;
10711 size_t hexl;
10712 size_t i;
10713 size_t mcode_size = codegen_size + ctx->mcode_size;
10714 uint8_t *mcode = mem_alloc(uint8_t *, mcode_size);
10715 memcpy(mcode, codegen_ptr, codegen_size);
10716 memcpy(mcode + codegen_size, ctx->mcode, ctx->mcode_size);
10717 #if 0
10718 if (!os_write_atomic(".", "dump.asm", cast_ptr(const char *, mcode), mcode_size, &ctx->err)) {
10719 warning("dump failed");
10720 }
10721
10722 str_init(&hex, &hexl);
10723 for (i = 0; i < mcode_size; i++) {
10724 uint8_t a = mcode[i];
10725 if (a < 16)
10726 str_add_char(&hex, &hexl, '0');
10727 str_add_unsigned(&hex, &hexl, a, 16);
10728 }
10729 if (!os_write_atomic(".", "dump.hex", hex, hexl, &ctx->err)) {
10730 warning("dump failed");
10731 }
10732 mem_free(hex);
10733 #endif
10734
10735 str_init(&hex, &hexl);
10736 #if defined(ARCH_RISCV64)
10737 str_add_string(&hex, &hexl, " .attribute arch, \"rv64i2p1_m2p0_a2p1_f2p2_d2p2_c2p0_zicsr2p0_zifencei2p0_zba1p0_zbb1p0_zbc1p0_zbs1p0\"\n");
10738 #endif
10739 for (i = 0; i < mcode_size; i++) {
10740 uint8_t a = mcode[i];
10741 str_add_string(&hex, &hexl, " .byte 0x");
10742 if (a < 16)
10743 str_add_char(&hex, &hexl, '0');
10744 str_add_unsigned(&hex, &hexl, a, 16);
10745 str_add_char(&hex, &hexl, '\n');
10746 }
10747 if (!os_write_atomic(".", "dump.s", hex, hexl, &ctx->err)) {
10748 warning("dump failed");
10749 }
10750 mem_free(hex);
10751 mem_free(mcode);
10752 }
10753 #endif
10754
10755 ctx->codegen = data_alloc_flexible(codegen, unoptimized_code, ctx->n_entries, &ctx->err);
10756 if (unlikely(!ctx->codegen))
10757 goto fail;
10758 da(ctx->codegen,codegen)->function = ctx->fn;
10759 da(ctx->codegen,codegen)->is_saved = false;
10760 da(ctx->codegen,codegen)->n_entries = 0;
10761 da(ctx->codegen,codegen)->offsets = NULL;
10762
10763 if (unlikely(!codegen_map(ctx)))
10764 goto fail;
10765
10766 codegen = ctx->codegen;
10767 ctx->codegen = NULL;
10768
10769 #ifdef HAVE_CODEGEN_TRAPS
10770 da(codegen,codegen)->trap_records = ctx->trap_records;
10771 da(codegen,codegen)->trap_records_size = ctx->trap_records_size;
10772 ctx->trap_records = NULL;
10773 data_trap_insert(codegen);
10774 #endif
10775
10776 have_codegen:
10777 done_ctx(ctx);
10778 return function_return(fp, pointer_data(codegen));
10779
10780 fail:
10781 done_ctx(ctx);
10782 return function_return(fp, pointer_thunk(thunk_alloc_exception_error(error_ajla(EC_SYNC, AJLA_ERROR_NOT_SUPPORTED), NULL, NULL, NULL pass_file_line)));
10783 }
10784
10785 void codegen_free(struct data *codegen)
10786 {
10787 if (unlikely(da(codegen,codegen)->offsets != NULL))
10788 mem_free(da(codegen,codegen)->offsets);
10789 if (likely(da(codegen,codegen)->is_saved))
10790 return;
10791 #ifdef HAVE_CODEGEN_TRAPS
10792 mem_free(da(codegen,codegen)->trap_records);
10793 #endif
10794 os_code_unmap(da(codegen,codegen)->unoptimized_code_base, da(codegen,codegen)->unoptimized_code_size);
10795 }
10796
10797 #if defined(ARCH_IA64)
10798 static uintptr_t ia64_stub[2];
10799 #endif
10800 #if defined(ARCH_PARISC32) && defined(ARCH_PARISC_USE_STUBS)
10801 static uintptr_t parisc_stub[2];
10802 #endif
10803 #if defined(ARCH_PARISC64) && defined(ARCH_PARISC_USE_STUBS)
10804 static uintptr_t parisc_stub[4];
10805 #endif
10806 #if defined(ARCH_POWER) && defined(AIX_CALL)
10807 static uintptr_t ppc_stub[3];
10808 #endif
10809
10810 void name(codegen_init)(void)
10811 {
10812 struct codegen_context ctx_;
10813 struct codegen_context *ctx = &ctx_;
10814 void *ptr;
10815
10816 init_ctx(ctx);
10817 ctx->fn = NULL;
10818
10819 array_init(uint8_t, &ctx->code, &ctx->code_size);
10820
10821 if (unlikely(!gen_entry(ctx)))
10822 goto fail;
10823
10824 array_init(uint8_t, &ctx->mcode, &ctx->mcode_size);
10825
10826 #ifdef ARCH_CONTEXT
10827 init_arch_context(ctx);
10828 #endif
10829
10830 if (unlikely(!gen_mcode(ctx)))
10831 goto fail;
10832
10833 array_finish(uint8_t, &ctx->mcode, &ctx->mcode_size);
10834 ptr = os_code_map(ctx->mcode, ctx->mcode_size, NULL);
10835 codegen_ptr = ptr;
10836 codegen_size = ctx->mcode_size;
10837 ctx->mcode = NULL;
10838 #if defined(ARCH_IA64)
10839 ia64_stub[0] = ptr_to_num(ptr);
10840 ia64_stub[1] = 0;
10841 codegen_entry = cast_ptr(codegen_type, ia64_stub);
10842 #elif defined(ARCH_PARISC32) && defined(ARCH_PARISC_USE_STUBS)
10843 parisc_stub[0] = ptr_to_num(ptr);
10844 parisc_stub[1] = 0;
10845 codegen_entry = cast_ptr(codegen_type, cast_ptr(char *, parisc_stub) + 2);
10846 #elif defined(ARCH_PARISC64) && defined(ARCH_PARISC_USE_STUBS)
10847 parisc_stub[0] = 0;
10848 parisc_stub[1] = 0;
10849 parisc_stub[2] = ptr_to_num(ptr);
10850 parisc_stub[3] = 0;
10851 codegen_entry = cast_ptr(codegen_type, parisc_stub);
10852 #elif defined(ARCH_POWER) && defined(AIX_CALL)
10853 ppc_stub[0] = ptr_to_num(ptr);
10854 ppc_stub[1] = 0;
10855 ppc_stub[2] = 0;
10856 codegen_entry = cast_ptr(codegen_type, ppc_stub);
10857 #else
10858 codegen_entry = ptr;
10859 #endif
10860
10861 done_ctx(ctx);
10862
10863 return;
10864
10865 fail:
10866 fatal("couldn't compile global entry");
10867 }
10868
10869 void name(codegen_done)(void)
10870 {
10871 os_code_unmap(codegen_ptr, codegen_size);
10872 }
10873
10874 #else
10875
10876 void name(codegen_init)(void)
10877 {
10878 }
10879
10880 void name(codegen_done)(void)
10881 {
10882 }
10883
10884 #endif
10885
10886 #endif
Ajla - a purely functional programming language