| blob | 97545d82285f92884b8b6ef94c5fd4b7353e2738 |
1 /*
2 * Copyright (C) 2024 Mikulas Patocka
3 *
4 * This file is part of Ajla.
5 *
6 * Ajla is free software: you can redistribute it and/or modify it under the
7 * terms of the GNU General Public License as published by the Free Software
8 * Foundation, either version 3 of the License, or (at your option) any later
9 * version.
10 *
11 * Ajla is distributed in the hope that it will be useful, but WITHOUT ANY
12 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
13 * A PARTICULAR PURPOSE. See the GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License along with
16 * Ajla. If not, see <https://www.gnu.org/licenses/>.
17 */
19 static bool attr_w gen_jump(struct codegen_context *ctx, int32_t jmp_offset, unsigned op_size, unsigned cond, unsigned reg1, unsigned reg2);
21 static bool attr_w gen_alu_upcall(struct codegen_context *ctx, size_t upcall, frame_t slot_1, frame_t slot_2, frame_t slot_r, uint32_t label_ovf)
22 {
23 if (ctx->registers[slot_1] >= 0)
24 g(spill(ctx, slot_1));
25 if (slot_2 != NO_FRAME_T && ctx->registers[slot_2] >= 0)
26 g(spill(ctx, slot_2));
27 g(gen_upcall_start(ctx, slot_2 != NO_FRAME_T ? 3 : 2));
28 g(gen_frame_address(ctx, slot_1, 0, R_ARG0));
29 g(gen_upcall_argument(ctx, 0));
30 if (slot_2 != NO_FRAME_T) {
31 g(gen_frame_address(ctx, slot_2, 0, R_ARG1));
32 g(gen_upcall_argument(ctx, 1));
33 g(gen_frame_address(ctx, slot_r, 0, R_ARG2));
34 g(gen_upcall_argument(ctx, 2));
35 g(gen_upcall(ctx, upcall, 3));
36 } else {
37 g(gen_frame_address(ctx, slot_r, 0, R_ARG1));
38 g(gen_upcall_argument(ctx, 1));
39 g(gen_upcall(ctx, upcall, 2));
40 }
41 if (ctx->registers[slot_r] >= 0)
42 g(unspill(ctx, slot_r));
43 if (label_ovf)
44 g(gen_jmp_on_zero(ctx, OP_SIZE_1, R_RET0, COND_E, label_ovf));
45 return true;
46 }
48 static bool attr_w gen_alu_typed_upcall(struct codegen_context *ctx, size_t upcall, unsigned op_size, frame_t slot_1, frame_t slot_2, frame_t slot_r, uint32_t label_ovf)
49 {
50 upcall += op_size * sizeof(void (*)(void));
51 return gen_alu_upcall(ctx, upcall, slot_1, slot_2, slot_r, label_ovf);
52 }
54 #define MODE_FIXED 0
55 #define MODE_INT 1
56 #define MODE_REAL 2
57 #define MODE_BOOL 3
58 #define MODE_ARRAY_LEN_GT 4
60 static bool attr_w gen_alu(struct codegen_context *ctx, unsigned mode, unsigned op_size, unsigned op, uint32_t label_ovf, frame_t slot_1, frame_t slot_2, frame_t slot_r)
61 {
62 unsigned alu;
63 bool sgn, mod;
64 unsigned reg1, reg2, reg3, target;
65 switch (mode) {
66 case MODE_FIXED: switch (op) {
67 case OPCODE_FIXED_OP_add: alu = ALU_ADD; goto do_alu;
68 case OPCODE_FIXED_OP_subtract: alu = ALU_SUB; goto do_alu;
69 case OPCODE_FIXED_OP_multiply: goto do_multiply;
70 case OPCODE_FIXED_OP_divide:
71 case OPCODE_FIXED_OP_divide_alt1: sgn = true; mod = false; goto do_divide;
72 case OPCODE_FIXED_OP_udivide:
73 case OPCODE_FIXED_OP_udivide_alt1: sgn = false; mod = false; goto do_divide;
74 case OPCODE_FIXED_OP_modulo:
75 case OPCODE_FIXED_OP_modulo_alt1: sgn = true; mod = true; goto do_divide;
76 case OPCODE_FIXED_OP_umodulo:
77 case OPCODE_FIXED_OP_umodulo_alt1: sgn = false; mod = true; goto do_divide;
78 case OPCODE_FIXED_OP_power: return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_binary_power_int8_t), op_size, slot_1, slot_2, slot_r, 0);
79 case OPCODE_FIXED_OP_and: alu = ALU_AND; goto do_alu;
80 case OPCODE_FIXED_OP_or: alu = ALU_OR; goto do_alu;
81 case OPCODE_FIXED_OP_xor: alu = ALU_XOR; goto do_alu;
82 case OPCODE_FIXED_OP_shl: alu = ROT_SHL; goto do_shift;
83 case OPCODE_FIXED_OP_shr: alu = ROT_SAR; goto do_shift;
84 case OPCODE_FIXED_OP_ushr: alu = ROT_SHR; goto do_shift;
85 case OPCODE_FIXED_OP_rol: alu = ROT_ROL; goto do_shift;
86 case OPCODE_FIXED_OP_ror: alu = ROT_ROR; goto do_shift;
87 case OPCODE_FIXED_OP_bts: alu = BTX_BTS; goto do_bt;
88 case OPCODE_FIXED_OP_btr: alu = BTX_BTR; goto do_bt;
89 case OPCODE_FIXED_OP_btc: alu = BTX_BTC; goto do_bt;
90 case OPCODE_FIXED_OP_equal: alu = COND_E; goto do_compare;
91 case OPCODE_FIXED_OP_not_equal: alu = COND_NE; goto do_compare;
92 case OPCODE_FIXED_OP_less: alu = COND_L; goto do_compare;
93 case OPCODE_FIXED_OP_less_equal: alu = COND_LE; goto do_compare;
94 case OPCODE_FIXED_OP_uless: alu = COND_B; goto do_compare;
95 case OPCODE_FIXED_OP_uless_equal: alu = COND_BE; goto do_compare;
96 case OPCODE_FIXED_OP_bt: alu = BTX_BT; goto do_bt;
97 default: internal(file_line, "gen_alu: unsupported fixed operation %u", op);
98 }
99 case MODE_INT: switch (op) {
100 case OPCODE_INT_OP_add: alu = ALU_ADD; goto do_alu;
101 case OPCODE_INT_OP_subtract: alu = ALU_SUB; goto do_alu;
102 case OPCODE_INT_OP_multiply: goto do_multiply;
103 case OPCODE_INT_OP_divide:
104 case OPCODE_INT_OP_divide_alt1: sgn = true; mod = false; goto do_divide;
105 case OPCODE_INT_OP_modulo:
106 case OPCODE_INT_OP_modulo_alt1: sgn = true; mod = true; goto do_divide;
107 case OPCODE_INT_OP_power: return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, INT_binary_power_int8_t), op_size, slot_1, slot_2, slot_r, label_ovf);
108 case OPCODE_INT_OP_and: alu = ALU_AND; mode = MODE_FIXED; goto do_alu;
109 case OPCODE_INT_OP_or: alu = ALU_OR; mode = MODE_FIXED; goto do_alu;
110 case OPCODE_INT_OP_xor: alu = ALU_XOR; mode = MODE_FIXED; goto do_alu;
111 case OPCODE_INT_OP_shl: alu = ROT_SHL; goto do_shift;
112 case OPCODE_INT_OP_shr: alu = ROT_SAR; goto do_shift;
113 case OPCODE_INT_OP_bts: alu = BTX_BTS; goto do_bt;
114 case OPCODE_INT_OP_btr: alu = BTX_BTR; goto do_bt;
115 case OPCODE_INT_OP_btc: alu = BTX_BTC; goto do_bt;
116 case OPCODE_INT_OP_equal: alu = COND_E; goto do_compare;
117 case OPCODE_INT_OP_not_equal: alu = COND_NE; goto do_compare;
118 case OPCODE_INT_OP_less: alu = COND_L; goto do_compare;
119 case OPCODE_INT_OP_less_equal: alu = COND_LE; goto do_compare;
120 case OPCODE_INT_OP_bt: alu = BTX_BT; goto do_bt;
121 default: internal(file_line, "gen_alu: unsupported int operation %u", op);
122 }
123 case MODE_BOOL: switch (op) {
124 case OPCODE_BOOL_OP_and: alu = ALU_AND; mode = MODE_FIXED; goto do_alu;
125 case OPCODE_BOOL_OP_or: alu = ALU_OR; mode = MODE_FIXED; goto do_alu;
126 case OPCODE_BOOL_OP_equal: alu = COND_E; goto do_compare;
127 case OPCODE_BOOL_OP_not_equal: alu = ALU_XOR; mode = MODE_FIXED; goto do_alu;
128 case OPCODE_BOOL_OP_less: alu = COND_L; goto do_compare;
129 case OPCODE_BOOL_OP_less_equal: alu = COND_LE; goto do_compare;
130 default: internal(file_line, "gen_alu: unsupported bool operation %u", op);
131 }
132 }
133 internal(file_line, "gen_alu: unsupported mode %u", mode);
135 /*******
136 * ALU *
137 *******/
138 do_alu: {
139 size_t attr_unused offset;
140 uint8_t attr_unused long_imm;
141 unsigned first_flags;
142 unsigned second_flags;
143 unsigned second_alu;
144 unsigned attr_unused op_size_flags;
145 if (unlikely(op_size > OP_SIZE_NATIVE)) {
146 #if !defined(ARCH_X86) && !defined(ARCH_ARM) && !defined(ARCH_PARISC) && !defined(ARCH_POWER) && !defined(ARCH_SPARC32)
147 if (mode == MODE_FIXED) {
148 if (alu == ALU_ADD) {
149 g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_add_,TYPE_INT_MAX)), slot_1, slot_2, slot_r, 0));
150 return true;
151 } else if (alu == ALU_SUB) {
152 g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_subtract_,TYPE_INT_MAX)), slot_1, slot_2, slot_r, 0));
153 return true;
154 }
155 } else if (mode == MODE_INT) {
156 if (alu == ALU_ADD) {
157 g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(INT_binary_add_,TYPE_INT_MAX)), slot_1, slot_2, slot_r, label_ovf));
158 return true;
159 } else if (alu == ALU_SUB) {
160 g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(INT_binary_subtract_,TYPE_INT_MAX)), slot_1, slot_2, slot_r, label_ovf));
161 return true;
162 }
163 }
164 #endif
165 first_flags = alu == ALU_ADD || alu == ALU_SUB ? 2 : 0;
166 second_flags = mode == MODE_INT ? 1 : 0;
167 second_alu = alu == ALU_ADD ? ALU_ADC : alu == ALU_SUB ? ALU_SBB : alu;
168 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
169 #if defined(ARCH_X86)
170 g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, alu, first_flags, slot_2, lo_word(OP_SIZE_NATIVE), R_SCRATCH_1));
171 g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, second_alu, second_flags, slot_2, hi_word(OP_SIZE_NATIVE), R_SCRATCH_2));
172 #else
173 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_2, 0, R_SCRATCH_3, R_SCRATCH_4));
174 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, alu, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_3, first_flags));
175 #if defined(ARCH_PARISC)
176 if (mode == MODE_INT) {
177 gen_insn(INSN_ALU_FLAGS_TRAP, OP_SIZE_NATIVE, second_alu, ALU_WRITES_FLAGS(second_alu, false));
178 gen_one(R_SCRATCH_2);
179 gen_one(R_SCRATCH_2);
180 gen_one(R_SCRATCH_4);
181 gen_four(label_ovf);
182 } else
183 #endif
184 {
185 gen_insn(first_flags ? INSN_ALU_FLAGS : INSN_ALU, OP_SIZE_NATIVE, second_alu, second_flags | ALU_WRITES_FLAGS(second_alu, false));
186 gen_one(R_SCRATCH_2);
187 gen_one(R_SCRATCH_2);
188 gen_one(R_SCRATCH_4);
189 }
190 #endif
191 #if !defined(ARCH_PARISC)
192 if (mode == MODE_INT) {
193 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_O, 0);
194 gen_four(label_ovf);
195 }
196 #endif
197 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_1, R_SCRATCH_2));
198 return true;
199 }
201 if ((ARCH_HAS_FLAGS || ARCH_SUPPORTS_TRAPS) && slot_2 == slot_r && slot_1 != slot_2 && alu_is_commutative(alu)) {
202 frame_t x = slot_1;
203 slot_1 = slot_2;
204 slot_2 = x;
205 }
206 if ((ARCH_HAS_FLAGS || ARCH_SUPPORTS_TRAPS) && slot_1 == slot_r && (slot_1 != slot_2 || mode != MODE_INT) && i_size_cmp(op_size) == op_size + zero
207 #if defined(ARCH_POWER)
208 && op_size == OP_SIZE_NATIVE
209 #endif
210 ) {
211 struct cg_exit *ce;
212 unsigned undo_alu = alu == ALU_ADD ? ALU_SUB : ALU_ADD;
213 if (ctx->registers[slot_1] >= 0) {
214 unsigned reg1 = ctx->registers[slot_1];
215 if (ctx->registers[slot_2] >= 0) {
216 unsigned reg2 = ctx->registers[slot_2];
217 if (mode == MODE_INT && ARCH_SUPPORTS_TRAPS) {
218 gen_insn(INSN_ALU_TRAP, op_size, alu, ALU_WRITES_FLAGS(alu, false));
219 gen_one(reg1);
220 gen_one(reg1);
221 gen_one(reg2);
222 if (ARCH_TRAP_BEFORE) {
223 gen_four(label_ovf);
224 return true;
225 } else {
226 ce = alloc_undo_label(ctx);
227 if (unlikely(!ce))
228 return false;
229 gen_four(ce->undo_label);
230 goto do_undo_opcode;
231 }
232 }
233 g(gen_3address_alu(ctx, i_size(op_size), alu, reg1, reg1, reg2, mode == MODE_INT));
234 if (mode == MODE_INT) {
235 ce = alloc_undo_label(ctx);
236 if (unlikely(!ce))
237 return false;
238 gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
239 gen_four(ce->undo_label);
240 do_undo_opcode:
241 ce->undo_opcode = INSN_ALU + ARCH_PARTIAL_ALU(op_size);
242 ce->undo_op_size = i_size(op_size);
243 ce->undo_aux = undo_alu;
244 ce->undo_writes_flags = ALU_WRITES_FLAGS(undo_alu, false);
245 ce->undo_parameters[0] = reg1;
246 ce->undo_parameters[1] = reg1;
247 ce->undo_parameters[2] = reg2;
248 ce->undo_parameters_len = 3;
249 }
250 return true;
251 }
252 #if defined(ARCH_S390) || defined(ARCH_X86)
253 else {
254 size_t m;
255 int64_t offset = (size_t)slot_2 * slot_size;
256 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDR_OFFSET, i_size(op_size)));
257 gen_insn(INSN_ALU + ARCH_PARTIAL_ALU(op_size), i_size(op_size), alu, 1);
258 gen_one(reg1);
259 gen_one(reg1);
260 gen_address_offset();
261 if (mode == MODE_INT) {
262 ce = alloc_undo_label(ctx);
263 if (unlikely(!ce))
264 return false;
265 ce->undo_opcode = INSN_ALU + ARCH_PARTIAL_ALU(op_size);
266 ce->undo_op_size = i_size(op_size);
267 ce->undo_aux = undo_alu;
268 ce->undo_writes_flags = ARCH_HAS_FLAGS;
269 m = mark_params(ctx);
270 gen_one(reg1);
271 gen_one(reg1);
272 gen_address_offset();
273 copy_params(ctx, ce, m);
274 gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
275 gen_four(ce->undo_label);
276 }
277 return true;
278 }
279 #endif
280 }
281 #if defined(ARCH_X86)
282 else {
283 unsigned reg2;
284 size_t m;
285 int64_t offset = (size_t)slot_1 * slot_size;
286 g(gen_frame_get(ctx, op_size, garbage, slot_2, 0, R_SCRATCH_1, ®2));
287 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDR_OFFSET, i_size(op_size)));
288 gen_insn(INSN_ALU + ARCH_PARTIAL_ALU(op_size), i_size(op_size), alu, 1);
289 gen_address_offset();
290 gen_address_offset();
291 gen_one(reg2);
292 if (mode == MODE_INT) {
293 ce = alloc_undo_label(ctx);
294 if (unlikely(!ce))
295 return false;
296 ce->undo_opcode = INSN_ALU + ARCH_PARTIAL_ALU(op_size);
297 ce->undo_op_size = i_size(op_size);
298 ce->undo_aux = undo_alu;
299 ce->undo_writes_flags = ARCH_HAS_FLAGS;
300 m = mark_params(ctx);
301 gen_address_offset();
302 gen_address_offset();
303 gen_one(reg2);
304 copy_params(ctx, ce, m);
305 gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
306 gen_four(ce->undo_label);
307 }
308 return true;
309 }
310 #endif
311 }
313 #if defined(ARCH_X86)
314 if (1)
315 #elif defined(ARCH_S390)
316 if (op_size >= OP_SIZE_4)
317 #else
318 if (0)
319 #endif
320 {
321 if (mode == MODE_INT) {
322 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
323 } else {
324 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, slot_2, R_SCRATCH_1);
325 }
326 g(gen_frame_load(ctx, op_size, garbage, slot_1, 0, target));
327 g(gen_frame_load_op(ctx, op_size, garbage, alu, mode == MODE_INT, slot_2, 0, target));
328 goto check_ovf_store;
329 }
330 op_size_flags = !ARCH_HAS_FLAGS && !ARCH_SUPPORTS_TRAPS ? OP_SIZE_NATIVE : OP_SIZE_4;
331 #if defined(ARCH_POWER)
332 op_size_flags = OP_SIZE_NATIVE;
333 #endif
334 g(gen_frame_get(ctx, op_size, mode == MODE_INT && (op_size < op_size_flags || ARCH_SUPPORTS_TRAPS) ? sign_x : garbage, slot_1, 0, R_SCRATCH_1, ®1));
335 g(gen_frame_get(ctx, op_size, mode == MODE_INT && (op_size < op_size_flags || ARCH_SUPPORTS_TRAPS) ? sign_x : garbage, slot_2, 0, R_SCRATCH_2, ®2));
336 #if !ARCH_HAS_FLAGS
337 if (mode == MODE_INT && op_size >= OP_SIZE_4) {
338 if (ARCH_SUPPORTS_TRAPS) {
339 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
340 gen_insn(INSN_ALU_TRAP, op_size, alu, ALU_WRITES_FLAGS(alu, false));
341 gen_one(target);
342 gen_one(reg1);
343 gen_one(reg2);
344 gen_four(label_ovf);
345 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
346 return true;
347 }
348 if (op_size >= OP_SIZE_NATIVE) {
349 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_3);
350 g(gen_3address_alu(ctx, i_size(op_size), alu, target, reg1, reg2, 0));
351 #if defined(ARCH_IA64)
352 g(gen_3address_alu(ctx, i_size(op_size), ALU_XOR, R_SCRATCH_1, reg1, reg2, 0));
353 g(gen_3address_alu(ctx, i_size(op_size), ALU_XOR, R_SCRATCH_2, reg2, target, 0));
354 if (alu == ALU_ADD) {
355 g(gen_3address_alu(ctx, i_size(op_size), ALU_ANDN, R_SCRATCH_1, R_SCRATCH_2, R_SCRATCH_1, 0));
356 } else {
357 g(gen_3address_alu(ctx, i_size(op_size), ALU_ANDN, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_2, 0));
358 }
359 g(gen_cmp_test_jmp(ctx, INSN_TEST, i_size(op_size), R_SCRATCH_1, R_SCRATCH_1, COND_S, label_ovf));
360 #else
361 gen_insn(INSN_CMP_DEST_REG, i_size(op_size), COND_L, 0);
362 gen_one(R_SCRATCH_1);
363 if (alu == ALU_ADD) {
364 gen_one(target);
365 gen_one(reg1);
366 } else {
367 gen_one(reg1);
368 gen_one(target);
369 }
371 g(gen_imm(ctx, 0, IMM_PURPOSE_CMP, i_size(op_size)));
372 gen_insn(INSN_CMP_DEST_REG, i_size(op_size), COND_L, 0);
373 gen_one(R_SCRATCH_2);
374 gen_one(reg2);
375 gen_imm_offset();
377 g(gen_cmp_test_jmp(ctx, INSN_CMP, i_size(op_size), R_SCRATCH_1, R_SCRATCH_2, COND_NE, label_ovf));
378 #endif
379 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
380 return true;
381 }
382 }
383 #endif
384 if (mode == MODE_INT) {
385 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
386 } else if (!ARCH_IS_3ADDRESS(alu, mode == MODE_INT && op_size >= op_size_flags) && !alu_is_commutative(alu)) {
387 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, slot_2, R_SCRATCH_1);
388 } else {
389 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
390 }
391 g(gen_3address_alu(ctx, i_size(op_size), alu, target, reg1, reg2, mode == MODE_INT && op_size >= op_size_flags));
393 if (mode == MODE_INT && unlikely(op_size < op_size_flags)) {
394 g(gen_cmp_extended(ctx, op_size_flags, op_size, target, R_SCRATCH_2, label_ovf));
395 } else
396 check_ovf_store:
397 if (mode == MODE_INT) {
398 gen_insn(INSN_JMP_COND, op_size, COND_O, 0);
399 gen_four(label_ovf);
400 }
401 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
402 return true;
403 }
405 /************
406 * MULTIPLY *
407 ************/
408 do_multiply: {
409 size_t attr_unused offset;
410 uint8_t attr_unused long_imm;
411 if (unlikely(op_size > OP_SIZE_NATIVE) || unlikely(!ARCH_HAS_MUL)) {
412 if (mode == MODE_INT) {
413 g(gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, INT_binary_multiply_int8_t), op_size, slot_1, slot_2, slot_r, label_ovf));
414 return true;
415 }
416 #if defined(ARCH_X86)
417 g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_1, hi_word(OP_SIZE_NATIVE), R_CX));
418 g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_2, hi_word(OP_SIZE_NATIVE), R_AX));
419 g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, ALU_MUL, true, slot_2, lo_word(OP_SIZE_NATIVE), R_CX));
420 g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, ALU_MUL, true, slot_1, lo_word(OP_SIZE_NATIVE), R_AX));
421 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_ADD, R_CX, R_CX, R_AX, 0));
422 g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_2, lo_word(OP_SIZE_NATIVE), R_AX));
424 offset = (size_t)slot_1 * slot_size + lo_word(OP_SIZE_NATIVE);
425 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
426 gen_insn(INSN_MUL_L, OP_SIZE_NATIVE, 0, 1);
427 gen_one(R_AX);
428 gen_one(R_DX);
429 gen_one(R_AX);
430 gen_address_offset();
432 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_ADD, R_DX, R_DX, R_CX, 0));
434 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_AX, R_DX));
436 return true;
437 #elif defined(ARCH_ARM32)
438 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
439 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_2, 0, R_SCRATCH_3, R_SCRATCH_4));
441 g(gen_mov(ctx, OP_SIZE_NATIVE, R_SCRATCH_NA_1, R_SCRATCH_1));
443 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_MUL, R_SCRATCH_4, R_SCRATCH_1, R_SCRATCH_4, 0));
445 gen_insn(INSN_MADD, OP_SIZE_NATIVE, 0, 0);
446 gen_one(R_SCRATCH_2);
447 gen_one(R_SCRATCH_3);
448 gen_one(R_SCRATCH_2);
449 gen_one(R_SCRATCH_4);
451 gen_insn(INSN_MUL_L, OP_SIZE_NATIVE, 0, 0);
452 gen_one(R_SCRATCH_1);
453 gen_one(R_SCRATCH_4);
454 gen_one(R_SCRATCH_NA_1);
455 gen_one(R_SCRATCH_3);
457 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_ADD, R_SCRATCH_2, R_SCRATCH_2, R_SCRATCH_4, 0));
459 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_1, R_SCRATCH_2));
461 return true;
462 #elif defined(ARCH_ARM64)
463 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
464 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_2, 0, R_SCRATCH_3, R_SCRATCH_4));
466 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_UMULH, R_SCRATCH_NA_1, R_SCRATCH_1, R_SCRATCH_3, 0));
468 gen_insn(INSN_MADD, OP_SIZE_NATIVE, 0, 0);
469 gen_one(R_SCRATCH_NA_1);
470 gen_one(R_SCRATCH_2);
471 gen_one(R_SCRATCH_3);
472 gen_one(R_SCRATCH_NA_1);
474 gen_insn(INSN_MADD, OP_SIZE_NATIVE, 0, 0);
475 gen_one(R_SCRATCH_2);
476 gen_one(R_SCRATCH_1);
477 gen_one(R_SCRATCH_4);
478 gen_one(R_SCRATCH_NA_1);
480 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_MUL, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_3, 0));
482 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_1, R_SCRATCH_2));
484 return true;
485 #else
486 g(gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_binary_multiply_int8_t), op_size, slot_1, slot_2, slot_r, 0));
487 return true;
488 #endif
489 }
491 #if defined(ARCH_X86)
492 if (mode == MODE_INT) {
493 if (op_size != OP_SIZE_1 && slot_r == slot_1 && ctx->registers[slot_1] >= 0) {
494 struct cg_exit *ce;
495 target = ctx->registers[slot_1];
496 g(gen_mov(ctx, op_size, R_SCRATCH_1, target));
497 g(gen_frame_load_op(ctx, op_size, garbage, ALU_MUL, mode == MODE_INT, slot_2, 0, target));
498 ce = alloc_undo_label(ctx);
499 if (unlikely(!ce))
500 return false;
501 ce->undo_opcode = INSN_MOV;
502 ce->undo_op_size = op_size;
503 ce->undo_aux = 0;
504 ce->undo_writes_flags = 0;
505 ce->undo_parameters[0] = target;
506 ce->undo_parameters[1] = R_SCRATCH_1;
507 ce->undo_parameters_len = 2;
508 gen_insn(INSN_JMP_COND, op_size, COND_O, 0);
509 gen_four(ce->undo_label);
510 return true;
511 }
512 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
513 } else {
514 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, slot_2, R_SCRATCH_1);
515 }
516 if (op_size == OP_SIZE_1)
517 target = R_SCRATCH_1;
518 g(gen_frame_load(ctx, op_size, garbage, slot_1, 0, target));
519 g(gen_frame_load_op(ctx, op_size, garbage, ALU_MUL, mode == MODE_INT, slot_2, 0, target));
520 if (mode == MODE_INT) {
521 gen_insn(INSN_JMP_COND, op_size, COND_O, 0);
522 gen_four(label_ovf);
523 }
524 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
525 return true;
526 #endif
527 #if defined(ARCH_ALPHA)
528 if (mode == MODE_INT && op_size >= OP_SIZE_4 && ARCH_SUPPORTS_TRAPS) {
529 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
530 g(gen_frame_get(ctx, op_size, garbage, slot_1, 0, R_SCRATCH_1, ®1));
531 g(gen_frame_get(ctx, op_size, garbage, slot_2, 0, R_SCRATCH_2, ®2));
533 gen_insn(INSN_ALU_TRAP, op_size, ALU_MUL, ALU_WRITES_FLAGS(ALU_MUL, false));
534 gen_one(target);
535 gen_one(reg1);
536 gen_one(reg2);
537 gen_four(label_ovf);
538 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
540 return true;
541 }
542 #endif
543 #if defined(ARCH_ARM32)
544 if (mode == MODE_INT && op_size == OP_SIZE_4) {
545 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_3);
546 g(gen_frame_get(ctx, op_size, garbage, slot_1, 0, R_SCRATCH_1, ®1));
547 g(gen_frame_get(ctx, op_size, garbage, slot_2, 0, R_SCRATCH_2, ®2));
549 gen_insn(INSN_MUL_L, OP_SIZE_NATIVE, 0, 0);
550 gen_one(target);
551 gen_one(R_SCRATCH_4);
552 gen_one(reg1);
553 gen_one(reg2);
555 gen_insn(INSN_CMP, OP_SIZE_NATIVE, 0, 1);
556 gen_one(R_SCRATCH_4);
557 gen_one(ARG_SHIFTED_REGISTER);
558 gen_one(ARG_SHIFT_ASR | 0x1f);
559 gen_one(target);
561 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_NE, 0);
562 gen_four(label_ovf);
564 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
566 return true;
567 }
568 #endif
569 #if defined(ARCH_ARM64)
570 if (mode == MODE_INT && op_size == OP_SIZE_4) {
571 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
572 g(gen_frame_get(ctx, op_size, op_size < OP_SIZE_4 ? sign_x : garbage, slot_1, 0, R_SCRATCH_1, ®1));
573 g(gen_frame_get(ctx, op_size, op_size < OP_SIZE_4 ? sign_x : garbage, slot_2, 0, R_SCRATCH_2, ®2));
574 gen_insn(INSN_ALU, OP_SIZE_8, ALU_MUL, ALU_WRITES_FLAGS(ALU_MUL, false));
575 gen_one(target);
576 gen_one(ARG_EXTENDED_REGISTER);
577 gen_one(ARG_EXTEND_SXTW);
578 gen_one(reg1);
579 gen_one(ARG_EXTENDED_REGISTER);
580 gen_one(ARG_EXTEND_SXTW);
581 gen_one(reg2);
583 gen_insn(INSN_CMP, OP_SIZE_8, 0, 1);
584 gen_one(target);
585 gen_one(ARG_EXTENDED_REGISTER);
586 gen_one(ARG_EXTEND_SXTW);
587 gen_one(target);
589 gen_insn(INSN_JMP_COND, OP_SIZE_8, COND_NE, 0);
590 gen_four(label_ovf);
592 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
594 return true;
595 }
596 if (mode == MODE_INT && op_size == OP_SIZE_8) {
597 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
598 g(gen_frame_get(ctx, op_size, garbage, slot_1, 0, R_SCRATCH_1, ®1));
599 g(gen_frame_get(ctx, op_size, garbage, slot_2, 0, R_SCRATCH_2, ®2));
600 g(gen_3address_alu(ctx, OP_SIZE_8, ALU_SMULH, R_SCRATCH_3, reg1, reg2, 0));
602 g(gen_3address_alu(ctx, OP_SIZE_8, ALU_MUL, target, reg1, reg2, 0));
604 gen_insn(INSN_CMP, OP_SIZE_8, 0, 1);
605 gen_one(R_SCRATCH_3);
606 gen_one(ARG_SHIFTED_REGISTER);
607 gen_one(ARG_SHIFT_ASR | 0x3f);
608 gen_one(target);
610 gen_insn(INSN_JMP_COND, OP_SIZE_8, COND_NE, 0);
611 gen_four(label_ovf);
613 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
615 return true;
616 }
617 #endif
618 #if defined(ARCH_POWER)
619 if (mode == MODE_INT && op_size >= OP_SIZE_4) {
620 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
621 g(gen_frame_get(ctx, op_size, garbage, slot_1, 0, R_SCRATCH_1, ®1));
622 g(gen_frame_get(ctx, op_size, garbage, slot_2, 0, R_SCRATCH_2, ®2));
624 g(gen_3address_alu(ctx, op_size, ALU_MUL, target, reg1, reg2, 1));
626 gen_insn(INSN_JMP_COND, op_size, COND_O, 0);
627 gen_four(label_ovf);
629 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
631 return true;
632 }
633 #endif
634 #if defined(ARCH_LOONGARCH64) || (defined(ARCH_MIPS) && MIPS_R6) || defined(ARCH_RISCV64)
635 if (mode == MODE_INT && op_size == OP_SIZE_NATIVE) {
636 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
637 g(gen_frame_get(ctx, op_size, garbage, slot_1, 0, R_SCRATCH_1, ®1));
638 g(gen_frame_get(ctx, op_size, garbage, slot_2, 0, R_SCRATCH_2, ®2));
640 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_SMULH, R_SCRATCH_3, reg1, reg2, 0));
642 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_MUL, target, reg1, reg2, 0));
644 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_4, target, (8U << OP_SIZE_NATIVE) - 1, 0));
646 g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, R_SCRATCH_3, R_SCRATCH_4, COND_NE, label_ovf));
648 g(gen_frame_store(ctx, OP_SIZE_NATIVE, slot_r, 0, target));
650 return true;
651 }
652 #endif
653 #if defined(ARCH_S390)
654 if (mode == MODE_INT && op_size >= OP_SIZE_4 && likely(cpu_test_feature(CPU_FEATURE_misc_insn_ext_2))) {
655 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
656 g(gen_frame_load(ctx, op_size, sign_x, slot_1, 0, target));
657 g(gen_frame_load_op(ctx, op_size, sign_x, ALU_MUL, 1, slot_2, 0, target));
659 gen_insn(INSN_JMP_COND, op_size, COND_O, 0);
660 gen_four(label_ovf);
662 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
663 return true;
664 }
665 #endif
666 #if (defined(ARCH_MIPS) && !MIPS_R6) || defined(ARCH_S390)
667 #if defined(ARCH_MIPS)
668 if (mode == MODE_INT && op_size >= OP_SIZE_4)
669 #endif
670 #if defined(ARCH_S390)
671 if (mode == MODE_INT && op_size == OP_SIZE_4)
672 #endif
673 {
674 #if defined(ARCH_S390)
675 target = R_SCRATCH_1;
676 #else
677 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
678 #endif
679 g(gen_frame_get(ctx, op_size, sign_x, slot_1, 0, R_SCRATCH_1, ®1));
680 g(gen_frame_get(ctx, op_size, sign_x, slot_2, 0, R_SCRATCH_3, ®2));
682 gen_insn(INSN_MUL_L, op_size, 0, 0);
683 gen_one(target);
684 gen_one(R_SCRATCH_2);
685 gen_one(reg1);
686 gen_one(reg2);
688 g(gen_3address_rot_imm(ctx, op_size, ROT_SAR, R_SCRATCH_4, target, (8U << op_size) - 1, false));
690 g(gen_cmp_test_jmp(ctx, INSN_CMP, op_size, R_SCRATCH_2, R_SCRATCH_4, COND_NE, label_ovf));
692 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
693 return true;
694 }
695 #endif
696 if (mode == MODE_INT && op_size == OP_SIZE_NATIVE) {
697 g(gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, INT_binary_multiply_int8_t), op_size, slot_1, slot_2, slot_r, label_ovf));
698 return true;
699 }
701 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
702 if (op_size < OP_SIZE_NATIVE && mode == MODE_INT) {
703 g(gen_frame_get(ctx, op_size, sign_x, slot_1, 0, R_SCRATCH_1, ®1));
704 g(gen_frame_get(ctx, op_size, sign_x, slot_2, 0, R_SCRATCH_2, ®2));
706 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_MUL, target, reg1, reg2, 0));
707 } else {
708 g(gen_frame_load(ctx, op_size, sign_x, slot_1, 0, target));
709 g(gen_frame_load_op(ctx, op_size, sign_x, ALU_MUL, 0, slot_2, 0, target));
710 }
712 if (mode == MODE_INT) {
713 g(gen_cmp_extended(ctx, OP_SIZE_NATIVE, op_size, target, R_SCRATCH_2, label_ovf));
714 }
716 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
718 return true;
719 }
721 /**********
722 * DIVIDE *
723 **********/
724 do_divide: {
725 uint32_t attr_unused label_skip = 0; /* avoid warning */
726 uint32_t attr_unused label_skip2 = 0; /* avoid warning */
727 uint32_t attr_unused label_end = 0; /* avoid warning */
728 uint32_t attr_unused label_div_0 = 0; /* avoid warning */
729 unsigned attr_unused divide_alu = 0; /* avoid warning */
730 bool attr_unused have_mod = false;
731 bool attr_unused force_sx = false;
732 unsigned attr_unused div_op_size = i_size(op_size);
733 if (unlikely(op_size > OP_SIZE_NATIVE) || unlikely(!ARCH_HAS_DIV)
734 #if defined(ARCH_S390)
735 || !(Z || (op_size <= OP_SIZE_4 && sgn))
736 #endif
737 ) {
738 size_t upcall;
739 if (mode == MODE_INT) {
740 upcall = !mod ? offsetof(struct cg_upcall_vector_s, INT_binary_divide_int8_t) : offsetof(struct cg_upcall_vector_s, INT_binary_modulo_int8_t);
741 } else if (sgn) {
742 upcall = !mod ? offsetof(struct cg_upcall_vector_s, FIXED_binary_divide_int8_t) : offsetof(struct cg_upcall_vector_s, FIXED_binary_modulo_int8_t);
743 } else {
744 upcall = !mod ? offsetof(struct cg_upcall_vector_s, FIXED_binary_udivide_int8_t) : offsetof(struct cg_upcall_vector_s, FIXED_binary_umodulo_int8_t);
745 }
746 g(gen_alu_typed_upcall(ctx, upcall, op_size, slot_1, slot_2, slot_r, mode == MODE_INT ? label_ovf : 0));
747 return true;
748 }
749 #if defined(ARCH_X86) || defined(ARCH_S390)
750 if (mode == MODE_FIXED) {
751 label_skip = alloc_label(ctx);
752 if (unlikely(!label_skip))
753 return false;
754 label_end = alloc_label(ctx);
755 if (unlikely(!label_end))
756 return false;
757 if (sgn) {
758 label_skip2 = alloc_label(ctx);
759 if (unlikely(!label_skip2))
760 return false;
761 }
762 }
763 #if defined(ARCH_X86)
764 if (R_SCRATCH_1 != R_AX || R_SCRATCH_2 != R_DX || R_SCRATCH_3 != R_CX)
765 internal(file_line, "gen_alu: bad scratch registers");
766 #endif
767 g(gen_frame_load(ctx, op_size, sgn ? sign_x : zero_x, slot_1, 0, R_SCRATCH_1));
768 g(gen_frame_load(ctx, op_size, sgn ? sign_x : zero_x, slot_2, 0, R_SCRATCH_3));
770 g(gen_jmp_on_zero(ctx, i_size(op_size), R_SCRATCH_3, COND_E, mode == MODE_INT ? label_ovf : label_skip));
772 if (sgn) {
773 uint64_t val;
774 uint32_t label_not_minus_1;
775 label_not_minus_1 = alloc_label(ctx);
776 if (unlikely(!label_not_minus_1))
777 return false;
779 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, op_size, R_SCRATCH_3, -1, COND_NE, label_not_minus_1));
781 val = -(uint64_t)0x80 << (((1 << op_size) - 1) * 8);
782 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, op_size, R_SCRATCH_1, val, COND_E, mode == MODE_INT ? label_ovf : label_skip2));
784 gen_label(label_not_minus_1);
785 }
787 #if defined(ARCH_X86)
788 if (op_size >= OP_SIZE_2) {
789 if (sgn) {
790 gen_insn(INSN_CWD + ARCH_PARTIAL_ALU(op_size), op_size, 0, 0);
791 gen_one(R_SCRATCH_2);
792 gen_one(R_SCRATCH_1);
793 if (op_size == OP_SIZE_2)
794 gen_one(R_SCRATCH_2);
795 } else {
796 g(gen_3address_alu(ctx, OP_SIZE_4, ALU_XOR, R_SCRATCH_2, R_SCRATCH_2, R_SCRATCH_2, 0));
797 }
798 }
799 gen_insn(INSN_DIV_L, op_size, sgn, 1);
800 gen_one(R_SCRATCH_1);
801 gen_one(i_size(op_size) == OP_SIZE_1 ? R_SCRATCH_1 : R_SCRATCH_2);
802 gen_one(R_SCRATCH_1);
803 gen_one(i_size(op_size) == OP_SIZE_1 ? R_SCRATCH_1 : R_SCRATCH_2);
804 gen_one(R_SCRATCH_3);
805 #else
806 if (!sgn) {
807 g(gen_load_constant(ctx, R_SCRATCH_2, 0));
808 } else if (op_size <= OP_SIZE_4) {
809 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_2, R_SCRATCH_1, (1U << (OP_SIZE_NATIVE + 3)) - 1, false));
810 }
811 gen_insn(INSN_DIV_L, i_size(op_size), sgn, 1);
812 gen_one(R_SCRATCH_2);
813 gen_one(R_SCRATCH_1);
814 gen_one(R_SCRATCH_2);
815 gen_one(R_SCRATCH_1);
816 gen_one(R_SCRATCH_3);
817 #endif
818 if (mod && i_size(op_size) == OP_SIZE_1) {
819 g(gen_3address_rot_imm(ctx, OP_SIZE_2, ROT_SHR, R_SCRATCH_1, R_SCRATCH_1, 8, 0));
820 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
821 } else if (mod) {
822 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_2));
823 } else {
824 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
825 }
826 if (mode == MODE_FIXED) {
827 gen_insn(INSN_JMP, 0, 0, 0);
828 gen_four(label_end);
830 if (sgn) {
831 gen_label(label_skip2);
833 if (mod)
834 g(gen_frame_clear(ctx, op_size, slot_r));
835 else
836 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
838 gen_insn(INSN_JMP, 0, 0, 0);
839 gen_four(label_end);
840 }
842 gen_label(label_skip);
843 if (!mod)
844 g(gen_frame_clear(ctx, op_size, slot_r));
845 else
846 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
847 gen_label(label_end);
848 }
849 return true;
850 #else
851 #if defined(ARCH_MIPS)
852 have_mod = true;
853 div_op_size = maximum(op_size, OP_SIZE_4);
854 if (op_size == OP_SIZE_4)
855 force_sx = true;
856 #endif
857 #if defined(ARCH_POWER)
858 have_mod = cpu_test_feature(CPU_FEATURE_v30);
859 div_op_size = maximum(op_size, OP_SIZE_4);
860 #endif
861 #if defined(ARCH_LOONGARCH64) || defined(ARCH_RISCV64)
862 have_mod = true;
863 div_op_size = maximum(op_size, OP_SIZE_4);
864 #endif
865 label_end = alloc_label(ctx);
866 if (unlikely(!label_end))
867 return false;
869 g(gen_frame_get(ctx, op_size, (sgn && op_size < i_size(op_size)) || force_sx ? sign_x : zero_x, slot_1, 0, R_SCRATCH_1, ®1));
870 g(gen_frame_get(ctx, op_size, (sgn && op_size < i_size(op_size)) || force_sx ? sign_x : zero_x, slot_2, 0, R_SCRATCH_2, ®2));
871 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_3);
873 if (ARCH_PREFERS_SX(op_size) && !sgn && op_size < i_size(op_size)) {
874 g(gen_extend(ctx, op_size, zero_x, R_SCRATCH_1, reg1));
875 reg1 = R_SCRATCH_1;
876 g(gen_extend(ctx, op_size, zero_x, R_SCRATCH_2, reg2));
877 reg2 = R_SCRATCH_2;
878 }
880 if (mode == MODE_INT) {
881 g(gen_jmp_on_zero(ctx, i_size(op_size), reg2, COND_E, label_ovf));
882 if (sgn) {
883 uint64_t val;
884 uint32_t label_not_minus_1;
885 label_not_minus_1 = alloc_label(ctx);
886 if (unlikely(!label_not_minus_1))
887 return false;
889 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, i_size(op_size), reg2, -1, COND_NE, label_not_minus_1));
891 val = 0xFFFFFFFFFFFFFF80ULL << (((1 << op_size) - 1) * 8);
892 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, i_size(op_size), reg1, val, COND_E, label_ovf));
894 gen_label(label_not_minus_1);
895 }
896 } else {
897 #if !(defined(ARCH_ARM) && ARM_ASM_DIV_NO_TRAP)
898 if (!mod) {
899 g(gen_load_constant(ctx, target, 0));
900 } else {
901 g(gen_mov(ctx, OP_SIZE_NATIVE, target, reg1));
902 }
903 g(gen_jmp_on_zero(ctx, i_size(op_size), reg2, COND_E, label_end));
904 if (sgn) {
905 uint64_t val;
906 uint32_t label_not_minus_1;
907 label_not_minus_1 = alloc_label(ctx);
908 if (unlikely(!label_not_minus_1))
909 return false;
911 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, i_size(op_size), reg2, -1, COND_NE, label_not_minus_1));
913 if (!mod) {
914 g(gen_mov(ctx, OP_SIZE_NATIVE, target, reg1));
915 } else {
916 g(gen_load_constant(ctx, target, 0));
917 }
919 val = 0xFFFFFFFFFFFFFF80ULL << (((1 << op_size) - 1) * 8);
920 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, i_size(op_size), reg1, val, COND_E, label_end));
922 gen_label(label_not_minus_1);
923 }
924 #endif
925 }
926 if (mod && have_mod) {
927 g(gen_3address_alu(ctx, div_op_size, sgn ? ALU_SREM : ALU_UREM, target, reg1, reg2, 0));
928 } else {
929 g(gen_3address_alu(ctx, div_op_size, sgn ? ALU_SDIV : ALU_UDIV, target, reg1, reg2, 0));
930 }
932 if (mod && !have_mod) {
933 #if defined(ARCH_ARM)
934 gen_insn(INSN_MADD, i_size(op_size), 1, 0);
935 gen_one(target);
936 gen_one(target);
937 gen_one(reg2);
938 gen_one(reg1);
939 #else
940 g(gen_3address_alu(ctx, i_size(op_size), ALU_MUL, R_SCRATCH_2, reg2, target, 0));
941 g(gen_3address_alu(ctx, i_size(op_size), ALU_SUB, target, reg1, R_SCRATCH_2, 0));
942 #endif
943 }
945 gen_label(label_end);
946 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
947 return true;
948 #endif
949 }
950 /*********
951 * SHIFT *
952 *********/
953 do_shift: {
954 bool sx;
955 bool must_mask;
956 unsigned op_s;
957 if (unlikely(op_size > OP_SIZE_NATIVE)) {
958 size_t upcall;
959 if (mode == MODE_FIXED) {
960 switch (alu) {
961 case ROT_SHL: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_shl_,TYPE_INT_MAX));
962 break;
963 case ROT_SAR: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_shr_,TYPE_INT_MAX));
964 break;
965 case ROT_SHR: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_ushr_,TYPE_INT_MAX));
966 break;
967 case ROT_ROL: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_rol_,TYPE_INT_MAX));
968 break;
969 case ROT_ROR: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_ror_,TYPE_INT_MAX));
970 break;
971 default: internal(file_line, "do_alu: invalid shift %u", alu);
972 }
973 } else {
974 switch (alu) {
975 case ROT_SHL: upcall = offsetof(struct cg_upcall_vector_s, cat(INT_binary_shl_,TYPE_INT_MAX));
976 break;
977 case ROT_SAR: upcall = offsetof(struct cg_upcall_vector_s, cat(INT_binary_shr_,TYPE_INT_MAX));
978 break;
979 default: internal(file_line, "do_alu: invalid shift %u", alu);
980 }
981 }
982 g(gen_alu_upcall(ctx, upcall, slot_1, slot_2, slot_r, mode == MODE_INT ? label_ovf : 0));
983 return true;
984 }
985 op_s = i_size_rot(op_size);
986 #if defined(ARCH_X86)
987 if (slot_1 == slot_r && ctx->registers[slot_1] < 0 && !(mode == MODE_INT && alu == ROT_SHL)) {
988 int64_t offset;
989 g(gen_frame_load(ctx, op_size, garbage, slot_2, 0, R_SCRATCH_3));
990 if (mode == MODE_INT) {
991 int64_t imm = (8U << op_size) - 1;
992 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, op_size, R_SCRATCH_3, imm, COND_A, label_ovf));
993 } else if ((alu != ROT_ROL && alu != ROT_ROR) && op_size < OP_SIZE_4) {
994 g(gen_3address_alu_imm(ctx, OP_SIZE_1, ALU_AND, R_SCRATCH_3, R_SCRATCH_3, (8U << op_size) - 1, 0));
995 }
996 offset = (size_t)slot_1 * slot_size;
997 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDR_OFFSET, op_size));
998 gen_insn(INSN_ROT + ARCH_PARTIAL_ALU(op_size), op_size, alu, 1);
999 gen_address_offset();
1000 gen_address_offset();
1001 gen_one(R_SCRATCH_3);
1002 return true;
1003 }
1004 if (mode == MODE_INT && alu == ROT_SHL && op_size < OP_SIZE_NATIVE)
1005 op_s = op_size + 1;
1006 #endif
1007 must_mask = op_size < ARCH_SHIFT_SIZE;
1008 sx = (alu == ROT_SAR && op_size < op_s) || (alu == ROT_SHL && op_size < OP_SIZE_NATIVE && mode == MODE_INT);
1009 #if defined(ARCH_MIPS)
1010 sx |= op_size == OP_SIZE_4;
1011 #endif
1012 g(gen_frame_get(ctx, op_size, sx ? sign_x : zero_x, slot_1, 0, R_SCRATCH_1, ®1));
1013 #if defined(ARCH_X86)
1014 if (!ARCH_IS_3ADDRESS_ROT(alu, op_size)) {
1015 g(gen_frame_load(ctx, op_size, garbage, slot_2, 0, R_SCRATCH_3));
1016 reg3 = R_SCRATCH_3;
1017 } else
1018 #endif
1019 g(gen_frame_get(ctx, op_size, garbage, slot_2, 0, R_SCRATCH_3, ®3));
1020 if (ARCH_PREFERS_SX(op_size) && !sx && op_size < op_s) {
1021 g(gen_extend(ctx, op_size, zero_x, R_SCRATCH_1, reg1));
1022 reg1 = R_SCRATCH_1;
1023 }
1025 if (mode == MODE_INT) {
1026 int64_t imm = (8U << op_size) - 1;
1027 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, i_size_cmp(op_size), reg3, imm, COND_A, label_ovf));
1028 } else {
1029 #if defined(ARCH_ARM)
1030 if (alu == ROT_ROL) {
1031 g(gen_2address_alu1(ctx, OP_SIZE_4, ALU1_NEG, R_SCRATCH_3, reg3, 0));
1032 reg3 = R_SCRATCH_3;
1033 alu = ROT_ROR;
1034 }
1035 #endif
1036 #if defined(ARCH_LOONGARCH64)
1037 if (alu == ROT_ROL && op_size >= OP_SIZE_4) {
1038 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NEG, R_SCRATCH_3, reg3, 0));
1039 reg3 = R_SCRATCH_3;
1040 alu = ROT_ROR;
1041 }
1042 #endif
1043 #if defined(ARCH_MIPS)
1044 if (MIPS_HAS_ROT && alu == ROT_ROL && op_size >= OP_SIZE_4) {
1045 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NEG, R_SCRATCH_3, reg3, 0));
1046 reg3 = R_SCRATCH_3;
1047 alu = ROT_ROR;
1048 }
1049 #endif
1050 #if defined(ARCH_POWER)
1051 if (alu == ROT_ROR && op_size >= OP_SIZE_4) {
1052 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NEG, R_SCRATCH_3, reg3, 0));
1053 reg3 = R_SCRATCH_3;
1054 alu = ROT_ROL;
1055 }
1056 #endif
1057 #if defined(ARCH_S390)
1058 if (Z && alu == ROT_ROR && op_size >= OP_SIZE_4) {
1059 g(gen_2address_alu1(ctx, OP_SIZE_4, ALU1_NEG, R_SCRATCH_3, reg3, 0));
1060 reg3 = R_SCRATCH_3;
1061 alu = ROT_ROL;
1062 }
1063 #endif
1064 if (must_mask) {
1065 g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_4), ALU_AND, R_SCRATCH_3, reg3, (8U << op_size) - 1, 0));
1066 reg3 = R_SCRATCH_3;
1067 }
1068 }
1070 #if defined(ARCH_X86)
1071 if (mode == MODE_INT && alu == ROT_SHL) {
1072 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_2);
1073 } else {
1074 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, slot_2, R_SCRATCH_2);
1075 }
1076 g(gen_3address_rot(ctx, op_s, alu, target, reg1, reg3));
1078 if (mode == MODE_INT && alu == ROT_SHL) {
1079 if (op_size < OP_SIZE_NATIVE) {
1080 gen_insn(INSN_MOVSX, op_size, 0, 0);
1081 gen_one(R_SCRATCH_4);
1082 gen_one(target);
1084 g(gen_cmp_test_jmp(ctx, INSN_CMP, op_s, target, R_SCRATCH_4, COND_NE, label_ovf));
1085 } else {
1086 g(gen_3address_rot(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_4, target, reg3));
1088 g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, reg1, R_SCRATCH_4, COND_NE, label_ovf));
1089 }
1090 }
1091 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1092 return true;
1093 #endif
1094 #if defined(ARCH_ARM)
1095 if (op_size <= OP_SIZE_2 && alu == ROT_ROR) {
1096 gen_insn(INSN_ALU, OP_SIZE_4, ALU_OR, ALU_WRITES_FLAGS(ALU_OR, false));
1097 gen_one(R_SCRATCH_1);
1098 gen_one(reg1);
1099 gen_one(ARG_SHIFTED_REGISTER);
1100 gen_one(ARG_SHIFT_LSL | (8U << op_size));
1101 gen_one(reg1);
1102 if (op_size == OP_SIZE_1)
1103 alu = ROT_SHR;
1104 reg1 = R_SCRATCH_1;
1105 }
1106 goto do_generic_shift;
1107 #endif
1108 #if defined(ARCH_LOONGARCH64)
1109 if (alu == ROT_ROR && op_size >= OP_SIZE_4)
1110 goto do_generic_shift;
1111 #endif
1112 #if defined(ARCH_MIPS)
1113 if (MIPS_HAS_ROT && alu == ROT_ROR && op_size >= OP_SIZE_4)
1114 goto do_generic_shift;
1115 #endif
1116 #if defined(ARCH_POWER)
1117 if (alu == ROT_ROL && op_size >= OP_SIZE_4)
1118 goto do_generic_shift;
1119 #endif
1120 #if defined(ARCH_RISCV64)
1121 if ((alu == ROT_ROL || alu == ROT_ROR) && likely(cpu_test_feature(CPU_FEATURE_zbb))) {
1122 if (likely(op_size >= OP_SIZE_4)) {
1123 goto do_generic_shift;
1124 }
1125 }
1126 #endif
1127 #if defined(ARCH_S390)
1128 if (Z && alu == ROT_ROL && op_size >= OP_SIZE_4)
1129 goto do_generic_shift;
1130 #endif
1131 if (alu == ROT_ROL || alu == ROT_ROR) {
1132 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
1133 g(gen_3address_rot(ctx, op_s, alu == ROT_ROL ? ROT_SHL : ROT_SHR, R_SCRATCH_2, reg1, reg3));
1134 g(gen_2address_alu1(ctx, i_size(OP_SIZE_4), ALU1_NEG, R_SCRATCH_3, reg3, 0));
1135 if (must_mask) {
1136 g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_4), ALU_AND, R_SCRATCH_3, R_SCRATCH_3, (8U << op_size) - 1, 0));
1137 }
1138 g(gen_3address_rot(ctx, op_s, alu == ROT_ROL ? ROT_SHR : ROT_SHL, target, reg1, R_SCRATCH_3));
1139 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_OR, target, target, R_SCRATCH_2, 0));
1140 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1141 return true;
1142 }
1144 goto do_generic_shift;
1145 do_generic_shift:
1146 if (mode == MODE_INT && alu == ROT_SHL) {
1147 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
1148 #if defined(ARCH_S390)
1149 if (op_size >= OP_SIZE_4) {
1150 g(gen_3address_rot(ctx, op_size, ROT_SAL, target, reg1, reg3));
1152 gen_insn(INSN_JMP_COND, op_size, COND_O, 0);
1153 gen_four(label_ovf);
1154 } else
1155 #endif
1156 if (op_size <= OP_SIZE_NATIVE - 1) {
1157 g(gen_3address_rot(ctx, OP_SIZE_NATIVE, alu, target, reg1, reg3));
1159 g(gen_cmp_extended(ctx, OP_SIZE_NATIVE, op_size, target, R_SCRATCH_2, label_ovf));
1160 } else {
1161 g(gen_3address_rot(ctx, OP_SIZE_NATIVE, alu, R_SCRATCH_2, reg1, reg3));
1163 g(gen_3address_rot(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_4, R_SCRATCH_2, reg3));
1165 g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, reg1, R_SCRATCH_4, COND_NE, label_ovf));
1167 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_2));
1169 return true;
1170 }
1171 } else {
1172 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, slot_2, R_SCRATCH_1);
1173 g(gen_3address_rot(ctx, op_s, alu, target, reg1, reg3));
1174 }
1176 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1177 return true;
1178 }
1179 /******
1180 * BT *
1181 ******/
1182 do_bt: {
1183 unsigned attr_unused op_s;
1184 bool need_mask;
1185 #if defined(ARCH_X86)
1186 if ((alu == BTX_BT || slot_1 == slot_r) && ctx->registers[slot_1] < 0) {
1187 unsigned n_op_size = minimum(op_size, OP_SIZE_NATIVE);
1188 g(gen_frame_get(ctx, n_op_size, garbage, slot_2, 0, R_SCRATCH_1, ®2));
1189 if (mode == MODE_INT) {
1190 int64_t imm = (8U << op_size) - 1;
1191 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, n_op_size, reg2, imm, alu == BTX_BT ? COND_A : COND_AE, label_ovf));
1192 if (unlikely(op_size > OP_SIZE_NATIVE)) {
1193 g(gen_address(ctx, R_FRAME, (size_t)slot_2 * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_STR_OFFSET, OP_SIZE_NATIVE));
1194 gen_insn(INSN_CMP, OP_SIZE_NATIVE, 0, 1);
1195 gen_address_offset();
1196 gen_one(ARG_IMM);
1197 gen_eight(0);
1198 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_NE, 0);
1199 gen_four(label_ovf);
1200 }
1201 } else {
1202 g(gen_3address_alu_imm(ctx, OP_SIZE_4, ALU_AND, R_SCRATCH_1, reg2, (8U << op_size) - 1, 0));
1203 reg2 = R_SCRATCH_1;
1204 }
1205 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size, IMM_PURPOSE_STR_OFFSET, maximum(n_op_size, OP_SIZE_2)));
1206 if (alu == BTX_BT) {
1207 gen_insn(INSN_BT, maximum(n_op_size, OP_SIZE_2), 0, 1);
1208 gen_address_offset();
1209 gen_one(reg2);
1210 g(gen_frame_set_cond(ctx, maximum(n_op_size, OP_SIZE_2), false, COND_B, slot_r));
1211 } else {
1212 gen_insn(INSN_BTX, maximum(n_op_size, OP_SIZE_2), alu, 1);
1213 gen_address_offset();
1214 gen_address_offset();
1215 gen_one(reg2);
1216 }
1217 return true;
1218 }
1219 #endif
1220 if (unlikely(op_size > OP_SIZE_NATIVE)) {
1221 size_t upcall;
1222 if (mode == MODE_FIXED) {
1223 switch (alu) {
1224 case BTX_BTS: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_bts_,TYPE_INT_MAX));
1225 break;
1226 case BTX_BTR: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_btr_,TYPE_INT_MAX));
1227 break;
1228 case BTX_BTC: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_btc_,TYPE_INT_MAX));
1229 break;
1230 case BTX_BT: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_bt_,TYPE_INT_MAX));
1231 break;
1232 default: internal(file_line, "do_alu: invalid bit test %u", alu);
1233 }
1234 } else {
1235 switch (alu) {
1236 case BTX_BTS: upcall = offsetof(struct cg_upcall_vector_s, cat(INT_binary_bts_,TYPE_INT_MAX));
1237 break;
1238 case BTX_BTR: upcall = offsetof(struct cg_upcall_vector_s, cat(INT_binary_btr_,TYPE_INT_MAX));
1239 break;
1240 case BTX_BTC: upcall = offsetof(struct cg_upcall_vector_s, cat(INT_binary_btc_,TYPE_INT_MAX));
1241 break;
1242 case BTX_BT: upcall = offsetof(struct cg_upcall_vector_s, cat(INT_binary_bt_,TYPE_INT_MAX));
1243 break;
1244 default: internal(file_line, "do_alu: invalid bit test %u", alu);
1245 }
1246 }
1247 g(gen_alu_upcall(ctx, upcall, slot_1, slot_2, slot_r, label_ovf));
1248 return true;
1249 }
1250 op_s = minimum(OP_SIZE_NATIVE, ARCH_SHIFT_SIZE);
1251 op_s = maximum(op_s, op_size);
1252 g(gen_frame_get(ctx, op_size, zero_x, slot_1, 0, R_SCRATCH_1, ®1));
1253 g(gen_frame_get(ctx, op_size, garbage, slot_2, 0, R_SCRATCH_2, ®2));
1254 if (mode == MODE_INT) {
1255 int64_t imm = (8U << op_size) - 1;
1256 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, i_size_cmp(op_size), reg2, imm, alu == BTX_BT ? COND_A : COND_AE, label_ovf));
1257 }
1258 if (alu != BTX_BT) {
1259 if (!ARCH_HAS_BTX(alu, OP_SIZE_NATIVE, false))
1260 goto do_generic_bt;
1261 need_mask = !ARCH_HAS_BTX(alu, op_size, false);
1262 } else {
1263 #if defined(ARCH_X86)
1264 need_mask = op_size < OP_SIZE_2;
1265 #else
1266 if (!ARCH_HAS_BTX(BTX_BTEXT, OP_SIZE_NATIVE, false))
1267 goto do_generic_bt;
1268 need_mask = !ARCH_HAS_BTX(BTX_BTEXT, op_size, false);
1269 #endif
1270 }
1271 if (need_mask) {
1272 g(gen_3address_alu_imm(ctx, OP_SIZE_4, ALU_AND, R_SCRATCH_2, reg2, (8U << op_size) - 1, 0));
1273 reg2 = R_SCRATCH_2;
1274 }
1275 if (alu == BTX_BT) {
1276 #if defined(ARCH_X86)
1277 gen_insn(INSN_BT, maximum(op_size, OP_SIZE_2), 0, 1);
1278 gen_one(reg1);
1279 gen_one(reg2);
1281 g(gen_frame_set_cond(ctx, maximum(op_size, OP_SIZE_2), false, COND_B, slot_r));
1282 #else
1283 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1284 gen_insn(INSN_BTX, need_mask ? OP_SIZE_NATIVE : op_size, BTX_BTEXT, 0);
1285 gen_one(target);
1286 gen_one(reg1);
1287 gen_one(reg2);
1289 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
1290 #endif
1291 } else {
1292 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, slot_2, R_SCRATCH_1);
1293 #if defined(ARCH_X86)
1294 if (target == reg2)
1295 target = R_SCRATCH_1;
1296 if (target != reg1) {
1297 g(gen_mov(ctx, op_size, target, reg1));
1298 reg1 = target;
1299 }
1300 gen_insn(INSN_BTX, maximum(op_size, OP_SIZE_2), alu, 1);
1301 #else
1302 gen_insn(INSN_BTX, need_mask ? OP_SIZE_NATIVE : op_size, alu, 0);
1303 #endif
1304 gen_one(target);
1305 gen_one(reg1);
1306 gen_one(reg2);
1308 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1309 }
1310 return true;
1312 goto do_generic_bt;
1313 do_generic_bt:
1314 if (mode == MODE_FIXED && op_size < ARCH_SHIFT_SIZE) {
1315 g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_4), ALU_AND, R_SCRATCH_2, reg2, (8U << op_size) - 1, 0));
1316 reg2 = R_SCRATCH_2;
1317 }
1318 g(gen_load_constant(ctx, R_SCRATCH_3, 1));
1320 g(gen_3address_rot(ctx, op_s, ROT_SHL, R_SCRATCH_3, R_SCRATCH_3, reg2));
1322 switch (alu) {
1323 case BTX_BT:
1324 #if ARCH_HAS_FLAGS
1325 #if defined(ARCH_S390) || defined(ARCH_POWER)
1326 g(gen_3address_alu(ctx, i_size(op_size), ALU_AND, R_SCRATCH_1, reg1, R_SCRATCH_3, 1));
1327 #else
1328 gen_insn(INSN_TEST, i_size(op_size), 0, 1);
1329 gen_one(reg1);
1330 gen_one(R_SCRATCH_3);
1331 #endif
1332 g(gen_frame_set_cond(ctx, i_size_cmp(op_size), false, COND_NE, slot_r));
1333 #else
1334 g(gen_3address_alu(ctx, i_size(op_size), ALU_AND, R_SCRATCH_1, reg1, R_SCRATCH_3, 0));
1335 g(gen_frame_cmp_imm_set_cond_reg(ctx, i_size(op_size), R_SCRATCH_1, 0, COND_NE, slot_r));
1336 #endif
1337 return true;
1338 case BTX_BTS:
1339 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1340 g(gen_3address_alu(ctx, i_size(op_size), ALU_OR, target, reg1, R_SCRATCH_3, 0));
1341 break;
1342 case BTX_BTR:
1343 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1344 if (!ARCH_HAS_ANDN) {
1345 g(gen_3address_alu_imm(ctx, i_size(op_size), ALU_XOR, R_SCRATCH_3, R_SCRATCH_3, -1, 0));
1347 g(gen_3address_alu(ctx, i_size(op_size), ALU_AND, target, reg1, R_SCRATCH_3, 0));
1348 break;
1349 }
1350 g(gen_3address_alu(ctx, i_size(op_size), ALU_ANDN, target, reg1, R_SCRATCH_3, 0));
1351 break;
1352 case BTX_BTC:
1353 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1354 g(gen_3address_alu(ctx, i_size(op_size), ALU_XOR, target, reg1, R_SCRATCH_3, 0));
1355 break;
1356 default:
1357 internal(file_line, "gen_alu: unsupported bit test %u", alu);
1358 }
1360 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1362 return true;
1363 }
1364 /***********
1365 * COMPARE *
1366 ***********/
1367 do_compare: {
1368 if (unlikely(op_size > OP_SIZE_NATIVE)) {
1369 size_t attr_unused upcall;
1370 switch (alu) {
1371 case COND_E:
1372 case COND_NE:
1373 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
1374 g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, ALU_XOR, 0, slot_2, lo_word(OP_SIZE_NATIVE), R_SCRATCH_1));
1375 g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, ALU_XOR, 0, slot_2, hi_word(OP_SIZE_NATIVE), R_SCRATCH_2));
1376 #if defined(ARCH_ARM64)
1377 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_OR, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_2, 0));
1379 gen_insn(INSN_CMP, OP_SIZE_NATIVE, 0, 1);
1380 gen_one(R_SCRATCH_1);
1381 gen_one(ARG_IMM);
1382 gen_eight(0);
1383 #else
1384 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_OR, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_2, ARCH_HAS_FLAGS));
1385 #endif
1386 #if ARCH_HAS_FLAGS
1387 g(gen_frame_set_cond(ctx, OP_SIZE_NATIVE, false, alu, slot_r));
1388 #else
1389 g(gen_frame_cmp_imm_set_cond_reg(ctx, OP_SIZE_NATIVE, R_SCRATCH_1, 0, alu, slot_r));
1390 #endif
1391 return true;
1392 #if defined(ARCH_X86) || defined(ARCH_ARM)
1393 case COND_L:
1394 case COND_B:
1395 g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_2, lo_word(OP_SIZE_NATIVE), R_SCRATCH_2));
1396 g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_1, hi_word(OP_SIZE_NATIVE), R_SCRATCH_1));
1397 g(gen_frame_load_cmp(ctx, OP_SIZE_NATIVE, false, garbage, true, slot_1, lo_word(OP_SIZE_NATIVE), R_SCRATCH_2));
1398 g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, ALU_SBB, 1, slot_2, hi_word(OP_SIZE_NATIVE), R_SCRATCH_1));
1399 g(gen_frame_set_cond(ctx, OP_SIZE_NATIVE, false, alu, slot_r));
1400 return true;
1401 case COND_LE:
1402 case COND_BE:
1403 g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_1, lo_word(OP_SIZE_NATIVE), R_SCRATCH_2));
1404 g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_2, hi_word(OP_SIZE_NATIVE), R_SCRATCH_1));
1405 g(gen_frame_load_cmp(ctx, OP_SIZE_NATIVE, false, garbage, true, slot_2, lo_word(OP_SIZE_NATIVE), R_SCRATCH_2));
1406 g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, ALU_SBB, 1, slot_1, hi_word(OP_SIZE_NATIVE), R_SCRATCH_1));
1407 g(gen_frame_set_cond(ctx, OP_SIZE_NATIVE, false, alu == COND_LE ? COND_GE : COND_AE, slot_r));
1408 return true;
1409 #else
1410 case COND_L: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_less_,TYPE_INT_MAX)); goto do_upcall;
1411 case COND_B: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_uless_,TYPE_INT_MAX)); goto do_upcall;
1412 case COND_LE: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_less_equal_,TYPE_INT_MAX)); goto do_upcall;
1413 case COND_BE: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_uless_equal_,TYPE_INT_MAX)); goto do_upcall;
1414 do_upcall: g(gen_alu_upcall(ctx, upcall, slot_1, slot_2, slot_r, 0));
1415 return true;
1416 #endif
1417 default:
1418 internal(file_line, "gen_alu: unsupported condition %u", alu);
1419 }
1420 return false;
1421 }
1422 #if defined(ARCH_X86)
1423 g(gen_frame_get(ctx, op_size, garbage, slot_1, 0, R_SCRATCH_1, ®1));
1424 g(gen_frame_load_cmp_set_cond(ctx, op_size, garbage, slot_2, 0, reg1, alu, slot_r));
1425 #else
1426 g(gen_frame_get(ctx, op_size, op_size == i_size_cmp(op_size) + (unsigned)zero ? garbage : alu == COND_L || alu == COND_LE || ARCH_PREFERS_SX(op_size) ? sign_x : zero_x, slot_1, 0, R_SCRATCH_1, ®1));
1427 g(gen_frame_load_cmp_set_cond(ctx, op_size, alu == COND_L || alu == COND_LE || ARCH_PREFERS_SX(op_size) ? sign_x : zero_x, slot_2, 0, reg1, alu, slot_r));
1428 #endif
1429 return true;
1430 }
1431 }
1433 static bool attr_w gen_alu1(struct codegen_context *ctx, unsigned mode, unsigned op_size, unsigned op, uint32_t label_ovf, frame_t slot_1, frame_t slot_r)
1434 {
1435 unsigned alu;
1436 unsigned reg1, target;
1437 switch (mode) {
1438 case MODE_FIXED: switch (op) {
1439 case OPCODE_FIXED_OP_not: alu = ALU1_NOT; goto do_alu;
1440 case OPCODE_FIXED_OP_neg: alu = ALU1_NEG; goto do_alu;
1441 case OPCODE_FIXED_OP_inc: alu = ALU1_INC; goto do_alu;
1442 case OPCODE_FIXED_OP_dec: alu = ALU1_DEC; goto do_alu;
1443 case OPCODE_FIXED_OP_bswap:
1444 case OPCODE_FIXED_OP_bswap_alt1: alu = ALU1_BSWAP; goto do_bswap;
1445 case OPCODE_FIXED_OP_brev:
1446 case OPCODE_FIXED_OP_brev_alt1: alu = ALU1_BREV; goto do_brev;
1447 case OPCODE_FIXED_OP_bsf:
1448 case OPCODE_FIXED_OP_bsf_alt1: alu = ALU1_BSF; goto do_bsf_bsr_popcnt;
1449 case OPCODE_FIXED_OP_bsr:
1450 case OPCODE_FIXED_OP_bsr_alt1: alu = ALU1_BSR; goto do_bsf_bsr_popcnt;
1451 case OPCODE_FIXED_OP_popcnt:
1452 case OPCODE_FIXED_OP_popcnt_alt1: alu = ALU1_POPCNT; goto do_bsf_bsr_popcnt;
1453 case OPCODE_FIXED_OP_to_int: goto do_fixed_conv;
1454 case OPCODE_FIXED_OP_from_int: goto do_fixed_conv;
1455 case OPCODE_FIXED_OP_uto_int: goto conv_uto_int;
1456 case OPCODE_FIXED_OP_ufrom_int: goto conv_ufrom_int;
1457 default: internal(file_line, "gen_alu1: unsupported fixed operation %u", op);
1458 }
1459 case MODE_INT: switch (op) {
1460 case OPCODE_INT_OP_not: alu = ALU1_NOT; mode = MODE_FIXED; goto do_alu;
1461 case OPCODE_INT_OP_neg: alu = ALU1_NEG; goto do_alu;
1462 case OPCODE_INT_OP_inc: alu = ALU1_INC; goto do_alu;
1463 case OPCODE_INT_OP_dec: alu = ALU1_DEC; goto do_alu;
1464 case OPCODE_INT_OP_bsf: alu = ALU1_BSF; goto do_bsf_bsr_popcnt;
1465 case OPCODE_INT_OP_bsr: alu = ALU1_BSR; goto do_bsf_bsr_popcnt;
1466 case OPCODE_INT_OP_popcnt:
1467 case OPCODE_INT_OP_popcnt_alt1: alu = ALU1_POPCNT; goto do_bsf_bsr_popcnt;
1468 case OPCODE_INT_OP_to_int: goto do_conv;
1469 case OPCODE_INT_OP_from_int: goto do_conv;
1470 default: internal(file_line, "gen_alu1: unsupported int operation %u", op);
1471 }
1472 case MODE_BOOL: switch (op) {
1473 case OPCODE_BOOL_OP_not: goto do_bool_not;
1474 default: internal(file_line, "gen_alu1: unsupported bool operation %u", op);
1475 }
1476 }
1477 internal(file_line, "gen_alu1: unsupported mode %u", mode);
1479 /*******
1480 * ALU *
1481 *******/
1482 do_alu: {
1483 bool arch_use_flags = ARCH_HAS_FLAGS;
1484 enum extend ex;
1485 #if defined(ARCH_POWER)
1486 arch_use_flags = false;
1487 #endif
1488 if (op_size > OP_SIZE_NATIVE) {
1489 #if !defined(ARCH_X86) && !defined(ARCH_ARM) && !defined(ARCH_POWER)
1490 if (alu == ALU1_NEG) {
1491 if (mode == MODE_FIXED)
1492 g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(FIXED_unary_neg_,TYPE_INT_MAX)), slot_1, NO_FRAME_T, slot_r, 0));
1493 else
1494 g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(INT_unary_neg_,TYPE_INT_MAX)), slot_1, NO_FRAME_T, slot_r, label_ovf));
1495 return true;
1496 }
1497 if (alu == ALU1_DEC) {
1498 if (mode == MODE_FIXED)
1499 g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(FIXED_unary_dec_,TYPE_INT_MAX)), slot_1, NO_FRAME_T, slot_r, 0));
1500 else
1501 g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(INT_unary_dec_,TYPE_INT_MAX)), slot_1, NO_FRAME_T, slot_r, label_ovf));
1502 return true;
1503 }
1504 if (alu == ALU1_INC) {
1505 if (mode == MODE_FIXED)
1506 g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(FIXED_unary_inc_,TYPE_INT_MAX)), slot_1, NO_FRAME_T, slot_r, 0));
1507 else
1508 g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(INT_unary_inc_,TYPE_INT_MAX)), slot_1, NO_FRAME_T, slot_r, label_ovf));
1509 return true;
1510 }
1511 #endif
1512 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
1513 #if defined(ARCH_S390)
1514 if (alu == ALU1_NOT) {
1515 g(gen_load_constant(ctx, R_SCRATCH_3, -1));
1517 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_XOR, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_3, 0));
1518 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_XOR, R_SCRATCH_2, R_SCRATCH_2, R_SCRATCH_3, 0));
1520 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_1, R_SCRATCH_2));
1521 return true;
1522 }
1523 #endif
1524 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, alu, R_SCRATCH_1, R_SCRATCH_1, alu == ALU1_INC || alu == ALU1_DEC || alu == ALU1_NEG ? 2 : 0));
1525 if (alu == ALU1_NOT) {
1526 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NOT, R_SCRATCH_2, R_SCRATCH_2, 0));
1527 } else if (alu == ALU1_INC || alu == ALU1_DEC) {
1528 g(gen_imm(ctx, 0, alu == ALU1_INC ? IMM_PURPOSE_ADD : IMM_PURPOSE_SUB, OP_SIZE_NATIVE));
1529 gen_insn(INSN_ALU, OP_SIZE_NATIVE, alu == ALU1_INC ? ALU_ADC : ALU_SBB, (mode == MODE_INT) | ALU_WRITES_FLAGS(alu == ALU1_INC ? ALU_ADC : ALU_SBB, is_imm()));
1530 gen_one(R_SCRATCH_2);
1531 gen_one(R_SCRATCH_2);
1532 gen_imm_offset();
1533 } else {
1534 #if defined(ARCH_X86)
1535 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NOT, R_SCRATCH_2, R_SCRATCH_2, 0));
1537 g(gen_imm(ctx, -1, IMM_PURPOSE_SUB, OP_SIZE_NATIVE));
1538 gen_insn(INSN_ALU, OP_SIZE_NATIVE, ALU_SBB, ALU_WRITES_FLAGS(ALU_SBB, is_imm()));
1539 gen_one(R_SCRATCH_2);
1540 gen_one(R_SCRATCH_2);
1541 gen_imm_offset();
1542 #else
1543 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NGC, R_SCRATCH_2, R_SCRATCH_2, (mode == MODE_INT)));
1544 #endif
1545 }
1546 if (mode == MODE_INT) {
1547 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_O, 0);
1548 gen_four(label_ovf);
1549 }
1550 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_1, R_SCRATCH_2));
1551 return true;
1552 }
1553 if ((arch_use_flags || ARCH_SUPPORTS_TRAPS) && slot_1 == slot_r && i_size_cmp(op_size) == op_size + zero) {
1554 struct cg_exit *ce;
1555 unsigned undo_alu = alu == ALU1_INC ? ALU1_DEC : alu == ALU1_DEC ? ALU1_INC : alu;
1556 if (ctx->registers[slot_1] >= 0) {
1557 unsigned reg = ctx->registers[slot_1];
1558 if (mode == MODE_INT && ARCH_SUPPORTS_TRAPS) {
1559 gen_insn(INSN_ALU1_TRAP, op_size, alu, ALU1_WRITES_FLAGS(alu));
1560 gen_one(reg);
1561 gen_one(reg);
1562 if (ARCH_TRAP_BEFORE || alu == undo_alu) {
1563 gen_four(label_ovf);
1564 return true;
1565 } else {
1566 ce = alloc_undo_label(ctx);
1567 if (unlikely(!ce))
1568 return false;
1569 gen_four(ce->undo_label);
1570 goto do_undo_opcode;
1571 }
1572 }
1573 g(gen_2address_alu1(ctx, i_size(op_size), alu, reg, reg, mode == MODE_INT));
1574 if (mode == MODE_INT) {
1575 if (alu != undo_alu) {
1576 ce = alloc_undo_label(ctx);
1577 if (unlikely(!ce))
1578 return false;
1579 gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
1580 gen_four(ce->undo_label);
1581 do_undo_opcode:
1582 ce->undo_opcode = INSN_ALU1 + ARCH_PARTIAL_ALU(op_size);
1583 ce->undo_op_size = i_size(op_size);
1584 ce->undo_aux = undo_alu;
1585 ce->undo_writes_flags = ALU1_WRITES_FLAGS(undo_alu);
1586 ce->undo_parameters[0] = reg;
1587 ce->undo_parameters[1] = reg;
1588 ce->undo_parameters_len = 2;
1589 } else {
1590 gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
1591 gen_four(label_ovf);
1592 }
1593 }
1594 return true;
1595 }
1596 #if defined(ARCH_X86)
1597 else {
1598 size_t m;
1599 int64_t offset = (size_t)slot_1 * slot_size;
1600 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDR_OFFSET, i_size(op_size)));
1601 gen_insn(INSN_ALU1 + ARCH_PARTIAL_ALU(op_size), i_size(op_size), alu, ALU1_WRITES_FLAGS(alu) | (mode == MODE_INT));
1602 gen_address_offset();
1603 gen_address_offset();
1604 if (mode == MODE_INT) {
1605 if (alu != undo_alu) {
1606 ce = alloc_undo_label(ctx);
1607 if (unlikely(!ce))
1608 return false;
1609 ce->undo_opcode = INSN_ALU1 + ARCH_PARTIAL_ALU(op_size);
1610 ce->undo_op_size = i_size(op_size);
1611 ce->undo_aux = undo_alu;
1612 ce->undo_writes_flags = ALU1_WRITES_FLAGS(undo_alu);
1613 m = mark_params(ctx);
1614 gen_address_offset();
1615 gen_address_offset();
1616 copy_params(ctx, ce, m);
1617 gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
1618 gen_four(ce->undo_label);
1619 } else {
1620 gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
1621 gen_four(label_ovf);
1622 }
1623 }
1624 return true;
1625 }
1626 #endif
1627 }
1628 target = gen_frame_target(ctx, slot_r, mode == MODE_INT ? slot_1 : NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1629 if (mode == MODE_FIXED) {
1630 ex = garbage;
1631 } else {
1632 ex = ARCH_PREFERS_SX(op_size) ? sign_x : zero_x;
1633 if (ARCH_SUPPORTS_TRAPS && op_size >= OP_SIZE_4)
1634 ex = garbage;
1635 if (op_size == i_size(op_size) + (unsigned)zero)
1636 ex = garbage;
1637 }
1638 g(gen_frame_get(ctx, op_size, ex, slot_1, 0, mode == MODE_INT ? R_SCRATCH_2 : target, ®1));
1639 #if defined(ARCH_S390)
1640 if (alu == ALU1_NOT) {
1641 g(gen_3address_alu_imm(ctx, i_size(op_size), ALU_XOR, target, reg1, -1, 0));
1643 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1644 return true;
1645 }
1646 #endif
1647 #if defined(ARCH_X86)
1648 g(gen_2address_alu1(ctx, op_size, alu, target, reg1, mode == MODE_INT));
1649 #else
1650 if (mode == MODE_INT) {
1651 #if defined(ARCH_POWER)
1652 if (op_size == OP_SIZE_NATIVE) {
1653 g(gen_2address_alu1(ctx, i_size(op_size), alu, target, reg1, 0));
1654 if (alu == ALU1_NEG) {
1655 g(gen_3address_alu(ctx, i_size(op_size), ALU_AND, R_CG_SCRATCH, target, reg1, 1));
1656 } else if (alu == ALU1_INC) {
1657 g(gen_3address_alu(ctx, i_size(op_size), ALU_ANDN, R_CG_SCRATCH, target, reg1, 1));
1658 } else if (alu == ALU1_DEC) {
1659 g(gen_3address_alu(ctx, i_size(op_size), ALU_ANDN, R_CG_SCRATCH, reg1, target, 1));
1660 }
1661 gen_insn(INSN_JMP_COND, op_size, COND_L, 0);
1662 gen_four(label_ovf);
1664 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1666 return true;
1667 }
1668 #endif
1669 if (ARCH_HAS_JMP_2REGS(COND_L) && ARCH_HAS_JMP_2REGS(COND_G) && (alu == ALU1_INC || alu == ALU1_DEC) && op_size == i_size_cmp(op_size) + (unsigned)zero) {
1670 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, alu, target, reg1, 0));
1672 g(gen_cmp_test_jmp(ctx, INSN_CMP, i_size_cmp(op_size), target, reg1, alu == ALU1_INC ? COND_L : COND_G, label_ovf));
1674 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1676 return true;
1677 }
1678 if (!arch_use_flags && !ARCH_SUPPORTS_TRAPS && ARCH_HAS_ANDN && op_size >= OP_SIZE_4) {
1679 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, alu, target, reg1, 0));
1681 if (alu == ALU1_NEG) {
1682 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_AND, R_SCRATCH_3, target, reg1, 0));
1683 } else if (alu == ALU1_INC) {
1684 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_ANDN, R_SCRATCH_3, target, reg1, 0));
1685 } else if (alu == ALU1_DEC) {
1686 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_ANDN, R_SCRATCH_3, reg1, target, 0));
1687 }
1689 if (op_size < OP_SIZE_NATIVE)
1690 g(gen_extend(ctx, op_size, sign_x, R_SCRATCH_3, R_SCRATCH_3));
1692 g(gen_jmp_on_zero(ctx, OP_SIZE_NATIVE, R_SCRATCH_3, COND_S, label_ovf));
1694 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1696 return true;
1697 }
1698 if (op_size <= OP_SIZE_2 || (!arch_use_flags && !ARCH_SUPPORTS_TRAPS)) {
1699 int64_t imm = ((alu != ALU1_INC && ARCH_PREFERS_SX(op_size) ? -0x80ULL : 0x80ULL) << (((1 << op_size) - 1) * 8)) - (alu == ALU1_INC);
1701 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, i_size_cmp(op_size), reg1, imm, COND_E, label_ovf));
1703 mode = MODE_FIXED;
1705 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1706 }
1707 }
1708 #if !ARCH_HAS_FLAGS
1709 if (mode == MODE_INT) {
1710 gen_insn(INSN_ALU1_TRAP, op_size, alu, ALU1_WRITES_FLAGS(alu));
1711 gen_one(target);
1712 gen_one(reg1);
1713 gen_four(label_ovf);
1714 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1715 return true;
1716 }
1717 #endif
1718 g(gen_2address_alu1(ctx, i_size(op_size), alu, target, reg1, mode == MODE_INT));
1719 #endif
1720 if (mode == MODE_INT) {
1721 gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
1722 gen_four(label_ovf);
1723 }
1724 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1725 return true;
1726 }
1728 /*******
1729 * NOT *
1730 *******/
1731 do_bool_not: {
1732 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1733 g(gen_frame_get(ctx, op_size, garbage, slot_1, 0, target, ®1));
1735 g(gen_3address_alu_imm(ctx, i_size(op_size), ALU_XOR, target, reg1, 1, 0));
1737 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1738 return true;
1739 }
1741 /*********
1742 * BSWAP *
1743 *********/
1744 do_bswap: {
1745 bool attr_unused sx = false;
1746 #if defined(ARCH_X86) || defined(ARCH_ARM) || defined(ARCH_IA64) || defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS) || defined(ARCH_RISCV64) || defined(ARCH_S390)
1747 #if defined(ARCH_ARM32)
1748 if (unlikely(!cpu_test_feature(CPU_FEATURE_armv6)))
1749 goto do_generic_bswap;
1750 #endif
1751 #if defined(ARCH_MIPS)
1752 if (unlikely(!MIPS_HAS_ROT))
1753 goto do_generic_bswap;
1754 sx = op_size == OP_SIZE_4;
1755 #endif
1756 #if defined(ARCH_RISCV64)
1757 if (unlikely(!cpu_test_feature(CPU_FEATURE_zbb)))
1758 goto do_generic_bswap;
1759 #endif
1760 #if defined(ARCH_S390)
1761 if (op_size == OP_SIZE_2)
1762 goto do_generic_bswap;
1763 #endif
1764 #if defined(ARCH_X86)
1765 if (op_size >= OP_SIZE_4 && !cpu_test_feature(CPU_FEATURE_bswap))
1766 goto do_generic_bswap;
1767 #endif
1768 if (op_size > OP_SIZE_NATIVE) {
1769 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
1770 reg1 = R_SCRATCH_1;
1771 target = R_SCRATCH_1;
1772 } else {
1773 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1774 g(gen_frame_get(ctx, op_size, sx ? sign_x : garbage, slot_1, 0, target, ®1));
1775 }
1777 if (op_size == OP_SIZE_1) {
1778 #if defined(ARCH_IA64) || defined(ARCH_RISCV64)
1779 } else if (op_size == OP_SIZE_2 || op_size == OP_SIZE_4) {
1780 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_BSWAP, target, reg1, 0));
1782 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SAR, target, target, op_size == OP_SIZE_2 ? 48 : 32, 0));
1783 #endif
1784 } else if (op_size == OP_SIZE_2) {
1785 #if defined(ARCH_X86)
1786 g(gen_3address_rot_imm(ctx, OP_SIZE_2, ROT_ROR, target, reg1, 8, 0));
1787 #else
1788 g(gen_2address_alu1(ctx, OP_SIZE_4, ALU1_BSWAP16, target, reg1, 0));
1789 #endif
1790 } else {
1791 g(gen_2address_alu1(ctx, minimum(op_size, OP_SIZE_NATIVE), ALU1_BSWAP, target, reg1, 0));
1792 }
1793 if (op_size > OP_SIZE_NATIVE) {
1794 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_BSWAP, R_SCRATCH_2, R_SCRATCH_2, 0));
1795 }
1797 if (op_size > OP_SIZE_NATIVE)
1798 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_2, R_SCRATCH_1));
1799 else
1800 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1801 return true;
1802 #endif
1803 goto do_generic_bswap;
1804 do_generic_bswap:
1805 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_unary_bswap_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, 0);
1806 }
1807 /********
1808 * BREV *
1809 ********/
1810 do_brev: {
1811 #if defined(ARCH_ARM) || defined(ARCH_LOONGARCH64) || (defined(ARCH_MIPS) && MIPS_R6)
1812 #if defined(ARCH_ARM32)
1813 if (unlikely(!cpu_test_feature(CPU_FEATURE_armv6t2)))
1814 goto do_generic_brev;
1815 #endif
1816 if (op_size > OP_SIZE_NATIVE) {
1817 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
1818 reg1 = R_SCRATCH_1;
1819 target = R_SCRATCH_1;
1820 } else {
1821 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1822 g(gen_frame_get(ctx, op_size, garbage, slot_1, 0, target, ®1));
1823 }
1825 g(gen_2address_alu1(ctx, minimum(maximum(OP_SIZE_4, op_size), OP_SIZE_NATIVE), ALU1_BREV, target, reg1, 0));
1826 if (op_size <= OP_SIZE_2) {
1827 g(gen_3address_rot_imm(ctx, OP_SIZE_4, ROT_SHR, target, target, op_size == OP_SIZE_1 ? 24 : 16, 0));
1828 }
1829 if (op_size > OP_SIZE_NATIVE) {
1830 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_BREV, R_SCRATCH_2, R_SCRATCH_2, 0));
1831 }
1833 if (op_size > OP_SIZE_NATIVE)
1834 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_2, R_SCRATCH_1));
1835 else
1836 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1837 return true;
1838 #endif
1839 goto do_generic_brev;
1840 do_generic_brev:
1841 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_unary_brev_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, 0);
1842 }
1843 /******************
1844 * BSF/BSR/POPCNT *
1845 ******************/
1846 do_bsf_bsr_popcnt: {
1847 if (op_size > OP_SIZE_NATIVE) {
1848 #if defined(ARCH_X86)
1849 uint32_t label_finish = 0; /* avoid warning */
1850 if (alu == ALU1_POPCNT && unlikely(!cpu_test_feature(CPU_FEATURE_popcnt)))
1851 goto do_generic_bsf_bsr_popcnt;
1852 if (alu == ALU1_BSR || alu == ALU1_POPCNT) {
1853 if (mode == MODE_INT) {
1854 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
1855 g(gen_imm(ctx, 0, IMM_PURPOSE_STORE_VALUE, OP_SIZE_NATIVE));
1856 gen_insn(INSN_CMP, OP_SIZE_NATIVE, 0, 1);
1857 gen_address_offset();
1858 gen_imm_offset();
1860 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_S, 0);
1861 gen_four(label_ovf);
1862 }
1863 }
1864 if (alu == ALU1_BSF) {
1865 label_finish = alloc_label(ctx);
1866 if (unlikely(!label_finish))
1867 return false;
1869 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + lo_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
1870 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_BSF, ALU1_WRITES_FLAGS(ALU1_BSF));
1871 gen_one(R_SCRATCH_1);
1872 gen_address_offset();
1874 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_NE, 0);
1875 gen_four(label_finish);
1877 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
1878 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_BSF, ALU1_WRITES_FLAGS(ALU1_BSF));
1879 gen_one(R_SCRATCH_1);
1880 gen_address_offset();
1882 g(gen_3address_alu_imm(ctx, OP_SIZE_4, ALU_ADD, R_SCRATCH_1, R_SCRATCH_1, 8U << OP_SIZE_NATIVE, 0));
1883 }
1884 if (alu == ALU1_BSR) {
1885 label_finish = alloc_label(ctx);
1886 if (unlikely(!label_finish))
1887 return false;
1889 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
1890 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_BSR, ALU1_WRITES_FLAGS(ALU1_BSR));
1891 gen_one(R_SCRATCH_1);
1892 gen_address_offset();
1894 g(gen_3address_alu_imm(ctx, OP_SIZE_4, ALU_ADD, R_SCRATCH_1, R_SCRATCH_1, 8U << OP_SIZE_NATIVE, 0));
1896 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_NE, 0);
1897 gen_four(label_finish);
1899 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + lo_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
1900 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_BSR, ALU1_WRITES_FLAGS(ALU1_BSR));
1901 gen_one(R_SCRATCH_1);
1902 gen_address_offset();
1903 }
1904 if (alu == ALU1_BSF || alu == ALU1_BSR) {
1905 if (mode == MODE_INT) {
1906 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_E, 0);
1907 gen_four(label_ovf);
1908 } else {
1909 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_NE, 0);
1910 gen_four(label_finish);
1912 g(gen_load_constant(ctx, R_SCRATCH_1, -1));
1913 }
1915 gen_label(label_finish);
1917 if (mode == MODE_INT)
1918 goto write_result;
1920 if (R_SCRATCH_1 != R_AX || R_SCRATCH_2 != R_DX)
1921 internal(file_line, "gen_alu1: bad scratch registers");
1922 gen_insn(INSN_CWD, OP_SIZE_NATIVE, 0, 0);
1923 gen_one(R_DX);
1924 gen_one(R_AX);
1926 g(gen_address(ctx, R_FRAME, (size_t)slot_r * slot_size + lo_word(OP_SIZE_NATIVE), IMM_PURPOSE_STR_OFFSET, OP_SIZE_NATIVE));
1927 gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
1928 gen_address_offset();
1929 gen_one(R_SCRATCH_1);
1931 g(gen_address(ctx, R_FRAME, (size_t)slot_r * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_STR_OFFSET, OP_SIZE_NATIVE));
1932 gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
1933 gen_address_offset();
1934 gen_one(R_SCRATCH_2);
1936 return true;
1937 }
1938 if (alu == ALU1_POPCNT) {
1939 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + lo_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
1940 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_POPCNT, ALU1_WRITES_FLAGS(ALU1_POPCNT));
1941 gen_one(R_SCRATCH_1);
1942 gen_address_offset();
1944 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
1945 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_POPCNT, ALU1_WRITES_FLAGS(ALU1_POPCNT));
1946 gen_one(R_SCRATCH_2);
1947 gen_address_offset();
1949 g(gen_3address_alu(ctx, OP_SIZE_4, ALU_ADD, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_2, 1));
1950 write_result:
1951 g(gen_address(ctx, R_FRAME, (size_t)slot_r * slot_size + lo_word(OP_SIZE_NATIVE), IMM_PURPOSE_STR_OFFSET, OP_SIZE_NATIVE));
1952 gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
1953 gen_address_offset();
1954 gen_one(R_SCRATCH_1);
1956 g(gen_address(ctx, R_FRAME, (size_t)slot_r * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_STR_OFFSET, OP_SIZE_NATIVE));
1957 g(gen_imm(ctx, 0, IMM_PURPOSE_STORE_VALUE, OP_SIZE_NATIVE));
1958 gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
1959 gen_address_offset();
1960 gen_imm_offset();
1962 return true;
1963 }
1964 #endif
1965 goto do_generic_bsf_bsr_popcnt;
1966 }
1967 #if defined(ARCH_X86)
1968 if (alu == ALU1_POPCNT && unlikely(!cpu_test_feature(CPU_FEATURE_popcnt)))
1969 goto do_generic_bsf_bsr_popcnt;
1970 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1971 if (op_size == OP_SIZE_1 || ((alu == ALU1_BSR || alu == ALU1_POPCNT) && mode == MODE_INT)) {
1972 g(gen_frame_get(ctx, op_size, zero_x, slot_1, 0, target, ®1));
1973 if ((alu == ALU1_BSR || alu == ALU1_POPCNT) && mode == MODE_INT) {
1974 g(gen_cmp_test_jmp(ctx, INSN_TEST, op_size, reg1, reg1, alu == ALU1_BSR ? COND_LE : COND_S, label_ovf));
1975 }
1976 g(gen_2address_alu1(ctx, maximum(op_size, OP_SIZE_2), alu, target, reg1, 1));
1977 if ((alu == ALU1_BSR || alu == ALU1_POPCNT) && mode == MODE_INT)
1978 goto x86_bsf_bsr_popcnt_finish;
1979 } else {
1980 g(gen_frame_load_op1(ctx, op_size, alu, 1, slot_1, 0, target));
1981 }
1982 if (alu == ALU1_POPCNT)
1983 goto x86_bsf_bsr_popcnt_finish;
1984 if (mode == MODE_FIXED) {
1985 uint32_t cmov_label;
1986 gen_insn(INSN_MOV, maximum(op_size, OP_SIZE_4), 0, 0);
1987 gen_one(R_SCRATCH_2);
1988 gen_one(ARG_IMM);
1989 gen_eight(-1);
1990 g(gen_cmov(ctx, maximum(op_size, OP_SIZE_4), COND_E, target, &cmov_label));
1991 gen_one(R_SCRATCH_2);
1992 if (cmov_label)
1993 gen_label(cmov_label);
1995 } else {
1996 gen_insn(INSN_JMP_COND, maximum(op_size, OP_SIZE_2), COND_E, 0);
1997 gen_four(label_ovf);
1998 }
1999 x86_bsf_bsr_popcnt_finish:
2000 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2001 return true;
2002 #endif
2003 #if defined(ARCH_ARM)
2004 #if defined(ARCH_ARM32)
2005 if (alu == ALU1_BSR && unlikely(!cpu_test_feature(CPU_FEATURE_armv6)))
2006 goto do_generic_bsf_bsr_popcnt;
2007 if (alu == ALU1_BSF && unlikely(!cpu_test_feature(CPU_FEATURE_armv6t2)))
2008 goto do_generic_bsf_bsr_popcnt;
2009 #endif
2010 if (alu == ALU1_POPCNT && unlikely(!cpu_test_feature(CPU_FEATURE_neon)))
2011 goto do_generic_bsf_bsr_popcnt;
2012 g(gen_frame_get(ctx, op_size, mode == MODE_INT ? sign_x : zero_x, slot_1, 0, R_SCRATCH_1, ®1));
2013 if (mode == MODE_INT) {
2014 g(gen_cmp_test_jmp(ctx, INSN_TEST, i_size(op_size), reg1, reg1, alu == ALU1_BSR ? COND_LE : alu == ALU1_BSF ? COND_E : COND_S, label_ovf));
2015 }
2017 if (alu == ALU1_POPCNT) {
2018 g(gen_mov(ctx, OP_SIZE_NATIVE, FR_SCRATCH_1, reg1));
2019 gen_insn(INSN_FP_ALU1, OP_SIZE_NATIVE, FP_ALU1_VCNT8, 0);
2020 gen_one(FR_SCRATCH_1);
2021 gen_one(FR_SCRATCH_1);
2022 #if defined(ARCH_ARM32)
2023 if (op_size > OP_SIZE_1) {
2024 gen_insn(INSN_FP_ALU1, OP_SIZE_1, FP_ALU1_VPADDL, 0);
2025 gen_one(FR_SCRATCH_1);
2026 gen_one(FR_SCRATCH_1);
2027 }
2028 if (op_size > OP_SIZE_2) {
2029 gen_insn(INSN_FP_ALU1, OP_SIZE_2, FP_ALU1_VPADDL, 0);
2030 gen_one(FR_SCRATCH_1);
2031 gen_one(FR_SCRATCH_1);
2032 }
2033 #else
2034 if (op_size > OP_SIZE_1) {
2035 gen_insn(INSN_FP_ALU1, OP_SIZE_1, FP_ALU1_ADDV, 0);
2036 gen_one(FR_SCRATCH_1);
2037 gen_one(FR_SCRATCH_1);
2038 }
2039 #endif
2040 g(gen_frame_store(ctx, op_size, slot_r, 0, FR_SCRATCH_1));
2041 if (ctx->registers[slot_r] >= 0)
2042 g(unspill(ctx, slot_r));
2043 return true;
2044 }
2046 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
2047 if (mode == MODE_FIXED && alu == ALU1_BSF) {
2048 gen_insn(INSN_TEST, i_size(op_size), 0, 1);
2049 gen_one(target);
2050 gen_one(reg1);
2051 reg1 = target;
2052 }
2054 if (alu == ALU1_BSF) {
2055 g(gen_2address_alu1(ctx, i_size(op_size), ALU1_BREV, target, reg1, 0));
2056 reg1 = target;
2057 }
2059 g(gen_2address_alu1(ctx, i_size(op_size), ALU1_LZCNT, target, reg1, 0));
2061 if (alu == ALU1_BSR) {
2062 g(gen_load_constant(ctx, R_SCRATCH_2, op_size == OP_SIZE_8 ? 63 : 31));
2063 g(gen_3address_alu(ctx, i_size(op_size), ALU_SUB, target, R_SCRATCH_2, target, 0));
2064 }
2066 if (mode == MODE_FIXED && alu == ALU1_BSF) {
2067 #if defined(ARCH_ARM32)
2068 g(gen_imm(ctx, -1, IMM_PURPOSE_CMOV, OP_SIZE_NATIVE));
2069 gen_insn(INSN_CMOV, OP_SIZE_NATIVE, COND_E, 0);
2070 gen_one(target);
2071 gen_one(target);
2072 gen_imm_offset();
2073 #else
2074 gen_insn(INSN_CSEL_INV, i_size(op_size), COND_NE, 0);
2075 gen_one(target);
2076 gen_one(ARG_IMM);
2077 gen_eight(0);
2078 gen_one(target);
2079 #endif
2080 }
2082 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2083 return true;
2084 #endif
2085 #if defined(ARCH_ALPHA)
2086 if (likely(cpu_test_feature(CPU_FEATURE_cix))) {
2087 g(gen_frame_get(ctx, op_size, mode == MODE_INT ? sign_x : zero_x, slot_1, 0, R_SCRATCH_1, ®1));
2088 target = gen_frame_target(ctx, slot_r, slot_1, NO_FRAME_T, R_SCRATCH_2);
2089 if (mode == MODE_INT) {
2090 g(gen_cmp_test_jmp(ctx, INSN_TEST, OP_SIZE_NATIVE, reg1, reg1, alu == ALU1_BSR ? COND_LE : alu == ALU1_BSF ? COND_E : COND_S, label_ovf));
2091 }
2092 if (alu == ALU1_POPCNT) {
2093 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_POPCNT, target, reg1, 0));
2094 }
2095 if (alu == ALU1_BSF) {
2096 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_BSF, target, reg1, 0));
2098 if (mode == MODE_FIXED) {
2099 g(gen_imm(ctx, -1, IMM_PURPOSE_MOVR, OP_SIZE_INT));
2100 gen_insn(INSN_MOVR, OP_SIZE_NATIVE, COND_E, 0);
2101 gen_one(target);
2102 gen_one(target);
2103 gen_one(reg1);
2104 gen_imm_offset();
2105 }
2106 }
2107 if (alu == ALU1_BSR) {
2108 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_LZCNT, target, reg1, 0));
2110 g(gen_load_constant(ctx, R_SCRATCH_3, OP_SIZE_NATIVE == OP_SIZE_8 ? 63 : 31));
2112 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_SUB, target, R_SCRATCH_3, target, 0));
2113 }
2114 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2115 return true;
2116 }
2117 #endif
2118 #if defined(ARCH_MIPS)
2119 if (MIPS_HAS_CLZ && alu != ALU1_POPCNT) {
2120 g(gen_frame_get(ctx, op_size, mode == MODE_INT ? sign_x : zero_x, slot_1, 0, R_SCRATCH_1, ®1));
2121 target = gen_frame_target(ctx, slot_r, slot_1, NO_FRAME_T, R_SCRATCH_2);
2122 if (mode == MODE_INT) {
2123 g(gen_cmp_test_jmp(ctx, INSN_TEST, OP_SIZE_NATIVE, reg1, reg1, alu == ALU1_BSR ? COND_LE : alu == ALU1_BSF ? COND_E : COND_S, label_ovf));
2124 }
2125 if (alu == ALU1_BSF) {
2126 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NEG, target, reg1, 0));
2128 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_AND, R_SCRATCH_1, reg1, target, 0));
2129 reg1 = R_SCRATCH_1;
2130 }
2131 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_LZCNT, target, reg1, 0));
2133 g(gen_load_constant(ctx, R_SCRATCH_3, OP_SIZE_NATIVE == OP_SIZE_8 ? 63 : 31));
2135 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_SUB, target, R_SCRATCH_3, target, 0));
2137 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2138 return true;
2139 }
2140 #endif
2141 #if defined(ARCH_POWER)
2142 if (alu == ALU1_BSF && (unlikely(!cpu_test_feature(CPU_FEATURE_v203)) || unlikely(!cpu_test_feature(CPU_FEATURE_v30))))
2143 goto do_generic_bsf_bsr_popcnt;
2144 if (alu == ALU1_POPCNT && unlikely(!cpu_test_feature(CPU_FEATURE_v206)))
2145 goto do_generic_bsf_bsr_popcnt;
2146 g(gen_frame_get(ctx, op_size, mode == MODE_INT ? sign_x : zero_x, slot_1, 0, R_SCRATCH_1, ®1));
2147 target = gen_frame_target(ctx, slot_r, slot_1, NO_FRAME_T, R_SCRATCH_2);
2148 if (mode == MODE_INT) {
2149 g(gen_cmp_test_jmp(ctx, INSN_TEST, i_size(op_size), reg1, reg1, alu == ALU1_BSR ? COND_LE : alu == ALU1_BSF ? COND_E : COND_S, label_ovf));
2150 }
2151 if (alu == ALU1_POPCNT) {
2152 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_POPCNT, target, reg1, 0));
2153 }
2154 if (alu == ALU1_BSF) {
2155 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_BSF, target, reg1, 0));
2157 if (mode == MODE_FIXED) {
2158 g(gen_3address_alu(ctx, i_size(op_size), ALU_AND, R_SCRATCH_3, reg1, reg1, 1));
2160 g(gen_imm(ctx, -1, IMM_PURPOSE_CMOV, OP_SIZE_NATIVE));
2161 gen_insn(INSN_CMOV, OP_SIZE_NATIVE, COND_E, 0);
2162 gen_one(target);
2163 gen_one(target);
2164 gen_imm_offset();
2165 }
2166 }
2167 if (alu == ALU1_BSR) {
2168 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_LZCNT, target, reg1, 0));
2170 g(gen_load_constant(ctx, R_SCRATCH_3, OP_SIZE_NATIVE == OP_SIZE_8 ? 63 : 31));
2172 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_SUB, target, R_SCRATCH_3, target, 0));
2173 }
2174 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2175 return true;
2176 #endif
2177 #if defined(ARCH_LOONGARCH64) || defined(ARCH_RISCV64)
2178 #if defined(ARCH_LOONGARCH64)
2179 if (alu == ALU1_POPCNT)
2180 goto do_generic_bsf_bsr_popcnt;
2181 #endif
2182 #if defined(ARCH_RISCV64)
2183 if (unlikely(!cpu_test_feature(CPU_FEATURE_zbb)))
2184 goto do_generic_bsf_bsr_popcnt;
2185 #endif
2186 g(gen_frame_get(ctx, op_size, mode == MODE_INT ? sign_x : zero_x, slot_1, 0, R_SCRATCH_1, ®1));
2187 target = gen_frame_target(ctx, slot_r, slot_1, NO_FRAME_T, R_SCRATCH_2);
2188 if (mode == MODE_INT) {
2189 g(gen_cmp_test_jmp(ctx, INSN_TEST, OP_SIZE_NATIVE, reg1, reg1, alu == ALU1_BSR ? COND_LE : alu == ALU1_BSF ? COND_E : COND_S, label_ovf));
2190 }
2191 if (alu == ALU1_POPCNT) {
2192 g(gen_2address_alu1(ctx, maximum(OP_SIZE_4, op_size), ALU1_POPCNT, target, reg1, 0));
2193 }
2194 if (alu == ALU1_BSF) {
2195 g(gen_2address_alu1(ctx, maximum(OP_SIZE_4, op_size), ALU1_BSF, target, reg1, 0));
2197 if (mode == MODE_FIXED) {
2198 g(gen_imm(ctx, 1, IMM_PURPOSE_CMP, OP_SIZE_NATIVE));
2199 gen_insn(INSN_CMP_DEST_REG, OP_SIZE_NATIVE, COND_B, 0);
2200 gen_one(R_SCRATCH_3);
2201 gen_one(reg1);
2202 gen_imm_offset();
2204 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NEG, R_SCRATCH_3, R_SCRATCH_3, 0));
2206 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_OR, target, target, R_SCRATCH_3, 0));
2207 }
2208 }
2209 if (alu == ALU1_BSR) {
2210 g(gen_2address_alu1(ctx, maximum(OP_SIZE_4, op_size), ALU1_LZCNT, target, reg1, 0));
2212 g(gen_load_constant(ctx, R_SCRATCH_3, op_size <= OP_SIZE_4 ? 31 : 63));
2214 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_SUB, target, R_SCRATCH_3, target, 0));
2215 }
2216 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2217 return true;
2218 #endif
2219 #if defined(ARCH_IA64) || defined(ARCH_S390) || defined(ARCH_SPARC)
2220 if (alu == ALU1_BSF && !ARCH_HAS_ANDN)
2221 goto do_generic_bsf_bsr_popcnt;
2222 #if defined(ARCH_S390)
2223 if (!cpu_test_feature(CPU_FEATURE_misc_45) || !cpu_test_feature(CPU_FEATURE_misc_insn_ext_3))
2224 goto do_generic_bsf_bsr_popcnt;
2225 #endif
2226 #if defined(ARCH_SPARC)
2227 if (!SPARC_9)
2228 goto do_generic_bsf_bsr_popcnt;
2229 #endif
2230 g(gen_frame_get(ctx, op_size, mode == MODE_INT ? sign_x : zero_x, slot_1, 0, R_SCRATCH_1, ®1));
2231 target = gen_frame_target(ctx, slot_r, slot_1, NO_FRAME_T, R_SCRATCH_2);
2232 if (mode == MODE_INT) {
2233 g(gen_cmp_test_jmp(ctx, INSN_TEST, maximum(op_size, OP_SIZE_4), reg1, reg1, alu == ALU1_BSR ? COND_LE : alu == ALU1_BSF ? COND_E : COND_S, label_ovf));
2234 } else {
2235 if (ARCH_PREFERS_SX(op_size) && alu == ALU1_POPCNT && op_size < OP_SIZE_NATIVE) {
2236 g(gen_extend(ctx, op_size, zero_x, R_SCRATCH_1, reg1));
2237 reg1 = R_SCRATCH_1;
2238 }
2239 }
2240 if (alu == ALU1_POPCNT) {
2241 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_POPCNT, R_SCRATCH_1, reg1, 0));
2242 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
2243 return true;
2244 }
2245 if (alu == ALU1_BSF) {
2246 g(gen_3address_alu_imm(ctx, OP_SIZE_NATIVE, ALU_SUB, target, reg1, 1, 0));
2248 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_ANDN, target, target, reg1, 0));
2250 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_POPCNT, target, target, 0));
2252 if (mode == MODE_FIXED) {
2253 unsigned attr_unused test_reg = R_SCRATCH_1;
2254 #if defined(ARCH_S390)
2255 g(gen_imm(ctx, 0, COND_IS_LOGICAL(COND_E) ? IMM_PURPOSE_CMP_LOGICAL : IMM_PURPOSE_CMP, OP_SIZE_NATIVE));
2256 gen_insn(INSN_CMP, OP_SIZE_NATIVE, 0, 1 + COND_IS_LOGICAL(COND_E));
2257 gen_one(reg1);
2258 gen_imm_offset();
2260 g(gen_imm(ctx, -1, IMM_PURPOSE_CMOV, OP_SIZE_NATIVE));
2261 gen_insn(INSN_CMOV, OP_SIZE_NATIVE, COND_E, 0);
2262 gen_one(target);
2263 gen_one(target);
2264 gen_imm_offset();
2265 #else
2266 #if defined(ARCH_IA64)
2267 g(gen_cmp_dest_reg(ctx, OP_SIZE_NATIVE, reg1, (unsigned)-1, R_CMP_RESULT, 0, COND_NE));
2268 test_reg = R_CMP_RESULT;
2269 #endif
2270 g(gen_imm(ctx, -1, IMM_PURPOSE_MOVR, OP_SIZE_NATIVE));
2271 gen_insn(INSN_MOVR, OP_SIZE_NATIVE, COND_E, 0);
2272 gen_one(target);
2273 gen_one(target);
2274 gen_one(test_reg);
2275 gen_imm_offset();
2276 #endif
2277 }
2279 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2280 return true;
2281 }
2282 #endif
2283 do_generic_bsf_bsr_popcnt:
2284 if (alu == ALU1_BSF) {
2285 if (mode == MODE_FIXED)
2286 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_unary_bsf_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, 0);
2287 else
2288 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, INT_unary_bsf_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, label_ovf);
2289 }
2290 if (alu == ALU1_BSR) {
2291 if (mode == MODE_FIXED)
2292 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_unary_bsr_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, 0);
2293 else
2294 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, INT_unary_bsr_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, label_ovf);
2295 }
2296 if (alu == ALU1_POPCNT) {
2297 if (mode == MODE_FIXED)
2298 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_unary_popcnt_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, 0);
2299 else
2300 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, INT_unary_popcnt_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, label_ovf);
2301 }
2302 }
2303 /**************
2304 * CONVERSION *
2305 **************/
2306 do_fixed_conv:
2307 do_conv: {
2308 unsigned src_op_size, dest_op_size;
2309 const struct type *src_type, *dest_type;
2310 src_type = get_type_of_local(ctx, slot_1);
2311 dest_type = get_type_of_local(ctx, slot_r);
2313 if (TYPE_TAG_IS_FIXED(src_type->tag)) {
2314 src_op_size = TYPE_TAG_IDX_FIXED(src_type->tag) >> 1;
2315 } else {
2316 src_op_size = TYPE_TAG_IDX_INT(src_type->tag);
2317 }
2319 if (TYPE_TAG_IS_FIXED(dest_type->tag)) {
2320 dest_op_size = TYPE_TAG_IDX_FIXED(dest_type->tag) >> 1;
2321 } else {
2322 dest_op_size = TYPE_TAG_IDX_INT(dest_type->tag);
2323 }
2325 if (src_op_size <= OP_SIZE_NATIVE) {
2326 g(gen_frame_get(ctx, src_op_size, sign_x, slot_1, 0, R_SCRATCH_1, ®1));
2327 } else {
2328 #if defined(ARCH_X86)
2329 if (dest_op_size < src_op_size)
2330 g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_1, 0, R_SCRATCH_1));
2331 else
2332 #endif
2333 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_3));
2334 reg1 = R_SCRATCH_1;
2335 }
2337 if (dest_op_size >= src_op_size) {
2338 if (dest_op_size <= OP_SIZE_NATIVE) {
2339 g(gen_frame_store(ctx, dest_op_size, slot_r, 0, reg1));
2340 } else {
2341 if (src_op_size <= OP_SIZE_NATIVE) {
2342 #if defined(ARCH_X86)
2343 if (R_SCRATCH_1 != R_AX || R_SCRATCH_2 != R_DX)
2344 internal(file_line, "gen_alu1: bad scratch registers");
2345 if (reg1 == R_SCRATCH_1) {
2346 gen_insn(INSN_CWD, OP_SIZE_NATIVE, 0, 0);
2347 gen_one(R_DX);
2348 gen_one(R_AX);
2349 } else
2350 #endif
2351 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_2, reg1, (1U << (OP_SIZE_NATIVE + 3)) - 1, false));
2352 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, reg1, R_SCRATCH_2));
2353 } else {
2354 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, reg1, R_SCRATCH_3));
2355 }
2356 }
2357 return true;
2358 } else {
2359 if (src_op_size > OP_SIZE_NATIVE) {
2360 #if defined(ARCH_ARM)
2361 gen_insn(INSN_CMP, OP_SIZE_NATIVE, 0, 1);
2362 gen_one(R_SCRATCH_3);
2363 gen_one(ARG_SHIFTED_REGISTER);
2364 gen_one(ARG_SHIFT_ASR | ((1U << (OP_SIZE_NATIVE + 3)) - 1));
2365 gen_one(R_SCRATCH_1);
2367 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_NE, 0);
2368 gen_four(label_ovf);
2369 #elif defined(ARCH_X86)
2370 if (R_SCRATCH_1 != R_AX || R_SCRATCH_2 != R_DX)
2371 internal(file_line, "gen_alu1: bad scratch registers");
2372 gen_insn(INSN_CWD, OP_SIZE_NATIVE, 0, 0);
2373 gen_one(R_DX);
2374 gen_one(R_AX);
2376 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
2377 gen_insn(INSN_CMP, OP_SIZE_NATIVE, 0, 1);
2378 gen_one(R_SCRATCH_2);
2379 gen_address_offset();
2381 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_NE, 0);
2382 gen_four(label_ovf);
2383 #else
2384 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_2, R_SCRATCH_1, (1U << (OP_SIZE_NATIVE + 3)) - 1, 0));
2386 g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, R_SCRATCH_3, R_SCRATCH_2, COND_NE, label_ovf));
2387 #endif
2389 src_op_size = OP_SIZE_NATIVE;
2390 }
2391 if (src_op_size > dest_op_size) {
2392 g(gen_cmp_extended(ctx, OP_SIZE_NATIVE, dest_op_size, reg1, R_SCRATCH_2, label_ovf));
2393 }
2394 g(gen_frame_store(ctx, dest_op_size, slot_r, 0, reg1));
2395 return true;
2396 }
2397 }
2399 conv_uto_int: {
2400 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_uto_int_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, label_ovf);
2401 }
2403 conv_ufrom_int: {
2404 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_ufrom_int_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, label_ovf);
2405 }
2406 }
2408 static bool attr_w gen_constant(struct codegen_context *ctx, bool real, unsigned op_size, bool shrt, frame_t slot_r)
2409 {
2410 uintbig_t c;
2411 if (shrt) {
2412 c = (int16_t)get_unaligned_16(ctx->current_position);
2413 } else switch (op_size) {
2414 #define fx(n, type, utype, sz, bits) \
2415 case n: \
2416 c = (type)cat(get_unaligned_,bits)(ctx->current_position);\
2417 break;
2418 for_all_fixed(fx);
2419 #undef fx
2420 default:
2421 internal(file_line, "gen_constant: invalid type %u", op_size);
2422 }
2423 if (op_size > OP_SIZE_NATIVE) {
2424 unsigned shift = (8U << OP_SIZE_NATIVE) - 1;
2425 g(gen_frame_store_imm_raw(ctx, OP_SIZE_NATIVE, slot_r, lo_word(OP_SIZE_NATIVE), c & ((2ULL << shift) - 1)));
2426 g(gen_frame_store_imm_raw(ctx, OP_SIZE_NATIVE, slot_r, hi_word(OP_SIZE_NATIVE), c >> 1 >> shift));
2427 if (real && ctx->registers[slot_r] >= 0)
2428 g(unspill(ctx, slot_r));
2429 return true;
2430 } else if (real && ctx->registers[slot_r] >= 0) {
2431 if (ARCH_HAS_FP_GP_MOV) {
2432 g(gen_load_constant(ctx, R_SCRATCH_1, c));
2433 g(gen_mov(ctx, op_size, ctx->registers[slot_r], R_SCRATCH_1));
2434 } else {
2435 g(gen_frame_store_imm_raw(ctx, op_size, slot_r, 0, c));
2436 g(unspill(ctx, slot_r));
2437 }
2438 } else {
2439 g(gen_frame_store_imm(ctx, op_size, slot_r, 0, c));
2440 }
2441 return true;
2442 }
2444 static bool attr_w gen_real_constant(struct codegen_context *ctx, const struct type *t, frame_t slot_r)
2445 {
2446 int64_t offset;
2447 if (is_power_of_2(t->size) && t->size <= sizeof(uintbig_t))
2448 return gen_constant(ctx, true, log_2(t->size), false, slot_r);
2450 g(load_function_offset(ctx, R_SCRATCH_3, offsetof(struct data, u_.function.code)));
2452 offset = (ctx->current_position - da(ctx->fn,function)->code) * sizeof(code_t);
2454 g(gen_memcpy_raw(ctx, R_FRAME, (size_t)slot_r * slot_size, R_SCRATCH_3, offset, t->size, minimum(t->align, sizeof(code_t))));
2455 if (ctx->registers[slot_r] >= 0)
2456 g(unspill(ctx, slot_r));
2458 return true;
2459 }
2461 static bool attr_w gen_copy(struct codegen_context *ctx, unsigned op_size, frame_t slot_1, frame_t slot_r)
2462 {
2463 unsigned reg1;
2464 if (unlikely(op_size > OP_SIZE_NATIVE)) {
2465 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
2466 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_1, R_SCRATCH_2));
2467 return true;
2468 } else {
2469 unsigned target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
2470 g(gen_frame_get(ctx, op_size, garbage, slot_1, 0, target, ®1));
2471 g(gen_frame_store(ctx, op_size, slot_r, 0, reg1));
2472 return true;
2473 }
2474 }
2476 static bool attr_w gen_fp_alu(struct codegen_context *ctx, unsigned real_type, unsigned op, uint32_t label_ovf, frame_t slot_1, frame_t slot_2, frame_t slot_r)
2477 {
2478 unsigned attr_unused fp_alu;
2479 size_t upc;
2480 unsigned attr_unused op_size = real_type_to_op_size(real_type);
2481 unsigned reg1, reg2, target;
2482 switch (op) {
2483 case OPCODE_REAL_OP_add:
2484 case OPCODE_REAL_OP_add_alt1:
2485 case OPCODE_REAL_OP_add_alt2: fp_alu = FP_ALU_ADD; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_add_real16_t); label_ovf = 0; goto do_alu;
2486 case OPCODE_REAL_OP_subtract:
2487 case OPCODE_REAL_OP_subtract_alt1:
2488 case OPCODE_REAL_OP_subtract_alt2: fp_alu = FP_ALU_SUB; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_subtract_real16_t); label_ovf = 0; goto do_alu;
2489 case OPCODE_REAL_OP_multiply:
2490 case OPCODE_REAL_OP_multiply_alt1:
2491 case OPCODE_REAL_OP_multiply_alt2: fp_alu = FP_ALU_MUL; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_multiply_real16_t); label_ovf = 0; goto do_alu;
2492 case OPCODE_REAL_OP_divide:
2493 case OPCODE_REAL_OP_divide_alt1:
2494 case OPCODE_REAL_OP_divide_alt2: fp_alu = FP_ALU_DIV; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_divide_real16_t); label_ovf = 0; goto do_alu;
2495 case OPCODE_REAL_OP_modulo:
2496 case OPCODE_REAL_OP_power:
2497 case OPCODE_REAL_OP_ldexp:
2498 case OPCODE_REAL_OP_atan2: upc = offsetof(struct cg_upcall_vector_s, REAL_binary_modulo_real16_t) + (op - OPCODE_REAL_OP_modulo) * TYPE_REAL_N * sizeof(void (*)(void)); goto do_upcall;
2499 case OPCODE_REAL_OP_equal:
2500 case OPCODE_REAL_OP_equal_alt1:
2501 case OPCODE_REAL_OP_equal_alt2: fp_alu = FP_COND_E; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_equal_real16_t); goto do_cmp;
2502 case OPCODE_REAL_OP_not_equal:
2503 case OPCODE_REAL_OP_not_equal_alt1:
2504 case OPCODE_REAL_OP_not_equal_alt2: fp_alu = FP_COND_NE; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_not_equal_real16_t); goto do_cmp;
2505 case OPCODE_REAL_OP_less:
2506 case OPCODE_REAL_OP_less_alt1:
2507 case OPCODE_REAL_OP_less_alt2: fp_alu = FP_COND_B; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_less_real16_t); goto do_cmp;
2508 case OPCODE_REAL_OP_less_equal:
2509 case OPCODE_REAL_OP_less_equal_alt1:
2510 case OPCODE_REAL_OP_less_equal_alt2: fp_alu = FP_COND_BE; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_less_equal_real16_t); goto do_cmp;
2511 default: internal(file_line, "gen_fp_alu: unsupported operation %u", op);
2512 }
2514 do_alu:
2515 if ((SUPPORTED_FP >> real_type) & 1) {
2516 #if defined(ARCH_IA64)
2517 if (unlikely(fp_alu == FP_ALU_DIV))
2518 goto do_upcall;
2519 #endif
2520 #if defined(ARCH_X86)
2521 if (1)
2522 #elif defined(ARCH_S390)
2523 if ((op_size <= OP_SIZE_8 && (size_t)slot_2 * slot_size < 4096) || ctx->registers[slot_2] >= 0)
2524 #else
2525 if (ctx->registers[slot_2] >= 0)
2526 #endif
2527 {
2528 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, slot_2, FR_SCRATCH_1);
2529 g(gen_frame_get(ctx, op_size, garbage, slot_1, 0, FR_SCRATCH_1, ®1));
2530 if (ctx->registers[slot_2] >= 0) {
2531 g(gen_3address_fp_alu(ctx, op_size, fp_alu, target, reg1, ctx->registers[slot_2]));
2532 } else {
2533 if (target != reg1 && !ARCH_IS_3ADDRESS_FP) {
2534 g(gen_mov(ctx, op_size, target, reg1));
2535 reg1 = target;
2536 }
2537 g(gen_address(ctx, R_FRAME, (size_t)slot_2 * slot_size, IMM_PURPOSE_VLDR_VSTR_OFFSET, op_size));
2538 gen_insn(INSN_FP_ALU, op_size, fp_alu, 0);
2539 gen_one(target);
2540 gen_one(reg1);
2541 gen_address_offset();
2542 }
2543 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2544 return true;
2545 }
2546 #if defined(ARCH_ALPHA)
2547 g(gen_frame_get(ctx, op_size, zero_x, slot_1, 0, FR_SCRATCH_1, ®1));
2548 g(gen_frame_get(ctx, op_size, zero_x, slot_2, 0, FR_SCRATCH_2, ®2));
2549 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, FR_SCRATCH_3);
2550 g(gen_3address_fp_alu(ctx, op_size, fp_alu, target, reg1, reg2));
2551 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2552 #else
2553 g(gen_frame_get(ctx, op_size, zero_x, slot_1, 0, FR_SCRATCH_1, ®1));
2554 g(gen_frame_get(ctx, op_size, zero_x, slot_2, 0, FR_SCRATCH_2, ®2));
2555 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_1);
2556 g(gen_3address_fp_alu(ctx, op_size, fp_alu, target, reg1, reg2));
2557 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2558 #endif
2559 return true;
2560 }
2561 #ifdef SUPPORTED_FP_X87
2562 if ((SUPPORTED_FP_X87 >> real_type) & 1) {
2563 if (real_type != 3) {
2564 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_2));
2565 g(gen_frame_load_x87(ctx, INSN_X87_ALU, op_size, fp_alu, slot_1));
2566 } else {
2567 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
2568 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_2));
2569 gen_insn(INSN_X87_ALUP, op_size, fp_alu, 0);
2570 gen_one(R_ST1);
2571 }
2572 g(gen_frame_store_x87(ctx, INSN_X87_FSTP, op_size, slot_r));
2573 return true;
2574 }
2575 #endif
2576 #ifdef SUPPORTED_FP_HALF_CVT
2577 if ((SUPPORTED_FP_HALF_CVT >> real_type) & 1) {
2578 g(gen_frame_get(ctx, op_size, zero_x, slot_1, 0, FR_SCRATCH_1, ®1));
2579 g(gen_frame_get(ctx, op_size, zero_x, slot_2, 0, FR_SCRATCH_2, ®2));
2580 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_1);
2581 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
2582 gen_one(FR_SCRATCH_1);
2583 gen_one(reg1);
2584 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
2585 gen_one(FR_SCRATCH_2);
2586 gen_one(reg2);
2587 gen_insn(INSN_FP_ALU, OP_SIZE_4, fp_alu, 0);
2588 gen_one(FR_SCRATCH_1);
2589 gen_one(FR_SCRATCH_1);
2590 gen_one(FR_SCRATCH_2);
2591 gen_insn(INSN_FP_CVT, OP_SIZE_4, op_size, 0);
2592 gen_one(target);
2593 gen_one(FR_SCRATCH_1);
2594 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2595 return true;
2596 }
2597 #endif
2598 goto do_upcall;
2600 do_cmp:
2601 if ((SUPPORTED_FP >> real_type) & 1
2602 #if defined(ARCH_ALPHA)
2603 && ARCH_SUPPORTS_TRAPS
2604 #endif
2605 ) {
2606 g(gen_frame_get(ctx, op_size, zero_x, slot_1, 0, FR_SCRATCH_1, ®1));
2607 g(gen_frame_get(ctx, op_size, zero_x, slot_2, 0, FR_SCRATCH_2, ®2));
2608 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
2609 #if defined(ARCH_ALPHA)
2610 gen_insn(INSN_FP_CMP_DEST_REG_TRAP, op_size, fp_alu == FP_COND_NE ? FP_COND_E : fp_alu, 0);
2611 gen_one(FR_SCRATCH_3);
2612 gen_one(reg1);
2613 gen_one(reg2);
2614 gen_four(label_ovf);
2616 if (!ARCH_HAS_FP_GP_MOV) {
2617 g(gen_frame_store_raw(ctx, OP_SIZE_4, slot_r, 0, FR_SCRATCH_3));
2618 g(gen_frame_load_raw(ctx, OP_SIZE_4, sign_x, slot_r, 0, target));
2619 } else {
2620 g(gen_mov(ctx, OP_SIZE_4, target, FR_SCRATCH_3));
2621 }
2623 if (fp_alu == FP_COND_NE) {
2624 g(gen_imm(ctx, 0, IMM_PURPOSE_CMP, OP_SIZE_NATIVE));
2625 gen_insn(INSN_CMP_DEST_REG, OP_SIZE_NATIVE, COND_E, 0);
2626 gen_one(target);
2627 gen_one(target);
2628 gen_imm_offset();
2629 } else {
2630 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SHR, target, target, 30, 0));
2631 }
2633 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
2635 return true;
2636 #elif defined(ARCH_IA64)
2637 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_P, 0);
2638 gen_one(R_CMP_RESULT);
2639 gen_one(reg1);
2640 gen_one(reg2);
2642 gen_insn(INSN_JMP_REG, OP_SIZE_NATIVE, COND_NE, 0);
2643 gen_one(R_CMP_RESULT);
2644 gen_four(label_ovf);
2646 gen_insn(INSN_FP_CMP_DEST_REG, op_size, fp_alu, 0);
2647 gen_one(R_CMP_RESULT);
2648 gen_one(reg1);
2649 gen_one(reg2);
2651 g(gen_mov(ctx, OP_SIZE_NATIVE, target, R_CMP_RESULT));
2653 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
2655 return true;
2656 #elif defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS) || defined(ARCH_PARISC)
2657 gen_insn(INSN_FP_CMP_COND, op_size, FP_COND_P, 1);
2658 gen_one(reg1);
2659 gen_one(reg2);
2661 gen_insn(INSN_JMP_FP_TEST, 0, FP_COND_P, 0);
2662 gen_four(label_ovf);
2664 gen_insn(INSN_FP_CMP_COND, op_size, fp_alu, 1);
2665 gen_one(reg1);
2666 gen_one(reg2);
2668 gen_insn(INSN_FP_TEST_REG, OP_SIZE_NATIVE, fp_alu, 0);
2669 gen_one(target);
2671 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
2673 return true;
2674 #elif defined(ARCH_RISCV64)
2675 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_E, 0);
2676 gen_one(R_SCRATCH_1);
2677 gen_one(reg1);
2678 gen_one(reg1);
2680 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_E, 0);
2681 gen_one(R_SCRATCH_2);
2682 gen_one(reg2);
2683 gen_one(reg2);
2685 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_AND, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_2, 0));
2687 g(gen_jmp_on_zero(ctx, OP_SIZE_NATIVE, R_SCRATCH_1, COND_E, label_ovf));
2689 gen_insn(INSN_FP_CMP_DEST_REG, op_size, fp_alu == FP_COND_NE ? FP_COND_E : fp_alu, 0);
2690 gen_one(target);
2691 gen_one(reg1);
2692 gen_one(reg2);
2694 if (fp_alu == FP_COND_NE) {
2695 g(gen_imm(ctx, 1, IMM_PURPOSE_XOR, OP_SIZE_NATIVE));
2696 gen_insn(INSN_ALU, OP_SIZE_NATIVE, ALU_XOR, ALU_WRITES_FLAGS(ALU_AND, false));
2697 gen_one(target);
2698 gen_one(target);
2699 gen_imm_offset();
2700 }
2702 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
2703 return true;
2704 #else
2705 gen_insn(INSN_FP_CMP, op_size, 0, 1);
2706 gen_one(reg1);
2707 gen_one(reg2);
2708 #if defined(ARCH_ARM32)
2709 gen_insn(INSN_FP_TO_INT_FLAGS, 0, 0, 1);
2710 #endif
2711 gen_insn(INSN_JMP_COND, op_size, FP_COND_P, 0);
2712 gen_four(label_ovf);
2713 g(gen_frame_set_cond(ctx, op_size, false, fp_alu, slot_r));
2714 return true;
2715 #endif
2716 }
2717 #ifdef SUPPORTED_FP_X87
2718 if ((SUPPORTED_FP_X87 >> real_type) & 1) {
2719 if (likely(cpu_test_feature(CPU_FEATURE_cmov))) {
2720 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_2));
2721 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
2722 gen_insn(INSN_X87_FCOMIP, op_size, 0, 0);
2723 gen_one(R_ST1);
2724 gen_insn(INSN_X87_FSTP, op_size, 0, 0);
2725 gen_one(R_ST0);
2726 gen_insn(INSN_JMP_COND, op_size, COND_P, 0);
2727 gen_four(label_ovf);
2728 g(gen_frame_set_cond(ctx, op_size, false, fp_alu & 0xf, slot_r));
2729 return true;
2730 }
2732 if (real_type != 3) {
2733 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
2734 g(gen_frame_load_x87(ctx, INSN_X87_FCOMP, op_size, 0, slot_2));
2735 } else {
2736 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_2));
2737 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
2738 gen_insn(INSN_X87_FCOMPP, op_size, 0, 0);
2739 }
2741 gen_insn(INSN_X87_FNSTSW, 0, 0, 0);
2742 gen_one(R_AX);
2743 gen_one(R_AX);
2745 gen_insn(INSN_TEST, OP_SIZE_2, 0, 1);
2746 gen_one(R_AX);
2747 gen_one(ARG_IMM);
2748 gen_eight(0x0400);
2750 gen_insn(INSN_JMP_COND, OP_SIZE_2, COND_NE, 0);
2751 gen_four(label_ovf);
2753 switch (fp_alu) {
2754 case FP_COND_E:
2755 gen_insn(INSN_TEST, OP_SIZE_2, 0, 1);
2756 gen_one(R_AX);
2757 gen_one(ARG_IMM);
2758 gen_eight(0x4000);
2759 g(gen_frame_set_cond(ctx, OP_SIZE_2, false, COND_NE, slot_r));
2760 break;
2761 case FP_COND_NE:
2762 gen_insn(INSN_TEST, OP_SIZE_2, 0, 1);
2763 gen_one(R_AX);
2764 gen_one(ARG_IMM);
2765 gen_eight(0x4000);
2766 g(gen_frame_set_cond(ctx, OP_SIZE_2, false, COND_E, slot_r));
2767 break;
2768 case FP_COND_B:
2769 gen_insn(INSN_TEST, OP_SIZE_2, 0, 1);
2770 gen_one(R_AX);
2771 gen_one(ARG_IMM);
2772 gen_eight(0x0100);
2773 g(gen_frame_set_cond(ctx, OP_SIZE_2, false, COND_NE, slot_r));
2774 break;
2775 case FP_COND_BE:
2776 gen_insn(INSN_TEST, OP_SIZE_2, 0, 1);
2777 gen_one(R_AX);
2778 gen_one(ARG_IMM);
2779 gen_eight(0x4100);
2780 g(gen_frame_set_cond(ctx, OP_SIZE_2, false, COND_NE, slot_r));
2781 break;
2782 default:
2783 internal(file_line, "gen_fp_alu: invalid condition %u", fp_alu);
2784 }
2785 return true;
2786 }
2787 #endif
2788 #ifdef SUPPORTED_FP_HALF_CVT
2789 if ((SUPPORTED_FP_HALF_CVT >> real_type) & 1) {
2790 g(gen_frame_get(ctx, op_size, zero_x, slot_1, 0, FR_SCRATCH_1, ®1));
2791 g(gen_frame_get(ctx, op_size, zero_x, slot_2, 0, FR_SCRATCH_2, ®2));
2792 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
2793 gen_one(FR_SCRATCH_1);
2794 gen_one(reg1);
2795 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
2796 gen_one(FR_SCRATCH_2);
2797 gen_one(reg2);
2798 gen_insn(INSN_FP_CMP, OP_SIZE_4, 0, 1);
2799 gen_one(FR_SCRATCH_1);
2800 gen_one(FR_SCRATCH_2);
2801 #if defined(ARCH_ARM32)
2802 gen_insn(INSN_FP_TO_INT_FLAGS, 0, 0, 1);
2803 #endif
2804 gen_insn(INSN_JMP_COND, op_size, FP_COND_P, 0);
2805 gen_four(label_ovf);
2806 g(gen_frame_set_cond(ctx, op_size, false, fp_alu, slot_r));
2807 return true;
2808 }
2809 #endif
2811 do_upcall:
2812 return gen_alu_typed_upcall(ctx, upc, real_type, slot_1, slot_2, slot_r, label_ovf);
2813 }
2815 #define OP_IS_ROUND(alu) ((alu) == FP_ALU1_ROUND || (alu) == FP_ALU1_FLOOR || (alu) == FP_ALU1_CEIL || (alu) == FP_ALU1_TRUNC)
2817 static bool attr_w gen_fp_alu1(struct codegen_context *ctx, unsigned real_type, unsigned op, uint32_t label_ovf, frame_t slot_1, frame_t slot_r)
2818 {
2819 unsigned attr_unused fp_alu;
2820 size_t upc;
2821 unsigned attr_unused op_size = real_type_to_op_size(real_type);
2822 unsigned reg1, target;
2823 switch (op) {
2824 case OPCODE_REAL_OP_neg:
2825 case OPCODE_REAL_OP_neg_alt1:
2826 case OPCODE_REAL_OP_neg_alt2: fp_alu = FP_ALU1_NEG; upc = offsetof(struct cg_upcall_vector_s, REAL_unary_neg_real16_t); label_ovf = 0; goto do_alu;
2827 case OPCODE_REAL_OP_sqrt:
2828 case OPCODE_REAL_OP_sqrt_alt1:
2829 case OPCODE_REAL_OP_sqrt_alt2: fp_alu = FP_ALU1_SQRT; upc = offsetof(struct cg_upcall_vector_s, REAL_unary_sqrt_real16_t); label_ovf = 0; goto do_alu;
2830 case OPCODE_REAL_OP_round: fp_alu = FP_ALU1_ROUND; upc = offsetof(struct cg_upcall_vector_s, REAL_unary_round_real16_t); label_ovf = 0; goto do_alu;
2831 case OPCODE_REAL_OP_floor: fp_alu = FP_ALU1_FLOOR; upc = offsetof(struct cg_upcall_vector_s, REAL_unary_floor_real16_t); label_ovf = 0; goto do_alu;
2832 case OPCODE_REAL_OP_ceil: fp_alu = FP_ALU1_CEIL; upc = offsetof(struct cg_upcall_vector_s, REAL_unary_ceil_real16_t); label_ovf = 0; goto do_alu;
2833 case OPCODE_REAL_OP_trunc: fp_alu = FP_ALU1_TRUNC; upc = offsetof(struct cg_upcall_vector_s, REAL_unary_trunc_real16_t); label_ovf = 0; goto do_alu;
2834 case OPCODE_REAL_OP_to_int:
2835 case OPCODE_REAL_OP_to_int_alt1:
2836 case OPCODE_REAL_OP_to_int_alt2: upc = offsetof(struct cg_upcall_vector_s, REAL_unary_to_int_real16_t); goto do_to_int;
2837 case OPCODE_REAL_OP_from_int:
2838 case OPCODE_REAL_OP_from_int_alt1:
2839 case OPCODE_REAL_OP_from_int_alt2: upc = offsetof(struct cg_upcall_vector_s, REAL_unary_from_int_real16_t); label_ovf = 0; goto do_from_int;
2840 case OPCODE_REAL_OP_is_exception:
2841 case OPCODE_REAL_OP_is_exception_alt1:
2842 case OPCODE_REAL_OP_is_exception_alt2: upc = offsetof(struct cg_upcall_vector_s, REAL_unary_is_exception_real16_t); label_ovf = 0; goto do_is_exception;
2843 default: upc = offsetof(struct cg_upcall_vector_s, REAL_unary_cbrt_real16_t) + (op - OPCODE_REAL_OP_cbrt) * TYPE_REAL_N * sizeof(void (*)(void)); label_ovf = 0; goto do_upcall;
2844 }
2846 do_alu:
2847 if ((SUPPORTED_FP >> real_type) & 1 && (
2848 #if defined(ARCH_ALPHA)
2849 fp_alu == FP_ALU1_NEG ||
2850 (fp_alu == FP_ALU1_SQRT && cpu_test_feature(CPU_FEATURE_fix)) ||
2851 #elif defined(ARCH_ARM32)
2852 fp_alu == FP_ALU1_NEG ||
2853 fp_alu == FP_ALU1_SQRT ||
2854 #elif defined(ARCH_ARM64)
2855 true ||
2856 #elif defined(ARCH_IA64)
2857 fp_alu == FP_ALU1_NEG ||
2858 #elif defined(ARCH_LOONGARCH64)
2859 fp_alu == FP_ALU1_NEG ||
2860 fp_alu == FP_ALU1_SQRT ||
2861 fp_alu == FP_ALU1_ROUND ||
2862 #elif defined(ARCH_MIPS)
2863 fp_alu == FP_ALU1_NEG ||
2864 (fp_alu == FP_ALU1_SQRT && MIPS_HAS_SQRT) ||
2865 #elif defined(ARCH_PARISC)
2866 (fp_alu == FP_ALU1_NEG && PA_20) ||
2867 fp_alu == FP_ALU1_SQRT ||
2868 #elif defined(ARCH_POWER)
2869 fp_alu == FP_ALU1_NEG ||
2870 (fp_alu == FP_ALU1_SQRT && cpu_test_feature(CPU_FEATURE_p2) && real_type != 4) ||
2871 #elif defined(ARCH_S390)
2872 true ||
2873 #elif defined(ARCH_SPARC)
2874 fp_alu == FP_ALU1_NEG ||
2875 fp_alu == FP_ALU1_SQRT ||
2876 #elif defined(ARCH_RISCV64)
2877 fp_alu == FP_ALU1_NEG ||
2878 fp_alu == FP_ALU1_SQRT ||
2879 #elif defined(ARCH_X86)
2880 fp_alu == FP_ALU1_SQRT ||
2881 (OP_IS_ROUND(fp_alu) && cpu_test_feature(CPU_FEATURE_sse41)) ||
2882 #endif
2883 false)) {
2884 #if defined(ARCH_S390)
2885 if (op_size <= OP_SIZE_8 && (size_t)slot_1 * slot_size < 4096 && fp_alu == FP_ALU1_SQRT) {
2886 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_1);
2887 if (ctx->registers[slot_1] >= 0) {
2888 gen_insn(INSN_FP_ALU1, op_size, fp_alu, 0);
2889 gen_one(target);
2890 gen_one(ctx->registers[slot_1]);
2891 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2892 } else {
2893 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size, IMM_PURPOSE_VLDR_VSTR_OFFSET, op_size));
2894 gen_insn(INSN_FP_ALU1, op_size, fp_alu, 0);
2895 gen_one(target);
2896 gen_address_offset();
2897 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2898 }
2899 return true;
2900 }
2901 #endif
2902 g(gen_frame_get(ctx, op_size, zero_x, slot_1, 0, FR_SCRATCH_1, ®1));
2903 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_2);
2904 gen_insn(INSN_FP_ALU1, op_size, fp_alu, 0);
2905 gen_one(target);
2906 gen_one(reg1);
2907 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2908 return true;
2909 }
2910 #ifdef SUPPORTED_FP_X87
2911 if ((SUPPORTED_FP_X87 >> real_type) & 1) {
2912 if (fp_alu == FP_ALU1_NEG) {
2913 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
2914 gen_insn(INSN_X87_FCHS, op_size, 0, 0);
2915 g(gen_frame_store_x87(ctx, INSN_X87_FSTP, op_size, slot_r));
2916 return true;
2917 } else if (fp_alu == FP_ALU1_SQRT) {
2918 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
2919 gen_insn(INSN_X87_FSQRT, op_size, 0, 0);
2920 g(gen_frame_store_x87(ctx, INSN_X87_FSTP, op_size, slot_r));
2921 return true;
2922 } else if (fp_alu == FP_ALU1_ROUND) {
2923 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
2924 gen_insn(INSN_X87_FRNDINT, op_size, 0, 0);
2925 g(gen_frame_store_x87(ctx, INSN_X87_FSTP, op_size, slot_r));
2926 return true;
2927 }
2928 }
2929 #endif
2930 #ifdef SUPPORTED_FP_HALF_CVT
2931 if ((SUPPORTED_FP_HALF_CVT >> real_type) & 1 && (
2932 #if defined(ARCH_ARM32)
2933 fp_alu == FP_ALU1_NEG ||
2934 fp_alu == FP_ALU1_SQRT ||
2935 #elif defined(ARCH_ARM64)
2936 true ||
2937 #elif defined(ARCH_X86)
2938 fp_alu == FP_ALU1_SQRT ||
2939 (OP_IS_ROUND(fp_alu) && cpu_test_feature(CPU_FEATURE_sse41)) ||
2940 #endif
2941 false)) {
2942 g(gen_frame_get(ctx, op_size, zero_x, slot_1, 0, FR_SCRATCH_1, ®1));
2943 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_1);
2944 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
2945 gen_one(target);
2946 gen_one(reg1);
2947 gen_insn(INSN_FP_ALU1, OP_SIZE_4, fp_alu, 0);
2948 gen_one(target);
2949 gen_one(target);
2950 gen_insn(INSN_FP_CVT, OP_SIZE_4, op_size, 0);
2951 gen_one(target);
2952 gen_one(target);
2953 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2954 return true;
2955 }
2956 #endif
2957 goto do_upcall;
2959 do_to_int:
2960 if ((SUPPORTED_FP >> real_type) & 1
2961 #if defined(ARCH_ALPHA)
2962 && ARCH_SUPPORTS_TRAPS
2963 #endif
2964 #if defined(ARCH_MIPS)
2965 && MIPS_HAS_TRUNC
2966 #endif
2967 ) {
2968 g(gen_frame_get(ctx, op_size, zero_x, slot_1, 0, FR_SCRATCH_1, ®1));
2969 goto do_cvt_to_int;
2970 do_cvt_to_int:
2971 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
2972 #if defined(ARCH_X86)
2973 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_TO_INT32 : INSN_FP_TO_INT64, op_size, 0, 0);
2974 gen_one(target);
2975 gen_one(reg1);
2977 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, OP_SIZE_INT, target, sign_bit(uint_default_t), COND_E, label_ovf));
2979 g(gen_frame_store(ctx, OP_SIZE_INT, slot_r, 0, target));
2980 return true;
2981 #endif
2982 #if defined(ARCH_ARM) || defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS)
2983 #if defined(ARCH_ARM)
2984 gen_insn(INSN_FP_CMP, op_size, 0, 1);
2985 gen_one(reg1);
2986 gen_one(reg1);
2987 #if defined(ARCH_ARM32)
2988 gen_insn(INSN_FP_TO_INT_FLAGS, 0, 0, 1);
2989 #endif
2990 gen_insn(INSN_JMP_COND, op_size, FP_COND_P, 0);
2991 gen_four(label_ovf);
2992 #else
2993 gen_insn(INSN_FP_CMP_COND, op_size, FP_COND_P, 1);
2994 gen_one(reg1);
2995 gen_one(reg1);
2997 gen_insn(INSN_JMP_FP_TEST, 0, FP_COND_P, 0);
2998 gen_four(label_ovf);
2999 #endif
3000 #if defined(ARCH_ARM32) || defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS)
3001 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_TO_INT32 : INSN_FP_TO_INT64, op_size, 0, 0);
3002 gen_one(FR_SCRATCH_1);
3003 gen_one(reg1);
3005 g(gen_mov(ctx, OP_SIZE_INT, target, FR_SCRATCH_1));
3006 #else
3007 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_TO_INT32 : INSN_FP_TO_INT64, op_size, 0, 0);
3008 gen_one(target);
3009 gen_one(reg1);
3010 #endif
3011 g(gen_imm(ctx, (int_default_t)(sign_bit(uint_default_t) + 1), IMM_PURPOSE_ADD, OP_SIZE_INT));
3012 gen_insn(INSN_ALU, OP_SIZE_INT, ALU_ADD, ALU_WRITES_FLAGS(ALU_ADD, is_imm()));
3013 gen_one(R_SCRATCH_2);
3014 gen_one(target);
3015 gen_imm_offset();
3017 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, OP_SIZE_INT, R_SCRATCH_2, 1, COND_BE, label_ovf));
3019 g(gen_frame_store(ctx, OP_SIZE_INT, slot_r, 0, target));
3020 return true;
3021 #endif
3022 #if defined(ARCH_IA64)
3023 gen_insn(INSN_FP_TO_INT64, op_size, 0, 0);
3024 gen_one(FR_SCRATCH_1);
3025 gen_one(reg1);
3027 g(gen_mov(ctx, OP_SIZE_NATIVE, target, FR_SCRATCH_1));
3029 if (OP_SIZE_INT == OP_SIZE_4) {
3030 g(gen_extend(ctx, OP_SIZE_4, sign_x, R_SCRATCH_2, target));
3031 g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, target, R_SCRATCH_2, COND_NE, label_ovf));
3032 } else {
3033 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, target, sign_bit(uint64_t), COND_E, label_ovf));
3034 }
3036 g(gen_frame_store(ctx, OP_SIZE_INT, slot_r, 0, target));
3037 return true;
3038 #endif
3039 #if defined(ARCH_PARISC) || defined(ARCH_POWER) || defined(ARCH_SPARC)
3040 #if defined(ARCH_POWER)
3041 if (!cpu_test_feature(CPU_FEATURE_ppc))
3042 goto do_upcall;
3043 if (OP_SIZE_INT == OP_SIZE_4)
3044 goto do_upcall;
3045 #endif
3046 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_TO_INT32 : INSN_FP_TO_INT64, op_size, 0, 0);
3047 gen_one(FR_SCRATCH_1);
3048 gen_one(reg1);
3050 g(gen_frame_store_raw(ctx, OP_SIZE_INT, slot_r, 0, FR_SCRATCH_1));
3051 if (ctx->registers[slot_r] >= 0)
3052 g(unspill(ctx, slot_r));
3053 g(gen_frame_load(ctx, OP_SIZE_INT, garbage, slot_r, 0, target));
3055 g(gen_imm(ctx, sign_bit(uint_default_t) + 1, IMM_PURPOSE_ADD, OP_SIZE_INT));
3056 gen_insn(INSN_ALU, i_size(OP_SIZE_INT), ALU_ADD, ALU_WRITES_FLAGS(ALU_ADD, is_imm()));
3057 gen_one(R_SCRATCH_2);
3058 gen_one(target);
3059 gen_imm_offset();
3061 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, OP_SIZE_INT, R_SCRATCH_2, 1, COND_BE, label_ovf));
3063 return true;
3064 #endif
3065 #if defined(ARCH_ALPHA)
3066 gen_insn(INSN_FP_TO_INT64_TRAP, op_size, 0, 0);
3067 gen_one(FR_SCRATCH_2);
3068 gen_one(reg1);
3069 gen_four(label_ovf);
3071 if (OP_SIZE_INT == OP_SIZE_4) {
3072 gen_insn(INSN_FP_INT64_TO_INT32_TRAP, 0, 0, 0);
3073 gen_one(FR_SCRATCH_3);
3074 gen_one(FR_SCRATCH_2);
3075 gen_four(label_ovf);
3076 g(gen_frame_store_raw(ctx, OP_SIZE_INT, slot_r, 0, FR_SCRATCH_3));
3077 } else {
3078 g(gen_frame_store_raw(ctx, OP_SIZE_INT, slot_r, 0, FR_SCRATCH_2));
3079 }
3080 if (ctx->registers[slot_r] >= 0)
3081 g(unspill(ctx, slot_r));
3082 return true;
3083 #endif
3084 #if defined(ARCH_S390)
3085 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_TO_INT32 : INSN_FP_TO_INT64, op_size, 0, 1);
3086 gen_one(target);
3087 gen_one(reg1);
3089 gen_insn(INSN_JMP_COND, op_size, FP_COND_P, 0);
3090 gen_four(label_ovf);
3092 g(gen_frame_store(ctx, OP_SIZE_INT, slot_r, 0, target));
3093 return true;
3094 #endif
3095 #if defined(ARCH_RISCV64)
3096 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_TO_INT32 : INSN_FP_TO_INT64, op_size, 0, 0);
3097 gen_one(target);
3098 gen_one(reg1);
3100 g(gen_load_constant(ctx, R_SCRATCH_2, sign_bit(int_default_t)));
3102 g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, target, R_SCRATCH_2, COND_E, label_ovf));
3104 g(gen_imm(ctx, -1, IMM_PURPOSE_XOR, i_size(size)));
3105 gen_insn(INSN_ALU, OP_SIZE_NATIVE, ALU_XOR, ALU_WRITES_FLAGS(ALU_XOR, is_imm()));
3106 gen_one(R_SCRATCH_2);
3107 gen_one(R_SCRATCH_2);
3108 gen_imm_offset();
3110 g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, target, R_SCRATCH_2, COND_E, label_ovf));
3112 g(gen_frame_store(ctx, OP_SIZE_INT, slot_r, 0, target));
3113 return true;
3114 #endif
3115 }
3116 #ifdef SUPPORTED_FP_X87
3117 if ((SUPPORTED_FP_X87 >> real_type) & 1) {
3118 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
3120 if (likely(cpu_test_feature(CPU_FEATURE_sse3))) {
3121 g(gen_frame_store_x87(ctx, INSN_X87_FISTTP, OP_SIZE_INT, slot_r));
3122 } else {
3123 gen_insn(INSN_PUSH, OP_SIZE_NATIVE, 0, 0);
3124 gen_one(ARG_IMM);
3125 gen_eight(0x0f7f);
3127 gen_insn(INSN_X87_FLDCW, 0, 0, 0);
3128 gen_one(ARG_ADDRESS_1);
3129 gen_one(R_SP);
3130 gen_eight(0);
3132 g(gen_frame_store_x87(ctx, INSN_X87_FISTP, OP_SIZE_INT, slot_r));
3134 gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
3135 gen_one(ARG_ADDRESS_1);
3136 gen_one(R_SP);
3137 gen_eight(0);
3138 gen_one(ARG_IMM);
3139 gen_eight(0x037f);
3141 gen_insn(INSN_X87_FLDCW, 0, 0, 0);
3142 gen_one(ARG_ADDRESS_1);
3143 gen_one(R_SP);
3144 gen_eight(0);
3146 gen_insn(INSN_ALU, i_size(OP_SIZE_ADDRESS), ALU_ADD, 1);
3147 gen_one(R_SP);
3148 gen_one(R_SP);
3149 gen_one(ARG_IMM);
3150 gen_eight(1 << OP_SIZE_NATIVE);
3151 }
3152 if (ctx->registers[slot_r] >= 0)
3153 g(unspill(ctx, slot_r));
3154 g(gen_frame_load(ctx, OP_SIZE_INT, garbage, slot_r, 0, R_SCRATCH_1));
3156 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, OP_SIZE_INT, R_SCRATCH_1, sign_bit(int_default_t), COND_E, label_ovf));
3158 return true;
3159 }
3160 #endif
3161 #ifdef SUPPORTED_FP_HALF_CVT
3162 if ((SUPPORTED_FP_HALF_CVT >> real_type) & 1) {
3163 g(gen_frame_get(ctx, op_size, zero_x, slot_1, 0, FR_SCRATCH_1, ®1));
3164 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
3165 gen_one(FR_SCRATCH_1);
3166 gen_one(reg1);
3167 reg1 = FR_SCRATCH_1;
3168 real_type = 1;
3169 op_size = real_type_to_op_size(real_type);
3170 goto do_cvt_to_int;
3171 }
3172 #endif
3173 goto do_upcall;
3175 do_from_int:
3176 if ((SUPPORTED_FP >> real_type) & 1) {
3177 #if defined(ARCH_ALPHA) || defined(ARCH_ARM32) || defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS) || defined(ARCH_PARISC) || defined(ARCH_POWER) || defined(ARCH_SPARC)
3178 int int_op_size = OP_SIZE_INT;
3179 #if defined(ARCH_POWER)
3180 if (int_op_size == OP_SIZE_4)
3181 goto do_upcall;
3182 if (op_size == OP_SIZE_4 && !cpu_test_feature(CPU_FEATURE_v206))
3183 goto do_upcall;
3184 if (op_size == OP_SIZE_8 && !cpu_test_feature(CPU_FEATURE_ppc))
3185 goto do_upcall;
3186 #endif
3187 if (ctx->registers[slot_1] >= 0)
3188 g(spill(ctx, slot_1));
3189 g(gen_frame_load_raw(ctx, int_op_size, zero_x, slot_1, 0, FR_SCRATCH_1));
3190 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_2);
3191 #if defined(ARCH_ALPHA)
3192 if (OP_SIZE_INT == OP_SIZE_4) {
3193 gen_insn(INSN_MOVSX, OP_SIZE_4, 0, 0);
3194 gen_one(FR_SCRATCH_1);
3195 gen_one(FR_SCRATCH_1);
3197 int_op_size = OP_SIZE_8;
3198 }
3199 #endif
3200 gen_insn(int_op_size == OP_SIZE_4 ? INSN_FP_FROM_INT32 : INSN_FP_FROM_INT64, op_size, 0, 0);
3201 gen_one(target);
3202 gen_one(FR_SCRATCH_1);
3204 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
3205 return true;
3206 #elif defined(ARCH_IA64)
3207 g(gen_frame_get(ctx, OP_SIZE_INT, sign_x, slot_1, 0, R_SCRATCH_1, ®1));
3208 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_1);
3210 g(gen_mov(ctx, OP_SIZE_NATIVE, target, reg1));
3212 gen_insn(INSN_FP_FROM_INT64, op_size, 0, 0);
3213 gen_one(target);
3214 gen_one(target);
3216 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
3217 return true;
3218 #else
3219 g(gen_frame_get(ctx, OP_SIZE_INT, garbage, slot_1, 0, R_SCRATCH_1, ®1));
3220 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_1);
3222 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_FROM_INT32 : INSN_FP_FROM_INT64, op_size, 0, 0);
3223 gen_one(target);
3224 gen_one(reg1);
3226 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
3227 return true;
3228 #endif
3229 }
3230 #ifdef SUPPORTED_FP_X87
3231 if ((SUPPORTED_FP_X87 >> real_type) & 1) {
3232 if (ctx->registers[slot_1] >= 0)
3233 g(spill(ctx, slot_1));
3234 g(gen_frame_load_x87(ctx, INSN_X87_FILD, OP_SIZE_INT, 0, slot_1));
3235 g(gen_frame_store_x87(ctx, INSN_X87_FSTP, op_size, slot_r));
3236 return true;
3237 }
3238 #endif
3239 #ifdef SUPPORTED_FP_HALF_CVT
3240 if ((SUPPORTED_FP_HALF_CVT >> real_type) & 1) {
3241 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_1);
3242 #if defined(ARCH_ARM32)
3243 g(gen_frame_get(ctx, OP_SIZE_INT, zero_x, slot_1, 0, FR_SCRATCH_1, ®1));
3245 gen_insn(INSN_FP_FROM_INT32, OP_SIZE_4, 0, 0);
3246 gen_one(target);
3247 gen_one(reg1);
3248 #else
3249 g(gen_frame_get(ctx, OP_SIZE_INT, garbage, slot_1, 0, R_SCRATCH_1, ®1));
3250 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_FROM_INT32 : INSN_FP_FROM_INT64, OP_SIZE_4, 0, 0);
3251 gen_one(target);
3252 gen_one(reg1);
3253 #endif
3254 gen_insn(INSN_FP_CVT, OP_SIZE_4, op_size, 0);
3255 gen_one(target);
3256 gen_one(target);
3257 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
3258 return true;
3259 }
3260 #endif
3261 goto do_upcall;
3263 do_is_exception:
3264 if ((SUPPORTED_FP >> real_type) & 1) {
3265 g(gen_frame_get(ctx, op_size, zero_x, slot_1, 0, FR_SCRATCH_1, ®1));
3266 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
3267 #if defined(ARCH_ALPHA)
3268 gen_insn(INSN_FP_CMP_UNORDERED_DEST_REG, op_size, 0, 0);
3269 gen_one(FR_SCRATCH_2);
3270 gen_one(FR_SCRATCH_1);
3271 gen_one(reg1);
3273 if (!cpu_test_feature(CPU_FEATURE_fix)) {
3274 g(gen_frame_store_raw(ctx, OP_SIZE_4, slot_r, 0, FR_SCRATCH_2));
3275 g(gen_frame_load_raw(ctx, OP_SIZE_4, sign_x, slot_r, 0, target));
3276 } else {
3277 g(gen_mov(ctx, OP_SIZE_4, target, FR_SCRATCH_2));
3278 }
3280 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SHR, target, target, 30, 0));
3282 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
3284 return true;
3285 #elif defined(ARCH_IA64)
3286 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_P, 0);
3287 gen_one(R_CMP_RESULT);
3288 gen_one(reg1);
3289 gen_one(reg1);
3291 g(gen_mov(ctx, OP_SIZE_NATIVE, target, R_CMP_RESULT));
3293 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
3294 #elif defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS) || defined(ARCH_PARISC)
3295 gen_insn(INSN_FP_CMP_COND, op_size, FP_COND_P, 1);
3296 gen_one(reg1);
3297 gen_one(reg1);
3299 gen_insn(INSN_FP_TEST_REG, OP_SIZE_NATIVE, FP_COND_P, 0);
3300 gen_one(target);
3302 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
3303 #elif defined(ARCH_RISCV64)
3304 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_E, 0);
3305 gen_one(target);
3306 gen_one(reg1);
3307 gen_one(reg1);
3309 g(gen_imm(ctx, 1, IMM_PURPOSE_XOR, OP_SIZE_NATIVE));
3310 gen_insn(INSN_ALU, OP_SIZE_NATIVE, ALU_XOR, ALU_WRITES_FLAGS(ALU_XOR, is_imm()));
3311 gen_one(target);
3312 gen_one(target);
3313 gen_imm_offset();
3315 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
3316 #else
3317 gen_insn(INSN_FP_CMP, op_size, 0, 1);
3318 gen_one(reg1);
3319 gen_one(reg1);
3320 #if defined(ARCH_ARM32)
3321 gen_insn(INSN_FP_TO_INT_FLAGS, 0, 0, 1);
3322 #endif
3323 g(gen_frame_set_cond(ctx, op_size, false, FP_COND_P, slot_r));
3324 #endif
3325 return true;
3326 }
3327 #ifdef SUPPORTED_FP_X87
3328 if ((SUPPORTED_FP_X87 >> real_type) & 1) {
3329 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
3330 if (likely(cpu_test_feature(CPU_FEATURE_cmov))) {
3331 gen_insn(INSN_X87_FCOMIP, op_size, 0, 0);
3332 gen_one(R_ST0);
3334 g(gen_frame_set_cond(ctx, op_size, false, COND_P, slot_r));
3335 return true;
3336 }
3338 gen_insn(INSN_X87_FCOMP, op_size, 0, 0);
3339 gen_one(R_ST0);
3341 gen_insn(INSN_X87_FNSTSW, 0, 0, 0);
3342 gen_one(R_AX);
3343 gen_one(R_AX);
3345 gen_insn(INSN_TEST, OP_SIZE_2, 0, 1);
3346 gen_one(R_AX);
3347 gen_one(ARG_IMM);
3348 gen_eight(0x0400);
3350 g(gen_frame_set_cond(ctx, op_size, false, COND_NE, slot_r));
3352 return true;
3353 }
3354 #endif
3355 #ifdef SUPPORTED_FP_HALF_CVT
3356 if ((SUPPORTED_FP_HALF_CVT >> real_type) & 1) {
3357 g(gen_frame_get(ctx, op_size, zero_x, slot_1, 0, FR_SCRATCH_1, ®1));
3358 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
3359 gen_one(FR_SCRATCH_1);
3360 gen_one(reg1);
3361 gen_insn(INSN_FP_CMP, OP_SIZE_4, 0, 1);
3362 gen_one(FR_SCRATCH_1);
3363 gen_one(FR_SCRATCH_1);
3364 #if defined(ARCH_ARM32)
3365 gen_insn(INSN_FP_TO_INT_FLAGS, 0, 0, 1);
3366 #endif
3367 g(gen_frame_set_cond(ctx, op_size, false, FP_COND_P, slot_r));
3368 return true;
3369 }
3370 #endif
3372 do_upcall:
3373 g(gen_alu_typed_upcall(ctx, upc, real_type, slot_1, NO_FRAME_T, slot_r, label_ovf));
3374 return true;
3375 }
3377 static bool attr_w gen_is_exception(struct codegen_context *ctx, frame_t slot_1, frame_t slot_r)
3378 {
3379 uint32_t no_ex_label, escape_label;
3380 const struct type *type = get_type_of_local(ctx, slot_1);
3382 no_ex_label = alloc_label(ctx);
3383 if (unlikely(!no_ex_label))
3384 return false;
3385 escape_label = alloc_escape_label(ctx);
3386 if (unlikely(!escape_label))
3387 return false;
3389 if (TYPE_IS_FLAT(type))
3390 g(gen_test_1_jz_cached(ctx, slot_1, no_ex_label));
3392 g(gen_frame_load(ctx, OP_SIZE_SLOT, zero_x, slot_1, 0, R_SCRATCH_1));
3393 g(gen_ptr_is_thunk(ctx, R_SCRATCH_1, slot_1, escape_label));
3395 if (!TYPE_IS_FLAT(type)) {
3396 g(gen_compare_da_tag(ctx, R_SCRATCH_1, DATA_TAG_flat, COND_E, escape_label, R_SCRATCH_1));
3397 }
3399 gen_label(no_ex_label);
3400 g(gen_frame_clear(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r));
3402 flag_set(ctx, slot_r, false);
3404 return true;
3405 }
3407 static bool attr_w gen_system_property(struct codegen_context *ctx, frame_t slot_1, frame_t slot_r)
3408 {
3409 uint32_t escape_label;
3411 escape_label = alloc_escape_label(ctx);
3412 if (unlikely(!escape_label))
3413 return false;
3415 g(gen_test_1_cached(ctx, slot_1, escape_label));
3417 g(gen_upcall_start(ctx, 1));
3419 g(gen_frame_load(ctx, OP_SIZE_INT, garbage, slot_1, 0, R_ARG0));
3420 g(gen_upcall_argument(ctx, 0));
3422 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, ipret_system_property), 1));
3424 g(gen_frame_store(ctx, OP_SIZE_INT, slot_r, 0, R_RET0));
3426 flag_set(ctx, slot_1, false);
3427 flag_set(ctx, slot_r, false);
3429 return true;
3430 }
3432 static bool attr_w gen_alu_jmp(struct codegen_context *ctx, unsigned mode, unsigned op_size, unsigned op, frame_t slot_1, frame_t slot_2, int32_t jmp_offset, bool *failed)
3433 {
3434 unsigned alu;
3435 enum extend ex;
3436 unsigned reg1;
3437 unsigned attr_unused reg2;
3438 #if 0
3439 *failed = true; return true;
3440 #endif
3441 switch (mode) {
3442 case MODE_FIXED: switch (op) {
3443 case OPCODE_FIXED_OP_equal: alu = COND_E; goto do_compare;
3444 case OPCODE_FIXED_OP_not_equal: alu = COND_NE; goto do_compare;
3445 case OPCODE_FIXED_OP_less: alu = COND_L; goto do_compare;
3446 case OPCODE_FIXED_OP_less_equal: alu = COND_LE; goto do_compare;
3447 case OPCODE_FIXED_OP_uless: alu = COND_B; goto do_compare;
3448 case OPCODE_FIXED_OP_uless_equal: alu = COND_BE; goto do_compare;
3449 case OPCODE_FIXED_OP_bt: *failed = true; return true;
3450 default: internal(file_line, "gen_alu_jmp: unsupported fixed operation %u", op);
3451 }
3452 case MODE_INT: switch (op) {
3453 case OPCODE_INT_OP_equal: alu = COND_E; goto do_compare;
3454 case OPCODE_INT_OP_not_equal: alu = COND_NE; goto do_compare;
3455 case OPCODE_INT_OP_less: alu = COND_L; goto do_compare;
3456 case OPCODE_INT_OP_less_equal: alu = COND_LE; goto do_compare;
3457 case OPCODE_INT_OP_bt: *failed = true; return true;
3458 default: internal(file_line, "gen_alu_jmp: unsupported int operation %u", op);
3459 }
3460 case MODE_BOOL: switch (op) {
3461 case OPCODE_BOOL_OP_and: alu = ALU_AND; mode = MODE_FIXED; goto do_alu;
3462 case OPCODE_BOOL_OP_or: alu = ALU_OR; mode = MODE_FIXED; goto do_alu;
3463 case OPCODE_BOOL_OP_equal: alu = COND_E; mode = MODE_FIXED; goto do_compare;
3464 case OPCODE_BOOL_OP_not_equal: alu = COND_NE; mode = MODE_FIXED; goto do_compare;
3465 case OPCODE_BOOL_OP_less: alu = COND_L; mode = MODE_FIXED; goto do_compare;
3466 case OPCODE_BOOL_OP_less_equal: alu = COND_LE; mode = MODE_FIXED; goto do_compare;
3467 default: internal(file_line, "gen_alu_jmp: unsupported bool operation %u", op);
3468 }
3469 }
3470 internal(file_line, "gen_alu_jmp: unsupported mode %u", mode);
3471 do_compare: {
3472 bool attr_unused logical;
3473 if (unlikely(op_size > OP_SIZE_NATIVE)) {
3474 *failed = true;
3475 return true;
3476 }
3477 if (ctx->registers[slot_2] >= 0 && ctx->registers[slot_1] < 0) {
3478 frame_t s = slot_1;
3479 slot_1 = slot_2;
3480 slot_2 = s;
3481 switch (alu) {
3482 case COND_L: alu = COND_G; break;
3483 case COND_LE: alu = COND_GE; break;
3484 case COND_B: alu = COND_A; break;
3485 case COND_BE: alu = COND_AE; break;
3486 }
3487 }
3488 ex = op_size == i_size_cmp(op_size) + (unsigned)zero ? garbage : alu == COND_L || alu == COND_LE || alu == COND_G || alu == COND_GE || ARCH_PREFERS_SX(op_size) ? sign_x : zero_x;
3489 g(gen_frame_get(ctx, op_size, ex, slot_1, 0, R_SCRATCH_1, ®1));
3490 if (ARCH_HAS_JMP_2REGS(alu)) {
3491 g(gen_frame_get(ctx, op_size, ex, slot_2, 0, R_SCRATCH_2, ®2));
3492 g(gen_jump(ctx, jmp_offset, i_size_cmp(op_size), alu ^ 1, reg1, reg2));
3493 return true;
3494 }
3495 #if ARCH_HAS_FLAGS
3496 logical = COND_IS_LOGICAL(alu ^ 1);
3497 g(gen_frame_load_cmp(ctx, op_size, logical, ex, false, slot_2, 0, reg1));
3498 g(gen_jump(ctx, jmp_offset, op_size, alu ^ 1, -1U, -1U));
3499 #else
3500 g(gen_frame_get(ctx, op_size, ex, slot_2, 0, R_SCRATCH_2, ®2));
3501 g(gen_cmp_dest_reg(ctx, op_size, reg1, reg2, R_CMP_RESULT, 0, alu));
3502 g(gen_jump(ctx, jmp_offset, OP_SIZE_NATIVE, COND_E, R_CMP_RESULT, -1U));
3503 #endif
3504 return true;
3505 }
3506 do_alu: {
3507 if (ctx->registers[slot_2] >= 0 && ctx->registers[slot_1] < 0) {
3508 frame_t s = slot_1;
3509 slot_1 = slot_2;
3510 slot_2 = s;
3511 }
3512 ex = op_size == i_size(op_size) + (unsigned)zero ? garbage : ARCH_PREFERS_SX(op_size) ? sign_x : zero_x;
3513 g(gen_frame_get(ctx, op_size, ex, slot_1, 0, R_SCRATCH_1, ®1));
3514 #if defined(ARCH_X86)
3515 if (alu == ALU_AND && ctx->registers[slot_2] < 0) {
3516 g(gen_address(ctx, R_FRAME, (size_t)slot_2 * slot_size, IMM_PURPOSE_LDR_OFFSET, op_size));
3517 gen_insn(INSN_TEST, i_size(op_size), 0, 1);
3518 gen_one(reg1);
3519 gen_address_offset();
3520 g(gen_jump(ctx, jmp_offset, op_size, COND_E, -1U, -1U));
3521 return true;
3522 }
3523 #endif
3524 g(gen_frame_get(ctx, op_size, ex, slot_2, 0, R_SCRATCH_2, ®2));
3525 #if ARCH_HAS_FLAGS && !defined(ARCH_S390)
3526 if (alu == ALU_AND) {
3527 gen_insn(INSN_TEST, i_size(op_size), 0, 1);
3528 gen_one(reg1);
3529 gen_one(reg2);
3530 g(gen_jump(ctx, jmp_offset, op_size, COND_E, -1U, -1U));
3531 return true;
3532 }
3533 #endif
3534 #if defined(ARCH_ARM64)
3535 if (alu == ALU_OR)
3536 goto skip_flags;
3537 #endif
3538 #if ARCH_HAS_FLAGS
3539 g(gen_3address_alu(ctx, i_size(op_size), alu, R_SCRATCH_1, reg1, reg2, 1));
3540 g(gen_jump(ctx, jmp_offset, i_size(op_size), COND_E, -1U, -1U));
3541 return true;
3542 #endif
3543 goto skip_flags;
3544 skip_flags:
3545 g(gen_3address_alu(ctx, i_size(op_size), alu, R_SCRATCH_1, reg1, reg2, 0));
3546 g(gen_jump(ctx, jmp_offset, i_size(op_size), COND_E, R_SCRATCH_1, -1U));
3547 return true;
3548 }
3549 }
3551 static bool attr_w gen_fp_alu_jmp(struct codegen_context *ctx, unsigned real_type, unsigned op, uint32_t label_ovf, frame_t slot_1, frame_t slot_2, int32_t jmp_offset, bool *failed)
3552 {
3553 unsigned attr_unused fp_alu;
3554 unsigned attr_unused op_size = real_type_to_op_size(real_type);
3555 unsigned reg1, reg2;
3556 unsigned attr_unused target;
3557 switch (op) {
3558 case OPCODE_REAL_OP_equal:
3559 case OPCODE_REAL_OP_equal_alt1:
3560 case OPCODE_REAL_OP_equal_alt2: fp_alu = FP_COND_E; goto do_cmp;
3561 case OPCODE_REAL_OP_not_equal:
3562 case OPCODE_REAL_OP_not_equal_alt1:
3563 case OPCODE_REAL_OP_not_equal_alt2: fp_alu = FP_COND_NE; goto do_cmp;
3564 case OPCODE_REAL_OP_less:
3565 case OPCODE_REAL_OP_less_alt1:
3566 case OPCODE_REAL_OP_less_alt2: fp_alu = FP_COND_B; goto do_cmp;
3567 case OPCODE_REAL_OP_less_equal:
3568 case OPCODE_REAL_OP_less_equal_alt1:
3569 case OPCODE_REAL_OP_less_equal_alt2: fp_alu = FP_COND_BE; goto do_cmp;
3570 default: internal(file_line, "gen_fp_alu_jmp: unsupported operation %u", op);
3571 }
3573 do_cmp:
3574 if ((SUPPORTED_FP >> real_type) & 1
3575 #if defined(ARCH_ALPHA)
3576 && ARCH_SUPPORTS_TRAPS && cpu_test_feature(CPU_FEATURE_fix)
3577 #endif
3578 ) {
3579 g(gen_frame_get(ctx, op_size, zero_x, slot_1, 0, FR_SCRATCH_1, ®1));
3580 g(gen_frame_get(ctx, op_size, zero_x, slot_2, 0, FR_SCRATCH_2, ®2));
3581 target = R_SCRATCH_1;
3582 #if defined(ARCH_ALPHA)
3583 gen_insn(INSN_FP_CMP_DEST_REG_TRAP, op_size, fp_alu == FP_COND_NE ? FP_COND_E : fp_alu, 0);
3584 gen_one(FR_SCRATCH_3);
3585 gen_one(reg1);
3586 gen_one(reg2);
3587 gen_four(label_ovf);
3589 g(gen_mov(ctx, OP_SIZE_4, target, FR_SCRATCH_3));
3591 if (fp_alu == FP_COND_NE) {
3592 g(gen_jump(ctx, jmp_offset, OP_SIZE_NATIVE, COND_NE, target, -1U));
3593 } else {
3594 g(gen_jump(ctx, jmp_offset, OP_SIZE_NATIVE, COND_E, target, -1U));
3595 }
3597 return true;
3598 #elif defined(ARCH_IA64)
3599 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_P, 0);
3600 gen_one(R_CMP_RESULT);
3601 gen_one(reg1);
3602 gen_one(reg2);
3604 gen_insn(INSN_JMP_REG, OP_SIZE_NATIVE, COND_NE, 0);
3605 gen_one(R_CMP_RESULT);
3606 gen_four(label_ovf);
3608 gen_insn(INSN_FP_CMP_DEST_REG, op_size, fp_alu, 0);
3609 gen_one(R_CMP_RESULT);
3610 gen_one(reg1);
3611 gen_one(reg2);
3613 g(gen_jump(ctx, jmp_offset, OP_SIZE_NATIVE, COND_E, R_CMP_RESULT, -1U);
3615 return true;
3616 #elif defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS) || defined(ARCH_PARISC)
3617 gen_insn(INSN_FP_CMP_COND, op_size, FP_COND_P, 1);
3618 gen_one(reg1);
3619 gen_one(reg2);
3621 gen_insn(INSN_JMP_FP_TEST, 0, FP_COND_P, 0);
3622 gen_four(label_ovf);
3624 gen_insn(INSN_FP_CMP_COND, op_size, fp_alu ^ 1, 1);
3625 gen_one(reg1);
3626 gen_one(reg2);
3628 g(gen_jump(ctx, jmp_offset, OP_SIZE_NATIVE, fp_alu ^ 1, -1U, -1U));
3630 return true;
3631 #elif defined(ARCH_RISCV64)
3632 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_E, 0);
3633 gen_one(R_SCRATCH_1);
3634 gen_one(reg1);
3635 gen_one(reg1);
3637 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_E, 0);
3638 gen_one(R_SCRATCH_2);
3639 gen_one(reg2);
3640 gen_one(reg2);
3642 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_AND, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_2, 0));
3644 g(gen_jmp_on_zero(ctx, OP_SIZE_NATIVE, R_SCRATCH_1, COND_E, label_ovf));
3646 gen_insn(INSN_FP_CMP_DEST_REG, op_size, fp_alu == FP_COND_NE ? FP_COND_E : fp_alu, 0);
3647 gen_one(target);
3648 gen_one(reg1);
3649 gen_one(reg2);
3651 if (fp_alu == FP_COND_NE) {
3652 g(gen_jump(ctx, jmp_offset, OP_SIZE_NATIVE, COND_NE, target, -1U));
3653 } else {
3654 g(gen_jump(ctx, jmp_offset, OP_SIZE_NATIVE, COND_E, target, -1U));
3655 }
3656 return true;
3657 #else
3658 gen_insn(INSN_FP_CMP, op_size, 0, 1);
3659 gen_one(reg1);
3660 gen_one(reg2);
3661 #if defined(ARCH_ARM32)
3662 gen_insn(INSN_FP_TO_INT_FLAGS, 0, 0, 1);
3663 #endif
3664 gen_insn(INSN_JMP_COND, op_size, FP_COND_P, 0);
3665 gen_four(label_ovf);
3666 g(gen_jump(ctx, jmp_offset, op_size, fp_alu ^ 1, -1U, -1U));
3667 return true;
3668 #endif
3669 }
3670 *failed = true;
3671 return true;
3672 }