| blob | cb13b902ca92d16b3bc787fb7f98da11eb588373 |
1 /*
2 * Copyright (C) 2024 Mikulas Patocka
3 *
4 * This file is part of Ajla.
5 *
6 * Ajla is free software: you can redistribute it and/or modify it under the
7 * terms of the GNU General Public License as published by the Free Software
8 * Foundation, either version 3 of the License, or (at your option) any later
9 * version.
10 *
11 * Ajla is distributed in the hope that it will be useful, but WITHOUT ANY
12 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
13 * A PARTICULAR PURPOSE. See the GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License along with
16 * Ajla. If not, see <https://www.gnu.org/licenses/>.
17 */
19 static bool attr_w gen_jump(struct codegen_context *ctx, int32_t jmp_offset, unsigned op_size, unsigned cond, unsigned reg1);
21 static bool attr_w gen_alu_upcall(struct codegen_context *ctx, size_t upcall, frame_t slot_1, frame_t slot_2, frame_t slot_r, uint32_t label_ovf)
22 {
23 if (ctx->registers[slot_1] >= 0)
24 g(spill(ctx, slot_1));
25 if (slot_2 != NO_FRAME_T && ctx->registers[slot_2] >= 0)
26 g(spill(ctx, slot_2));
27 g(gen_upcall_start(ctx, slot_2 != NO_FRAME_T ? 3 : 2));
28 g(gen_frame_address(ctx, slot_1, 0, R_ARG0));
29 g(gen_upcall_argument(ctx, 0));
30 if (slot_2 != NO_FRAME_T) {
31 g(gen_frame_address(ctx, slot_2, 0, R_ARG1));
32 g(gen_upcall_argument(ctx, 1));
33 g(gen_frame_address(ctx, slot_r, 0, R_ARG2));
34 g(gen_upcall_argument(ctx, 2));
35 g(gen_upcall(ctx, upcall, 3));
36 } else {
37 g(gen_frame_address(ctx, slot_r, 0, R_ARG1));
38 g(gen_upcall_argument(ctx, 1));
39 g(gen_upcall(ctx, upcall, 2));
40 }
41 if (ctx->registers[slot_r] >= 0)
42 g(unspill(ctx, slot_r));
43 if (label_ovf)
44 g(gen_jmp_on_zero(ctx, OP_SIZE_1, R_RET0, COND_E, label_ovf));
45 return true;
46 }
48 static bool attr_w gen_alu_typed_upcall(struct codegen_context *ctx, size_t upcall, unsigned op_size, frame_t slot_1, frame_t slot_2, frame_t slot_r, uint32_t label_ovf)
49 {
50 upcall += op_size * sizeof(void (*)(void));
51 return gen_alu_upcall(ctx, upcall, slot_1, slot_2, slot_r, label_ovf);
52 }
54 #define MODE_FIXED 0
55 #define MODE_INT 1
56 #define MODE_REAL 2
57 #define MODE_BOOL 3
59 static bool attr_w gen_alu(struct codegen_context *ctx, unsigned mode, unsigned op_size, unsigned op, uint32_t label_ovf, frame_t slot_1, frame_t slot_2, frame_t slot_r)
60 {
61 unsigned alu;
62 bool sgn, mod;
63 unsigned reg1, reg2, reg3, target;
64 switch (mode) {
65 case MODE_FIXED: switch (op) {
66 case OPCODE_FIXED_OP_add: alu = ALU_ADD; goto do_alu;
67 case OPCODE_FIXED_OP_subtract: alu = ALU_SUB; goto do_alu;
68 case OPCODE_FIXED_OP_multiply: goto do_multiply;
69 case OPCODE_FIXED_OP_divide:
70 case OPCODE_FIXED_OP_divide_alt1: sgn = true; mod = false; goto do_divide;
71 case OPCODE_FIXED_OP_udivide:
72 case OPCODE_FIXED_OP_udivide_alt1: sgn = false; mod = false; goto do_divide;
73 case OPCODE_FIXED_OP_modulo:
74 case OPCODE_FIXED_OP_modulo_alt1: sgn = true; mod = true; goto do_divide;
75 case OPCODE_FIXED_OP_umodulo:
76 case OPCODE_FIXED_OP_umodulo_alt1: sgn = false; mod = true; goto do_divide;
77 case OPCODE_FIXED_OP_power: return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_binary_power_int8_t), op_size, slot_1, slot_2, slot_r, 0);
78 case OPCODE_FIXED_OP_and: alu = ALU_AND; goto do_alu;
79 case OPCODE_FIXED_OP_or: alu = ALU_OR; goto do_alu;
80 case OPCODE_FIXED_OP_xor: alu = ALU_XOR; goto do_alu;
81 case OPCODE_FIXED_OP_shl: alu = ROT_SHL; goto do_shift;
82 case OPCODE_FIXED_OP_shr: alu = ROT_SAR; goto do_shift;
83 case OPCODE_FIXED_OP_ushr: alu = ROT_SHR; goto do_shift;
84 case OPCODE_FIXED_OP_rol: alu = ROT_ROL; goto do_shift;
85 case OPCODE_FIXED_OP_ror: alu = ROT_ROR; goto do_shift;
86 case OPCODE_FIXED_OP_bts: alu = BTX_BTS; goto do_bt;
87 case OPCODE_FIXED_OP_btr: alu = BTX_BTR; goto do_bt;
88 case OPCODE_FIXED_OP_btc: alu = BTX_BTC; goto do_bt;
89 case OPCODE_FIXED_OP_equal: alu = COND_E; goto do_compare;
90 case OPCODE_FIXED_OP_not_equal: alu = COND_NE; goto do_compare;
91 case OPCODE_FIXED_OP_less: alu = COND_L; goto do_compare;
92 case OPCODE_FIXED_OP_less_equal: alu = COND_LE; goto do_compare;
93 case OPCODE_FIXED_OP_uless: alu = COND_B; goto do_compare;
94 case OPCODE_FIXED_OP_uless_equal: alu = COND_BE; goto do_compare;
95 case OPCODE_FIXED_OP_bt: alu = BTX_BT; goto do_bt;
96 default: internal(file_line, "gen_alu: unsupported fixed operation %u", op);
97 }
98 case MODE_INT: switch (op) {
99 case OPCODE_INT_OP_add: alu = ALU_ADD; goto do_alu;
100 case OPCODE_INT_OP_subtract: alu = ALU_SUB; goto do_alu;
101 case OPCODE_INT_OP_multiply: goto do_multiply;
102 case OPCODE_INT_OP_divide:
103 case OPCODE_INT_OP_divide_alt1: sgn = true; mod = false; goto do_divide;
104 case OPCODE_INT_OP_modulo:
105 case OPCODE_INT_OP_modulo_alt1: sgn = true; mod = true; goto do_divide;
106 case OPCODE_INT_OP_power: return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, INT_binary_power_int8_t), op_size, slot_1, slot_2, slot_r, label_ovf);
107 case OPCODE_INT_OP_and: alu = ALU_AND; mode = MODE_FIXED; goto do_alu;
108 case OPCODE_INT_OP_or: alu = ALU_OR; mode = MODE_FIXED; goto do_alu;
109 case OPCODE_INT_OP_xor: alu = ALU_XOR; mode = MODE_FIXED; goto do_alu;
110 case OPCODE_INT_OP_shl: alu = ROT_SHL; goto do_shift;
111 case OPCODE_INT_OP_shr: alu = ROT_SAR; goto do_shift;
112 case OPCODE_INT_OP_bts: alu = BTX_BTS; goto do_bt;
113 case OPCODE_INT_OP_btr: alu = BTX_BTR; goto do_bt;
114 case OPCODE_INT_OP_btc: alu = BTX_BTC; goto do_bt;
115 case OPCODE_INT_OP_equal: alu = COND_E; goto do_compare;
116 case OPCODE_INT_OP_not_equal: alu = COND_NE; goto do_compare;
117 case OPCODE_INT_OP_less: alu = COND_L; goto do_compare;
118 case OPCODE_INT_OP_less_equal: alu = COND_LE; goto do_compare;
119 case OPCODE_INT_OP_bt: alu = BTX_BT; goto do_bt;
120 default: internal(file_line, "gen_alu: unsupported int operation %u", op);
121 }
122 case MODE_BOOL: switch (op) {
123 case OPCODE_BOOL_OP_and: alu = ALU_AND; mode = MODE_FIXED; goto do_alu;
124 case OPCODE_BOOL_OP_or: alu = ALU_OR; mode = MODE_FIXED; goto do_alu;
125 case OPCODE_BOOL_OP_equal: alu = COND_E; goto do_compare;
126 case OPCODE_BOOL_OP_not_equal: alu = ALU_XOR; mode = MODE_FIXED; goto do_alu;
127 case OPCODE_BOOL_OP_less: alu = COND_L; goto do_compare;
128 case OPCODE_BOOL_OP_less_equal: alu = COND_LE; goto do_compare;
129 default: internal(file_line, "gen_alu: unsupported bool operation %u", op);
130 }
131 }
132 internal(file_line, "gen_alu: unsupported mode %u", mode);
134 /*******
135 * ALU *
136 *******/
137 do_alu: {
138 size_t attr_unused offset;
139 uint8_t attr_unused long_imm;
140 unsigned first_flags;
141 unsigned second_flags;
142 unsigned second_alu;
143 unsigned attr_unused op_size_flags;
144 if (unlikely(op_size > OP_SIZE_NATIVE)) {
145 #if !defined(ARCH_X86) && !defined(ARCH_ARM) && !defined(ARCH_PARISC) && !defined(ARCH_POWER) && !defined(ARCH_SPARC32)
146 if (mode == MODE_FIXED) {
147 if (alu == ALU_ADD) {
148 g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_add_,TYPE_INT_MAX)), slot_1, slot_2, slot_r, 0));
149 return true;
150 } else if (alu == ALU_SUB) {
151 g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_subtract_,TYPE_INT_MAX)), slot_1, slot_2, slot_r, 0));
152 return true;
153 }
154 } else if (mode == MODE_INT) {
155 if (alu == ALU_ADD) {
156 g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(INT_binary_add_,TYPE_INT_MAX)), slot_1, slot_2, slot_r, label_ovf));
157 return true;
158 } else if (alu == ALU_SUB) {
159 g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(INT_binary_subtract_,TYPE_INT_MAX)), slot_1, slot_2, slot_r, label_ovf));
160 return true;
161 }
162 }
163 #endif
164 first_flags = alu == ALU_ADD || alu == ALU_SUB ? 2 : 0;
165 second_flags = mode == MODE_INT ? 1 : 0;
166 second_alu = alu == ALU_ADD ? ALU_ADC : alu == ALU_SUB ? ALU_SBB : alu;
167 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
168 #if defined(ARCH_X86)
169 g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, alu, first_flags, slot_2, lo_word(OP_SIZE_NATIVE), R_SCRATCH_1));
170 g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, second_alu, second_flags, slot_2, hi_word(OP_SIZE_NATIVE), R_SCRATCH_2));
171 #else
172 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_2, 0, R_SCRATCH_3, R_SCRATCH_4));
173 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, alu, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_3, first_flags));
174 #if defined(ARCH_PARISC)
175 if (mode == MODE_INT) {
176 gen_insn(INSN_ALU_FLAGS_TRAP, OP_SIZE_NATIVE, second_alu, ALU_WRITES_FLAGS(second_alu, false));
177 gen_one(R_SCRATCH_2);
178 gen_one(R_SCRATCH_2);
179 gen_one(R_SCRATCH_4);
180 gen_four(label_ovf);
181 } else
182 #endif
183 {
184 gen_insn(first_flags ? INSN_ALU_FLAGS : INSN_ALU, OP_SIZE_NATIVE, second_alu, second_flags | ALU_WRITES_FLAGS(second_alu, false));
185 gen_one(R_SCRATCH_2);
186 gen_one(R_SCRATCH_2);
187 gen_one(R_SCRATCH_4);
188 }
189 #endif
190 #if !defined(ARCH_PARISC)
191 if (mode == MODE_INT) {
192 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_O, 0);
193 gen_four(label_ovf);
194 }
195 #endif
196 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_1, R_SCRATCH_2));
197 return true;
198 }
200 if ((ARCH_HAS_FLAGS || ARCH_SUPPORTS_TRAPS) && slot_2 == slot_r && slot_1 != slot_2 && alu_is_commutative(alu)) {
201 frame_t x = slot_1;
202 slot_1 = slot_2;
203 slot_2 = x;
204 }
205 if ((ARCH_HAS_FLAGS || ARCH_SUPPORTS_TRAPS) && slot_1 == slot_r && (slot_1 != slot_2 || mode != MODE_INT) && i_size_cmp(op_size) == op_size + zero
206 #if defined(ARCH_POWER)
207 && op_size == OP_SIZE_NATIVE
208 #endif
209 ) {
210 struct cg_exit *ce;
211 unsigned undo_alu = alu == ALU_ADD ? ALU_SUB : ALU_ADD;
212 if (ctx->registers[slot_1] >= 0) {
213 unsigned reg1 = ctx->registers[slot_1];
214 if (ctx->registers[slot_2] >= 0) {
215 unsigned reg2 = ctx->registers[slot_2];
216 if (mode == MODE_INT && ARCH_SUPPORTS_TRAPS) {
217 gen_insn(INSN_ALU_TRAP, op_size, alu, ALU_WRITES_FLAGS(alu, false));
218 gen_one(reg1);
219 gen_one(reg1);
220 gen_one(reg2);
221 if (ARCH_TRAP_BEFORE) {
222 gen_four(label_ovf);
223 return true;
224 } else {
225 ce = alloc_undo_label(ctx);
226 if (unlikely(!ce))
227 return false;
228 gen_four(ce->undo_label);
229 goto do_undo_opcode;
230 }
231 }
232 g(gen_3address_alu(ctx, i_size(op_size), alu, reg1, reg1, reg2, mode == MODE_INT));
233 if (mode == MODE_INT) {
234 ce = alloc_undo_label(ctx);
235 if (unlikely(!ce))
236 return false;
237 gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
238 gen_four(ce->undo_label);
239 do_undo_opcode:
240 ce->undo_opcode = INSN_ALU + ARCH_PARTIAL_ALU(op_size);
241 ce->undo_op_size = i_size(op_size);
242 ce->undo_aux = undo_alu;
243 ce->undo_writes_flags = ALU_WRITES_FLAGS(undo_alu, false);
244 ce->undo_parameters[0] = reg1;
245 ce->undo_parameters[1] = reg1;
246 ce->undo_parameters[2] = reg2;
247 ce->undo_parameters_len = 3;
248 }
249 return true;
250 }
251 #if defined(ARCH_S390) || defined(ARCH_X86)
252 else {
253 size_t m;
254 int64_t offset = (size_t)slot_2 * slot_size;
255 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDR_OFFSET, i_size(op_size)));
256 gen_insn(INSN_ALU + ARCH_PARTIAL_ALU(op_size), i_size(op_size), alu, 1);
257 gen_one(reg1);
258 gen_one(reg1);
259 gen_address_offset();
260 if (mode == MODE_INT) {
261 ce = alloc_undo_label(ctx);
262 if (unlikely(!ce))
263 return false;
264 ce->undo_opcode = INSN_ALU + ARCH_PARTIAL_ALU(op_size);
265 ce->undo_op_size = i_size(op_size);
266 ce->undo_aux = undo_alu;
267 ce->undo_writes_flags = ARCH_HAS_FLAGS;
268 m = mark_params(ctx);
269 gen_one(reg1);
270 gen_one(reg1);
271 gen_address_offset();
272 copy_params(ctx, ce, m);
273 gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
274 gen_four(ce->undo_label);
275 }
276 return true;
277 }
278 #endif
279 }
280 #if defined(ARCH_X86)
281 else {
282 unsigned reg2;
283 size_t m;
284 int64_t offset = (size_t)slot_1 * slot_size;
285 g(gen_frame_get(ctx, op_size, garbage, slot_2, 0, R_SCRATCH_1, ®2));
286 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDR_OFFSET, i_size(op_size)));
287 gen_insn(INSN_ALU + ARCH_PARTIAL_ALU(op_size), i_size(op_size), alu, 1);
288 gen_address_offset();
289 gen_address_offset();
290 gen_one(reg2);
291 if (mode == MODE_INT) {
292 ce = alloc_undo_label(ctx);
293 if (unlikely(!ce))
294 return false;
295 ce->undo_opcode = INSN_ALU + ARCH_PARTIAL_ALU(op_size);
296 ce->undo_op_size = i_size(op_size);
297 ce->undo_aux = undo_alu;
298 ce->undo_writes_flags = ARCH_HAS_FLAGS;
299 m = mark_params(ctx);
300 gen_address_offset();
301 gen_address_offset();
302 gen_one(reg2);
303 copy_params(ctx, ce, m);
304 gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
305 gen_four(ce->undo_label);
306 }
307 return true;
308 }
309 #endif
310 }
312 #if defined(ARCH_X86)
313 if (1)
314 #elif defined(ARCH_S390)
315 if (op_size >= OP_SIZE_4)
316 #else
317 if (0)
318 #endif
319 {
320 if (mode == MODE_INT) {
321 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
322 } else {
323 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, slot_2, R_SCRATCH_1);
324 }
325 g(gen_frame_load(ctx, op_size, garbage, slot_1, 0, target));
326 g(gen_frame_load_op(ctx, op_size, garbage, alu, mode == MODE_INT, slot_2, 0, target));
327 goto check_ovf_store;
328 }
329 op_size_flags = !ARCH_HAS_FLAGS && !ARCH_SUPPORTS_TRAPS ? OP_SIZE_NATIVE : OP_SIZE_4;
330 #if defined(ARCH_POWER)
331 op_size_flags = OP_SIZE_NATIVE;
332 #endif
333 g(gen_frame_get(ctx, op_size, mode == MODE_INT && (op_size < op_size_flags || ARCH_SUPPORTS_TRAPS) ? sign_x : garbage, slot_1, 0, R_SCRATCH_1, ®1));
334 g(gen_frame_get(ctx, op_size, mode == MODE_INT && (op_size < op_size_flags || ARCH_SUPPORTS_TRAPS) ? sign_x : garbage, slot_2, 0, R_SCRATCH_2, ®2));
335 #if !ARCH_HAS_FLAGS
336 if (mode == MODE_INT && op_size >= OP_SIZE_4) {
337 if (ARCH_SUPPORTS_TRAPS) {
338 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
339 gen_insn(INSN_ALU_TRAP, op_size, alu, ALU_WRITES_FLAGS(alu, false));
340 gen_one(target);
341 gen_one(reg1);
342 gen_one(reg2);
343 gen_four(label_ovf);
344 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
345 return true;
346 }
347 if (op_size >= OP_SIZE_NATIVE) {
348 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_3);
349 g(gen_3address_alu(ctx, i_size(op_size), alu, target, reg1, reg2, 0));
350 #if defined(ARCH_IA64)
351 g(gen_3address_alu(ctx, i_size(op_size), ALU_XOR, R_SCRATCH_1, reg1, reg2, 0));
352 g(gen_3address_alu(ctx, i_size(op_size), ALU_XOR, R_SCRATCH_2, reg2, target, 0));
353 if (alu == ALU_ADD) {
354 g(gen_3address_alu(ctx, i_size(op_size), ALU_ANDN, R_SCRATCH_1, R_SCRATCH_2, R_SCRATCH_1, 0));
355 } else {
356 g(gen_3address_alu(ctx, i_size(op_size), ALU_ANDN, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_2, 0));
357 }
358 g(gen_cmp_test_jmp(ctx, INSN_TEST, i_size(op_size), R_SCRATCH_1, R_SCRATCH_1, COND_S, label_ovf));
359 #else
360 gen_insn(INSN_CMP_DEST_REG, i_size(op_size), COND_L, 0);
361 gen_one(R_SCRATCH_1);
362 if (alu == ALU_ADD) {
363 gen_one(target);
364 gen_one(reg1);
365 } else {
366 gen_one(reg1);
367 gen_one(target);
368 }
370 g(gen_imm(ctx, 0, IMM_PURPOSE_CMP, i_size(op_size)));
371 gen_insn(INSN_CMP_DEST_REG, i_size(op_size), COND_L, 0);
372 gen_one(R_SCRATCH_2);
373 gen_one(reg2);
374 gen_imm_offset();
376 g(gen_cmp_test_jmp(ctx, INSN_CMP, i_size(op_size), R_SCRATCH_1, R_SCRATCH_2, COND_NE, label_ovf));
377 #endif
378 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
379 return true;
380 }
381 }
382 #endif
383 if (mode == MODE_INT) {
384 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
385 } else if (!ARCH_IS_3ADDRESS(alu, mode == MODE_INT && op_size >= op_size_flags) && !alu_is_commutative(alu)) {
386 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, slot_2, R_SCRATCH_1);
387 } else {
388 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
389 }
390 g(gen_3address_alu(ctx, i_size(op_size), alu, target, reg1, reg2, mode == MODE_INT && op_size >= op_size_flags));
392 if (mode == MODE_INT && unlikely(op_size < op_size_flags)) {
393 g(gen_cmp_extended(ctx, op_size_flags, op_size, target, R_SCRATCH_2, label_ovf));
394 } else
395 check_ovf_store:
396 if (mode == MODE_INT) {
397 gen_insn(INSN_JMP_COND, op_size, COND_O, 0);
398 gen_four(label_ovf);
399 }
400 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
401 return true;
402 }
404 /************
405 * MULTIPLY *
406 ************/
407 do_multiply: {
408 size_t attr_unused offset;
409 uint8_t attr_unused long_imm;
410 if (unlikely(op_size > OP_SIZE_NATIVE) || unlikely(!ARCH_HAS_MUL)) {
411 if (mode == MODE_INT) {
412 g(gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, INT_binary_multiply_int8_t), op_size, slot_1, slot_2, slot_r, label_ovf));
413 return true;
414 }
415 #if defined(ARCH_X86)
416 g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_1, hi_word(OP_SIZE_NATIVE), R_CX));
417 g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_2, hi_word(OP_SIZE_NATIVE), R_AX));
418 g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, ALU_MUL, true, slot_2, lo_word(OP_SIZE_NATIVE), R_CX));
419 g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, ALU_MUL, true, slot_1, lo_word(OP_SIZE_NATIVE), R_AX));
420 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_ADD, R_CX, R_CX, R_AX, 0));
421 g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_2, lo_word(OP_SIZE_NATIVE), R_AX));
423 offset = (size_t)slot_1 * slot_size + lo_word(OP_SIZE_NATIVE);
424 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
425 gen_insn(INSN_MUL_L, OP_SIZE_NATIVE, 0, 1);
426 gen_one(R_AX);
427 gen_one(R_DX);
428 gen_one(R_AX);
429 gen_address_offset();
431 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_ADD, R_DX, R_DX, R_CX, 0));
433 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_AX, R_DX));
435 return true;
436 #elif defined(ARCH_ARM32)
437 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
438 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_2, 0, R_SCRATCH_3, R_SCRATCH_4));
440 g(gen_mov(ctx, OP_SIZE_NATIVE, R_SCRATCH_NA_1, R_SCRATCH_1));
442 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_MUL, R_SCRATCH_4, R_SCRATCH_1, R_SCRATCH_4, 0));
444 gen_insn(INSN_MADD, OP_SIZE_NATIVE, 0, 0);
445 gen_one(R_SCRATCH_2);
446 gen_one(R_SCRATCH_3);
447 gen_one(R_SCRATCH_2);
448 gen_one(R_SCRATCH_4);
450 gen_insn(INSN_MUL_L, OP_SIZE_NATIVE, 0, 0);
451 gen_one(R_SCRATCH_1);
452 gen_one(R_SCRATCH_4);
453 gen_one(R_SCRATCH_NA_1);
454 gen_one(R_SCRATCH_3);
456 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_ADD, R_SCRATCH_2, R_SCRATCH_2, R_SCRATCH_4, 0));
458 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_1, R_SCRATCH_2));
460 return true;
461 #elif defined(ARCH_ARM64)
462 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
463 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_2, 0, R_SCRATCH_3, R_SCRATCH_4));
465 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_UMULH, R_SCRATCH_NA_1, R_SCRATCH_1, R_SCRATCH_3, 0));
467 gen_insn(INSN_MADD, OP_SIZE_NATIVE, 0, 0);
468 gen_one(R_SCRATCH_NA_1);
469 gen_one(R_SCRATCH_2);
470 gen_one(R_SCRATCH_3);
471 gen_one(R_SCRATCH_NA_1);
473 gen_insn(INSN_MADD, OP_SIZE_NATIVE, 0, 0);
474 gen_one(R_SCRATCH_2);
475 gen_one(R_SCRATCH_1);
476 gen_one(R_SCRATCH_4);
477 gen_one(R_SCRATCH_NA_1);
479 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_MUL, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_3, 0));
481 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_1, R_SCRATCH_2));
483 return true;
484 #else
485 g(gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_binary_multiply_int8_t), op_size, slot_1, slot_2, slot_r, 0));
486 return true;
487 #endif
488 }
490 #if defined(ARCH_X86)
491 if (mode == MODE_INT) {
492 if (op_size != OP_SIZE_1 && slot_r == slot_1 && ctx->registers[slot_1] >= 0) {
493 struct cg_exit *ce;
494 target = ctx->registers[slot_1];
495 g(gen_mov(ctx, op_size, R_SCRATCH_1, target));
496 g(gen_frame_load_op(ctx, op_size, garbage, ALU_MUL, mode == MODE_INT, slot_2, 0, target));
497 ce = alloc_undo_label(ctx);
498 if (unlikely(!ce))
499 return false;
500 ce->undo_opcode = INSN_MOV;
501 ce->undo_op_size = op_size;
502 ce->undo_aux = 0;
503 ce->undo_writes_flags = 0;
504 ce->undo_parameters[0] = target;
505 ce->undo_parameters[1] = R_SCRATCH_1;
506 ce->undo_parameters_len = 2;
507 gen_insn(INSN_JMP_COND, op_size, COND_O, 0);
508 gen_four(ce->undo_label);
509 return true;
510 }
511 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
512 } else {
513 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, slot_2, R_SCRATCH_1);
514 }
515 if (op_size == OP_SIZE_1)
516 target = R_SCRATCH_1;
517 g(gen_frame_load(ctx, op_size, garbage, slot_1, 0, target));
518 g(gen_frame_load_op(ctx, op_size, garbage, ALU_MUL, mode == MODE_INT, slot_2, 0, target));
519 if (mode == MODE_INT) {
520 gen_insn(INSN_JMP_COND, op_size, COND_O, 0);
521 gen_four(label_ovf);
522 }
523 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
524 return true;
525 #endif
526 #if defined(ARCH_ALPHA)
527 if (mode == MODE_INT && op_size >= OP_SIZE_4 && ARCH_SUPPORTS_TRAPS) {
528 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
529 g(gen_frame_get(ctx, op_size, garbage, slot_1, 0, R_SCRATCH_1, ®1));
530 g(gen_frame_get(ctx, op_size, garbage, slot_2, 0, R_SCRATCH_2, ®2));
532 gen_insn(INSN_ALU_TRAP, op_size, ALU_MUL, ALU_WRITES_FLAGS(ALU_MUL, false));
533 gen_one(target);
534 gen_one(reg1);
535 gen_one(reg2);
536 gen_four(label_ovf);
537 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
539 return true;
540 }
541 #endif
542 #if defined(ARCH_ARM32)
543 if (mode == MODE_INT && op_size == OP_SIZE_4) {
544 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_3);
545 g(gen_frame_get(ctx, op_size, garbage, slot_1, 0, R_SCRATCH_1, ®1));
546 g(gen_frame_get(ctx, op_size, garbage, slot_2, 0, R_SCRATCH_2, ®2));
548 gen_insn(INSN_MUL_L, OP_SIZE_NATIVE, 0, 0);
549 gen_one(target);
550 gen_one(R_SCRATCH_4);
551 gen_one(reg1);
552 gen_one(reg2);
554 gen_insn(INSN_CMP, OP_SIZE_NATIVE, 0, 1);
555 gen_one(R_SCRATCH_4);
556 gen_one(ARG_SHIFTED_REGISTER);
557 gen_one(ARG_SHIFT_ASR | 0x1f);
558 gen_one(target);
560 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_NE, 0);
561 gen_four(label_ovf);
563 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
565 return true;
566 }
567 #endif
568 #if defined(ARCH_ARM64)
569 if (mode == MODE_INT && op_size == OP_SIZE_4) {
570 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
571 g(gen_frame_get(ctx, op_size, op_size < OP_SIZE_4 ? sign_x : garbage, slot_1, 0, R_SCRATCH_1, ®1));
572 g(gen_frame_get(ctx, op_size, op_size < OP_SIZE_4 ? sign_x : garbage, slot_2, 0, R_SCRATCH_2, ®2));
573 gen_insn(INSN_ALU, OP_SIZE_8, ALU_MUL, ALU_WRITES_FLAGS(ALU_MUL, false));
574 gen_one(target);
575 gen_one(ARG_EXTENDED_REGISTER);
576 gen_one(ARG_EXTEND_SXTW);
577 gen_one(reg1);
578 gen_one(ARG_EXTENDED_REGISTER);
579 gen_one(ARG_EXTEND_SXTW);
580 gen_one(reg2);
582 gen_insn(INSN_CMP, OP_SIZE_8, 0, 1);
583 gen_one(target);
584 gen_one(ARG_EXTENDED_REGISTER);
585 gen_one(ARG_EXTEND_SXTW);
586 gen_one(target);
588 gen_insn(INSN_JMP_COND, OP_SIZE_8, COND_NE, 0);
589 gen_four(label_ovf);
591 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
593 return true;
594 }
595 if (mode == MODE_INT && op_size == OP_SIZE_8) {
596 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
597 g(gen_frame_get(ctx, op_size, garbage, slot_1, 0, R_SCRATCH_1, ®1));
598 g(gen_frame_get(ctx, op_size, garbage, slot_2, 0, R_SCRATCH_2, ®2));
599 g(gen_3address_alu(ctx, OP_SIZE_8, ALU_SMULH, R_SCRATCH_3, reg1, reg2, 0));
601 g(gen_3address_alu(ctx, OP_SIZE_8, ALU_MUL, target, reg1, reg2, 0));
603 gen_insn(INSN_CMP, OP_SIZE_8, 0, 1);
604 gen_one(R_SCRATCH_3);
605 gen_one(ARG_SHIFTED_REGISTER);
606 gen_one(ARG_SHIFT_ASR | 0x3f);
607 gen_one(target);
609 gen_insn(INSN_JMP_COND, OP_SIZE_8, COND_NE, 0);
610 gen_four(label_ovf);
612 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
614 return true;
615 }
616 #endif
617 #if defined(ARCH_POWER)
618 if (mode == MODE_INT && op_size >= OP_SIZE_4) {
619 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
620 g(gen_frame_get(ctx, op_size, garbage, slot_1, 0, R_SCRATCH_1, ®1));
621 g(gen_frame_get(ctx, op_size, garbage, slot_2, 0, R_SCRATCH_2, ®2));
623 g(gen_3address_alu(ctx, op_size, ALU_MUL, target, reg1, reg2, 1));
625 gen_insn(INSN_JMP_COND, op_size, COND_O, 0);
626 gen_four(label_ovf);
628 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
630 return true;
631 }
632 #endif
633 #if defined(ARCH_LOONGARCH64) || (defined(ARCH_MIPS) && MIPS_R6) || defined(ARCH_RISCV64)
634 if (mode == MODE_INT && op_size == OP_SIZE_NATIVE) {
635 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
636 g(gen_frame_get(ctx, op_size, garbage, slot_1, 0, R_SCRATCH_1, ®1));
637 g(gen_frame_get(ctx, op_size, garbage, slot_2, 0, R_SCRATCH_2, ®2));
639 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_SMULH, R_SCRATCH_3, reg1, reg2, 0));
641 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_MUL, target, reg1, reg2, 0));
643 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_4, target, (8U << OP_SIZE_NATIVE) - 1, 0));
645 g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, R_SCRATCH_3, R_SCRATCH_4, COND_NE, label_ovf));
647 g(gen_frame_store(ctx, OP_SIZE_NATIVE, slot_r, 0, target));
649 return true;
650 }
651 #endif
652 #if defined(ARCH_S390)
653 if (mode == MODE_INT && op_size >= OP_SIZE_4 && likely(cpu_test_feature(CPU_FEATURE_misc_insn_ext_2))) {
654 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
655 g(gen_frame_load(ctx, op_size, sign_x, slot_1, 0, target));
656 g(gen_frame_load_op(ctx, op_size, sign_x, ALU_MUL, 1, slot_2, 0, target));
658 gen_insn(INSN_JMP_COND, op_size, COND_O, 0);
659 gen_four(label_ovf);
661 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
662 return true;
663 }
664 #endif
665 #if (defined(ARCH_MIPS) && !MIPS_R6) || defined(ARCH_S390)
666 #if defined(ARCH_MIPS)
667 if (mode == MODE_INT && op_size >= OP_SIZE_4)
668 #endif
669 #if defined(ARCH_S390)
670 if (mode == MODE_INT && op_size == OP_SIZE_4)
671 #endif
672 {
673 #if defined(ARCH_S390)
674 target = R_SCRATCH_1;
675 #else
676 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
677 #endif
678 g(gen_frame_get(ctx, op_size, sign_x, slot_1, 0, R_SCRATCH_1, ®1));
679 g(gen_frame_get(ctx, op_size, sign_x, slot_2, 0, R_SCRATCH_3, ®2));
681 gen_insn(INSN_MUL_L, op_size, 0, 0);
682 gen_one(target);
683 gen_one(R_SCRATCH_2);
684 gen_one(reg1);
685 gen_one(reg2);
687 g(gen_3address_rot_imm(ctx, op_size, ROT_SAR, R_SCRATCH_4, target, (8U << op_size) - 1, false));
689 g(gen_cmp_test_jmp(ctx, INSN_CMP, op_size, R_SCRATCH_2, R_SCRATCH_4, COND_NE, label_ovf));
691 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
692 return true;
693 }
694 #endif
695 if (mode == MODE_INT && op_size == OP_SIZE_NATIVE) {
696 g(gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, INT_binary_multiply_int8_t), op_size, slot_1, slot_2, slot_r, label_ovf));
697 return true;
698 }
700 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
701 if (op_size < OP_SIZE_NATIVE && mode == MODE_INT) {
702 g(gen_frame_get(ctx, op_size, sign_x, slot_1, 0, R_SCRATCH_1, ®1));
703 g(gen_frame_get(ctx, op_size, sign_x, slot_2, 0, R_SCRATCH_2, ®2));
705 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_MUL, target, reg1, reg2, 0));
706 } else {
707 g(gen_frame_load(ctx, op_size, sign_x, slot_1, 0, target));
708 g(gen_frame_load_op(ctx, op_size, sign_x, ALU_MUL, 0, slot_2, 0, target));
709 }
711 if (mode == MODE_INT) {
712 g(gen_cmp_extended(ctx, OP_SIZE_NATIVE, op_size, target, R_SCRATCH_2, label_ovf));
713 }
715 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
717 return true;
718 }
720 /**********
721 * DIVIDE *
722 **********/
723 do_divide: {
724 uint32_t attr_unused label_skip = 0; /* avoid warning */
725 uint32_t attr_unused label_skip2 = 0; /* avoid warning */
726 uint32_t attr_unused label_end = 0; /* avoid warning */
727 uint32_t attr_unused label_div_0 = 0; /* avoid warning */
728 unsigned attr_unused divide_alu = 0; /* avoid warning */
729 bool attr_unused have_mod = false;
730 bool attr_unused force_sx = false;
731 unsigned attr_unused div_op_size = i_size(op_size);
732 if (unlikely(op_size > OP_SIZE_NATIVE) || unlikely(!ARCH_HAS_DIV)
733 #if defined(ARCH_S390)
734 || !(Z || (op_size <= OP_SIZE_4 && sgn))
735 #endif
736 ) {
737 size_t upcall;
738 if (mode == MODE_INT) {
739 upcall = !mod ? offsetof(struct cg_upcall_vector_s, INT_binary_divide_int8_t) : offsetof(struct cg_upcall_vector_s, INT_binary_modulo_int8_t);
740 } else if (sgn) {
741 upcall = !mod ? offsetof(struct cg_upcall_vector_s, FIXED_binary_divide_int8_t) : offsetof(struct cg_upcall_vector_s, FIXED_binary_modulo_int8_t);
742 } else {
743 upcall = !mod ? offsetof(struct cg_upcall_vector_s, FIXED_binary_udivide_int8_t) : offsetof(struct cg_upcall_vector_s, FIXED_binary_umodulo_int8_t);
744 }
745 g(gen_alu_typed_upcall(ctx, upcall, op_size, slot_1, slot_2, slot_r, mode == MODE_INT ? label_ovf : 0));
746 return true;
747 }
748 #if defined(ARCH_X86) || defined(ARCH_S390)
749 if (mode == MODE_FIXED) {
750 label_skip = alloc_label(ctx);
751 if (unlikely(!label_skip))
752 return false;
753 label_end = alloc_label(ctx);
754 if (unlikely(!label_end))
755 return false;
756 if (sgn) {
757 label_skip2 = alloc_label(ctx);
758 if (unlikely(!label_skip2))
759 return false;
760 }
761 }
762 #if defined(ARCH_X86)
763 if (R_SCRATCH_1 != R_AX || R_SCRATCH_2 != R_DX || R_SCRATCH_3 != R_CX)
764 internal(file_line, "gen_alu: bad scratch registers");
765 #endif
766 g(gen_frame_load(ctx, op_size, sgn ? sign_x : zero_x, slot_1, 0, R_SCRATCH_1));
767 g(gen_frame_load(ctx, op_size, sgn ? sign_x : zero_x, slot_2, 0, R_SCRATCH_3));
769 g(gen_jmp_on_zero(ctx, i_size(op_size), R_SCRATCH_3, COND_E, mode == MODE_INT ? label_ovf : label_skip));
771 if (sgn) {
772 uint64_t val;
773 uint32_t label_not_minus_1;
774 label_not_minus_1 = alloc_label(ctx);
775 if (unlikely(!label_not_minus_1))
776 return false;
778 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, op_size, R_SCRATCH_3, -1, COND_NE, label_not_minus_1));
780 val = -(uint64_t)0x80 << (((1 << op_size) - 1) * 8);
781 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, op_size, R_SCRATCH_1, val, COND_E, mode == MODE_INT ? label_ovf : label_skip2));
783 gen_label(label_not_minus_1);
784 }
786 #if defined(ARCH_X86)
787 if (op_size >= OP_SIZE_2) {
788 if (sgn) {
789 gen_insn(INSN_CWD + ARCH_PARTIAL_ALU(op_size), op_size, 0, 0);
790 gen_one(R_SCRATCH_2);
791 gen_one(R_SCRATCH_1);
792 if (op_size == OP_SIZE_2)
793 gen_one(R_SCRATCH_2);
794 } else {
795 g(gen_3address_alu(ctx, OP_SIZE_4, ALU_XOR, R_SCRATCH_2, R_SCRATCH_2, R_SCRATCH_2, 0));
796 }
797 }
798 gen_insn(INSN_DIV_L, op_size, sgn, 1);
799 gen_one(R_SCRATCH_1);
800 gen_one(i_size(op_size) == OP_SIZE_1 ? R_SCRATCH_1 : R_SCRATCH_2);
801 gen_one(R_SCRATCH_1);
802 gen_one(i_size(op_size) == OP_SIZE_1 ? R_SCRATCH_1 : R_SCRATCH_2);
803 gen_one(R_SCRATCH_3);
804 #else
805 if (!sgn) {
806 g(gen_load_constant(ctx, R_SCRATCH_2, 0));
807 } else if (op_size <= OP_SIZE_4) {
808 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_2, R_SCRATCH_1, (1U << (OP_SIZE_NATIVE + 3)) - 1, false));
809 }
810 gen_insn(INSN_DIV_L, i_size(op_size), sgn, 1);
811 gen_one(R_SCRATCH_2);
812 gen_one(R_SCRATCH_1);
813 gen_one(R_SCRATCH_2);
814 gen_one(R_SCRATCH_1);
815 gen_one(R_SCRATCH_3);
816 #endif
817 if (mod && i_size(op_size) == OP_SIZE_1) {
818 g(gen_3address_rot_imm(ctx, OP_SIZE_2, ROT_SHR, R_SCRATCH_1, R_SCRATCH_1, 8, 0));
819 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
820 } else if (mod) {
821 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_2));
822 } else {
823 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
824 }
825 if (mode == MODE_FIXED) {
826 gen_insn(INSN_JMP, 0, 0, 0);
827 gen_four(label_end);
829 if (sgn) {
830 gen_label(label_skip2);
832 if (mod)
833 g(gen_frame_clear(ctx, op_size, slot_r));
834 else
835 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
837 gen_insn(INSN_JMP, 0, 0, 0);
838 gen_four(label_end);
839 }
841 gen_label(label_skip);
842 if (!mod)
843 g(gen_frame_clear(ctx, op_size, slot_r));
844 else
845 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
846 gen_label(label_end);
847 }
848 return true;
849 #else
850 #if defined(ARCH_MIPS)
851 have_mod = true;
852 div_op_size = maximum(op_size, OP_SIZE_4);
853 if (op_size == OP_SIZE_4)
854 force_sx = true;
855 #endif
856 #if defined(ARCH_POWER)
857 have_mod = cpu_test_feature(CPU_FEATURE_v30);
858 div_op_size = maximum(op_size, OP_SIZE_4);
859 #endif
860 #if defined(ARCH_LOONGARCH64) || defined(ARCH_RISCV64)
861 have_mod = true;
862 div_op_size = maximum(op_size, OP_SIZE_4);
863 #endif
864 label_end = alloc_label(ctx);
865 if (unlikely(!label_end))
866 return false;
868 g(gen_frame_get(ctx, op_size, (sgn && op_size < i_size(op_size)) || force_sx ? sign_x : zero_x, slot_1, 0, R_SCRATCH_1, ®1));
869 g(gen_frame_get(ctx, op_size, (sgn && op_size < i_size(op_size)) || force_sx ? sign_x : zero_x, slot_2, 0, R_SCRATCH_2, ®2));
870 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_3);
872 if (ARCH_PREFERS_SX(op_size) && !sgn && op_size < i_size(op_size)) {
873 g(gen_extend(ctx, op_size, zero_x, R_SCRATCH_1, reg1));
874 reg1 = R_SCRATCH_1;
875 g(gen_extend(ctx, op_size, zero_x, R_SCRATCH_2, reg2));
876 reg2 = R_SCRATCH_2;
877 }
879 if (mode == MODE_INT) {
880 g(gen_jmp_on_zero(ctx, i_size(op_size), reg2, COND_E, label_ovf));
881 if (sgn) {
882 uint64_t val;
883 uint32_t label_not_minus_1;
884 label_not_minus_1 = alloc_label(ctx);
885 if (unlikely(!label_not_minus_1))
886 return false;
888 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, i_size(op_size), reg2, -1, COND_NE, label_not_minus_1));
890 val = 0xFFFFFFFFFFFFFF80ULL << (((1 << op_size) - 1) * 8);
891 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, i_size(op_size), reg1, val, COND_E, label_ovf));
893 gen_label(label_not_minus_1);
894 }
895 } else {
896 #if !(defined(ARCH_ARM) && ARM_ASM_DIV_NO_TRAP)
897 if (!mod) {
898 g(gen_load_constant(ctx, target, 0));
899 } else {
900 g(gen_mov(ctx, OP_SIZE_NATIVE, target, reg1));
901 }
902 g(gen_jmp_on_zero(ctx, i_size(op_size), reg2, COND_E, label_end));
903 if (sgn) {
904 uint64_t val;
905 uint32_t label_not_minus_1;
906 label_not_minus_1 = alloc_label(ctx);
907 if (unlikely(!label_not_minus_1))
908 return false;
910 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, i_size(op_size), reg2, -1, COND_NE, label_not_minus_1));
912 if (!mod) {
913 g(gen_mov(ctx, OP_SIZE_NATIVE, target, reg1));
914 } else {
915 g(gen_load_constant(ctx, target, 0));
916 }
918 val = 0xFFFFFFFFFFFFFF80ULL << (((1 << op_size) - 1) * 8);
919 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, i_size(op_size), reg1, val, COND_E, label_end));
921 gen_label(label_not_minus_1);
922 }
923 #endif
924 }
925 if (mod && have_mod) {
926 g(gen_3address_alu(ctx, div_op_size, sgn ? ALU_SREM : ALU_UREM, target, reg1, reg2, 0));
927 } else {
928 g(gen_3address_alu(ctx, div_op_size, sgn ? ALU_SDIV : ALU_UDIV, target, reg1, reg2, 0));
929 }
931 if (mod && !have_mod) {
932 #if defined(ARCH_ARM)
933 gen_insn(INSN_MADD, i_size(op_size), 1, 0);
934 gen_one(target);
935 gen_one(target);
936 gen_one(reg2);
937 gen_one(reg1);
938 #else
939 g(gen_3address_alu(ctx, i_size(op_size), ALU_MUL, R_SCRATCH_2, reg2, target, 0));
940 g(gen_3address_alu(ctx, i_size(op_size), ALU_SUB, target, reg1, R_SCRATCH_2, 0));
941 #endif
942 }
944 gen_label(label_end);
945 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
946 return true;
947 #endif
948 }
949 /*********
950 * SHIFT *
951 *********/
952 do_shift: {
953 bool sx;
954 bool must_mask;
955 unsigned op_s;
956 if (unlikely(op_size > OP_SIZE_NATIVE)) {
957 size_t upcall;
958 if (mode == MODE_FIXED) {
959 switch (alu) {
960 case ROT_SHL: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_shl_,TYPE_INT_MAX));
961 break;
962 case ROT_SAR: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_shr_,TYPE_INT_MAX));
963 break;
964 case ROT_SHR: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_ushr_,TYPE_INT_MAX));
965 break;
966 case ROT_ROL: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_rol_,TYPE_INT_MAX));
967 break;
968 case ROT_ROR: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_ror_,TYPE_INT_MAX));
969 break;
970 default: internal(file_line, "do_alu: invalid shift %u", alu);
971 }
972 } else {
973 switch (alu) {
974 case ROT_SHL: upcall = offsetof(struct cg_upcall_vector_s, cat(INT_binary_shl_,TYPE_INT_MAX));
975 break;
976 case ROT_SAR: upcall = offsetof(struct cg_upcall_vector_s, cat(INT_binary_shr_,TYPE_INT_MAX));
977 break;
978 default: internal(file_line, "do_alu: invalid shift %u", alu);
979 }
980 }
981 g(gen_alu_upcall(ctx, upcall, slot_1, slot_2, slot_r, mode == MODE_INT ? label_ovf : 0));
982 return true;
983 }
984 op_s = i_size_rot(op_size);
985 #if defined(ARCH_X86)
986 if (slot_1 == slot_r && ctx->registers[slot_1] < 0 && !(mode == MODE_INT && alu == ROT_SHL)) {
987 int64_t offset;
988 g(gen_frame_load(ctx, op_size, garbage, slot_2, 0, R_SCRATCH_3));
989 if (mode == MODE_INT) {
990 int64_t imm = (8U << op_size) - 1;
991 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, op_size, R_SCRATCH_3, imm, COND_A, label_ovf));
992 } else if ((alu != ROT_ROL && alu != ROT_ROR) && op_size < OP_SIZE_4) {
993 g(gen_3address_alu_imm(ctx, OP_SIZE_1, ALU_AND, R_SCRATCH_3, R_SCRATCH_3, (8U << op_size) - 1, 0));
994 }
995 offset = (size_t)slot_1 * slot_size;
996 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDR_OFFSET, op_size));
997 gen_insn(INSN_ROT + ARCH_PARTIAL_ALU(op_size), op_size, alu, 1);
998 gen_address_offset();
999 gen_address_offset();
1000 gen_one(R_SCRATCH_3);
1001 return true;
1002 }
1003 if (mode == MODE_INT && alu == ROT_SHL && op_size < OP_SIZE_NATIVE)
1004 op_s = op_size + 1;
1005 #endif
1006 must_mask = op_size < ARCH_SHIFT_SIZE;
1007 sx = (alu == ROT_SAR && op_size < op_s) || (alu == ROT_SHL && op_size < OP_SIZE_NATIVE && mode == MODE_INT);
1008 #if defined(ARCH_MIPS)
1009 sx |= op_size == OP_SIZE_4;
1010 #endif
1011 g(gen_frame_get(ctx, op_size, sx ? sign_x : zero_x, slot_1, 0, R_SCRATCH_1, ®1));
1012 #if defined(ARCH_X86)
1013 if (!ARCH_IS_3ADDRESS_ROT(alu, op_size)) {
1014 g(gen_frame_load(ctx, op_size, garbage, slot_2, 0, R_SCRATCH_3));
1015 reg3 = R_SCRATCH_3;
1016 } else
1017 #endif
1018 g(gen_frame_get(ctx, op_size, garbage, slot_2, 0, R_SCRATCH_3, ®3));
1019 if (ARCH_PREFERS_SX(op_size) && !sx && op_size < op_s) {
1020 g(gen_extend(ctx, op_size, zero_x, R_SCRATCH_1, reg1));
1021 reg1 = R_SCRATCH_1;
1022 }
1024 if (mode == MODE_INT) {
1025 int64_t imm = (8U << op_size) - 1;
1026 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, i_size_cmp(op_size), reg3, imm, COND_A, label_ovf));
1027 } else {
1028 #if defined(ARCH_ARM)
1029 if (alu == ROT_ROL) {
1030 g(gen_2address_alu1(ctx, OP_SIZE_4, ALU1_NEG, R_SCRATCH_3, reg3, 0));
1031 reg3 = R_SCRATCH_3;
1032 alu = ROT_ROR;
1033 }
1034 #endif
1035 #if defined(ARCH_LOONGARCH64)
1036 if (alu == ROT_ROL && op_size >= OP_SIZE_4) {
1037 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NEG, R_SCRATCH_3, reg3, 0));
1038 reg3 = R_SCRATCH_3;
1039 alu = ROT_ROR;
1040 }
1041 #endif
1042 #if defined(ARCH_MIPS)
1043 if (MIPS_HAS_ROT && alu == ROT_ROL && op_size >= OP_SIZE_4) {
1044 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NEG, R_SCRATCH_3, reg3, 0));
1045 reg3 = R_SCRATCH_3;
1046 alu = ROT_ROR;
1047 }
1048 #endif
1049 #if defined(ARCH_POWER)
1050 if (alu == ROT_ROR && op_size >= OP_SIZE_4) {
1051 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NEG, R_SCRATCH_3, reg3, 0));
1052 reg3 = R_SCRATCH_3;
1053 alu = ROT_ROL;
1054 }
1055 #endif
1056 #if defined(ARCH_S390)
1057 if (Z && alu == ROT_ROR && op_size >= OP_SIZE_4) {
1058 g(gen_2address_alu1(ctx, OP_SIZE_4, ALU1_NEG, R_SCRATCH_3, reg3, 0));
1059 reg3 = R_SCRATCH_3;
1060 alu = ROT_ROL;
1061 }
1062 #endif
1063 if (must_mask) {
1064 g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_4), ALU_AND, R_SCRATCH_3, reg3, (8U << op_size) - 1, 0));
1065 reg3 = R_SCRATCH_3;
1066 }
1067 }
1069 #if defined(ARCH_X86)
1070 if (mode == MODE_INT && alu == ROT_SHL) {
1071 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_2);
1072 } else {
1073 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, slot_2, R_SCRATCH_2);
1074 }
1075 g(gen_3address_rot(ctx, op_s, alu, target, reg1, reg3));
1077 if (mode == MODE_INT && alu == ROT_SHL) {
1078 if (op_size < OP_SIZE_NATIVE) {
1079 gen_insn(INSN_MOVSX, op_size, 0, 0);
1080 gen_one(R_SCRATCH_4);
1081 gen_one(target);
1083 g(gen_cmp_test_jmp(ctx, INSN_CMP, op_s, target, R_SCRATCH_4, COND_NE, label_ovf));
1084 } else {
1085 g(gen_3address_rot(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_4, target, reg3));
1087 g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, reg1, R_SCRATCH_4, COND_NE, label_ovf));
1088 }
1089 }
1090 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1091 return true;
1092 #endif
1093 #if defined(ARCH_ARM)
1094 if (op_size <= OP_SIZE_2 && alu == ROT_ROR) {
1095 gen_insn(INSN_ALU, OP_SIZE_4, ALU_OR, ALU_WRITES_FLAGS(ALU_OR, false));
1096 gen_one(R_SCRATCH_1);
1097 gen_one(reg1);
1098 gen_one(ARG_SHIFTED_REGISTER);
1099 gen_one(ARG_SHIFT_LSL | (8U << op_size));
1100 gen_one(reg1);
1101 if (op_size == OP_SIZE_1)
1102 alu = ROT_SHR;
1103 reg1 = R_SCRATCH_1;
1104 }
1105 goto do_generic_shift;
1106 #endif
1107 #if defined(ARCH_LOONGARCH64)
1108 if (alu == ROT_ROR && op_size >= OP_SIZE_4)
1109 goto do_generic_shift;
1110 #endif
1111 #if defined(ARCH_MIPS)
1112 if (MIPS_HAS_ROT && alu == ROT_ROR && op_size >= OP_SIZE_4)
1113 goto do_generic_shift;
1114 #endif
1115 #if defined(ARCH_POWER)
1116 if (alu == ROT_ROL && op_size >= OP_SIZE_4)
1117 goto do_generic_shift;
1118 #endif
1119 #if defined(ARCH_RISCV64)
1120 if ((alu == ROT_ROL || alu == ROT_ROR) && likely(cpu_test_feature(CPU_FEATURE_zbb))) {
1121 if (likely(op_size >= OP_SIZE_4)) {
1122 goto do_generic_shift;
1123 }
1124 }
1125 #endif
1126 #if defined(ARCH_S390)
1127 if (Z && alu == ROT_ROL && op_size >= OP_SIZE_4)
1128 goto do_generic_shift;
1129 #endif
1130 if (alu == ROT_ROL || alu == ROT_ROR) {
1131 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
1132 g(gen_3address_rot(ctx, op_s, alu == ROT_ROL ? ROT_SHL : ROT_SHR, R_SCRATCH_2, reg1, reg3));
1133 g(gen_2address_alu1(ctx, i_size(OP_SIZE_4), ALU1_NEG, R_SCRATCH_3, reg3, 0));
1134 if (must_mask) {
1135 g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_4), ALU_AND, R_SCRATCH_3, R_SCRATCH_3, (8U << op_size) - 1, 0));
1136 }
1137 g(gen_3address_rot(ctx, op_s, alu == ROT_ROL ? ROT_SHR : ROT_SHL, target, reg1, R_SCRATCH_3));
1138 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_OR, target, target, R_SCRATCH_2, 0));
1139 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1140 return true;
1141 }
1143 goto do_generic_shift;
1144 do_generic_shift:
1145 if (mode == MODE_INT && alu == ROT_SHL) {
1146 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
1147 #if defined(ARCH_S390)
1148 if (op_size >= OP_SIZE_4) {
1149 g(gen_3address_rot(ctx, op_size, ROT_SAL, target, reg1, reg3));
1151 gen_insn(INSN_JMP_COND, op_size, COND_O, 0);
1152 gen_four(label_ovf);
1153 } else
1154 #endif
1155 if (op_size <= OP_SIZE_NATIVE - 1) {
1156 g(gen_3address_rot(ctx, OP_SIZE_NATIVE, alu, target, reg1, reg3));
1158 g(gen_cmp_extended(ctx, OP_SIZE_NATIVE, op_size, target, R_SCRATCH_2, label_ovf));
1159 } else {
1160 g(gen_3address_rot(ctx, OP_SIZE_NATIVE, alu, R_SCRATCH_2, reg1, reg3));
1162 g(gen_3address_rot(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_4, R_SCRATCH_2, reg3));
1164 g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, reg1, R_SCRATCH_4, COND_NE, label_ovf));
1166 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_2));
1168 return true;
1169 }
1170 } else {
1171 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, slot_2, R_SCRATCH_1);
1172 g(gen_3address_rot(ctx, op_s, alu, target, reg1, reg3));
1173 }
1175 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1176 return true;
1177 }
1178 /******
1179 * BT *
1180 ******/
1181 do_bt: {
1182 unsigned attr_unused op_s;
1183 bool need_mask;
1184 #if defined(ARCH_X86)
1185 if ((alu == BTX_BT || slot_1 == slot_r) && ctx->registers[slot_1] < 0) {
1186 unsigned n_op_size = minimum(op_size, OP_SIZE_NATIVE);
1187 g(gen_frame_get(ctx, n_op_size, garbage, slot_2, 0, R_SCRATCH_1, ®2));
1188 if (mode == MODE_INT) {
1189 int64_t imm = (8U << op_size) - 1;
1190 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, n_op_size, reg2, imm, alu == BTX_BT ? COND_A : COND_AE, label_ovf));
1191 if (unlikely(op_size > OP_SIZE_NATIVE)) {
1192 g(gen_address(ctx, R_FRAME, (size_t)slot_2 * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_STR_OFFSET, OP_SIZE_NATIVE));
1193 gen_insn(INSN_CMP, OP_SIZE_NATIVE, 0, 1);
1194 gen_address_offset();
1195 gen_one(ARG_IMM);
1196 gen_eight(0);
1197 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_NE, 0);
1198 gen_four(label_ovf);
1199 }
1200 } else {
1201 g(gen_3address_alu_imm(ctx, OP_SIZE_4, ALU_AND, R_SCRATCH_1, reg2, (8U << op_size) - 1, 0));
1202 reg2 = R_SCRATCH_1;
1203 }
1204 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size, IMM_PURPOSE_STR_OFFSET, maximum(n_op_size, OP_SIZE_2)));
1205 if (alu == BTX_BT) {
1206 gen_insn(INSN_BT, maximum(n_op_size, OP_SIZE_2), 0, 1);
1207 gen_address_offset();
1208 gen_one(reg2);
1209 g(gen_frame_set_cond(ctx, maximum(n_op_size, OP_SIZE_2), false, COND_B, slot_r));
1210 } else {
1211 gen_insn(INSN_BTX, maximum(n_op_size, OP_SIZE_2), alu, 1);
1212 gen_address_offset();
1213 gen_address_offset();
1214 gen_one(reg2);
1215 }
1216 return true;
1217 }
1218 #endif
1219 if (unlikely(op_size > OP_SIZE_NATIVE)) {
1220 size_t upcall;
1221 if (mode == MODE_FIXED) {
1222 switch (alu) {
1223 case BTX_BTS: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_bts_,TYPE_INT_MAX));
1224 break;
1225 case BTX_BTR: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_btr_,TYPE_INT_MAX));
1226 break;
1227 case BTX_BTC: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_btc_,TYPE_INT_MAX));
1228 break;
1229 case BTX_BT: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_bt_,TYPE_INT_MAX));
1230 break;
1231 default: internal(file_line, "do_alu: invalid bit test %u", alu);
1232 }
1233 } else {
1234 switch (alu) {
1235 case BTX_BTS: upcall = offsetof(struct cg_upcall_vector_s, cat(INT_binary_bts_,TYPE_INT_MAX));
1236 break;
1237 case BTX_BTR: upcall = offsetof(struct cg_upcall_vector_s, cat(INT_binary_btr_,TYPE_INT_MAX));
1238 break;
1239 case BTX_BTC: upcall = offsetof(struct cg_upcall_vector_s, cat(INT_binary_btc_,TYPE_INT_MAX));
1240 break;
1241 case BTX_BT: upcall = offsetof(struct cg_upcall_vector_s, cat(INT_binary_bt_,TYPE_INT_MAX));
1242 break;
1243 default: internal(file_line, "do_alu: invalid bit test %u", alu);
1244 }
1245 }
1246 g(gen_alu_upcall(ctx, upcall, slot_1, slot_2, slot_r, label_ovf));
1247 return true;
1248 }
1249 op_s = minimum(OP_SIZE_NATIVE, ARCH_SHIFT_SIZE);
1250 op_s = maximum(op_s, op_size);
1251 g(gen_frame_get(ctx, op_size, zero_x, slot_1, 0, R_SCRATCH_1, ®1));
1252 g(gen_frame_get(ctx, op_size, garbage, slot_2, 0, R_SCRATCH_2, ®2));
1253 if (mode == MODE_INT) {
1254 int64_t imm = (8U << op_size) - 1;
1255 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, i_size_cmp(op_size), reg2, imm, alu == BTX_BT ? COND_A : COND_AE, label_ovf));
1256 }
1257 if (alu != BTX_BT) {
1258 if (!ARCH_HAS_BTX(alu, OP_SIZE_NATIVE, false))
1259 goto do_generic_bt;
1260 need_mask = !ARCH_HAS_BTX(alu, op_size, false);
1261 } else {
1262 #if defined(ARCH_X86)
1263 need_mask = op_size < OP_SIZE_2;
1264 #else
1265 if (!ARCH_HAS_BTX(BTX_BTEXT, OP_SIZE_NATIVE, false))
1266 goto do_generic_bt;
1267 need_mask = !ARCH_HAS_BTX(BTX_BTEXT, op_size, false);
1268 #endif
1269 }
1270 if (need_mask) {
1271 g(gen_3address_alu_imm(ctx, OP_SIZE_4, ALU_AND, R_SCRATCH_2, reg2, (8U << op_size) - 1, 0));
1272 reg2 = R_SCRATCH_2;
1273 }
1274 if (alu == BTX_BT) {
1275 #if defined(ARCH_X86)
1276 gen_insn(INSN_BT, maximum(op_size, OP_SIZE_2), 0, 1);
1277 gen_one(reg1);
1278 gen_one(reg2);
1280 g(gen_frame_set_cond(ctx, maximum(op_size, OP_SIZE_2), false, COND_B, slot_r));
1281 #else
1282 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1283 gen_insn(INSN_BTX, need_mask ? OP_SIZE_NATIVE : op_size, BTX_BTEXT, 0);
1284 gen_one(target);
1285 gen_one(reg1);
1286 gen_one(reg2);
1288 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
1289 #endif
1290 } else {
1291 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, slot_2, R_SCRATCH_1);
1292 #if defined(ARCH_X86)
1293 if (target == reg2)
1294 target = R_SCRATCH_1;
1295 if (target != reg1) {
1296 g(gen_mov(ctx, op_size, target, reg1));
1297 reg1 = target;
1298 }
1299 gen_insn(INSN_BTX, maximum(op_size, OP_SIZE_2), alu, 1);
1300 #else
1301 gen_insn(INSN_BTX, need_mask ? OP_SIZE_NATIVE : op_size, alu, 0);
1302 #endif
1303 gen_one(target);
1304 gen_one(reg1);
1305 gen_one(reg2);
1307 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1308 }
1309 return true;
1311 goto do_generic_bt;
1312 do_generic_bt:
1313 if (mode == MODE_FIXED && op_size < ARCH_SHIFT_SIZE) {
1314 g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_4), ALU_AND, R_SCRATCH_2, reg2, (8U << op_size) - 1, 0));
1315 reg2 = R_SCRATCH_2;
1316 }
1317 g(gen_load_constant(ctx, R_SCRATCH_3, 1));
1319 g(gen_3address_rot(ctx, op_s, ROT_SHL, R_SCRATCH_3, R_SCRATCH_3, reg2));
1321 switch (alu) {
1322 case BTX_BT:
1323 #if ARCH_HAS_FLAGS
1324 #if defined(ARCH_S390) || defined(ARCH_POWER)
1325 g(gen_3address_alu(ctx, i_size(op_size), ALU_AND, R_SCRATCH_1, reg1, R_SCRATCH_3, 1));
1326 #else
1327 gen_insn(INSN_TEST, i_size(op_size), 0, 1);
1328 gen_one(reg1);
1329 gen_one(R_SCRATCH_3);
1330 #endif
1331 g(gen_frame_set_cond(ctx, i_size_cmp(op_size), false, COND_NE, slot_r));
1332 #else
1333 g(gen_3address_alu(ctx, i_size(op_size), ALU_AND, R_SCRATCH_1, reg1, R_SCRATCH_3, 0));
1334 g(gen_frame_cmp_imm_set_cond_reg(ctx, i_size(op_size), R_SCRATCH_1, 0, COND_NE, slot_r));
1335 #endif
1336 return true;
1337 case BTX_BTS:
1338 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1339 g(gen_3address_alu(ctx, i_size(op_size), ALU_OR, target, reg1, R_SCRATCH_3, 0));
1340 break;
1341 case BTX_BTR:
1342 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1343 if (!ARCH_HAS_ANDN) {
1344 g(gen_3address_alu_imm(ctx, i_size(op_size), ALU_XOR, R_SCRATCH_3, R_SCRATCH_3, -1, 0));
1346 g(gen_3address_alu(ctx, i_size(op_size), ALU_AND, target, reg1, R_SCRATCH_3, 0));
1347 break;
1348 }
1349 g(gen_3address_alu(ctx, i_size(op_size), ALU_ANDN, target, reg1, R_SCRATCH_3, 0));
1350 break;
1351 case BTX_BTC:
1352 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1353 g(gen_3address_alu(ctx, i_size(op_size), ALU_XOR, target, reg1, R_SCRATCH_3, 0));
1354 break;
1355 default:
1356 internal(file_line, "gen_alu: unsupported bit test %u", alu);
1357 }
1359 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1361 return true;
1362 }
1363 /***********
1364 * COMPARE *
1365 ***********/
1366 do_compare: {
1367 if (unlikely(op_size > OP_SIZE_NATIVE)) {
1368 size_t attr_unused upcall;
1369 switch (alu) {
1370 case COND_E:
1371 case COND_NE:
1372 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
1373 g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, ALU_XOR, 0, slot_2, lo_word(OP_SIZE_NATIVE), R_SCRATCH_1));
1374 g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, ALU_XOR, 0, slot_2, hi_word(OP_SIZE_NATIVE), R_SCRATCH_2));
1375 #if defined(ARCH_ARM64)
1376 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_OR, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_2, 0));
1378 gen_insn(INSN_CMP, OP_SIZE_NATIVE, 0, 1);
1379 gen_one(R_SCRATCH_1);
1380 gen_one(ARG_IMM);
1381 gen_eight(0);
1382 #else
1383 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_OR, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_2, ARCH_HAS_FLAGS));
1384 #endif
1385 #if ARCH_HAS_FLAGS
1386 g(gen_frame_set_cond(ctx, OP_SIZE_NATIVE, false, alu, slot_r));
1387 #else
1388 g(gen_frame_cmp_imm_set_cond_reg(ctx, OP_SIZE_NATIVE, R_SCRATCH_1, 0, alu, slot_r));
1389 #endif
1390 return true;
1391 #if defined(ARCH_X86) || defined(ARCH_ARM)
1392 case COND_L:
1393 case COND_B:
1394 g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_2, lo_word(OP_SIZE_NATIVE), R_SCRATCH_2));
1395 g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_1, hi_word(OP_SIZE_NATIVE), R_SCRATCH_1));
1396 g(gen_frame_load_cmp(ctx, OP_SIZE_NATIVE, false, garbage, true, slot_1, lo_word(OP_SIZE_NATIVE), R_SCRATCH_2));
1397 g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, ALU_SBB, 1, slot_2, hi_word(OP_SIZE_NATIVE), R_SCRATCH_1));
1398 g(gen_frame_set_cond(ctx, OP_SIZE_NATIVE, false, alu, slot_r));
1399 return true;
1400 case COND_LE:
1401 case COND_BE:
1402 g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_1, lo_word(OP_SIZE_NATIVE), R_SCRATCH_2));
1403 g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_2, hi_word(OP_SIZE_NATIVE), R_SCRATCH_1));
1404 g(gen_frame_load_cmp(ctx, OP_SIZE_NATIVE, false, garbage, true, slot_2, lo_word(OP_SIZE_NATIVE), R_SCRATCH_2));
1405 g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, ALU_SBB, 1, slot_1, hi_word(OP_SIZE_NATIVE), R_SCRATCH_1));
1406 g(gen_frame_set_cond(ctx, OP_SIZE_NATIVE, false, alu == COND_LE ? COND_GE : COND_AE, slot_r));
1407 return true;
1408 #else
1409 case COND_L: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_less_,TYPE_INT_MAX)); goto do_upcall;
1410 case COND_B: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_uless_,TYPE_INT_MAX)); goto do_upcall;
1411 case COND_LE: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_less_equal_,TYPE_INT_MAX)); goto do_upcall;
1412 case COND_BE: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_uless_equal_,TYPE_INT_MAX)); goto do_upcall;
1413 do_upcall: g(gen_alu_upcall(ctx, upcall, slot_1, slot_2, slot_r, 0));
1414 return true;
1415 #endif
1416 default:
1417 internal(file_line, "gen_alu: unsupported condition %u", alu);
1418 }
1419 return false;
1420 }
1421 #if defined(ARCH_X86)
1422 g(gen_frame_get(ctx, op_size, garbage, slot_1, 0, R_SCRATCH_1, ®1));
1423 g(gen_frame_load_cmp_set_cond(ctx, op_size, garbage, slot_2, 0, reg1, alu, slot_r));
1424 #else
1425 g(gen_frame_get(ctx, op_size, op_size == i_size_cmp(op_size) + (unsigned)zero ? garbage : alu == COND_L || alu == COND_LE || ARCH_PREFERS_SX(op_size) ? sign_x : zero_x, slot_1, 0, R_SCRATCH_1, ®1));
1426 g(gen_frame_load_cmp_set_cond(ctx, op_size, alu == COND_L || alu == COND_LE || ARCH_PREFERS_SX(op_size) ? sign_x : zero_x, slot_2, 0, reg1, alu, slot_r));
1427 #endif
1428 return true;
1429 }
1430 }
1432 static bool attr_w gen_alu1(struct codegen_context *ctx, unsigned mode, unsigned op_size, unsigned op, uint32_t label_ovf, frame_t slot_1, frame_t slot_r)
1433 {
1434 unsigned alu;
1435 unsigned reg1, target;
1436 switch (mode) {
1437 case MODE_FIXED: switch (op) {
1438 case OPCODE_FIXED_OP_not: alu = ALU1_NOT; goto do_alu;
1439 case OPCODE_FIXED_OP_neg: alu = ALU1_NEG; goto do_alu;
1440 case OPCODE_FIXED_OP_inc: alu = ALU1_INC; goto do_alu;
1441 case OPCODE_FIXED_OP_dec: alu = ALU1_DEC; goto do_alu;
1442 case OPCODE_FIXED_OP_bswap:
1443 case OPCODE_FIXED_OP_bswap_alt1: alu = ALU1_BSWAP; goto do_bswap;
1444 case OPCODE_FIXED_OP_brev:
1445 case OPCODE_FIXED_OP_brev_alt1: alu = ALU1_BREV; goto do_brev;
1446 case OPCODE_FIXED_OP_bsf:
1447 case OPCODE_FIXED_OP_bsf_alt1: alu = ALU1_BSF; goto do_bsf_bsr_popcnt;
1448 case OPCODE_FIXED_OP_bsr:
1449 case OPCODE_FIXED_OP_bsr_alt1: alu = ALU1_BSR; goto do_bsf_bsr_popcnt;
1450 case OPCODE_FIXED_OP_popcnt:
1451 case OPCODE_FIXED_OP_popcnt_alt1: alu = ALU1_POPCNT; goto do_bsf_bsr_popcnt;
1452 case OPCODE_FIXED_OP_to_int: goto do_fixed_conv;
1453 case OPCODE_FIXED_OP_from_int: goto do_fixed_conv;
1454 case OPCODE_FIXED_OP_uto_int: goto conv_uto_int;
1455 case OPCODE_FIXED_OP_ufrom_int: goto conv_ufrom_int;
1456 default: internal(file_line, "gen_alu1: unsupported fixed operation %u", op);
1457 }
1458 case MODE_INT: switch (op) {
1459 case OPCODE_INT_OP_not: alu = ALU1_NOT; mode = MODE_FIXED; goto do_alu;
1460 case OPCODE_INT_OP_neg: alu = ALU1_NEG; goto do_alu;
1461 case OPCODE_INT_OP_inc: alu = ALU1_INC; goto do_alu;
1462 case OPCODE_INT_OP_dec: alu = ALU1_DEC; goto do_alu;
1463 case OPCODE_INT_OP_bsf: alu = ALU1_BSF; goto do_bsf_bsr_popcnt;
1464 case OPCODE_INT_OP_bsr: alu = ALU1_BSR; goto do_bsf_bsr_popcnt;
1465 case OPCODE_INT_OP_popcnt:
1466 case OPCODE_INT_OP_popcnt_alt1: alu = ALU1_POPCNT; goto do_bsf_bsr_popcnt;
1467 case OPCODE_INT_OP_to_int: goto do_conv;
1468 case OPCODE_INT_OP_from_int: goto do_conv;
1469 default: internal(file_line, "gen_alu1: unsupported int operation %u", op);
1470 }
1471 case MODE_BOOL: switch (op) {
1472 case OPCODE_BOOL_OP_not: goto do_bool_not;
1473 default: internal(file_line, "gen_alu1: unsupported bool operation %u", op);
1474 }
1475 }
1476 internal(file_line, "gen_alu1: unsupported mode %u", mode);
1478 /*******
1479 * ALU *
1480 *******/
1481 do_alu: {
1482 bool arch_use_flags = ARCH_HAS_FLAGS;
1483 enum extend ex;
1484 #if defined(ARCH_POWER)
1485 arch_use_flags = false;
1486 #endif
1487 if (op_size > OP_SIZE_NATIVE) {
1488 #if !defined(ARCH_X86) && !defined(ARCH_ARM) && !defined(ARCH_POWER)
1489 if (alu == ALU1_NEG) {
1490 if (mode == MODE_FIXED)
1491 g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(FIXED_unary_neg_,TYPE_INT_MAX)), slot_1, NO_FRAME_T, slot_r, 0));
1492 else
1493 g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(INT_unary_neg_,TYPE_INT_MAX)), slot_1, NO_FRAME_T, slot_r, label_ovf));
1494 return true;
1495 }
1496 if (alu == ALU1_DEC) {
1497 if (mode == MODE_FIXED)
1498 g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(FIXED_unary_dec_,TYPE_INT_MAX)), slot_1, NO_FRAME_T, slot_r, 0));
1499 else
1500 g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(INT_unary_dec_,TYPE_INT_MAX)), slot_1, NO_FRAME_T, slot_r, label_ovf));
1501 return true;
1502 }
1503 if (alu == ALU1_INC) {
1504 if (mode == MODE_FIXED)
1505 g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(FIXED_unary_inc_,TYPE_INT_MAX)), slot_1, NO_FRAME_T, slot_r, 0));
1506 else
1507 g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(INT_unary_inc_,TYPE_INT_MAX)), slot_1, NO_FRAME_T, slot_r, label_ovf));
1508 return true;
1509 }
1510 #endif
1511 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
1512 #if defined(ARCH_S390)
1513 if (alu == ALU1_NOT) {
1514 g(gen_load_constant(ctx, R_SCRATCH_3, -1));
1516 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_XOR, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_3, 0));
1517 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_XOR, R_SCRATCH_2, R_SCRATCH_2, R_SCRATCH_3, 0));
1519 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_1, R_SCRATCH_2));
1520 return true;
1521 }
1522 #endif
1523 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, alu, R_SCRATCH_1, R_SCRATCH_1, alu == ALU1_INC || alu == ALU1_DEC || alu == ALU1_NEG ? 2 : 0));
1524 if (alu == ALU1_NOT) {
1525 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NOT, R_SCRATCH_2, R_SCRATCH_2, 0));
1526 } else if (alu == ALU1_INC || alu == ALU1_DEC) {
1527 g(gen_imm(ctx, 0, alu == ALU1_INC ? IMM_PURPOSE_ADD : IMM_PURPOSE_SUB, OP_SIZE_NATIVE));
1528 gen_insn(INSN_ALU, OP_SIZE_NATIVE, alu == ALU1_INC ? ALU_ADC : ALU_SBB, (mode == MODE_INT) | ALU_WRITES_FLAGS(alu == ALU1_INC ? ALU_ADC : ALU_SBB, is_imm()));
1529 gen_one(R_SCRATCH_2);
1530 gen_one(R_SCRATCH_2);
1531 gen_imm_offset();
1532 } else {
1533 #if defined(ARCH_X86)
1534 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NOT, R_SCRATCH_2, R_SCRATCH_2, 0));
1536 g(gen_imm(ctx, -1, IMM_PURPOSE_SUB, OP_SIZE_NATIVE));
1537 gen_insn(INSN_ALU, OP_SIZE_NATIVE, ALU_SBB, ALU_WRITES_FLAGS(ALU_SBB, is_imm()));
1538 gen_one(R_SCRATCH_2);
1539 gen_one(R_SCRATCH_2);
1540 gen_imm_offset();
1541 #else
1542 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NGC, R_SCRATCH_2, R_SCRATCH_2, (mode == MODE_INT)));
1543 #endif
1544 }
1545 if (mode == MODE_INT) {
1546 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_O, 0);
1547 gen_four(label_ovf);
1548 }
1549 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_1, R_SCRATCH_2));
1550 return true;
1551 }
1552 if ((arch_use_flags || ARCH_SUPPORTS_TRAPS) && slot_1 == slot_r && i_size_cmp(op_size) == op_size + zero) {
1553 struct cg_exit *ce;
1554 unsigned undo_alu = alu == ALU1_INC ? ALU1_DEC : alu == ALU1_DEC ? ALU1_INC : alu;
1555 if (ctx->registers[slot_1] >= 0) {
1556 unsigned reg = ctx->registers[slot_1];
1557 if (mode == MODE_INT && ARCH_SUPPORTS_TRAPS) {
1558 gen_insn(INSN_ALU1_TRAP, op_size, alu, ALU1_WRITES_FLAGS(alu));
1559 gen_one(reg);
1560 gen_one(reg);
1561 if (ARCH_TRAP_BEFORE || alu == undo_alu) {
1562 gen_four(label_ovf);
1563 return true;
1564 } else {
1565 ce = alloc_undo_label(ctx);
1566 if (unlikely(!ce))
1567 return false;
1568 gen_four(ce->undo_label);
1569 goto do_undo_opcode;
1570 }
1571 }
1572 g(gen_2address_alu1(ctx, i_size(op_size), alu, reg, reg, mode == MODE_INT));
1573 if (mode == MODE_INT) {
1574 if (alu != undo_alu) {
1575 ce = alloc_undo_label(ctx);
1576 if (unlikely(!ce))
1577 return false;
1578 gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
1579 gen_four(ce->undo_label);
1580 do_undo_opcode:
1581 ce->undo_opcode = INSN_ALU1 + ARCH_PARTIAL_ALU(op_size);
1582 ce->undo_op_size = i_size(op_size);
1583 ce->undo_aux = undo_alu;
1584 ce->undo_writes_flags = ALU1_WRITES_FLAGS(undo_alu);
1585 ce->undo_parameters[0] = reg;
1586 ce->undo_parameters[1] = reg;
1587 ce->undo_parameters_len = 2;
1588 } else {
1589 gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
1590 gen_four(label_ovf);
1591 }
1592 }
1593 return true;
1594 }
1595 #if defined(ARCH_X86)
1596 else {
1597 size_t m;
1598 int64_t offset = (size_t)slot_1 * slot_size;
1599 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDR_OFFSET, i_size(op_size)));
1600 gen_insn(INSN_ALU1 + ARCH_PARTIAL_ALU(op_size), i_size(op_size), alu, ALU1_WRITES_FLAGS(alu) | (mode == MODE_INT));
1601 gen_address_offset();
1602 gen_address_offset();
1603 if (mode == MODE_INT) {
1604 if (alu != undo_alu) {
1605 ce = alloc_undo_label(ctx);
1606 if (unlikely(!ce))
1607 return false;
1608 ce->undo_opcode = INSN_ALU1 + ARCH_PARTIAL_ALU(op_size);
1609 ce->undo_op_size = i_size(op_size);
1610 ce->undo_aux = undo_alu;
1611 ce->undo_writes_flags = ALU1_WRITES_FLAGS(undo_alu);
1612 m = mark_params(ctx);
1613 gen_address_offset();
1614 gen_address_offset();
1615 copy_params(ctx, ce, m);
1616 gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
1617 gen_four(ce->undo_label);
1618 } else {
1619 gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
1620 gen_four(label_ovf);
1621 }
1622 }
1623 return true;
1624 }
1625 #endif
1626 }
1627 target = gen_frame_target(ctx, slot_r, mode == MODE_INT ? slot_1 : NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1628 if (mode == MODE_FIXED) {
1629 ex = garbage;
1630 } else {
1631 ex = ARCH_PREFERS_SX(op_size) ? sign_x : zero_x;
1632 if (ARCH_SUPPORTS_TRAPS && op_size >= OP_SIZE_4)
1633 ex = garbage;
1634 if (op_size == i_size(op_size) + (unsigned)zero)
1635 ex = garbage;
1636 }
1637 g(gen_frame_get(ctx, op_size, ex, slot_1, 0, target, ®1));
1638 #if defined(ARCH_S390)
1639 if (alu == ALU1_NOT) {
1640 g(gen_3address_alu_imm(ctx, i_size(op_size), ALU_XOR, target, reg1, -1, 0));
1642 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1643 return true;
1644 }
1645 #endif
1646 #if defined(ARCH_X86)
1647 g(gen_2address_alu1(ctx, op_size, alu, target, reg1, mode == MODE_INT));
1648 #else
1649 if (mode == MODE_INT) {
1650 #if defined(ARCH_POWER)
1651 if (op_size == OP_SIZE_NATIVE) {
1652 g(gen_2address_alu1(ctx, i_size(op_size), alu, target, reg1, 0));
1653 if (alu == ALU1_NEG) {
1654 g(gen_3address_alu(ctx, i_size(op_size), ALU_AND, R_CG_SCRATCH, target, reg1, 1));
1655 } else if (alu == ALU1_INC) {
1656 g(gen_3address_alu(ctx, i_size(op_size), ALU_ANDN, R_CG_SCRATCH, target, reg1, 1));
1657 } else if (alu == ALU1_DEC) {
1658 g(gen_3address_alu(ctx, i_size(op_size), ALU_ANDN, R_CG_SCRATCH, reg1, target, 1));
1659 }
1660 gen_insn(INSN_JMP_COND, op_size, COND_L, 0);
1661 gen_four(label_ovf);
1663 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1665 return true;
1666 }
1667 #endif
1668 if (!arch_use_flags && !ARCH_SUPPORTS_TRAPS && ARCH_HAS_ANDN && op_size == OP_SIZE_NATIVE) {
1669 g(gen_2address_alu1(ctx, i_size(op_size), alu, target, reg1, 0));
1671 if (alu == ALU1_NEG) {
1672 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_AND, R_SCRATCH_3, target, reg1, 0));
1673 } else if (alu == ALU1_INC) {
1674 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_ANDN, R_SCRATCH_3, target, reg1, 0));
1675 } else if (alu == ALU1_DEC) {
1676 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_ANDN, R_SCRATCH_3, reg1, target, 0));
1677 }
1678 g(gen_jmp_on_zero(ctx, OP_SIZE_NATIVE, R_SCRATCH_3, COND_S, label_ovf));
1680 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1682 return true;
1683 }
1684 if (op_size <= OP_SIZE_2 || (!arch_use_flags && !ARCH_SUPPORTS_TRAPS)) {
1685 int64_t imm = ((alu != ALU1_INC && ARCH_PREFERS_SX(op_size) ? -0x80ULL : 0x80ULL) << (((1 << op_size) - 1) * 8)) - (alu == ALU1_INC);
1687 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, i_size_cmp(op_size), reg1, imm, COND_E, label_ovf));
1689 mode = MODE_FIXED;
1691 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1692 }
1693 }
1694 #if !ARCH_HAS_FLAGS
1695 if (mode == MODE_INT) {
1696 gen_insn(INSN_ALU1_TRAP, op_size, alu, ALU1_WRITES_FLAGS(alu));
1697 gen_one(target);
1698 gen_one(reg1);
1699 gen_four(label_ovf);
1700 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1701 return true;
1702 }
1703 #endif
1704 g(gen_2address_alu1(ctx, i_size(op_size), alu, target, reg1, mode == MODE_INT));
1705 #endif
1706 if (mode == MODE_INT) {
1707 gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
1708 gen_four(label_ovf);
1709 }
1710 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1711 return true;
1712 }
1714 /*******
1715 * NOT *
1716 *******/
1717 do_bool_not: {
1718 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1719 g(gen_frame_get(ctx, op_size, garbage, slot_1, 0, target, ®1));
1721 g(gen_3address_alu_imm(ctx, i_size(op_size), ALU_XOR, target, reg1, 1, 0));
1723 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1724 return true;
1725 }
1727 /*********
1728 * BSWAP *
1729 *********/
1730 do_bswap: {
1731 bool attr_unused sx = false;
1732 #if defined(ARCH_X86) || defined(ARCH_ARM) || defined(ARCH_IA64) || defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS) || defined(ARCH_RISCV64) || defined(ARCH_S390)
1733 #if defined(ARCH_ARM32)
1734 if (unlikely(!cpu_test_feature(CPU_FEATURE_armv6)))
1735 goto do_generic_bswap;
1736 #endif
1737 #if defined(ARCH_MIPS)
1738 if (unlikely(!MIPS_HAS_ROT))
1739 goto do_generic_bswap;
1740 sx = op_size == OP_SIZE_4;
1741 #endif
1742 #if defined(ARCH_RISCV64)
1743 if (unlikely(!cpu_test_feature(CPU_FEATURE_zbb)))
1744 goto do_generic_bswap;
1745 #endif
1746 #if defined(ARCH_S390)
1747 if (op_size == OP_SIZE_2)
1748 goto do_generic_bswap;
1749 #endif
1750 #if defined(ARCH_X86)
1751 if (op_size >= OP_SIZE_4 && !cpu_test_feature(CPU_FEATURE_bswap))
1752 goto do_generic_bswap;
1753 #endif
1754 if (op_size > OP_SIZE_NATIVE) {
1755 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
1756 reg1 = R_SCRATCH_1;
1757 target = R_SCRATCH_1;
1758 } else {
1759 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1760 g(gen_frame_get(ctx, op_size, sx ? sign_x : garbage, slot_1, 0, target, ®1));
1761 }
1763 if (op_size == OP_SIZE_1) {
1764 #if defined(ARCH_IA64) || defined(ARCH_RISCV64)
1765 } else if (op_size == OP_SIZE_2 || op_size == OP_SIZE_4) {
1766 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_BSWAP, target, reg1, 0));
1768 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SAR, target, target, op_size == OP_SIZE_2 ? 48 : 32, 0));
1769 #endif
1770 } else if (op_size == OP_SIZE_2) {
1771 #if defined(ARCH_X86)
1772 g(gen_3address_rot_imm(ctx, OP_SIZE_2, ROT_ROR, target, reg1, 8, 0));
1773 #else
1774 g(gen_2address_alu1(ctx, OP_SIZE_4, ALU1_BSWAP16, target, reg1, 0));
1775 #endif
1776 } else {
1777 g(gen_2address_alu1(ctx, minimum(op_size, OP_SIZE_NATIVE), ALU1_BSWAP, target, reg1, 0));
1778 }
1779 if (op_size > OP_SIZE_NATIVE) {
1780 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_BSWAP, R_SCRATCH_2, R_SCRATCH_2, 0));
1781 }
1783 if (op_size > OP_SIZE_NATIVE)
1784 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_2, R_SCRATCH_1));
1785 else
1786 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1787 return true;
1788 #endif
1789 goto do_generic_bswap;
1790 do_generic_bswap:
1791 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_unary_bswap_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, 0);
1792 }
1793 /********
1794 * BREV *
1795 ********/
1796 do_brev: {
1797 #if defined(ARCH_ARM) || defined(ARCH_LOONGARCH64) || (defined(ARCH_MIPS) && MIPS_R6)
1798 #if defined(ARCH_ARM32)
1799 if (unlikely(!cpu_test_feature(CPU_FEATURE_armv6t2)))
1800 goto do_generic_brev;
1801 #endif
1802 if (op_size > OP_SIZE_NATIVE) {
1803 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
1804 reg1 = R_SCRATCH_1;
1805 target = R_SCRATCH_1;
1806 } else {
1807 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1808 g(gen_frame_get(ctx, op_size, garbage, slot_1, 0, target, ®1));
1809 }
1811 g(gen_2address_alu1(ctx, minimum(maximum(OP_SIZE_4, op_size), OP_SIZE_NATIVE), ALU1_BREV, target, reg1, 0));
1812 if (op_size <= OP_SIZE_2) {
1813 g(gen_3address_rot_imm(ctx, OP_SIZE_4, ROT_SHR, target, target, op_size == OP_SIZE_1 ? 24 : 16, 0));
1814 }
1815 if (op_size > OP_SIZE_NATIVE) {
1816 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_BREV, R_SCRATCH_2, R_SCRATCH_2, 0));
1817 }
1819 if (op_size > OP_SIZE_NATIVE)
1820 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_2, R_SCRATCH_1));
1821 else
1822 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1823 return true;
1824 #endif
1825 goto do_generic_brev;
1826 do_generic_brev:
1827 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_unary_brev_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, 0);
1828 }
1829 /******************
1830 * BSF/BSR/POPCNT *
1831 ******************/
1832 do_bsf_bsr_popcnt: {
1833 if (op_size > OP_SIZE_NATIVE) {
1834 #if defined(ARCH_X86)
1835 uint32_t label_finish = 0; /* avoid warning */
1836 if (alu == ALU1_POPCNT && unlikely(!cpu_test_feature(CPU_FEATURE_popcnt)))
1837 goto do_generic_bsf_bsr_popcnt;
1838 if (alu == ALU1_BSR || alu == ALU1_POPCNT) {
1839 if (mode == MODE_INT) {
1840 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
1841 g(gen_imm(ctx, 0, IMM_PURPOSE_STORE_VALUE, OP_SIZE_NATIVE));
1842 gen_insn(INSN_CMP, OP_SIZE_NATIVE, 0, 1);
1843 gen_address_offset();
1844 gen_imm_offset();
1846 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_S, 0);
1847 gen_four(label_ovf);
1848 }
1849 }
1850 if (alu == ALU1_BSF) {
1851 label_finish = alloc_label(ctx);
1852 if (unlikely(!label_finish))
1853 return false;
1855 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + lo_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
1856 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_BSF, ALU1_WRITES_FLAGS(ALU1_BSF));
1857 gen_one(R_SCRATCH_1);
1858 gen_address_offset();
1860 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_NE, 0);
1861 gen_four(label_finish);
1863 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
1864 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_BSF, ALU1_WRITES_FLAGS(ALU1_BSF));
1865 gen_one(R_SCRATCH_1);
1866 gen_address_offset();
1868 g(gen_3address_alu_imm(ctx, OP_SIZE_4, ALU_ADD, R_SCRATCH_1, R_SCRATCH_1, 8U << OP_SIZE_NATIVE, 0));
1869 }
1870 if (alu == ALU1_BSR) {
1871 label_finish = alloc_label(ctx);
1872 if (unlikely(!label_finish))
1873 return false;
1875 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
1876 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_BSR, ALU1_WRITES_FLAGS(ALU1_BSR));
1877 gen_one(R_SCRATCH_1);
1878 gen_address_offset();
1880 g(gen_3address_alu_imm(ctx, OP_SIZE_4, ALU_ADD, R_SCRATCH_1, R_SCRATCH_1, 8U << OP_SIZE_NATIVE, 0));
1882 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_NE, 0);
1883 gen_four(label_finish);
1885 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + lo_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
1886 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_BSR, ALU1_WRITES_FLAGS(ALU1_BSR));
1887 gen_one(R_SCRATCH_1);
1888 gen_address_offset();
1889 }
1890 if (alu == ALU1_BSF || alu == ALU1_BSR) {
1891 if (mode == MODE_INT) {
1892 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_E, 0);
1893 gen_four(label_ovf);
1894 } else {
1895 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_NE, 0);
1896 gen_four(label_finish);
1898 g(gen_load_constant(ctx, R_SCRATCH_1, -1));
1899 }
1901 gen_label(label_finish);
1903 if (mode == MODE_INT)
1904 goto write_result;
1906 if (R_SCRATCH_1 != R_AX || R_SCRATCH_2 != R_DX)
1907 internal(file_line, "gen_alu1: bad scratch registers");
1908 gen_insn(INSN_CWD, OP_SIZE_NATIVE, 0, 0);
1909 gen_one(R_DX);
1910 gen_one(R_AX);
1912 g(gen_address(ctx, R_FRAME, (size_t)slot_r * slot_size + lo_word(OP_SIZE_NATIVE), IMM_PURPOSE_STR_OFFSET, OP_SIZE_NATIVE));
1913 gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
1914 gen_address_offset();
1915 gen_one(R_SCRATCH_1);
1917 g(gen_address(ctx, R_FRAME, (size_t)slot_r * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_STR_OFFSET, OP_SIZE_NATIVE));
1918 gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
1919 gen_address_offset();
1920 gen_one(R_SCRATCH_2);
1922 return true;
1923 }
1924 if (alu == ALU1_POPCNT) {
1925 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + lo_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
1926 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_POPCNT, ALU1_WRITES_FLAGS(ALU1_POPCNT));
1927 gen_one(R_SCRATCH_1);
1928 gen_address_offset();
1930 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
1931 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_POPCNT, ALU1_WRITES_FLAGS(ALU1_POPCNT));
1932 gen_one(R_SCRATCH_2);
1933 gen_address_offset();
1935 g(gen_3address_alu(ctx, OP_SIZE_4, ALU_ADD, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_2, 1));
1936 write_result:
1937 g(gen_address(ctx, R_FRAME, (size_t)slot_r * slot_size + lo_word(OP_SIZE_NATIVE), IMM_PURPOSE_STR_OFFSET, OP_SIZE_NATIVE));
1938 gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
1939 gen_address_offset();
1940 gen_one(R_SCRATCH_1);
1942 g(gen_address(ctx, R_FRAME, (size_t)slot_r * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_STR_OFFSET, OP_SIZE_NATIVE));
1943 g(gen_imm(ctx, 0, IMM_PURPOSE_STORE_VALUE, OP_SIZE_NATIVE));
1944 gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
1945 gen_address_offset();
1946 gen_imm_offset();
1948 return true;
1949 }
1950 #endif
1951 goto do_generic_bsf_bsr_popcnt;
1952 }
1953 #if defined(ARCH_X86)
1954 if (alu == ALU1_POPCNT && unlikely(!cpu_test_feature(CPU_FEATURE_popcnt)))
1955 goto do_generic_bsf_bsr_popcnt;
1956 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1957 if (op_size == OP_SIZE_1 || ((alu == ALU1_BSR || alu == ALU1_POPCNT) && mode == MODE_INT)) {
1958 g(gen_frame_get(ctx, op_size, zero_x, slot_1, 0, target, ®1));
1959 if ((alu == ALU1_BSR || alu == ALU1_POPCNT) && mode == MODE_INT) {
1960 g(gen_cmp_test_jmp(ctx, INSN_TEST, op_size, reg1, reg1, alu == ALU1_BSR ? COND_LE : COND_S, label_ovf));
1961 }
1962 g(gen_2address_alu1(ctx, maximum(op_size, OP_SIZE_2), alu, target, reg1, 1));
1963 if ((alu == ALU1_BSR || alu == ALU1_POPCNT) && mode == MODE_INT)
1964 goto x86_bsf_bsr_popcnt_finish;
1965 } else {
1966 g(gen_frame_load_op1(ctx, op_size, alu, 1, slot_1, 0, target));
1967 }
1968 if (alu == ALU1_POPCNT)
1969 goto x86_bsf_bsr_popcnt_finish;
1970 if (mode == MODE_FIXED) {
1971 uint32_t cmov_label;
1972 gen_insn(INSN_MOV, maximum(op_size, OP_SIZE_4), 0, 0);
1973 gen_one(R_SCRATCH_2);
1974 gen_one(ARG_IMM);
1975 gen_eight(-1);
1976 g(gen_cmov(ctx, maximum(op_size, OP_SIZE_4), COND_E, target, &cmov_label));
1977 gen_one(R_SCRATCH_2);
1978 if (cmov_label)
1979 gen_label(cmov_label);
1981 } else {
1982 gen_insn(INSN_JMP_COND, maximum(op_size, OP_SIZE_2), COND_E, 0);
1983 gen_four(label_ovf);
1984 }
1985 x86_bsf_bsr_popcnt_finish:
1986 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1987 return true;
1988 #endif
1989 #if defined(ARCH_ARM)
1990 #if defined(ARCH_ARM32)
1991 if (alu == ALU1_BSR && unlikely(!cpu_test_feature(CPU_FEATURE_armv6)))
1992 goto do_generic_bsf_bsr_popcnt;
1993 if (alu == ALU1_BSF && unlikely(!cpu_test_feature(CPU_FEATURE_armv6t2)))
1994 goto do_generic_bsf_bsr_popcnt;
1995 #endif
1996 if (alu == ALU1_POPCNT && unlikely(!cpu_test_feature(CPU_FEATURE_neon)))
1997 goto do_generic_bsf_bsr_popcnt;
1998 g(gen_frame_get(ctx, op_size, mode == MODE_INT ? sign_x : zero_x, slot_1, 0, R_SCRATCH_1, ®1));
1999 if (mode == MODE_INT) {
2000 g(gen_cmp_test_jmp(ctx, INSN_TEST, i_size(op_size), reg1, reg1, alu == ALU1_BSR ? COND_LE : alu == ALU1_BSF ? COND_E : COND_S, label_ovf));
2001 }
2003 if (alu == ALU1_POPCNT) {
2004 g(gen_mov(ctx, OP_SIZE_NATIVE, FR_SCRATCH_1, reg1));
2005 gen_insn(INSN_FP_ALU1, OP_SIZE_NATIVE, FP_ALU1_VCNT8, 0);
2006 gen_one(FR_SCRATCH_1);
2007 gen_one(FR_SCRATCH_1);
2008 #if defined(ARCH_ARM32)
2009 if (op_size > OP_SIZE_1) {
2010 gen_insn(INSN_FP_ALU1, OP_SIZE_1, FP_ALU1_VPADDL, 0);
2011 gen_one(FR_SCRATCH_1);
2012 gen_one(FR_SCRATCH_1);
2013 }
2014 if (op_size > OP_SIZE_2) {
2015 gen_insn(INSN_FP_ALU1, OP_SIZE_2, FP_ALU1_VPADDL, 0);
2016 gen_one(FR_SCRATCH_1);
2017 gen_one(FR_SCRATCH_1);
2018 }
2019 #else
2020 if (op_size > OP_SIZE_1) {
2021 gen_insn(INSN_FP_ALU1, OP_SIZE_1, FP_ALU1_ADDV, 0);
2022 gen_one(FR_SCRATCH_1);
2023 gen_one(FR_SCRATCH_1);
2024 }
2025 #endif
2026 g(gen_frame_store(ctx, op_size, slot_r, 0, FR_SCRATCH_1));
2027 if (ctx->registers[slot_r] >= 0)
2028 g(unspill(ctx, slot_r));
2029 return true;
2030 }
2032 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
2033 if (mode == MODE_FIXED && alu == ALU1_BSF) {
2034 gen_insn(INSN_TEST, i_size(op_size), 0, 1);
2035 gen_one(target);
2036 gen_one(reg1);
2037 reg1 = target;
2038 }
2040 if (alu == ALU1_BSF) {
2041 g(gen_2address_alu1(ctx, i_size(op_size), ALU1_BREV, target, reg1, 0));
2042 reg1 = target;
2043 }
2045 g(gen_2address_alu1(ctx, i_size(op_size), ALU1_LZCNT, target, reg1, 0));
2047 if (alu == ALU1_BSR) {
2048 g(gen_load_constant(ctx, R_SCRATCH_2, op_size == OP_SIZE_8 ? 63 : 31));
2049 g(gen_3address_alu(ctx, i_size(op_size), ALU_SUB, target, R_SCRATCH_2, target, 0));
2050 }
2052 if (mode == MODE_FIXED && alu == ALU1_BSF) {
2053 #if defined(ARCH_ARM32)
2054 g(gen_imm(ctx, -1, IMM_PURPOSE_CMOV, OP_SIZE_NATIVE));
2055 gen_insn(INSN_CMOV, OP_SIZE_NATIVE, COND_E, 0);
2056 gen_one(target);
2057 gen_one(target);
2058 gen_imm_offset();
2059 #else
2060 gen_insn(INSN_CSEL_INV, i_size(op_size), COND_NE, 0);
2061 gen_one(target);
2062 gen_one(ARG_IMM);
2063 gen_eight(0);
2064 gen_one(target);
2065 #endif
2066 }
2068 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2069 return true;
2070 #endif
2071 #if defined(ARCH_ALPHA)
2072 if (likely(cpu_test_feature(CPU_FEATURE_cix))) {
2073 g(gen_frame_get(ctx, op_size, mode == MODE_INT ? sign_x : zero_x, slot_1, 0, R_SCRATCH_1, ®1));
2074 target = gen_frame_target(ctx, slot_r, slot_1, NO_FRAME_T, R_SCRATCH_2);
2075 if (mode == MODE_INT) {
2076 g(gen_cmp_test_jmp(ctx, INSN_TEST, OP_SIZE_NATIVE, reg1, reg1, alu == ALU1_BSR ? COND_LE : alu == ALU1_BSF ? COND_E : COND_S, label_ovf));
2077 }
2078 if (alu == ALU1_POPCNT) {
2079 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_POPCNT, target, reg1, 0));
2080 }
2081 if (alu == ALU1_BSF) {
2082 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_BSF, target, reg1, 0));
2084 if (mode == MODE_FIXED) {
2085 g(gen_imm(ctx, -1, IMM_PURPOSE_MOVR, OP_SIZE_INT));
2086 gen_insn(INSN_MOVR, OP_SIZE_NATIVE, COND_E, 0);
2087 gen_one(target);
2088 gen_one(target);
2089 gen_one(reg1);
2090 gen_imm_offset();
2091 }
2092 }
2093 if (alu == ALU1_BSR) {
2094 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_LZCNT, target, reg1, 0));
2096 g(gen_load_constant(ctx, R_SCRATCH_3, OP_SIZE_NATIVE == OP_SIZE_8 ? 63 : 31));
2098 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_SUB, target, R_SCRATCH_3, target, 0));
2099 }
2100 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2101 return true;
2102 }
2103 #endif
2104 #if defined(ARCH_MIPS)
2105 if (MIPS_HAS_CLZ && alu != ALU1_POPCNT) {
2106 g(gen_frame_get(ctx, op_size, mode == MODE_INT ? sign_x : zero_x, slot_1, 0, R_SCRATCH_1, ®1));
2107 target = gen_frame_target(ctx, slot_r, slot_1, NO_FRAME_T, R_SCRATCH_2);
2108 if (mode == MODE_INT) {
2109 g(gen_cmp_test_jmp(ctx, INSN_TEST, OP_SIZE_NATIVE, reg1, reg1, alu == ALU1_BSR ? COND_LE : alu == ALU1_BSF ? COND_E : COND_S, label_ovf));
2110 }
2111 if (alu == ALU1_BSF) {
2112 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NEG, target, reg1, 0));
2114 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_AND, R_SCRATCH_1, reg1, target, 0));
2115 reg1 = R_SCRATCH_1;
2116 }
2117 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_LZCNT, target, reg1, 0));
2119 g(gen_load_constant(ctx, R_SCRATCH_3, OP_SIZE_NATIVE == OP_SIZE_8 ? 63 : 31));
2121 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_SUB, target, R_SCRATCH_3, target, 0));
2123 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2124 return true;
2125 }
2126 #endif
2127 #if defined(ARCH_POWER)
2128 if (alu == ALU1_BSF && (unlikely(!cpu_test_feature(CPU_FEATURE_v203)) || unlikely(!cpu_test_feature(CPU_FEATURE_v30))))
2129 goto do_generic_bsf_bsr_popcnt;
2130 if (alu == ALU1_POPCNT && unlikely(!cpu_test_feature(CPU_FEATURE_v206)))
2131 goto do_generic_bsf_bsr_popcnt;
2132 g(gen_frame_get(ctx, op_size, mode == MODE_INT ? sign_x : zero_x, slot_1, 0, R_SCRATCH_1, ®1));
2133 target = gen_frame_target(ctx, slot_r, slot_1, NO_FRAME_T, R_SCRATCH_2);
2134 if (mode == MODE_INT) {
2135 g(gen_cmp_test_jmp(ctx, INSN_TEST, i_size(op_size), reg1, reg1, alu == ALU1_BSR ? COND_LE : alu == ALU1_BSF ? COND_E : COND_S, label_ovf));
2136 }
2137 if (alu == ALU1_POPCNT) {
2138 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_POPCNT, target, reg1, 0));
2139 }
2140 if (alu == ALU1_BSF) {
2141 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_BSF, target, reg1, 0));
2143 if (mode == MODE_FIXED) {
2144 g(gen_3address_alu(ctx, i_size(op_size), ALU_AND, R_SCRATCH_3, reg1, reg1, 1));
2146 g(gen_imm(ctx, -1, IMM_PURPOSE_CMOV, OP_SIZE_NATIVE));
2147 gen_insn(INSN_CMOV, OP_SIZE_NATIVE, COND_E, 0);
2148 gen_one(target);
2149 gen_one(target);
2150 gen_imm_offset();
2151 }
2152 }
2153 if (alu == ALU1_BSR) {
2154 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_LZCNT, target, reg1, 0));
2156 g(gen_load_constant(ctx, R_SCRATCH_3, OP_SIZE_NATIVE == OP_SIZE_8 ? 63 : 31));
2158 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_SUB, target, R_SCRATCH_3, target, 0));
2159 }
2160 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2161 return true;
2162 #endif
2163 #if defined(ARCH_LOONGARCH64) || defined(ARCH_RISCV64)
2164 #if defined(ARCH_LOONGARCH64)
2165 if (alu == ALU1_POPCNT)
2166 goto do_generic_bsf_bsr_popcnt;
2167 #endif
2168 #if defined(ARCH_RISCV64)
2169 if (unlikely(!cpu_test_feature(CPU_FEATURE_zbb)))
2170 goto do_generic_bsf_bsr_popcnt;
2171 #endif
2172 g(gen_frame_get(ctx, op_size, mode == MODE_INT ? sign_x : zero_x, slot_1, 0, R_SCRATCH_1, ®1));
2173 target = gen_frame_target(ctx, slot_r, slot_1, NO_FRAME_T, R_SCRATCH_2);
2174 if (mode == MODE_INT) {
2175 g(gen_cmp_test_jmp(ctx, INSN_TEST, OP_SIZE_NATIVE, reg1, reg1, alu == ALU1_BSR ? COND_LE : alu == ALU1_BSF ? COND_E : COND_S, label_ovf));
2176 }
2177 if (alu == ALU1_POPCNT) {
2178 g(gen_2address_alu1(ctx, maximum(OP_SIZE_4, op_size), ALU1_POPCNT, target, reg1, 0));
2179 }
2180 if (alu == ALU1_BSF) {
2181 g(gen_2address_alu1(ctx, maximum(OP_SIZE_4, op_size), ALU1_BSF, target, reg1, 0));
2183 if (mode == MODE_FIXED) {
2184 g(gen_imm(ctx, 1, IMM_PURPOSE_CMP, OP_SIZE_NATIVE));
2185 gen_insn(INSN_CMP_DEST_REG, OP_SIZE_NATIVE, COND_B, 0);
2186 gen_one(R_SCRATCH_3);
2187 gen_one(reg1);
2188 gen_imm_offset();
2190 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NEG, R_SCRATCH_3, R_SCRATCH_3, 0));
2192 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_OR, target, target, R_SCRATCH_3, 0));
2193 }
2194 }
2195 if (alu == ALU1_BSR) {
2196 g(gen_2address_alu1(ctx, maximum(OP_SIZE_4, op_size), ALU1_LZCNT, target, reg1, 0));
2198 g(gen_load_constant(ctx, R_SCRATCH_3, op_size <= OP_SIZE_4 ? 31 : 63));
2200 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_SUB, target, R_SCRATCH_3, target, 0));
2201 }
2202 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2203 return true;
2204 #endif
2205 #if defined(ARCH_IA64) || defined(ARCH_S390) || defined(ARCH_SPARC)
2206 if (alu == ALU1_BSF && !ARCH_HAS_ANDN)
2207 goto do_generic_bsf_bsr_popcnt;
2208 #if defined(ARCH_S390)
2209 if (!cpu_test_feature(CPU_FEATURE_misc_45) || !cpu_test_feature(CPU_FEATURE_misc_insn_ext_3))
2210 goto do_generic_bsf_bsr_popcnt;
2211 #endif
2212 #if defined(ARCH_SPARC)
2213 if (!SPARC_9)
2214 goto do_generic_bsf_bsr_popcnt;
2215 #endif
2216 g(gen_frame_get(ctx, op_size, mode == MODE_INT ? sign_x : zero_x, slot_1, 0, R_SCRATCH_1, ®1));
2217 target = gen_frame_target(ctx, slot_r, slot_1, NO_FRAME_T, R_SCRATCH_2);
2218 if (mode == MODE_INT) {
2219 g(gen_cmp_test_jmp(ctx, INSN_TEST, maximum(op_size, OP_SIZE_4), reg1, reg1, alu == ALU1_BSR ? COND_LE : alu == ALU1_BSF ? COND_E : COND_S, label_ovf));
2220 } else {
2221 if (ARCH_PREFERS_SX(op_size) && alu == ALU1_POPCNT && op_size < OP_SIZE_NATIVE) {
2222 g(gen_extend(ctx, op_size, zero_x, R_SCRATCH_1, reg1));
2223 reg1 = R_SCRATCH_1;
2224 }
2225 }
2226 if (alu == ALU1_POPCNT) {
2227 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_POPCNT, R_SCRATCH_1, reg1, 0));
2228 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
2229 return true;
2230 }
2231 if (alu == ALU1_BSF) {
2232 g(gen_3address_alu_imm(ctx, OP_SIZE_NATIVE, ALU_SUB, target, reg1, 1, 0));
2234 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_ANDN, target, target, reg1, 0));
2236 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_POPCNT, target, target, 0));
2238 if (mode == MODE_FIXED) {
2239 unsigned attr_unused test_reg = R_SCRATCH_1;
2240 #if defined(ARCH_S390)
2241 g(gen_imm(ctx, 0, COND_IS_LOGICAL(COND_E) ? IMM_PURPOSE_CMP_LOGICAL : IMM_PURPOSE_CMP, OP_SIZE_NATIVE));
2242 gen_insn(INSN_CMP, OP_SIZE_NATIVE, 0, 1 + COND_IS_LOGICAL(COND_E));
2243 gen_one(reg1);
2244 gen_imm_offset();
2246 g(gen_imm(ctx, -1, IMM_PURPOSE_CMOV, OP_SIZE_NATIVE));
2247 gen_insn(INSN_CMOV, OP_SIZE_NATIVE, COND_E, 0);
2248 gen_one(target);
2249 gen_one(target);
2250 gen_imm_offset();
2251 #else
2252 #if defined(ARCH_IA64)
2253 g(gen_cmp_dest_reg(ctx, OP_SIZE_NATIVE, reg1, (unsigned)-1, R_CMP_RESULT, 0, COND_NE));
2254 test_reg = R_CMP_RESULT;
2255 #endif
2256 g(gen_imm(ctx, -1, IMM_PURPOSE_MOVR, OP_SIZE_NATIVE));
2257 gen_insn(INSN_MOVR, OP_SIZE_NATIVE, COND_E, 0);
2258 gen_one(target);
2259 gen_one(target);
2260 gen_one(test_reg);
2261 gen_imm_offset();
2262 #endif
2263 }
2265 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2266 return true;
2267 }
2268 #endif
2269 do_generic_bsf_bsr_popcnt:
2270 if (alu == ALU1_BSF) {
2271 if (mode == MODE_FIXED)
2272 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_unary_bsf_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, 0);
2273 else
2274 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, INT_unary_bsf_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, label_ovf);
2275 }
2276 if (alu == ALU1_BSR) {
2277 if (mode == MODE_FIXED)
2278 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_unary_bsr_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, 0);
2279 else
2280 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, INT_unary_bsr_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, label_ovf);
2281 }
2282 if (alu == ALU1_POPCNT) {
2283 if (mode == MODE_FIXED)
2284 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_unary_popcnt_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, 0);
2285 else
2286 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, INT_unary_popcnt_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, label_ovf);
2287 }
2288 }
2289 /**************
2290 * CONVERSION *
2291 **************/
2292 do_fixed_conv:
2293 do_conv: {
2294 unsigned src_op_size, dest_op_size;
2295 const struct type *src_type, *dest_type;
2296 src_type = get_type_of_local(ctx, slot_1);
2297 dest_type = get_type_of_local(ctx, slot_r);
2299 if (TYPE_TAG_IS_FIXED(src_type->tag)) {
2300 src_op_size = TYPE_TAG_IDX_FIXED(src_type->tag) >> 1;
2301 } else {
2302 src_op_size = TYPE_TAG_IDX_INT(src_type->tag);
2303 }
2305 if (TYPE_TAG_IS_FIXED(dest_type->tag)) {
2306 dest_op_size = TYPE_TAG_IDX_FIXED(dest_type->tag) >> 1;
2307 } else {
2308 dest_op_size = TYPE_TAG_IDX_INT(dest_type->tag);
2309 }
2311 if (src_op_size <= OP_SIZE_NATIVE) {
2312 g(gen_frame_get(ctx, src_op_size, sign_x, slot_1, 0, R_SCRATCH_1, ®1));
2313 } else {
2314 #if defined(ARCH_X86)
2315 if (dest_op_size < src_op_size)
2316 g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_1, 0, R_SCRATCH_1));
2317 else
2318 #endif
2319 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_3));
2320 reg1 = R_SCRATCH_1;
2321 }
2323 if (dest_op_size >= src_op_size) {
2324 if (dest_op_size <= OP_SIZE_NATIVE) {
2325 g(gen_frame_store(ctx, dest_op_size, slot_r, 0, reg1));
2326 } else {
2327 if (src_op_size <= OP_SIZE_NATIVE) {
2328 #if defined(ARCH_X86)
2329 if (R_SCRATCH_1 != R_AX || R_SCRATCH_2 != R_DX)
2330 internal(file_line, "gen_alu1: bad scratch registers");
2331 if (reg1 == R_SCRATCH_1) {
2332 gen_insn(INSN_CWD, OP_SIZE_NATIVE, 0, 0);
2333 gen_one(R_DX);
2334 gen_one(R_AX);
2335 } else
2336 #endif
2337 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_2, reg1, (1U << (OP_SIZE_NATIVE + 3)) - 1, false));
2338 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, reg1, R_SCRATCH_2));
2339 } else {
2340 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, reg1, R_SCRATCH_3));
2341 }
2342 }
2343 return true;
2344 } else {
2345 if (src_op_size > OP_SIZE_NATIVE) {
2346 #if defined(ARCH_ARM)
2347 gen_insn(INSN_CMP, OP_SIZE_NATIVE, 0, 1);
2348 gen_one(R_SCRATCH_3);
2349 gen_one(ARG_SHIFTED_REGISTER);
2350 gen_one(ARG_SHIFT_ASR | ((1U << (OP_SIZE_NATIVE + 3)) - 1));
2351 gen_one(R_SCRATCH_1);
2353 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_NE, 0);
2354 gen_four(label_ovf);
2355 #elif defined(ARCH_X86)
2356 if (R_SCRATCH_1 != R_AX || R_SCRATCH_2 != R_DX)
2357 internal(file_line, "gen_alu1: bad scratch registers");
2358 gen_insn(INSN_CWD, OP_SIZE_NATIVE, 0, 0);
2359 gen_one(R_DX);
2360 gen_one(R_AX);
2362 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
2363 gen_insn(INSN_CMP, OP_SIZE_NATIVE, 0, 1);
2364 gen_one(R_SCRATCH_2);
2365 gen_address_offset();
2367 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_NE, 0);
2368 gen_four(label_ovf);
2369 #else
2370 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_2, R_SCRATCH_1, (1U << (OP_SIZE_NATIVE + 3)) - 1, 0));
2372 g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, R_SCRATCH_3, R_SCRATCH_2, COND_NE, label_ovf));
2373 #endif
2375 src_op_size = OP_SIZE_NATIVE;
2376 }
2377 if (src_op_size > dest_op_size) {
2378 g(gen_cmp_extended(ctx, OP_SIZE_NATIVE, dest_op_size, reg1, R_SCRATCH_2, label_ovf));
2379 }
2380 g(gen_frame_store(ctx, dest_op_size, slot_r, 0, reg1));
2381 return true;
2382 }
2383 }
2385 conv_uto_int: {
2386 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_uto_int_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, label_ovf);
2387 }
2389 conv_ufrom_int: {
2390 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_ufrom_int_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, label_ovf);
2391 }
2392 }
2394 static bool attr_w gen_constant(struct codegen_context *ctx, bool real, unsigned op_size, bool shrt, frame_t slot_r)
2395 {
2396 uintbig_t c;
2397 if (shrt) {
2398 c = (int16_t)get_unaligned_16(ctx->current_position);
2399 } else switch (op_size) {
2400 #define fx(n, type, utype, sz, bits) \
2401 case n: \
2402 c = (type)cat(get_unaligned_,bits)(ctx->current_position);\
2403 break;
2404 for_all_fixed(fx);
2405 #undef fx
2406 default:
2407 internal(file_line, "gen_constant: invalid type %u", op_size);
2408 }
2409 if (op_size > OP_SIZE_NATIVE) {
2410 unsigned shift = (8U << OP_SIZE_NATIVE) - 1;
2411 g(gen_frame_store_imm_raw(ctx, OP_SIZE_NATIVE, slot_r, lo_word(OP_SIZE_NATIVE), c & ((2ULL << shift) - 1)));
2412 g(gen_frame_store_imm_raw(ctx, OP_SIZE_NATIVE, slot_r, hi_word(OP_SIZE_NATIVE), c >> 1 >> shift));
2413 if (real && ctx->registers[slot_r] >= 0)
2414 g(unspill(ctx, slot_r));
2415 return true;
2416 } else if (real && ctx->registers[slot_r] >= 0) {
2417 if (ARCH_HAS_FP_GP_MOV) {
2418 g(gen_load_constant(ctx, R_SCRATCH_1, c));
2419 g(gen_mov(ctx, op_size, ctx->registers[slot_r], R_SCRATCH_1));
2420 } else {
2421 g(gen_frame_store_imm_raw(ctx, op_size, slot_r, 0, c));
2422 g(unspill(ctx, slot_r));
2423 }
2424 } else {
2425 g(gen_frame_store_imm(ctx, op_size, slot_r, 0, c));
2426 }
2427 return true;
2428 }
2430 static bool attr_w gen_real_constant(struct codegen_context *ctx, const struct type *t, frame_t slot_r)
2431 {
2432 int64_t offset;
2433 if (is_power_of_2(t->size) && t->size <= sizeof(uintbig_t))
2434 return gen_constant(ctx, true, log_2(t->size), false, slot_r);
2436 g(load_function_offset(ctx, R_SCRATCH_3, offsetof(struct data, u_.function.code)));
2438 offset = (ctx->current_position - da(ctx->fn,function)->code) * sizeof(code_t);
2440 g(gen_memcpy_raw(ctx, R_FRAME, (size_t)slot_r * slot_size, R_SCRATCH_3, offset, t->size, minimum(t->align, sizeof(code_t))));
2441 if (ctx->registers[slot_r] >= 0)
2442 g(unspill(ctx, slot_r));
2444 return true;
2445 }
2447 static bool attr_w gen_copy(struct codegen_context *ctx, unsigned op_size, frame_t slot_1, frame_t slot_r)
2448 {
2449 unsigned reg1;
2450 if (unlikely(op_size > OP_SIZE_NATIVE)) {
2451 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
2452 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_1, R_SCRATCH_2));
2453 return true;
2454 } else {
2455 unsigned target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
2456 g(gen_frame_get(ctx, op_size, garbage, slot_1, 0, target, ®1));
2457 g(gen_frame_store(ctx, op_size, slot_r, 0, reg1));
2458 return true;
2459 }
2460 }
2462 static bool attr_w gen_fp_alu(struct codegen_context *ctx, unsigned real_type, unsigned op, uint32_t label_ovf, frame_t slot_1, frame_t slot_2, frame_t slot_r)
2463 {
2464 unsigned attr_unused fp_alu;
2465 size_t upc;
2466 unsigned attr_unused op_size = real_type_to_op_size(real_type);
2467 unsigned reg1, reg2, target;
2468 switch (op) {
2469 case OPCODE_REAL_OP_add:
2470 case OPCODE_REAL_OP_add_alt1:
2471 case OPCODE_REAL_OP_add_alt2: fp_alu = FP_ALU_ADD; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_add_real16_t); label_ovf = 0; goto do_alu;
2472 case OPCODE_REAL_OP_subtract:
2473 case OPCODE_REAL_OP_subtract_alt1:
2474 case OPCODE_REAL_OP_subtract_alt2: fp_alu = FP_ALU_SUB; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_subtract_real16_t); label_ovf = 0; goto do_alu;
2475 case OPCODE_REAL_OP_multiply:
2476 case OPCODE_REAL_OP_multiply_alt1:
2477 case OPCODE_REAL_OP_multiply_alt2: fp_alu = FP_ALU_MUL; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_multiply_real16_t); label_ovf = 0; goto do_alu;
2478 case OPCODE_REAL_OP_divide:
2479 case OPCODE_REAL_OP_divide_alt1:
2480 case OPCODE_REAL_OP_divide_alt2: fp_alu = FP_ALU_DIV; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_divide_real16_t); label_ovf = 0; goto do_alu;
2481 case OPCODE_REAL_OP_modulo:
2482 case OPCODE_REAL_OP_power:
2483 case OPCODE_REAL_OP_ldexp:
2484 case OPCODE_REAL_OP_atan2: upc = offsetof(struct cg_upcall_vector_s, REAL_binary_modulo_real16_t) + (op - OPCODE_REAL_OP_modulo) * TYPE_REAL_N * sizeof(void (*)(void)); goto do_upcall;
2485 case OPCODE_REAL_OP_equal:
2486 case OPCODE_REAL_OP_equal_alt1:
2487 case OPCODE_REAL_OP_equal_alt2: fp_alu = FP_COND_E; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_equal_real16_t); goto do_cmp;
2488 case OPCODE_REAL_OP_not_equal:
2489 case OPCODE_REAL_OP_not_equal_alt1:
2490 case OPCODE_REAL_OP_not_equal_alt2: fp_alu = FP_COND_NE; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_not_equal_real16_t); goto do_cmp;
2491 case OPCODE_REAL_OP_less:
2492 case OPCODE_REAL_OP_less_alt1:
2493 case OPCODE_REAL_OP_less_alt2: fp_alu = FP_COND_B; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_less_real16_t); goto do_cmp;
2494 case OPCODE_REAL_OP_less_equal:
2495 case OPCODE_REAL_OP_less_equal_alt1:
2496 case OPCODE_REAL_OP_less_equal_alt2: fp_alu = FP_COND_BE; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_less_equal_real16_t); goto do_cmp;
2497 default: internal(file_line, "gen_fp_alu: unsupported operation %u", op);
2498 }
2500 do_alu:
2501 if ((SUPPORTED_FP >> real_type) & 1) {
2502 #if defined(ARCH_IA64)
2503 if (unlikely(fp_alu == FP_ALU_DIV))
2504 goto do_upcall;
2505 #endif
2506 #if defined(ARCH_X86)
2507 if (1)
2508 #elif defined(ARCH_S390)
2509 if ((op_size <= OP_SIZE_8 && (size_t)slot_2 * slot_size < 4096) || ctx->registers[slot_2] >= 0)
2510 #else
2511 if (ctx->registers[slot_2] >= 0)
2512 #endif
2513 {
2514 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, slot_2, FR_SCRATCH_1);
2515 g(gen_frame_get(ctx, op_size, garbage, slot_1, 0, FR_SCRATCH_1, ®1));
2516 if (ctx->registers[slot_2] >= 0) {
2517 g(gen_3address_fp_alu(ctx, op_size, fp_alu, target, reg1, ctx->registers[slot_2]));
2518 } else {
2519 if (target != reg1 && !ARCH_IS_3ADDRESS_FP) {
2520 g(gen_mov(ctx, op_size, target, reg1));
2521 reg1 = target;
2522 }
2523 g(gen_address(ctx, R_FRAME, (size_t)slot_2 * slot_size, IMM_PURPOSE_VLDR_VSTR_OFFSET, op_size));
2524 gen_insn(INSN_FP_ALU, op_size, fp_alu, 0);
2525 gen_one(target);
2526 gen_one(reg1);
2527 gen_address_offset();
2528 }
2529 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2530 return true;
2531 }
2532 #if defined(ARCH_ALPHA)
2533 g(gen_frame_get(ctx, op_size, zero_x, slot_1, 0, FR_SCRATCH_1, ®1));
2534 g(gen_frame_get(ctx, op_size, zero_x, slot_2, 0, FR_SCRATCH_2, ®2));
2535 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, FR_SCRATCH_3);
2536 g(gen_3address_fp_alu(ctx, op_size, fp_alu, target, reg1, reg2));
2537 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2538 #else
2539 g(gen_frame_get(ctx, op_size, zero_x, slot_1, 0, FR_SCRATCH_1, ®1));
2540 g(gen_frame_get(ctx, op_size, zero_x, slot_2, 0, FR_SCRATCH_2, ®2));
2541 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_1);
2542 g(gen_3address_fp_alu(ctx, op_size, fp_alu, target, reg1, reg2));
2543 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2544 #endif
2545 return true;
2546 }
2547 #ifdef SUPPORTED_FP_X87
2548 if ((SUPPORTED_FP_X87 >> real_type) & 1) {
2549 if (real_type != 3) {
2550 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_2));
2551 g(gen_frame_load_x87(ctx, INSN_X87_ALU, op_size, fp_alu, slot_1));
2552 } else {
2553 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
2554 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_2));
2555 gen_insn(INSN_X87_ALUP, op_size, fp_alu, 0);
2556 gen_one(R_ST1);
2557 }
2558 g(gen_frame_store_x87(ctx, INSN_X87_FSTP, op_size, slot_r));
2559 return true;
2560 }
2561 #endif
2562 #ifdef SUPPORTED_FP_HALF_CVT
2563 if ((SUPPORTED_FP_HALF_CVT >> real_type) & 1) {
2564 g(gen_frame_get(ctx, op_size, zero_x, slot_1, 0, FR_SCRATCH_1, ®1));
2565 g(gen_frame_get(ctx, op_size, zero_x, slot_2, 0, FR_SCRATCH_2, ®2));
2566 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_1);
2567 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
2568 gen_one(FR_SCRATCH_1);
2569 gen_one(reg1);
2570 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
2571 gen_one(FR_SCRATCH_2);
2572 gen_one(reg2);
2573 gen_insn(INSN_FP_ALU, OP_SIZE_4, fp_alu, 0);
2574 gen_one(FR_SCRATCH_1);
2575 gen_one(FR_SCRATCH_1);
2576 gen_one(FR_SCRATCH_2);
2577 gen_insn(INSN_FP_CVT, OP_SIZE_4, op_size, 0);
2578 gen_one(target);
2579 gen_one(FR_SCRATCH_1);
2580 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2581 return true;
2582 }
2583 #endif
2584 goto do_upcall;
2586 do_cmp:
2587 if ((SUPPORTED_FP >> real_type) & 1
2588 #if defined(ARCH_ALPHA)
2589 && ARCH_SUPPORTS_TRAPS
2590 #endif
2591 ) {
2592 g(gen_frame_get(ctx, op_size, zero_x, slot_1, 0, FR_SCRATCH_1, ®1));
2593 g(gen_frame_get(ctx, op_size, zero_x, slot_2, 0, FR_SCRATCH_2, ®2));
2594 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
2595 #if defined(ARCH_ALPHA)
2596 gen_insn(INSN_FP_CMP_DEST_REG_TRAP, op_size, fp_alu == FP_COND_NE ? FP_COND_E : fp_alu, 0);
2597 gen_one(FR_SCRATCH_3);
2598 gen_one(reg1);
2599 gen_one(reg2);
2600 gen_four(label_ovf);
2602 if (!ARCH_HAS_FP_GP_MOV) {
2603 g(gen_frame_store_raw(ctx, OP_SIZE_4, slot_r, 0, FR_SCRATCH_3));
2604 g(gen_frame_load_raw(ctx, OP_SIZE_4, sign_x, slot_r, 0, target));
2605 } else {
2606 g(gen_mov(ctx, OP_SIZE_4, target, FR_SCRATCH_3));
2607 }
2609 if (fp_alu == FP_COND_NE) {
2610 g(gen_imm(ctx, 0, IMM_PURPOSE_CMP, OP_SIZE_NATIVE));
2611 gen_insn(INSN_CMP_DEST_REG, OP_SIZE_NATIVE, COND_E, 0);
2612 gen_one(target);
2613 gen_one(target);
2614 gen_imm_offset();
2615 } else {
2616 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SHR, target, target, 30, 0));
2617 }
2619 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
2621 return true;
2622 #elif defined(ARCH_IA64)
2623 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_P, 0);
2624 gen_one(R_CMP_RESULT);
2625 gen_one(reg1);
2626 gen_one(reg2);
2628 gen_insn(INSN_JMP_REG, OP_SIZE_NATIVE, COND_NE, 0);
2629 gen_one(R_CMP_RESULT);
2630 gen_four(label_ovf);
2632 gen_insn(INSN_FP_CMP_DEST_REG, op_size, fp_alu, 0);
2633 gen_one(R_CMP_RESULT);
2634 gen_one(reg1);
2635 gen_one(reg2);
2637 g(gen_mov(ctx, OP_SIZE_NATIVE, target, R_CMP_RESULT));
2639 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
2641 return true;
2642 #elif defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS) || defined(ARCH_PARISC)
2643 gen_insn(INSN_FP_CMP_COND, op_size, FP_COND_P, 1);
2644 gen_one(reg1);
2645 gen_one(reg2);
2647 gen_insn(INSN_JMP_FP_TEST, 0, FP_COND_P, 0);
2648 gen_four(label_ovf);
2650 gen_insn(INSN_FP_CMP_COND, op_size, fp_alu, 1);
2651 gen_one(reg1);
2652 gen_one(reg2);
2654 gen_insn(INSN_FP_TEST_REG, OP_SIZE_NATIVE, fp_alu, 0);
2655 gen_one(target);
2657 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
2659 return true;
2660 #elif defined(ARCH_RISCV64)
2661 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_E, 0);
2662 gen_one(R_SCRATCH_1);
2663 gen_one(reg1);
2664 gen_one(reg1);
2666 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_E, 0);
2667 gen_one(R_SCRATCH_2);
2668 gen_one(reg2);
2669 gen_one(reg2);
2671 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_AND, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_2, 0));
2673 g(gen_jmp_on_zero(ctx, OP_SIZE_NATIVE, R_SCRATCH_1, COND_E, label_ovf));
2675 gen_insn(INSN_FP_CMP_DEST_REG, op_size, fp_alu == FP_COND_NE ? FP_COND_E : fp_alu, 0);
2676 gen_one(target);
2677 gen_one(reg1);
2678 gen_one(reg2);
2680 if (fp_alu == FP_COND_NE) {
2681 g(gen_imm(ctx, 1, IMM_PURPOSE_XOR, OP_SIZE_NATIVE));
2682 gen_insn(INSN_ALU, OP_SIZE_NATIVE, ALU_XOR, ALU_WRITES_FLAGS(ALU_AND, false));
2683 gen_one(target);
2684 gen_one(target);
2685 gen_imm_offset();
2686 }
2688 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
2689 return true;
2690 #else
2691 gen_insn(INSN_FP_CMP, op_size, 0, 1);
2692 gen_one(reg1);
2693 gen_one(reg2);
2694 #if defined(ARCH_ARM32)
2695 gen_insn(INSN_FP_TO_INT_FLAGS, 0, 0, 1);
2696 #endif
2697 gen_insn(INSN_JMP_COND, op_size, FP_COND_P, 0);
2698 gen_four(label_ovf);
2699 g(gen_frame_set_cond(ctx, op_size, false, fp_alu, slot_r));
2700 return true;
2701 #endif
2702 }
2703 #ifdef SUPPORTED_FP_X87
2704 if ((SUPPORTED_FP_X87 >> real_type) & 1) {
2705 if (likely(cpu_test_feature(CPU_FEATURE_cmov))) {
2706 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_2));
2707 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
2708 gen_insn(INSN_X87_FCOMIP, op_size, 0, 0);
2709 gen_one(R_ST1);
2710 gen_insn(INSN_X87_FSTP, op_size, 0, 0);
2711 gen_one(R_ST0);
2712 gen_insn(INSN_JMP_COND, op_size, COND_P, 0);
2713 gen_four(label_ovf);
2714 g(gen_frame_set_cond(ctx, op_size, false, fp_alu & 0xf, slot_r));
2715 return true;
2716 }
2718 if (real_type != 3) {
2719 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
2720 g(gen_frame_load_x87(ctx, INSN_X87_FCOMP, op_size, 0, slot_2));
2721 } else {
2722 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_2));
2723 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
2724 gen_insn(INSN_X87_FCOMPP, op_size, 0, 0);
2725 }
2727 gen_insn(INSN_X87_FNSTSW, 0, 0, 0);
2728 gen_one(R_AX);
2729 gen_one(R_AX);
2731 gen_insn(INSN_TEST, OP_SIZE_2, 0, 1);
2732 gen_one(R_AX);
2733 gen_one(ARG_IMM);
2734 gen_eight(0x0400);
2736 gen_insn(INSN_JMP_COND, OP_SIZE_2, COND_NE, 0);
2737 gen_four(label_ovf);
2739 switch (fp_alu) {
2740 case FP_COND_E:
2741 gen_insn(INSN_TEST, OP_SIZE_2, 0, 1);
2742 gen_one(R_AX);
2743 gen_one(ARG_IMM);
2744 gen_eight(0x4000);
2745 g(gen_frame_set_cond(ctx, OP_SIZE_2, false, COND_NE, slot_r));
2746 break;
2747 case FP_COND_NE:
2748 gen_insn(INSN_TEST, OP_SIZE_2, 0, 1);
2749 gen_one(R_AX);
2750 gen_one(ARG_IMM);
2751 gen_eight(0x4000);
2752 g(gen_frame_set_cond(ctx, OP_SIZE_2, false, COND_E, slot_r));
2753 break;
2754 case FP_COND_B:
2755 gen_insn(INSN_TEST, OP_SIZE_2, 0, 1);
2756 gen_one(R_AX);
2757 gen_one(ARG_IMM);
2758 gen_eight(0x0100);
2759 g(gen_frame_set_cond(ctx, OP_SIZE_2, false, COND_NE, slot_r));
2760 break;
2761 case FP_COND_BE:
2762 gen_insn(INSN_TEST, OP_SIZE_2, 0, 1);
2763 gen_one(R_AX);
2764 gen_one(ARG_IMM);
2765 gen_eight(0x4100);
2766 g(gen_frame_set_cond(ctx, OP_SIZE_2, false, COND_NE, slot_r));
2767 break;
2768 default:
2769 internal(file_line, "gen_fp_alu: invalid condition %u", fp_alu);
2770 }
2771 return true;
2772 }
2773 #endif
2774 #ifdef SUPPORTED_FP_HALF_CVT
2775 if ((SUPPORTED_FP_HALF_CVT >> real_type) & 1) {
2776 g(gen_frame_get(ctx, op_size, zero_x, slot_1, 0, FR_SCRATCH_1, ®1));
2777 g(gen_frame_get(ctx, op_size, zero_x, slot_2, 0, FR_SCRATCH_2, ®2));
2778 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
2779 gen_one(FR_SCRATCH_1);
2780 gen_one(reg1);
2781 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
2782 gen_one(FR_SCRATCH_2);
2783 gen_one(reg2);
2784 gen_insn(INSN_FP_CMP, OP_SIZE_4, 0, 1);
2785 gen_one(FR_SCRATCH_1);
2786 gen_one(FR_SCRATCH_2);
2787 #if defined(ARCH_ARM32)
2788 gen_insn(INSN_FP_TO_INT_FLAGS, 0, 0, 1);
2789 #endif
2790 gen_insn(INSN_JMP_COND, op_size, FP_COND_P, 0);
2791 gen_four(label_ovf);
2792 g(gen_frame_set_cond(ctx, op_size, false, fp_alu, slot_r));
2793 return true;
2794 }
2795 #endif
2797 do_upcall:
2798 return gen_alu_typed_upcall(ctx, upc, real_type, slot_1, slot_2, slot_r, label_ovf);
2799 }
2801 #define OP_IS_ROUND(alu) ((alu) == FP_ALU1_ROUND || (alu) == FP_ALU1_FLOOR || (alu) == FP_ALU1_CEIL || (alu) == FP_ALU1_TRUNC)
2803 static bool attr_w gen_fp_alu1(struct codegen_context *ctx, unsigned real_type, unsigned op, uint32_t label_ovf, frame_t slot_1, frame_t slot_r)
2804 {
2805 unsigned attr_unused fp_alu;
2806 size_t upc;
2807 unsigned attr_unused op_size = real_type_to_op_size(real_type);
2808 unsigned reg1, target;
2809 switch (op) {
2810 case OPCODE_REAL_OP_neg:
2811 case OPCODE_REAL_OP_neg_alt1:
2812 case OPCODE_REAL_OP_neg_alt2: fp_alu = FP_ALU1_NEG; upc = offsetof(struct cg_upcall_vector_s, REAL_unary_neg_real16_t); label_ovf = 0; goto do_alu;
2813 case OPCODE_REAL_OP_sqrt:
2814 case OPCODE_REAL_OP_sqrt_alt1:
2815 case OPCODE_REAL_OP_sqrt_alt2: fp_alu = FP_ALU1_SQRT; upc = offsetof(struct cg_upcall_vector_s, REAL_unary_sqrt_real16_t); label_ovf = 0; goto do_alu;
2816 case OPCODE_REAL_OP_round: fp_alu = FP_ALU1_ROUND; upc = offsetof(struct cg_upcall_vector_s, REAL_unary_round_real16_t); label_ovf = 0; goto do_alu;
2817 case OPCODE_REAL_OP_floor: fp_alu = FP_ALU1_FLOOR; upc = offsetof(struct cg_upcall_vector_s, REAL_unary_floor_real16_t); label_ovf = 0; goto do_alu;
2818 case OPCODE_REAL_OP_ceil: fp_alu = FP_ALU1_CEIL; upc = offsetof(struct cg_upcall_vector_s, REAL_unary_ceil_real16_t); label_ovf = 0; goto do_alu;
2819 case OPCODE_REAL_OP_trunc: fp_alu = FP_ALU1_TRUNC; upc = offsetof(struct cg_upcall_vector_s, REAL_unary_trunc_real16_t); label_ovf = 0; goto do_alu;
2820 case OPCODE_REAL_OP_to_int:
2821 case OPCODE_REAL_OP_to_int_alt1:
2822 case OPCODE_REAL_OP_to_int_alt2: upc = offsetof(struct cg_upcall_vector_s, REAL_unary_to_int_real16_t); goto do_to_int;
2823 case OPCODE_REAL_OP_from_int:
2824 case OPCODE_REAL_OP_from_int_alt1:
2825 case OPCODE_REAL_OP_from_int_alt2: upc = offsetof(struct cg_upcall_vector_s, REAL_unary_from_int_real16_t); label_ovf = 0; goto do_from_int;
2826 case OPCODE_REAL_OP_is_exception:
2827 case OPCODE_REAL_OP_is_exception_alt1:
2828 case OPCODE_REAL_OP_is_exception_alt2: upc = offsetof(struct cg_upcall_vector_s, REAL_unary_is_exception_real16_t); label_ovf = 0; goto do_is_exception;
2829 default: upc = offsetof(struct cg_upcall_vector_s, REAL_unary_cbrt_real16_t) + (op - OPCODE_REAL_OP_cbrt) * TYPE_REAL_N * sizeof(void (*)(void)); label_ovf = 0; goto do_upcall;
2830 }
2832 do_alu:
2833 if ((SUPPORTED_FP >> real_type) & 1 && (
2834 #if defined(ARCH_ALPHA)
2835 fp_alu == FP_ALU1_NEG ||
2836 (fp_alu == FP_ALU1_SQRT && cpu_test_feature(CPU_FEATURE_fix)) ||
2837 #elif defined(ARCH_ARM32)
2838 fp_alu == FP_ALU1_NEG ||
2839 fp_alu == FP_ALU1_SQRT ||
2840 #elif defined(ARCH_ARM64)
2841 true ||
2842 #elif defined(ARCH_IA64)
2843 fp_alu == FP_ALU1_NEG ||
2844 #elif defined(ARCH_LOONGARCH64)
2845 fp_alu == FP_ALU1_NEG ||
2846 fp_alu == FP_ALU1_SQRT ||
2847 fp_alu == FP_ALU1_ROUND ||
2848 #elif defined(ARCH_MIPS)
2849 fp_alu == FP_ALU1_NEG ||
2850 (fp_alu == FP_ALU1_SQRT && MIPS_HAS_SQRT) ||
2851 #elif defined(ARCH_PARISC)
2852 (fp_alu == FP_ALU1_NEG && PA_20) ||
2853 fp_alu == FP_ALU1_SQRT ||
2854 #elif defined(ARCH_POWER)
2855 fp_alu == FP_ALU1_NEG ||
2856 (fp_alu == FP_ALU1_SQRT && cpu_test_feature(CPU_FEATURE_p2) && real_type != 4) ||
2857 #elif defined(ARCH_S390)
2858 true ||
2859 #elif defined(ARCH_SPARC)
2860 fp_alu == FP_ALU1_NEG ||
2861 fp_alu == FP_ALU1_SQRT ||
2862 #elif defined(ARCH_RISCV64)
2863 fp_alu == FP_ALU1_NEG ||
2864 fp_alu == FP_ALU1_SQRT ||
2865 #elif defined(ARCH_X86)
2866 fp_alu == FP_ALU1_SQRT ||
2867 (OP_IS_ROUND(fp_alu) && cpu_test_feature(CPU_FEATURE_sse41)) ||
2868 #endif
2869 false)) {
2870 #if defined(ARCH_S390)
2871 if (op_size <= OP_SIZE_8 && (size_t)slot_1 * slot_size < 4096 && fp_alu == FP_ALU1_SQRT) {
2872 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_1);
2873 if (ctx->registers[slot_1] >= 0) {
2874 gen_insn(INSN_FP_ALU1, op_size, fp_alu, 0);
2875 gen_one(target);
2876 gen_one(ctx->registers[slot_1]);
2877 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2878 } else {
2879 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size, IMM_PURPOSE_VLDR_VSTR_OFFSET, op_size));
2880 gen_insn(INSN_FP_ALU1, op_size, fp_alu, 0);
2881 gen_one(target);
2882 gen_address_offset();
2883 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2884 }
2885 return true;
2886 }
2887 #endif
2888 g(gen_frame_get(ctx, op_size, zero_x, slot_1, 0, FR_SCRATCH_1, ®1));
2889 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_2);
2890 gen_insn(INSN_FP_ALU1, op_size, fp_alu, 0);
2891 gen_one(target);
2892 gen_one(reg1);
2893 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2894 return true;
2895 }
2896 #ifdef SUPPORTED_FP_X87
2897 if ((SUPPORTED_FP_X87 >> real_type) & 1) {
2898 if (fp_alu == FP_ALU1_NEG) {
2899 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
2900 gen_insn(INSN_X87_FCHS, op_size, 0, 0);
2901 g(gen_frame_store_x87(ctx, INSN_X87_FSTP, op_size, slot_r));
2902 return true;
2903 } else if (fp_alu == FP_ALU1_SQRT) {
2904 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
2905 gen_insn(INSN_X87_FSQRT, op_size, 0, 0);
2906 g(gen_frame_store_x87(ctx, INSN_X87_FSTP, op_size, slot_r));
2907 return true;
2908 } else if (fp_alu == FP_ALU1_ROUND) {
2909 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
2910 gen_insn(INSN_X87_FRNDINT, op_size, 0, 0);
2911 g(gen_frame_store_x87(ctx, INSN_X87_FSTP, op_size, slot_r));
2912 return true;
2913 }
2914 }
2915 #endif
2916 #ifdef SUPPORTED_FP_HALF_CVT
2917 if ((SUPPORTED_FP_HALF_CVT >> real_type) & 1 && (
2918 #if defined(ARCH_ARM32)
2919 fp_alu == FP_ALU1_NEG ||
2920 fp_alu == FP_ALU1_SQRT ||
2921 #elif defined(ARCH_ARM64)
2922 true ||
2923 #elif defined(ARCH_X86)
2924 fp_alu == FP_ALU1_SQRT ||
2925 (OP_IS_ROUND(fp_alu) && cpu_test_feature(CPU_FEATURE_sse41)) ||
2926 #endif
2927 false)) {
2928 g(gen_frame_get(ctx, op_size, zero_x, slot_1, 0, FR_SCRATCH_1, ®1));
2929 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_1);
2930 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
2931 gen_one(target);
2932 gen_one(reg1);
2933 gen_insn(INSN_FP_ALU1, OP_SIZE_4, fp_alu, 0);
2934 gen_one(target);
2935 gen_one(target);
2936 gen_insn(INSN_FP_CVT, OP_SIZE_4, op_size, 0);
2937 gen_one(target);
2938 gen_one(target);
2939 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2940 return true;
2941 }
2942 #endif
2943 goto do_upcall;
2945 do_to_int:
2946 if ((SUPPORTED_FP >> real_type) & 1
2947 #if defined(ARCH_ALPHA)
2948 && ARCH_SUPPORTS_TRAPS
2949 #endif
2950 #if defined(ARCH_MIPS)
2951 && MIPS_HAS_TRUNC
2952 #endif
2953 ) {
2954 g(gen_frame_get(ctx, op_size, zero_x, slot_1, 0, FR_SCRATCH_1, ®1));
2955 goto do_cvt_to_int;
2956 do_cvt_to_int:
2957 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
2958 #if defined(ARCH_X86)
2959 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_TO_INT32 : INSN_FP_TO_INT64, op_size, 0, 0);
2960 gen_one(target);
2961 gen_one(reg1);
2963 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, OP_SIZE_INT, target, sign_bit(uint_default_t), COND_E, label_ovf));
2965 g(gen_frame_store(ctx, OP_SIZE_INT, slot_r, 0, target));
2966 return true;
2967 #endif
2968 #if defined(ARCH_ARM) || defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS)
2969 #if defined(ARCH_ARM)
2970 gen_insn(INSN_FP_CMP, op_size, 0, 1);
2971 gen_one(reg1);
2972 gen_one(reg1);
2973 #if defined(ARCH_ARM32)
2974 gen_insn(INSN_FP_TO_INT_FLAGS, 0, 0, 1);
2975 #endif
2976 gen_insn(INSN_JMP_COND, op_size, FP_COND_P, 0);
2977 gen_four(label_ovf);
2978 #else
2979 gen_insn(INSN_FP_CMP_COND, op_size, FP_COND_P, 1);
2980 gen_one(reg1);
2981 gen_one(reg1);
2983 gen_insn(INSN_JMP_FP_TEST, 0, FP_COND_P, 0);
2984 gen_four(label_ovf);
2985 #endif
2986 #if defined(ARCH_ARM32) || defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS)
2987 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_TO_INT32 : INSN_FP_TO_INT64, op_size, 0, 0);
2988 gen_one(FR_SCRATCH_1);
2989 gen_one(reg1);
2991 g(gen_mov(ctx, OP_SIZE_INT, target, FR_SCRATCH_1));
2992 #else
2993 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_TO_INT32 : INSN_FP_TO_INT64, op_size, 0, 0);
2994 gen_one(target);
2995 gen_one(reg1);
2996 #endif
2997 g(gen_imm(ctx, (int_default_t)(sign_bit(uint_default_t) + 1), IMM_PURPOSE_ADD, OP_SIZE_INT));
2998 gen_insn(INSN_ALU, OP_SIZE_INT, ALU_ADD, ALU_WRITES_FLAGS(ALU_ADD, is_imm()));
2999 gen_one(R_SCRATCH_2);
3000 gen_one(target);
3001 gen_imm_offset();
3003 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, OP_SIZE_INT, R_SCRATCH_2, 1, COND_BE, label_ovf));
3005 g(gen_frame_store(ctx, OP_SIZE_INT, slot_r, 0, target));
3006 return true;
3007 #endif
3008 #if defined(ARCH_IA64)
3009 gen_insn(INSN_FP_TO_INT64, op_size, 0, 0);
3010 gen_one(FR_SCRATCH_1);
3011 gen_one(reg1);
3013 g(gen_mov(ctx, OP_SIZE_NATIVE, target, FR_SCRATCH_1));
3015 if (OP_SIZE_INT == OP_SIZE_4) {
3016 g(gen_extend(ctx, OP_SIZE_4, sign_x, R_SCRATCH_2, target));
3017 g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, target, R_SCRATCH_2, COND_NE, label_ovf));
3018 } else {
3019 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, target, sign_bit(uint64_t), COND_E, label_ovf));
3020 }
3022 g(gen_frame_store(ctx, OP_SIZE_INT, slot_r, 0, target));
3023 return true;
3024 #endif
3025 #if defined(ARCH_PARISC) || defined(ARCH_POWER) || defined(ARCH_SPARC)
3026 #if defined(ARCH_POWER)
3027 if (!cpu_test_feature(CPU_FEATURE_ppc))
3028 goto do_upcall;
3029 if (OP_SIZE_INT == OP_SIZE_4)
3030 goto do_upcall;
3031 #endif
3032 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_TO_INT32 : INSN_FP_TO_INT64, op_size, 0, 0);
3033 gen_one(FR_SCRATCH_1);
3034 gen_one(reg1);
3036 g(gen_frame_store_raw(ctx, OP_SIZE_INT, slot_r, 0, FR_SCRATCH_1));
3037 if (ctx->registers[slot_r] >= 0)
3038 g(unspill(ctx, slot_r));
3039 g(gen_frame_load(ctx, OP_SIZE_INT, garbage, slot_r, 0, target));
3041 g(gen_imm(ctx, sign_bit(uint_default_t) + 1, IMM_PURPOSE_ADD, OP_SIZE_INT));
3042 gen_insn(INSN_ALU, i_size(OP_SIZE_INT), ALU_ADD, ALU_WRITES_FLAGS(ALU_ADD, is_imm()));
3043 gen_one(R_SCRATCH_2);
3044 gen_one(target);
3045 gen_imm_offset();
3047 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, OP_SIZE_INT, R_SCRATCH_2, 1, COND_BE, label_ovf));
3049 return true;
3050 #endif
3051 #if defined(ARCH_ALPHA)
3052 gen_insn(INSN_FP_TO_INT64_TRAP, op_size, 0, 0);
3053 gen_one(FR_SCRATCH_2);
3054 gen_one(reg1);
3055 gen_four(label_ovf);
3057 if (OP_SIZE_INT == OP_SIZE_4) {
3058 gen_insn(INSN_FP_INT64_TO_INT32_TRAP, 0, 0, 0);
3059 gen_one(FR_SCRATCH_3);
3060 gen_one(FR_SCRATCH_2);
3061 gen_four(label_ovf);
3062 g(gen_frame_store_raw(ctx, OP_SIZE_INT, slot_r, 0, FR_SCRATCH_3));
3063 } else {
3064 g(gen_frame_store_raw(ctx, OP_SIZE_INT, slot_r, 0, FR_SCRATCH_2));
3065 }
3066 if (ctx->registers[slot_r] >= 0)
3067 g(unspill(ctx, slot_r));
3068 return true;
3069 #endif
3070 #if defined(ARCH_S390)
3071 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_TO_INT32 : INSN_FP_TO_INT64, op_size, 0, 1);
3072 gen_one(target);
3073 gen_one(reg1);
3075 gen_insn(INSN_JMP_COND, op_size, FP_COND_P, 0);
3076 gen_four(label_ovf);
3078 g(gen_frame_store(ctx, OP_SIZE_INT, slot_r, 0, target));
3079 return true;
3080 #endif
3081 #if defined(ARCH_RISCV64)
3082 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_TO_INT32 : INSN_FP_TO_INT64, op_size, 0, 0);
3083 gen_one(target);
3084 gen_one(reg1);
3086 g(gen_load_constant(ctx, R_SCRATCH_2, sign_bit(int_default_t)));
3088 g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, target, R_SCRATCH_2, COND_E, label_ovf));
3090 g(gen_imm(ctx, -1, IMM_PURPOSE_XOR, i_size(size)));
3091 gen_insn(INSN_ALU, OP_SIZE_NATIVE, ALU_XOR, ALU_WRITES_FLAGS(ALU_XOR, is_imm()));
3092 gen_one(R_SCRATCH_2);
3093 gen_one(R_SCRATCH_2);
3094 gen_imm_offset();
3096 g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, target, R_SCRATCH_2, COND_E, label_ovf));
3098 g(gen_frame_store(ctx, OP_SIZE_INT, slot_r, 0, target));
3099 return true;
3100 #endif
3101 }
3102 #ifdef SUPPORTED_FP_X87
3103 if ((SUPPORTED_FP_X87 >> real_type) & 1) {
3104 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
3106 if (likely(cpu_test_feature(CPU_FEATURE_sse3))) {
3107 g(gen_frame_store_x87(ctx, INSN_X87_FISTTP, OP_SIZE_INT, slot_r));
3108 } else {
3109 gen_insn(INSN_PUSH, OP_SIZE_NATIVE, 0, 0);
3110 gen_one(ARG_IMM);
3111 gen_eight(0x0f7f);
3113 gen_insn(INSN_X87_FLDCW, 0, 0, 0);
3114 gen_one(ARG_ADDRESS_1);
3115 gen_one(R_SP);
3116 gen_eight(0);
3118 g(gen_frame_store_x87(ctx, INSN_X87_FISTP, OP_SIZE_INT, slot_r));
3120 gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
3121 gen_one(ARG_ADDRESS_1);
3122 gen_one(R_SP);
3123 gen_eight(0);
3124 gen_one(ARG_IMM);
3125 gen_eight(0x037f);
3127 gen_insn(INSN_X87_FLDCW, 0, 0, 0);
3128 gen_one(ARG_ADDRESS_1);
3129 gen_one(R_SP);
3130 gen_eight(0);
3132 gen_insn(INSN_ALU, i_size(OP_SIZE_ADDRESS), ALU_ADD, 1);
3133 gen_one(R_SP);
3134 gen_one(R_SP);
3135 gen_one(ARG_IMM);
3136 gen_eight(1 << OP_SIZE_NATIVE);
3137 }
3138 if (ctx->registers[slot_r] >= 0)
3139 g(unspill(ctx, slot_r));
3140 g(gen_frame_load(ctx, OP_SIZE_INT, garbage, slot_r, 0, R_SCRATCH_1));
3142 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, OP_SIZE_INT, R_SCRATCH_1, sign_bit(int_default_t), COND_E, label_ovf));
3144 return true;
3145 }
3146 #endif
3147 #ifdef SUPPORTED_FP_HALF_CVT
3148 if ((SUPPORTED_FP_HALF_CVT >> real_type) & 1) {
3149 g(gen_frame_get(ctx, op_size, zero_x, slot_1, 0, FR_SCRATCH_1, ®1));
3150 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
3151 gen_one(FR_SCRATCH_1);
3152 gen_one(reg1);
3153 reg1 = FR_SCRATCH_1;
3154 real_type = 1;
3155 op_size = real_type_to_op_size(real_type);
3156 goto do_cvt_to_int;
3157 }
3158 #endif
3159 goto do_upcall;
3161 do_from_int:
3162 if ((SUPPORTED_FP >> real_type) & 1) {
3163 #if defined(ARCH_ALPHA) || defined(ARCH_ARM32) || defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS) || defined(ARCH_PARISC) || defined(ARCH_POWER) || defined(ARCH_SPARC)
3164 int int_op_size = OP_SIZE_INT;
3165 #if defined(ARCH_POWER)
3166 if (int_op_size == OP_SIZE_4)
3167 goto do_upcall;
3168 if (op_size == OP_SIZE_4 && !cpu_test_feature(CPU_FEATURE_v206))
3169 goto do_upcall;
3170 if (op_size == OP_SIZE_8 && !cpu_test_feature(CPU_FEATURE_ppc))
3171 goto do_upcall;
3172 #endif
3173 if (ctx->registers[slot_1] >= 0)
3174 g(spill(ctx, slot_1));
3175 g(gen_frame_load_raw(ctx, int_op_size, zero_x, slot_1, 0, FR_SCRATCH_1));
3176 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_2);
3177 #if defined(ARCH_ALPHA)
3178 if (OP_SIZE_INT == OP_SIZE_4) {
3179 gen_insn(INSN_MOVSX, OP_SIZE_4, 0, 0);
3180 gen_one(FR_SCRATCH_1);
3181 gen_one(FR_SCRATCH_1);
3183 int_op_size = OP_SIZE_8;
3184 }
3185 #endif
3186 gen_insn(int_op_size == OP_SIZE_4 ? INSN_FP_FROM_INT32 : INSN_FP_FROM_INT64, op_size, 0, 0);
3187 gen_one(target);
3188 gen_one(FR_SCRATCH_1);
3190 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
3191 return true;
3192 #elif defined(ARCH_IA64)
3193 g(gen_frame_get(ctx, OP_SIZE_INT, sign_x, slot_1, 0, R_SCRATCH_1, ®1));
3194 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_1);
3196 g(gen_mov(ctx, OP_SIZE_NATIVE, target, reg1));
3198 gen_insn(INSN_FP_FROM_INT64, op_size, 0, 0);
3199 gen_one(target);
3200 gen_one(target);
3202 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
3203 return true;
3204 #else
3205 g(gen_frame_get(ctx, OP_SIZE_INT, garbage, slot_1, 0, R_SCRATCH_1, ®1));
3206 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_1);
3208 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_FROM_INT32 : INSN_FP_FROM_INT64, op_size, 0, 0);
3209 gen_one(target);
3210 gen_one(reg1);
3212 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
3213 return true;
3214 #endif
3215 }
3216 #ifdef SUPPORTED_FP_X87
3217 if ((SUPPORTED_FP_X87 >> real_type) & 1) {
3218 if (ctx->registers[slot_1] >= 0)
3219 g(spill(ctx, slot_1));
3220 g(gen_frame_load_x87(ctx, INSN_X87_FILD, OP_SIZE_INT, 0, slot_1));
3221 g(gen_frame_store_x87(ctx, INSN_X87_FSTP, op_size, slot_r));
3222 return true;
3223 }
3224 #endif
3225 #ifdef SUPPORTED_FP_HALF_CVT
3226 if ((SUPPORTED_FP_HALF_CVT >> real_type) & 1) {
3227 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_1);
3228 #if defined(ARCH_ARM32)
3229 g(gen_frame_get(ctx, OP_SIZE_INT, zero_x, slot_1, 0, FR_SCRATCH_1, ®1));
3231 gen_insn(INSN_FP_FROM_INT32, OP_SIZE_4, 0, 0);
3232 gen_one(target);
3233 gen_one(reg1);
3234 #else
3235 g(gen_frame_get(ctx, OP_SIZE_INT, garbage, slot_1, 0, R_SCRATCH_1, ®1));
3236 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_FROM_INT32 : INSN_FP_FROM_INT64, OP_SIZE_4, 0, 0);
3237 gen_one(target);
3238 gen_one(reg1);
3239 #endif
3240 gen_insn(INSN_FP_CVT, OP_SIZE_4, op_size, 0);
3241 gen_one(target);
3242 gen_one(target);
3243 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
3244 return true;
3245 }
3246 #endif
3247 goto do_upcall;
3249 do_is_exception:
3250 if ((SUPPORTED_FP >> real_type) & 1) {
3251 g(gen_frame_get(ctx, op_size, zero_x, slot_1, 0, FR_SCRATCH_1, ®1));
3252 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
3253 #if defined(ARCH_ALPHA)
3254 gen_insn(INSN_FP_CMP_UNORDERED_DEST_REG, op_size, 0, 0);
3255 gen_one(FR_SCRATCH_2);
3256 gen_one(FR_SCRATCH_1);
3257 gen_one(reg1);
3259 if (!cpu_test_feature(CPU_FEATURE_fix)) {
3260 g(gen_frame_store_raw(ctx, OP_SIZE_4, slot_r, 0, FR_SCRATCH_2));
3261 g(gen_frame_load_raw(ctx, OP_SIZE_4, sign_x, slot_r, 0, target));
3262 } else {
3263 g(gen_mov(ctx, OP_SIZE_4, target, FR_SCRATCH_2));
3264 }
3266 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SHR, target, target, 30, 0));
3268 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
3270 return true;
3271 #elif defined(ARCH_IA64)
3272 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_P, 0);
3273 gen_one(R_CMP_RESULT);
3274 gen_one(reg1);
3275 gen_one(reg1);
3277 g(gen_mov(ctx, OP_SIZE_NATIVE, target, R_CMP_RESULT));
3279 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
3280 #elif defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS) || defined(ARCH_PARISC)
3281 gen_insn(INSN_FP_CMP_COND, op_size, FP_COND_P, 1);
3282 gen_one(reg1);
3283 gen_one(reg1);
3285 gen_insn(INSN_FP_TEST_REG, OP_SIZE_NATIVE, FP_COND_P, 0);
3286 gen_one(target);
3288 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
3289 #elif defined(ARCH_RISCV64)
3290 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_E, 0);
3291 gen_one(target);
3292 gen_one(reg1);
3293 gen_one(reg1);
3295 g(gen_imm(ctx, 1, IMM_PURPOSE_XOR, OP_SIZE_NATIVE));
3296 gen_insn(INSN_ALU, OP_SIZE_NATIVE, ALU_XOR, ALU_WRITES_FLAGS(ALU_XOR, is_imm()));
3297 gen_one(target);
3298 gen_one(target);
3299 gen_imm_offset();
3301 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
3302 #else
3303 gen_insn(INSN_FP_CMP, op_size, 0, 1);
3304 gen_one(reg1);
3305 gen_one(reg1);
3306 #if defined(ARCH_ARM32)
3307 gen_insn(INSN_FP_TO_INT_FLAGS, 0, 0, 1);
3308 #endif
3309 g(gen_frame_set_cond(ctx, op_size, false, FP_COND_P, slot_r));
3310 #endif
3311 return true;
3312 }
3313 #ifdef SUPPORTED_FP_X87
3314 if ((SUPPORTED_FP_X87 >> real_type) & 1) {
3315 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
3316 if (likely(cpu_test_feature(CPU_FEATURE_cmov))) {
3317 gen_insn(INSN_X87_FCOMIP, op_size, 0, 0);
3318 gen_one(R_ST0);
3320 g(gen_frame_set_cond(ctx, op_size, false, COND_P, slot_r));
3321 return true;
3322 }
3324 gen_insn(INSN_X87_FCOMP, op_size, 0, 0);
3325 gen_one(R_ST0);
3327 gen_insn(INSN_X87_FNSTSW, 0, 0, 0);
3328 gen_one(R_AX);
3329 gen_one(R_AX);
3331 gen_insn(INSN_TEST, OP_SIZE_2, 0, 1);
3332 gen_one(R_AX);
3333 gen_one(ARG_IMM);
3334 gen_eight(0x0400);
3336 g(gen_frame_set_cond(ctx, op_size, false, COND_NE, slot_r));
3338 return true;
3339 }
3340 #endif
3341 #ifdef SUPPORTED_FP_HALF_CVT
3342 if ((SUPPORTED_FP_HALF_CVT >> real_type) & 1) {
3343 g(gen_frame_get(ctx, op_size, zero_x, slot_1, 0, FR_SCRATCH_1, ®1));
3344 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
3345 gen_one(FR_SCRATCH_1);
3346 gen_one(reg1);
3347 gen_insn(INSN_FP_CMP, OP_SIZE_4, 0, 1);
3348 gen_one(FR_SCRATCH_1);
3349 gen_one(FR_SCRATCH_1);
3350 #if defined(ARCH_ARM32)
3351 gen_insn(INSN_FP_TO_INT_FLAGS, 0, 0, 1);
3352 #endif
3353 g(gen_frame_set_cond(ctx, op_size, false, FP_COND_P, slot_r));
3354 return true;
3355 }
3356 #endif
3358 do_upcall:
3359 g(gen_alu_typed_upcall(ctx, upc, real_type, slot_1, NO_FRAME_T, slot_r, label_ovf));
3360 return true;
3361 }
3363 static bool attr_w gen_is_exception(struct codegen_context *ctx, frame_t slot_1, frame_t slot_r)
3364 {
3365 uint32_t no_ex_label, escape_label;
3366 const struct type *type = get_type_of_local(ctx, slot_1);
3368 no_ex_label = alloc_label(ctx);
3369 if (unlikely(!no_ex_label))
3370 return false;
3371 escape_label = alloc_escape_label(ctx);
3372 if (unlikely(!escape_label))
3373 return false;
3375 if (TYPE_IS_FLAT(type))
3376 g(gen_test_1_jz_cached(ctx, slot_1, no_ex_label));
3378 g(gen_frame_load(ctx, OP_SIZE_SLOT, zero_x, slot_1, 0, R_SCRATCH_1));
3379 g(gen_ptr_is_thunk(ctx, R_SCRATCH_1, true, escape_label));
3380 g(gen_barrier(ctx));
3382 if (!TYPE_IS_FLAT(type)) {
3383 g(gen_compare_da_tag(ctx, R_SCRATCH_1, DATA_TAG_flat, COND_E, escape_label, R_SCRATCH_1));
3384 }
3386 gen_label(no_ex_label);
3387 g(gen_frame_clear(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r));
3389 flag_set(ctx, slot_r, false);
3391 return true;
3392 }
3394 static bool attr_w gen_system_property(struct codegen_context *ctx, frame_t slot_1, frame_t slot_r)
3395 {
3396 uint32_t escape_label;
3398 escape_label = alloc_escape_label(ctx);
3399 if (unlikely(!escape_label))
3400 return false;
3402 g(gen_test_1_cached(ctx, slot_1, escape_label));
3404 g(gen_upcall_start(ctx, 1));
3406 g(gen_frame_load(ctx, OP_SIZE_INT, garbage, slot_1, 0, R_ARG0));
3407 g(gen_upcall_argument(ctx, 0));
3409 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, ipret_system_property), 1));
3411 g(gen_frame_store(ctx, OP_SIZE_INT, slot_r, 0, R_RET0));
3413 flag_set(ctx, slot_1, false);
3414 flag_set(ctx, slot_r, false);
3416 return true;
3417 }
3419 static bool attr_w gen_alu_jmp(struct codegen_context *ctx, unsigned mode, unsigned op_size, unsigned op, frame_t slot_1, frame_t slot_2, int32_t jmp_offset, bool *failed)
3420 {
3421 unsigned alu;
3422 enum extend ex;
3423 unsigned reg1;
3424 unsigned attr_unused reg2;
3425 #if 0
3426 *failed = true; return true;
3427 #endif
3428 switch (mode) {
3429 case MODE_FIXED: switch (op) {
3430 case OPCODE_FIXED_OP_equal: alu = COND_E; goto do_compare;
3431 case OPCODE_FIXED_OP_not_equal: alu = COND_NE; goto do_compare;
3432 case OPCODE_FIXED_OP_less: alu = COND_L; goto do_compare;
3433 case OPCODE_FIXED_OP_less_equal: alu = COND_LE; goto do_compare;
3434 case OPCODE_FIXED_OP_uless: alu = COND_B; goto do_compare;
3435 case OPCODE_FIXED_OP_uless_equal: alu = COND_BE; goto do_compare;
3436 case OPCODE_FIXED_OP_bt: *failed = true; return true;
3437 default: internal(file_line, "gen_alu_jmp: unsupported fixed operation %u", op);
3438 }
3439 case MODE_INT: switch (op) {
3440 case OPCODE_INT_OP_equal: alu = COND_E; goto do_compare;
3441 case OPCODE_INT_OP_not_equal: alu = COND_NE; goto do_compare;
3442 case OPCODE_INT_OP_less: alu = COND_L; goto do_compare;
3443 case OPCODE_INT_OP_less_equal: alu = COND_LE; goto do_compare;
3444 case OPCODE_INT_OP_bt: *failed = true; return true;
3445 default: internal(file_line, "gen_alu_jmp: unsupported int operation %u", op);
3446 }
3447 case MODE_BOOL: switch (op) {
3448 case OPCODE_BOOL_OP_and: alu = ALU_AND; mode = MODE_FIXED; goto do_alu;
3449 case OPCODE_BOOL_OP_or: alu = ALU_OR; mode = MODE_FIXED; goto do_alu;
3450 case OPCODE_BOOL_OP_equal: alu = COND_E; mode = MODE_FIXED; goto do_compare;
3451 case OPCODE_BOOL_OP_not_equal: alu = COND_NE; mode = MODE_FIXED; goto do_compare;
3452 case OPCODE_BOOL_OP_less: alu = COND_L; mode = MODE_FIXED; goto do_compare;
3453 case OPCODE_BOOL_OP_less_equal: alu = COND_LE; mode = MODE_FIXED; goto do_compare;
3454 default: internal(file_line, "gen_alu_jmp: unsupported bool operation %u", op);
3455 }
3456 }
3457 internal(file_line, "gen_alu_jmp: unsupported mode %u", mode);
3458 do_compare: {
3459 bool attr_unused logical;
3460 if (unlikely(op_size > OP_SIZE_NATIVE)) {
3461 *failed = true;
3462 return true;
3463 }
3464 if (ctx->registers[slot_2] >= 0 && ctx->registers[slot_1] < 0) {
3465 frame_t s = slot_1;
3466 slot_1 = slot_2;
3467 slot_2 = s;
3468 switch (alu) {
3469 case COND_L: alu = COND_G; break;
3470 case COND_LE: alu = COND_GE; break;
3471 case COND_B: alu = COND_A; break;
3472 case COND_BE: alu = COND_AE; break;
3473 }
3474 }
3475 ex = op_size == i_size_cmp(op_size) + (unsigned)zero ? garbage : alu == COND_L || alu == COND_LE || alu == COND_G || alu == COND_GE || ARCH_PREFERS_SX(op_size) ? sign_x : zero_x;
3476 g(gen_frame_get(ctx, op_size, ex, slot_1, 0, R_SCRATCH_1, ®1));
3477 #if ARCH_HAS_FLAGS
3478 logical = COND_IS_LOGICAL(alu ^ 1);
3479 g(gen_frame_load_cmp(ctx, op_size, logical, ex, false, slot_2, 0, reg1));
3480 g(gen_jump(ctx, jmp_offset, op_size, alu ^ 1, -1U));
3481 #else
3482 g(gen_frame_get(ctx, op_size, ex, slot_2, 0, R_SCRATCH_2, ®2));
3483 g(gen_cmp_dest_reg(ctx, op_size, reg1, reg2, R_CMP_RESULT, 0, alu));
3484 g(gen_jump(ctx, jmp_offset, OP_SIZE_NATIVE, COND_E, R_CMP_RESULT));
3485 #endif
3486 return true;
3487 }
3488 do_alu: {
3489 if (ctx->registers[slot_2] >= 0 && ctx->registers[slot_1] < 0) {
3490 frame_t s = slot_1;
3491 slot_1 = slot_2;
3492 slot_2 = s;
3493 }
3494 ex = op_size == i_size(op_size) + (unsigned)zero ? garbage : ARCH_PREFERS_SX(op_size) ? sign_x : zero_x;
3495 g(gen_frame_get(ctx, op_size, ex, slot_1, 0, R_SCRATCH_1, ®1));
3496 #if defined(ARCH_X86)
3497 if (alu == ALU_AND && ctx->registers[slot_2] < 0) {
3498 g(gen_address(ctx, R_FRAME, (size_t)slot_2 * slot_size, IMM_PURPOSE_LDR_OFFSET, op_size));
3499 gen_insn(INSN_TEST, i_size(op_size), 0, 1);
3500 gen_one(reg1);
3501 gen_address_offset();
3502 g(gen_jump(ctx, jmp_offset, op_size, COND_E, -1U));
3503 return true;
3504 }
3505 #endif
3506 g(gen_frame_get(ctx, op_size, ex, slot_2, 0, R_SCRATCH_2, ®2));
3507 #if ARCH_HAS_FLAGS && !defined(ARCH_S390)
3508 if (alu == ALU_AND) {
3509 gen_insn(INSN_TEST, i_size(op_size), 0, 1);
3510 gen_one(reg1);
3511 gen_one(reg2);
3512 g(gen_jump(ctx, jmp_offset, op_size, COND_E, -1U));
3513 return true;
3514 }
3515 #endif
3516 #if defined(ARCH_ARM64)
3517 if (alu == ALU_OR)
3518 goto skip_flags;
3519 #endif
3520 #if ARCH_HAS_FLAGS
3521 g(gen_3address_alu(ctx, i_size(op_size), alu, R_SCRATCH_1, reg1, reg2, 1));
3522 g(gen_jump(ctx, jmp_offset, i_size(op_size), COND_E, -1U));
3523 return true;
3524 #endif
3525 goto skip_flags;
3526 skip_flags:
3527 g(gen_3address_alu(ctx, i_size(op_size), alu, R_SCRATCH_1, reg1, reg2, 0));
3528 g(gen_jump(ctx, jmp_offset, i_size(op_size), COND_E, R_SCRATCH_1));
3529 return true;
3530 }
3531 }
3533 static bool attr_w gen_fp_alu_jmp(struct codegen_context *ctx, unsigned real_type, unsigned op, uint32_t label_ovf, frame_t slot_1, frame_t slot_2, int32_t jmp_offset, bool *failed)
3534 {
3535 unsigned attr_unused fp_alu;
3536 unsigned attr_unused op_size = real_type_to_op_size(real_type);
3537 unsigned reg1, reg2;
3538 unsigned attr_unused target;
3539 switch (op) {
3540 case OPCODE_REAL_OP_equal:
3541 case OPCODE_REAL_OP_equal_alt1:
3542 case OPCODE_REAL_OP_equal_alt2: fp_alu = FP_COND_E; goto do_cmp;
3543 case OPCODE_REAL_OP_not_equal:
3544 case OPCODE_REAL_OP_not_equal_alt1:
3545 case OPCODE_REAL_OP_not_equal_alt2: fp_alu = FP_COND_NE; goto do_cmp;
3546 case OPCODE_REAL_OP_less:
3547 case OPCODE_REAL_OP_less_alt1:
3548 case OPCODE_REAL_OP_less_alt2: fp_alu = FP_COND_B; goto do_cmp;
3549 case OPCODE_REAL_OP_less_equal:
3550 case OPCODE_REAL_OP_less_equal_alt1:
3551 case OPCODE_REAL_OP_less_equal_alt2: fp_alu = FP_COND_BE; goto do_cmp;
3552 default: internal(file_line, "gen_fp_alu_jmp: unsupported operation %u", op);
3553 }
3555 do_cmp:
3556 if ((SUPPORTED_FP >> real_type) & 1
3557 #if defined(ARCH_ALPHA)
3558 && ARCH_SUPPORTS_TRAPS && cpu_test_feature(CPU_FEATURE_fix)
3559 #endif
3560 ) {
3561 g(gen_frame_get(ctx, op_size, zero_x, slot_1, 0, FR_SCRATCH_1, ®1));
3562 g(gen_frame_get(ctx, op_size, zero_x, slot_2, 0, FR_SCRATCH_2, ®2));
3563 target = R_SCRATCH_1;
3564 #if defined(ARCH_ALPHA)
3565 gen_insn(INSN_FP_CMP_DEST_REG_TRAP, op_size, fp_alu == FP_COND_NE ? FP_COND_E : fp_alu, 0);
3566 gen_one(FR_SCRATCH_3);
3567 gen_one(reg1);
3568 gen_one(reg2);
3569 gen_four(label_ovf);
3571 g(gen_mov(ctx, OP_SIZE_4, target, FR_SCRATCH_3));
3573 if (fp_alu == FP_COND_NE) {
3574 g(gen_jump(ctx, jmp_offset, OP_SIZE_NATIVE, COND_NE, target));
3575 } else {
3576 g(gen_jump(ctx, jmp_offset, OP_SIZE_NATIVE, COND_E, target));
3577 }
3579 return true;
3580 #elif defined(ARCH_IA64)
3581 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_P, 0);
3582 gen_one(R_CMP_RESULT);
3583 gen_one(reg1);
3584 gen_one(reg2);
3586 gen_insn(INSN_JMP_REG, OP_SIZE_NATIVE, COND_NE, 0);
3587 gen_one(R_CMP_RESULT);
3588 gen_four(label_ovf);
3590 gen_insn(INSN_FP_CMP_DEST_REG, op_size, fp_alu, 0);
3591 gen_one(R_CMP_RESULT);
3592 gen_one(reg1);
3593 gen_one(reg2);
3595 g(gen_jump(ctx, jmp_offset, OP_SIZE_NATIVE, COND_E, R_CMP_RESULT);
3597 return true;
3598 #elif defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS) || defined(ARCH_PARISC)
3599 gen_insn(INSN_FP_CMP_COND, op_size, FP_COND_P, 1);
3600 gen_one(reg1);
3601 gen_one(reg2);
3603 gen_insn(INSN_JMP_FP_TEST, 0, FP_COND_P, 0);
3604 gen_four(label_ovf);
3606 gen_insn(INSN_FP_CMP_COND, op_size, fp_alu, 1);
3607 gen_one(reg1);
3608 gen_one(reg2);
3610 gen_insn(INSN_FP_TEST_REG, OP_SIZE_NATIVE, fp_alu, 0);
3611 gen_one(target);
3613 g(gen_jump(ctx, jmp_offset, OP_SIZE_NATIVE, COND_E, target));
3615 return true;
3616 #elif defined(ARCH_RISCV64)
3617 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_E, 0);
3618 gen_one(R_SCRATCH_1);
3619 gen_one(reg1);
3620 gen_one(reg1);
3622 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_E, 0);
3623 gen_one(R_SCRATCH_2);
3624 gen_one(reg2);
3625 gen_one(reg2);
3627 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_AND, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_2, 0));
3629 g(gen_jmp_on_zero(ctx, OP_SIZE_NATIVE, R_SCRATCH_1, COND_E, label_ovf));
3631 gen_insn(INSN_FP_CMP_DEST_REG, op_size, fp_alu == FP_COND_NE ? FP_COND_E : fp_alu, 0);
3632 gen_one(target);
3633 gen_one(reg1);
3634 gen_one(reg2);
3636 if (fp_alu == FP_COND_NE) {
3637 g(gen_jump(ctx, jmp_offset, OP_SIZE_NATIVE, COND_NE, target));
3638 } else {
3639 g(gen_jump(ctx, jmp_offset, OP_SIZE_NATIVE, COND_E, target));
3640 }
3641 return true;
3642 #else
3643 gen_insn(INSN_FP_CMP, op_size, 0, 1);
3644 gen_one(reg1);
3645 gen_one(reg2);
3646 #if defined(ARCH_ARM32)
3647 gen_insn(INSN_FP_TO_INT_FLAGS, 0, 0, 1);
3648 #endif
3649 gen_insn(INSN_JMP_COND, op_size, FP_COND_P, 0);
3650 gen_four(label_ovf);
3651 g(gen_jump(ctx, jmp_offset, op_size, fp_alu ^ 1, -1U));
3652 return true;
3653 #endif
3654 }
3655 *failed = true;
3656 return true;
3657 }