| blob | 2877f31d177097ebc075390d2ed3977fdc79c455 |
1 /*
2 * Copyright (C) 2024 Mikulas Patocka
3 *
4 * This file is part of Ajla.
5 *
6 * Ajla is free software: you can redistribute it and/or modify it under the
7 * terms of the GNU General Public License as published by the Free Software
8 * Foundation, either version 3 of the License, or (at your option) any later
9 * version.
10 *
11 * Ajla is distributed in the hope that it will be useful, but WITHOUT ANY
12 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
13 * A PARTICULAR PURPOSE. See the GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License along with
16 * Ajla. If not, see <https://www.gnu.org/licenses/>.
17 */
19 static bool attr_w gen_jump(struct codegen_context *ctx, int32_t jmp_offset, unsigned op_size, unsigned cond, unsigned reg1, unsigned reg2);
21 static bool attr_w gen_alu_upcall(struct codegen_context *ctx, size_t upcall, unsigned op_size, frame_t slot_1, frame_t slot_2, frame_t slot_r, uint32_t label_ovf)
22 {
23 if (slot_is_register(ctx, slot_1))
24 g(spill(ctx, slot_1));
25 if (slot_2 != NO_FRAME_T && slot_is_register(ctx, slot_2))
26 g(spill(ctx, slot_2));
27 g(gen_upcall_start(ctx, frame_t_is_const(slot_2) ? 4 : slot_2 != NO_FRAME_T ? 3 : 2));
28 g(gen_frame_address(ctx, slot_1, 0, R_ARG0));
29 g(gen_upcall_argument(ctx, 0));
30 if (frame_t_is_const(slot_2)) {
31 size_t x_offs;
32 g(gen_load_constant(ctx, R_ARG1, frame_t_get_const(slot_2)));
33 g(gen_upcall_argument(ctx, 1));
34 g(gen_frame_address(ctx, slot_r, 0, R_ARG2));
35 g(gen_upcall_argument(ctx, 2));
36 g(gen_get_upcall_pointer(ctx, upcall, R_ARG3));
37 g(gen_upcall_argument(ctx, 3));
38 x_offs = offsetof(struct cg_upcall_vector_s, INT_binary_const_int8_t) + op_size * sizeof(void (*)(void));
39 g(gen_upcall(ctx, x_offs, 4));
40 } else if (slot_2 != NO_FRAME_T) {
41 g(gen_frame_address(ctx, slot_2, 0, R_ARG1));
42 g(gen_upcall_argument(ctx, 1));
43 g(gen_frame_address(ctx, slot_r, 0, R_ARG2));
44 g(gen_upcall_argument(ctx, 2));
45 g(gen_upcall(ctx, upcall, 3));
46 } else {
47 g(gen_frame_address(ctx, slot_r, 0, R_ARG1));
48 g(gen_upcall_argument(ctx, 1));
49 g(gen_upcall(ctx, upcall, 2));
50 }
51 if (slot_is_register(ctx, slot_r))
52 g(unspill(ctx, slot_r));
53 if (label_ovf)
54 g(gen_jmp_on_zero(ctx, OP_SIZE_1, R_RET0, COND_E, label_ovf));
55 return true;
56 }
58 static bool attr_w gen_alu_typed_upcall(struct codegen_context *ctx, size_t upcall, unsigned op_size, frame_t slot_1, frame_t slot_2, frame_t slot_r, uint32_t label_ovf)
59 {
60 upcall += op_size * sizeof(void (*)(void));
61 return gen_alu_upcall(ctx, upcall, op_size, slot_1, slot_2, slot_r, label_ovf);
62 }
64 #define MODE_FIXED 0
65 #define MODE_INT 1
66 #define MODE_REAL 2
67 #define MODE_BOOL 3
68 #define MODE_ARRAY_LEN_GT 4
70 static bool attr_w gen_alu(struct codegen_context *ctx, unsigned mode, unsigned op_size, unsigned op, uint32_t label_ovf, frame_t slot_1, frame_t slot_2, frame_t slot_r)
71 {
72 unsigned alu;
73 bool sgn, mod;
74 unsigned reg1, reg2, reg3, target;
75 switch (mode) {
76 case MODE_FIXED: switch (op) {
77 case OPCODE_FIXED_OP_add: alu = ALU_ADD; goto do_alu;
78 case OPCODE_FIXED_OP_subtract: alu = ALU_SUB; goto do_alu;
79 case OPCODE_FIXED_OP_multiply: goto do_multiply;
80 case OPCODE_FIXED_OP_divide:
81 case OPCODE_FIXED_OP_divide_alt1: sgn = true; mod = false; goto do_divide;
82 case OPCODE_FIXED_OP_udivide:
83 case OPCODE_FIXED_OP_udivide_alt1: sgn = false; mod = false; goto do_divide;
84 case OPCODE_FIXED_OP_modulo:
85 case OPCODE_FIXED_OP_modulo_alt1: sgn = true; mod = true; goto do_divide;
86 case OPCODE_FIXED_OP_umodulo:
87 case OPCODE_FIXED_OP_umodulo_alt1: sgn = false; mod = true; goto do_divide;
88 case OPCODE_FIXED_OP_power: return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_binary_power_int8_t), op_size, slot_1, slot_2, slot_r, 0);
89 case OPCODE_FIXED_OP_and: alu = ALU_AND; goto do_alu;
90 case OPCODE_FIXED_OP_or: alu = ALU_OR; goto do_alu;
91 case OPCODE_FIXED_OP_xor: alu = ALU_XOR; goto do_alu;
92 case OPCODE_FIXED_OP_shl: alu = ROT_SHL; goto do_shift;
93 case OPCODE_FIXED_OP_shr: alu = ROT_SAR; goto do_shift;
94 case OPCODE_FIXED_OP_ushr: alu = ROT_SHR; goto do_shift;
95 case OPCODE_FIXED_OP_rol: alu = ROT_ROL; goto do_shift;
96 case OPCODE_FIXED_OP_ror: alu = ROT_ROR; goto do_shift;
97 case OPCODE_FIXED_OP_bts: alu = BTX_BTS; goto do_bt;
98 case OPCODE_FIXED_OP_btr: alu = BTX_BTR; goto do_bt;
99 case OPCODE_FIXED_OP_btc: alu = BTX_BTC; goto do_bt;
100 case OPCODE_FIXED_OP_equal: alu = COND_E; goto do_compare;
101 case OPCODE_FIXED_OP_not_equal: alu = COND_NE; goto do_compare;
102 case OPCODE_FIXED_OP_less: alu = COND_L; goto do_compare;
103 case OPCODE_FIXED_OP_less_equal: alu = COND_LE; goto do_compare;
104 case OPCODE_FIXED_OP_greater: alu = COND_G; goto do_compare;
105 case OPCODE_FIXED_OP_greater_equal: alu = COND_GE; goto do_compare;
106 case OPCODE_FIXED_OP_uless: alu = COND_B; goto do_compare;
107 case OPCODE_FIXED_OP_uless_equal: alu = COND_BE; goto do_compare;
108 case OPCODE_FIXED_OP_ugreater: alu = COND_A; goto do_compare;
109 case OPCODE_FIXED_OP_ugreater_equal: alu = COND_AE; goto do_compare;
110 case OPCODE_FIXED_OP_bt: alu = BTX_BT; goto do_bt;
111 default: internal(file_line, "gen_alu: unsupported fixed operation %u", op);
112 }
113 case MODE_INT: switch (op) {
114 case OPCODE_INT_OP_add: alu = ALU_ADD; goto do_alu;
115 case OPCODE_INT_OP_subtract: alu = ALU_SUB; goto do_alu;
116 case OPCODE_INT_OP_multiply: goto do_multiply;
117 case OPCODE_INT_OP_divide:
118 case OPCODE_INT_OP_divide_alt1: sgn = true; mod = false; goto do_divide;
119 case OPCODE_INT_OP_modulo:
120 case OPCODE_INT_OP_modulo_alt1: sgn = true; mod = true; goto do_divide;
121 case OPCODE_INT_OP_power: return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, INT_binary_power_int8_t), op_size, slot_1, slot_2, slot_r, label_ovf);
122 case OPCODE_INT_OP_and: alu = ALU_AND; mode = MODE_FIXED; goto do_alu;
123 case OPCODE_INT_OP_or: alu = ALU_OR; mode = MODE_FIXED; goto do_alu;
124 case OPCODE_INT_OP_xor: alu = ALU_XOR; mode = MODE_FIXED; goto do_alu;
125 case OPCODE_INT_OP_shl: alu = ROT_SHL; goto do_shift;
126 case OPCODE_INT_OP_shr: alu = ROT_SAR; goto do_shift;
127 case OPCODE_INT_OP_bts: alu = BTX_BTS; goto do_bt;
128 case OPCODE_INT_OP_btr: alu = BTX_BTR; goto do_bt;
129 case OPCODE_INT_OP_btc: alu = BTX_BTC; goto do_bt;
130 case OPCODE_INT_OP_equal: alu = COND_E; goto do_compare;
131 case OPCODE_INT_OP_not_equal: alu = COND_NE; goto do_compare;
132 case OPCODE_INT_OP_less: alu = COND_L; goto do_compare;
133 case OPCODE_INT_OP_less_equal: alu = COND_LE; goto do_compare;
134 case OPCODE_INT_OP_greater: alu = COND_G; goto do_compare;
135 case OPCODE_INT_OP_greater_equal: alu = COND_GE; goto do_compare;
136 case OPCODE_INT_OP_bt: alu = BTX_BT; goto do_bt;
137 default: internal(file_line, "gen_alu: unsupported int operation %u", op);
138 }
139 case MODE_BOOL: switch (op) {
140 case OPCODE_BOOL_OP_and: alu = ALU_AND; mode = MODE_FIXED; goto do_alu;
141 case OPCODE_BOOL_OP_or: alu = ALU_OR; mode = MODE_FIXED; goto do_alu;
142 case OPCODE_BOOL_OP_equal: alu = COND_E; goto do_compare;
143 case OPCODE_BOOL_OP_not_equal: alu = ALU_XOR; mode = MODE_FIXED; goto do_alu;
144 case OPCODE_BOOL_OP_less: alu = COND_L; goto do_compare;
145 case OPCODE_BOOL_OP_less_equal: alu = COND_LE; goto do_compare;
146 case OPCODE_BOOL_OP_greater: alu = COND_G; goto do_compare;
147 case OPCODE_BOOL_OP_greater_equal: alu = COND_GE; goto do_compare;
148 default: internal(file_line, "gen_alu: unsupported bool operation %u", op);
149 }
150 }
151 internal(file_line, "gen_alu: unsupported mode %u", mode);
153 /*******
154 * ALU *
155 *******/
156 do_alu: {
157 size_t attr_unused offset;
158 uint8_t attr_unused long_imm;
159 unsigned first_flags;
160 unsigned second_flags;
161 unsigned second_alu;
162 unsigned attr_unused op_size_flags;
163 if (unlikely(op_size > OP_SIZE_NATIVE)) {
164 #if !defined(ARCH_X86) && !defined(ARCH_ARM) && !defined(ARCH_PARISC) && !defined(ARCH_POWER) && !defined(ARCH_SPARC32)
165 if (mode == MODE_FIXED) {
166 if (alu == ALU_ADD) {
167 g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_add_,TYPE_INT_MAX)), op_size, slot_1, slot_2, slot_r, 0));
168 return true;
169 } else if (alu == ALU_SUB) {
170 g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_subtract_,TYPE_INT_MAX)), op_size, slot_1, slot_2, slot_r, 0));
171 return true;
172 }
173 } else if (mode == MODE_INT) {
174 if (alu == ALU_ADD) {
175 g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(INT_binary_add_,TYPE_INT_MAX)), op_size, slot_1, slot_2, slot_r, label_ovf));
176 return true;
177 } else if (alu == ALU_SUB) {
178 g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(INT_binary_subtract_,TYPE_INT_MAX)), op_size, slot_1, slot_2, slot_r, label_ovf));
179 return true;
180 }
181 }
182 #endif
183 first_flags = alu == ALU_ADD || alu == ALU_SUB ? 2 : 0;
184 second_flags = mode == MODE_INT ? 1 : 0;
185 second_alu = alu == ALU_ADD ? ALU_ADC : alu == ALU_SUB ? ALU_SBB : alu;
186 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
187 #if defined(ARCH_X86)
188 g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, alu, first_flags, slot_2, lo_word(OP_SIZE_NATIVE), true, R_SCRATCH_1));
189 g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, second_alu, second_flags, slot_2, hi_word(OP_SIZE_NATIVE), true, R_SCRATCH_2));
190 #else
191 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_2, 0, R_SCRATCH_3, R_SCRATCH_4));
192 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, alu, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_3, first_flags));
193 #if defined(ARCH_PARISC)
194 if (mode == MODE_INT) {
195 gen_insn(INSN_ALU_FLAGS_TRAP, OP_SIZE_NATIVE, second_alu, ALU_WRITES_FLAGS(second_alu, false));
196 gen_one(R_SCRATCH_2);
197 gen_one(R_SCRATCH_2);
198 gen_one(R_SCRATCH_4);
199 gen_four(label_ovf);
200 } else
201 #endif
202 {
203 gen_insn(first_flags ? INSN_ALU_FLAGS : INSN_ALU, OP_SIZE_NATIVE, second_alu, second_flags | ALU_WRITES_FLAGS(second_alu, false));
204 gen_one(R_SCRATCH_2);
205 gen_one(R_SCRATCH_2);
206 gen_one(R_SCRATCH_4);
207 }
208 #endif
209 #if !defined(ARCH_PARISC)
210 if (mode == MODE_INT) {
211 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_O, 0);
212 gen_four(label_ovf);
213 }
214 #endif
215 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_1, R_SCRATCH_2));
216 return true;
217 }
219 if ((ARCH_HAS_FLAGS || ARCH_SUPPORTS_TRAPS) && slot_2 == slot_r && slot_1 != slot_2 && alu_is_commutative(alu)) {
220 frame_t x = slot_1;
221 slot_1 = slot_2;
222 slot_2 = x;
223 }
224 if ((ARCH_HAS_FLAGS || ARCH_SUPPORTS_TRAPS) && slot_1 == slot_r && (slot_1 != slot_2 || mode != MODE_INT) && i_size_cmp(op_size) == op_size + zero
225 #if defined(ARCH_POWER)
226 && op_size == OP_SIZE_NATIVE
227 #endif
228 ) {
229 struct cg_exit *ce;
230 unsigned undo_alu = alu == ALU_ADD ? ALU_SUB : ALU_ADD;
231 if (slot_is_register(ctx, slot_1)) {
232 unsigned reg1 = ctx->registers[slot_1];
233 if (slot_is_register(ctx, slot_2) || frame_t_is_const(slot_2)) {
234 unsigned reg2 = frame_t_is_const(slot_2) ? 0xff /* avoid warning */ : ctx->registers[slot_2];
235 if (mode == MODE_INT && ARCH_SUPPORTS_TRAPS) {
236 if (frame_t_is_const(slot_2))
237 g(gen_imm(ctx, frame_t_get_const(slot_2), alu_trap_purpose(alu), i_size(op_size)));
238 gen_insn(INSN_ALU_TRAP, op_size, alu, ALU_WRITES_FLAGS(alu, frame_t_is_const(slot_2) && is_imm()));
239 gen_one(reg1);
240 gen_one(reg1);
241 if (frame_t_is_const(slot_2))
242 gen_imm_offset();
243 else
244 gen_one(reg2);
245 if (ARCH_TRAP_BEFORE) {
246 gen_four(label_ovf);
247 return true;
248 } else {
249 ce = alloc_undo_label(ctx);
250 if (unlikely(!ce))
251 return false;
252 gen_four(ce->undo_label);
253 goto do_undo_opcode;
254 }
255 }
256 if (frame_t_is_const(slot_2))
257 g(gen_3address_alu_imm(ctx, i_size(op_size), alu, reg1, reg1, frame_t_get_const(slot_2), mode == MODE_INT));
258 else
259 g(gen_3address_alu(ctx, i_size(op_size), alu, reg1, reg1, reg2, mode == MODE_INT));
260 if (mode == MODE_INT) {
261 size_t m;
262 ce = alloc_undo_label(ctx);
263 if (unlikely(!ce))
264 return false;
265 gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
266 gen_four(ce->undo_label);
267 do_undo_opcode:
268 ce->undo_opcode = INSN_ALU + ARCH_PARTIAL_ALU(op_size);
269 ce->undo_op_size = i_size(op_size);
270 ce->undo_aux = undo_alu;
271 ce->undo_writes_flags = ALU_WRITES_FLAGS(undo_alu, frame_t_is_const(slot_2) && is_imm());
272 m = mark_params(ctx);
273 gen_one(reg1);
274 gen_one(reg1);
275 if (frame_t_is_const(slot_2))
276 gen_imm_offset();
277 else
278 gen_one(reg2);
279 copy_params(ctx, ce, m);
280 }
281 return true;
282 }
283 #if defined(ARCH_S390) || defined(ARCH_X86)
284 else if (!frame_t_is_const(slot_2)) {
285 size_t m;
286 int64_t offset = (size_t)slot_2 * slot_size;
287 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDR_OFFSET, i_size(op_size)));
288 gen_insn(INSN_ALU + ARCH_PARTIAL_ALU(op_size), i_size(op_size), alu, 1);
289 gen_one(reg1);
290 gen_one(reg1);
291 gen_address_offset();
292 if (mode == MODE_INT) {
293 ce = alloc_undo_label(ctx);
294 if (unlikely(!ce))
295 return false;
296 ce->undo_opcode = INSN_ALU + ARCH_PARTIAL_ALU(op_size);
297 ce->undo_op_size = i_size(op_size);
298 ce->undo_aux = undo_alu;
299 ce->undo_writes_flags = ARCH_HAS_FLAGS;
300 m = mark_params(ctx);
301 gen_one(reg1);
302 gen_one(reg1);
303 gen_address_offset();
304 copy_params(ctx, ce, m);
305 gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
306 gen_four(ce->undo_label);
307 }
308 return true;
309 }
310 #endif
311 }
312 #if defined(ARCH_X86)
313 else {
314 unsigned reg2;
315 size_t m;
316 int64_t offset = (size_t)slot_1 * slot_size;
317 if (!frame_t_is_const(slot_2))
318 g(gen_frame_get(ctx, op_size, garbage, slot_2, R_SCRATCH_1, ®2));
319 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDR_OFFSET, i_size(op_size)));
320 if (frame_t_is_const(slot_2))
321 g(gen_imm(ctx, frame_t_get_const(slot_2), alu_purpose(alu), i_size(op_size)));
322 gen_insn(INSN_ALU + ARCH_PARTIAL_ALU(op_size), i_size(op_size), alu, 1);
323 gen_address_offset();
324 gen_address_offset();
325 if (frame_t_is_const(slot_2))
326 gen_imm_offset();
327 else
328 gen_one(reg2);
329 if (mode == MODE_INT) {
330 ce = alloc_undo_label(ctx);
331 if (unlikely(!ce))
332 return false;
333 ce->undo_opcode = INSN_ALU + ARCH_PARTIAL_ALU(op_size);
334 ce->undo_op_size = i_size(op_size);
335 ce->undo_aux = undo_alu;
336 ce->undo_writes_flags = ARCH_HAS_FLAGS;
337 m = mark_params(ctx);
338 gen_address_offset();
339 gen_address_offset();
340 if (frame_t_is_const(slot_2))
341 gen_imm_offset();
342 else
343 gen_one(reg2);
344 copy_params(ctx, ce, m);
345 gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
346 gen_four(ce->undo_label);
347 }
348 return true;
349 }
350 #endif
351 }
353 #if defined(ARCH_X86)
354 if (1)
355 #elif defined(ARCH_S390)
356 if (op_size >= OP_SIZE_4)
357 #elif ARCH_HAS_FLAGS && !defined(ARCH_POWER)
358 if (op_size == i_size(op_size) + (unsigned)zero && frame_t_is_const(slot_2))
359 #else
360 if (mode != MODE_INT && op_size == i_size(op_size) + (unsigned)zero && frame_t_is_const(slot_2))
361 #endif
362 {
363 if (mode == MODE_INT) {
364 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
365 } else {
366 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, slot_2, R_SCRATCH_1);
367 }
368 g(gen_frame_load(ctx, op_size, garbage, slot_1, 0, false, target));
369 g(gen_frame_load_op(ctx, op_size, garbage, alu, mode == MODE_INT, slot_2, 0, false, target));
370 goto check_ovf_store;
371 }
372 op_size_flags = !ARCH_HAS_FLAGS && !ARCH_SUPPORTS_TRAPS ? OP_SIZE_NATIVE : OP_SIZE_4;
373 #if defined(ARCH_POWER)
374 op_size_flags = OP_SIZE_NATIVE;
375 #endif
376 g(gen_frame_get(ctx, op_size, mode == MODE_INT && (op_size < op_size_flags || ARCH_SUPPORTS_TRAPS) ? sign_x : garbage, slot_1, R_SCRATCH_1, ®1));
377 g(gen_frame_get(ctx, op_size, mode == MODE_INT && (op_size < op_size_flags || ARCH_SUPPORTS_TRAPS) ? sign_x : garbage, slot_2, R_SCRATCH_2, ®2));
378 #if !ARCH_HAS_FLAGS
379 if (mode == MODE_INT && op_size >= OP_SIZE_4) {
380 if (ARCH_SUPPORTS_TRAPS) {
381 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
382 gen_insn(INSN_ALU_TRAP, op_size, alu, ALU_WRITES_FLAGS(alu, false));
383 gen_one(target);
384 gen_one(reg1);
385 gen_one(reg2);
386 gen_four(label_ovf);
387 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
388 return true;
389 }
390 if (op_size >= OP_SIZE_NATIVE) {
391 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_3);
392 g(gen_3address_alu(ctx, i_size(op_size), alu, target, reg1, reg2, 0));
393 #if defined(ARCH_IA64)
394 g(gen_3address_alu(ctx, i_size(op_size), ALU_XOR, R_SCRATCH_1, reg1, reg2, 0));
395 g(gen_3address_alu(ctx, i_size(op_size), ALU_XOR, R_SCRATCH_2, reg2, target, 0));
396 if (alu == ALU_ADD) {
397 g(gen_3address_alu(ctx, i_size(op_size), ALU_ANDN, R_SCRATCH_1, R_SCRATCH_2, R_SCRATCH_1, 0));
398 } else {
399 g(gen_3address_alu(ctx, i_size(op_size), ALU_ANDN, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_2, 0));
400 }
401 g(gen_cmp_test_jmp(ctx, INSN_TEST, i_size(op_size), R_SCRATCH_1, R_SCRATCH_1, COND_S, label_ovf));
402 #else
403 gen_insn(INSN_CMP_DEST_REG, i_size(op_size), COND_L, 0);
404 gen_one(R_SCRATCH_1);
405 if (alu == ALU_ADD) {
406 gen_one(target);
407 gen_one(reg1);
408 } else {
409 gen_one(reg1);
410 gen_one(target);
411 }
413 g(gen_imm(ctx, 0, IMM_PURPOSE_CMP, i_size(op_size)));
414 gen_insn(INSN_CMP_DEST_REG, i_size(op_size), COND_L, 0);
415 gen_one(R_SCRATCH_2);
416 gen_one(reg2);
417 gen_imm_offset();
419 g(gen_cmp_test_jmp(ctx, INSN_CMP, i_size(op_size), R_SCRATCH_1, R_SCRATCH_2, COND_NE, label_ovf));
420 #endif
421 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
422 return true;
423 }
424 }
425 #endif
426 if (mode == MODE_INT) {
427 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
428 } else if (!ARCH_IS_3ADDRESS(alu, mode == MODE_INT && op_size >= op_size_flags) && !alu_is_commutative(alu)) {
429 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, slot_2, R_SCRATCH_1);
430 } else {
431 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
432 }
433 g(gen_3address_alu(ctx, i_size(op_size), alu, target, reg1, reg2, mode == MODE_INT && op_size >= op_size_flags));
435 if (mode == MODE_INT && unlikely(op_size < op_size_flags)) {
436 g(gen_cmp_extended(ctx, op_size_flags, op_size, target, R_SCRATCH_2, label_ovf));
437 } else
438 check_ovf_store:
439 if (mode == MODE_INT) {
440 gen_insn(INSN_JMP_COND, op_size, COND_O, 0);
441 gen_four(label_ovf);
442 }
443 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
444 return true;
445 }
447 /************
448 * MULTIPLY *
449 ************/
450 do_multiply: {
451 size_t attr_unused offset;
452 uint8_t attr_unused long_imm;
453 if (unlikely(op_size > OP_SIZE_NATIVE) || unlikely(!ARCH_HAS_MUL)) {
454 if (mode == MODE_INT) {
455 g(gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, INT_binary_multiply_int8_t), op_size, slot_1, slot_2, slot_r, label_ovf));
456 return true;
457 }
458 #if defined(ARCH_X86)
459 g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_1, hi_word(OP_SIZE_NATIVE), true, R_CX));
460 g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_2, hi_word(OP_SIZE_NATIVE), true, R_AX));
461 g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, ALU_MUL, true, slot_2, lo_word(OP_SIZE_NATIVE), true, R_CX));
462 g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, ALU_MUL, true, slot_1, lo_word(OP_SIZE_NATIVE), true, R_AX));
463 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_ADD, R_CX, R_CX, R_AX, 0));
464 g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_2, lo_word(OP_SIZE_NATIVE), true, R_AX));
466 offset = (size_t)slot_1 * slot_size + lo_word(OP_SIZE_NATIVE);
467 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
468 gen_insn(INSN_MUL_L, OP_SIZE_NATIVE, 0, 1);
469 gen_one(R_AX);
470 gen_one(R_DX);
471 gen_one(R_AX);
472 gen_address_offset();
474 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_ADD, R_DX, R_DX, R_CX, 0));
476 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_AX, R_DX));
478 return true;
479 #elif defined(ARCH_ARM32)
480 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
481 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_2, 0, R_SCRATCH_3, R_SCRATCH_4));
483 g(gen_mov(ctx, OP_SIZE_NATIVE, R_SCRATCH_NA_1, R_SCRATCH_1));
485 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_MUL, R_SCRATCH_4, R_SCRATCH_1, R_SCRATCH_4, 0));
487 gen_insn(INSN_MADD, OP_SIZE_NATIVE, 0, 0);
488 gen_one(R_SCRATCH_2);
489 gen_one(R_SCRATCH_3);
490 gen_one(R_SCRATCH_2);
491 gen_one(R_SCRATCH_4);
493 gen_insn(INSN_MUL_L, OP_SIZE_NATIVE, 0, 0);
494 gen_one(R_SCRATCH_1);
495 gen_one(R_SCRATCH_4);
496 gen_one(R_SCRATCH_NA_1);
497 gen_one(R_SCRATCH_3);
499 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_ADD, R_SCRATCH_2, R_SCRATCH_2, R_SCRATCH_4, 0));
501 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_1, R_SCRATCH_2));
503 return true;
504 #elif defined(ARCH_ARM64)
505 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
506 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_2, 0, R_SCRATCH_3, R_SCRATCH_4));
508 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_UMULH, R_SCRATCH_NA_1, R_SCRATCH_1, R_SCRATCH_3, 0));
510 gen_insn(INSN_MADD, OP_SIZE_NATIVE, 0, 0);
511 gen_one(R_SCRATCH_NA_1);
512 gen_one(R_SCRATCH_2);
513 gen_one(R_SCRATCH_3);
514 gen_one(R_SCRATCH_NA_1);
516 gen_insn(INSN_MADD, OP_SIZE_NATIVE, 0, 0);
517 gen_one(R_SCRATCH_2);
518 gen_one(R_SCRATCH_1);
519 gen_one(R_SCRATCH_4);
520 gen_one(R_SCRATCH_NA_1);
522 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_MUL, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_3, 0));
524 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_1, R_SCRATCH_2));
526 return true;
527 #else
528 g(gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_binary_multiply_int8_t), op_size, slot_1, slot_2, slot_r, 0));
529 return true;
530 #endif
531 }
533 #if defined(ARCH_X86)
534 if (mode == MODE_INT) {
535 if (op_size != OP_SIZE_1 && slot_r == slot_1 && slot_is_register(ctx, slot_1)) {
536 struct cg_exit *ce;
537 target = ctx->registers[slot_1];
538 g(gen_mov(ctx, op_size, R_SCRATCH_1, target));
539 g(gen_frame_load_op(ctx, op_size, garbage, ALU_MUL, mode == MODE_INT, slot_2, 0, false, target));
540 ce = alloc_undo_label(ctx);
541 if (unlikely(!ce))
542 return false;
543 ce->undo_opcode = INSN_MOV;
544 ce->undo_op_size = op_size;
545 ce->undo_aux = 0;
546 ce->undo_writes_flags = 0;
547 ce->undo_parameters[0] = target;
548 ce->undo_parameters[1] = R_SCRATCH_1;
549 ce->undo_parameters_len = 2;
550 gen_insn(INSN_JMP_COND, op_size, COND_O, 0);
551 gen_four(ce->undo_label);
552 return true;
553 }
554 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
555 } else {
556 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, slot_2, R_SCRATCH_1);
557 }
558 if (op_size == OP_SIZE_1)
559 target = R_SCRATCH_1;
560 g(gen_frame_load(ctx, op_size, garbage, slot_1, 0, false, target));
561 if (op_size == OP_SIZE_1 && frame_t_is_const(slot_2)) {
562 g(gen_load_constant(ctx, R_SCRATCH_3, frame_t_get_const(slot_2)));
563 gen_insn(INSN_ALU + ARCH_PARTIAL_ALU(op_size), op_size, ALU_MUL, 1);
564 gen_one(target);
565 gen_one(target);
566 gen_one(R_SCRATCH_3);
567 } else {
568 g(gen_frame_load_op(ctx, op_size, garbage, ALU_MUL, mode == MODE_INT, slot_2, 0, false, target));
569 }
570 if (mode == MODE_INT) {
571 gen_insn(INSN_JMP_COND, op_size, COND_O, 0);
572 gen_four(label_ovf);
573 }
574 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
575 return true;
576 #endif
577 #if defined(ARCH_ALPHA)
578 if (mode == MODE_INT && op_size >= OP_SIZE_4 && ARCH_SUPPORTS_TRAPS) {
579 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
580 g(gen_frame_get(ctx, op_size, garbage, slot_1, R_SCRATCH_1, ®1));
581 g(gen_frame_get(ctx, op_size, garbage, slot_2, R_SCRATCH_2, ®2));
583 gen_insn(INSN_ALU_TRAP, op_size, ALU_MUL, ALU_WRITES_FLAGS(ALU_MUL, false));
584 gen_one(target);
585 gen_one(reg1);
586 gen_one(reg2);
587 gen_four(label_ovf);
588 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
590 return true;
591 }
592 #endif
593 #if defined(ARCH_ARM32)
594 if (mode == MODE_INT && op_size == OP_SIZE_4) {
595 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_3);
596 g(gen_frame_get(ctx, op_size, garbage, slot_1, R_SCRATCH_1, ®1));
597 g(gen_frame_get(ctx, op_size, garbage, slot_2, R_SCRATCH_2, ®2));
599 gen_insn(INSN_MUL_L, OP_SIZE_NATIVE, 0, 0);
600 gen_one(target);
601 gen_one(R_SCRATCH_4);
602 gen_one(reg1);
603 gen_one(reg2);
605 gen_insn(INSN_CMP, OP_SIZE_NATIVE, 0, 1);
606 gen_one(R_SCRATCH_4);
607 gen_one(ARG_SHIFTED_REGISTER);
608 gen_one(ARG_SHIFT_ASR | 0x1f);
609 gen_one(target);
611 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_NE, 0);
612 gen_four(label_ovf);
614 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
616 return true;
617 }
618 #endif
619 #if defined(ARCH_ARM64)
620 if (mode == MODE_INT && op_size == OP_SIZE_4) {
621 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
622 g(gen_frame_get(ctx, op_size, op_size < OP_SIZE_4 ? sign_x : garbage, slot_1, R_SCRATCH_1, ®1));
623 g(gen_frame_get(ctx, op_size, op_size < OP_SIZE_4 ? sign_x : garbage, slot_2, R_SCRATCH_2, ®2));
624 gen_insn(INSN_ALU, OP_SIZE_8, ALU_MUL, ALU_WRITES_FLAGS(ALU_MUL, false));
625 gen_one(target);
626 gen_one(ARG_EXTENDED_REGISTER);
627 gen_one(ARG_EXTEND_SXTW);
628 gen_one(reg1);
629 gen_one(ARG_EXTENDED_REGISTER);
630 gen_one(ARG_EXTEND_SXTW);
631 gen_one(reg2);
633 gen_insn(INSN_CMP, OP_SIZE_8, 0, 1);
634 gen_one(target);
635 gen_one(ARG_EXTENDED_REGISTER);
636 gen_one(ARG_EXTEND_SXTW);
637 gen_one(target);
639 gen_insn(INSN_JMP_COND, OP_SIZE_8, COND_NE, 0);
640 gen_four(label_ovf);
642 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
644 return true;
645 }
646 if (mode == MODE_INT && op_size == OP_SIZE_8) {
647 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
648 g(gen_frame_get(ctx, op_size, garbage, slot_1, R_SCRATCH_1, ®1));
649 g(gen_frame_get(ctx, op_size, garbage, slot_2, R_SCRATCH_2, ®2));
650 g(gen_3address_alu(ctx, OP_SIZE_8, ALU_SMULH, R_SCRATCH_3, reg1, reg2, 0));
652 g(gen_3address_alu(ctx, OP_SIZE_8, ALU_MUL, target, reg1, reg2, 0));
654 gen_insn(INSN_CMP, OP_SIZE_8, 0, 1);
655 gen_one(R_SCRATCH_3);
656 gen_one(ARG_SHIFTED_REGISTER);
657 gen_one(ARG_SHIFT_ASR | 0x3f);
658 gen_one(target);
660 gen_insn(INSN_JMP_COND, OP_SIZE_8, COND_NE, 0);
661 gen_four(label_ovf);
663 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
665 return true;
666 }
667 #endif
668 #if defined(ARCH_POWER)
669 if (mode == MODE_INT && op_size >= OP_SIZE_4) {
670 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
671 g(gen_frame_get(ctx, op_size, garbage, slot_1, R_SCRATCH_1, ®1));
672 g(gen_frame_get(ctx, op_size, garbage, slot_2, R_SCRATCH_2, ®2));
674 g(gen_3address_alu(ctx, op_size, ALU_MUL, target, reg1, reg2, 1));
676 gen_insn(INSN_JMP_COND, op_size, COND_O, 0);
677 gen_four(label_ovf);
679 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
681 return true;
682 }
683 #endif
684 #if defined(ARCH_LOONGARCH64) || (defined(ARCH_MIPS) && MIPS_R6) || defined(ARCH_RISCV64)
685 if (mode == MODE_INT && op_size == OP_SIZE_NATIVE) {
686 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
687 g(gen_frame_get(ctx, op_size, garbage, slot_1, R_SCRATCH_1, ®1));
688 g(gen_frame_get(ctx, op_size, garbage, slot_2, R_SCRATCH_2, ®2));
690 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_SMULH, R_SCRATCH_3, reg1, reg2, 0));
692 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_MUL, target, reg1, reg2, 0));
694 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_4, target, (8U << OP_SIZE_NATIVE) - 1, 0));
696 g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, R_SCRATCH_3, R_SCRATCH_4, COND_NE, label_ovf));
698 g(gen_frame_store(ctx, OP_SIZE_NATIVE, slot_r, 0, target));
700 return true;
701 }
702 #endif
703 #if defined(ARCH_S390)
704 if (mode == MODE_INT && op_size >= OP_SIZE_4 && likely(cpu_test_feature(CPU_FEATURE_misc_insn_ext_2))) {
705 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
706 g(gen_frame_load(ctx, op_size, sign_x, slot_1, 0, false, target));
707 g(gen_frame_load_op(ctx, op_size, sign_x, ALU_MUL, 1, slot_2, 0, false, target));
709 gen_insn(INSN_JMP_COND, op_size, COND_O, 0);
710 gen_four(label_ovf);
712 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
713 return true;
714 }
715 #endif
716 #if (defined(ARCH_MIPS) && !MIPS_R6) || defined(ARCH_S390)
717 #if defined(ARCH_MIPS)
718 if (mode == MODE_INT && op_size >= OP_SIZE_4)
719 #endif
720 #if defined(ARCH_S390)
721 if (mode == MODE_INT && op_size == OP_SIZE_4)
722 #endif
723 {
724 #if defined(ARCH_S390)
725 target = R_SCRATCH_1;
726 #else
727 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
728 #endif
729 g(gen_frame_get(ctx, op_size, sign_x, slot_1, R_SCRATCH_1, ®1));
730 g(gen_frame_get(ctx, op_size, sign_x, slot_2, R_SCRATCH_3, ®2));
732 gen_insn(INSN_MUL_L, op_size, 0, 0);
733 gen_one(target);
734 gen_one(R_SCRATCH_2);
735 gen_one(reg1);
736 gen_one(reg2);
738 g(gen_3address_rot_imm(ctx, op_size, ROT_SAR, R_SCRATCH_4, target, (8U << op_size) - 1, false));
740 g(gen_cmp_test_jmp(ctx, INSN_CMP, op_size, R_SCRATCH_2, R_SCRATCH_4, COND_NE, label_ovf));
742 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
743 return true;
744 }
745 #endif
746 if (mode == MODE_INT && op_size == OP_SIZE_NATIVE) {
747 g(gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, INT_binary_multiply_int8_t), op_size, slot_1, slot_2, slot_r, label_ovf));
748 return true;
749 }
751 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
752 if (op_size < OP_SIZE_NATIVE && mode == MODE_INT) {
753 g(gen_frame_get(ctx, op_size, sign_x, slot_1, R_SCRATCH_1, ®1));
754 g(gen_frame_get(ctx, op_size, sign_x, slot_2, R_SCRATCH_2, ®2));
756 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_MUL, target, reg1, reg2, 0));
757 } else {
758 g(gen_frame_load(ctx, op_size, sign_x, slot_1, 0, false, target));
759 g(gen_frame_load_op(ctx, op_size, sign_x, ALU_MUL, 0, slot_2, 0, false, target));
760 }
762 if (mode == MODE_INT) {
763 g(gen_cmp_extended(ctx, OP_SIZE_NATIVE, op_size, target, R_SCRATCH_2, label_ovf));
764 }
766 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
768 return true;
769 }
771 /**********
772 * DIVIDE *
773 **********/
774 do_divide: {
775 uint32_t attr_unused label_skip = 0; /* avoid warning */
776 uint32_t attr_unused label_skip2 = 0; /* avoid warning */
777 uint32_t attr_unused label_end = 0; /* avoid warning */
778 uint32_t attr_unused label_div_0 = 0; /* avoid warning */
779 unsigned attr_unused divide_alu = 0; /* avoid warning */
780 bool attr_unused have_mod = false;
781 bool attr_unused force_sx = false;
782 unsigned attr_unused div_op_size = i_size(op_size);
783 if (unlikely(op_size > OP_SIZE_NATIVE) || unlikely(!ARCH_HAS_DIV)
784 #if defined(ARCH_S390)
785 || !(Z || (op_size <= OP_SIZE_4 && sgn))
786 #endif
787 ) {
788 size_t upcall;
789 if (mode == MODE_INT) {
790 upcall = !mod ? offsetof(struct cg_upcall_vector_s, INT_binary_divide_int8_t) : offsetof(struct cg_upcall_vector_s, INT_binary_modulo_int8_t);
791 } else if (sgn) {
792 upcall = !mod ? offsetof(struct cg_upcall_vector_s, FIXED_binary_divide_int8_t) : offsetof(struct cg_upcall_vector_s, FIXED_binary_modulo_int8_t);
793 } else {
794 upcall = !mod ? offsetof(struct cg_upcall_vector_s, FIXED_binary_udivide_int8_t) : offsetof(struct cg_upcall_vector_s, FIXED_binary_umodulo_int8_t);
795 }
796 g(gen_alu_typed_upcall(ctx, upcall, op_size, slot_1, slot_2, slot_r, mode == MODE_INT ? label_ovf : 0));
797 return true;
798 }
799 #if defined(ARCH_X86) || defined(ARCH_S390)
800 if (mode == MODE_FIXED) {
801 label_skip = alloc_label(ctx);
802 if (unlikely(!label_skip))
803 return false;
804 label_end = alloc_label(ctx);
805 if (unlikely(!label_end))
806 return false;
807 if (sgn) {
808 label_skip2 = alloc_label(ctx);
809 if (unlikely(!label_skip2))
810 return false;
811 }
812 }
813 #if defined(ARCH_X86)
814 if (R_SCRATCH_1 != R_AX || R_SCRATCH_2 != R_DX || R_SCRATCH_3 != R_CX)
815 internal(file_line, "gen_alu: bad scratch registers");
816 #endif
817 g(gen_frame_load(ctx, op_size, sgn ? sign_x : zero_x, slot_1, 0, false, R_SCRATCH_1));
818 g(gen_frame_load(ctx, op_size, sgn ? sign_x : zero_x, slot_2, 0, false, R_SCRATCH_3));
820 g(gen_jmp_on_zero(ctx, i_size(op_size), R_SCRATCH_3, COND_E, mode == MODE_INT ? label_ovf : label_skip));
822 if (sgn) {
823 uint64_t val;
824 uint32_t label_not_minus_1;
825 label_not_minus_1 = alloc_label(ctx);
826 if (unlikely(!label_not_minus_1))
827 return false;
829 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, op_size, R_SCRATCH_3, -1, COND_NE, label_not_minus_1));
831 val = -(uint64_t)0x80 << (((1 << op_size) - 1) * 8);
832 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, op_size, R_SCRATCH_1, val, COND_E, mode == MODE_INT ? label_ovf : label_skip2));
834 gen_label(label_not_minus_1);
835 }
837 #if defined(ARCH_X86)
838 if (op_size >= OP_SIZE_2) {
839 if (sgn) {
840 gen_insn(INSN_CWD + ARCH_PARTIAL_ALU(op_size), op_size, 0, 0);
841 gen_one(R_SCRATCH_2);
842 gen_one(R_SCRATCH_1);
843 if (op_size == OP_SIZE_2)
844 gen_one(R_SCRATCH_2);
845 } else {
846 g(gen_3address_alu(ctx, OP_SIZE_4, ALU_XOR, R_SCRATCH_2, R_SCRATCH_2, R_SCRATCH_2, 0));
847 }
848 }
849 gen_insn(INSN_DIV_L, op_size, sgn, 1);
850 gen_one(R_SCRATCH_1);
851 gen_one(i_size(op_size) == OP_SIZE_1 ? R_SCRATCH_1 : R_SCRATCH_2);
852 gen_one(R_SCRATCH_1);
853 gen_one(i_size(op_size) == OP_SIZE_1 ? R_SCRATCH_1 : R_SCRATCH_2);
854 gen_one(R_SCRATCH_3);
855 #else
856 if (!sgn) {
857 g(gen_load_constant(ctx, R_SCRATCH_2, 0));
858 } else if (op_size <= OP_SIZE_4) {
859 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_2, R_SCRATCH_1, (1U << (OP_SIZE_NATIVE + 3)) - 1, false));
860 }
861 gen_insn(INSN_DIV_L, i_size(op_size), sgn, 1);
862 gen_one(R_SCRATCH_2);
863 gen_one(R_SCRATCH_1);
864 gen_one(R_SCRATCH_2);
865 gen_one(R_SCRATCH_1);
866 gen_one(R_SCRATCH_3);
867 #endif
868 if (mod && i_size(op_size) == OP_SIZE_1) {
869 g(gen_3address_rot_imm(ctx, OP_SIZE_2, ROT_SHR, R_SCRATCH_1, R_SCRATCH_1, 8, 0));
870 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
871 } else if (mod) {
872 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_2));
873 } else {
874 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
875 }
876 if (mode == MODE_FIXED) {
877 gen_insn(INSN_JMP, 0, 0, 0);
878 gen_four(label_end);
880 if (sgn) {
881 gen_label(label_skip2);
883 if (mod)
884 g(gen_frame_clear(ctx, op_size, slot_r));
885 else
886 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
888 gen_insn(INSN_JMP, 0, 0, 0);
889 gen_four(label_end);
890 }
892 gen_label(label_skip);
893 if (!mod)
894 g(gen_frame_clear(ctx, op_size, slot_r));
895 else
896 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
897 gen_label(label_end);
898 }
899 return true;
900 #else
901 #if defined(ARCH_MIPS)
902 have_mod = true;
903 div_op_size = maximum(op_size, OP_SIZE_4);
904 if (op_size == OP_SIZE_4)
905 force_sx = true;
906 #endif
907 #if defined(ARCH_POWER)
908 have_mod = cpu_test_feature(CPU_FEATURE_v30);
909 div_op_size = maximum(op_size, OP_SIZE_4);
910 #endif
911 #if defined(ARCH_LOONGARCH64) || defined(ARCH_RISCV64)
912 have_mod = true;
913 div_op_size = maximum(op_size, OP_SIZE_4);
914 #endif
915 label_end = alloc_label(ctx);
916 if (unlikely(!label_end))
917 return false;
919 g(gen_frame_get(ctx, op_size, (sgn && op_size < i_size(op_size)) || force_sx ? sign_x : zero_x, slot_1, R_SCRATCH_1, ®1));
920 g(gen_frame_get(ctx, op_size, (sgn && op_size < i_size(op_size)) || force_sx ? sign_x : zero_x, slot_2, R_SCRATCH_2, ®2));
921 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_3);
923 if (ARCH_PREFERS_SX(op_size) && !sgn && op_size < i_size(op_size)) {
924 g(gen_extend(ctx, op_size, zero_x, R_SCRATCH_1, reg1));
925 reg1 = R_SCRATCH_1;
926 g(gen_extend(ctx, op_size, zero_x, R_SCRATCH_2, reg2));
927 reg2 = R_SCRATCH_2;
928 }
930 if (mode == MODE_INT) {
931 g(gen_jmp_on_zero(ctx, i_size(op_size), reg2, COND_E, label_ovf));
932 if (sgn) {
933 uint64_t val;
934 uint32_t label_not_minus_1;
935 label_not_minus_1 = alloc_label(ctx);
936 if (unlikely(!label_not_minus_1))
937 return false;
939 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, i_size(op_size), reg2, -1, COND_NE, label_not_minus_1));
941 val = 0xFFFFFFFFFFFFFF80ULL << (((1 << op_size) - 1) * 8);
942 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, i_size(op_size), reg1, val, COND_E, label_ovf));
944 gen_label(label_not_minus_1);
945 }
946 } else {
947 #if !(defined(ARCH_ARM) && ARM_ASM_DIV_NO_TRAP)
948 if (!mod) {
949 g(gen_load_constant(ctx, target, 0));
950 } else {
951 g(gen_mov(ctx, OP_SIZE_NATIVE, target, reg1));
952 }
953 g(gen_jmp_on_zero(ctx, i_size(op_size), reg2, COND_E, label_end));
954 if (sgn) {
955 uint64_t val;
956 uint32_t label_not_minus_1;
957 label_not_minus_1 = alloc_label(ctx);
958 if (unlikely(!label_not_minus_1))
959 return false;
961 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, i_size(op_size), reg2, -1, COND_NE, label_not_minus_1));
963 if (!mod) {
964 g(gen_mov(ctx, OP_SIZE_NATIVE, target, reg1));
965 } else {
966 g(gen_load_constant(ctx, target, 0));
967 }
969 val = 0xFFFFFFFFFFFFFF80ULL << (((1 << op_size) - 1) * 8);
970 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, i_size(op_size), reg1, val, COND_E, label_end));
972 gen_label(label_not_minus_1);
973 }
974 #endif
975 }
976 if (mod && have_mod) {
977 g(gen_3address_alu(ctx, div_op_size, sgn ? ALU_SREM : ALU_UREM, target, reg1, reg2, 0));
978 } else {
979 g(gen_3address_alu(ctx, div_op_size, sgn ? ALU_SDIV : ALU_UDIV, target, reg1, reg2, 0));
980 }
982 if (mod && !have_mod) {
983 #if defined(ARCH_ARM)
984 gen_insn(INSN_MADD, i_size(op_size), 1, 0);
985 gen_one(target);
986 gen_one(target);
987 gen_one(reg2);
988 gen_one(reg1);
989 #else
990 g(gen_3address_alu(ctx, i_size(op_size), ALU_MUL, R_SCRATCH_2, reg2, target, 0));
991 g(gen_3address_alu(ctx, i_size(op_size), ALU_SUB, target, reg1, R_SCRATCH_2, 0));
992 #endif
993 }
995 gen_label(label_end);
996 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
997 return true;
998 #endif
999 }
1000 /*********
1001 * SHIFT *
1002 *********/
1003 do_shift: {
1004 bool sx;
1005 bool must_mask;
1006 unsigned op_s;
1007 int64_t cnst = 0; /* avoid warning */
1008 bool c = frame_t_is_const(slot_2);
1009 if (unlikely(op_size > OP_SIZE_NATIVE)) {
1010 size_t upcall;
1011 if (mode == MODE_FIXED) {
1012 switch (alu) {
1013 case ROT_SHL: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_shl_,TYPE_INT_MAX));
1014 break;
1015 case ROT_SAR: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_shr_,TYPE_INT_MAX));
1016 break;
1017 case ROT_SHR: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_ushr_,TYPE_INT_MAX));
1018 break;
1019 case ROT_ROL: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_rol_,TYPE_INT_MAX));
1020 break;
1021 case ROT_ROR: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_ror_,TYPE_INT_MAX));
1022 break;
1023 default: internal(file_line, "do_alu: invalid shift %u", alu);
1024 }
1025 } else {
1026 switch (alu) {
1027 case ROT_SHL: upcall = offsetof(struct cg_upcall_vector_s, cat(INT_binary_shl_,TYPE_INT_MAX));
1028 break;
1029 case ROT_SAR: upcall = offsetof(struct cg_upcall_vector_s, cat(INT_binary_shr_,TYPE_INT_MAX));
1030 break;
1031 default: internal(file_line, "do_alu: invalid shift %u", alu);
1032 }
1033 }
1034 g(gen_alu_upcall(ctx, upcall, op_size, slot_1, slot_2, slot_r, mode == MODE_INT ? label_ovf : 0));
1035 return true;
1036 }
1037 op_s = i_size_rot(op_size);
1038 #if defined(ARCH_X86)
1039 if (slot_1 == slot_r && !slot_is_register(ctx, slot_1) && !(mode == MODE_INT && alu == ROT_SHL)) {
1040 int64_t offset = (size_t)slot_1 * slot_size;
1041 if (c) {
1042 cnst = frame_t_get_const(slot_2);
1043 if (mode == MODE_INT) {
1044 if ((uint64_t)cnst > (8U << op_size) - 1) {
1045 gen_insn(INSN_JMP, 0, 0, 0);
1046 gen_four(label_ovf);
1047 return true;
1048 }
1049 } else {
1050 cnst &= (8U << op_size) - 1;
1051 }
1052 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDR_OFFSET, op_size));
1053 gen_insn(INSN_ROT + ARCH_PARTIAL_ALU(op_size), op_size, alu, 1);
1054 gen_address_offset();
1055 gen_address_offset();
1056 gen_one(ARG_IMM);
1057 gen_eight(cnst);
1058 } else {
1059 g(gen_frame_load(ctx, op_size, garbage, slot_2, 0, false, R_SCRATCH_3));
1060 if (mode == MODE_INT) {
1061 int64_t imm = (8U << op_size) - 1;
1062 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, op_size, R_SCRATCH_3, imm, COND_A, label_ovf));
1063 } else if ((alu != ROT_ROL && alu != ROT_ROR) && op_size < OP_SIZE_4) {
1064 g(gen_3address_alu_imm(ctx, OP_SIZE_1, ALU_AND, R_SCRATCH_3, R_SCRATCH_3, (8U << op_size) - 1, 0));
1065 }
1066 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDR_OFFSET, op_size));
1067 gen_insn(INSN_ROT + ARCH_PARTIAL_ALU(op_size), op_size, alu, 1);
1068 gen_address_offset();
1069 gen_address_offset();
1070 gen_one(R_SCRATCH_3);
1071 }
1072 return true;
1073 }
1074 if (mode == MODE_INT && alu == ROT_SHL && op_size < OP_SIZE_NATIVE)
1075 op_s = op_size + 1;
1076 #endif
1077 must_mask = op_size < ARCH_SHIFT_SIZE;
1078 sx = (alu == ROT_SAR && op_size < op_s) || (alu == ROT_SHL && op_size < OP_SIZE_NATIVE && mode == MODE_INT);
1079 #if defined(ARCH_MIPS)
1080 sx |= op_size == OP_SIZE_4;
1081 #endif
1082 g(gen_frame_get(ctx, op_size, sx ? sign_x : zero_x, slot_1, R_SCRATCH_1, ®1));
1083 if (c) {
1084 reg3 = 0xff; /* avoid warning */
1085 cnst = frame_t_get_const(slot_2);
1086 } else
1087 #if defined(ARCH_X86)
1088 if (!ARCH_IS_3ADDRESS_ROT(alu, op_size)) {
1089 g(gen_frame_load(ctx, op_size, garbage, slot_2, 0, false, R_SCRATCH_3));
1090 reg3 = R_SCRATCH_3;
1091 } else
1092 #endif
1093 g(gen_frame_get(ctx, op_size, garbage, slot_2, R_SCRATCH_3, ®3));
1095 if (ARCH_PREFERS_SX(op_size) && !sx && op_size < op_s) {
1096 g(gen_extend(ctx, op_size, zero_x, R_SCRATCH_1, reg1));
1097 reg1 = R_SCRATCH_1;
1098 }
1100 if (mode == MODE_INT) {
1101 int64_t imm = (8U << op_size) - 1;
1102 if (c) {
1103 if ((uint64_t)cnst > (uint64_t)imm) {
1104 gen_insn(INSN_JMP, 0, 0, 0);
1105 gen_four(label_ovf);
1106 return true;
1107 }
1108 } else {
1109 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, i_size_cmp(op_size), reg3, imm, COND_A, label_ovf));
1110 }
1111 } else {
1112 #if defined(ARCH_ARM)
1113 if (alu == ROT_ROL) {
1114 if (c) {
1115 cnst = -(uint64_t)cnst;
1116 } else {
1117 g(gen_2address_alu1(ctx, OP_SIZE_4, ALU1_NEG, R_SCRATCH_3, reg3, 0));
1118 reg3 = R_SCRATCH_3;
1119 }
1120 alu = ROT_ROR;
1121 }
1122 #endif
1123 #if defined(ARCH_LOONGARCH64)
1124 if (alu == ROT_ROL && op_size >= OP_SIZE_4) {
1125 if (c) {
1126 cnst = -(uint64_t)cnst;
1127 } else {
1128 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NEG, R_SCRATCH_3, reg3, 0));
1129 reg3 = R_SCRATCH_3;
1130 }
1131 alu = ROT_ROR;
1132 }
1133 #endif
1134 #if defined(ARCH_MIPS)
1135 if (MIPS_HAS_ROT && alu == ROT_ROL && op_size >= OP_SIZE_4) {
1136 if (c) {
1137 cnst = -(uint64_t)cnst;
1138 } else {
1139 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NEG, R_SCRATCH_3, reg3, 0));
1140 reg3 = R_SCRATCH_3;
1141 }
1142 alu = ROT_ROR;
1143 }
1144 #endif
1145 #if defined(ARCH_POWER)
1146 if (alu == ROT_ROR && op_size >= OP_SIZE_4) {
1147 if (c) {
1148 cnst = -(uint64_t)cnst;
1149 } else {
1150 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NEG, R_SCRATCH_3, reg3, 0));
1151 reg3 = R_SCRATCH_3;
1152 }
1153 alu = ROT_ROL;
1154 }
1155 #endif
1156 #if defined(ARCH_S390)
1157 if (Z && alu == ROT_ROR && op_size >= OP_SIZE_4) {
1158 if (c) {
1159 cnst = -(uint64_t)cnst;
1160 } else {
1161 g(gen_2address_alu1(ctx, OP_SIZE_4, ALU1_NEG, R_SCRATCH_3, reg3, 0));
1162 reg3 = R_SCRATCH_3;
1163 }
1164 alu = ROT_ROL;
1165 }
1166 #endif
1167 if (c) {
1168 cnst &= (8U << op_size) - 1;
1169 } else if (must_mask) {
1170 g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_4), ALU_AND, R_SCRATCH_3, reg3, (8U << op_size) - 1, 0));
1171 reg3 = R_SCRATCH_3;
1172 }
1173 }
1175 #if defined(ARCH_X86)
1176 if (mode == MODE_INT && alu == ROT_SHL) {
1177 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_2);
1178 } else {
1179 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, slot_2, R_SCRATCH_2);
1180 }
1181 if (c) {
1182 g(gen_3address_rot_imm(ctx, op_s, alu, target, reg1, cnst, 0));
1183 } else {
1184 g(gen_3address_rot(ctx, op_s, alu, target, reg1, reg3));
1185 }
1187 if (mode == MODE_INT && alu == ROT_SHL) {
1188 if (op_size < OP_SIZE_NATIVE) {
1189 gen_insn(INSN_MOVSX, op_size, 0, 0);
1190 gen_one(R_SCRATCH_4);
1191 gen_one(target);
1193 g(gen_cmp_test_jmp(ctx, INSN_CMP, op_s, target, R_SCRATCH_4, COND_NE, label_ovf));
1194 } else {
1195 if (c) {
1196 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_4, target, cnst, 0));
1197 } else {
1198 g(gen_3address_rot(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_4, target, reg3));
1199 }
1201 g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, reg1, R_SCRATCH_4, COND_NE, label_ovf));
1202 }
1203 }
1204 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1205 return true;
1206 #endif
1207 #if defined(ARCH_ARM)
1208 if (op_size <= OP_SIZE_2 && alu == ROT_ROR) {
1209 gen_insn(INSN_ALU, OP_SIZE_4, ALU_OR, ALU_WRITES_FLAGS(ALU_OR, false));
1210 gen_one(R_SCRATCH_1);
1211 gen_one(reg1);
1212 gen_one(ARG_SHIFTED_REGISTER);
1213 gen_one(ARG_SHIFT_LSL | (8U << op_size));
1214 gen_one(reg1);
1215 if (op_size == OP_SIZE_1)
1216 alu = ROT_SHR;
1217 reg1 = R_SCRATCH_1;
1218 }
1219 goto do_generic_shift;
1220 #endif
1221 #if defined(ARCH_LOONGARCH64)
1222 if (alu == ROT_ROR && op_size >= OP_SIZE_4)
1223 goto do_generic_shift;
1224 #endif
1225 #if defined(ARCH_MIPS)
1226 if (MIPS_HAS_ROT && alu == ROT_ROR && op_size >= OP_SIZE_4)
1227 goto do_generic_shift;
1228 #endif
1229 #if defined(ARCH_POWER)
1230 if (alu == ROT_ROL && op_size >= OP_SIZE_4)
1231 goto do_generic_shift;
1232 #endif
1233 #if defined(ARCH_RISCV64)
1234 if ((alu == ROT_ROL || alu == ROT_ROR) && likely(cpu_test_feature(CPU_FEATURE_zbb))) {
1235 if (likely(op_size >= OP_SIZE_4)) {
1236 goto do_generic_shift;
1237 }
1238 }
1239 #endif
1240 #if defined(ARCH_S390)
1241 if (Z && alu == ROT_ROL && op_size >= OP_SIZE_4)
1242 goto do_generic_shift;
1243 #endif
1244 if (alu == ROT_ROL || alu == ROT_ROR) {
1245 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
1246 if (c) {
1247 g(gen_3address_rot_imm(ctx, op_s, alu == ROT_ROL ? ROT_SHL : ROT_SHR, R_SCRATCH_2, reg1, cnst, 0));
1248 g(gen_3address_rot_imm(ctx, op_s, alu == ROT_ROL ? ROT_SHR : ROT_SHL, target, reg1, -(uint64_t)cnst & ((8U << op_size) - 1), 0));
1249 } else {
1250 g(gen_3address_rot(ctx, op_s, alu == ROT_ROL ? ROT_SHL : ROT_SHR, R_SCRATCH_2, reg1, reg3));
1251 g(gen_2address_alu1(ctx, i_size(OP_SIZE_4), ALU1_NEG, R_SCRATCH_3, reg3, 0));
1252 if (must_mask) {
1253 g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_4), ALU_AND, R_SCRATCH_3, R_SCRATCH_3, (8U << op_size) - 1, 0));
1254 }
1255 g(gen_3address_rot(ctx, op_s, alu == ROT_ROL ? ROT_SHR : ROT_SHL, target, reg1, R_SCRATCH_3));
1256 }
1257 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_OR, target, target, R_SCRATCH_2, 0));
1258 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1259 return true;
1260 }
1262 goto do_generic_shift;
1263 do_generic_shift:
1264 if (mode == MODE_INT && alu == ROT_SHL) {
1265 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
1266 #if defined(ARCH_S390)
1267 if (op_size >= OP_SIZE_4) {
1268 if (c)
1269 g(gen_3address_rot_imm(ctx, op_size, ROT_SAL, target, reg1, cnst, 0));
1270 else
1271 g(gen_3address_rot(ctx, op_size, ROT_SAL, target, reg1, reg3));
1273 gen_insn(INSN_JMP_COND, op_size, COND_O, 0);
1274 gen_four(label_ovf);
1275 } else
1276 #endif
1277 if (op_size <= OP_SIZE_NATIVE - 1) {
1278 if (c)
1279 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, alu, target, reg1, cnst, 0));
1280 else
1281 g(gen_3address_rot(ctx, OP_SIZE_NATIVE, alu, target, reg1, reg3));
1283 g(gen_cmp_extended(ctx, OP_SIZE_NATIVE, op_size, target, R_SCRATCH_2, label_ovf));
1284 } else {
1285 if (c) {
1286 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, alu, R_SCRATCH_2, reg1, cnst, 0));
1287 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_4, R_SCRATCH_2, cnst, 0));
1288 } else {
1289 g(gen_3address_rot(ctx, OP_SIZE_NATIVE, alu, R_SCRATCH_2, reg1, reg3));
1290 g(gen_3address_rot(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_4, R_SCRATCH_2, reg3));
1291 }
1293 g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, reg1, R_SCRATCH_4, COND_NE, label_ovf));
1295 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_2));
1297 return true;
1298 }
1299 } else {
1300 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, slot_2, R_SCRATCH_1);
1301 if (c)
1302 g(gen_3address_rot_imm(ctx, op_s, alu, target, reg1, cnst, 0));
1303 else
1304 g(gen_3address_rot(ctx, op_s, alu, target, reg1, reg3));
1305 }
1307 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1308 return true;
1309 }
1310 /******
1311 * BT *
1312 ******/
1313 do_bt: {
1314 unsigned attr_unused op_s;
1315 bool need_mask;
1316 bool c = frame_t_is_const(slot_2);
1317 int64_t cnst = !c ? 0 : frame_t_get_const(slot_2);
1318 int64_t max_imm = (8U << op_size) - 1;
1319 if (c) {
1320 if (mode == MODE_INT) {
1321 if (alu == BTX_BT ? (uint64_t)cnst > (uint64_t)max_imm : (uint64_t)cnst >= (uint64_t)max_imm) {
1322 gen_insn(INSN_JMP, 0, 0, 0);
1323 gen_four(label_ovf);
1324 return true;
1325 }
1326 }
1327 cnst &= max_imm;
1328 }
1329 #if defined(ARCH_X86)
1330 if ((alu == BTX_BT || slot_1 == slot_r) && !slot_is_register(ctx, slot_1)) {
1331 int64_t offset;
1332 unsigned n_op_size = minimum(op_size, OP_SIZE_NATIVE);
1333 g(gen_frame_get(ctx, n_op_size, garbage, slot_2, R_SCRATCH_1, ®2));
1334 if (mode == MODE_INT) {
1335 if (!c) {
1336 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, n_op_size, reg2, max_imm, alu == BTX_BT ? COND_A : COND_AE, label_ovf));
1337 if (unlikely(op_size > OP_SIZE_NATIVE)) {
1338 g(gen_address(ctx, R_FRAME, (size_t)slot_2 * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_STR_OFFSET, OP_SIZE_NATIVE));
1339 gen_insn(INSN_CMP, OP_SIZE_NATIVE, 0, 1);
1340 gen_address_offset();
1341 gen_one(ARG_IMM);
1342 gen_eight(0);
1343 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_NE, 0);
1344 gen_four(label_ovf);
1345 }
1346 }
1347 } else {
1348 if (!c) {
1349 g(gen_3address_alu_imm(ctx, OP_SIZE_4, ALU_AND, R_SCRATCH_1, reg2, (8U << op_size) - 1, 0));
1350 reg2 = R_SCRATCH_1;
1351 }
1352 }
1353 offset = (size_t)slot_1 * slot_size;
1354 if (c && cnst >= 8U << OP_SIZE_NATIVE) {
1355 offset += 1U << OP_SIZE_NATIVE;
1356 cnst -= 8U << OP_SIZE_NATIVE;
1357 }
1358 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_STR_OFFSET, maximum(n_op_size, OP_SIZE_2)));
1359 if (alu == BTX_BT) {
1360 gen_insn(INSN_BT, maximum(n_op_size, OP_SIZE_2), 0, 1);
1361 gen_address_offset();
1362 if (c) {
1363 gen_one(ARG_IMM);
1364 gen_eight(cnst);
1365 } else {
1366 gen_one(reg2);
1367 }
1368 g(gen_frame_set_cond(ctx, maximum(n_op_size, OP_SIZE_2), false, COND_B, slot_r));
1369 } else {
1370 gen_insn(INSN_BTX, maximum(n_op_size, OP_SIZE_2), alu, 1);
1371 gen_address_offset();
1372 gen_address_offset();
1373 if (c) {
1374 gen_one(ARG_IMM);
1375 gen_eight(cnst);
1376 } else {
1377 gen_one(reg2);
1378 }
1379 }
1380 return true;
1381 }
1382 #endif
1383 if (unlikely(op_size > OP_SIZE_NATIVE)) {
1384 size_t upcall;
1385 if (mode == MODE_FIXED) {
1386 switch (alu) {
1387 case BTX_BTS: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_bts_,TYPE_INT_MAX));
1388 break;
1389 case BTX_BTR: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_btr_,TYPE_INT_MAX));
1390 break;
1391 case BTX_BTC: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_btc_,TYPE_INT_MAX));
1392 break;
1393 case BTX_BT: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_bt_,TYPE_INT_MAX));
1394 break;
1395 default: internal(file_line, "do_alu: invalid bit test %u", alu);
1396 }
1397 } else {
1398 switch (alu) {
1399 case BTX_BTS: upcall = offsetof(struct cg_upcall_vector_s, cat(INT_binary_bts_,TYPE_INT_MAX));
1400 break;
1401 case BTX_BTR: upcall = offsetof(struct cg_upcall_vector_s, cat(INT_binary_btr_,TYPE_INT_MAX));
1402 break;
1403 case BTX_BTC: upcall = offsetof(struct cg_upcall_vector_s, cat(INT_binary_btc_,TYPE_INT_MAX));
1404 break;
1405 case BTX_BT: upcall = offsetof(struct cg_upcall_vector_s, cat(INT_binary_bt_,TYPE_INT_MAX));
1406 break;
1407 default: internal(file_line, "do_alu: invalid bit test %u", alu);
1408 }
1409 }
1410 g(gen_alu_upcall(ctx, upcall, op_size, slot_1, slot_2, slot_r, label_ovf));
1411 return true;
1412 }
1413 op_s = minimum(OP_SIZE_NATIVE, ARCH_SHIFT_SIZE);
1414 op_s = maximum(op_s, op_size);
1415 g(gen_frame_get(ctx, op_size, zero_x, slot_1, R_SCRATCH_1, ®1));
1416 if (c) {
1417 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1418 if (ARCH_HAS_BTX(alu == BTX_BT ? BTX_BTEXT : alu, OP_SIZE_NATIVE, true)) {
1419 #ifdef ARCH_X86
1420 g(gen_mov(ctx, OP_SIZE_NATIVE, target, reg1));
1421 reg1 = target;
1422 gen_insn(INSN_BTX, OP_SIZE_NATIVE, alu == BTX_BT ? BTX_BTEXT : alu, 1);
1423 #else
1424 gen_insn(INSN_BTX, OP_SIZE_NATIVE, alu == BTX_BT ? BTX_BTEXT : alu, 0);
1425 #endif
1426 gen_one(target);
1427 gen_one(reg1);
1428 gen_one(ARG_IMM);
1429 gen_eight(cnst);
1430 } else switch (alu) {
1431 case BTX_BTS:
1432 g(gen_3address_alu_imm(ctx, i_size(op_size), ALU_OR, target, reg1, 1ULL << cnst, 0));
1433 break;
1434 case BTX_BTR:
1435 g(gen_3address_alu_imm(ctx, i_size(op_size), ALU_AND, target, reg1, ~(1ULL << cnst), 0));
1436 break;
1437 case BTX_BTC:
1438 g(gen_3address_alu_imm(ctx, i_size(op_size), ALU_XOR, target, reg1, 1ULL << cnst, 0));
1439 break;
1440 case BTX_BT:
1441 if (cnst) {
1442 g(gen_3address_rot_imm(ctx, i_size(op_size), ROT_SHR, target, reg1, cnst, 0));
1443 }
1444 g(gen_3address_alu_imm(ctx, i_size(op_size), ALU_AND, target, target, 1, 0));
1445 break;
1446 default:
1447 internal(file_line, "do_alu: invalid bit test %u", alu);
1448 }
1449 if (alu == BTX_BT)
1450 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
1451 else
1452 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1453 return true;
1454 }
1455 g(gen_frame_get(ctx, op_size, garbage, slot_2, R_SCRATCH_2, ®2));
1456 if (mode == MODE_INT) {
1457 if (!c)
1458 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, i_size_cmp(op_size), reg2, max_imm, alu == BTX_BT ? COND_A : COND_AE, label_ovf));
1459 }
1460 if (alu != BTX_BT) {
1461 if (!ARCH_HAS_BTX(alu, OP_SIZE_NATIVE, false))
1462 goto do_generic_bt;
1463 need_mask = !ARCH_HAS_BTX(alu, op_size, false);
1464 } else {
1465 #if defined(ARCH_X86)
1466 need_mask = op_size < OP_SIZE_2;
1467 #else
1468 if (!ARCH_HAS_BTX(BTX_BTEXT, OP_SIZE_NATIVE, false))
1469 goto do_generic_bt;
1470 need_mask = !ARCH_HAS_BTX(BTX_BTEXT, op_size, false);
1471 #endif
1472 }
1473 if (need_mask) {
1474 g(gen_3address_alu_imm(ctx, OP_SIZE_4, ALU_AND, R_SCRATCH_2, reg2, (8U << op_size) - 1, 0));
1475 reg2 = R_SCRATCH_2;
1476 }
1477 if (alu == BTX_BT) {
1478 #if defined(ARCH_X86)
1479 gen_insn(INSN_BT, maximum(op_size, OP_SIZE_2), 0, 1);
1480 gen_one(reg1);
1481 gen_one(reg2);
1483 g(gen_frame_set_cond(ctx, maximum(op_size, OP_SIZE_2), false, COND_B, slot_r));
1484 #else
1485 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1486 gen_insn(INSN_BTX, need_mask ? OP_SIZE_NATIVE : op_size, BTX_BTEXT, 0);
1487 gen_one(target);
1488 gen_one(reg1);
1489 gen_one(reg2);
1491 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
1492 #endif
1493 } else {
1494 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, slot_2, R_SCRATCH_1);
1495 #if defined(ARCH_X86)
1496 if (target == reg2)
1497 target = R_SCRATCH_1;
1498 if (target != reg1) {
1499 g(gen_mov(ctx, op_size, target, reg1));
1500 reg1 = target;
1501 }
1502 gen_insn(INSN_BTX, maximum(op_size, OP_SIZE_2), alu, 1);
1503 #else
1504 gen_insn(INSN_BTX, need_mask ? OP_SIZE_NATIVE : op_size, alu, 0);
1505 #endif
1506 gen_one(target);
1507 gen_one(reg1);
1508 gen_one(reg2);
1510 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1511 }
1512 return true;
1514 goto do_generic_bt;
1515 do_generic_bt:
1516 if (mode == MODE_FIXED && op_size < ARCH_SHIFT_SIZE) {
1517 g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_4), ALU_AND, R_SCRATCH_2, reg2, (8U << op_size) - 1, 0));
1518 reg2 = R_SCRATCH_2;
1519 }
1520 g(gen_load_constant(ctx, R_SCRATCH_3, 1));
1522 g(gen_3address_rot(ctx, op_s, ROT_SHL, R_SCRATCH_3, R_SCRATCH_3, reg2));
1524 switch (alu) {
1525 case BTX_BT:
1526 #if ARCH_HAS_FLAGS
1527 #if defined(ARCH_S390) || defined(ARCH_POWER)
1528 g(gen_3address_alu(ctx, i_size(op_size), ALU_AND, R_SCRATCH_1, reg1, R_SCRATCH_3, 1));
1529 #else
1530 gen_insn(INSN_TEST, i_size(op_size), 0, 1);
1531 gen_one(reg1);
1532 gen_one(R_SCRATCH_3);
1533 #endif
1534 g(gen_frame_set_cond(ctx, i_size_cmp(op_size), false, COND_NE, slot_r));
1535 #else
1536 g(gen_3address_alu(ctx, i_size(op_size), ALU_AND, R_SCRATCH_1, reg1, R_SCRATCH_3, 0));
1537 g(gen_frame_cmp_imm_set_cond_reg(ctx, i_size(op_size), R_SCRATCH_1, 0, COND_NE, slot_r));
1538 #endif
1539 return true;
1540 case BTX_BTS:
1541 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1542 g(gen_3address_alu(ctx, i_size(op_size), ALU_OR, target, reg1, R_SCRATCH_3, 0));
1543 break;
1544 case BTX_BTR:
1545 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1546 if (!ARCH_HAS_ANDN) {
1547 g(gen_3address_alu_imm(ctx, i_size(op_size), ALU_XOR, R_SCRATCH_3, R_SCRATCH_3, -1, 0));
1549 g(gen_3address_alu(ctx, i_size(op_size), ALU_AND, target, reg1, R_SCRATCH_3, 0));
1550 break;
1551 }
1552 g(gen_3address_alu(ctx, i_size(op_size), ALU_ANDN, target, reg1, R_SCRATCH_3, 0));
1553 break;
1554 case BTX_BTC:
1555 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1556 g(gen_3address_alu(ctx, i_size(op_size), ALU_XOR, target, reg1, R_SCRATCH_3, 0));
1557 break;
1558 default:
1559 internal(file_line, "gen_alu: unsupported bit test %u", alu);
1560 }
1562 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1564 return true;
1565 }
1566 /***********
1567 * COMPARE *
1568 ***********/
1569 do_compare: {
1570 if (unlikely(op_size > OP_SIZE_NATIVE)) {
1571 size_t attr_unused upcall;
1572 frame_t attr_unused swap;
1573 switch (alu) {
1574 case COND_E:
1575 case COND_NE:
1576 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
1577 g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, ALU_XOR, 0, slot_2, lo_word(OP_SIZE_NATIVE), true, R_SCRATCH_1));
1578 g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, ALU_XOR, 0, slot_2, hi_word(OP_SIZE_NATIVE), true, R_SCRATCH_2));
1579 #if defined(ARCH_ARM64)
1580 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_OR, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_2, 0));
1582 gen_insn(INSN_CMP, OP_SIZE_NATIVE, 0, 1);
1583 gen_one(R_SCRATCH_1);
1584 gen_one(ARG_IMM);
1585 gen_eight(0);
1586 #else
1587 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_OR, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_2, ARCH_HAS_FLAGS));
1588 #endif
1589 #if ARCH_HAS_FLAGS
1590 g(gen_frame_set_cond(ctx, OP_SIZE_NATIVE, false, alu, slot_r));
1591 #else
1592 g(gen_frame_cmp_imm_set_cond_reg(ctx, OP_SIZE_NATIVE, R_SCRATCH_1, 0, alu, slot_r));
1593 #endif
1594 return true;
1595 #if defined(ARCH_X86_64) || defined(ARCH_X86_X32) || defined(ARCH_ARM)
1596 case COND_G:
1597 case COND_A:
1598 swap = slot_1; slot_1 = slot_2; slot_2 = swap;
1599 alu = alu == COND_G ? COND_L : COND_B;
1600 /*-fallthrough*/
1601 case COND_L:
1602 case COND_B:
1603 g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_2, lo_word(OP_SIZE_NATIVE), true, R_SCRATCH_2));
1604 g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_1, hi_word(OP_SIZE_NATIVE), true, R_SCRATCH_1));
1605 g(gen_frame_load_cmp(ctx, OP_SIZE_NATIVE, false, garbage, true, slot_1, lo_word(OP_SIZE_NATIVE), true, R_SCRATCH_2));
1606 g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, ALU_SBB, 1, slot_2, hi_word(OP_SIZE_NATIVE), true, R_SCRATCH_1));
1607 g(gen_frame_set_cond(ctx, OP_SIZE_NATIVE, false, alu, slot_r));
1608 return true;
1609 case COND_GE:
1610 case COND_AE:
1611 swap = slot_1; slot_1 = slot_2; slot_2 = swap;
1612 alu = alu == COND_GE ? COND_LE : COND_BE;
1613 /*-fallthrough*/
1614 case COND_LE:
1615 case COND_BE:
1616 g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_1, lo_word(OP_SIZE_NATIVE), true, R_SCRATCH_2));
1617 g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_2, hi_word(OP_SIZE_NATIVE), true, R_SCRATCH_1));
1618 g(gen_frame_load_cmp(ctx, OP_SIZE_NATIVE, false, garbage, true, slot_2, lo_word(OP_SIZE_NATIVE), true, R_SCRATCH_2));
1619 g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, ALU_SBB, 1, slot_1, hi_word(OP_SIZE_NATIVE), true, R_SCRATCH_1));
1620 g(gen_frame_set_cond(ctx, OP_SIZE_NATIVE, false, alu == COND_LE ? COND_GE : COND_AE, slot_r));
1621 return true;
1622 #else
1623 case COND_L: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_less_,TYPE_INT_MAX)); goto do_upcall;
1624 case COND_LE: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_less_equal_,TYPE_INT_MAX)); goto do_upcall;
1625 case COND_G: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_greater_,TYPE_INT_MAX)); goto do_upcall;
1626 case COND_GE: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_greater_equal_,TYPE_INT_MAX)); goto do_upcall;
1627 case COND_B: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_uless_,TYPE_INT_MAX)); goto do_upcall;
1628 case COND_BE: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_uless_equal_,TYPE_INT_MAX)); goto do_upcall;
1629 case COND_A: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_ugreater_,TYPE_INT_MAX)); goto do_upcall;
1630 case COND_AE: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_ugreater_equal_,TYPE_INT_MAX)); goto do_upcall;
1631 do_upcall: g(gen_alu_upcall(ctx, upcall, op_size, slot_1, slot_2, slot_r, 0));
1632 return true;
1633 #endif
1634 default:
1635 internal(file_line, "gen_alu: unsupported condition %u", alu);
1636 }
1637 return false;
1638 }
1639 #if defined(ARCH_X86)
1640 g(gen_frame_get(ctx, op_size, garbage, slot_1, R_SCRATCH_1, ®1));
1641 g(gen_frame_load_cmp_set_cond(ctx, op_size, garbage, slot_2, reg1, alu, slot_r));
1642 #else
1643 g(gen_frame_get(ctx, op_size, op_size == i_size_cmp(op_size) + (unsigned)zero ? garbage : alu == COND_L || alu == COND_LE || alu == COND_G || alu == COND_GE || ARCH_PREFERS_SX(op_size) ? sign_x : zero_x, slot_1, R_SCRATCH_1, ®1));
1644 g(gen_frame_load_cmp_set_cond(ctx, op_size, alu == COND_L || alu == COND_LE || alu == COND_G || alu == COND_GE || ARCH_PREFERS_SX(op_size) ? sign_x : zero_x, slot_2, reg1, alu, slot_r));
1645 #endif
1646 return true;
1647 }
1648 }
1650 static bool attr_w gen_alu1(struct codegen_context *ctx, unsigned mode, unsigned op_size, unsigned op, uint32_t label_ovf, frame_t slot_1, frame_t slot_r)
1651 {
1652 unsigned alu;
1653 unsigned reg1, target;
1654 switch (mode) {
1655 case MODE_FIXED: switch (op) {
1656 case OPCODE_FIXED_OP_not: alu = ALU1_NOT; goto do_alu;
1657 case OPCODE_FIXED_OP_neg: alu = ALU1_NEG; goto do_alu;
1658 case OPCODE_FIXED_OP_inc: alu = ALU1_INC; goto do_alu;
1659 case OPCODE_FIXED_OP_dec: alu = ALU1_DEC; goto do_alu;
1660 case OPCODE_FIXED_OP_bswap:
1661 case OPCODE_FIXED_OP_bswap_alt1: alu = ALU1_BSWAP; goto do_bswap;
1662 case OPCODE_FIXED_OP_brev:
1663 case OPCODE_FIXED_OP_brev_alt1: alu = ALU1_BREV; goto do_brev;
1664 case OPCODE_FIXED_OP_bsf:
1665 case OPCODE_FIXED_OP_bsf_alt1: alu = ALU1_BSF; goto do_bsf_bsr_popcnt;
1666 case OPCODE_FIXED_OP_bsr:
1667 case OPCODE_FIXED_OP_bsr_alt1: alu = ALU1_BSR; goto do_bsf_bsr_popcnt;
1668 case OPCODE_FIXED_OP_popcnt:
1669 case OPCODE_FIXED_OP_popcnt_alt1: alu = ALU1_POPCNT; goto do_bsf_bsr_popcnt;
1670 case OPCODE_FIXED_OP_to_int: goto do_fixed_conv;
1671 case OPCODE_FIXED_OP_from_int: goto do_fixed_conv;
1672 case OPCODE_FIXED_OP_uto_int: goto conv_uto_int;
1673 case OPCODE_FIXED_OP_ufrom_int: goto conv_ufrom_int;
1674 default: internal(file_line, "gen_alu1: unsupported fixed operation %u", op);
1675 }
1676 case MODE_INT: switch (op) {
1677 case OPCODE_INT_OP_not: alu = ALU1_NOT; mode = MODE_FIXED; goto do_alu;
1678 case OPCODE_INT_OP_neg: alu = ALU1_NEG; goto do_alu;
1679 case OPCODE_INT_OP_inc: alu = ALU1_INC; goto do_alu;
1680 case OPCODE_INT_OP_dec: alu = ALU1_DEC; goto do_alu;
1681 case OPCODE_INT_OP_bsf: alu = ALU1_BSF; goto do_bsf_bsr_popcnt;
1682 case OPCODE_INT_OP_bsr: alu = ALU1_BSR; goto do_bsf_bsr_popcnt;
1683 case OPCODE_INT_OP_popcnt:
1684 case OPCODE_INT_OP_popcnt_alt1: alu = ALU1_POPCNT; goto do_bsf_bsr_popcnt;
1685 case OPCODE_INT_OP_to_int: goto do_conv;
1686 case OPCODE_INT_OP_from_int: goto do_conv;
1687 default: internal(file_line, "gen_alu1: unsupported int operation %u", op);
1688 }
1689 case MODE_BOOL: switch (op) {
1690 case OPCODE_BOOL_OP_not: goto do_bool_not;
1691 default: internal(file_line, "gen_alu1: unsupported bool operation %u", op);
1692 }
1693 }
1694 internal(file_line, "gen_alu1: unsupported mode %u", mode);
1696 /*******
1697 * ALU *
1698 *******/
1699 do_alu: {
1700 bool arch_use_flags = ARCH_HAS_FLAGS;
1701 enum extend ex;
1702 #if defined(ARCH_POWER)
1703 arch_use_flags = false;
1704 #endif
1705 if (op_size > OP_SIZE_NATIVE) {
1706 #if !defined(ARCH_X86) && !defined(ARCH_ARM) && !defined(ARCH_POWER)
1707 if (alu == ALU1_NEG) {
1708 if (mode == MODE_FIXED)
1709 g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(FIXED_unary_neg_,TYPE_INT_MAX)), op_size, slot_1, NO_FRAME_T, slot_r, 0));
1710 else
1711 g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(INT_unary_neg_,TYPE_INT_MAX)), op_size, slot_1, NO_FRAME_T, slot_r, label_ovf));
1712 return true;
1713 }
1714 if (alu == ALU1_DEC) {
1715 if (mode == MODE_FIXED)
1716 g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(FIXED_unary_dec_,TYPE_INT_MAX)), op_size, slot_1, NO_FRAME_T, slot_r, 0));
1717 else
1718 g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(INT_unary_dec_,TYPE_INT_MAX)), op_size, slot_1, NO_FRAME_T, slot_r, label_ovf));
1719 return true;
1720 }
1721 if (alu == ALU1_INC) {
1722 if (mode == MODE_FIXED)
1723 g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(FIXED_unary_inc_,TYPE_INT_MAX)), op_size, slot_1, NO_FRAME_T, slot_r, 0));
1724 else
1725 g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(INT_unary_inc_,TYPE_INT_MAX)), op_size, slot_1, NO_FRAME_T, slot_r, label_ovf));
1726 return true;
1727 }
1728 #endif
1729 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
1730 #if defined(ARCH_S390)
1731 if (alu == ALU1_NOT) {
1732 g(gen_load_constant(ctx, R_SCRATCH_3, -1));
1734 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_XOR, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_3, 0));
1735 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_XOR, R_SCRATCH_2, R_SCRATCH_2, R_SCRATCH_3, 0));
1737 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_1, R_SCRATCH_2));
1738 return true;
1739 }
1740 #endif
1741 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, alu, R_SCRATCH_1, R_SCRATCH_1, alu == ALU1_INC || alu == ALU1_DEC || alu == ALU1_NEG ? 2 : 0));
1742 if (alu == ALU1_NOT) {
1743 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NOT, R_SCRATCH_2, R_SCRATCH_2, 0));
1744 } else if (alu == ALU1_INC || alu == ALU1_DEC) {
1745 g(gen_imm(ctx, 0, alu == ALU1_INC ? IMM_PURPOSE_ADD : IMM_PURPOSE_SUB, OP_SIZE_NATIVE));
1746 gen_insn(INSN_ALU, OP_SIZE_NATIVE, alu == ALU1_INC ? ALU_ADC : ALU_SBB, (mode == MODE_INT) | ALU_WRITES_FLAGS(alu == ALU1_INC ? ALU_ADC : ALU_SBB, is_imm()));
1747 gen_one(R_SCRATCH_2);
1748 gen_one(R_SCRATCH_2);
1749 gen_imm_offset();
1750 } else {
1751 #if defined(ARCH_X86)
1752 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NOT, R_SCRATCH_2, R_SCRATCH_2, 0));
1754 g(gen_imm(ctx, -1, IMM_PURPOSE_SUB, OP_SIZE_NATIVE));
1755 gen_insn(INSN_ALU, OP_SIZE_NATIVE, ALU_SBB, ALU_WRITES_FLAGS(ALU_SBB, is_imm()));
1756 gen_one(R_SCRATCH_2);
1757 gen_one(R_SCRATCH_2);
1758 gen_imm_offset();
1759 #else
1760 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NGC, R_SCRATCH_2, R_SCRATCH_2, (mode == MODE_INT)));
1761 #endif
1762 }
1763 if (mode == MODE_INT) {
1764 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_O, 0);
1765 gen_four(label_ovf);
1766 }
1767 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_1, R_SCRATCH_2));
1768 return true;
1769 }
1770 if ((arch_use_flags || ARCH_SUPPORTS_TRAPS) && slot_1 == slot_r && i_size_cmp(op_size) == op_size + zero) {
1771 struct cg_exit *ce;
1772 unsigned undo_alu = alu == ALU1_INC ? ALU1_DEC : alu == ALU1_DEC ? ALU1_INC : alu;
1773 if (slot_is_register(ctx, slot_1)) {
1774 unsigned reg = ctx->registers[slot_1];
1775 if (mode == MODE_INT && ARCH_SUPPORTS_TRAPS) {
1776 gen_insn(INSN_ALU1_TRAP, op_size, alu, ALU1_WRITES_FLAGS(alu));
1777 gen_one(reg);
1778 gen_one(reg);
1779 if (ARCH_TRAP_BEFORE || alu == undo_alu) {
1780 gen_four(label_ovf);
1781 return true;
1782 } else {
1783 ce = alloc_undo_label(ctx);
1784 if (unlikely(!ce))
1785 return false;
1786 gen_four(ce->undo_label);
1787 goto do_undo_opcode;
1788 }
1789 }
1790 g(gen_2address_alu1(ctx, i_size(op_size), alu, reg, reg, mode == MODE_INT));
1791 if (mode == MODE_INT) {
1792 if (alu != undo_alu) {
1793 ce = alloc_undo_label(ctx);
1794 if (unlikely(!ce))
1795 return false;
1796 gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
1797 gen_four(ce->undo_label);
1798 do_undo_opcode:
1799 ce->undo_opcode = INSN_ALU1 + ARCH_PARTIAL_ALU(op_size);
1800 ce->undo_op_size = i_size(op_size);
1801 ce->undo_aux = undo_alu;
1802 ce->undo_writes_flags = ALU1_WRITES_FLAGS(undo_alu);
1803 ce->undo_parameters[0] = reg;
1804 ce->undo_parameters[1] = reg;
1805 ce->undo_parameters_len = 2;
1806 } else {
1807 gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
1808 gen_four(label_ovf);
1809 }
1810 }
1811 return true;
1812 }
1813 #if defined(ARCH_X86)
1814 else {
1815 size_t m;
1816 int64_t offset = (size_t)slot_1 * slot_size;
1817 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDR_OFFSET, i_size(op_size)));
1818 gen_insn(INSN_ALU1 + ARCH_PARTIAL_ALU(op_size), i_size(op_size), alu, ALU1_WRITES_FLAGS(alu) | (mode == MODE_INT));
1819 gen_address_offset();
1820 gen_address_offset();
1821 if (mode == MODE_INT) {
1822 if (alu != undo_alu) {
1823 ce = alloc_undo_label(ctx);
1824 if (unlikely(!ce))
1825 return false;
1826 ce->undo_opcode = INSN_ALU1 + ARCH_PARTIAL_ALU(op_size);
1827 ce->undo_op_size = i_size(op_size);
1828 ce->undo_aux = undo_alu;
1829 ce->undo_writes_flags = ALU1_WRITES_FLAGS(undo_alu);
1830 m = mark_params(ctx);
1831 gen_address_offset();
1832 gen_address_offset();
1833 copy_params(ctx, ce, m);
1834 gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
1835 gen_four(ce->undo_label);
1836 } else {
1837 gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
1838 gen_four(label_ovf);
1839 }
1840 }
1841 return true;
1842 }
1843 #endif
1844 }
1845 target = gen_frame_target(ctx, slot_r, mode == MODE_INT ? slot_1 : NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1846 if (mode == MODE_FIXED) {
1847 ex = garbage;
1848 } else {
1849 ex = ARCH_PREFERS_SX(op_size) ? sign_x : zero_x;
1850 if (ARCH_SUPPORTS_TRAPS && op_size >= OP_SIZE_4)
1851 ex = garbage;
1852 if (op_size == i_size(op_size) + (unsigned)zero)
1853 ex = garbage;
1854 }
1855 g(gen_frame_get(ctx, op_size, ex, slot_1, mode == MODE_INT ? R_SCRATCH_2 : target, ®1));
1856 #if defined(ARCH_S390)
1857 if (alu == ALU1_NOT) {
1858 g(gen_3address_alu_imm(ctx, i_size(op_size), ALU_XOR, target, reg1, -1, 0));
1860 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1861 return true;
1862 }
1863 #endif
1864 #if defined(ARCH_X86)
1865 g(gen_2address_alu1(ctx, op_size, alu, target, reg1, mode == MODE_INT));
1866 #else
1867 if (mode == MODE_INT) {
1868 #if defined(ARCH_POWER)
1869 if (op_size == OP_SIZE_NATIVE) {
1870 g(gen_2address_alu1(ctx, i_size(op_size), alu, target, reg1, 0));
1871 if (alu == ALU1_NEG) {
1872 g(gen_3address_alu(ctx, i_size(op_size), ALU_AND, R_CG_SCRATCH, target, reg1, 1));
1873 } else if (alu == ALU1_INC) {
1874 g(gen_3address_alu(ctx, i_size(op_size), ALU_ANDN, R_CG_SCRATCH, target, reg1, 1));
1875 } else if (alu == ALU1_DEC) {
1876 g(gen_3address_alu(ctx, i_size(op_size), ALU_ANDN, R_CG_SCRATCH, reg1, target, 1));
1877 }
1878 gen_insn(INSN_JMP_COND, op_size, COND_L, 0);
1879 gen_four(label_ovf);
1881 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1883 return true;
1884 }
1885 #endif
1886 if (ARCH_HAS_JMP_2REGS(COND_L) && ARCH_HAS_JMP_2REGS(COND_G) && (alu == ALU1_INC || alu == ALU1_DEC) && op_size == i_size_cmp(op_size) + (unsigned)zero) {
1887 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, alu, target, reg1, 0));
1889 g(gen_cmp_test_jmp(ctx, INSN_CMP, i_size_cmp(op_size), target, reg1, alu == ALU1_INC ? COND_L : COND_G, label_ovf));
1891 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1893 return true;
1894 }
1895 if (!arch_use_flags && !ARCH_SUPPORTS_TRAPS && ARCH_HAS_ANDN && op_size >= OP_SIZE_4) {
1896 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, alu, target, reg1, 0));
1898 if (alu == ALU1_NEG) {
1899 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_AND, R_SCRATCH_3, target, reg1, 0));
1900 } else if (alu == ALU1_INC) {
1901 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_ANDN, R_SCRATCH_3, target, reg1, 0));
1902 } else if (alu == ALU1_DEC) {
1903 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_ANDN, R_SCRATCH_3, reg1, target, 0));
1904 }
1906 if (op_size < OP_SIZE_NATIVE)
1907 g(gen_extend(ctx, op_size, sign_x, R_SCRATCH_3, R_SCRATCH_3));
1909 g(gen_jmp_on_zero(ctx, OP_SIZE_NATIVE, R_SCRATCH_3, COND_S, label_ovf));
1911 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1913 return true;
1914 }
1915 if (op_size <= OP_SIZE_2 || (!arch_use_flags && !ARCH_SUPPORTS_TRAPS)) {
1916 int64_t imm = ((alu != ALU1_INC && ARCH_PREFERS_SX(op_size) ? -0x80ULL : 0x80ULL) << (((1 << op_size) - 1) * 8)) - (alu == ALU1_INC);
1918 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, i_size_cmp(op_size), reg1, imm, COND_E, label_ovf));
1920 mode = MODE_FIXED;
1922 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1923 }
1924 }
1925 #if !ARCH_HAS_FLAGS
1926 if (mode == MODE_INT) {
1927 gen_insn(INSN_ALU1_TRAP, op_size, alu, ALU1_WRITES_FLAGS(alu));
1928 gen_one(target);
1929 gen_one(reg1);
1930 gen_four(label_ovf);
1931 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1932 return true;
1933 }
1934 #endif
1935 g(gen_2address_alu1(ctx, i_size(op_size), alu, target, reg1, mode == MODE_INT));
1936 #endif
1937 if (mode == MODE_INT) {
1938 gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
1939 gen_four(label_ovf);
1940 }
1941 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1942 return true;
1943 }
1945 /*******
1946 * NOT *
1947 *******/
1948 do_bool_not: {
1949 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1950 g(gen_frame_get(ctx, op_size, garbage, slot_1, target, ®1));
1952 g(gen_3address_alu_imm(ctx, i_size(op_size), ALU_XOR, target, reg1, 1, 0));
1954 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1955 return true;
1956 }
1958 /*********
1959 * BSWAP *
1960 *********/
1961 do_bswap: {
1962 bool attr_unused sx = false;
1963 #if defined(ARCH_X86) || defined(ARCH_ARM) || defined(ARCH_IA64) || defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS) || defined(ARCH_RISCV64) || defined(ARCH_S390)
1964 #if defined(ARCH_ARM32)
1965 if (unlikely(!cpu_test_feature(CPU_FEATURE_armv6)))
1966 goto do_generic_bswap;
1967 #endif
1968 #if defined(ARCH_MIPS)
1969 if (unlikely(!MIPS_HAS_ROT))
1970 goto do_generic_bswap;
1971 sx = op_size == OP_SIZE_4;
1972 #endif
1973 #if defined(ARCH_RISCV64)
1974 if (unlikely(!cpu_test_feature(CPU_FEATURE_zbb)))
1975 goto do_generic_bswap;
1976 #endif
1977 #if defined(ARCH_S390)
1978 if (op_size == OP_SIZE_2)
1979 goto do_generic_bswap;
1980 #endif
1981 #if defined(ARCH_X86)
1982 if (op_size >= OP_SIZE_4 && !cpu_test_feature(CPU_FEATURE_bswap))
1983 goto do_generic_bswap;
1984 #endif
1985 if (op_size > OP_SIZE_NATIVE) {
1986 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
1987 reg1 = R_SCRATCH_1;
1988 target = R_SCRATCH_1;
1989 } else {
1990 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1991 g(gen_frame_get(ctx, op_size, sx ? sign_x : garbage, slot_1, target, ®1));
1992 }
1994 if (op_size == OP_SIZE_1) {
1995 #if defined(ARCH_IA64) || defined(ARCH_RISCV64)
1996 } else if (op_size == OP_SIZE_2 || op_size == OP_SIZE_4) {
1997 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_BSWAP, target, reg1, 0));
1999 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SAR, target, target, op_size == OP_SIZE_2 ? 48 : 32, 0));
2000 #endif
2001 } else if (op_size == OP_SIZE_2) {
2002 #if defined(ARCH_X86)
2003 g(gen_3address_rot_imm(ctx, OP_SIZE_2, ROT_ROR, target, reg1, 8, 0));
2004 #else
2005 g(gen_2address_alu1(ctx, OP_SIZE_4, ALU1_BSWAP16, target, reg1, 0));
2006 #endif
2007 } else {
2008 g(gen_2address_alu1(ctx, minimum(op_size, OP_SIZE_NATIVE), ALU1_BSWAP, target, reg1, 0));
2009 }
2010 if (op_size > OP_SIZE_NATIVE) {
2011 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_BSWAP, R_SCRATCH_2, R_SCRATCH_2, 0));
2012 }
2014 if (op_size > OP_SIZE_NATIVE)
2015 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_2, R_SCRATCH_1));
2016 else
2017 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2018 return true;
2019 #endif
2020 goto do_generic_bswap;
2021 do_generic_bswap:
2022 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_unary_bswap_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, 0);
2023 }
2024 /********
2025 * BREV *
2026 ********/
2027 do_brev: {
2028 #if defined(ARCH_ARM) || defined(ARCH_LOONGARCH64) || (defined(ARCH_MIPS) && MIPS_R6)
2029 #if defined(ARCH_ARM32)
2030 if (unlikely(!cpu_test_feature(CPU_FEATURE_armv6t2)))
2031 goto do_generic_brev;
2032 #endif
2033 if (op_size > OP_SIZE_NATIVE) {
2034 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
2035 reg1 = R_SCRATCH_1;
2036 target = R_SCRATCH_1;
2037 } else {
2038 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
2039 g(gen_frame_get(ctx, op_size, garbage, slot_1, target, ®1));
2040 }
2042 g(gen_2address_alu1(ctx, minimum(maximum(OP_SIZE_4, op_size), OP_SIZE_NATIVE), ALU1_BREV, target, reg1, 0));
2043 if (op_size <= OP_SIZE_2) {
2044 g(gen_3address_rot_imm(ctx, OP_SIZE_4, ROT_SHR, target, target, op_size == OP_SIZE_1 ? 24 : 16, 0));
2045 }
2046 if (op_size > OP_SIZE_NATIVE) {
2047 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_BREV, R_SCRATCH_2, R_SCRATCH_2, 0));
2048 }
2050 if (op_size > OP_SIZE_NATIVE)
2051 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_2, R_SCRATCH_1));
2052 else
2053 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2054 return true;
2055 #endif
2056 goto do_generic_brev;
2057 do_generic_brev:
2058 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_unary_brev_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, 0);
2059 }
2060 /******************
2061 * BSF/BSR/POPCNT *
2062 ******************/
2063 do_bsf_bsr_popcnt: {
2064 if (op_size > OP_SIZE_NATIVE) {
2065 #if defined(ARCH_X86)
2066 uint32_t label_finish = 0; /* avoid warning */
2067 if (alu == ALU1_POPCNT && unlikely(!cpu_test_feature(CPU_FEATURE_popcnt)))
2068 goto do_generic_bsf_bsr_popcnt;
2069 if (alu == ALU1_BSR || alu == ALU1_POPCNT) {
2070 if (mode == MODE_INT) {
2071 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
2072 g(gen_imm(ctx, 0, IMM_PURPOSE_STORE_VALUE, OP_SIZE_NATIVE));
2073 gen_insn(INSN_CMP, OP_SIZE_NATIVE, 0, 1);
2074 gen_address_offset();
2075 gen_imm_offset();
2077 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_S, 0);
2078 gen_four(label_ovf);
2079 }
2080 }
2081 if (alu == ALU1_BSF) {
2082 label_finish = alloc_label(ctx);
2083 if (unlikely(!label_finish))
2084 return false;
2086 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + lo_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
2087 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_BSF, ALU1_WRITES_FLAGS(ALU1_BSF));
2088 gen_one(R_SCRATCH_1);
2089 gen_address_offset();
2091 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_NE, 0);
2092 gen_four(label_finish);
2094 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
2095 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_BSF, ALU1_WRITES_FLAGS(ALU1_BSF));
2096 gen_one(R_SCRATCH_1);
2097 gen_address_offset();
2099 g(gen_3address_alu_imm(ctx, OP_SIZE_4, ALU_ADD, R_SCRATCH_1, R_SCRATCH_1, 8U << OP_SIZE_NATIVE, 0));
2100 }
2101 if (alu == ALU1_BSR) {
2102 label_finish = alloc_label(ctx);
2103 if (unlikely(!label_finish))
2104 return false;
2106 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
2107 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_BSR, ALU1_WRITES_FLAGS(ALU1_BSR));
2108 gen_one(R_SCRATCH_1);
2109 gen_address_offset();
2111 g(gen_3address_alu_imm(ctx, OP_SIZE_4, ALU_ADD, R_SCRATCH_1, R_SCRATCH_1, 8U << OP_SIZE_NATIVE, 0));
2113 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_NE, 0);
2114 gen_four(label_finish);
2116 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + lo_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
2117 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_BSR, ALU1_WRITES_FLAGS(ALU1_BSR));
2118 gen_one(R_SCRATCH_1);
2119 gen_address_offset();
2120 }
2121 if (alu == ALU1_BSF || alu == ALU1_BSR) {
2122 if (mode == MODE_INT) {
2123 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_E, 0);
2124 gen_four(label_ovf);
2125 } else {
2126 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_NE, 0);
2127 gen_four(label_finish);
2129 g(gen_load_constant(ctx, R_SCRATCH_1, -1));
2130 }
2132 gen_label(label_finish);
2134 if (mode == MODE_INT)
2135 goto write_result;
2137 if (R_SCRATCH_1 != R_AX || R_SCRATCH_2 != R_DX)
2138 internal(file_line, "gen_alu1: bad scratch registers");
2139 gen_insn(INSN_CWD, OP_SIZE_NATIVE, 0, 0);
2140 gen_one(R_DX);
2141 gen_one(R_AX);
2143 g(gen_address(ctx, R_FRAME, (size_t)slot_r * slot_size + lo_word(OP_SIZE_NATIVE), IMM_PURPOSE_STR_OFFSET, OP_SIZE_NATIVE));
2144 gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
2145 gen_address_offset();
2146 gen_one(R_SCRATCH_1);
2148 g(gen_address(ctx, R_FRAME, (size_t)slot_r * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_STR_OFFSET, OP_SIZE_NATIVE));
2149 gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
2150 gen_address_offset();
2151 gen_one(R_SCRATCH_2);
2153 return true;
2154 }
2155 if (alu == ALU1_POPCNT) {
2156 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + lo_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
2157 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_POPCNT, ALU1_WRITES_FLAGS(ALU1_POPCNT));
2158 gen_one(R_SCRATCH_1);
2159 gen_address_offset();
2161 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
2162 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_POPCNT, ALU1_WRITES_FLAGS(ALU1_POPCNT));
2163 gen_one(R_SCRATCH_2);
2164 gen_address_offset();
2166 g(gen_3address_alu(ctx, OP_SIZE_4, ALU_ADD, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_2, 1));
2167 write_result:
2168 g(gen_address(ctx, R_FRAME, (size_t)slot_r * slot_size + lo_word(OP_SIZE_NATIVE), IMM_PURPOSE_STR_OFFSET, OP_SIZE_NATIVE));
2169 gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
2170 gen_address_offset();
2171 gen_one(R_SCRATCH_1);
2173 g(gen_address(ctx, R_FRAME, (size_t)slot_r * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_STR_OFFSET, OP_SIZE_NATIVE));
2174 g(gen_imm(ctx, 0, IMM_PURPOSE_STORE_VALUE, OP_SIZE_NATIVE));
2175 gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
2176 gen_address_offset();
2177 gen_imm_offset();
2179 return true;
2180 }
2181 #endif
2182 goto do_generic_bsf_bsr_popcnt;
2183 }
2184 #if defined(ARCH_X86)
2185 if (alu == ALU1_POPCNT && unlikely(!cpu_test_feature(CPU_FEATURE_popcnt)))
2186 goto do_generic_bsf_bsr_popcnt;
2187 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
2188 if (op_size == OP_SIZE_1 || ((alu == ALU1_BSR || alu == ALU1_POPCNT) && mode == MODE_INT)) {
2189 g(gen_frame_get(ctx, op_size, zero_x, slot_1, target, ®1));
2190 if ((alu == ALU1_BSR || alu == ALU1_POPCNT) && mode == MODE_INT) {
2191 g(gen_cmp_test_jmp(ctx, INSN_TEST, op_size, reg1, reg1, alu == ALU1_BSR ? COND_LE : COND_S, label_ovf));
2192 }
2193 g(gen_2address_alu1(ctx, maximum(op_size, OP_SIZE_2), alu, target, reg1, 1));
2194 if ((alu == ALU1_BSR || alu == ALU1_POPCNT) && mode == MODE_INT)
2195 goto x86_bsf_bsr_popcnt_finish;
2196 } else {
2197 g(gen_frame_load_op1(ctx, op_size, alu, 1, slot_1, target));
2198 }
2199 if (alu == ALU1_POPCNT)
2200 goto x86_bsf_bsr_popcnt_finish;
2201 if (mode == MODE_FIXED) {
2202 uint32_t cmov_label;
2203 gen_insn(INSN_MOV, maximum(op_size, OP_SIZE_4), 0, 0);
2204 gen_one(R_SCRATCH_2);
2205 gen_one(ARG_IMM);
2206 gen_eight(-1);
2207 g(gen_cmov(ctx, maximum(op_size, OP_SIZE_4), COND_E, target, &cmov_label));
2208 gen_one(R_SCRATCH_2);
2209 if (cmov_label)
2210 gen_label(cmov_label);
2212 } else {
2213 gen_insn(INSN_JMP_COND, maximum(op_size, OP_SIZE_2), COND_E, 0);
2214 gen_four(label_ovf);
2215 }
2216 x86_bsf_bsr_popcnt_finish:
2217 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2218 return true;
2219 #endif
2220 #if defined(ARCH_ARM)
2221 #if defined(ARCH_ARM32)
2222 if (alu == ALU1_BSR && unlikely(!cpu_test_feature(CPU_FEATURE_armv6)))
2223 goto do_generic_bsf_bsr_popcnt;
2224 if (alu == ALU1_BSF && unlikely(!cpu_test_feature(CPU_FEATURE_armv6t2)))
2225 goto do_generic_bsf_bsr_popcnt;
2226 #endif
2227 if (alu == ALU1_POPCNT && unlikely(!cpu_test_feature(CPU_FEATURE_neon)))
2228 goto do_generic_bsf_bsr_popcnt;
2229 g(gen_frame_get(ctx, op_size, mode == MODE_INT ? sign_x : zero_x, slot_1, R_SCRATCH_1, ®1));
2230 if (mode == MODE_INT) {
2231 g(gen_cmp_test_jmp(ctx, INSN_TEST, i_size(op_size), reg1, reg1, alu == ALU1_BSR ? COND_LE : alu == ALU1_BSF ? COND_E : COND_S, label_ovf));
2232 }
2234 if (alu == ALU1_POPCNT) {
2235 g(gen_mov(ctx, OP_SIZE_NATIVE, FR_SCRATCH_1, reg1));
2236 gen_insn(INSN_FP_ALU1, OP_SIZE_NATIVE, FP_ALU1_VCNT8, 0);
2237 gen_one(FR_SCRATCH_1);
2238 gen_one(FR_SCRATCH_1);
2239 #if defined(ARCH_ARM32)
2240 if (op_size > OP_SIZE_1) {
2241 gen_insn(INSN_FP_ALU1, OP_SIZE_1, FP_ALU1_VPADDL, 0);
2242 gen_one(FR_SCRATCH_1);
2243 gen_one(FR_SCRATCH_1);
2244 }
2245 if (op_size > OP_SIZE_2) {
2246 gen_insn(INSN_FP_ALU1, OP_SIZE_2, FP_ALU1_VPADDL, 0);
2247 gen_one(FR_SCRATCH_1);
2248 gen_one(FR_SCRATCH_1);
2249 }
2250 #else
2251 if (op_size > OP_SIZE_1) {
2252 gen_insn(INSN_FP_ALU1, OP_SIZE_1, FP_ALU1_ADDV, 0);
2253 gen_one(FR_SCRATCH_1);
2254 gen_one(FR_SCRATCH_1);
2255 }
2256 #endif
2257 g(gen_frame_store(ctx, op_size, slot_r, 0, FR_SCRATCH_1));
2258 if (slot_is_register(ctx, slot_r))
2259 g(unspill(ctx, slot_r));
2260 return true;
2261 }
2263 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
2264 if (mode == MODE_FIXED && alu == ALU1_BSF) {
2265 gen_insn(INSN_TEST, i_size(op_size), 0, 1);
2266 gen_one(target);
2267 gen_one(reg1);
2268 reg1 = target;
2269 }
2271 if (alu == ALU1_BSF) {
2272 g(gen_2address_alu1(ctx, i_size(op_size), ALU1_BREV, target, reg1, 0));
2273 reg1 = target;
2274 }
2276 g(gen_2address_alu1(ctx, i_size(op_size), ALU1_LZCNT, target, reg1, 0));
2278 if (alu == ALU1_BSR) {
2279 g(gen_load_constant(ctx, R_SCRATCH_2, op_size == OP_SIZE_8 ? 63 : 31));
2280 g(gen_3address_alu(ctx, i_size(op_size), ALU_SUB, target, R_SCRATCH_2, target, 0));
2281 }
2283 if (mode == MODE_FIXED && alu == ALU1_BSF) {
2284 #if defined(ARCH_ARM32)
2285 g(gen_imm(ctx, -1, IMM_PURPOSE_CMOV, OP_SIZE_NATIVE));
2286 gen_insn(INSN_CMOV, OP_SIZE_NATIVE, COND_E, 0);
2287 gen_one(target);
2288 gen_one(target);
2289 gen_imm_offset();
2290 #else
2291 gen_insn(INSN_CSEL_INV, i_size(op_size), COND_NE, 0);
2292 gen_one(target);
2293 gen_one(ARG_IMM);
2294 gen_eight(0);
2295 gen_one(target);
2296 #endif
2297 }
2299 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2300 return true;
2301 #endif
2302 #if defined(ARCH_ALPHA)
2303 if (likely(cpu_test_feature(CPU_FEATURE_cix))) {
2304 g(gen_frame_get(ctx, op_size, mode == MODE_INT ? sign_x : zero_x, slot_1, R_SCRATCH_1, ®1));
2305 target = gen_frame_target(ctx, slot_r, slot_1, NO_FRAME_T, R_SCRATCH_2);
2306 if (mode == MODE_INT) {
2307 g(gen_cmp_test_jmp(ctx, INSN_TEST, OP_SIZE_NATIVE, reg1, reg1, alu == ALU1_BSR ? COND_LE : alu == ALU1_BSF ? COND_E : COND_S, label_ovf));
2308 }
2309 if (alu == ALU1_POPCNT) {
2310 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_POPCNT, target, reg1, 0));
2311 }
2312 if (alu == ALU1_BSF) {
2313 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_BSF, target, reg1, 0));
2315 if (mode == MODE_FIXED) {
2316 g(gen_imm(ctx, -1, IMM_PURPOSE_MOVR, OP_SIZE_INT));
2317 gen_insn(INSN_MOVR, OP_SIZE_NATIVE, COND_E, 0);
2318 gen_one(target);
2319 gen_one(target);
2320 gen_one(reg1);
2321 gen_imm_offset();
2322 }
2323 }
2324 if (alu == ALU1_BSR) {
2325 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_LZCNT, target, reg1, 0));
2327 g(gen_load_constant(ctx, R_SCRATCH_3, OP_SIZE_NATIVE == OP_SIZE_8 ? 63 : 31));
2329 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_SUB, target, R_SCRATCH_3, target, 0));
2330 }
2331 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2332 return true;
2333 }
2334 #endif
2335 #if defined(ARCH_MIPS)
2336 if (MIPS_HAS_CLZ && alu != ALU1_POPCNT) {
2337 g(gen_frame_get(ctx, op_size, mode == MODE_INT ? sign_x : zero_x, slot_1, R_SCRATCH_1, ®1));
2338 target = gen_frame_target(ctx, slot_r, slot_1, NO_FRAME_T, R_SCRATCH_2);
2339 if (mode == MODE_INT) {
2340 g(gen_cmp_test_jmp(ctx, INSN_TEST, OP_SIZE_NATIVE, reg1, reg1, alu == ALU1_BSR ? COND_LE : alu == ALU1_BSF ? COND_E : COND_S, label_ovf));
2341 }
2342 if (alu == ALU1_BSF) {
2343 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NEG, target, reg1, 0));
2345 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_AND, R_SCRATCH_1, reg1, target, 0));
2346 reg1 = R_SCRATCH_1;
2347 }
2348 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_LZCNT, target, reg1, 0));
2350 g(gen_load_constant(ctx, R_SCRATCH_3, OP_SIZE_NATIVE == OP_SIZE_8 ? 63 : 31));
2352 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_SUB, target, R_SCRATCH_3, target, 0));
2354 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2355 return true;
2356 }
2357 #endif
2358 #if defined(ARCH_POWER)
2359 if (alu == ALU1_BSF && (unlikely(!cpu_test_feature(CPU_FEATURE_v203)) || unlikely(!cpu_test_feature(CPU_FEATURE_v30))))
2360 goto do_generic_bsf_bsr_popcnt;
2361 if (alu == ALU1_POPCNT && unlikely(!cpu_test_feature(CPU_FEATURE_v206)))
2362 goto do_generic_bsf_bsr_popcnt;
2363 g(gen_frame_get(ctx, op_size, mode == MODE_INT ? sign_x : zero_x, slot_1, R_SCRATCH_1, ®1));
2364 target = gen_frame_target(ctx, slot_r, slot_1, NO_FRAME_T, R_SCRATCH_2);
2365 if (mode == MODE_INT) {
2366 g(gen_cmp_test_jmp(ctx, INSN_TEST, i_size(op_size), reg1, reg1, alu == ALU1_BSR ? COND_LE : alu == ALU1_BSF ? COND_E : COND_S, label_ovf));
2367 }
2368 if (alu == ALU1_POPCNT) {
2369 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_POPCNT, target, reg1, 0));
2370 }
2371 if (alu == ALU1_BSF) {
2372 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_BSF, target, reg1, 0));
2374 if (mode == MODE_FIXED) {
2375 g(gen_3address_alu(ctx, i_size(op_size), ALU_AND, R_SCRATCH_3, reg1, reg1, 1));
2377 g(gen_imm(ctx, -1, IMM_PURPOSE_CMOV, OP_SIZE_NATIVE));
2378 gen_insn(INSN_CMOV, OP_SIZE_NATIVE, COND_E, 0);
2379 gen_one(target);
2380 gen_one(target);
2381 gen_imm_offset();
2382 }
2383 }
2384 if (alu == ALU1_BSR) {
2385 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_LZCNT, target, reg1, 0));
2387 g(gen_load_constant(ctx, R_SCRATCH_3, OP_SIZE_NATIVE == OP_SIZE_8 ? 63 : 31));
2389 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_SUB, target, R_SCRATCH_3, target, 0));
2390 }
2391 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2392 return true;
2393 #endif
2394 #if defined(ARCH_LOONGARCH64) || defined(ARCH_RISCV64)
2395 #if defined(ARCH_LOONGARCH64)
2396 if (alu == ALU1_POPCNT)
2397 goto do_generic_bsf_bsr_popcnt;
2398 #endif
2399 #if defined(ARCH_RISCV64)
2400 if (unlikely(!cpu_test_feature(CPU_FEATURE_zbb)))
2401 goto do_generic_bsf_bsr_popcnt;
2402 #endif
2403 g(gen_frame_get(ctx, op_size, mode == MODE_INT ? sign_x : zero_x, slot_1, R_SCRATCH_1, ®1));
2404 target = gen_frame_target(ctx, slot_r, slot_1, NO_FRAME_T, R_SCRATCH_2);
2405 if (mode == MODE_INT) {
2406 g(gen_cmp_test_jmp(ctx, INSN_TEST, OP_SIZE_NATIVE, reg1, reg1, alu == ALU1_BSR ? COND_LE : alu == ALU1_BSF ? COND_E : COND_S, label_ovf));
2407 }
2408 if (alu == ALU1_POPCNT) {
2409 g(gen_2address_alu1(ctx, maximum(OP_SIZE_4, op_size), ALU1_POPCNT, target, reg1, 0));
2410 }
2411 if (alu == ALU1_BSF) {
2412 g(gen_2address_alu1(ctx, maximum(OP_SIZE_4, op_size), ALU1_BSF, target, reg1, 0));
2414 if (mode == MODE_FIXED) {
2415 g(gen_imm(ctx, 1, IMM_PURPOSE_CMP, OP_SIZE_NATIVE));
2416 gen_insn(INSN_CMP_DEST_REG, OP_SIZE_NATIVE, COND_B, 0);
2417 gen_one(R_SCRATCH_3);
2418 gen_one(reg1);
2419 gen_imm_offset();
2421 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NEG, R_SCRATCH_3, R_SCRATCH_3, 0));
2423 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_OR, target, target, R_SCRATCH_3, 0));
2424 }
2425 }
2426 if (alu == ALU1_BSR) {
2427 g(gen_2address_alu1(ctx, maximum(OP_SIZE_4, op_size), ALU1_LZCNT, target, reg1, 0));
2429 g(gen_load_constant(ctx, R_SCRATCH_3, op_size <= OP_SIZE_4 ? 31 : 63));
2431 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_SUB, target, R_SCRATCH_3, target, 0));
2432 }
2433 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2434 return true;
2435 #endif
2436 #if defined(ARCH_IA64) || defined(ARCH_S390) || defined(ARCH_SPARC)
2437 if (alu == ALU1_BSF && !ARCH_HAS_ANDN)
2438 goto do_generic_bsf_bsr_popcnt;
2439 #if defined(ARCH_S390)
2440 if (!cpu_test_feature(CPU_FEATURE_misc_45) || !cpu_test_feature(CPU_FEATURE_misc_insn_ext_3))
2441 goto do_generic_bsf_bsr_popcnt;
2442 #endif
2443 #if defined(ARCH_SPARC)
2444 if (!SPARC_9)
2445 goto do_generic_bsf_bsr_popcnt;
2446 #endif
2447 g(gen_frame_get(ctx, op_size, mode == MODE_INT ? sign_x : zero_x, slot_1, R_SCRATCH_1, ®1));
2448 target = gen_frame_target(ctx, slot_r, slot_1, NO_FRAME_T, R_SCRATCH_2);
2449 if (mode == MODE_INT) {
2450 g(gen_cmp_test_jmp(ctx, INSN_TEST, maximum(op_size, OP_SIZE_4), reg1, reg1, alu == ALU1_BSR ? COND_LE : alu == ALU1_BSF ? COND_E : COND_S, label_ovf));
2451 } else {
2452 if (ARCH_PREFERS_SX(op_size) && alu == ALU1_POPCNT && op_size < OP_SIZE_NATIVE) {
2453 g(gen_extend(ctx, op_size, zero_x, R_SCRATCH_1, reg1));
2454 reg1 = R_SCRATCH_1;
2455 }
2456 }
2457 if (alu == ALU1_POPCNT) {
2458 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_POPCNT, R_SCRATCH_1, reg1, 0));
2459 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
2460 return true;
2461 }
2462 if (alu == ALU1_BSF) {
2463 g(gen_3address_alu_imm(ctx, OP_SIZE_NATIVE, ALU_SUB, target, reg1, 1, 0));
2465 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_ANDN, target, target, reg1, 0));
2467 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_POPCNT, target, target, 0));
2469 if (mode == MODE_FIXED) {
2470 unsigned attr_unused test_reg = R_SCRATCH_1;
2471 #if defined(ARCH_S390)
2472 g(gen_imm(ctx, 0, COND_IS_LOGICAL(COND_E) ? IMM_PURPOSE_CMP_LOGICAL : IMM_PURPOSE_CMP, OP_SIZE_NATIVE));
2473 gen_insn(INSN_CMP, OP_SIZE_NATIVE, 0, 1 + COND_IS_LOGICAL(COND_E));
2474 gen_one(reg1);
2475 gen_imm_offset();
2477 g(gen_imm(ctx, -1, IMM_PURPOSE_CMOV, OP_SIZE_NATIVE));
2478 gen_insn(INSN_CMOV, OP_SIZE_NATIVE, COND_E, 0);
2479 gen_one(target);
2480 gen_one(target);
2481 gen_imm_offset();
2482 #else
2483 #if defined(ARCH_IA64)
2484 g(gen_cmp_dest_reg(ctx, OP_SIZE_NATIVE, reg1, (unsigned)-1, R_CMP_RESULT, 0, COND_NE));
2485 test_reg = R_CMP_RESULT;
2486 #endif
2487 g(gen_imm(ctx, -1, IMM_PURPOSE_MOVR, OP_SIZE_NATIVE));
2488 gen_insn(INSN_MOVR, OP_SIZE_NATIVE, COND_E, 0);
2489 gen_one(target);
2490 gen_one(target);
2491 gen_one(test_reg);
2492 gen_imm_offset();
2493 #endif
2494 }
2496 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2497 return true;
2498 }
2499 #endif
2500 do_generic_bsf_bsr_popcnt:
2501 if (alu == ALU1_BSF) {
2502 if (mode == MODE_FIXED)
2503 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_unary_bsf_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, 0);
2504 else
2505 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, INT_unary_bsf_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, label_ovf);
2506 }
2507 if (alu == ALU1_BSR) {
2508 if (mode == MODE_FIXED)
2509 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_unary_bsr_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, 0);
2510 else
2511 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, INT_unary_bsr_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, label_ovf);
2512 }
2513 if (alu == ALU1_POPCNT) {
2514 if (mode == MODE_FIXED)
2515 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_unary_popcnt_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, 0);
2516 else
2517 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, INT_unary_popcnt_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, label_ovf);
2518 }
2519 }
2520 /**************
2521 * CONVERSION *
2522 **************/
2523 do_fixed_conv:
2524 do_conv: {
2525 unsigned src_op_size, dest_op_size;
2526 const struct type *src_type, *dest_type;
2527 src_type = get_type_of_local(ctx, slot_1);
2528 dest_type = get_type_of_local(ctx, slot_r);
2530 if (TYPE_TAG_IS_FIXED(src_type->tag)) {
2531 src_op_size = TYPE_TAG_IDX_FIXED(src_type->tag) >> 1;
2532 } else {
2533 src_op_size = TYPE_TAG_IDX_INT(src_type->tag);
2534 }
2536 if (TYPE_TAG_IS_FIXED(dest_type->tag)) {
2537 dest_op_size = TYPE_TAG_IDX_FIXED(dest_type->tag) >> 1;
2538 } else {
2539 dest_op_size = TYPE_TAG_IDX_INT(dest_type->tag);
2540 }
2542 if (src_op_size <= OP_SIZE_NATIVE) {
2543 g(gen_frame_get(ctx, src_op_size, sign_x, slot_1, R_SCRATCH_1, ®1));
2544 } else {
2545 #if defined(ARCH_X86)
2546 if (dest_op_size < src_op_size)
2547 g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_1, 0, false, R_SCRATCH_1));
2548 else
2549 #endif
2550 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_3));
2551 reg1 = R_SCRATCH_1;
2552 }
2554 if (dest_op_size >= src_op_size) {
2555 if (dest_op_size <= OP_SIZE_NATIVE) {
2556 g(gen_frame_store(ctx, dest_op_size, slot_r, 0, reg1));
2557 } else {
2558 if (src_op_size <= OP_SIZE_NATIVE) {
2559 #if defined(ARCH_X86)
2560 if (R_SCRATCH_1 != R_AX || R_SCRATCH_2 != R_DX)
2561 internal(file_line, "gen_alu1: bad scratch registers");
2562 if (reg1 == R_SCRATCH_1) {
2563 gen_insn(INSN_CWD, OP_SIZE_NATIVE, 0, 0);
2564 gen_one(R_DX);
2565 gen_one(R_AX);
2566 } else
2567 #endif
2568 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_2, reg1, (1U << (OP_SIZE_NATIVE + 3)) - 1, false));
2569 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, reg1, R_SCRATCH_2));
2570 } else {
2571 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, reg1, R_SCRATCH_3));
2572 }
2573 }
2574 return true;
2575 } else {
2576 if (src_op_size > OP_SIZE_NATIVE) {
2577 #if defined(ARCH_ARM)
2578 gen_insn(INSN_CMP, OP_SIZE_NATIVE, 0, 1);
2579 gen_one(R_SCRATCH_3);
2580 gen_one(ARG_SHIFTED_REGISTER);
2581 gen_one(ARG_SHIFT_ASR | ((1U << (OP_SIZE_NATIVE + 3)) - 1));
2582 gen_one(R_SCRATCH_1);
2584 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_NE, 0);
2585 gen_four(label_ovf);
2586 #elif defined(ARCH_X86)
2587 if (R_SCRATCH_1 != R_AX || R_SCRATCH_2 != R_DX)
2588 internal(file_line, "gen_alu1: bad scratch registers");
2589 gen_insn(INSN_CWD, OP_SIZE_NATIVE, 0, 0);
2590 gen_one(R_DX);
2591 gen_one(R_AX);
2593 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
2594 gen_insn(INSN_CMP, OP_SIZE_NATIVE, 0, 1);
2595 gen_one(R_SCRATCH_2);
2596 gen_address_offset();
2598 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_NE, 0);
2599 gen_four(label_ovf);
2600 #else
2601 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_2, R_SCRATCH_1, (1U << (OP_SIZE_NATIVE + 3)) - 1, 0));
2603 g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, R_SCRATCH_3, R_SCRATCH_2, COND_NE, label_ovf));
2604 #endif
2606 src_op_size = OP_SIZE_NATIVE;
2607 }
2608 if (src_op_size > dest_op_size) {
2609 g(gen_cmp_extended(ctx, OP_SIZE_NATIVE, dest_op_size, reg1, R_SCRATCH_2, label_ovf));
2610 }
2611 g(gen_frame_store(ctx, dest_op_size, slot_r, 0, reg1));
2612 return true;
2613 }
2614 }
2616 conv_uto_int: {
2617 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_uto_int_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, label_ovf);
2618 }
2620 conv_ufrom_int: {
2621 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_ufrom_int_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, label_ovf);
2622 }
2623 }
2625 static bool attr_w gen_constant(struct codegen_context *ctx, bool real, unsigned op_size, bool shrt, frame_t slot_r)
2626 {
2627 uintbig_t c;
2628 if (shrt) {
2629 c = (int16_t)get_unaligned_16(ctx->current_position);
2630 } else switch (op_size) {
2631 #define fx(n, type, utype, sz, bits) \
2632 case n: \
2633 c = (type)cat(get_unaligned_,bits)(ctx->current_position);\
2634 break;
2635 for_all_fixed(fx);
2636 #undef fx
2637 default:
2638 internal(file_line, "gen_constant: invalid type %u", op_size);
2639 }
2640 if (op_size > OP_SIZE_NATIVE) {
2641 unsigned shift = (8U << OP_SIZE_NATIVE) - 1;
2642 g(gen_frame_store_imm_raw(ctx, OP_SIZE_NATIVE, slot_r, lo_word(OP_SIZE_NATIVE), c & ((2ULL << shift) - 1)));
2643 g(gen_frame_store_imm_raw(ctx, OP_SIZE_NATIVE, slot_r, hi_word(OP_SIZE_NATIVE), c >> 1 >> shift));
2644 if (real && slot_is_register(ctx, slot_r))
2645 g(unspill(ctx, slot_r));
2646 return true;
2647 } else if (real && slot_is_register(ctx, slot_r)) {
2648 if (ARCH_HAS_FP_GP_MOV) {
2649 g(gen_load_constant(ctx, R_SCRATCH_1, c));
2650 g(gen_mov(ctx, op_size, ctx->registers[slot_r], R_SCRATCH_1));
2651 } else {
2652 g(gen_frame_store_imm_raw(ctx, op_size, slot_r, 0, c));
2653 g(unspill(ctx, slot_r));
2654 }
2655 } else {
2656 g(gen_frame_store_imm(ctx, op_size, slot_r, 0, c));
2657 }
2658 return true;
2659 }
2661 static bool attr_w gen_real_constant(struct codegen_context *ctx, const struct type *t, frame_t slot_r)
2662 {
2663 int64_t offset;
2664 if (is_power_of_2(t->size) && t->size <= sizeof(uintbig_t))
2665 return gen_constant(ctx, true, log_2(t->size), false, slot_r);
2667 g(load_function_offset(ctx, R_SCRATCH_3, offsetof(struct data, u_.function.code)));
2669 offset = (ctx->current_position - da(ctx->fn,function)->code) * sizeof(code_t);
2671 g(gen_memcpy_raw(ctx, R_FRAME, (size_t)slot_r * slot_size, R_SCRATCH_3, offset, t->size, minimum(t->align, sizeof(code_t))));
2672 if (slot_is_register(ctx, slot_r))
2673 g(unspill(ctx, slot_r));
2675 return true;
2676 }
2678 static bool attr_w gen_copy(struct codegen_context *ctx, unsigned op_size, frame_t slot_1, frame_t slot_r)
2679 {
2680 unsigned reg1;
2681 if (unlikely(op_size > OP_SIZE_NATIVE)) {
2682 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
2683 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_1, R_SCRATCH_2));
2684 return true;
2685 } else {
2686 unsigned target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
2687 g(gen_frame_get(ctx, op_size, garbage, slot_1, target, ®1));
2688 g(gen_frame_store(ctx, op_size, slot_r, 0, reg1));
2689 return true;
2690 }
2691 }
2693 static bool attr_w gen_fp_alu(struct codegen_context *ctx, unsigned real_type, unsigned op, uint32_t label_ovf, frame_t slot_1, frame_t slot_2, frame_t slot_r)
2694 {
2695 unsigned attr_unused fp_alu;
2696 size_t upc;
2697 unsigned attr_unused op_size = real_type_to_op_size(real_type);
2698 unsigned reg1, reg2, target;
2699 switch (op) {
2700 case OPCODE_REAL_OP_add:
2701 case OPCODE_REAL_OP_add_alt1:
2702 case OPCODE_REAL_OP_add_alt2: fp_alu = FP_ALU_ADD; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_add_real16_t); label_ovf = 0; goto do_alu;
2703 case OPCODE_REAL_OP_subtract:
2704 case OPCODE_REAL_OP_subtract_alt1:
2705 case OPCODE_REAL_OP_subtract_alt2: fp_alu = FP_ALU_SUB; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_subtract_real16_t); label_ovf = 0; goto do_alu;
2706 case OPCODE_REAL_OP_multiply:
2707 case OPCODE_REAL_OP_multiply_alt1:
2708 case OPCODE_REAL_OP_multiply_alt2: fp_alu = FP_ALU_MUL; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_multiply_real16_t); label_ovf = 0; goto do_alu;
2709 case OPCODE_REAL_OP_divide:
2710 case OPCODE_REAL_OP_divide_alt1:
2711 case OPCODE_REAL_OP_divide_alt2: fp_alu = FP_ALU_DIV; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_divide_real16_t); label_ovf = 0; goto do_alu;
2712 case OPCODE_REAL_OP_modulo:
2713 case OPCODE_REAL_OP_power:
2714 case OPCODE_REAL_OP_ldexp:
2715 case OPCODE_REAL_OP_atan2: upc = offsetof(struct cg_upcall_vector_s, REAL_binary_modulo_real16_t) + (op - OPCODE_REAL_OP_modulo) * TYPE_REAL_N * sizeof(void (*)(void)); goto do_upcall;
2716 case OPCODE_REAL_OP_equal:
2717 case OPCODE_REAL_OP_equal_alt1:
2718 case OPCODE_REAL_OP_equal_alt2: fp_alu = FP_COND_E; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_equal_real16_t); goto do_cmp;
2719 case OPCODE_REAL_OP_not_equal:
2720 case OPCODE_REAL_OP_not_equal_alt1:
2721 case OPCODE_REAL_OP_not_equal_alt2: fp_alu = FP_COND_NE; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_not_equal_real16_t); goto do_cmp;
2722 case OPCODE_REAL_OP_less:
2723 case OPCODE_REAL_OP_less_alt1:
2724 case OPCODE_REAL_OP_less_alt2: fp_alu = FP_COND_B; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_less_real16_t); goto do_cmp;
2725 case OPCODE_REAL_OP_less_equal:
2726 case OPCODE_REAL_OP_less_equal_alt1:
2727 case OPCODE_REAL_OP_less_equal_alt2: fp_alu = FP_COND_BE; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_less_equal_real16_t); goto do_cmp;
2728 case OPCODE_REAL_OP_greater:
2729 case OPCODE_REAL_OP_greater_alt1:
2730 case OPCODE_REAL_OP_greater_alt2: fp_alu = FP_COND_A; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_greater_real16_t); goto do_cmp;
2731 case OPCODE_REAL_OP_greater_equal:
2732 case OPCODE_REAL_OP_greater_equal_alt1:
2733 case OPCODE_REAL_OP_greater_equal_alt2: fp_alu = FP_COND_AE; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_greater_equal_real16_t); goto do_cmp;
2734 default: internal(file_line, "gen_fp_alu: unsupported operation %u", op);
2735 }
2737 do_alu:
2738 if ((SUPPORTED_FP >> real_type) & 1) {
2739 #if defined(ARCH_IA64)
2740 if (unlikely(fp_alu == FP_ALU_DIV))
2741 goto do_upcall;
2742 #endif
2743 #if defined(ARCH_X86)
2744 if (1)
2745 #elif defined(ARCH_S390)
2746 if ((op_size <= OP_SIZE_8 && (size_t)slot_2 * slot_size < 4096) || slot_is_register(ctx, slot_2))
2747 #else
2748 if (slot_is_register(ctx, slot_2))
2749 #endif
2750 {
2751 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, slot_2, FR_SCRATCH_1);
2752 g(gen_frame_get(ctx, op_size, garbage, slot_1, FR_SCRATCH_1, ®1));
2753 if (slot_is_register(ctx, slot_2)) {
2754 g(gen_3address_fp_alu(ctx, op_size, fp_alu, target, reg1, ctx->registers[slot_2]));
2755 } else {
2756 if (target != reg1 && !ARCH_IS_3ADDRESS_FP) {
2757 g(gen_mov(ctx, op_size, target, reg1));
2758 reg1 = target;
2759 }
2760 g(gen_address(ctx, R_FRAME, (size_t)slot_2 * slot_size, IMM_PURPOSE_VLDR_VSTR_OFFSET, op_size));
2761 gen_insn(INSN_FP_ALU, op_size, fp_alu, 0);
2762 gen_one(target);
2763 gen_one(reg1);
2764 gen_address_offset();
2765 }
2766 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2767 return true;
2768 }
2769 #if defined(ARCH_ALPHA)
2770 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, ®1));
2771 g(gen_frame_get(ctx, op_size, zero_x, slot_2, FR_SCRATCH_2, ®2));
2772 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, FR_SCRATCH_3);
2773 g(gen_3address_fp_alu(ctx, op_size, fp_alu, target, reg1, reg2));
2774 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2775 #else
2776 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, ®1));
2777 g(gen_frame_get(ctx, op_size, zero_x, slot_2, FR_SCRATCH_2, ®2));
2778 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_1);
2779 g(gen_3address_fp_alu(ctx, op_size, fp_alu, target, reg1, reg2));
2780 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2781 #endif
2782 return true;
2783 }
2784 #ifdef SUPPORTED_FP_X87
2785 if ((SUPPORTED_FP_X87 >> real_type) & 1) {
2786 if (real_type != 3) {
2787 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_2));
2788 g(gen_frame_load_x87(ctx, INSN_X87_ALU, op_size, fp_alu, slot_1));
2789 } else {
2790 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
2791 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_2));
2792 gen_insn(INSN_X87_ALUP, op_size, fp_alu, 0);
2793 gen_one(R_ST1);
2794 }
2795 g(gen_frame_store_x87(ctx, INSN_X87_FSTP, op_size, slot_r));
2796 return true;
2797 }
2798 #endif
2799 #ifdef SUPPORTED_FP_HALF_CVT
2800 if ((SUPPORTED_FP_HALF_CVT >> real_type) & 1) {
2801 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, ®1));
2802 g(gen_frame_get(ctx, op_size, zero_x, slot_2, FR_SCRATCH_2, ®2));
2803 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_1);
2804 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
2805 gen_one(FR_SCRATCH_1);
2806 gen_one(reg1);
2807 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
2808 gen_one(FR_SCRATCH_2);
2809 gen_one(reg2);
2810 gen_insn(INSN_FP_ALU, OP_SIZE_4, fp_alu, 0);
2811 gen_one(FR_SCRATCH_1);
2812 gen_one(FR_SCRATCH_1);
2813 gen_one(FR_SCRATCH_2);
2814 gen_insn(INSN_FP_CVT, OP_SIZE_4, op_size, 0);
2815 gen_one(target);
2816 gen_one(FR_SCRATCH_1);
2817 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2818 return true;
2819 }
2820 #endif
2821 goto do_upcall;
2823 do_cmp:
2824 if ((SUPPORTED_FP >> real_type) & 1
2825 #if defined(ARCH_ALPHA)
2826 && ARCH_SUPPORTS_TRAPS
2827 #endif
2828 ) {
2829 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, ®1));
2830 g(gen_frame_get(ctx, op_size, zero_x, slot_2, FR_SCRATCH_2, ®2));
2831 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
2832 #if defined(ARCH_ALPHA)
2833 gen_insn(INSN_FP_CMP_DEST_REG_TRAP, op_size, fp_alu == FP_COND_NE ? FP_COND_E : fp_alu, 0);
2834 gen_one(FR_SCRATCH_3);
2835 gen_one(reg1);
2836 gen_one(reg2);
2837 gen_four(label_ovf);
2839 if (!ARCH_HAS_FP_GP_MOV) {
2840 g(gen_frame_store_raw(ctx, OP_SIZE_4, slot_r, 0, FR_SCRATCH_3));
2841 g(gen_frame_load_raw(ctx, OP_SIZE_4, sign_x, slot_r, 0, false, target));
2842 } else {
2843 g(gen_mov(ctx, OP_SIZE_4, target, FR_SCRATCH_3));
2844 }
2846 if (fp_alu == FP_COND_NE) {
2847 g(gen_imm(ctx, 0, IMM_PURPOSE_CMP, OP_SIZE_NATIVE));
2848 gen_insn(INSN_CMP_DEST_REG, OP_SIZE_NATIVE, COND_E, 0);
2849 gen_one(target);
2850 gen_one(target);
2851 gen_imm_offset();
2852 } else {
2853 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SHR, target, target, 30, 0));
2854 }
2856 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
2858 return true;
2859 #elif defined(ARCH_IA64)
2860 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_P, 0);
2861 gen_one(R_CMP_RESULT);
2862 gen_one(reg1);
2863 gen_one(reg2);
2865 gen_insn(INSN_JMP_REG, OP_SIZE_NATIVE, COND_NE, 0);
2866 gen_one(R_CMP_RESULT);
2867 gen_four(label_ovf);
2869 gen_insn(INSN_FP_CMP_DEST_REG, op_size, fp_alu, 0);
2870 gen_one(R_CMP_RESULT);
2871 gen_one(reg1);
2872 gen_one(reg2);
2874 g(gen_mov(ctx, OP_SIZE_NATIVE, target, R_CMP_RESULT));
2876 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
2878 return true;
2879 #elif defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS) || defined(ARCH_PARISC)
2880 gen_insn(INSN_FP_CMP_COND, op_size, FP_COND_P, 1);
2881 gen_one(reg1);
2882 gen_one(reg2);
2884 gen_insn(INSN_JMP_FP_TEST, 0, FP_COND_P, 0);
2885 gen_four(label_ovf);
2887 gen_insn(INSN_FP_CMP_COND, op_size, fp_alu, 1);
2888 gen_one(reg1);
2889 gen_one(reg2);
2891 gen_insn(INSN_FP_TEST_REG, OP_SIZE_NATIVE, fp_alu, 0);
2892 gen_one(target);
2894 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
2896 return true;
2897 #elif defined(ARCH_RISCV64)
2898 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_E, 0);
2899 gen_one(R_SCRATCH_1);
2900 gen_one(reg1);
2901 gen_one(reg1);
2903 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_E, 0);
2904 gen_one(R_SCRATCH_2);
2905 gen_one(reg2);
2906 gen_one(reg2);
2908 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_AND, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_2, 0));
2910 g(gen_jmp_on_zero(ctx, OP_SIZE_NATIVE, R_SCRATCH_1, COND_E, label_ovf));
2912 gen_insn(INSN_FP_CMP_DEST_REG, op_size, fp_alu == FP_COND_NE ? FP_COND_E : fp_alu, 0);
2913 gen_one(target);
2914 gen_one(reg1);
2915 gen_one(reg2);
2917 if (fp_alu == FP_COND_NE) {
2918 g(gen_imm(ctx, 1, IMM_PURPOSE_XOR, OP_SIZE_NATIVE));
2919 gen_insn(INSN_ALU, OP_SIZE_NATIVE, ALU_XOR, ALU_WRITES_FLAGS(ALU_AND, false));
2920 gen_one(target);
2921 gen_one(target);
2922 gen_imm_offset();
2923 }
2925 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
2926 return true;
2927 #else
2928 gen_insn(INSN_FP_CMP, op_size, 0, 1);
2929 gen_one(reg1);
2930 gen_one(reg2);
2931 #if defined(ARCH_ARM32)
2932 gen_insn(INSN_FP_TO_INT_FLAGS, 0, 0, 1);
2933 #endif
2934 gen_insn(INSN_JMP_COND, op_size, FP_COND_P, 0);
2935 gen_four(label_ovf);
2936 g(gen_frame_set_cond(ctx, op_size, false, fp_alu, slot_r));
2937 return true;
2938 #endif
2939 }
2940 #ifdef SUPPORTED_FP_X87
2941 if ((SUPPORTED_FP_X87 >> real_type) & 1) {
2942 if (likely(cpu_test_feature(CPU_FEATURE_cmov))) {
2943 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_2));
2944 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
2945 gen_insn(INSN_X87_FCOMIP, op_size, 0, 0);
2946 gen_one(R_ST1);
2947 gen_insn(INSN_X87_FSTP, op_size, 0, 0);
2948 gen_one(R_ST0);
2949 gen_insn(INSN_JMP_COND, op_size, COND_P, 0);
2950 gen_four(label_ovf);
2951 g(gen_frame_set_cond(ctx, op_size, false, fp_alu & 0xf, slot_r));
2952 return true;
2953 }
2955 if (real_type != 3) {
2956 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
2957 g(gen_frame_load_x87(ctx, INSN_X87_FCOMP, op_size, 0, slot_2));
2958 } else {
2959 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_2));
2960 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
2961 gen_insn(INSN_X87_FCOMPP, op_size, 0, 0);
2962 }
2964 gen_insn(INSN_X87_FNSTSW, 0, 0, 0);
2965 gen_one(R_AX);
2966 gen_one(R_AX);
2968 gen_insn(INSN_TEST, OP_SIZE_2, 0, 1);
2969 gen_one(R_AX);
2970 gen_one(ARG_IMM);
2971 gen_eight(0x0400);
2973 gen_insn(INSN_JMP_COND, OP_SIZE_2, COND_NE, 0);
2974 gen_four(label_ovf);
2976 switch (fp_alu) {
2977 case FP_COND_E:
2978 gen_insn(INSN_TEST, OP_SIZE_2, 0, 1);
2979 gen_one(R_AX);
2980 gen_one(ARG_IMM);
2981 gen_eight(0x4000);
2982 g(gen_frame_set_cond(ctx, OP_SIZE_2, false, COND_NE, slot_r));
2983 break;
2984 case FP_COND_NE:
2985 gen_insn(INSN_TEST, OP_SIZE_2, 0, 1);
2986 gen_one(R_AX);
2987 gen_one(ARG_IMM);
2988 gen_eight(0x4000);
2989 g(gen_frame_set_cond(ctx, OP_SIZE_2, false, COND_E, slot_r));
2990 break;
2991 case FP_COND_B:
2992 gen_insn(INSN_TEST, OP_SIZE_2, 0, 1);
2993 gen_one(R_AX);
2994 gen_one(ARG_IMM);
2995 gen_eight(0x0100);
2996 g(gen_frame_set_cond(ctx, OP_SIZE_2, false, COND_NE, slot_r));
2997 break;
2998 case FP_COND_BE:
2999 gen_insn(INSN_TEST, OP_SIZE_2, 0, 1);
3000 gen_one(R_AX);
3001 gen_one(ARG_IMM);
3002 gen_eight(0x4100);
3003 g(gen_frame_set_cond(ctx, OP_SIZE_2, false, COND_NE, slot_r));
3004 break;
3005 default:
3006 internal(file_line, "gen_fp_alu: invalid condition %u", fp_alu);
3007 }
3008 return true;
3009 }
3010 #endif
3011 #ifdef SUPPORTED_FP_HALF_CVT
3012 if ((SUPPORTED_FP_HALF_CVT >> real_type) & 1) {
3013 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, ®1));
3014 g(gen_frame_get(ctx, op_size, zero_x, slot_2, FR_SCRATCH_2, ®2));
3015 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
3016 gen_one(FR_SCRATCH_1);
3017 gen_one(reg1);
3018 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
3019 gen_one(FR_SCRATCH_2);
3020 gen_one(reg2);
3021 gen_insn(INSN_FP_CMP, OP_SIZE_4, 0, 1);
3022 gen_one(FR_SCRATCH_1);
3023 gen_one(FR_SCRATCH_2);
3024 #if defined(ARCH_ARM32)
3025 gen_insn(INSN_FP_TO_INT_FLAGS, 0, 0, 1);
3026 #endif
3027 gen_insn(INSN_JMP_COND, op_size, FP_COND_P, 0);
3028 gen_four(label_ovf);
3029 g(gen_frame_set_cond(ctx, op_size, false, fp_alu, slot_r));
3030 return true;
3031 }
3032 #endif
3034 do_upcall:
3035 return gen_alu_typed_upcall(ctx, upc, real_type, slot_1, slot_2, slot_r, label_ovf);
3036 }
3038 #define OP_IS_ROUND(alu) ((alu) == FP_ALU1_ROUND || (alu) == FP_ALU1_FLOOR || (alu) == FP_ALU1_CEIL || (alu) == FP_ALU1_TRUNC)
3040 static bool attr_w gen_fp_alu1(struct codegen_context *ctx, unsigned real_type, unsigned op, uint32_t label_ovf, frame_t slot_1, frame_t slot_r)
3041 {
3042 unsigned attr_unused fp_alu;
3043 size_t upc;
3044 unsigned attr_unused op_size = real_type_to_op_size(real_type);
3045 unsigned reg1, target;
3046 switch (op) {
3047 case OPCODE_REAL_OP_neg:
3048 case OPCODE_REAL_OP_neg_alt1:
3049 case OPCODE_REAL_OP_neg_alt2: fp_alu = FP_ALU1_NEG; upc = offsetof(struct cg_upcall_vector_s, REAL_unary_neg_real16_t); label_ovf = 0; goto do_alu;
3050 case OPCODE_REAL_OP_sqrt:
3051 case OPCODE_REAL_OP_sqrt_alt1:
3052 case OPCODE_REAL_OP_sqrt_alt2: fp_alu = FP_ALU1_SQRT; upc = offsetof(struct cg_upcall_vector_s, REAL_unary_sqrt_real16_t); label_ovf = 0; goto do_alu;
3053 case OPCODE_REAL_OP_round: fp_alu = FP_ALU1_ROUND; upc = offsetof(struct cg_upcall_vector_s, REAL_unary_round_real16_t); label_ovf = 0; goto do_alu;
3054 case OPCODE_REAL_OP_floor: fp_alu = FP_ALU1_FLOOR; upc = offsetof(struct cg_upcall_vector_s, REAL_unary_floor_real16_t); label_ovf = 0; goto do_alu;
3055 case OPCODE_REAL_OP_ceil: fp_alu = FP_ALU1_CEIL; upc = offsetof(struct cg_upcall_vector_s, REAL_unary_ceil_real16_t); label_ovf = 0; goto do_alu;
3056 case OPCODE_REAL_OP_trunc: fp_alu = FP_ALU1_TRUNC; upc = offsetof(struct cg_upcall_vector_s, REAL_unary_trunc_real16_t); label_ovf = 0; goto do_alu;
3057 case OPCODE_REAL_OP_to_int:
3058 case OPCODE_REAL_OP_to_int_alt1:
3059 case OPCODE_REAL_OP_to_int_alt2: upc = offsetof(struct cg_upcall_vector_s, REAL_unary_to_int_real16_t); goto do_to_int;
3060 case OPCODE_REAL_OP_from_int:
3061 case OPCODE_REAL_OP_from_int_alt1:
3062 case OPCODE_REAL_OP_from_int_alt2: upc = offsetof(struct cg_upcall_vector_s, REAL_unary_from_int_real16_t); label_ovf = 0; goto do_from_int;
3063 case OPCODE_REAL_OP_is_exception:
3064 case OPCODE_REAL_OP_is_exception_alt1:
3065 case OPCODE_REAL_OP_is_exception_alt2: upc = offsetof(struct cg_upcall_vector_s, REAL_unary_is_exception_real16_t); label_ovf = 0; goto do_is_exception;
3066 default: upc = offsetof(struct cg_upcall_vector_s, REAL_unary_cbrt_real16_t) + (op - OPCODE_REAL_OP_cbrt) * TYPE_REAL_N * sizeof(void (*)(void)); label_ovf = 0; goto do_upcall;
3067 }
3069 do_alu:
3070 if ((SUPPORTED_FP >> real_type) & 1 && (
3071 #if defined(ARCH_ALPHA)
3072 fp_alu == FP_ALU1_NEG ||
3073 (fp_alu == FP_ALU1_SQRT && cpu_test_feature(CPU_FEATURE_fix)) ||
3074 #elif defined(ARCH_ARM32)
3075 fp_alu == FP_ALU1_NEG ||
3076 fp_alu == FP_ALU1_SQRT ||
3077 #elif defined(ARCH_ARM64)
3078 true ||
3079 #elif defined(ARCH_IA64)
3080 fp_alu == FP_ALU1_NEG ||
3081 #elif defined(ARCH_LOONGARCH64)
3082 fp_alu == FP_ALU1_NEG ||
3083 fp_alu == FP_ALU1_SQRT ||
3084 fp_alu == FP_ALU1_ROUND ||
3085 #elif defined(ARCH_MIPS)
3086 fp_alu == FP_ALU1_NEG ||
3087 (fp_alu == FP_ALU1_SQRT && MIPS_HAS_SQRT) ||
3088 #elif defined(ARCH_PARISC)
3089 (fp_alu == FP_ALU1_NEG && PA_20) ||
3090 fp_alu == FP_ALU1_SQRT ||
3091 #elif defined(ARCH_POWER)
3092 fp_alu == FP_ALU1_NEG ||
3093 (fp_alu == FP_ALU1_SQRT && cpu_test_feature(CPU_FEATURE_p2) && real_type != 4) ||
3094 #elif defined(ARCH_S390)
3095 true ||
3096 #elif defined(ARCH_SPARC)
3097 fp_alu == FP_ALU1_NEG ||
3098 fp_alu == FP_ALU1_SQRT ||
3099 #elif defined(ARCH_RISCV64)
3100 fp_alu == FP_ALU1_NEG ||
3101 fp_alu == FP_ALU1_SQRT ||
3102 #elif defined(ARCH_X86)
3103 fp_alu == FP_ALU1_SQRT ||
3104 (OP_IS_ROUND(fp_alu) && cpu_test_feature(CPU_FEATURE_sse41)) ||
3105 #endif
3106 false)) {
3107 #if defined(ARCH_S390)
3108 if (op_size <= OP_SIZE_8 && (size_t)slot_1 * slot_size < 4096 && fp_alu == FP_ALU1_SQRT) {
3109 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_1);
3110 if (slot_is_register(ctx, slot_1)) {
3111 gen_insn(INSN_FP_ALU1, op_size, fp_alu, 0);
3112 gen_one(target);
3113 gen_one(ctx->registers[slot_1]);
3114 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
3115 } else {
3116 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size, IMM_PURPOSE_VLDR_VSTR_OFFSET, op_size));
3117 gen_insn(INSN_FP_ALU1, op_size, fp_alu, 0);
3118 gen_one(target);
3119 gen_address_offset();
3120 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
3121 }
3122 return true;
3123 }
3124 #endif
3125 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, ®1));
3126 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_2);
3127 gen_insn(INSN_FP_ALU1, op_size, fp_alu, 0);
3128 gen_one(target);
3129 gen_one(reg1);
3130 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
3131 return true;
3132 }
3133 #ifdef SUPPORTED_FP_X87
3134 if ((SUPPORTED_FP_X87 >> real_type) & 1) {
3135 if (fp_alu == FP_ALU1_NEG) {
3136 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
3137 gen_insn(INSN_X87_FCHS, op_size, 0, 0);
3138 g(gen_frame_store_x87(ctx, INSN_X87_FSTP, op_size, slot_r));
3139 return true;
3140 } else if (fp_alu == FP_ALU1_SQRT) {
3141 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
3142 gen_insn(INSN_X87_FSQRT, op_size, 0, 0);
3143 g(gen_frame_store_x87(ctx, INSN_X87_FSTP, op_size, slot_r));
3144 return true;
3145 } else if (fp_alu == FP_ALU1_ROUND) {
3146 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
3147 gen_insn(INSN_X87_FRNDINT, op_size, 0, 0);
3148 g(gen_frame_store_x87(ctx, INSN_X87_FSTP, op_size, slot_r));
3149 return true;
3150 }
3151 }
3152 #endif
3153 #ifdef SUPPORTED_FP_HALF_CVT
3154 if ((SUPPORTED_FP_HALF_CVT >> real_type) & 1 && (
3155 #if defined(ARCH_ARM32)
3156 fp_alu == FP_ALU1_NEG ||
3157 fp_alu == FP_ALU1_SQRT ||
3158 #elif defined(ARCH_ARM64)
3159 true ||
3160 #elif defined(ARCH_X86)
3161 fp_alu == FP_ALU1_SQRT ||
3162 (OP_IS_ROUND(fp_alu) && cpu_test_feature(CPU_FEATURE_sse41)) ||
3163 #endif
3164 false)) {
3165 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, ®1));
3166 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_1);
3167 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
3168 gen_one(target);
3169 gen_one(reg1);
3170 gen_insn(INSN_FP_ALU1, OP_SIZE_4, fp_alu, 0);
3171 gen_one(target);
3172 gen_one(target);
3173 gen_insn(INSN_FP_CVT, OP_SIZE_4, op_size, 0);
3174 gen_one(target);
3175 gen_one(target);
3176 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
3177 return true;
3178 }
3179 #endif
3180 goto do_upcall;
3182 do_to_int:
3183 if ((SUPPORTED_FP >> real_type) & 1
3184 #if defined(ARCH_ALPHA)
3185 && ARCH_SUPPORTS_TRAPS
3186 #endif
3187 #if defined(ARCH_MIPS)
3188 && MIPS_HAS_TRUNC
3189 #endif
3190 ) {
3191 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, ®1));
3192 goto do_cvt_to_int;
3193 do_cvt_to_int:
3194 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
3195 #if defined(ARCH_X86)
3196 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_TO_INT32 : INSN_FP_TO_INT64, op_size, 0, 0);
3197 gen_one(target);
3198 gen_one(reg1);
3200 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, OP_SIZE_INT, target, sign_bit(uint_default_t), COND_E, label_ovf));
3202 g(gen_frame_store(ctx, OP_SIZE_INT, slot_r, 0, target));
3203 return true;
3204 #endif
3205 #if defined(ARCH_ARM) || defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS)
3206 #if defined(ARCH_ARM)
3207 gen_insn(INSN_FP_CMP, op_size, 0, 1);
3208 gen_one(reg1);
3209 gen_one(reg1);
3210 #if defined(ARCH_ARM32)
3211 gen_insn(INSN_FP_TO_INT_FLAGS, 0, 0, 1);
3212 #endif
3213 gen_insn(INSN_JMP_COND, op_size, FP_COND_P, 0);
3214 gen_four(label_ovf);
3215 #else
3216 gen_insn(INSN_FP_CMP_COND, op_size, FP_COND_P, 1);
3217 gen_one(reg1);
3218 gen_one(reg1);
3220 gen_insn(INSN_JMP_FP_TEST, 0, FP_COND_P, 0);
3221 gen_four(label_ovf);
3222 #endif
3223 #if defined(ARCH_ARM32) || defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS)
3224 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_TO_INT32 : INSN_FP_TO_INT64, op_size, 0, 0);
3225 gen_one(FR_SCRATCH_1);
3226 gen_one(reg1);
3228 g(gen_mov(ctx, OP_SIZE_INT, target, FR_SCRATCH_1));
3229 #else
3230 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_TO_INT32 : INSN_FP_TO_INT64, op_size, 0, 0);
3231 gen_one(target);
3232 gen_one(reg1);
3233 #endif
3234 g(gen_imm(ctx, (int_default_t)(sign_bit(uint_default_t) + 1), IMM_PURPOSE_ADD, OP_SIZE_INT));
3235 gen_insn(INSN_ALU, OP_SIZE_INT, ALU_ADD, ALU_WRITES_FLAGS(ALU_ADD, is_imm()));
3236 gen_one(R_SCRATCH_2);
3237 gen_one(target);
3238 gen_imm_offset();
3240 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, OP_SIZE_INT, R_SCRATCH_2, 1, COND_BE, label_ovf));
3242 g(gen_frame_store(ctx, OP_SIZE_INT, slot_r, 0, target));
3243 return true;
3244 #endif
3245 #if defined(ARCH_IA64)
3246 gen_insn(INSN_FP_TO_INT64, op_size, 0, 0);
3247 gen_one(FR_SCRATCH_1);
3248 gen_one(reg1);
3250 g(gen_mov(ctx, OP_SIZE_NATIVE, target, FR_SCRATCH_1));
3252 if (OP_SIZE_INT == OP_SIZE_4) {
3253 g(gen_extend(ctx, OP_SIZE_4, sign_x, R_SCRATCH_2, target));
3254 g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, target, R_SCRATCH_2, COND_NE, label_ovf));
3255 } else {
3256 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, target, sign_bit(uint64_t), COND_E, label_ovf));
3257 }
3259 g(gen_frame_store(ctx, OP_SIZE_INT, slot_r, 0, target));
3260 return true;
3261 #endif
3262 #if defined(ARCH_PARISC) || defined(ARCH_POWER) || defined(ARCH_SPARC)
3263 #if defined(ARCH_POWER)
3264 if (!cpu_test_feature(CPU_FEATURE_ppc))
3265 goto do_upcall;
3266 if (OP_SIZE_INT == OP_SIZE_4)
3267 goto do_upcall;
3268 #endif
3269 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_TO_INT32 : INSN_FP_TO_INT64, op_size, 0, 0);
3270 gen_one(FR_SCRATCH_1);
3271 gen_one(reg1);
3273 g(gen_frame_store_raw(ctx, OP_SIZE_INT, slot_r, 0, FR_SCRATCH_1));
3274 if (slot_is_register(ctx, slot_r))
3275 g(unspill(ctx, slot_r));
3276 g(gen_frame_load(ctx, OP_SIZE_INT, garbage, slot_r, 0, false, target));
3278 g(gen_imm(ctx, sign_bit(uint_default_t) + 1, IMM_PURPOSE_ADD, OP_SIZE_INT));
3279 gen_insn(INSN_ALU, i_size(OP_SIZE_INT), ALU_ADD, ALU_WRITES_FLAGS(ALU_ADD, is_imm()));
3280 gen_one(R_SCRATCH_2);
3281 gen_one(target);
3282 gen_imm_offset();
3284 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, OP_SIZE_INT, R_SCRATCH_2, 1, COND_BE, label_ovf));
3286 return true;
3287 #endif
3288 #if defined(ARCH_ALPHA)
3289 gen_insn(INSN_FP_TO_INT64_TRAP, op_size, 0, 0);
3290 gen_one(FR_SCRATCH_2);
3291 gen_one(reg1);
3292 gen_four(label_ovf);
3294 if (OP_SIZE_INT == OP_SIZE_4) {
3295 gen_insn(INSN_FP_INT64_TO_INT32_TRAP, 0, 0, 0);
3296 gen_one(FR_SCRATCH_3);
3297 gen_one(FR_SCRATCH_2);
3298 gen_four(label_ovf);
3299 g(gen_frame_store_raw(ctx, OP_SIZE_INT, slot_r, 0, FR_SCRATCH_3));
3300 } else {
3301 g(gen_frame_store_raw(ctx, OP_SIZE_INT, slot_r, 0, FR_SCRATCH_2));
3302 }
3303 if (slot_is_register(ctx, slot_r))
3304 g(unspill(ctx, slot_r));
3305 return true;
3306 #endif
3307 #if defined(ARCH_S390)
3308 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_TO_INT32 : INSN_FP_TO_INT64, op_size, 0, 1);
3309 gen_one(target);
3310 gen_one(reg1);
3312 gen_insn(INSN_JMP_COND, op_size, FP_COND_P, 0);
3313 gen_four(label_ovf);
3315 g(gen_frame_store(ctx, OP_SIZE_INT, slot_r, 0, target));
3316 return true;
3317 #endif
3318 #if defined(ARCH_RISCV64)
3319 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_TO_INT32 : INSN_FP_TO_INT64, op_size, 0, 0);
3320 gen_one(target);
3321 gen_one(reg1);
3323 g(gen_load_constant(ctx, R_SCRATCH_2, sign_bit(int_default_t)));
3325 g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, target, R_SCRATCH_2, COND_E, label_ovf));
3327 g(gen_imm(ctx, -1, IMM_PURPOSE_XOR, i_size(size)));
3328 gen_insn(INSN_ALU, OP_SIZE_NATIVE, ALU_XOR, ALU_WRITES_FLAGS(ALU_XOR, is_imm()));
3329 gen_one(R_SCRATCH_2);
3330 gen_one(R_SCRATCH_2);
3331 gen_imm_offset();
3333 g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, target, R_SCRATCH_2, COND_E, label_ovf));
3335 g(gen_frame_store(ctx, OP_SIZE_INT, slot_r, 0, target));
3336 return true;
3337 #endif
3338 }
3339 #ifdef SUPPORTED_FP_X87
3340 if ((SUPPORTED_FP_X87 >> real_type) & 1) {
3341 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
3343 if (likely(cpu_test_feature(CPU_FEATURE_sse3))) {
3344 g(gen_frame_store_x87(ctx, INSN_X87_FISTTP, OP_SIZE_INT, slot_r));
3345 } else {
3346 gen_insn(INSN_PUSH, OP_SIZE_NATIVE, 0, 0);
3347 gen_one(ARG_IMM);
3348 gen_eight(0x0f7f);
3350 gen_insn(INSN_X87_FLDCW, 0, 0, 0);
3351 gen_one(ARG_ADDRESS_1);
3352 gen_one(R_SP);
3353 gen_eight(0);
3355 g(gen_frame_store_x87(ctx, INSN_X87_FISTP, OP_SIZE_INT, slot_r));
3357 gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
3358 gen_one(ARG_ADDRESS_1);
3359 gen_one(R_SP);
3360 gen_eight(0);
3361 gen_one(ARG_IMM);
3362 gen_eight(0x037f);
3364 gen_insn(INSN_X87_FLDCW, 0, 0, 0);
3365 gen_one(ARG_ADDRESS_1);
3366 gen_one(R_SP);
3367 gen_eight(0);
3369 gen_insn(INSN_ALU, i_size(OP_SIZE_ADDRESS), ALU_ADD, 1);
3370 gen_one(R_SP);
3371 gen_one(R_SP);
3372 gen_one(ARG_IMM);
3373 gen_eight(1 << OP_SIZE_NATIVE);
3374 }
3375 if (slot_is_register(ctx, slot_r))
3376 g(unspill(ctx, slot_r));
3377 g(gen_frame_load(ctx, OP_SIZE_INT, garbage, slot_r, 0, false, R_SCRATCH_1));
3379 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, OP_SIZE_INT, R_SCRATCH_1, sign_bit(int_default_t), COND_E, label_ovf));
3381 return true;
3382 }
3383 #endif
3384 #ifdef SUPPORTED_FP_HALF_CVT
3385 if ((SUPPORTED_FP_HALF_CVT >> real_type) & 1) {
3386 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, ®1));
3387 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
3388 gen_one(FR_SCRATCH_1);
3389 gen_one(reg1);
3390 reg1 = FR_SCRATCH_1;
3391 real_type = 1;
3392 op_size = real_type_to_op_size(real_type);
3393 goto do_cvt_to_int;
3394 }
3395 #endif
3396 goto do_upcall;
3398 do_from_int:
3399 if ((SUPPORTED_FP >> real_type) & 1) {
3400 #if defined(ARCH_ALPHA) || defined(ARCH_ARM32) || defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS) || defined(ARCH_PARISC) || defined(ARCH_POWER) || defined(ARCH_SPARC)
3401 int int_op_size = OP_SIZE_INT;
3402 #if defined(ARCH_POWER)
3403 if (int_op_size == OP_SIZE_4)
3404 goto do_upcall;
3405 if (op_size == OP_SIZE_4 && !cpu_test_feature(CPU_FEATURE_v206))
3406 goto do_upcall;
3407 if (op_size == OP_SIZE_8 && !cpu_test_feature(CPU_FEATURE_ppc))
3408 goto do_upcall;
3409 #endif
3410 if (slot_is_register(ctx, slot_1))
3411 g(spill(ctx, slot_1));
3412 g(gen_frame_load_raw(ctx, int_op_size, zero_x, slot_1, 0, false, FR_SCRATCH_1));
3413 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_2);
3414 #if defined(ARCH_ALPHA)
3415 if (OP_SIZE_INT == OP_SIZE_4) {
3416 gen_insn(INSN_MOVSX, OP_SIZE_4, 0, 0);
3417 gen_one(FR_SCRATCH_1);
3418 gen_one(FR_SCRATCH_1);
3420 int_op_size = OP_SIZE_8;
3421 }
3422 #endif
3423 gen_insn(int_op_size == OP_SIZE_4 ? INSN_FP_FROM_INT32 : INSN_FP_FROM_INT64, op_size, 0, 0);
3424 gen_one(target);
3425 gen_one(FR_SCRATCH_1);
3427 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
3428 return true;
3429 #elif defined(ARCH_IA64)
3430 g(gen_frame_get(ctx, OP_SIZE_INT, sign_x, slot_1, R_SCRATCH_1, ®1));
3431 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_1);
3433 g(gen_mov(ctx, OP_SIZE_NATIVE, target, reg1));
3435 gen_insn(INSN_FP_FROM_INT64, op_size, 0, 0);
3436 gen_one(target);
3437 gen_one(target);
3439 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
3440 return true;
3441 #else
3442 g(gen_frame_get(ctx, OP_SIZE_INT, garbage, slot_1, R_SCRATCH_1, ®1));
3443 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_1);
3445 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_FROM_INT32 : INSN_FP_FROM_INT64, op_size, 0, 0);
3446 gen_one(target);
3447 gen_one(reg1);
3449 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
3450 return true;
3451 #endif
3452 }
3453 #ifdef SUPPORTED_FP_X87
3454 if ((SUPPORTED_FP_X87 >> real_type) & 1) {
3455 if (slot_is_register(ctx, slot_1))
3456 g(spill(ctx, slot_1));
3457 g(gen_frame_load_x87(ctx, INSN_X87_FILD, OP_SIZE_INT, 0, slot_1));
3458 g(gen_frame_store_x87(ctx, INSN_X87_FSTP, op_size, slot_r));
3459 return true;
3460 }
3461 #endif
3462 #ifdef SUPPORTED_FP_HALF_CVT
3463 if ((SUPPORTED_FP_HALF_CVT >> real_type) & 1) {
3464 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_1);
3465 #if defined(ARCH_ARM32)
3466 g(gen_frame_get(ctx, OP_SIZE_INT, zero_x, slot_1, FR_SCRATCH_1, ®1));
3468 gen_insn(INSN_FP_FROM_INT32, OP_SIZE_4, 0, 0);
3469 gen_one(target);
3470 gen_one(reg1);
3471 #else
3472 g(gen_frame_get(ctx, OP_SIZE_INT, garbage, slot_1, R_SCRATCH_1, ®1));
3473 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_FROM_INT32 : INSN_FP_FROM_INT64, OP_SIZE_4, 0, 0);
3474 gen_one(target);
3475 gen_one(reg1);
3476 #endif
3477 gen_insn(INSN_FP_CVT, OP_SIZE_4, op_size, 0);
3478 gen_one(target);
3479 gen_one(target);
3480 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
3481 return true;
3482 }
3483 #endif
3484 goto do_upcall;
3486 do_is_exception:
3487 if ((SUPPORTED_FP >> real_type) & 1) {
3488 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, ®1));
3489 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
3490 #if defined(ARCH_ALPHA)
3491 gen_insn(INSN_FP_CMP_UNORDERED_DEST_REG, op_size, 0, 0);
3492 gen_one(FR_SCRATCH_2);
3493 gen_one(FR_SCRATCH_1);
3494 gen_one(reg1);
3496 if (!cpu_test_feature(CPU_FEATURE_fix)) {
3497 g(gen_frame_store_raw(ctx, OP_SIZE_4, slot_r, 0, FR_SCRATCH_2));
3498 g(gen_frame_load_raw(ctx, OP_SIZE_4, sign_x, slot_r, 0, false, target));
3499 } else {
3500 g(gen_mov(ctx, OP_SIZE_4, target, FR_SCRATCH_2));
3501 }
3503 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SHR, target, target, 30, 0));
3505 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
3507 return true;
3508 #elif defined(ARCH_IA64)
3509 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_P, 0);
3510 gen_one(R_CMP_RESULT);
3511 gen_one(reg1);
3512 gen_one(reg1);
3514 g(gen_mov(ctx, OP_SIZE_NATIVE, target, R_CMP_RESULT));
3516 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
3517 #elif defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS) || defined(ARCH_PARISC)
3518 gen_insn(INSN_FP_CMP_COND, op_size, FP_COND_P, 1);
3519 gen_one(reg1);
3520 gen_one(reg1);
3522 gen_insn(INSN_FP_TEST_REG, OP_SIZE_NATIVE, FP_COND_P, 0);
3523 gen_one(target);
3525 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
3526 #elif defined(ARCH_RISCV64)
3527 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_E, 0);
3528 gen_one(target);
3529 gen_one(reg1);
3530 gen_one(reg1);
3532 g(gen_imm(ctx, 1, IMM_PURPOSE_XOR, OP_SIZE_NATIVE));
3533 gen_insn(INSN_ALU, OP_SIZE_NATIVE, ALU_XOR, ALU_WRITES_FLAGS(ALU_XOR, is_imm()));
3534 gen_one(target);
3535 gen_one(target);
3536 gen_imm_offset();
3538 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
3539 #else
3540 gen_insn(INSN_FP_CMP, op_size, 0, 1);
3541 gen_one(reg1);
3542 gen_one(reg1);
3543 #if defined(ARCH_ARM32)
3544 gen_insn(INSN_FP_TO_INT_FLAGS, 0, 0, 1);
3545 #endif
3546 g(gen_frame_set_cond(ctx, op_size, false, FP_COND_P, slot_r));
3547 #endif
3548 return true;
3549 }
3550 #ifdef SUPPORTED_FP_X87
3551 if ((SUPPORTED_FP_X87 >> real_type) & 1) {
3552 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
3553 if (likely(cpu_test_feature(CPU_FEATURE_cmov))) {
3554 gen_insn(INSN_X87_FCOMIP, op_size, 0, 0);
3555 gen_one(R_ST0);
3557 g(gen_frame_set_cond(ctx, op_size, false, COND_P, slot_r));
3558 return true;
3559 }
3561 gen_insn(INSN_X87_FCOMP, op_size, 0, 0);
3562 gen_one(R_ST0);
3564 gen_insn(INSN_X87_FNSTSW, 0, 0, 0);
3565 gen_one(R_AX);
3566 gen_one(R_AX);
3568 gen_insn(INSN_TEST, OP_SIZE_2, 0, 1);
3569 gen_one(R_AX);
3570 gen_one(ARG_IMM);
3571 gen_eight(0x0400);
3573 g(gen_frame_set_cond(ctx, op_size, false, COND_NE, slot_r));
3575 return true;
3576 }
3577 #endif
3578 #ifdef SUPPORTED_FP_HALF_CVT
3579 if ((SUPPORTED_FP_HALF_CVT >> real_type) & 1) {
3580 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, ®1));
3581 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
3582 gen_one(FR_SCRATCH_1);
3583 gen_one(reg1);
3584 gen_insn(INSN_FP_CMP, OP_SIZE_4, 0, 1);
3585 gen_one(FR_SCRATCH_1);
3586 gen_one(FR_SCRATCH_1);
3587 #if defined(ARCH_ARM32)
3588 gen_insn(INSN_FP_TO_INT_FLAGS, 0, 0, 1);
3589 #endif
3590 g(gen_frame_set_cond(ctx, op_size, false, FP_COND_P, slot_r));
3591 return true;
3592 }
3593 #endif
3595 do_upcall:
3596 g(gen_alu_typed_upcall(ctx, upc, real_type, slot_1, NO_FRAME_T, slot_r, label_ovf));
3597 return true;
3598 }
3600 static bool attr_w gen_is_exception(struct codegen_context *ctx, frame_t slot_1, frame_t slot_r)
3601 {
3602 uint32_t no_ex_label, escape_label;
3603 const struct type *type = get_type_of_local(ctx, slot_1);
3605 no_ex_label = alloc_label(ctx);
3606 if (unlikely(!no_ex_label))
3607 return false;
3608 escape_label = alloc_escape_label(ctx);
3609 if (unlikely(!escape_label))
3610 return false;
3612 if (TYPE_IS_FLAT(type))
3613 g(gen_test_1_jz_cached(ctx, slot_1, no_ex_label));
3615 g(gen_frame_load(ctx, OP_SIZE_SLOT, zero_x, slot_1, 0, false, R_SCRATCH_1));
3616 g(gen_ptr_is_thunk(ctx, R_SCRATCH_1, slot_1, escape_label));
3618 if (!TYPE_IS_FLAT(type)) {
3619 g(gen_compare_da_tag(ctx, R_SCRATCH_1, DATA_TAG_flat, COND_E, escape_label, R_SCRATCH_1));
3620 }
3622 gen_label(no_ex_label);
3623 g(gen_frame_clear(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r));
3625 flag_set(ctx, slot_r, false);
3627 return true;
3628 }
3630 static bool attr_w gen_system_property(struct codegen_context *ctx, frame_t slot_1, frame_t slot_r)
3631 {
3632 uint32_t escape_label;
3634 escape_label = alloc_escape_label(ctx);
3635 if (unlikely(!escape_label))
3636 return false;
3638 g(gen_test_1_cached(ctx, slot_1, escape_label));
3640 g(gen_upcall_start(ctx, 1));
3642 g(gen_frame_load(ctx, OP_SIZE_INT, garbage, slot_1, 0, false, R_ARG0));
3643 g(gen_upcall_argument(ctx, 0));
3645 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_ipret_system_property), 1));
3647 g(gen_frame_store(ctx, OP_SIZE_INT, slot_r, 0, R_RET0));
3649 flag_set(ctx, slot_1, false);
3650 flag_set(ctx, slot_r, false);
3652 return true;
3653 }
3655 static bool attr_w gen_alu_jmp(struct codegen_context *ctx, unsigned mode, unsigned op_size, unsigned op, frame_t slot_1, frame_t slot_2, int32_t jmp_offset, bool *failed)
3656 {
3657 unsigned alu;
3658 enum extend ex;
3659 unsigned reg1;
3660 unsigned attr_unused reg2;
3661 #if 0
3662 *failed = true; return true;
3663 #endif
3664 switch (mode) {
3665 case MODE_FIXED: switch (op) {
3666 case OPCODE_FIXED_OP_equal: alu = COND_E; goto do_compare;
3667 case OPCODE_FIXED_OP_not_equal: alu = COND_NE; goto do_compare;
3668 case OPCODE_FIXED_OP_less: alu = COND_L; goto do_compare;
3669 case OPCODE_FIXED_OP_less_equal: alu = COND_LE; goto do_compare;
3670 case OPCODE_FIXED_OP_greater: alu = COND_G; goto do_compare;
3671 case OPCODE_FIXED_OP_greater_equal: alu = COND_GE; goto do_compare;
3672 case OPCODE_FIXED_OP_uless: alu = COND_B; goto do_compare;
3673 case OPCODE_FIXED_OP_uless_equal: alu = COND_BE; goto do_compare;
3674 case OPCODE_FIXED_OP_ugreater: alu = COND_A; goto do_compare;
3675 case OPCODE_FIXED_OP_ugreater_equal: alu = COND_AE; goto do_compare;
3676 case OPCODE_FIXED_OP_bt: *failed = true; return true;
3677 default: internal(file_line, "gen_alu_jmp: unsupported fixed operation %u", op);
3678 }
3679 case MODE_INT: switch (op) {
3680 case OPCODE_INT_OP_equal: alu = COND_E; goto do_compare;
3681 case OPCODE_INT_OP_not_equal: alu = COND_NE; goto do_compare;
3682 case OPCODE_INT_OP_less: alu = COND_L; goto do_compare;
3683 case OPCODE_INT_OP_less_equal: alu = COND_LE; goto do_compare;
3684 case OPCODE_INT_OP_greater: alu = COND_G; goto do_compare;
3685 case OPCODE_INT_OP_greater_equal: alu = COND_GE; goto do_compare;
3686 case OPCODE_INT_OP_bt: *failed = true; return true;
3687 default: internal(file_line, "gen_alu_jmp: unsupported int operation %u", op);
3688 }
3689 case MODE_BOOL: switch (op) {
3690 case OPCODE_BOOL_OP_and: alu = ALU_AND; mode = MODE_FIXED; goto do_alu;
3691 case OPCODE_BOOL_OP_or: alu = ALU_OR; mode = MODE_FIXED; goto do_alu;
3692 case OPCODE_BOOL_OP_equal: alu = COND_E; mode = MODE_FIXED; goto do_compare;
3693 case OPCODE_BOOL_OP_not_equal: alu = COND_NE; mode = MODE_FIXED; goto do_compare;
3694 case OPCODE_BOOL_OP_less: alu = COND_L; mode = MODE_FIXED; goto do_compare;
3695 case OPCODE_BOOL_OP_less_equal: alu = COND_LE; mode = MODE_FIXED; goto do_compare;
3696 case OPCODE_BOOL_OP_greater: alu = COND_G; mode = MODE_FIXED; goto do_compare;
3697 case OPCODE_BOOL_OP_greater_equal: alu = COND_GE; mode = MODE_FIXED; goto do_compare;
3698 default: internal(file_line, "gen_alu_jmp: unsupported bool operation %u", op);
3699 }
3700 }
3701 internal(file_line, "gen_alu_jmp: unsupported mode %u", mode);
3702 do_compare: {
3703 bool attr_unused logical;
3704 if (unlikely(op_size > OP_SIZE_NATIVE)) {
3705 *failed = true;
3706 return true;
3707 }
3708 if (slot_is_register(ctx, slot_2) && !slot_is_register(ctx, slot_1)) {
3709 frame_t s = slot_1;
3710 slot_1 = slot_2;
3711 slot_2 = s;
3712 switch (alu) {
3713 case COND_L: alu = COND_G; break;
3714 case COND_LE: alu = COND_GE; break;
3715 case COND_G: alu = COND_L; break;
3716 case COND_GE: alu = COND_LE; break;
3717 case COND_B: alu = COND_A; break;
3718 case COND_BE: alu = COND_AE; break;
3719 case COND_A: alu = COND_B; break;
3720 case COND_AE: alu = COND_BE; break;
3721 }
3722 }
3723 ex = op_size == i_size_cmp(op_size) + (unsigned)zero ? garbage : alu == COND_L || alu == COND_LE || alu == COND_G || alu == COND_GE || ARCH_PREFERS_SX(op_size) ? sign_x : zero_x;
3724 g(gen_frame_get(ctx, op_size, ex, slot_1, R_SCRATCH_1, ®1));
3725 if (ARCH_HAS_JMP_2REGS(alu)) {
3726 g(gen_frame_get(ctx, op_size, ex, slot_2, R_SCRATCH_2, ®2));
3727 g(gen_jump(ctx, jmp_offset, i_size_cmp(op_size), alu ^ 1, reg1, reg2));
3728 return true;
3729 }
3730 #if ARCH_HAS_FLAGS
3731 logical = COND_IS_LOGICAL(alu ^ 1);
3732 g(gen_frame_load_cmp(ctx, op_size, logical, ex, false, slot_2, 0, false, reg1));
3733 g(gen_jump(ctx, jmp_offset, op_size, alu ^ 1, -1U, -1U));
3734 #else
3735 g(gen_frame_get(ctx, op_size, ex, slot_2, R_SCRATCH_2, ®2));
3736 g(gen_cmp_dest_reg(ctx, op_size, reg1, reg2, R_CMP_RESULT, 0, alu));
3737 g(gen_jump(ctx, jmp_offset, OP_SIZE_NATIVE, COND_E, R_CMP_RESULT, -1U));
3738 #endif
3739 return true;
3740 }
3741 do_alu: {
3742 if (slot_is_register(ctx, slot_2) && !slot_is_register(ctx, slot_1)) {
3743 frame_t s = slot_1;
3744 slot_1 = slot_2;
3745 slot_2 = s;
3746 }
3747 ex = op_size == i_size(op_size) + (unsigned)zero ? garbage : ARCH_PREFERS_SX(op_size) ? sign_x : zero_x;
3748 g(gen_frame_get(ctx, op_size, ex, slot_1, R_SCRATCH_1, ®1));
3749 #if defined(ARCH_X86)
3750 if (alu == ALU_AND && !slot_is_register(ctx, slot_2)) {
3751 g(gen_address(ctx, R_FRAME, (size_t)slot_2 * slot_size, IMM_PURPOSE_LDR_OFFSET, op_size));
3752 gen_insn(INSN_TEST, i_size(op_size), 0, 1);
3753 gen_one(reg1);
3754 gen_address_offset();
3755 g(gen_jump(ctx, jmp_offset, op_size, COND_E, -1U, -1U));
3756 return true;
3757 }
3758 #endif
3759 g(gen_frame_get(ctx, op_size, ex, slot_2, R_SCRATCH_2, ®2));
3760 #if ARCH_HAS_FLAGS && !defined(ARCH_S390)
3761 if (alu == ALU_AND) {
3762 gen_insn(INSN_TEST, i_size(op_size), 0, 1);
3763 gen_one(reg1);
3764 gen_one(reg2);
3765 g(gen_jump(ctx, jmp_offset, op_size, COND_E, -1U, -1U));
3766 return true;
3767 }
3768 #endif
3769 #if defined(ARCH_ARM64)
3770 if (alu == ALU_OR)
3771 goto skip_flags;
3772 #endif
3773 #if ARCH_HAS_FLAGS
3774 g(gen_3address_alu(ctx, i_size(op_size), alu, R_SCRATCH_1, reg1, reg2, 1));
3775 g(gen_jump(ctx, jmp_offset, i_size(op_size), COND_E, -1U, -1U));
3776 return true;
3777 #endif
3778 goto skip_flags;
3779 skip_flags:
3780 g(gen_3address_alu(ctx, i_size(op_size), alu, R_SCRATCH_1, reg1, reg2, 0));
3781 g(gen_jump(ctx, jmp_offset, i_size(op_size), COND_E, R_SCRATCH_1, -1U));
3782 return true;
3783 }
3784 }
3786 static bool attr_w gen_fp_alu_jmp(struct codegen_context *ctx, unsigned real_type, unsigned op, uint32_t label_ovf, frame_t slot_1, frame_t slot_2, int32_t jmp_offset, bool *failed)
3787 {
3788 unsigned attr_unused fp_alu;
3789 unsigned attr_unused op_size = real_type_to_op_size(real_type);
3790 unsigned reg1, reg2;
3791 unsigned attr_unused target;
3792 switch (op) {
3793 case OPCODE_REAL_OP_equal:
3794 case OPCODE_REAL_OP_equal_alt1:
3795 case OPCODE_REAL_OP_equal_alt2: fp_alu = FP_COND_E; goto do_cmp;
3796 case OPCODE_REAL_OP_not_equal:
3797 case OPCODE_REAL_OP_not_equal_alt1:
3798 case OPCODE_REAL_OP_not_equal_alt2: fp_alu = FP_COND_NE; goto do_cmp;
3799 case OPCODE_REAL_OP_less:
3800 case OPCODE_REAL_OP_less_alt1:
3801 case OPCODE_REAL_OP_less_alt2: fp_alu = FP_COND_B; goto do_cmp;
3802 case OPCODE_REAL_OP_less_equal:
3803 case OPCODE_REAL_OP_less_equal_alt1:
3804 case OPCODE_REAL_OP_less_equal_alt2: fp_alu = FP_COND_BE; goto do_cmp;
3805 case OPCODE_REAL_OP_greater:
3806 case OPCODE_REAL_OP_greater_alt1:
3807 case OPCODE_REAL_OP_greater_alt2: fp_alu = FP_COND_A; goto do_cmp;
3808 case OPCODE_REAL_OP_greater_equal:
3809 case OPCODE_REAL_OP_greater_equal_alt1:
3810 case OPCODE_REAL_OP_greater_equal_alt2: fp_alu = FP_COND_AE; goto do_cmp;
3811 default: internal(file_line, "gen_fp_alu_jmp: unsupported operation %u", op);
3812 }
3814 do_cmp:
3815 if ((SUPPORTED_FP >> real_type) & 1
3816 #if defined(ARCH_ALPHA)
3817 && ARCH_SUPPORTS_TRAPS && cpu_test_feature(CPU_FEATURE_fix)
3818 #endif
3819 ) {
3820 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, ®1));
3821 g(gen_frame_get(ctx, op_size, zero_x, slot_2, FR_SCRATCH_2, ®2));
3822 target = R_SCRATCH_1;
3823 #if defined(ARCH_ALPHA)
3824 gen_insn(INSN_FP_CMP_DEST_REG_TRAP, op_size, fp_alu == FP_COND_NE ? FP_COND_E : fp_alu, 0);
3825 gen_one(FR_SCRATCH_3);
3826 gen_one(reg1);
3827 gen_one(reg2);
3828 gen_four(label_ovf);
3830 g(gen_mov(ctx, OP_SIZE_4, target, FR_SCRATCH_3));
3832 if (fp_alu == FP_COND_NE) {
3833 g(gen_jump(ctx, jmp_offset, OP_SIZE_NATIVE, COND_NE, target, -1U));
3834 } else {
3835 g(gen_jump(ctx, jmp_offset, OP_SIZE_NATIVE, COND_E, target, -1U));
3836 }
3838 return true;
3839 #elif defined(ARCH_IA64)
3840 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_P, 0);
3841 gen_one(R_CMP_RESULT);
3842 gen_one(reg1);
3843 gen_one(reg2);
3845 gen_insn(INSN_JMP_REG, OP_SIZE_NATIVE, COND_NE, 0);
3846 gen_one(R_CMP_RESULT);
3847 gen_four(label_ovf);
3849 gen_insn(INSN_FP_CMP_DEST_REG, op_size, fp_alu, 0);
3850 gen_one(R_CMP_RESULT);
3851 gen_one(reg1);
3852 gen_one(reg2);
3854 g(gen_jump(ctx, jmp_offset, OP_SIZE_NATIVE, COND_E, R_CMP_RESULT, -1U);
3856 return true;
3857 #elif defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS) || defined(ARCH_PARISC)
3858 gen_insn(INSN_FP_CMP_COND, op_size, FP_COND_P, 1);
3859 gen_one(reg1);
3860 gen_one(reg2);
3862 gen_insn(INSN_JMP_FP_TEST, 0, FP_COND_P, 0);
3863 gen_four(label_ovf);
3865 gen_insn(INSN_FP_CMP_COND, op_size, fp_alu ^ 1, 1);
3866 gen_one(reg1);
3867 gen_one(reg2);
3869 g(gen_jump(ctx, jmp_offset, OP_SIZE_NATIVE, fp_alu ^ 1, -1U, -1U));
3871 return true;
3872 #elif defined(ARCH_RISCV64)
3873 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_E, 0);
3874 gen_one(R_SCRATCH_1);
3875 gen_one(reg1);
3876 gen_one(reg1);
3878 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_E, 0);
3879 gen_one(R_SCRATCH_2);
3880 gen_one(reg2);
3881 gen_one(reg2);
3883 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_AND, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_2, 0));
3885 g(gen_jmp_on_zero(ctx, OP_SIZE_NATIVE, R_SCRATCH_1, COND_E, label_ovf));
3887 gen_insn(INSN_FP_CMP_DEST_REG, op_size, fp_alu == FP_COND_NE ? FP_COND_E : fp_alu, 0);
3888 gen_one(target);
3889 gen_one(reg1);
3890 gen_one(reg2);
3892 if (fp_alu == FP_COND_NE) {
3893 g(gen_jump(ctx, jmp_offset, OP_SIZE_NATIVE, COND_NE, target, -1U));
3894 } else {
3895 g(gen_jump(ctx, jmp_offset, OP_SIZE_NATIVE, COND_E, target, -1U));
3896 }
3897 return true;
3898 #else
3899 gen_insn(INSN_FP_CMP, op_size, 0, 1);
3900 gen_one(reg1);
3901 gen_one(reg2);
3902 #if defined(ARCH_ARM32)
3903 gen_insn(INSN_FP_TO_INT_FLAGS, 0, 0, 1);
3904 #endif
3905 gen_insn(INSN_JMP_COND, op_size, FP_COND_P, 0);
3906 gen_four(label_ovf);
3907 g(gen_jump(ctx, jmp_offset, op_size, fp_alu ^ 1, -1U, -1U));
3908 return true;
3909 #endif
3910 }
3911 *failed = true;
3912 return true;
3913 }