| blob | 54388aca8a6873ee8de9a7e75511520f5aef873f |
1 /*
2 * Copyright (C) 2024 Mikulas Patocka
3 *
4 * This file is part of Ajla.
5 *
6 * Ajla is free software: you can redistribute it and/or modify it under the
7 * terms of the GNU General Public License as published by the Free Software
8 * Foundation, either version 3 of the License, or (at your option) any later
9 * version.
10 *
11 * Ajla is distributed in the hope that it will be useful, but WITHOUT ANY
12 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
13 * A PARTICULAR PURPOSE. See the GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License along with
16 * Ajla. If not, see <https://www.gnu.org/licenses/>.
17 */
19 static bool attr_w gen_jump(struct codegen_context *ctx, int32_t jmp_offset, unsigned op_size, unsigned cond, unsigned reg1, unsigned reg2);
21 static bool attr_w gen_alu_upcall(struct codegen_context *ctx, size_t upcall, unsigned op_size, frame_t slot_1, frame_t slot_2, frame_t slot_r, uint32_t label_ovf)
22 {
23 if (slot_is_register(ctx, slot_1))
24 g(spill(ctx, slot_1));
25 if (slot_2 != NO_FRAME_T && slot_is_register(ctx, slot_2))
26 g(spill(ctx, slot_2));
27 if (frame_t_is_const(slot_2)) {
28 size_t x_offs = offsetof(struct cg_upcall_vector_s, INT_binary_const_int8_t) + op_size * sizeof(void (*)(void));
29 g(gen_upcall_start(ctx, x_offs, 4));
30 g(gen_frame_address(ctx, slot_1, 0, R_ARG0));
31 g(gen_upcall_argument(ctx, 0));
32 g(gen_load_constant(ctx, R_ARG1, frame_t_get_const(slot_2)));
33 g(gen_upcall_argument(ctx, 1));
34 g(gen_frame_address(ctx, slot_r, 0, R_ARG2));
35 g(gen_upcall_argument(ctx, 2));
36 g(gen_get_upcall_pointer(ctx, upcall, R_ARG3));
37 g(gen_upcall_argument(ctx, 3));
38 g(gen_upcall(ctx, x_offs, 4));
39 } else if (slot_2 != NO_FRAME_T) {
40 g(gen_upcall_start(ctx, upcall, 3));
41 g(gen_frame_address(ctx, slot_1, 0, R_ARG0));
42 g(gen_upcall_argument(ctx, 0));
43 g(gen_frame_address(ctx, slot_2, 0, R_ARG1));
44 g(gen_upcall_argument(ctx, 1));
45 g(gen_frame_address(ctx, slot_r, 0, R_ARG2));
46 g(gen_upcall_argument(ctx, 2));
47 g(gen_upcall(ctx, upcall, 3));
48 } else {
49 g(gen_upcall_start(ctx, upcall, 2));
50 g(gen_frame_address(ctx, slot_1, 0, R_ARG0));
51 g(gen_upcall_argument(ctx, 0));
52 g(gen_frame_address(ctx, slot_r, 0, R_ARG1));
53 g(gen_upcall_argument(ctx, 1));
54 g(gen_upcall(ctx, upcall, 2));
55 }
56 if (slot_is_register(ctx, slot_r))
57 g(unspill(ctx, slot_r));
58 if (label_ovf)
59 g(gen_jmp_on_zero(ctx, OP_SIZE_1, R_RET0, COND_E, label_ovf));
60 return true;
61 }
63 static bool attr_w gen_alu_typed_upcall(struct codegen_context *ctx, size_t upcall, unsigned op_size, frame_t slot_1, frame_t slot_2, frame_t slot_r, uint32_t label_ovf)
64 {
65 upcall += op_size * sizeof(void (*)(void));
66 return gen_alu_upcall(ctx, upcall, op_size, slot_1, slot_2, slot_r, label_ovf);
67 }
69 #define MODE_FIXED 0
70 #define MODE_INT 1
71 #define MODE_REAL 2
72 #define MODE_BOOL 3
73 #define MODE_ARRAY_LEN_GT 4
75 static bool attr_w gen_alu(struct codegen_context *ctx, unsigned mode, unsigned op_size, unsigned op, uint32_t label_ovf, frame_t slot_1, frame_t slot_2, frame_t slot_r)
76 {
77 unsigned alu;
78 bool sgn, mod;
79 unsigned reg1, reg2, reg3, target;
80 switch (mode) {
81 case MODE_FIXED: switch (op) {
82 case OPCODE_FIXED_OP_add: alu = ALU_ADD; goto do_alu;
83 case OPCODE_FIXED_OP_subtract: alu = ALU_SUB; goto do_alu;
84 case OPCODE_FIXED_OP_multiply: goto do_multiply;
85 case OPCODE_FIXED_OP_divide:
86 case OPCODE_FIXED_OP_divide_alt1: sgn = true; mod = false; goto do_divide;
87 case OPCODE_FIXED_OP_udivide:
88 case OPCODE_FIXED_OP_udivide_alt1: sgn = false; mod = false; goto do_divide;
89 case OPCODE_FIXED_OP_modulo:
90 case OPCODE_FIXED_OP_modulo_alt1: sgn = true; mod = true; goto do_divide;
91 case OPCODE_FIXED_OP_umodulo:
92 case OPCODE_FIXED_OP_umodulo_alt1: sgn = false; mod = true; goto do_divide;
93 case OPCODE_FIXED_OP_power: return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_binary_power_int8_t), op_size, slot_1, slot_2, slot_r, 0);
94 case OPCODE_FIXED_OP_and: alu = ALU_AND; goto do_alu;
95 case OPCODE_FIXED_OP_or: alu = ALU_OR; goto do_alu;
96 case OPCODE_FIXED_OP_xor: alu = ALU_XOR; goto do_alu;
97 case OPCODE_FIXED_OP_shl: alu = ROT_SHL; goto do_shift;
98 case OPCODE_FIXED_OP_shr: alu = ROT_SAR; goto do_shift;
99 case OPCODE_FIXED_OP_ushr: alu = ROT_SHR; goto do_shift;
100 case OPCODE_FIXED_OP_rol: alu = ROT_ROL; goto do_shift;
101 case OPCODE_FIXED_OP_ror: alu = ROT_ROR; goto do_shift;
102 case OPCODE_FIXED_OP_bts: alu = BTX_BTS; goto do_bt;
103 case OPCODE_FIXED_OP_btr: alu = BTX_BTR; goto do_bt;
104 case OPCODE_FIXED_OP_btc: alu = BTX_BTC; goto do_bt;
105 case OPCODE_FIXED_OP_equal: alu = COND_E; goto do_compare;
106 case OPCODE_FIXED_OP_not_equal: alu = COND_NE; goto do_compare;
107 case OPCODE_FIXED_OP_less: alu = COND_L; goto do_compare;
108 case OPCODE_FIXED_OP_less_equal: alu = COND_LE; goto do_compare;
109 case OPCODE_FIXED_OP_greater: alu = COND_G; goto do_compare;
110 case OPCODE_FIXED_OP_greater_equal: alu = COND_GE; goto do_compare;
111 case OPCODE_FIXED_OP_uless: alu = COND_B; goto do_compare;
112 case OPCODE_FIXED_OP_uless_equal: alu = COND_BE; goto do_compare;
113 case OPCODE_FIXED_OP_ugreater: alu = COND_A; goto do_compare;
114 case OPCODE_FIXED_OP_ugreater_equal: alu = COND_AE; goto do_compare;
115 case OPCODE_FIXED_OP_bt: alu = BTX_BT; goto do_bt;
116 default: internal(file_line, "gen_alu: unsupported fixed operation %u", op);
117 }
118 case MODE_INT: switch (op) {
119 case OPCODE_INT_OP_add: alu = ALU_ADD; goto do_alu;
120 case OPCODE_INT_OP_subtract: alu = ALU_SUB; goto do_alu;
121 case OPCODE_INT_OP_multiply: goto do_multiply;
122 case OPCODE_INT_OP_divide:
123 case OPCODE_INT_OP_divide_alt1: sgn = true; mod = false; goto do_divide;
124 case OPCODE_INT_OP_modulo:
125 case OPCODE_INT_OP_modulo_alt1: sgn = true; mod = true; goto do_divide;
126 case OPCODE_INT_OP_power: return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, INT_binary_power_int8_t), op_size, slot_1, slot_2, slot_r, label_ovf);
127 case OPCODE_INT_OP_and: alu = ALU_AND; mode = MODE_FIXED; goto do_alu;
128 case OPCODE_INT_OP_or: alu = ALU_OR; mode = MODE_FIXED; goto do_alu;
129 case OPCODE_INT_OP_xor: alu = ALU_XOR; mode = MODE_FIXED; goto do_alu;
130 case OPCODE_INT_OP_shl: alu = ROT_SHL; goto do_shift;
131 case OPCODE_INT_OP_shr: alu = ROT_SAR; goto do_shift;
132 case OPCODE_INT_OP_bts: alu = BTX_BTS; goto do_bt;
133 case OPCODE_INT_OP_btr: alu = BTX_BTR; goto do_bt;
134 case OPCODE_INT_OP_btc: alu = BTX_BTC; goto do_bt;
135 case OPCODE_INT_OP_equal: alu = COND_E; goto do_compare;
136 case OPCODE_INT_OP_not_equal: alu = COND_NE; goto do_compare;
137 case OPCODE_INT_OP_less: alu = COND_L; goto do_compare;
138 case OPCODE_INT_OP_less_equal: alu = COND_LE; goto do_compare;
139 case OPCODE_INT_OP_greater: alu = COND_G; goto do_compare;
140 case OPCODE_INT_OP_greater_equal: alu = COND_GE; goto do_compare;
141 case OPCODE_INT_OP_bt: alu = BTX_BT; goto do_bt;
142 default: internal(file_line, "gen_alu: unsupported int operation %u", op);
143 }
144 case MODE_BOOL: switch (op) {
145 case OPCODE_BOOL_OP_and: alu = ALU_AND; mode = MODE_FIXED; goto do_alu;
146 case OPCODE_BOOL_OP_or: alu = ALU_OR; mode = MODE_FIXED; goto do_alu;
147 case OPCODE_BOOL_OP_equal: alu = COND_E; goto do_compare;
148 case OPCODE_BOOL_OP_not_equal: alu = ALU_XOR; mode = MODE_FIXED; goto do_alu;
149 case OPCODE_BOOL_OP_less: alu = COND_L; goto do_compare;
150 case OPCODE_BOOL_OP_less_equal: alu = COND_LE; goto do_compare;
151 case OPCODE_BOOL_OP_greater: alu = COND_G; goto do_compare;
152 case OPCODE_BOOL_OP_greater_equal: alu = COND_GE; goto do_compare;
153 default: internal(file_line, "gen_alu: unsupported bool operation %u", op);
154 }
155 }
156 internal(file_line, "gen_alu: unsupported mode %u", mode);
158 /*******
159 * ALU *
160 *******/
161 do_alu: {
162 size_t attr_unused offset;
163 uint8_t attr_unused long_imm;
164 unsigned first_flags;
165 unsigned second_flags;
166 unsigned second_alu;
167 unsigned attr_unused op_size_flags;
168 bool c;
169 if (unlikely(op_size > OP_SIZE_NATIVE)) {
170 #if !defined(ARCH_X86) && !defined(ARCH_ARM) && !defined(ARCH_PARISC) && !defined(ARCH_POWER) && !defined(ARCH_SPARC32)
171 if (mode == MODE_FIXED) {
172 if (alu == ALU_ADD) {
173 g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_add_,TYPE_INT_MAX)), op_size, slot_1, slot_2, slot_r, 0));
174 return true;
175 } else if (alu == ALU_SUB) {
176 g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_subtract_,TYPE_INT_MAX)), op_size, slot_1, slot_2, slot_r, 0));
177 return true;
178 }
179 } else if (mode == MODE_INT) {
180 if (alu == ALU_ADD) {
181 g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(INT_binary_add_,TYPE_INT_MAX)), op_size, slot_1, slot_2, slot_r, label_ovf));
182 return true;
183 } else if (alu == ALU_SUB) {
184 g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(INT_binary_subtract_,TYPE_INT_MAX)), op_size, slot_1, slot_2, slot_r, label_ovf));
185 return true;
186 }
187 }
188 #endif
189 first_flags = alu == ALU_ADD || alu == ALU_SUB ? 2 : 0;
190 second_flags = mode == MODE_INT ? 1 : 0;
191 second_alu = alu == ALU_ADD ? ALU_ADC : alu == ALU_SUB ? ALU_SBB : alu;
192 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
193 #if defined(ARCH_X86)
194 g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, alu, first_flags, slot_2, lo_word(OP_SIZE_NATIVE), true, R_SCRATCH_1));
195 g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, second_alu, second_flags, slot_2, hi_word(OP_SIZE_NATIVE), true, R_SCRATCH_2));
196 #else
197 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_2, 0, R_SCRATCH_3, R_SCRATCH_4));
198 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, alu, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_3, first_flags));
199 #if defined(ARCH_PARISC)
200 if (mode == MODE_INT) {
201 gen_insn(INSN_ALU_FLAGS_TRAP, OP_SIZE_NATIVE, second_alu, ALU_WRITES_FLAGS(OP_SIZE_NATIVE, second_alu, false, false, 0));
202 gen_one(R_SCRATCH_2);
203 gen_one(R_SCRATCH_2);
204 gen_one(R_SCRATCH_4);
205 gen_four(label_ovf);
206 } else
207 #endif
208 {
209 gen_insn(first_flags ? INSN_ALU_FLAGS : INSN_ALU, OP_SIZE_NATIVE, second_alu, second_flags | ALU_WRITES_FLAGS(OP_SIZE_NATIVE, second_alu, false, false, 0));
210 gen_one(R_SCRATCH_2);
211 gen_one(R_SCRATCH_2);
212 gen_one(R_SCRATCH_4);
213 }
214 #endif
215 #if !defined(ARCH_PARISC)
216 if (mode == MODE_INT) {
217 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_O, 0);
218 gen_four(label_ovf);
219 }
220 #endif
221 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_1, R_SCRATCH_2));
222 return true;
223 }
225 if (((ARCH_HAS_FLAGS && i_size_cmp(op_size) == op_size + zero) || ARCH_SUPPORTS_TRAPS(op_size)) && slot_2 == slot_r && slot_1 != slot_2 && alu_is_commutative(alu)) {
226 frame_t x = slot_1;
227 slot_1 = slot_2;
228 slot_2 = x;
229 }
230 if (((ARCH_HAS_FLAGS && i_size_cmp(op_size) == op_size + zero) || ARCH_SUPPORTS_TRAPS(op_size)) && slot_1 == slot_r && (slot_1 != slot_2 || mode != MODE_INT)
231 #if defined(ARCH_POWER)
232 && op_size == OP_SIZE_NATIVE
233 #endif
234 ) {
235 struct cg_exit *ce;
236 unsigned undo_alu = alu == ALU_ADD ? ALU_SUB : ALU_ADD;
237 if (slot_is_register(ctx, slot_1)) {
238 unsigned reg1 = ctx->registers[slot_1];
239 if (slot_is_register(ctx, slot_2)
240 #if !defined(ARCH_POWER)
241 || frame_t_is_const(slot_2)
242 #endif
243 ) {
244 unsigned reg2 = frame_t_is_const(slot_2) ? 0xff /* avoid warning */ : ctx->registers[slot_2];
245 if (mode == MODE_INT && ARCH_SUPPORTS_TRAPS(op_size)) {
246 if (frame_t_is_const(slot_2))
247 g(gen_imm(ctx, frame_t_get_const(slot_2), alu_trap_purpose(alu) | (alu_purpose(undo_alu) << 8), i_size(op_size)));
248 gen_insn(INSN_ALU_TRAP, op_size, alu, ALU_WRITES_FLAGS(op_size, alu, false, frame_t_is_const(slot_2) && is_imm(), ctx->const_imm));
249 gen_one(reg1);
250 gen_one(reg1);
251 if (frame_t_is_const(slot_2))
252 gen_imm_offset();
253 else
254 gen_one(reg2);
255 if (ARCH_TRAP_BEFORE) {
256 gen_four(label_ovf);
257 return true;
258 } else {
259 ce = alloc_undo_label(ctx);
260 if (unlikely(!ce))
261 return false;
262 gen_four(ce->undo_label);
263 goto do_undo_opcode;
264 }
265 }
266 if (frame_t_is_const(slot_2))
267 g(gen_imm(ctx, frame_t_get_const(slot_2), alu_purpose(alu) | (alu_purpose(undo_alu) << 8), i_size(op_size)));
268 gen_insn(INSN_ALU + ARCH_PARTIAL_ALU(i_size(op_size)), i_size(op_size), alu, ALU_WRITES_FLAGS(i_size(op_size), alu, false, frame_t_is_const(slot_2) && is_imm(), ctx->const_imm));
269 gen_one(reg1);
270 gen_one(reg1);
271 if (frame_t_is_const(slot_2))
272 gen_imm_offset();
273 else
274 gen_one(reg2);
275 if (mode == MODE_INT) {
276 size_t m;
277 ce = alloc_undo_label(ctx);
278 if (unlikely(!ce))
279 return false;
280 gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
281 gen_four(ce->undo_label);
282 do_undo_opcode:
283 ce->undo_opcode = INSN_ALU + ARCH_PARTIAL_ALU(i_size(op_size));
284 ce->undo_op_size = i_size(op_size);
285 ce->undo_aux = undo_alu;
286 ce->undo_writes_flags = ALU_WRITES_FLAGS(i_size(op_size), undo_alu, false, frame_t_is_const(slot_2) && is_imm(), ctx->const_imm);
287 m = mark_params(ctx);
288 gen_one(reg1);
289 gen_one(reg1);
290 if (frame_t_is_const(slot_2))
291 gen_imm_offset();
292 else
293 gen_one(reg2);
294 copy_params(ctx, ce, m);
295 }
296 return true;
297 }
298 #if defined(ARCH_S390) || defined(ARCH_X86)
299 else if (!frame_t_is_const(slot_2)) {
300 size_t m;
301 int64_t offset = (size_t)slot_2 * slot_size;
302 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDR_OFFSET, i_size(op_size)));
303 gen_insn(INSN_ALU + ARCH_PARTIAL_ALU(i_size(op_size)), i_size(op_size), alu, ALU_WRITES_FLAGS(i_size(op_size), alu, true, false, 0) | 1);
304 gen_one(reg1);
305 gen_one(reg1);
306 gen_address_offset();
307 if (mode == MODE_INT) {
308 ce = alloc_undo_label(ctx);
309 if (unlikely(!ce))
310 return false;
311 ce->undo_opcode = INSN_ALU + ARCH_PARTIAL_ALU(i_size(op_size));
312 ce->undo_op_size = i_size(op_size);
313 ce->undo_aux = undo_alu;
314 ce->undo_writes_flags = ALU_WRITES_FLAGS(i_size(op_size), undo_alu, true, false, 0);
315 m = mark_params(ctx);
316 gen_one(reg1);
317 gen_one(reg1);
318 gen_address_offset();
319 copy_params(ctx, ce, m);
320 gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
321 gen_four(ce->undo_label);
322 }
323 return true;
324 }
325 #endif
326 }
327 #if defined(ARCH_X86)
328 else {
329 unsigned reg2;
330 size_t m;
331 int64_t offset = (size_t)slot_1 * slot_size;
332 if (!frame_t_is_const(slot_2))
333 g(gen_frame_get(ctx, op_size, garbage, slot_2, R_SCRATCH_1, ®2));
334 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDR_OFFSET, i_size(op_size)));
335 if (frame_t_is_const(slot_2))
336 g(gen_imm(ctx, frame_t_get_const(slot_2), alu_purpose(alu) | (alu_purpose(undo_alu) << 8), i_size(op_size)));
337 gen_insn(INSN_ALU + ARCH_PARTIAL_ALU(op_size), i_size(op_size), alu, ALU_WRITES_FLAGS(i_size(op_size), undo_alu, true, frame_t_is_const(slot_2) && is_imm(), ctx->const_imm) | 1);
338 gen_address_offset();
339 gen_address_offset();
340 if (frame_t_is_const(slot_2))
341 gen_imm_offset();
342 else
343 gen_one(reg2);
344 if (mode == MODE_INT) {
345 ce = alloc_undo_label(ctx);
346 if (unlikely(!ce))
347 return false;
348 ce->undo_opcode = INSN_ALU + ARCH_PARTIAL_ALU(op_size);
349 ce->undo_op_size = i_size(op_size);
350 ce->undo_aux = undo_alu;
351 ce->undo_writes_flags = ALU_WRITES_FLAGS(i_size(op_size), undo_alu, true, false, 0);
352 m = mark_params(ctx);
353 gen_address_offset();
354 gen_address_offset();
355 if (frame_t_is_const(slot_2))
356 gen_imm_offset();
357 else
358 gen_one(reg2);
359 copy_params(ctx, ce, m);
360 gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
361 gen_four(ce->undo_label);
362 }
363 return true;
364 }
365 #endif
366 }
368 #if defined(ARCH_X86)
369 if (1)
370 #elif defined(ARCH_S390)
371 if (op_size >= OP_SIZE_4)
372 #elif ARCH_HAS_FLAGS && !defined(ARCH_POWER)
373 if (op_size == i_size(op_size) + (unsigned)zero && frame_t_is_const(slot_2))
374 #else
375 if (mode != MODE_INT && op_size == i_size(op_size) + (unsigned)zero && frame_t_is_const(slot_2))
376 #endif
377 {
378 if (mode == MODE_INT) {
379 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
380 } else {
381 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, slot_2, R_SCRATCH_1);
382 }
383 g(gen_frame_load(ctx, op_size, garbage, slot_1, 0, false, target));
384 g(gen_frame_load_op(ctx, op_size, garbage, alu, mode == MODE_INT, slot_2, 0, false, target));
385 goto check_ovf_store;
386 }
387 op_size_flags = !ARCH_HAS_FLAGS && !ARCH_SUPPORTS_TRAPS(op_size) ? OP_SIZE_NATIVE : OP_SIZE_4;
388 #if defined(ARCH_POWER)
389 op_size_flags = OP_SIZE_NATIVE;
390 #endif
391 g(gen_frame_get(ctx, op_size, mode == MODE_INT && (op_size < op_size_flags || ARCH_SUPPORTS_TRAPS(op_size)) ? sign_x : garbage, slot_1, R_SCRATCH_1, ®1));
392 if (frame_t_is_const(slot_2)
393 #if defined(ARCH_POWER)
394 && 0
395 #endif
396 ) {
397 reg2 = 0xff;
398 c = true;
399 } else {
400 g(gen_frame_get(ctx, op_size, mode == MODE_INT && (op_size < op_size_flags || ARCH_SUPPORTS_TRAPS(op_size)) ? sign_x : garbage, slot_2, R_SCRATCH_2, ®2));
401 c = false;
402 }
403 #if !ARCH_HAS_FLAGS
404 if (mode == MODE_INT && op_size >= OP_SIZE_4) {
405 if (ARCH_SUPPORTS_TRAPS(op_size)) {
406 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
407 if (c)
408 g(gen_imm(ctx, frame_t_get_const(slot_2), alu_trap_purpose(alu), op_size));
409 gen_insn(INSN_ALU_TRAP, op_size, alu, ALU_WRITES_FLAGS(op_size, alu, false, c && is_imm(), ctx->const_imm));
410 gen_one(target);
411 gen_one(reg1);
412 if (c)
413 gen_imm_offset();
414 else
415 gen_one(reg2);
416 gen_four(label_ovf);
417 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
418 return true;
419 }
420 if (op_size >= OP_SIZE_NATIVE) {
421 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_3);
422 if (c)
423 g(gen_3address_alu_imm(ctx, i_size(op_size), alu, target, reg1, frame_t_get_const(slot_2), 0));
424 else
425 g(gen_3address_alu(ctx, i_size(op_size), alu, target, reg1, reg2, 0));
426 #if defined(ARCH_IA64)
427 if (c) {
428 g(gen_3address_alu_imm(ctx, i_size(op_size), ALU_XOR, R_SCRATCH_1, reg1, frame_t_get_const(slot_2), 0));
429 g(gen_3address_alu_imm(ctx, i_size(op_size), ALU_XOR, R_SCRATCH_2, target, frame_t_get_const(slot_2), 0));
430 } else {
431 g(gen_3address_alu(ctx, i_size(op_size), ALU_XOR, R_SCRATCH_1, reg1, reg2, 0));
432 g(gen_3address_alu(ctx, i_size(op_size), ALU_XOR, R_SCRATCH_2, target, reg2, 0));
433 }
434 if (alu == ALU_ADD) {
435 g(gen_3address_alu(ctx, i_size(op_size), ALU_ANDN, R_SCRATCH_1, R_SCRATCH_2, R_SCRATCH_1, 0));
436 } else {
437 g(gen_3address_alu(ctx, i_size(op_size), ALU_ANDN, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_2, 0));
438 }
439 g(gen_cmp_test_jmp(ctx, INSN_TEST, i_size(op_size), R_SCRATCH_1, R_SCRATCH_1, COND_S, label_ovf));
440 #else
441 if (c) {
442 g(gen_cmp_test_jmp(ctx, INSN_CMP, i_size(op_size), reg1, target, (frame_t_get_const(slot_2) >= 0) ^ (alu != ALU_ADD) ? COND_G : COND_L, label_ovf));
443 } else {
444 gen_insn(INSN_CMP_DEST_REG, i_size(op_size), COND_L, 0);
445 gen_one(R_SCRATCH_1);
446 if (alu == ALU_ADD) {
447 gen_one(target);
448 gen_one(reg1);
449 } else {
450 gen_one(reg1);
451 gen_one(target);
452 }
454 g(gen_imm(ctx, 0, IMM_PURPOSE_CMP, i_size(op_size)));
455 gen_insn(INSN_CMP_DEST_REG, i_size(op_size), COND_L, 0);
456 gen_one(R_SCRATCH_2);
457 gen_one(reg2);
458 gen_imm_offset();
460 g(gen_cmp_test_jmp(ctx, INSN_CMP, i_size(op_size), R_SCRATCH_1, R_SCRATCH_2, COND_NE, label_ovf));
461 }
462 #endif
463 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
464 return true;
465 }
466 }
467 #endif
468 if (mode == MODE_INT) {
469 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
470 } else if (!ARCH_IS_3ADDRESS(alu, mode == MODE_INT && op_size >= op_size_flags) && !alu_is_commutative(alu)) {
471 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, slot_2, R_SCRATCH_1);
472 } else {
473 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
474 }
475 if (c) {
476 g(gen_3address_alu_imm(ctx, i_size(op_size), alu, target, reg1, frame_t_get_const(slot_2), mode == MODE_INT && op_size >= op_size_flags));
477 } else {
478 g(gen_3address_alu(ctx, i_size(op_size), alu, target, reg1, reg2, mode == MODE_INT && op_size >= op_size_flags));
479 }
481 if (mode == MODE_INT && unlikely(op_size < op_size_flags)) {
482 g(gen_cmp_extended(ctx, op_size_flags, op_size, target, R_SCRATCH_2, label_ovf));
483 } else
484 check_ovf_store:
485 if (mode == MODE_INT) {
486 gen_insn(INSN_JMP_COND, op_size, COND_O, 0);
487 gen_four(label_ovf);
488 }
489 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
490 return true;
491 }
493 /************
494 * MULTIPLY *
495 ************/
496 do_multiply: {
497 size_t attr_unused offset;
498 uint8_t attr_unused long_imm;
499 if (unlikely(op_size > OP_SIZE_NATIVE) || unlikely(!ARCH_HAS_MUL)) {
500 if (mode == MODE_INT) {
501 g(gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, INT_binary_multiply_int8_t), op_size, slot_1, slot_2, slot_r, label_ovf));
502 return true;
503 }
504 #if defined(ARCH_X86)
505 g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_1, hi_word(OP_SIZE_NATIVE), true, R_CX));
506 g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_2, hi_word(OP_SIZE_NATIVE), true, R_AX));
507 g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, ALU_MUL, true, slot_2, lo_word(OP_SIZE_NATIVE), true, R_CX));
508 g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, ALU_MUL, true, slot_1, lo_word(OP_SIZE_NATIVE), true, R_AX));
509 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_ADD, R_CX, R_CX, R_AX, 0));
510 g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_2, lo_word(OP_SIZE_NATIVE), true, R_AX));
512 offset = (size_t)slot_1 * slot_size + lo_word(OP_SIZE_NATIVE);
513 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
514 gen_insn(INSN_MUL_L, OP_SIZE_NATIVE, 0, 1);
515 gen_one(R_AX);
516 gen_one(R_DX);
517 gen_one(R_AX);
518 gen_address_offset();
520 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_ADD, R_DX, R_DX, R_CX, 0));
522 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_AX, R_DX));
524 return true;
525 #elif defined(ARCH_ARM32)
526 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
527 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_2, 0, R_SCRATCH_3, R_SCRATCH_4));
529 g(gen_mov(ctx, OP_SIZE_NATIVE, R_SCRATCH_NA_1, R_SCRATCH_1));
531 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_MUL, R_SCRATCH_4, R_SCRATCH_1, R_SCRATCH_4, 0));
533 gen_insn(INSN_MADD, OP_SIZE_NATIVE, 0, 0);
534 gen_one(R_SCRATCH_2);
535 gen_one(R_SCRATCH_3);
536 gen_one(R_SCRATCH_2);
537 gen_one(R_SCRATCH_4);
539 gen_insn(INSN_MUL_L, OP_SIZE_NATIVE, 0, 0);
540 gen_one(R_SCRATCH_1);
541 gen_one(R_SCRATCH_4);
542 gen_one(R_SCRATCH_NA_1);
543 gen_one(R_SCRATCH_3);
545 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_ADD, R_SCRATCH_2, R_SCRATCH_2, R_SCRATCH_4, 0));
547 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_1, R_SCRATCH_2));
549 return true;
550 #elif defined(ARCH_ARM64)
551 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
552 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_2, 0, R_SCRATCH_3, R_SCRATCH_4));
554 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_UMULH, R_SCRATCH_NA_1, R_SCRATCH_1, R_SCRATCH_3, 0));
556 gen_insn(INSN_MADD, OP_SIZE_NATIVE, 0, 0);
557 gen_one(R_SCRATCH_NA_1);
558 gen_one(R_SCRATCH_2);
559 gen_one(R_SCRATCH_3);
560 gen_one(R_SCRATCH_NA_1);
562 gen_insn(INSN_MADD, OP_SIZE_NATIVE, 0, 0);
563 gen_one(R_SCRATCH_2);
564 gen_one(R_SCRATCH_1);
565 gen_one(R_SCRATCH_4);
566 gen_one(R_SCRATCH_NA_1);
568 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_MUL, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_3, 0));
570 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_1, R_SCRATCH_2));
572 return true;
573 #else
574 g(gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_binary_multiply_int8_t), op_size, slot_1, slot_2, slot_r, 0));
575 return true;
576 #endif
577 }
579 #if defined(ARCH_X86)
580 if (mode == MODE_INT) {
581 if (op_size != OP_SIZE_1 && slot_r == slot_1 && slot_is_register(ctx, slot_1)) {
582 struct cg_exit *ce;
583 target = ctx->registers[slot_1];
584 g(gen_mov(ctx, op_size, R_SCRATCH_1, target));
585 g(gen_frame_load_op(ctx, op_size, garbage, ALU_MUL, mode == MODE_INT, slot_2, 0, false, target));
586 ce = alloc_undo_label(ctx);
587 if (unlikely(!ce))
588 return false;
589 ce->undo_opcode = INSN_MOV;
590 ce->undo_op_size = op_size;
591 ce->undo_aux = 0;
592 ce->undo_writes_flags = 0;
593 ce->undo_parameters[0] = target;
594 ce->undo_parameters[1] = R_SCRATCH_1;
595 ce->undo_parameters_len = 2;
596 gen_insn(INSN_JMP_COND, op_size, COND_O, 0);
597 gen_four(ce->undo_label);
598 return true;
599 }
600 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
601 } else {
602 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, slot_2, R_SCRATCH_1);
603 }
604 if (op_size == OP_SIZE_1)
605 target = R_SCRATCH_1;
606 g(gen_frame_load(ctx, op_size, garbage, slot_1, 0, false, target));
607 if (op_size == OP_SIZE_1 && frame_t_is_const(slot_2)) {
608 g(gen_load_constant(ctx, R_SCRATCH_3, frame_t_get_const(slot_2)));
609 gen_insn(INSN_ALU + ARCH_PARTIAL_ALU(op_size), op_size, ALU_MUL, 1);
610 gen_one(target);
611 gen_one(target);
612 gen_one(R_SCRATCH_3);
613 } else {
614 g(gen_frame_load_op(ctx, op_size, garbage, ALU_MUL, mode == MODE_INT, slot_2, 0, false, target));
615 }
616 if (mode == MODE_INT) {
617 gen_insn(INSN_JMP_COND, op_size, COND_O, 0);
618 gen_four(label_ovf);
619 }
620 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
621 return true;
622 #endif
623 #if defined(ARCH_ALPHA)
624 if (mode == MODE_INT && ARCH_SUPPORTS_TRAPS(op_size)) {
625 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
626 g(gen_frame_get(ctx, op_size, garbage, slot_1, R_SCRATCH_1, ®1));
627 g(gen_frame_get(ctx, op_size, garbage, slot_2, R_SCRATCH_2, ®2));
629 gen_insn(INSN_ALU_TRAP, op_size, ALU_MUL, ALU_WRITES_FLAGS(op_size, ALU_MUL, false, false, 0));
630 gen_one(target);
631 gen_one(reg1);
632 gen_one(reg2);
633 gen_four(label_ovf);
634 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
636 return true;
637 }
638 #endif
639 #if defined(ARCH_ARM32)
640 if (mode == MODE_INT && op_size == OP_SIZE_4) {
641 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_3);
642 g(gen_frame_get(ctx, op_size, garbage, slot_1, R_SCRATCH_1, ®1));
643 g(gen_frame_get(ctx, op_size, garbage, slot_2, R_SCRATCH_2, ®2));
645 gen_insn(INSN_MUL_L, OP_SIZE_NATIVE, 0, 0);
646 gen_one(target);
647 gen_one(R_SCRATCH_4);
648 gen_one(reg1);
649 gen_one(reg2);
651 gen_insn(INSN_CMP, OP_SIZE_NATIVE, 0, 1);
652 gen_one(R_SCRATCH_4);
653 gen_one(ARG_SHIFTED_REGISTER);
654 gen_one(ARG_SHIFT_ASR | 0x1f);
655 gen_one(target);
657 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_NE, 0);
658 gen_four(label_ovf);
660 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
662 return true;
663 }
664 #endif
665 #if defined(ARCH_ARM64)
666 if (mode == MODE_INT && op_size == OP_SIZE_4) {
667 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
668 g(gen_frame_get(ctx, op_size, op_size < OP_SIZE_4 ? sign_x : garbage, slot_1, R_SCRATCH_1, ®1));
669 g(gen_frame_get(ctx, op_size, op_size < OP_SIZE_4 ? sign_x : garbage, slot_2, R_SCRATCH_2, ®2));
670 gen_insn(INSN_ALU, OP_SIZE_8, ALU_MUL, ALU_WRITES_FLAGS(OP_SIZE_8, ALU_MUL, false, false, 0));
671 gen_one(target);
672 gen_one(ARG_EXTENDED_REGISTER);
673 gen_one(ARG_EXTEND_SXTW);
674 gen_one(reg1);
675 gen_one(ARG_EXTENDED_REGISTER);
676 gen_one(ARG_EXTEND_SXTW);
677 gen_one(reg2);
679 gen_insn(INSN_CMP, OP_SIZE_8, 0, 1);
680 gen_one(target);
681 gen_one(ARG_EXTENDED_REGISTER);
682 gen_one(ARG_EXTEND_SXTW);
683 gen_one(target);
685 gen_insn(INSN_JMP_COND, OP_SIZE_8, COND_NE, 0);
686 gen_four(label_ovf);
688 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
690 return true;
691 }
692 if (mode == MODE_INT && op_size == OP_SIZE_8) {
693 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
694 g(gen_frame_get(ctx, op_size, garbage, slot_1, R_SCRATCH_1, ®1));
695 g(gen_frame_get(ctx, op_size, garbage, slot_2, R_SCRATCH_2, ®2));
696 g(gen_3address_alu(ctx, OP_SIZE_8, ALU_SMULH, R_SCRATCH_3, reg1, reg2, 0));
698 g(gen_3address_alu(ctx, OP_SIZE_8, ALU_MUL, target, reg1, reg2, 0));
700 gen_insn(INSN_CMP, OP_SIZE_8, 0, 1);
701 gen_one(R_SCRATCH_3);
702 gen_one(ARG_SHIFTED_REGISTER);
703 gen_one(ARG_SHIFT_ASR | 0x3f);
704 gen_one(target);
706 gen_insn(INSN_JMP_COND, OP_SIZE_8, COND_NE, 0);
707 gen_four(label_ovf);
709 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
711 return true;
712 }
713 #endif
714 #if defined(ARCH_POWER)
715 if (mode == MODE_INT && op_size >= OP_SIZE_4) {
716 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
717 g(gen_frame_get(ctx, op_size, garbage, slot_1, R_SCRATCH_1, ®1));
718 g(gen_frame_get(ctx, op_size, garbage, slot_2, R_SCRATCH_2, ®2));
720 g(gen_3address_alu(ctx, op_size, ALU_MUL, target, reg1, reg2, 1));
722 gen_insn(INSN_JMP_COND, op_size, COND_O, 0);
723 gen_four(label_ovf);
725 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
727 return true;
728 }
729 #endif
730 #if defined(ARCH_LOONGARCH64) || (defined(ARCH_MIPS) && MIPS_R6) || defined(ARCH_RISCV64)
731 if (mode == MODE_INT && op_size == OP_SIZE_NATIVE) {
732 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
733 g(gen_frame_get(ctx, op_size, garbage, slot_1, R_SCRATCH_1, ®1));
734 g(gen_frame_get(ctx, op_size, garbage, slot_2, R_SCRATCH_2, ®2));
736 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_SMULH, R_SCRATCH_3, reg1, reg2, 0));
738 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_MUL, target, reg1, reg2, 0));
740 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_4, target, (8U << OP_SIZE_NATIVE) - 1, 0));
742 g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, R_SCRATCH_3, R_SCRATCH_4, COND_NE, label_ovf));
744 g(gen_frame_store(ctx, OP_SIZE_NATIVE, slot_r, 0, target));
746 return true;
747 }
748 #endif
749 #if defined(ARCH_S390)
750 if (mode == MODE_INT && op_size >= OP_SIZE_4 && likely(cpu_test_feature(CPU_FEATURE_misc_insn_ext_2))) {
751 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
752 g(gen_frame_load(ctx, op_size, sign_x, slot_1, 0, false, target));
753 g(gen_frame_load_op(ctx, op_size, sign_x, ALU_MUL, 1, slot_2, 0, false, target));
755 gen_insn(INSN_JMP_COND, op_size, COND_O, 0);
756 gen_four(label_ovf);
758 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
759 return true;
760 }
761 #endif
762 #if (defined(ARCH_MIPS) && !MIPS_R6) || defined(ARCH_S390)
763 #if defined(ARCH_MIPS)
764 if (mode == MODE_INT && op_size >= OP_SIZE_4)
765 #endif
766 #if defined(ARCH_S390)
767 if (mode == MODE_INT && op_size == OP_SIZE_4)
768 #endif
769 {
770 #if defined(ARCH_S390)
771 target = R_SCRATCH_1;
772 #else
773 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
774 #endif
775 g(gen_frame_get(ctx, op_size, sign_x, slot_1, R_SCRATCH_1, ®1));
776 g(gen_frame_get(ctx, op_size, sign_x, slot_2, R_SCRATCH_3, ®2));
778 gen_insn(INSN_MUL_L, op_size, 0, 0);
779 gen_one(target);
780 gen_one(R_SCRATCH_2);
781 gen_one(reg1);
782 gen_one(reg2);
784 g(gen_3address_rot_imm(ctx, op_size, ROT_SAR, R_SCRATCH_4, target, (8U << op_size) - 1, false));
786 g(gen_cmp_test_jmp(ctx, INSN_CMP, op_size, R_SCRATCH_2, R_SCRATCH_4, COND_NE, label_ovf));
788 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
789 return true;
790 }
791 #endif
792 if (mode == MODE_INT && op_size == OP_SIZE_NATIVE) {
793 g(gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, INT_binary_multiply_int8_t), op_size, slot_1, slot_2, slot_r, label_ovf));
794 return true;
795 }
797 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
798 if (op_size < OP_SIZE_NATIVE && mode == MODE_INT) {
799 g(gen_frame_get(ctx, op_size, sign_x, slot_1, R_SCRATCH_1, ®1));
800 g(gen_frame_get(ctx, op_size, sign_x, slot_2, R_SCRATCH_2, ®2));
802 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_MUL, target, reg1, reg2, 0));
803 } else {
804 g(gen_frame_load(ctx, op_size, sign_x, slot_1, 0, false, target));
805 g(gen_frame_load_op(ctx, op_size, sign_x, ALU_MUL, 0, slot_2, 0, false, target));
806 }
808 if (mode == MODE_INT) {
809 g(gen_cmp_extended(ctx, OP_SIZE_NATIVE, op_size, target, R_SCRATCH_2, label_ovf));
810 }
812 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
814 return true;
815 }
817 /**********
818 * DIVIDE *
819 **********/
820 do_divide: {
821 uint32_t attr_unused label_skip = 0; /* avoid warning */
822 uint32_t attr_unused label_skip2 = 0; /* avoid warning */
823 uint32_t attr_unused label_end = 0; /* avoid warning */
824 uint32_t attr_unused label_div_0 = 0; /* avoid warning */
825 unsigned attr_unused divide_alu = 0; /* avoid warning */
826 bool attr_unused have_mod = false;
827 bool attr_unused force_sx = false;
828 unsigned attr_unused div_op_size = i_size(op_size);
829 if (unlikely(op_size > OP_SIZE_NATIVE) || unlikely(!ARCH_HAS_DIV)
830 #if defined(ARCH_S390)
831 || !(Z || (op_size <= OP_SIZE_4 && sgn))
832 #endif
833 ) {
834 size_t upcall;
835 if (mode == MODE_INT) {
836 upcall = !mod ? offsetof(struct cg_upcall_vector_s, INT_binary_divide_int8_t) : offsetof(struct cg_upcall_vector_s, INT_binary_modulo_int8_t);
837 } else if (sgn) {
838 upcall = !mod ? offsetof(struct cg_upcall_vector_s, FIXED_binary_divide_int8_t) : offsetof(struct cg_upcall_vector_s, FIXED_binary_modulo_int8_t);
839 } else {
840 upcall = !mod ? offsetof(struct cg_upcall_vector_s, FIXED_binary_udivide_int8_t) : offsetof(struct cg_upcall_vector_s, FIXED_binary_umodulo_int8_t);
841 }
842 g(gen_alu_typed_upcall(ctx, upcall, op_size, slot_1, slot_2, slot_r, mode == MODE_INT ? label_ovf : 0));
843 return true;
844 }
845 #if defined(ARCH_X86) || defined(ARCH_S390)
846 if (mode == MODE_FIXED) {
847 label_skip = alloc_label(ctx);
848 if (unlikely(!label_skip))
849 return false;
850 label_end = alloc_label(ctx);
851 if (unlikely(!label_end))
852 return false;
853 if (sgn) {
854 label_skip2 = alloc_label(ctx);
855 if (unlikely(!label_skip2))
856 return false;
857 }
858 }
859 #if defined(ARCH_X86)
860 if (R_SCRATCH_1 != R_AX || R_SCRATCH_2 != R_DX || R_SCRATCH_3 != R_CX)
861 internal(file_line, "gen_alu: bad scratch registers");
862 #endif
863 g(gen_frame_load(ctx, op_size, sgn ? sign_x : zero_x, slot_1, 0, false, R_SCRATCH_1));
864 g(gen_frame_load(ctx, op_size, sgn ? sign_x : zero_x, slot_2, 0, false, R_SCRATCH_3));
866 g(gen_jmp_on_zero(ctx, i_size(op_size), R_SCRATCH_3, COND_E, mode == MODE_INT ? label_ovf : label_skip));
868 if (sgn) {
869 uint64_t val;
870 uint32_t label_not_minus_1;
871 label_not_minus_1 = alloc_label(ctx);
872 if (unlikely(!label_not_minus_1))
873 return false;
875 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, op_size, R_SCRATCH_3, -1, COND_NE, label_not_minus_1));
877 val = -(uint64_t)0x80 << (((1 << op_size) - 1) * 8);
878 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, op_size, R_SCRATCH_1, val, COND_E, mode == MODE_INT ? label_ovf : label_skip2));
880 gen_label(label_not_minus_1);
881 }
883 #if defined(ARCH_X86)
884 if (op_size >= OP_SIZE_2) {
885 if (sgn) {
886 gen_insn(INSN_CWD + ARCH_PARTIAL_ALU(op_size), op_size, 0, 0);
887 gen_one(R_SCRATCH_2);
888 gen_one(R_SCRATCH_1);
889 if (op_size == OP_SIZE_2)
890 gen_one(R_SCRATCH_2);
891 } else {
892 g(gen_3address_alu(ctx, OP_SIZE_4, ALU_XOR, R_SCRATCH_2, R_SCRATCH_2, R_SCRATCH_2, 0));
893 }
894 }
895 gen_insn(INSN_DIV_L, op_size, sgn, 1);
896 gen_one(R_SCRATCH_1);
897 gen_one(i_size(op_size) == OP_SIZE_1 ? R_SCRATCH_1 : R_SCRATCH_2);
898 gen_one(R_SCRATCH_1);
899 gen_one(i_size(op_size) == OP_SIZE_1 ? R_SCRATCH_1 : R_SCRATCH_2);
900 gen_one(R_SCRATCH_3);
901 #else
902 if (!sgn) {
903 g(gen_load_constant(ctx, R_SCRATCH_2, 0));
904 } else if (op_size <= OP_SIZE_4) {
905 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_2, R_SCRATCH_1, (1U << (OP_SIZE_NATIVE + 3)) - 1, false));
906 }
907 gen_insn(INSN_DIV_L, i_size(op_size), sgn, 1);
908 gen_one(R_SCRATCH_2);
909 gen_one(R_SCRATCH_1);
910 gen_one(R_SCRATCH_2);
911 gen_one(R_SCRATCH_1);
912 gen_one(R_SCRATCH_3);
913 #endif
914 if (mod && i_size(op_size) == OP_SIZE_1) {
915 g(gen_3address_rot_imm(ctx, OP_SIZE_2, ROT_SHR, R_SCRATCH_1, R_SCRATCH_1, 8, 0));
916 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
917 } else if (mod) {
918 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_2));
919 } else {
920 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
921 }
922 if (mode == MODE_FIXED) {
923 gen_insn(INSN_JMP, 0, 0, 0);
924 gen_four(label_end);
926 if (sgn) {
927 gen_label(label_skip2);
929 if (mod)
930 g(gen_frame_clear(ctx, op_size, slot_r));
931 else
932 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
934 gen_insn(INSN_JMP, 0, 0, 0);
935 gen_four(label_end);
936 }
938 gen_label(label_skip);
939 if (!mod)
940 g(gen_frame_clear(ctx, op_size, slot_r));
941 else
942 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
943 gen_label(label_end);
944 }
945 return true;
946 #else
947 #if defined(ARCH_MIPS)
948 have_mod = true;
949 div_op_size = maximum(op_size, OP_SIZE_4);
950 if (op_size == OP_SIZE_4)
951 force_sx = true;
952 #endif
953 #if defined(ARCH_POWER)
954 have_mod = cpu_test_feature(CPU_FEATURE_v30);
955 div_op_size = maximum(op_size, OP_SIZE_4);
956 #endif
957 #if defined(ARCH_LOONGARCH64) || defined(ARCH_RISCV64)
958 have_mod = true;
959 div_op_size = maximum(op_size, OP_SIZE_4);
960 #endif
961 label_end = alloc_label(ctx);
962 if (unlikely(!label_end))
963 return false;
965 g(gen_frame_get(ctx, op_size, (sgn && op_size < i_size(op_size)) || force_sx ? sign_x : zero_x, slot_1, R_SCRATCH_1, ®1));
966 g(gen_frame_get(ctx, op_size, (sgn && op_size < i_size(op_size)) || force_sx ? sign_x : zero_x, slot_2, R_SCRATCH_2, ®2));
967 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_3);
969 if (ARCH_PREFERS_SX(op_size) && !sgn && op_size < i_size(op_size)) {
970 g(gen_extend(ctx, op_size, zero_x, R_SCRATCH_1, reg1));
971 reg1 = R_SCRATCH_1;
972 g(gen_extend(ctx, op_size, zero_x, R_SCRATCH_2, reg2));
973 reg2 = R_SCRATCH_2;
974 }
976 if (mode == MODE_INT) {
977 g(gen_jmp_on_zero(ctx, i_size(op_size), reg2, COND_E, label_ovf));
978 if (sgn) {
979 uint64_t val;
980 uint32_t label_not_minus_1;
981 label_not_minus_1 = alloc_label(ctx);
982 if (unlikely(!label_not_minus_1))
983 return false;
985 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, i_size(op_size), reg2, -1, COND_NE, label_not_minus_1));
987 val = 0xFFFFFFFFFFFFFF80ULL << (((1 << op_size) - 1) * 8);
988 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, i_size(op_size), reg1, val, COND_E, label_ovf));
990 gen_label(label_not_minus_1);
991 }
992 } else {
993 #if !(defined(ARCH_ARM) && ARM_ASM_DIV_NO_TRAP)
994 if (!mod) {
995 g(gen_load_constant(ctx, target, 0));
996 } else {
997 g(gen_mov(ctx, OP_SIZE_NATIVE, target, reg1));
998 }
999 g(gen_jmp_on_zero(ctx, i_size(op_size), reg2, COND_E, label_end));
1000 if (sgn) {
1001 uint64_t val;
1002 uint32_t label_not_minus_1;
1003 label_not_minus_1 = alloc_label(ctx);
1004 if (unlikely(!label_not_minus_1))
1005 return false;
1007 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, i_size(op_size), reg2, -1, COND_NE, label_not_minus_1));
1009 if (!mod) {
1010 g(gen_mov(ctx, OP_SIZE_NATIVE, target, reg1));
1011 } else {
1012 g(gen_load_constant(ctx, target, 0));
1013 }
1015 val = 0xFFFFFFFFFFFFFF80ULL << (((1 << op_size) - 1) * 8);
1016 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, i_size(op_size), reg1, val, COND_E, label_end));
1018 gen_label(label_not_minus_1);
1019 }
1020 #endif
1021 }
1022 if (mod && have_mod) {
1023 g(gen_3address_alu(ctx, div_op_size, sgn ? ALU_SREM : ALU_UREM, target, reg1, reg2, 0));
1024 } else {
1025 g(gen_3address_alu(ctx, div_op_size, sgn ? ALU_SDIV : ALU_UDIV, target, reg1, reg2, 0));
1026 }
1028 if (mod && !have_mod) {
1029 #if defined(ARCH_ARM)
1030 gen_insn(INSN_MADD, i_size(op_size), 1, 0);
1031 gen_one(target);
1032 gen_one(target);
1033 gen_one(reg2);
1034 gen_one(reg1);
1035 #else
1036 g(gen_3address_alu(ctx, i_size(op_size), ALU_MUL, R_SCRATCH_2, reg2, target, 0));
1037 g(gen_3address_alu(ctx, i_size(op_size), ALU_SUB, target, reg1, R_SCRATCH_2, 0));
1038 #endif
1039 }
1041 gen_label(label_end);
1042 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1043 return true;
1044 #endif
1045 }
1046 /*********
1047 * SHIFT *
1048 *********/
1049 do_shift: {
1050 bool sx;
1051 bool must_mask;
1052 unsigned op_s;
1053 int64_t cnst = 0; /* avoid warning */
1054 bool c = frame_t_is_const(slot_2);
1055 if (unlikely(op_size > OP_SIZE_NATIVE)) {
1056 size_t upcall;
1057 if (mode == MODE_FIXED) {
1058 switch (alu) {
1059 case ROT_SHL: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_shl_,TYPE_INT_MAX));
1060 break;
1061 case ROT_SAR: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_shr_,TYPE_INT_MAX));
1062 break;
1063 case ROT_SHR: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_ushr_,TYPE_INT_MAX));
1064 break;
1065 case ROT_ROL: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_rol_,TYPE_INT_MAX));
1066 break;
1067 case ROT_ROR: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_ror_,TYPE_INT_MAX));
1068 break;
1069 default: internal(file_line, "do_alu: invalid shift %u", alu);
1070 }
1071 } else {
1072 switch (alu) {
1073 case ROT_SHL: upcall = offsetof(struct cg_upcall_vector_s, cat(INT_binary_shl_,TYPE_INT_MAX));
1074 break;
1075 case ROT_SAR: upcall = offsetof(struct cg_upcall_vector_s, cat(INT_binary_shr_,TYPE_INT_MAX));
1076 break;
1077 default: internal(file_line, "do_alu: invalid shift %u", alu);
1078 }
1079 }
1080 g(gen_alu_upcall(ctx, upcall, op_size, slot_1, slot_2, slot_r, mode == MODE_INT ? label_ovf : 0));
1081 return true;
1082 }
1083 op_s = i_size_rot(op_size);
1084 #if defined(ARCH_X86)
1085 if (slot_1 == slot_r && !slot_is_register(ctx, slot_1) && !(mode == MODE_INT && alu == ROT_SHL)) {
1086 int64_t offset = (size_t)slot_1 * slot_size;
1087 if (c) {
1088 cnst = frame_t_get_const(slot_2);
1089 if (mode == MODE_INT) {
1090 if ((uint64_t)cnst > (8U << op_size) - 1) {
1091 gen_insn(INSN_JMP, 0, 0, 0);
1092 gen_four(label_ovf);
1093 return true;
1094 }
1095 } else {
1096 cnst &= (8U << op_size) - 1;
1097 }
1098 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDR_OFFSET, op_size));
1099 gen_insn(INSN_ROT + ARCH_PARTIAL_ALU(op_size), op_size, alu, 1);
1100 gen_address_offset();
1101 gen_address_offset();
1102 gen_one(ARG_IMM);
1103 gen_eight(cnst);
1104 } else {
1105 g(gen_frame_load(ctx, op_size, garbage, slot_2, 0, false, R_SCRATCH_3));
1106 if (mode == MODE_INT) {
1107 int64_t imm = (8U << op_size) - 1;
1108 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, op_size, R_SCRATCH_3, imm, COND_A, label_ovf));
1109 } else if ((alu != ROT_ROL && alu != ROT_ROR) && op_size < OP_SIZE_4) {
1110 g(gen_3address_alu_imm(ctx, OP_SIZE_1, ALU_AND, R_SCRATCH_3, R_SCRATCH_3, (8U << op_size) - 1, 0));
1111 }
1112 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDR_OFFSET, op_size));
1113 gen_insn(INSN_ROT + ARCH_PARTIAL_ALU(op_size), op_size, alu, 1);
1114 gen_address_offset();
1115 gen_address_offset();
1116 gen_one(R_SCRATCH_3);
1117 }
1118 return true;
1119 }
1120 if (mode == MODE_INT && alu == ROT_SHL && op_size < OP_SIZE_NATIVE)
1121 op_s = op_size + 1;
1122 #endif
1123 must_mask = op_size < ARCH_SHIFT_SIZE;
1124 sx = (alu == ROT_SAR && op_size < op_s) || (alu == ROT_SHL && op_size < OP_SIZE_NATIVE && mode == MODE_INT);
1125 #if defined(ARCH_MIPS)
1126 sx |= op_size == OP_SIZE_4;
1127 #endif
1128 g(gen_frame_get(ctx, op_size, sx ? sign_x : zero_x, slot_1, R_SCRATCH_1, ®1));
1129 if (c) {
1130 reg3 = 0xff; /* avoid warning */
1131 cnst = frame_t_get_const(slot_2);
1132 } else
1133 #if defined(ARCH_X86)
1134 if (!ARCH_IS_3ADDRESS_ROT(alu, op_size)) {
1135 g(gen_frame_load(ctx, op_size, garbage, slot_2, 0, false, R_SCRATCH_3));
1136 reg3 = R_SCRATCH_3;
1137 } else
1138 #endif
1139 g(gen_frame_get(ctx, op_size, garbage, slot_2, R_SCRATCH_3, ®3));
1141 if (ARCH_PREFERS_SX(op_size) && !sx && op_size < op_s) {
1142 g(gen_extend(ctx, op_size, zero_x, R_SCRATCH_1, reg1));
1143 reg1 = R_SCRATCH_1;
1144 }
1146 if (mode == MODE_INT) {
1147 int64_t imm = (8U << op_size) - 1;
1148 if (c) {
1149 if ((uint64_t)cnst > (uint64_t)imm) {
1150 gen_insn(INSN_JMP, 0, 0, 0);
1151 gen_four(label_ovf);
1152 return true;
1153 }
1154 } else {
1155 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, i_size_cmp(op_size), reg3, imm, COND_A, label_ovf));
1156 }
1157 } else {
1158 #if defined(ARCH_ARM)
1159 if (alu == ROT_ROL) {
1160 if (c) {
1161 cnst = -(uint64_t)cnst;
1162 } else {
1163 g(gen_2address_alu1(ctx, OP_SIZE_4, ALU1_NEG, R_SCRATCH_3, reg3, 0));
1164 reg3 = R_SCRATCH_3;
1165 }
1166 alu = ROT_ROR;
1167 }
1168 #endif
1169 #if defined(ARCH_LOONGARCH64)
1170 if (alu == ROT_ROL && op_size >= OP_SIZE_4) {
1171 if (c) {
1172 cnst = -(uint64_t)cnst;
1173 } else {
1174 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NEG, R_SCRATCH_3, reg3, 0));
1175 reg3 = R_SCRATCH_3;
1176 }
1177 alu = ROT_ROR;
1178 }
1179 #endif
1180 #if defined(ARCH_MIPS)
1181 if (MIPS_HAS_ROT && alu == ROT_ROL && op_size >= OP_SIZE_4) {
1182 if (c) {
1183 cnst = -(uint64_t)cnst;
1184 } else {
1185 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NEG, R_SCRATCH_3, reg3, 0));
1186 reg3 = R_SCRATCH_3;
1187 }
1188 alu = ROT_ROR;
1189 }
1190 #endif
1191 #if defined(ARCH_POWER)
1192 if (alu == ROT_ROR && op_size >= OP_SIZE_4) {
1193 if (c) {
1194 cnst = -(uint64_t)cnst;
1195 } else {
1196 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NEG, R_SCRATCH_3, reg3, 0));
1197 reg3 = R_SCRATCH_3;
1198 }
1199 alu = ROT_ROL;
1200 }
1201 #endif
1202 #if defined(ARCH_S390)
1203 if (Z && alu == ROT_ROR && op_size >= OP_SIZE_4) {
1204 if (c) {
1205 cnst = -(uint64_t)cnst;
1206 } else {
1207 g(gen_2address_alu1(ctx, OP_SIZE_4, ALU1_NEG, R_SCRATCH_3, reg3, 0));
1208 reg3 = R_SCRATCH_3;
1209 }
1210 alu = ROT_ROL;
1211 }
1212 #endif
1213 if (c) {
1214 cnst &= (8U << op_size) - 1;
1215 } else if (must_mask) {
1216 g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_4), ALU_AND, R_SCRATCH_3, reg3, (8U << op_size) - 1, 0));
1217 reg3 = R_SCRATCH_3;
1218 }
1219 }
1221 #if defined(ARCH_X86)
1222 if (mode == MODE_INT && alu == ROT_SHL) {
1223 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_2);
1224 } else {
1225 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, slot_2, R_SCRATCH_2);
1226 }
1227 if (c) {
1228 g(gen_3address_rot_imm(ctx, op_s, alu, target, reg1, cnst, 0));
1229 } else {
1230 g(gen_3address_rot(ctx, op_s, alu, target, reg1, reg3));
1231 }
1233 if (mode == MODE_INT && alu == ROT_SHL) {
1234 if (op_size < OP_SIZE_NATIVE) {
1235 gen_insn(INSN_MOVSX, op_size, 0, 0);
1236 gen_one(R_SCRATCH_4);
1237 gen_one(target);
1239 g(gen_cmp_test_jmp(ctx, INSN_CMP, op_s, target, R_SCRATCH_4, COND_NE, label_ovf));
1240 } else {
1241 if (c) {
1242 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_4, target, cnst, 0));
1243 } else {
1244 g(gen_3address_rot(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_4, target, reg3));
1245 }
1247 g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, reg1, R_SCRATCH_4, COND_NE, label_ovf));
1248 }
1249 }
1250 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1251 return true;
1252 #endif
1253 #if defined(ARCH_ARM)
1254 if (op_size <= OP_SIZE_2 && alu == ROT_ROR) {
1255 gen_insn(INSN_ALU, OP_SIZE_4, ALU_OR, ALU_WRITES_FLAGS(OP_SIZE_4, ALU_OR, false, false, 0));
1256 gen_one(R_SCRATCH_1);
1257 gen_one(reg1);
1258 gen_one(ARG_SHIFTED_REGISTER);
1259 gen_one(ARG_SHIFT_LSL | (8U << op_size));
1260 gen_one(reg1);
1261 if (op_size == OP_SIZE_1)
1262 alu = ROT_SHR;
1263 reg1 = R_SCRATCH_1;
1264 }
1265 goto do_generic_shift;
1266 #endif
1267 #if defined(ARCH_LOONGARCH64)
1268 if (alu == ROT_ROR && op_size >= OP_SIZE_4)
1269 goto do_generic_shift;
1270 #endif
1271 #if defined(ARCH_MIPS)
1272 if (MIPS_HAS_ROT && alu == ROT_ROR && op_size >= OP_SIZE_4)
1273 goto do_generic_shift;
1274 #endif
1275 #if defined(ARCH_POWER)
1276 if (alu == ROT_ROL && op_size >= OP_SIZE_4)
1277 goto do_generic_shift;
1278 #endif
1279 #if defined(ARCH_RISCV64)
1280 if ((alu == ROT_ROL || alu == ROT_ROR) && likely(cpu_test_feature(CPU_FEATURE_zbb))) {
1281 if (likely(op_size >= OP_SIZE_4)) {
1282 goto do_generic_shift;
1283 }
1284 }
1285 #endif
1286 #if defined(ARCH_S390)
1287 if (Z && alu == ROT_ROL && op_size >= OP_SIZE_4)
1288 goto do_generic_shift;
1289 #endif
1290 if (alu == ROT_ROL || alu == ROT_ROR) {
1291 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
1292 if (c) {
1293 g(gen_3address_rot_imm(ctx, op_s, alu == ROT_ROL ? ROT_SHL : ROT_SHR, R_SCRATCH_2, reg1, cnst, 0));
1294 g(gen_3address_rot_imm(ctx, op_s, alu == ROT_ROL ? ROT_SHR : ROT_SHL, target, reg1, -(uint64_t)cnst & ((8U << op_size) - 1), 0));
1295 } else {
1296 g(gen_3address_rot(ctx, op_s, alu == ROT_ROL ? ROT_SHL : ROT_SHR, R_SCRATCH_2, reg1, reg3));
1297 g(gen_2address_alu1(ctx, i_size(OP_SIZE_4), ALU1_NEG, R_SCRATCH_3, reg3, 0));
1298 if (must_mask) {
1299 g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_4), ALU_AND, R_SCRATCH_3, R_SCRATCH_3, (8U << op_size) - 1, 0));
1300 }
1301 g(gen_3address_rot(ctx, op_s, alu == ROT_ROL ? ROT_SHR : ROT_SHL, target, reg1, R_SCRATCH_3));
1302 }
1303 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_OR, target, target, R_SCRATCH_2, 0));
1304 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1305 return true;
1306 }
1308 goto do_generic_shift;
1309 do_generic_shift:
1310 if (mode == MODE_INT && alu == ROT_SHL) {
1311 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
1312 #if defined(ARCH_S390)
1313 if (op_size >= OP_SIZE_4) {
1314 if (c)
1315 g(gen_3address_rot_imm(ctx, op_size, ROT_SAL, target, reg1, cnst, 0));
1316 else
1317 g(gen_3address_rot(ctx, op_size, ROT_SAL, target, reg1, reg3));
1319 gen_insn(INSN_JMP_COND, op_size, COND_O, 0);
1320 gen_four(label_ovf);
1321 } else
1322 #endif
1323 if (op_size <= OP_SIZE_NATIVE - 1) {
1324 if (c)
1325 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, alu, target, reg1, cnst, 0));
1326 else
1327 g(gen_3address_rot(ctx, OP_SIZE_NATIVE, alu, target, reg1, reg3));
1329 g(gen_cmp_extended(ctx, OP_SIZE_NATIVE, op_size, target, R_SCRATCH_2, label_ovf));
1330 } else {
1331 if (c) {
1332 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, alu, R_SCRATCH_2, reg1, cnst, 0));
1333 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_4, R_SCRATCH_2, cnst, 0));
1334 } else {
1335 g(gen_3address_rot(ctx, OP_SIZE_NATIVE, alu, R_SCRATCH_2, reg1, reg3));
1336 g(gen_3address_rot(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_4, R_SCRATCH_2, reg3));
1337 }
1339 g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, reg1, R_SCRATCH_4, COND_NE, label_ovf));
1341 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_2));
1343 return true;
1344 }
1345 } else {
1346 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, slot_2, R_SCRATCH_1);
1347 if (c)
1348 g(gen_3address_rot_imm(ctx, op_s, alu, target, reg1, cnst, 0));
1349 else
1350 g(gen_3address_rot(ctx, op_s, alu, target, reg1, reg3));
1351 }
1353 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1354 return true;
1355 }
1356 /******
1357 * BT *
1358 ******/
1359 do_bt: {
1360 unsigned attr_unused op_s;
1361 bool need_mask;
1362 bool c = frame_t_is_const(slot_2);
1363 int64_t cnst = !c ? 0 : frame_t_get_const(slot_2);
1364 int64_t max_imm = (8U << op_size) - 1;
1365 if (c) {
1366 if (mode == MODE_INT) {
1367 if (alu == BTX_BT ? (uint64_t)cnst > (uint64_t)max_imm : (uint64_t)cnst >= (uint64_t)max_imm) {
1368 gen_insn(INSN_JMP, 0, 0, 0);
1369 gen_four(label_ovf);
1370 return true;
1371 }
1372 }
1373 cnst &= max_imm;
1374 }
1375 #if defined(ARCH_X86)
1376 if ((alu == BTX_BT || slot_1 == slot_r) && !slot_is_register(ctx, slot_1)) {
1377 int64_t offset;
1378 unsigned n_op_size = minimum(op_size, OP_SIZE_NATIVE);
1379 g(gen_frame_get(ctx, n_op_size, garbage, slot_2, R_SCRATCH_1, ®2));
1380 if (mode == MODE_INT) {
1381 if (!c) {
1382 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, n_op_size, reg2, max_imm, alu == BTX_BT ? COND_A : COND_AE, label_ovf));
1383 if (unlikely(op_size > OP_SIZE_NATIVE)) {
1384 g(gen_address(ctx, R_FRAME, (size_t)slot_2 * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_STR_OFFSET, OP_SIZE_NATIVE));
1385 gen_insn(INSN_CMP, OP_SIZE_NATIVE, 0, 1);
1386 gen_address_offset();
1387 gen_one(ARG_IMM);
1388 gen_eight(0);
1389 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_NE, 0);
1390 gen_four(label_ovf);
1391 }
1392 }
1393 } else {
1394 if (!c) {
1395 g(gen_3address_alu_imm(ctx, OP_SIZE_4, ALU_AND, R_SCRATCH_1, reg2, (8U << op_size) - 1, 0));
1396 reg2 = R_SCRATCH_1;
1397 }
1398 }
1399 offset = (size_t)slot_1 * slot_size;
1400 if (c && cnst >= 8U << OP_SIZE_NATIVE) {
1401 offset += 1U << OP_SIZE_NATIVE;
1402 cnst -= 8U << OP_SIZE_NATIVE;
1403 }
1404 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_STR_OFFSET, maximum(n_op_size, OP_SIZE_2)));
1405 if (alu == BTX_BT) {
1406 gen_insn(INSN_BT, maximum(n_op_size, OP_SIZE_2), 0, 1);
1407 gen_address_offset();
1408 if (c) {
1409 gen_one(ARG_IMM);
1410 gen_eight(cnst);
1411 } else {
1412 gen_one(reg2);
1413 }
1414 g(gen_frame_set_cond(ctx, maximum(n_op_size, OP_SIZE_2), false, COND_B, slot_r));
1415 } else {
1416 gen_insn(INSN_BTX, maximum(n_op_size, OP_SIZE_2), alu, 1);
1417 gen_address_offset();
1418 gen_address_offset();
1419 if (c) {
1420 gen_one(ARG_IMM);
1421 gen_eight(cnst);
1422 } else {
1423 gen_one(reg2);
1424 }
1425 }
1426 return true;
1427 }
1428 #endif
1429 if (unlikely(op_size > OP_SIZE_NATIVE)) {
1430 size_t upcall;
1431 if (mode == MODE_FIXED) {
1432 switch (alu) {
1433 case BTX_BTS: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_bts_,TYPE_INT_MAX));
1434 break;
1435 case BTX_BTR: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_btr_,TYPE_INT_MAX));
1436 break;
1437 case BTX_BTC: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_btc_,TYPE_INT_MAX));
1438 break;
1439 case BTX_BT: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_bt_,TYPE_INT_MAX));
1440 break;
1441 default: internal(file_line, "do_alu: invalid bit test %u", alu);
1442 }
1443 } else {
1444 switch (alu) {
1445 case BTX_BTS: upcall = offsetof(struct cg_upcall_vector_s, cat(INT_binary_bts_,TYPE_INT_MAX));
1446 break;
1447 case BTX_BTR: upcall = offsetof(struct cg_upcall_vector_s, cat(INT_binary_btr_,TYPE_INT_MAX));
1448 break;
1449 case BTX_BTC: upcall = offsetof(struct cg_upcall_vector_s, cat(INT_binary_btc_,TYPE_INT_MAX));
1450 break;
1451 case BTX_BT: upcall = offsetof(struct cg_upcall_vector_s, cat(INT_binary_bt_,TYPE_INT_MAX));
1452 break;
1453 default: internal(file_line, "do_alu: invalid bit test %u", alu);
1454 }
1455 }
1456 g(gen_alu_upcall(ctx, upcall, op_size, slot_1, slot_2, slot_r, label_ovf));
1457 return true;
1458 }
1459 op_s = minimum(OP_SIZE_NATIVE, ARCH_SHIFT_SIZE);
1460 op_s = maximum(op_s, op_size);
1461 g(gen_frame_get(ctx, op_size, zero_x, slot_1, R_SCRATCH_1, ®1));
1462 if (c) {
1463 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1464 if (ARCH_HAS_BTX(alu == BTX_BT ? BTX_BTEXT : alu, OP_SIZE_NATIVE, true)) {
1465 #ifdef ARCH_X86
1466 g(gen_mov(ctx, OP_SIZE_NATIVE, target, reg1));
1467 reg1 = target;
1468 gen_insn(INSN_BTX, OP_SIZE_NATIVE, alu == BTX_BT ? BTX_BTEXT : alu, 1);
1469 #else
1470 gen_insn(INSN_BTX, OP_SIZE_NATIVE, alu == BTX_BT ? BTX_BTEXT : alu, 0);
1471 #endif
1472 gen_one(target);
1473 gen_one(reg1);
1474 gen_one(ARG_IMM);
1475 gen_eight(cnst);
1476 } else switch (alu) {
1477 case BTX_BTS:
1478 g(gen_3address_alu_imm(ctx, i_size(op_size), ALU_OR, target, reg1, 1ULL << cnst, 0));
1479 break;
1480 case BTX_BTR:
1481 g(gen_3address_alu_imm(ctx, i_size(op_size), ALU_AND, target, reg1, ~(1ULL << cnst), 0));
1482 break;
1483 case BTX_BTC:
1484 g(gen_3address_alu_imm(ctx, i_size(op_size), ALU_XOR, target, reg1, 1ULL << cnst, 0));
1485 break;
1486 case BTX_BT:
1487 if (cnst) {
1488 g(gen_3address_rot_imm(ctx, i_size(op_size), ROT_SHR, target, reg1, cnst, 0));
1489 }
1490 g(gen_3address_alu_imm(ctx, i_size(op_size), ALU_AND, target, target, 1, 0));
1491 break;
1492 default:
1493 internal(file_line, "do_alu: invalid bit test %u", alu);
1494 }
1495 if (alu == BTX_BT)
1496 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
1497 else
1498 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1499 return true;
1500 }
1501 g(gen_frame_get(ctx, op_size, garbage, slot_2, R_SCRATCH_2, ®2));
1502 if (mode == MODE_INT) {
1503 if (!c)
1504 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, i_size_cmp(op_size), reg2, max_imm, alu == BTX_BT ? COND_A : COND_AE, label_ovf));
1505 }
1506 if (alu != BTX_BT) {
1507 if (!ARCH_HAS_BTX(alu, OP_SIZE_NATIVE, false))
1508 goto do_generic_bt;
1509 need_mask = !ARCH_HAS_BTX(alu, op_size, false);
1510 } else {
1511 #if defined(ARCH_X86)
1512 need_mask = op_size < OP_SIZE_2;
1513 #else
1514 if (!ARCH_HAS_BTX(BTX_BTEXT, OP_SIZE_NATIVE, false))
1515 goto do_generic_bt;
1516 need_mask = !ARCH_HAS_BTX(BTX_BTEXT, op_size, false);
1517 #endif
1518 }
1519 if (need_mask) {
1520 g(gen_3address_alu_imm(ctx, OP_SIZE_4, ALU_AND, R_SCRATCH_2, reg2, (8U << op_size) - 1, 0));
1521 reg2 = R_SCRATCH_2;
1522 }
1523 if (alu == BTX_BT) {
1524 #if defined(ARCH_X86)
1525 gen_insn(INSN_BT, maximum(op_size, OP_SIZE_2), 0, 1);
1526 gen_one(reg1);
1527 gen_one(reg2);
1529 g(gen_frame_set_cond(ctx, maximum(op_size, OP_SIZE_2), false, COND_B, slot_r));
1530 #else
1531 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1532 gen_insn(INSN_BTX, need_mask ? OP_SIZE_NATIVE : op_size, BTX_BTEXT, 0);
1533 gen_one(target);
1534 gen_one(reg1);
1535 gen_one(reg2);
1537 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
1538 #endif
1539 } else {
1540 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, slot_2, R_SCRATCH_1);
1541 #if defined(ARCH_X86)
1542 if (target == reg2)
1543 target = R_SCRATCH_1;
1544 if (target != reg1) {
1545 g(gen_mov(ctx, op_size, target, reg1));
1546 reg1 = target;
1547 }
1548 gen_insn(INSN_BTX, maximum(op_size, OP_SIZE_2), alu, 1);
1549 #else
1550 gen_insn(INSN_BTX, need_mask ? OP_SIZE_NATIVE : op_size, alu, 0);
1551 #endif
1552 gen_one(target);
1553 gen_one(reg1);
1554 gen_one(reg2);
1556 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1557 }
1558 return true;
1560 goto do_generic_bt;
1561 do_generic_bt:
1562 if (mode == MODE_FIXED && op_size < ARCH_SHIFT_SIZE) {
1563 g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_4), ALU_AND, R_SCRATCH_2, reg2, (8U << op_size) - 1, 0));
1564 reg2 = R_SCRATCH_2;
1565 }
1566 g(gen_load_constant(ctx, R_SCRATCH_3, 1));
1568 g(gen_3address_rot(ctx, op_s, ROT_SHL, R_SCRATCH_3, R_SCRATCH_3, reg2));
1570 switch (alu) {
1571 case BTX_BT:
1572 #if ARCH_HAS_FLAGS
1573 #if defined(ARCH_S390) || defined(ARCH_POWER)
1574 g(gen_3address_alu(ctx, i_size(op_size), ALU_AND, R_SCRATCH_1, reg1, R_SCRATCH_3, 1));
1575 #else
1576 gen_insn(INSN_TEST, i_size(op_size), 0, 1);
1577 gen_one(reg1);
1578 gen_one(R_SCRATCH_3);
1579 #endif
1580 g(gen_frame_set_cond(ctx, i_size_cmp(op_size), false, COND_NE, slot_r));
1581 #else
1582 g(gen_3address_alu(ctx, i_size(op_size), ALU_AND, R_SCRATCH_1, reg1, R_SCRATCH_3, 0));
1583 g(gen_frame_cmp_imm_set_cond_reg(ctx, i_size(op_size), R_SCRATCH_1, 0, COND_NE, slot_r));
1584 #endif
1585 return true;
1586 case BTX_BTS:
1587 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1588 g(gen_3address_alu(ctx, i_size(op_size), ALU_OR, target, reg1, R_SCRATCH_3, 0));
1589 break;
1590 case BTX_BTR:
1591 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1592 if (!ARCH_HAS_ANDN) {
1593 g(gen_3address_alu_imm(ctx, i_size(op_size), ALU_XOR, R_SCRATCH_3, R_SCRATCH_3, -1, 0));
1595 g(gen_3address_alu(ctx, i_size(op_size), ALU_AND, target, reg1, R_SCRATCH_3, 0));
1596 break;
1597 }
1598 g(gen_3address_alu(ctx, i_size(op_size), ALU_ANDN, target, reg1, R_SCRATCH_3, 0));
1599 break;
1600 case BTX_BTC:
1601 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1602 g(gen_3address_alu(ctx, i_size(op_size), ALU_XOR, target, reg1, R_SCRATCH_3, 0));
1603 break;
1604 default:
1605 internal(file_line, "gen_alu: unsupported bit test %u", alu);
1606 }
1608 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1610 return true;
1611 }
1612 /***********
1613 * COMPARE *
1614 ***********/
1615 do_compare: {
1616 if (unlikely(op_size > OP_SIZE_NATIVE)) {
1617 size_t attr_unused upcall;
1618 frame_t attr_unused swap;
1619 switch (alu) {
1620 case COND_E:
1621 case COND_NE:
1622 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
1623 g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, ALU_XOR, 0, slot_2, lo_word(OP_SIZE_NATIVE), true, R_SCRATCH_1));
1624 g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, ALU_XOR, 0, slot_2, hi_word(OP_SIZE_NATIVE), true, R_SCRATCH_2));
1625 #if defined(ARCH_ARM64)
1626 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_OR, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_2, 0));
1628 gen_insn(INSN_CMP, OP_SIZE_NATIVE, 0, 1);
1629 gen_one(R_SCRATCH_1);
1630 gen_one(ARG_IMM);
1631 gen_eight(0);
1632 #else
1633 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_OR, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_2, ARCH_HAS_FLAGS));
1634 #endif
1635 #if ARCH_HAS_FLAGS
1636 g(gen_frame_set_cond(ctx, OP_SIZE_NATIVE, false, alu, slot_r));
1637 #else
1638 g(gen_frame_cmp_imm_set_cond_reg(ctx, OP_SIZE_NATIVE, R_SCRATCH_1, 0, alu, slot_r));
1639 #endif
1640 return true;
1641 #if defined(ARCH_X86_64) || defined(ARCH_X86_X32) || defined(ARCH_ARM)
1642 case COND_G:
1643 case COND_A:
1644 swap = slot_1; slot_1 = slot_2; slot_2 = swap;
1645 alu = alu == COND_G ? COND_L : COND_B;
1646 /*-fallthrough*/
1647 case COND_L:
1648 case COND_B:
1649 g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_2, lo_word(OP_SIZE_NATIVE), true, R_SCRATCH_2));
1650 g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_1, hi_word(OP_SIZE_NATIVE), true, R_SCRATCH_1));
1651 g(gen_frame_load_cmp(ctx, OP_SIZE_NATIVE, false, garbage, true, slot_1, lo_word(OP_SIZE_NATIVE), true, R_SCRATCH_2));
1652 g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, ALU_SBB, 1, slot_2, hi_word(OP_SIZE_NATIVE), true, R_SCRATCH_1));
1653 g(gen_frame_set_cond(ctx, OP_SIZE_NATIVE, false, alu, slot_r));
1654 return true;
1655 case COND_GE:
1656 case COND_AE:
1657 swap = slot_1; slot_1 = slot_2; slot_2 = swap;
1658 alu = alu == COND_GE ? COND_LE : COND_BE;
1659 /*-fallthrough*/
1660 case COND_LE:
1661 case COND_BE:
1662 g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_1, lo_word(OP_SIZE_NATIVE), true, R_SCRATCH_2));
1663 g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_2, hi_word(OP_SIZE_NATIVE), true, R_SCRATCH_1));
1664 g(gen_frame_load_cmp(ctx, OP_SIZE_NATIVE, false, garbage, true, slot_2, lo_word(OP_SIZE_NATIVE), true, R_SCRATCH_2));
1665 g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, ALU_SBB, 1, slot_1, hi_word(OP_SIZE_NATIVE), true, R_SCRATCH_1));
1666 g(gen_frame_set_cond(ctx, OP_SIZE_NATIVE, false, alu == COND_LE ? COND_GE : COND_AE, slot_r));
1667 return true;
1668 #else
1669 case COND_L: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_less_,TYPE_INT_MAX)); goto do_upcall;
1670 case COND_LE: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_less_equal_,TYPE_INT_MAX)); goto do_upcall;
1671 case COND_G: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_greater_,TYPE_INT_MAX)); goto do_upcall;
1672 case COND_GE: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_greater_equal_,TYPE_INT_MAX)); goto do_upcall;
1673 case COND_B: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_uless_,TYPE_INT_MAX)); goto do_upcall;
1674 case COND_BE: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_uless_equal_,TYPE_INT_MAX)); goto do_upcall;
1675 case COND_A: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_ugreater_,TYPE_INT_MAX)); goto do_upcall;
1676 case COND_AE: upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_ugreater_equal_,TYPE_INT_MAX)); goto do_upcall;
1677 do_upcall: g(gen_alu_upcall(ctx, upcall, op_size, slot_1, slot_2, slot_r, 0));
1678 return true;
1679 #endif
1680 default:
1681 internal(file_line, "gen_alu: unsupported condition %u", alu);
1682 }
1683 return false;
1684 }
1685 #if defined(ARCH_X86)
1686 g(gen_frame_get(ctx, op_size, garbage, slot_1, R_SCRATCH_1, ®1));
1687 g(gen_frame_load_cmp_set_cond(ctx, op_size, garbage, slot_2, reg1, alu, slot_r));
1688 #else
1689 g(gen_frame_get(ctx, op_size, op_size == i_size_cmp(op_size) + (unsigned)zero ? garbage : alu == COND_L || alu == COND_LE || alu == COND_G || alu == COND_GE || ARCH_PREFERS_SX(op_size) ? sign_x : zero_x, slot_1, R_SCRATCH_1, ®1));
1690 g(gen_frame_load_cmp_set_cond(ctx, op_size, alu == COND_L || alu == COND_LE || alu == COND_G || alu == COND_GE || ARCH_PREFERS_SX(op_size) ? sign_x : zero_x, slot_2, reg1, alu, slot_r));
1691 #endif
1692 return true;
1693 }
1694 }
1696 static bool attr_w gen_alu1(struct codegen_context *ctx, unsigned mode, unsigned op_size, unsigned op, uint32_t label_ovf, frame_t slot_1, frame_t slot_r)
1697 {
1698 unsigned alu;
1699 unsigned reg1, target;
1700 switch (mode) {
1701 case MODE_FIXED: switch (op) {
1702 case OPCODE_FIXED_OP_not: alu = ALU1_NOT; goto do_alu;
1703 case OPCODE_FIXED_OP_neg: alu = ALU1_NEG; goto do_alu;
1704 case OPCODE_FIXED_OP_bswap:
1705 case OPCODE_FIXED_OP_bswap_alt1: alu = ALU1_BSWAP; goto do_bswap;
1706 case OPCODE_FIXED_OP_brev:
1707 case OPCODE_FIXED_OP_brev_alt1: alu = ALU1_BREV; goto do_brev;
1708 case OPCODE_FIXED_OP_bsf:
1709 case OPCODE_FIXED_OP_bsf_alt1: alu = ALU1_BSF; goto do_bsf_bsr_popcnt;
1710 case OPCODE_FIXED_OP_bsr:
1711 case OPCODE_FIXED_OP_bsr_alt1: alu = ALU1_BSR; goto do_bsf_bsr_popcnt;
1712 case OPCODE_FIXED_OP_popcnt:
1713 case OPCODE_FIXED_OP_popcnt_alt1: alu = ALU1_POPCNT; goto do_bsf_bsr_popcnt;
1714 case OPCODE_FIXED_OP_to_int: goto do_fixed_conv;
1715 case OPCODE_FIXED_OP_from_int: goto do_fixed_conv;
1716 case OPCODE_FIXED_OP_uto_int: goto conv_uto_int;
1717 case OPCODE_FIXED_OP_ufrom_int: goto conv_ufrom_int;
1718 default: internal(file_line, "gen_alu1: unsupported fixed operation %u", op);
1719 }
1720 case MODE_INT: switch (op) {
1721 case OPCODE_INT_OP_not: alu = ALU1_NOT; mode = MODE_FIXED; goto do_alu;
1722 case OPCODE_INT_OP_neg: alu = ALU1_NEG; goto do_alu;
1723 case OPCODE_INT_OP_bsf: alu = ALU1_BSF; goto do_bsf_bsr_popcnt;
1724 case OPCODE_INT_OP_bsr: alu = ALU1_BSR; goto do_bsf_bsr_popcnt;
1725 case OPCODE_INT_OP_popcnt:
1726 case OPCODE_INT_OP_popcnt_alt1: alu = ALU1_POPCNT; goto do_bsf_bsr_popcnt;
1727 case OPCODE_INT_OP_to_int: goto do_conv;
1728 case OPCODE_INT_OP_from_int: goto do_conv;
1729 default: internal(file_line, "gen_alu1: unsupported int operation %u", op);
1730 }
1731 case MODE_BOOL: switch (op) {
1732 case OPCODE_BOOL_OP_not: goto do_bool_not;
1733 default: internal(file_line, "gen_alu1: unsupported bool operation %u", op);
1734 }
1735 }
1736 internal(file_line, "gen_alu1: unsupported mode %u", mode);
1738 /*******
1739 * ALU *
1740 *******/
1741 do_alu: {
1742 bool arch_use_flags = ARCH_HAS_FLAGS;
1743 enum extend ex;
1744 #if defined(ARCH_POWER)
1745 arch_use_flags = false;
1746 #endif
1747 if (op_size > OP_SIZE_NATIVE) {
1748 #if !defined(ARCH_X86) && !defined(ARCH_ARM) && !defined(ARCH_POWER)
1749 if (alu == ALU1_NEG) {
1750 if (mode == MODE_FIXED)
1751 g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(FIXED_unary_neg_,TYPE_INT_MAX)), op_size, slot_1, NO_FRAME_T, slot_r, 0));
1752 else
1753 g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(INT_unary_neg_,TYPE_INT_MAX)), op_size, slot_1, NO_FRAME_T, slot_r, label_ovf));
1754 return true;
1755 }
1756 #endif
1757 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
1758 #if defined(ARCH_S390)
1759 if (alu == ALU1_NOT) {
1760 g(gen_load_constant(ctx, R_SCRATCH_3, -1));
1762 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_XOR, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_3, 0));
1763 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_XOR, R_SCRATCH_2, R_SCRATCH_2, R_SCRATCH_3, 0));
1765 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_1, R_SCRATCH_2));
1766 return true;
1767 }
1768 #endif
1769 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, alu, R_SCRATCH_1, R_SCRATCH_1, alu == ALU1_NEG ? 2 : 0));
1770 if (alu == ALU1_NOT) {
1771 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NOT, R_SCRATCH_2, R_SCRATCH_2, 0));
1772 } else {
1773 #if defined(ARCH_X86)
1774 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NOT, R_SCRATCH_2, R_SCRATCH_2, 0));
1776 g(gen_imm(ctx, -1, IMM_PURPOSE_SUB, OP_SIZE_NATIVE));
1777 gen_insn(INSN_ALU, OP_SIZE_NATIVE, ALU_SBB, ALU_WRITES_FLAGS(OP_SIZE_NATIVE, ALU_SBB, false, is_imm(), ctx->const_imm));
1778 gen_one(R_SCRATCH_2);
1779 gen_one(R_SCRATCH_2);
1780 gen_imm_offset();
1781 #else
1782 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NGC, R_SCRATCH_2, R_SCRATCH_2, (mode == MODE_INT)));
1783 #endif
1784 }
1785 if (mode == MODE_INT) {
1786 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_O, 0);
1787 gen_four(label_ovf);
1788 }
1789 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_1, R_SCRATCH_2));
1790 return true;
1791 }
1792 if ((arch_use_flags || ARCH_SUPPORTS_TRAPS(op_size)) && slot_1 == slot_r && i_size_cmp(op_size) == op_size + zero) {
1793 struct cg_exit *ce;
1794 unsigned undo_alu = alu;
1795 if (slot_is_register(ctx, slot_1)) {
1796 unsigned reg = ctx->registers[slot_1];
1797 if (mode == MODE_INT && ARCH_SUPPORTS_TRAPS(op_size)) {
1798 gen_insn(INSN_ALU1_TRAP, op_size, alu, ALU1_WRITES_FLAGS(alu));
1799 gen_one(reg);
1800 gen_one(reg);
1801 if (ARCH_TRAP_BEFORE || alu == undo_alu) {
1802 gen_four(label_ovf);
1803 return true;
1804 } else {
1805 ce = alloc_undo_label(ctx);
1806 if (unlikely(!ce))
1807 return false;
1808 gen_four(ce->undo_label);
1809 goto do_undo_opcode;
1810 }
1811 }
1812 g(gen_2address_alu1(ctx, i_size(op_size), alu, reg, reg, mode == MODE_INT));
1813 if (mode == MODE_INT) {
1814 if (alu != undo_alu) {
1815 ce = alloc_undo_label(ctx);
1816 if (unlikely(!ce))
1817 return false;
1818 gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
1819 gen_four(ce->undo_label);
1820 do_undo_opcode:
1821 ce->undo_opcode = INSN_ALU1 + ARCH_PARTIAL_ALU(op_size);
1822 ce->undo_op_size = i_size(op_size);
1823 ce->undo_aux = undo_alu;
1824 ce->undo_writes_flags = ALU1_WRITES_FLAGS(undo_alu);
1825 ce->undo_parameters[0] = reg;
1826 ce->undo_parameters[1] = reg;
1827 ce->undo_parameters_len = 2;
1828 } else {
1829 gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
1830 gen_four(label_ovf);
1831 }
1832 }
1833 return true;
1834 }
1835 #if defined(ARCH_X86)
1836 else {
1837 size_t m;
1838 int64_t offset = (size_t)slot_1 * slot_size;
1839 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDR_OFFSET, i_size(op_size)));
1840 gen_insn(INSN_ALU1 + ARCH_PARTIAL_ALU(op_size), i_size(op_size), alu, ALU1_WRITES_FLAGS(alu) | (mode == MODE_INT));
1841 gen_address_offset();
1842 gen_address_offset();
1843 if (mode == MODE_INT) {
1844 if (alu != undo_alu) {
1845 ce = alloc_undo_label(ctx);
1846 if (unlikely(!ce))
1847 return false;
1848 ce->undo_opcode = INSN_ALU1 + ARCH_PARTIAL_ALU(op_size);
1849 ce->undo_op_size = i_size(op_size);
1850 ce->undo_aux = undo_alu;
1851 ce->undo_writes_flags = ALU1_WRITES_FLAGS(undo_alu);
1852 m = mark_params(ctx);
1853 gen_address_offset();
1854 gen_address_offset();
1855 copy_params(ctx, ce, m);
1856 gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
1857 gen_four(ce->undo_label);
1858 } else {
1859 gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
1860 gen_four(label_ovf);
1861 }
1862 }
1863 return true;
1864 }
1865 #endif
1866 }
1867 target = gen_frame_target(ctx, slot_r, mode == MODE_INT ? slot_1 : NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1868 if (mode == MODE_FIXED) {
1869 ex = garbage;
1870 } else {
1871 ex = ARCH_PREFERS_SX(op_size) ? sign_x : zero_x;
1872 if (ARCH_SUPPORTS_TRAPS(op_size))
1873 ex = garbage;
1874 if (op_size == i_size(op_size) + (unsigned)zero)
1875 ex = garbage;
1876 }
1877 g(gen_frame_get(ctx, op_size, ex, slot_1, mode == MODE_INT ? R_SCRATCH_2 : target, ®1));
1878 #if defined(ARCH_S390)
1879 if (alu == ALU1_NOT) {
1880 g(gen_3address_alu_imm(ctx, i_size(op_size), ALU_XOR, target, reg1, -1, 0));
1882 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1883 return true;
1884 }
1885 #endif
1886 #if defined(ARCH_X86)
1887 g(gen_2address_alu1(ctx, op_size, alu, target, reg1, mode == MODE_INT));
1888 #else
1889 if (mode == MODE_INT) {
1890 #if defined(ARCH_POWER)
1891 if (op_size == OP_SIZE_NATIVE) {
1892 g(gen_2address_alu1(ctx, i_size(op_size), alu, target, reg1, 0));
1893 if (alu == ALU1_NEG) {
1894 g(gen_3address_alu(ctx, i_size(op_size), ALU_AND, R_CG_SCRATCH, target, reg1, 1));
1895 }
1896 gen_insn(INSN_JMP_COND, op_size, COND_L, 0);
1897 gen_four(label_ovf);
1899 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1901 return true;
1902 }
1903 #endif
1904 if (!arch_use_flags && !ARCH_SUPPORTS_TRAPS(op_size) && ARCH_HAS_ANDN && op_size >= OP_SIZE_4) {
1905 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, alu, target, reg1, 0));
1907 if (alu == ALU1_NEG) {
1908 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_AND, R_SCRATCH_3, target, reg1, 0));
1909 }
1911 if (op_size < OP_SIZE_NATIVE)
1912 g(gen_extend(ctx, op_size, sign_x, R_SCRATCH_3, R_SCRATCH_3));
1914 g(gen_jmp_on_zero(ctx, OP_SIZE_NATIVE, R_SCRATCH_3, COND_S, label_ovf));
1916 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1918 return true;
1919 }
1920 if (op_size <= OP_SIZE_2 || (!arch_use_flags && !ARCH_SUPPORTS_TRAPS(op_size))) {
1921 int64_t imm = (ARCH_PREFERS_SX(op_size) ? -0x80ULL : 0x80ULL) << (((1 << op_size) - 1) * 8);
1923 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, i_size_cmp(op_size), reg1, imm, COND_E, label_ovf));
1925 mode = MODE_FIXED;
1927 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1928 }
1929 }
1930 #if !ARCH_HAS_FLAGS
1931 if (mode == MODE_INT) {
1932 gen_insn(INSN_ALU1_TRAP, op_size, alu, ALU1_WRITES_FLAGS(alu));
1933 gen_one(target);
1934 gen_one(reg1);
1935 gen_four(label_ovf);
1936 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1937 return true;
1938 }
1939 #endif
1940 g(gen_2address_alu1(ctx, i_size(op_size), alu, target, reg1, mode == MODE_INT));
1941 #endif
1942 if (mode == MODE_INT) {
1943 gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
1944 gen_four(label_ovf);
1945 }
1946 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1947 return true;
1948 }
1950 /*******
1951 * NOT *
1952 *******/
1953 do_bool_not: {
1954 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1955 g(gen_frame_get(ctx, op_size, garbage, slot_1, target, ®1));
1957 g(gen_3address_alu_imm(ctx, i_size(op_size), ALU_XOR, target, reg1, 1, 0));
1959 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1960 return true;
1961 }
1963 /*********
1964 * BSWAP *
1965 *********/
1966 do_bswap: {
1967 bool attr_unused sx = false;
1968 #if defined(ARCH_X86) || defined(ARCH_ARM) || defined(ARCH_IA64) || defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS) || defined(ARCH_RISCV64) || defined(ARCH_S390)
1969 #if defined(ARCH_ARM32)
1970 if (unlikely(!cpu_test_feature(CPU_FEATURE_armv6)))
1971 goto do_generic_bswap;
1972 #endif
1973 #if defined(ARCH_MIPS)
1974 if (unlikely(!MIPS_HAS_ROT))
1975 goto do_generic_bswap;
1976 sx = op_size == OP_SIZE_4;
1977 #endif
1978 #if defined(ARCH_RISCV64)
1979 if (unlikely(!cpu_test_feature(CPU_FEATURE_zbb)))
1980 goto do_generic_bswap;
1981 #endif
1982 #if defined(ARCH_S390)
1983 if (op_size == OP_SIZE_2)
1984 goto do_generic_bswap;
1985 #endif
1986 #if defined(ARCH_X86)
1987 if (op_size >= OP_SIZE_4 && !cpu_test_feature(CPU_FEATURE_bswap))
1988 goto do_generic_bswap;
1989 #endif
1990 if (op_size > OP_SIZE_NATIVE) {
1991 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
1992 reg1 = R_SCRATCH_1;
1993 target = R_SCRATCH_1;
1994 } else {
1995 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1996 g(gen_frame_get(ctx, op_size, sx ? sign_x : garbage, slot_1, target, ®1));
1997 }
1999 if (op_size == OP_SIZE_1) {
2000 #if defined(ARCH_IA64) || defined(ARCH_RISCV64)
2001 } else if (op_size == OP_SIZE_2 || op_size == OP_SIZE_4) {
2002 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_BSWAP, target, reg1, 0));
2004 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SAR, target, target, op_size == OP_SIZE_2 ? 48 : 32, 0));
2005 #endif
2006 } else if (op_size == OP_SIZE_2) {
2007 #if defined(ARCH_X86)
2008 g(gen_3address_rot_imm(ctx, OP_SIZE_2, ROT_ROR, target, reg1, 8, 0));
2009 #else
2010 g(gen_2address_alu1(ctx, OP_SIZE_4, ALU1_BSWAP16, target, reg1, 0));
2011 #endif
2012 } else {
2013 g(gen_2address_alu1(ctx, minimum(op_size, OP_SIZE_NATIVE), ALU1_BSWAP, target, reg1, 0));
2014 }
2015 if (op_size > OP_SIZE_NATIVE) {
2016 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_BSWAP, R_SCRATCH_2, R_SCRATCH_2, 0));
2017 }
2019 if (op_size > OP_SIZE_NATIVE)
2020 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_2, R_SCRATCH_1));
2021 else
2022 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2023 return true;
2024 #endif
2025 goto do_generic_bswap;
2026 do_generic_bswap:
2027 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_unary_bswap_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, 0);
2028 }
2029 /********
2030 * BREV *
2031 ********/
2032 do_brev: {
2033 #if defined(ARCH_ARM) || defined(ARCH_LOONGARCH64) || (defined(ARCH_MIPS) && MIPS_R6)
2034 #if defined(ARCH_ARM32)
2035 if (unlikely(!cpu_test_feature(CPU_FEATURE_armv6t2)))
2036 goto do_generic_brev;
2037 #endif
2038 if (op_size > OP_SIZE_NATIVE) {
2039 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
2040 reg1 = R_SCRATCH_1;
2041 target = R_SCRATCH_1;
2042 } else {
2043 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
2044 g(gen_frame_get(ctx, op_size, garbage, slot_1, target, ®1));
2045 }
2047 g(gen_2address_alu1(ctx, minimum(maximum(OP_SIZE_4, op_size), OP_SIZE_NATIVE), ALU1_BREV, target, reg1, 0));
2048 if (op_size <= OP_SIZE_2) {
2049 g(gen_3address_rot_imm(ctx, OP_SIZE_4, ROT_SHR, target, target, op_size == OP_SIZE_1 ? 24 : 16, 0));
2050 }
2051 if (op_size > OP_SIZE_NATIVE) {
2052 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_BREV, R_SCRATCH_2, R_SCRATCH_2, 0));
2053 }
2055 if (op_size > OP_SIZE_NATIVE)
2056 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_2, R_SCRATCH_1));
2057 else
2058 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2059 return true;
2060 #endif
2061 goto do_generic_brev;
2062 do_generic_brev:
2063 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_unary_brev_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, 0);
2064 }
2065 /******************
2066 * BSF/BSR/POPCNT *
2067 ******************/
2068 do_bsf_bsr_popcnt: {
2069 if (op_size > OP_SIZE_NATIVE) {
2070 #if defined(ARCH_X86)
2071 uint32_t label_finish = 0; /* avoid warning */
2072 if (alu == ALU1_POPCNT && unlikely(!cpu_test_feature(CPU_FEATURE_popcnt)))
2073 goto do_generic_bsf_bsr_popcnt;
2074 if (alu == ALU1_BSR || alu == ALU1_POPCNT) {
2075 if (mode == MODE_INT) {
2076 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
2077 g(gen_imm(ctx, 0, IMM_PURPOSE_STORE_VALUE, OP_SIZE_NATIVE));
2078 gen_insn(INSN_CMP, OP_SIZE_NATIVE, 0, 1);
2079 gen_address_offset();
2080 gen_imm_offset();
2082 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_S, 0);
2083 gen_four(label_ovf);
2084 }
2085 }
2086 if (alu == ALU1_BSF) {
2087 label_finish = alloc_label(ctx);
2088 if (unlikely(!label_finish))
2089 return false;
2091 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + lo_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
2092 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_BSF, ALU1_WRITES_FLAGS(ALU1_BSF));
2093 gen_one(R_SCRATCH_1);
2094 gen_address_offset();
2096 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_NE, 0);
2097 gen_four(label_finish);
2099 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
2100 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_BSF, ALU1_WRITES_FLAGS(ALU1_BSF));
2101 gen_one(R_SCRATCH_1);
2102 gen_address_offset();
2104 g(gen_3address_alu_imm(ctx, OP_SIZE_4, ALU_ADD, R_SCRATCH_1, R_SCRATCH_1, 8U << OP_SIZE_NATIVE, 0));
2105 }
2106 if (alu == ALU1_BSR) {
2107 label_finish = alloc_label(ctx);
2108 if (unlikely(!label_finish))
2109 return false;
2111 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
2112 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_BSR, ALU1_WRITES_FLAGS(ALU1_BSR));
2113 gen_one(R_SCRATCH_1);
2114 gen_address_offset();
2116 g(gen_3address_alu_imm(ctx, OP_SIZE_4, ALU_ADD, R_SCRATCH_1, R_SCRATCH_1, 8U << OP_SIZE_NATIVE, 0));
2118 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_NE, 0);
2119 gen_four(label_finish);
2121 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + lo_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
2122 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_BSR, ALU1_WRITES_FLAGS(ALU1_BSR));
2123 gen_one(R_SCRATCH_1);
2124 gen_address_offset();
2125 }
2126 if (alu == ALU1_BSF || alu == ALU1_BSR) {
2127 if (mode == MODE_INT) {
2128 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_E, 0);
2129 gen_four(label_ovf);
2130 } else {
2131 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_NE, 0);
2132 gen_four(label_finish);
2134 g(gen_load_constant(ctx, R_SCRATCH_1, -1));
2135 }
2137 gen_label(label_finish);
2139 if (mode == MODE_INT)
2140 goto write_result;
2142 if (R_SCRATCH_1 != R_AX || R_SCRATCH_2 != R_DX)
2143 internal(file_line, "gen_alu1: bad scratch registers");
2144 gen_insn(INSN_CWD, OP_SIZE_NATIVE, 0, 0);
2145 gen_one(R_DX);
2146 gen_one(R_AX);
2148 g(gen_address(ctx, R_FRAME, (size_t)slot_r * slot_size + lo_word(OP_SIZE_NATIVE), IMM_PURPOSE_STR_OFFSET, OP_SIZE_NATIVE));
2149 gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
2150 gen_address_offset();
2151 gen_one(R_SCRATCH_1);
2153 g(gen_address(ctx, R_FRAME, (size_t)slot_r * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_STR_OFFSET, OP_SIZE_NATIVE));
2154 gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
2155 gen_address_offset();
2156 gen_one(R_SCRATCH_2);
2158 return true;
2159 }
2160 if (alu == ALU1_POPCNT) {
2161 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + lo_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
2162 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_POPCNT, ALU1_WRITES_FLAGS(ALU1_POPCNT));
2163 gen_one(R_SCRATCH_1);
2164 gen_address_offset();
2166 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
2167 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_POPCNT, ALU1_WRITES_FLAGS(ALU1_POPCNT));
2168 gen_one(R_SCRATCH_2);
2169 gen_address_offset();
2171 g(gen_3address_alu(ctx, OP_SIZE_4, ALU_ADD, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_2, 1));
2172 write_result:
2173 g(gen_address(ctx, R_FRAME, (size_t)slot_r * slot_size + lo_word(OP_SIZE_NATIVE), IMM_PURPOSE_STR_OFFSET, OP_SIZE_NATIVE));
2174 gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
2175 gen_address_offset();
2176 gen_one(R_SCRATCH_1);
2178 g(gen_address(ctx, R_FRAME, (size_t)slot_r * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_STR_OFFSET, OP_SIZE_NATIVE));
2179 g(gen_imm(ctx, 0, IMM_PURPOSE_STORE_VALUE, OP_SIZE_NATIVE));
2180 gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
2181 gen_address_offset();
2182 gen_imm_offset();
2184 return true;
2185 }
2186 #endif
2187 goto do_generic_bsf_bsr_popcnt;
2188 }
2189 #if defined(ARCH_X86)
2190 if (alu == ALU1_POPCNT && unlikely(!cpu_test_feature(CPU_FEATURE_popcnt)))
2191 goto do_generic_bsf_bsr_popcnt;
2192 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
2193 if (op_size == OP_SIZE_1 || ((alu == ALU1_BSR || alu == ALU1_POPCNT) && mode == MODE_INT)) {
2194 g(gen_frame_get(ctx, op_size, zero_x, slot_1, target, ®1));
2195 if ((alu == ALU1_BSR || alu == ALU1_POPCNT) && mode == MODE_INT) {
2196 g(gen_cmp_test_jmp(ctx, INSN_TEST, op_size, reg1, reg1, alu == ALU1_BSR ? COND_LE : COND_S, label_ovf));
2197 }
2198 g(gen_2address_alu1(ctx, maximum(op_size, OP_SIZE_2), alu, target, reg1, 1));
2199 if ((alu == ALU1_BSR || alu == ALU1_POPCNT) && mode == MODE_INT)
2200 goto x86_bsf_bsr_popcnt_finish;
2201 } else {
2202 g(gen_frame_load_op1(ctx, op_size, alu, 1, slot_1, target));
2203 }
2204 if (alu == ALU1_POPCNT)
2205 goto x86_bsf_bsr_popcnt_finish;
2206 if (mode == MODE_FIXED) {
2207 uint32_t cmov_label;
2208 gen_insn(INSN_MOV, maximum(op_size, OP_SIZE_4), 0, 0);
2209 gen_one(R_SCRATCH_2);
2210 gen_one(ARG_IMM);
2211 gen_eight(-1);
2212 g(gen_cmov(ctx, maximum(op_size, OP_SIZE_4), COND_E, target, &cmov_label));
2213 gen_one(R_SCRATCH_2);
2214 if (cmov_label)
2215 gen_label(cmov_label);
2217 } else {
2218 gen_insn(INSN_JMP_COND, maximum(op_size, OP_SIZE_2), COND_E, 0);
2219 gen_four(label_ovf);
2220 }
2221 x86_bsf_bsr_popcnt_finish:
2222 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2223 return true;
2224 #endif
2225 #if defined(ARCH_ARM)
2226 #if defined(ARCH_ARM32)
2227 if (alu == ALU1_BSR && unlikely(!cpu_test_feature(CPU_FEATURE_armv6)))
2228 goto do_generic_bsf_bsr_popcnt;
2229 if (alu == ALU1_BSF && unlikely(!cpu_test_feature(CPU_FEATURE_armv6t2)))
2230 goto do_generic_bsf_bsr_popcnt;
2231 #endif
2232 if (alu == ALU1_POPCNT && unlikely(!cpu_test_feature(CPU_FEATURE_neon)))
2233 goto do_generic_bsf_bsr_popcnt;
2234 g(gen_frame_get(ctx, op_size, mode == MODE_INT ? sign_x : zero_x, slot_1, R_SCRATCH_1, ®1));
2235 if (mode == MODE_INT) {
2236 g(gen_cmp_test_jmp(ctx, INSN_TEST, i_size(op_size), reg1, reg1, alu == ALU1_BSR ? COND_LE : alu == ALU1_BSF ? COND_E : COND_S, label_ovf));
2237 }
2239 if (alu == ALU1_POPCNT) {
2240 g(gen_mov(ctx, OP_SIZE_NATIVE, FR_SCRATCH_1, reg1));
2241 gen_insn(INSN_FP_ALU1, OP_SIZE_NATIVE, FP_ALU1_VCNT8, 0);
2242 gen_one(FR_SCRATCH_1);
2243 gen_one(FR_SCRATCH_1);
2244 #if defined(ARCH_ARM32)
2245 if (op_size > OP_SIZE_1) {
2246 gen_insn(INSN_FP_ALU1, OP_SIZE_1, FP_ALU1_VPADDL, 0);
2247 gen_one(FR_SCRATCH_1);
2248 gen_one(FR_SCRATCH_1);
2249 }
2250 if (op_size > OP_SIZE_2) {
2251 gen_insn(INSN_FP_ALU1, OP_SIZE_2, FP_ALU1_VPADDL, 0);
2252 gen_one(FR_SCRATCH_1);
2253 gen_one(FR_SCRATCH_1);
2254 }
2255 #else
2256 if (op_size > OP_SIZE_1) {
2257 gen_insn(INSN_FP_ALU1, OP_SIZE_1, FP_ALU1_ADDV, 0);
2258 gen_one(FR_SCRATCH_1);
2259 gen_one(FR_SCRATCH_1);
2260 }
2261 #endif
2262 g(gen_frame_store(ctx, op_size, slot_r, 0, FR_SCRATCH_1));
2263 if (slot_is_register(ctx, slot_r))
2264 g(unspill(ctx, slot_r));
2265 return true;
2266 }
2268 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
2269 if (mode == MODE_FIXED && alu == ALU1_BSF) {
2270 gen_insn(INSN_TEST, i_size(op_size), 0, 1);
2271 gen_one(target);
2272 gen_one(reg1);
2273 reg1 = target;
2274 }
2276 if (alu == ALU1_BSF) {
2277 g(gen_2address_alu1(ctx, i_size(op_size), ALU1_BREV, target, reg1, 0));
2278 reg1 = target;
2279 }
2281 g(gen_2address_alu1(ctx, i_size(op_size), ALU1_LZCNT, target, reg1, 0));
2283 if (alu == ALU1_BSR) {
2284 g(gen_load_constant(ctx, R_SCRATCH_2, op_size == OP_SIZE_8 ? 63 : 31));
2285 g(gen_3address_alu(ctx, i_size(op_size), ALU_SUB, target, R_SCRATCH_2, target, 0));
2286 }
2288 if (mode == MODE_FIXED && alu == ALU1_BSF) {
2289 #if defined(ARCH_ARM32)
2290 g(gen_imm(ctx, -1, IMM_PURPOSE_CMOV, OP_SIZE_NATIVE));
2291 gen_insn(INSN_CMOV, OP_SIZE_NATIVE, COND_E, 0);
2292 gen_one(target);
2293 gen_one(target);
2294 gen_imm_offset();
2295 #else
2296 gen_insn(INSN_CSEL_INV, i_size(op_size), COND_NE, 0);
2297 gen_one(target);
2298 gen_one(ARG_IMM);
2299 gen_eight(0);
2300 gen_one(target);
2301 #endif
2302 }
2304 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2305 return true;
2306 #endif
2307 #if defined(ARCH_ALPHA)
2308 if (likely(cpu_test_feature(CPU_FEATURE_cix))) {
2309 g(gen_frame_get(ctx, op_size, mode == MODE_INT ? sign_x : zero_x, slot_1, R_SCRATCH_1, ®1));
2310 target = gen_frame_target(ctx, slot_r, slot_1, NO_FRAME_T, R_SCRATCH_2);
2311 if (mode == MODE_INT) {
2312 g(gen_cmp_test_jmp(ctx, INSN_TEST, OP_SIZE_NATIVE, reg1, reg1, alu == ALU1_BSR ? COND_LE : alu == ALU1_BSF ? COND_E : COND_S, label_ovf));
2313 }
2314 if (alu == ALU1_POPCNT) {
2315 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_POPCNT, target, reg1, 0));
2316 }
2317 if (alu == ALU1_BSF) {
2318 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_BSF, target, reg1, 0));
2320 if (mode == MODE_FIXED) {
2321 g(gen_imm(ctx, -1, IMM_PURPOSE_MOVR, OP_SIZE_INT));
2322 gen_insn(INSN_MOVR, OP_SIZE_NATIVE, COND_E, 0);
2323 gen_one(target);
2324 gen_one(target);
2325 gen_one(reg1);
2326 gen_imm_offset();
2327 }
2328 }
2329 if (alu == ALU1_BSR) {
2330 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_LZCNT, target, reg1, 0));
2332 g(gen_load_constant(ctx, R_SCRATCH_3, OP_SIZE_NATIVE == OP_SIZE_8 ? 63 : 31));
2334 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_SUB, target, R_SCRATCH_3, target, 0));
2335 }
2336 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2337 return true;
2338 }
2339 #endif
2340 #if defined(ARCH_MIPS)
2341 if (MIPS_HAS_CLZ && alu != ALU1_POPCNT) {
2342 g(gen_frame_get(ctx, op_size, mode == MODE_INT ? sign_x : zero_x, slot_1, R_SCRATCH_1, ®1));
2343 target = gen_frame_target(ctx, slot_r, slot_1, NO_FRAME_T, R_SCRATCH_2);
2344 if (mode == MODE_INT) {
2345 g(gen_cmp_test_jmp(ctx, INSN_TEST, OP_SIZE_NATIVE, reg1, reg1, alu == ALU1_BSR ? COND_LE : alu == ALU1_BSF ? COND_E : COND_S, label_ovf));
2346 }
2347 if (alu == ALU1_BSF) {
2348 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NEG, target, reg1, 0));
2350 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_AND, R_SCRATCH_1, reg1, target, 0));
2351 reg1 = R_SCRATCH_1;
2352 }
2353 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_LZCNT, target, reg1, 0));
2355 g(gen_load_constant(ctx, R_SCRATCH_3, OP_SIZE_NATIVE == OP_SIZE_8 ? 63 : 31));
2357 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_SUB, target, R_SCRATCH_3, target, 0));
2359 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2360 return true;
2361 }
2362 #endif
2363 #if defined(ARCH_POWER)
2364 if (alu == ALU1_BSF && (unlikely(!cpu_test_feature(CPU_FEATURE_v203)) || unlikely(!cpu_test_feature(CPU_FEATURE_v30))))
2365 goto do_generic_bsf_bsr_popcnt;
2366 if (alu == ALU1_POPCNT && unlikely(!cpu_test_feature(CPU_FEATURE_v206)))
2367 goto do_generic_bsf_bsr_popcnt;
2368 g(gen_frame_get(ctx, op_size, mode == MODE_INT ? sign_x : zero_x, slot_1, R_SCRATCH_1, ®1));
2369 target = gen_frame_target(ctx, slot_r, slot_1, NO_FRAME_T, R_SCRATCH_2);
2370 if (mode == MODE_INT) {
2371 g(gen_cmp_test_jmp(ctx, INSN_TEST, i_size(op_size), reg1, reg1, alu == ALU1_BSR ? COND_LE : alu == ALU1_BSF ? COND_E : COND_S, label_ovf));
2372 }
2373 if (alu == ALU1_POPCNT) {
2374 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_POPCNT, target, reg1, 0));
2375 }
2376 if (alu == ALU1_BSF) {
2377 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_BSF, target, reg1, 0));
2379 if (mode == MODE_FIXED) {
2380 g(gen_3address_alu(ctx, i_size(op_size), ALU_AND, R_SCRATCH_3, reg1, reg1, 1));
2382 g(gen_imm(ctx, -1, IMM_PURPOSE_CMOV, OP_SIZE_NATIVE));
2383 gen_insn(INSN_CMOV, OP_SIZE_NATIVE, COND_E, 0);
2384 gen_one(target);
2385 gen_one(target);
2386 gen_imm_offset();
2387 }
2388 }
2389 if (alu == ALU1_BSR) {
2390 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_LZCNT, target, reg1, 0));
2392 g(gen_load_constant(ctx, R_SCRATCH_3, OP_SIZE_NATIVE == OP_SIZE_8 ? 63 : 31));
2394 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_SUB, target, R_SCRATCH_3, target, 0));
2395 }
2396 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2397 return true;
2398 #endif
2399 #if defined(ARCH_LOONGARCH64) || defined(ARCH_RISCV64)
2400 #if defined(ARCH_LOONGARCH64)
2401 if (alu == ALU1_POPCNT)
2402 goto do_generic_bsf_bsr_popcnt;
2403 #endif
2404 #if defined(ARCH_RISCV64)
2405 if (unlikely(!cpu_test_feature(CPU_FEATURE_zbb)))
2406 goto do_generic_bsf_bsr_popcnt;
2407 #endif
2408 g(gen_frame_get(ctx, op_size, mode == MODE_INT ? sign_x : zero_x, slot_1, R_SCRATCH_1, ®1));
2409 target = gen_frame_target(ctx, slot_r, slot_1, NO_FRAME_T, R_SCRATCH_2);
2410 if (mode == MODE_INT) {
2411 g(gen_cmp_test_jmp(ctx, INSN_TEST, OP_SIZE_NATIVE, reg1, reg1, alu == ALU1_BSR ? COND_LE : alu == ALU1_BSF ? COND_E : COND_S, label_ovf));
2412 }
2413 if (alu == ALU1_POPCNT) {
2414 g(gen_2address_alu1(ctx, maximum(OP_SIZE_4, op_size), ALU1_POPCNT, target, reg1, 0));
2415 }
2416 if (alu == ALU1_BSF) {
2417 g(gen_2address_alu1(ctx, maximum(OP_SIZE_4, op_size), ALU1_BSF, target, reg1, 0));
2419 if (mode == MODE_FIXED) {
2420 g(gen_imm(ctx, 1, IMM_PURPOSE_CMP, OP_SIZE_NATIVE));
2421 gen_insn(INSN_CMP_DEST_REG, OP_SIZE_NATIVE, COND_B, 0);
2422 gen_one(R_SCRATCH_3);
2423 gen_one(reg1);
2424 gen_imm_offset();
2426 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NEG, R_SCRATCH_3, R_SCRATCH_3, 0));
2428 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_OR, target, target, R_SCRATCH_3, 0));
2429 }
2430 }
2431 if (alu == ALU1_BSR) {
2432 g(gen_2address_alu1(ctx, maximum(OP_SIZE_4, op_size), ALU1_LZCNT, target, reg1, 0));
2434 g(gen_load_constant(ctx, R_SCRATCH_3, op_size <= OP_SIZE_4 ? 31 : 63));
2436 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_SUB, target, R_SCRATCH_3, target, 0));
2437 }
2438 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2439 return true;
2440 #endif
2441 #if defined(ARCH_IA64) || defined(ARCH_S390) || defined(ARCH_SPARC)
2442 if (alu == ALU1_BSF && !ARCH_HAS_ANDN)
2443 goto do_generic_bsf_bsr_popcnt;
2444 #if defined(ARCH_S390)
2445 if (!cpu_test_feature(CPU_FEATURE_misc_45) || !cpu_test_feature(CPU_FEATURE_misc_insn_ext_3))
2446 goto do_generic_bsf_bsr_popcnt;
2447 #endif
2448 #if defined(ARCH_SPARC)
2449 if (!SPARC_9)
2450 goto do_generic_bsf_bsr_popcnt;
2451 #endif
2452 g(gen_frame_get(ctx, op_size, mode == MODE_INT ? sign_x : zero_x, slot_1, R_SCRATCH_1, ®1));
2453 target = gen_frame_target(ctx, slot_r, slot_1, NO_FRAME_T, R_SCRATCH_2);
2454 if (mode == MODE_INT) {
2455 g(gen_cmp_test_jmp(ctx, INSN_TEST, maximum(op_size, OP_SIZE_4), reg1, reg1, alu == ALU1_BSR ? COND_LE : alu == ALU1_BSF ? COND_E : COND_S, label_ovf));
2456 } else {
2457 if (ARCH_PREFERS_SX(op_size) && alu == ALU1_POPCNT && op_size < OP_SIZE_NATIVE) {
2458 g(gen_extend(ctx, op_size, zero_x, R_SCRATCH_1, reg1));
2459 reg1 = R_SCRATCH_1;
2460 }
2461 }
2462 if (alu == ALU1_POPCNT) {
2463 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_POPCNT, R_SCRATCH_1, reg1, 0));
2464 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
2465 return true;
2466 }
2467 if (alu == ALU1_BSF) {
2468 g(gen_3address_alu_imm(ctx, OP_SIZE_NATIVE, ALU_SUB, target, reg1, 1, 0));
2470 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_ANDN, target, target, reg1, 0));
2472 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_POPCNT, target, target, 0));
2474 if (mode == MODE_FIXED) {
2475 unsigned attr_unused test_reg = R_SCRATCH_1;
2476 #if defined(ARCH_S390)
2477 g(gen_imm(ctx, 0, COND_IS_LOGICAL(COND_E) ? IMM_PURPOSE_CMP_LOGICAL : IMM_PURPOSE_CMP, OP_SIZE_NATIVE));
2478 gen_insn(INSN_CMP, OP_SIZE_NATIVE, 0, 1 + COND_IS_LOGICAL(COND_E));
2479 gen_one(reg1);
2480 gen_imm_offset();
2482 g(gen_imm(ctx, -1, IMM_PURPOSE_CMOV, OP_SIZE_NATIVE));
2483 gen_insn(INSN_CMOV, OP_SIZE_NATIVE, COND_E, 0);
2484 gen_one(target);
2485 gen_one(target);
2486 gen_imm_offset();
2487 #else
2488 #if defined(ARCH_IA64)
2489 g(gen_cmp_dest_reg(ctx, OP_SIZE_NATIVE, reg1, (unsigned)-1, R_CMP_RESULT, 0, COND_NE));
2490 test_reg = R_CMP_RESULT;
2491 #endif
2492 g(gen_imm(ctx, -1, IMM_PURPOSE_MOVR, OP_SIZE_NATIVE));
2493 gen_insn(INSN_MOVR, OP_SIZE_NATIVE, COND_E, 0);
2494 gen_one(target);
2495 gen_one(target);
2496 gen_one(test_reg);
2497 gen_imm_offset();
2498 #endif
2499 }
2501 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2502 return true;
2503 }
2504 #endif
2505 do_generic_bsf_bsr_popcnt:
2506 if (alu == ALU1_BSF) {
2507 if (mode == MODE_FIXED)
2508 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_unary_bsf_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, 0);
2509 else
2510 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, INT_unary_bsf_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, label_ovf);
2511 }
2512 if (alu == ALU1_BSR) {
2513 if (mode == MODE_FIXED)
2514 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_unary_bsr_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, 0);
2515 else
2516 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, INT_unary_bsr_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, label_ovf);
2517 }
2518 if (alu == ALU1_POPCNT) {
2519 if (mode == MODE_FIXED)
2520 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_unary_popcnt_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, 0);
2521 else
2522 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, INT_unary_popcnt_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, label_ovf);
2523 }
2524 }
2525 /**************
2526 * CONVERSION *
2527 **************/
2528 do_fixed_conv:
2529 do_conv: {
2530 unsigned src_op_size, dest_op_size;
2531 const struct type *src_type, *dest_type;
2532 src_type = get_type_of_local(ctx, slot_1);
2533 dest_type = get_type_of_local(ctx, slot_r);
2535 if (TYPE_TAG_IS_FIXED(src_type->tag)) {
2536 src_op_size = TYPE_TAG_IDX_FIXED(src_type->tag) >> 1;
2537 } else {
2538 src_op_size = TYPE_TAG_IDX_INT(src_type->tag);
2539 }
2541 if (TYPE_TAG_IS_FIXED(dest_type->tag)) {
2542 dest_op_size = TYPE_TAG_IDX_FIXED(dest_type->tag) >> 1;
2543 } else {
2544 dest_op_size = TYPE_TAG_IDX_INT(dest_type->tag);
2545 }
2547 if (src_op_size <= OP_SIZE_NATIVE) {
2548 g(gen_frame_get(ctx, src_op_size, sign_x, slot_1, R_SCRATCH_1, ®1));
2549 } else {
2550 #if defined(ARCH_X86)
2551 if (dest_op_size < src_op_size)
2552 g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_1, 0, false, R_SCRATCH_1));
2553 else
2554 #endif
2555 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_3));
2556 reg1 = R_SCRATCH_1;
2557 }
2559 if (dest_op_size >= src_op_size) {
2560 if (dest_op_size <= OP_SIZE_NATIVE) {
2561 g(gen_frame_store(ctx, dest_op_size, slot_r, 0, reg1));
2562 } else {
2563 if (src_op_size <= OP_SIZE_NATIVE) {
2564 #if defined(ARCH_X86)
2565 if (R_SCRATCH_1 != R_AX || R_SCRATCH_2 != R_DX)
2566 internal(file_line, "gen_alu1: bad scratch registers");
2567 if (reg1 == R_SCRATCH_1) {
2568 gen_insn(INSN_CWD, OP_SIZE_NATIVE, 0, 0);
2569 gen_one(R_DX);
2570 gen_one(R_AX);
2571 } else
2572 #endif
2573 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_2, reg1, (1U << (OP_SIZE_NATIVE + 3)) - 1, false));
2574 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, reg1, R_SCRATCH_2));
2575 } else {
2576 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, reg1, R_SCRATCH_3));
2577 }
2578 }
2579 return true;
2580 } else {
2581 if (src_op_size > OP_SIZE_NATIVE) {
2582 #if defined(ARCH_ARM)
2583 gen_insn(INSN_CMP, OP_SIZE_NATIVE, 0, 1);
2584 gen_one(R_SCRATCH_3);
2585 gen_one(ARG_SHIFTED_REGISTER);
2586 gen_one(ARG_SHIFT_ASR | ((1U << (OP_SIZE_NATIVE + 3)) - 1));
2587 gen_one(R_SCRATCH_1);
2589 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_NE, 0);
2590 gen_four(label_ovf);
2591 #elif defined(ARCH_X86)
2592 if (R_SCRATCH_1 != R_AX || R_SCRATCH_2 != R_DX)
2593 internal(file_line, "gen_alu1: bad scratch registers");
2594 gen_insn(INSN_CWD, OP_SIZE_NATIVE, 0, 0);
2595 gen_one(R_DX);
2596 gen_one(R_AX);
2598 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
2599 gen_insn(INSN_CMP, OP_SIZE_NATIVE, 0, 1);
2600 gen_one(R_SCRATCH_2);
2601 gen_address_offset();
2603 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_NE, 0);
2604 gen_four(label_ovf);
2605 #else
2606 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_2, R_SCRATCH_1, (1U << (OP_SIZE_NATIVE + 3)) - 1, 0));
2608 g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, R_SCRATCH_3, R_SCRATCH_2, COND_NE, label_ovf));
2609 #endif
2611 src_op_size = OP_SIZE_NATIVE;
2612 }
2613 if (src_op_size > dest_op_size) {
2614 g(gen_cmp_extended(ctx, OP_SIZE_NATIVE, dest_op_size, reg1, R_SCRATCH_2, label_ovf));
2615 }
2616 g(gen_frame_store(ctx, dest_op_size, slot_r, 0, reg1));
2617 return true;
2618 }
2619 }
2621 conv_uto_int: {
2622 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_uto_int_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, label_ovf);
2623 }
2625 conv_ufrom_int: {
2626 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_ufrom_int_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, label_ovf);
2627 }
2628 }
2630 static bool attr_w gen_constant(struct codegen_context *ctx, bool real, unsigned op_size, bool shrt, frame_t slot_r)
2631 {
2632 uintbig_t c;
2633 if (shrt) {
2634 c = (int16_t)get_unaligned_16(ctx->current_position);
2635 } else switch (op_size) {
2636 #define fx(n, type, utype, sz, bits) \
2637 case n: \
2638 c = (type)cat(get_unaligned_,bits)(ctx->current_position);\
2639 break;
2640 for_all_fixed(fx);
2641 #undef fx
2642 default:
2643 internal(file_line, "gen_constant: invalid type %u", op_size);
2644 }
2645 if (op_size > OP_SIZE_NATIVE) {
2646 unsigned shift = (8U << OP_SIZE_NATIVE) - 1;
2647 g(gen_frame_store_imm_raw(ctx, OP_SIZE_NATIVE, slot_r, lo_word(OP_SIZE_NATIVE), c & ((2ULL << shift) - 1)));
2648 g(gen_frame_store_imm_raw(ctx, OP_SIZE_NATIVE, slot_r, hi_word(OP_SIZE_NATIVE), c >> 1 >> shift));
2649 if (real && slot_is_register(ctx, slot_r))
2650 g(unspill(ctx, slot_r));
2651 return true;
2652 } else if (real && slot_is_register(ctx, slot_r)) {
2653 if (ARCH_HAS_FP_GP_MOV) {
2654 g(gen_load_constant(ctx, R_SCRATCH_1, c));
2655 g(gen_mov(ctx, op_size, ctx->registers[slot_r], R_SCRATCH_1));
2656 } else {
2657 g(gen_frame_store_imm_raw(ctx, op_size, slot_r, 0, c));
2658 g(unspill(ctx, slot_r));
2659 }
2660 } else {
2661 g(gen_frame_store_imm(ctx, op_size, slot_r, 0, c));
2662 }
2663 return true;
2664 }
2666 static bool attr_w gen_real_constant(struct codegen_context *ctx, const struct type *t, frame_t slot_r)
2667 {
2668 int64_t offset;
2669 if (is_power_of_2(t->size) && t->size <= sizeof(uintbig_t))
2670 return gen_constant(ctx, true, log_2(t->size), false, slot_r);
2672 g(load_function_offset(ctx, R_SCRATCH_3, offsetof(struct data, u_.function.code)));
2674 offset = (ctx->current_position - da(ctx->fn,function)->code) * sizeof(code_t);
2676 g(gen_memcpy_raw(ctx, R_FRAME, (size_t)slot_r * slot_size, R_SCRATCH_3, offset, t->size, minimum(t->align, sizeof(code_t))));
2677 if (slot_is_register(ctx, slot_r))
2678 g(unspill(ctx, slot_r));
2680 return true;
2681 }
2683 static bool attr_w gen_copy(struct codegen_context *ctx, unsigned op_size, frame_t slot_1, frame_t slot_r)
2684 {
2685 unsigned reg1;
2686 if (unlikely(op_size > OP_SIZE_NATIVE)) {
2687 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
2688 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_1, R_SCRATCH_2));
2689 return true;
2690 } else {
2691 unsigned target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
2692 g(gen_frame_get(ctx, op_size, garbage, slot_1, target, ®1));
2693 g(gen_frame_store(ctx, op_size, slot_r, 0, reg1));
2694 return true;
2695 }
2696 }
2698 static bool attr_w gen_fp_alu(struct codegen_context *ctx, unsigned real_type, unsigned op, uint32_t label_ovf, frame_t slot_1, frame_t slot_2, frame_t slot_r)
2699 {
2700 unsigned attr_unused fp_alu;
2701 size_t upc;
2702 unsigned attr_unused op_size = real_type_to_op_size(real_type);
2703 unsigned reg1, reg2, target;
2704 switch (op) {
2705 case OPCODE_REAL_OP_add:
2706 case OPCODE_REAL_OP_add_alt1:
2707 case OPCODE_REAL_OP_add_alt2: fp_alu = FP_ALU_ADD; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_add_real16_t); label_ovf = 0; goto do_alu;
2708 case OPCODE_REAL_OP_subtract:
2709 case OPCODE_REAL_OP_subtract_alt1:
2710 case OPCODE_REAL_OP_subtract_alt2: fp_alu = FP_ALU_SUB; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_subtract_real16_t); label_ovf = 0; goto do_alu;
2711 case OPCODE_REAL_OP_multiply:
2712 case OPCODE_REAL_OP_multiply_alt1:
2713 case OPCODE_REAL_OP_multiply_alt2: fp_alu = FP_ALU_MUL; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_multiply_real16_t); label_ovf = 0; goto do_alu;
2714 case OPCODE_REAL_OP_divide:
2715 case OPCODE_REAL_OP_divide_alt1:
2716 case OPCODE_REAL_OP_divide_alt2: fp_alu = FP_ALU_DIV; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_divide_real16_t); label_ovf = 0; goto do_alu;
2717 case OPCODE_REAL_OP_modulo:
2718 case OPCODE_REAL_OP_power:
2719 case OPCODE_REAL_OP_ldexp:
2720 case OPCODE_REAL_OP_atan2: upc = offsetof(struct cg_upcall_vector_s, REAL_binary_modulo_real16_t) + (op - OPCODE_REAL_OP_modulo) * TYPE_REAL_N * sizeof(void (*)(void)); goto do_upcall;
2721 case OPCODE_REAL_OP_equal:
2722 case OPCODE_REAL_OP_equal_alt1:
2723 case OPCODE_REAL_OP_equal_alt2: fp_alu = FP_COND_E; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_equal_real16_t); goto do_cmp;
2724 case OPCODE_REAL_OP_not_equal:
2725 case OPCODE_REAL_OP_not_equal_alt1:
2726 case OPCODE_REAL_OP_not_equal_alt2: fp_alu = FP_COND_NE; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_not_equal_real16_t); goto do_cmp;
2727 case OPCODE_REAL_OP_less:
2728 case OPCODE_REAL_OP_less_alt1:
2729 case OPCODE_REAL_OP_less_alt2: fp_alu = FP_COND_B; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_less_real16_t); goto do_cmp;
2730 case OPCODE_REAL_OP_less_equal:
2731 case OPCODE_REAL_OP_less_equal_alt1:
2732 case OPCODE_REAL_OP_less_equal_alt2: fp_alu = FP_COND_BE; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_less_equal_real16_t); goto do_cmp;
2733 case OPCODE_REAL_OP_greater:
2734 case OPCODE_REAL_OP_greater_alt1:
2735 case OPCODE_REAL_OP_greater_alt2: fp_alu = FP_COND_A; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_greater_real16_t); goto do_cmp;
2736 case OPCODE_REAL_OP_greater_equal:
2737 case OPCODE_REAL_OP_greater_equal_alt1:
2738 case OPCODE_REAL_OP_greater_equal_alt2: fp_alu = FP_COND_AE; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_greater_equal_real16_t); goto do_cmp;
2739 default: internal(file_line, "gen_fp_alu: unsupported operation %u", op);
2740 }
2742 do_alu:
2743 if ((SUPPORTED_FP >> real_type) & 1) {
2744 #if defined(ARCH_IA64)
2745 if (unlikely(fp_alu == FP_ALU_DIV))
2746 goto do_upcall;
2747 #endif
2748 #if defined(ARCH_X86)
2749 if (1)
2750 #elif defined(ARCH_S390)
2751 if ((op_size <= OP_SIZE_8 && (size_t)slot_2 * slot_size < 4096) || slot_is_register(ctx, slot_2))
2752 #else
2753 if (slot_is_register(ctx, slot_2))
2754 #endif
2755 {
2756 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, slot_2, FR_SCRATCH_1);
2757 g(gen_frame_get(ctx, op_size, garbage, slot_1, FR_SCRATCH_1, ®1));
2758 if (slot_is_register(ctx, slot_2)) {
2759 g(gen_3address_fp_alu(ctx, op_size, fp_alu, target, reg1, ctx->registers[slot_2]));
2760 } else {
2761 if (target != reg1 && !ARCH_IS_3ADDRESS_FP) {
2762 g(gen_mov(ctx, op_size, target, reg1));
2763 reg1 = target;
2764 }
2765 g(gen_address(ctx, R_FRAME, (size_t)slot_2 * slot_size, IMM_PURPOSE_VLDR_VSTR_OFFSET, op_size));
2766 gen_insn(INSN_FP_ALU, op_size, fp_alu, 0);
2767 gen_one(target);
2768 gen_one(reg1);
2769 gen_address_offset();
2770 }
2771 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2772 return true;
2773 }
2774 #if defined(ARCH_ALPHA)
2775 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, ®1));
2776 g(gen_frame_get(ctx, op_size, zero_x, slot_2, FR_SCRATCH_2, ®2));
2777 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, FR_SCRATCH_3);
2778 g(gen_3address_fp_alu(ctx, op_size, fp_alu, target, reg1, reg2));
2779 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2780 #else
2781 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, ®1));
2782 g(gen_frame_get(ctx, op_size, zero_x, slot_2, FR_SCRATCH_2, ®2));
2783 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_1);
2784 g(gen_3address_fp_alu(ctx, op_size, fp_alu, target, reg1, reg2));
2785 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2786 #endif
2787 return true;
2788 }
2789 #ifdef SUPPORTED_FP_X87
2790 if ((SUPPORTED_FP_X87 >> real_type) & 1) {
2791 if (real_type != 3) {
2792 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_2));
2793 g(gen_frame_load_x87(ctx, INSN_X87_ALU, op_size, fp_alu, slot_1));
2794 } else {
2795 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
2796 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_2));
2797 gen_insn(INSN_X87_ALUP, op_size, fp_alu, 0);
2798 gen_one(R_ST1);
2799 }
2800 g(gen_frame_store_x87(ctx, INSN_X87_FSTP, op_size, slot_r));
2801 return true;
2802 }
2803 #endif
2804 #ifdef SUPPORTED_FP_HALF_CVT
2805 if ((SUPPORTED_FP_HALF_CVT >> real_type) & 1) {
2806 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, ®1));
2807 g(gen_frame_get(ctx, op_size, zero_x, slot_2, FR_SCRATCH_2, ®2));
2808 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_1);
2809 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
2810 gen_one(FR_SCRATCH_1);
2811 gen_one(reg1);
2812 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
2813 gen_one(FR_SCRATCH_2);
2814 gen_one(reg2);
2815 gen_insn(INSN_FP_ALU, OP_SIZE_4, fp_alu, 0);
2816 gen_one(FR_SCRATCH_1);
2817 gen_one(FR_SCRATCH_1);
2818 gen_one(FR_SCRATCH_2);
2819 gen_insn(INSN_FP_CVT, OP_SIZE_4, op_size, 0);
2820 gen_one(target);
2821 gen_one(FR_SCRATCH_1);
2822 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2823 return true;
2824 }
2825 #endif
2826 goto do_upcall;
2828 do_cmp:
2829 if ((SUPPORTED_FP >> real_type) & 1
2830 #if defined(ARCH_ALPHA)
2831 && OS_SUPPORTS_TRAPS
2832 #endif
2833 ) {
2834 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, ®1));
2835 g(gen_frame_get(ctx, op_size, zero_x, slot_2, FR_SCRATCH_2, ®2));
2836 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
2837 #if defined(ARCH_ALPHA)
2838 gen_insn(INSN_FP_CMP_DEST_REG_TRAP, op_size, fp_alu == FP_COND_NE ? FP_COND_E : fp_alu, 0);
2839 gen_one(FR_SCRATCH_3);
2840 gen_one(reg1);
2841 gen_one(reg2);
2842 gen_four(label_ovf);
2844 if (!ARCH_HAS_FP_GP_MOV) {
2845 g(gen_frame_store_raw(ctx, OP_SIZE_4, slot_r, 0, FR_SCRATCH_3));
2846 g(gen_frame_load_raw(ctx, OP_SIZE_4, sign_x, slot_r, 0, false, target));
2847 } else {
2848 g(gen_mov(ctx, OP_SIZE_4, target, FR_SCRATCH_3));
2849 }
2851 if (fp_alu == FP_COND_NE) {
2852 g(gen_imm(ctx, 0, IMM_PURPOSE_CMP, OP_SIZE_NATIVE));
2853 gen_insn(INSN_CMP_DEST_REG, OP_SIZE_NATIVE, COND_E, 0);
2854 gen_one(target);
2855 gen_one(target);
2856 gen_imm_offset();
2857 } else {
2858 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SHR, target, target, 30, 0));
2859 }
2861 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
2863 return true;
2864 #elif defined(ARCH_IA64)
2865 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_P, 0);
2866 gen_one(R_CMP_RESULT);
2867 gen_one(reg1);
2868 gen_one(reg2);
2870 gen_insn(INSN_JMP_REG, OP_SIZE_NATIVE, COND_NE, 0);
2871 gen_one(R_CMP_RESULT);
2872 gen_four(label_ovf);
2874 gen_insn(INSN_FP_CMP_DEST_REG, op_size, fp_alu, 0);
2875 gen_one(R_CMP_RESULT);
2876 gen_one(reg1);
2877 gen_one(reg2);
2879 g(gen_mov(ctx, OP_SIZE_NATIVE, target, R_CMP_RESULT));
2881 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
2883 return true;
2884 #elif defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS) || defined(ARCH_PARISC)
2885 gen_insn(INSN_FP_CMP_COND, op_size, FP_COND_P, 1);
2886 gen_one(reg1);
2887 gen_one(reg2);
2889 gen_insn(INSN_JMP_FP_TEST, 0, FP_COND_P, 0);
2890 gen_four(label_ovf);
2892 gen_insn(INSN_FP_CMP_COND, op_size, fp_alu, 1);
2893 gen_one(reg1);
2894 gen_one(reg2);
2896 gen_insn(INSN_FP_TEST_REG, OP_SIZE_NATIVE, fp_alu, 0);
2897 gen_one(target);
2899 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
2901 return true;
2902 #elif defined(ARCH_RISCV64)
2903 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_E, 0);
2904 gen_one(R_SCRATCH_1);
2905 gen_one(reg1);
2906 gen_one(reg1);
2908 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_E, 0);
2909 gen_one(R_SCRATCH_2);
2910 gen_one(reg2);
2911 gen_one(reg2);
2913 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_AND, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_2, 0));
2915 g(gen_jmp_on_zero(ctx, OP_SIZE_NATIVE, R_SCRATCH_1, COND_E, label_ovf));
2917 gen_insn(INSN_FP_CMP_DEST_REG, op_size, fp_alu == FP_COND_NE ? FP_COND_E : fp_alu, 0);
2918 gen_one(target);
2919 gen_one(reg1);
2920 gen_one(reg2);
2922 if (fp_alu == FP_COND_NE) {
2923 g(gen_imm(ctx, 1, IMM_PURPOSE_XOR, OP_SIZE_NATIVE));
2924 gen_insn(INSN_ALU, OP_SIZE_NATIVE, ALU_XOR, ALU_WRITES_FLAGS(OP_SIZE_NATIVE, ALU_XOR, false, is_imm(), ctx->const_imm));
2925 gen_one(target);
2926 gen_one(target);
2927 gen_imm_offset();
2928 }
2930 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
2931 return true;
2932 #else
2933 gen_insn(INSN_FP_CMP, op_size, 0, 1);
2934 gen_one(reg1);
2935 gen_one(reg2);
2936 #if defined(ARCH_ARM32)
2937 gen_insn(INSN_FP_TO_INT_FLAGS, 0, 0, 1);
2938 #endif
2939 gen_insn(INSN_JMP_COND, op_size, FP_COND_P, 0);
2940 gen_four(label_ovf);
2941 g(gen_frame_set_cond(ctx, op_size, false, fp_alu, slot_r));
2942 return true;
2943 #endif
2944 }
2945 #ifdef SUPPORTED_FP_X87
2946 if ((SUPPORTED_FP_X87 >> real_type) & 1) {
2947 if (likely(cpu_test_feature(CPU_FEATURE_cmov))) {
2948 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_2));
2949 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
2950 gen_insn(INSN_X87_FCOMIP, op_size, 0, 0);
2951 gen_one(R_ST1);
2952 gen_insn(INSN_X87_FSTP, op_size, 0, 0);
2953 gen_one(R_ST0);
2954 gen_insn(INSN_JMP_COND, op_size, COND_P, 0);
2955 gen_four(label_ovf);
2956 g(gen_frame_set_cond(ctx, op_size, false, fp_alu & 0xf, slot_r));
2957 return true;
2958 }
2960 if (real_type != 3) {
2961 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
2962 g(gen_frame_load_x87(ctx, INSN_X87_FCOMP, op_size, 0, slot_2));
2963 } else {
2964 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_2));
2965 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
2966 gen_insn(INSN_X87_FCOMPP, op_size, 0, 0);
2967 }
2969 gen_insn(INSN_X87_FNSTSW, 0, 0, 0);
2970 gen_one(R_AX);
2971 gen_one(R_AX);
2973 gen_insn(INSN_TEST, OP_SIZE_2, 0, 1);
2974 gen_one(R_AX);
2975 gen_one(ARG_IMM);
2976 gen_eight(0x0400);
2978 gen_insn(INSN_JMP_COND, OP_SIZE_2, COND_NE, 0);
2979 gen_four(label_ovf);
2981 switch (fp_alu) {
2982 case FP_COND_E:
2983 gen_insn(INSN_TEST, OP_SIZE_2, 0, 1);
2984 gen_one(R_AX);
2985 gen_one(ARG_IMM);
2986 gen_eight(0x4000);
2987 g(gen_frame_set_cond(ctx, OP_SIZE_2, false, COND_NE, slot_r));
2988 break;
2989 case FP_COND_NE:
2990 gen_insn(INSN_TEST, OP_SIZE_2, 0, 1);
2991 gen_one(R_AX);
2992 gen_one(ARG_IMM);
2993 gen_eight(0x4000);
2994 g(gen_frame_set_cond(ctx, OP_SIZE_2, false, COND_E, slot_r));
2995 break;
2996 case FP_COND_B:
2997 gen_insn(INSN_TEST, OP_SIZE_2, 0, 1);
2998 gen_one(R_AX);
2999 gen_one(ARG_IMM);
3000 gen_eight(0x0100);
3001 g(gen_frame_set_cond(ctx, OP_SIZE_2, false, COND_NE, slot_r));
3002 break;
3003 case FP_COND_BE:
3004 gen_insn(INSN_TEST, OP_SIZE_2, 0, 1);
3005 gen_one(R_AX);
3006 gen_one(ARG_IMM);
3007 gen_eight(0x4100);
3008 g(gen_frame_set_cond(ctx, OP_SIZE_2, false, COND_NE, slot_r));
3009 break;
3010 default:
3011 internal(file_line, "gen_fp_alu: invalid condition %u", fp_alu);
3012 }
3013 return true;
3014 }
3015 #endif
3016 #ifdef SUPPORTED_FP_HALF_CVT
3017 if ((SUPPORTED_FP_HALF_CVT >> real_type) & 1) {
3018 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, ®1));
3019 g(gen_frame_get(ctx, op_size, zero_x, slot_2, FR_SCRATCH_2, ®2));
3020 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
3021 gen_one(FR_SCRATCH_1);
3022 gen_one(reg1);
3023 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
3024 gen_one(FR_SCRATCH_2);
3025 gen_one(reg2);
3026 gen_insn(INSN_FP_CMP, OP_SIZE_4, 0, 1);
3027 gen_one(FR_SCRATCH_1);
3028 gen_one(FR_SCRATCH_2);
3029 #if defined(ARCH_ARM32)
3030 gen_insn(INSN_FP_TO_INT_FLAGS, 0, 0, 1);
3031 #endif
3032 gen_insn(INSN_JMP_COND, op_size, FP_COND_P, 0);
3033 gen_four(label_ovf);
3034 g(gen_frame_set_cond(ctx, op_size, false, fp_alu, slot_r));
3035 return true;
3036 }
3037 #endif
3039 do_upcall:
3040 return gen_alu_typed_upcall(ctx, upc, real_type, slot_1, slot_2, slot_r, label_ovf);
3041 }
3043 #define OP_IS_ROUND(alu) ((alu) == FP_ALU1_ROUND || (alu) == FP_ALU1_FLOOR || (alu) == FP_ALU1_CEIL || (alu) == FP_ALU1_TRUNC)
3045 static bool attr_w gen_fp_alu1(struct codegen_context *ctx, unsigned real_type, unsigned op, uint32_t label_ovf, frame_t slot_1, frame_t slot_r)
3046 {
3047 unsigned attr_unused fp_alu;
3048 size_t upc;
3049 unsigned attr_unused op_size = real_type_to_op_size(real_type);
3050 unsigned reg1, target;
3051 switch (op) {
3052 case OPCODE_REAL_OP_neg:
3053 case OPCODE_REAL_OP_neg_alt1:
3054 case OPCODE_REAL_OP_neg_alt2: fp_alu = FP_ALU1_NEG; upc = offsetof(struct cg_upcall_vector_s, REAL_unary_neg_real16_t); label_ovf = 0; goto do_alu;
3055 case OPCODE_REAL_OP_sqrt:
3056 case OPCODE_REAL_OP_sqrt_alt1:
3057 case OPCODE_REAL_OP_sqrt_alt2: fp_alu = FP_ALU1_SQRT; upc = offsetof(struct cg_upcall_vector_s, REAL_unary_sqrt_real16_t); label_ovf = 0; goto do_alu;
3058 case OPCODE_REAL_OP_round: fp_alu = FP_ALU1_ROUND; upc = offsetof(struct cg_upcall_vector_s, REAL_unary_round_real16_t); label_ovf = 0; goto do_alu;
3059 case OPCODE_REAL_OP_floor: fp_alu = FP_ALU1_FLOOR; upc = offsetof(struct cg_upcall_vector_s, REAL_unary_floor_real16_t); label_ovf = 0; goto do_alu;
3060 case OPCODE_REAL_OP_ceil: fp_alu = FP_ALU1_CEIL; upc = offsetof(struct cg_upcall_vector_s, REAL_unary_ceil_real16_t); label_ovf = 0; goto do_alu;
3061 case OPCODE_REAL_OP_trunc: fp_alu = FP_ALU1_TRUNC; upc = offsetof(struct cg_upcall_vector_s, REAL_unary_trunc_real16_t); label_ovf = 0; goto do_alu;
3062 case OPCODE_REAL_OP_to_int:
3063 case OPCODE_REAL_OP_to_int_alt1:
3064 case OPCODE_REAL_OP_to_int_alt2: upc = offsetof(struct cg_upcall_vector_s, REAL_unary_to_int_real16_t); goto do_to_int;
3065 case OPCODE_REAL_OP_from_int:
3066 case OPCODE_REAL_OP_from_int_alt1:
3067 case OPCODE_REAL_OP_from_int_alt2: upc = offsetof(struct cg_upcall_vector_s, REAL_unary_from_int_real16_t); label_ovf = 0; goto do_from_int;
3068 case OPCODE_REAL_OP_is_exception:
3069 case OPCODE_REAL_OP_is_exception_alt1:
3070 case OPCODE_REAL_OP_is_exception_alt2: upc = offsetof(struct cg_upcall_vector_s, REAL_unary_is_exception_real16_t); label_ovf = 0; goto do_is_exception;
3071 default: upc = offsetof(struct cg_upcall_vector_s, REAL_unary_cbrt_real16_t) + (op - OPCODE_REAL_OP_cbrt) * TYPE_REAL_N * sizeof(void (*)(void)); label_ovf = 0; goto do_upcall;
3072 }
3074 do_alu:
3075 if ((SUPPORTED_FP >> real_type) & 1 && (
3076 #if defined(ARCH_ALPHA)
3077 fp_alu == FP_ALU1_NEG ||
3078 (fp_alu == FP_ALU1_SQRT && cpu_test_feature(CPU_FEATURE_fix)) ||
3079 #elif defined(ARCH_ARM32)
3080 fp_alu == FP_ALU1_NEG ||
3081 fp_alu == FP_ALU1_SQRT ||
3082 #elif defined(ARCH_ARM64)
3083 true ||
3084 #elif defined(ARCH_IA64)
3085 fp_alu == FP_ALU1_NEG ||
3086 #elif defined(ARCH_LOONGARCH64)
3087 fp_alu == FP_ALU1_NEG ||
3088 fp_alu == FP_ALU1_SQRT ||
3089 fp_alu == FP_ALU1_ROUND ||
3090 #elif defined(ARCH_MIPS)
3091 fp_alu == FP_ALU1_NEG ||
3092 (fp_alu == FP_ALU1_SQRT && MIPS_HAS_SQRT) ||
3093 #elif defined(ARCH_PARISC)
3094 (fp_alu == FP_ALU1_NEG && PA_20) ||
3095 fp_alu == FP_ALU1_SQRT ||
3096 #elif defined(ARCH_POWER)
3097 fp_alu == FP_ALU1_NEG ||
3098 (fp_alu == FP_ALU1_SQRT && cpu_test_feature(CPU_FEATURE_p2) && real_type != 4) ||
3099 #elif defined(ARCH_S390)
3100 true ||
3101 #elif defined(ARCH_SPARC)
3102 fp_alu == FP_ALU1_NEG ||
3103 fp_alu == FP_ALU1_SQRT ||
3104 #elif defined(ARCH_RISCV64)
3105 fp_alu == FP_ALU1_NEG ||
3106 fp_alu == FP_ALU1_SQRT ||
3107 #elif defined(ARCH_X86)
3108 fp_alu == FP_ALU1_SQRT ||
3109 (OP_IS_ROUND(fp_alu) && cpu_test_feature(CPU_FEATURE_sse41)) ||
3110 #endif
3111 false)) {
3112 #if defined(ARCH_S390)
3113 if (op_size <= OP_SIZE_8 && (size_t)slot_1 * slot_size < 4096 && fp_alu == FP_ALU1_SQRT) {
3114 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_1);
3115 if (slot_is_register(ctx, slot_1)) {
3116 gen_insn(INSN_FP_ALU1, op_size, fp_alu, 0);
3117 gen_one(target);
3118 gen_one(ctx->registers[slot_1]);
3119 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
3120 } else {
3121 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size, IMM_PURPOSE_VLDR_VSTR_OFFSET, op_size));
3122 gen_insn(INSN_FP_ALU1, op_size, fp_alu, 0);
3123 gen_one(target);
3124 gen_address_offset();
3125 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
3126 }
3127 return true;
3128 }
3129 #endif
3130 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, ®1));
3131 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_2);
3132 gen_insn(INSN_FP_ALU1, op_size, fp_alu, 0);
3133 gen_one(target);
3134 gen_one(reg1);
3135 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
3136 return true;
3137 }
3138 #ifdef SUPPORTED_FP_X87
3139 if ((SUPPORTED_FP_X87 >> real_type) & 1) {
3140 if (fp_alu == FP_ALU1_NEG) {
3141 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
3142 gen_insn(INSN_X87_FCHS, op_size, 0, 0);
3143 g(gen_frame_store_x87(ctx, INSN_X87_FSTP, op_size, slot_r));
3144 return true;
3145 } else if (fp_alu == FP_ALU1_SQRT) {
3146 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
3147 gen_insn(INSN_X87_FSQRT, op_size, 0, 0);
3148 g(gen_frame_store_x87(ctx, INSN_X87_FSTP, op_size, slot_r));
3149 return true;
3150 } else if (fp_alu == FP_ALU1_ROUND) {
3151 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
3152 gen_insn(INSN_X87_FRNDINT, op_size, 0, 0);
3153 g(gen_frame_store_x87(ctx, INSN_X87_FSTP, op_size, slot_r));
3154 return true;
3155 }
3156 }
3157 #endif
3158 #ifdef SUPPORTED_FP_HALF_CVT
3159 if ((SUPPORTED_FP_HALF_CVT >> real_type) & 1 && (
3160 #if defined(ARCH_ARM32)
3161 fp_alu == FP_ALU1_NEG ||
3162 fp_alu == FP_ALU1_SQRT ||
3163 #elif defined(ARCH_ARM64)
3164 true ||
3165 #elif defined(ARCH_X86)
3166 fp_alu == FP_ALU1_SQRT ||
3167 (OP_IS_ROUND(fp_alu) && cpu_test_feature(CPU_FEATURE_sse41)) ||
3168 #endif
3169 false)) {
3170 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, ®1));
3171 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_1);
3172 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
3173 gen_one(target);
3174 gen_one(reg1);
3175 gen_insn(INSN_FP_ALU1, OP_SIZE_4, fp_alu, 0);
3176 gen_one(target);
3177 gen_one(target);
3178 gen_insn(INSN_FP_CVT, OP_SIZE_4, op_size, 0);
3179 gen_one(target);
3180 gen_one(target);
3181 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
3182 return true;
3183 }
3184 #endif
3185 goto do_upcall;
3187 do_to_int:
3188 if ((SUPPORTED_FP >> real_type) & 1
3189 #if defined(ARCH_ALPHA)
3190 && OS_SUPPORTS_TRAPS
3191 #endif
3192 #if defined(ARCH_MIPS)
3193 && MIPS_HAS_TRUNC
3194 #endif
3195 ) {
3196 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, ®1));
3197 goto do_cvt_to_int;
3198 do_cvt_to_int:
3199 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
3200 #if defined(ARCH_X86)
3201 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_TO_INT32 : INSN_FP_TO_INT64, op_size, 0, 0);
3202 gen_one(target);
3203 gen_one(reg1);
3205 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, OP_SIZE_INT, target, sign_bit(uint_default_t), COND_E, label_ovf));
3207 g(gen_frame_store(ctx, OP_SIZE_INT, slot_r, 0, target));
3208 return true;
3209 #endif
3210 #if defined(ARCH_ARM) || defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS)
3211 #if defined(ARCH_ARM)
3212 gen_insn(INSN_FP_CMP, op_size, 0, 1);
3213 gen_one(reg1);
3214 gen_one(reg1);
3215 #if defined(ARCH_ARM32)
3216 gen_insn(INSN_FP_TO_INT_FLAGS, 0, 0, 1);
3217 #endif
3218 gen_insn(INSN_JMP_COND, op_size, FP_COND_P, 0);
3219 gen_four(label_ovf);
3220 #else
3221 gen_insn(INSN_FP_CMP_COND, op_size, FP_COND_P, 1);
3222 gen_one(reg1);
3223 gen_one(reg1);
3225 gen_insn(INSN_JMP_FP_TEST, 0, FP_COND_P, 0);
3226 gen_four(label_ovf);
3227 #endif
3228 #if defined(ARCH_ARM32) || defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS)
3229 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_TO_INT32 : INSN_FP_TO_INT64, op_size, 0, 0);
3230 gen_one(FR_SCRATCH_1);
3231 gen_one(reg1);
3233 g(gen_mov(ctx, OP_SIZE_INT, target, FR_SCRATCH_1));
3234 #else
3235 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_TO_INT32 : INSN_FP_TO_INT64, op_size, 0, 0);
3236 gen_one(target);
3237 gen_one(reg1);
3238 #endif
3239 g(gen_imm(ctx, (int_default_t)(sign_bit(uint_default_t) + 1), IMM_PURPOSE_ADD, OP_SIZE_INT));
3240 gen_insn(INSN_ALU, OP_SIZE_INT, ALU_ADD, ALU_WRITES_FLAGS(OP_SIZE_INT, ALU_ADD, false, is_imm(), ctx->const_imm));
3241 gen_one(R_SCRATCH_2);
3242 gen_one(target);
3243 gen_imm_offset();
3245 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, OP_SIZE_INT, R_SCRATCH_2, 1, COND_BE, label_ovf));
3247 g(gen_frame_store(ctx, OP_SIZE_INT, slot_r, 0, target));
3248 return true;
3249 #endif
3250 #if defined(ARCH_IA64)
3251 gen_insn(INSN_FP_TO_INT64, op_size, 0, 0);
3252 gen_one(FR_SCRATCH_1);
3253 gen_one(reg1);
3255 g(gen_mov(ctx, OP_SIZE_NATIVE, target, FR_SCRATCH_1));
3257 if (OP_SIZE_INT == OP_SIZE_4) {
3258 g(gen_extend(ctx, OP_SIZE_4, sign_x, R_SCRATCH_2, target));
3259 g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, target, R_SCRATCH_2, COND_NE, label_ovf));
3260 } else {
3261 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, target, sign_bit(uint64_t), COND_E, label_ovf));
3262 }
3264 g(gen_frame_store(ctx, OP_SIZE_INT, slot_r, 0, target));
3265 return true;
3266 #endif
3267 #if defined(ARCH_PARISC) || defined(ARCH_POWER) || defined(ARCH_SPARC)
3268 #if defined(ARCH_POWER)
3269 if (!cpu_test_feature(CPU_FEATURE_ppc))
3270 goto do_upcall;
3271 if (OP_SIZE_INT == OP_SIZE_4)
3272 goto do_upcall;
3273 #endif
3274 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_TO_INT32 : INSN_FP_TO_INT64, op_size, 0, 0);
3275 gen_one(FR_SCRATCH_1);
3276 gen_one(reg1);
3278 g(gen_frame_store_raw(ctx, OP_SIZE_INT, slot_r, 0, FR_SCRATCH_1));
3279 if (slot_is_register(ctx, slot_r))
3280 g(unspill(ctx, slot_r));
3281 g(gen_frame_load(ctx, OP_SIZE_INT, garbage, slot_r, 0, false, target));
3283 g(gen_imm(ctx, sign_bit(uint_default_t) + 1, IMM_PURPOSE_ADD, OP_SIZE_INT));
3284 gen_insn(INSN_ALU, i_size(OP_SIZE_INT), ALU_ADD, ALU_WRITES_FLAGS(i_size(OP_SIZE_INT), ALU_ADD, false, is_imm(), ctx->const_imm));
3285 gen_one(R_SCRATCH_2);
3286 gen_one(target);
3287 gen_imm_offset();
3289 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, OP_SIZE_INT, R_SCRATCH_2, 1, COND_BE, label_ovf));
3291 return true;
3292 #endif
3293 #if defined(ARCH_ALPHA)
3294 gen_insn(INSN_FP_TO_INT64_TRAP, op_size, 0, 0);
3295 gen_one(FR_SCRATCH_2);
3296 gen_one(reg1);
3297 gen_four(label_ovf);
3299 if (OP_SIZE_INT == OP_SIZE_4) {
3300 gen_insn(INSN_FP_INT64_TO_INT32_TRAP, 0, 0, 0);
3301 gen_one(FR_SCRATCH_3);
3302 gen_one(FR_SCRATCH_2);
3303 gen_four(label_ovf);
3304 g(gen_frame_store_raw(ctx, OP_SIZE_INT, slot_r, 0, FR_SCRATCH_3));
3305 } else {
3306 g(gen_frame_store_raw(ctx, OP_SIZE_INT, slot_r, 0, FR_SCRATCH_2));
3307 }
3308 if (slot_is_register(ctx, slot_r))
3309 g(unspill(ctx, slot_r));
3310 return true;
3311 #endif
3312 #if defined(ARCH_S390)
3313 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_TO_INT32 : INSN_FP_TO_INT64, op_size, 0, 1);
3314 gen_one(target);
3315 gen_one(reg1);
3317 gen_insn(INSN_JMP_COND, op_size, FP_COND_P, 0);
3318 gen_four(label_ovf);
3320 g(gen_frame_store(ctx, OP_SIZE_INT, slot_r, 0, target));
3321 return true;
3322 #endif
3323 #if defined(ARCH_RISCV64)
3324 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_TO_INT32 : INSN_FP_TO_INT64, op_size, 0, 0);
3325 gen_one(target);
3326 gen_one(reg1);
3328 g(gen_load_constant(ctx, R_SCRATCH_2, sign_bit(int_default_t)));
3330 g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, target, R_SCRATCH_2, COND_E, label_ovf));
3332 g(gen_imm(ctx, -1, IMM_PURPOSE_XOR, i_size(size)));
3333 gen_insn(INSN_ALU, OP_SIZE_NATIVE, ALU_XOR, ALU_WRITES_FLAGS(OP_SIZE_NATIVE, ALU_XOR, false, is_imm(), ctx->const_imm));
3334 gen_one(R_SCRATCH_2);
3335 gen_one(R_SCRATCH_2);
3336 gen_imm_offset();
3338 g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, target, R_SCRATCH_2, COND_E, label_ovf));
3340 g(gen_frame_store(ctx, OP_SIZE_INT, slot_r, 0, target));
3341 return true;
3342 #endif
3343 }
3344 #ifdef SUPPORTED_FP_X87
3345 if ((SUPPORTED_FP_X87 >> real_type) & 1) {
3346 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
3348 if (likely(cpu_test_feature(CPU_FEATURE_sse3))) {
3349 g(gen_frame_store_x87(ctx, INSN_X87_FISTTP, OP_SIZE_INT, slot_r));
3350 } else {
3351 gen_insn(INSN_PUSH, OP_SIZE_NATIVE, 0, 0);
3352 gen_one(ARG_IMM);
3353 gen_eight(0x0f7f);
3355 gen_insn(INSN_X87_FLDCW, 0, 0, 0);
3356 gen_one(ARG_ADDRESS_1);
3357 gen_one(R_SP);
3358 gen_eight(0);
3360 g(gen_frame_store_x87(ctx, INSN_X87_FISTP, OP_SIZE_INT, slot_r));
3362 gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
3363 gen_one(ARG_ADDRESS_1);
3364 gen_one(R_SP);
3365 gen_eight(0);
3366 gen_one(ARG_IMM);
3367 gen_eight(0x037f);
3369 gen_insn(INSN_X87_FLDCW, 0, 0, 0);
3370 gen_one(ARG_ADDRESS_1);
3371 gen_one(R_SP);
3372 gen_eight(0);
3374 gen_insn(INSN_ALU, i_size(OP_SIZE_ADDRESS), ALU_ADD, 1);
3375 gen_one(R_SP);
3376 gen_one(R_SP);
3377 gen_one(ARG_IMM);
3378 gen_eight(1 << OP_SIZE_NATIVE);
3379 }
3380 if (slot_is_register(ctx, slot_r))
3381 g(unspill(ctx, slot_r));
3382 g(gen_frame_load(ctx, OP_SIZE_INT, garbage, slot_r, 0, false, R_SCRATCH_1));
3384 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, OP_SIZE_INT, R_SCRATCH_1, sign_bit(int_default_t), COND_E, label_ovf));
3386 return true;
3387 }
3388 #endif
3389 #ifdef SUPPORTED_FP_HALF_CVT
3390 if ((SUPPORTED_FP_HALF_CVT >> real_type) & 1) {
3391 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, ®1));
3392 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
3393 gen_one(FR_SCRATCH_1);
3394 gen_one(reg1);
3395 reg1 = FR_SCRATCH_1;
3396 real_type = 1;
3397 op_size = real_type_to_op_size(real_type);
3398 goto do_cvt_to_int;
3399 }
3400 #endif
3401 goto do_upcall;
3403 do_from_int:
3404 if ((SUPPORTED_FP >> real_type) & 1) {
3405 #if defined(ARCH_ALPHA) || defined(ARCH_ARM32) || defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS) || defined(ARCH_PARISC) || defined(ARCH_POWER) || defined(ARCH_SPARC)
3406 int int_op_size = OP_SIZE_INT;
3407 #if defined(ARCH_POWER)
3408 if (int_op_size == OP_SIZE_4)
3409 goto do_upcall;
3410 if (op_size == OP_SIZE_4 && !cpu_test_feature(CPU_FEATURE_v206))
3411 goto do_upcall;
3412 if (op_size == OP_SIZE_8 && !cpu_test_feature(CPU_FEATURE_ppc))
3413 goto do_upcall;
3414 #endif
3415 if (slot_is_register(ctx, slot_1))
3416 g(spill(ctx, slot_1));
3417 g(gen_frame_load_raw(ctx, int_op_size, zero_x, slot_1, 0, false, FR_SCRATCH_1));
3418 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_2);
3419 #if defined(ARCH_ALPHA)
3420 if (OP_SIZE_INT == OP_SIZE_4) {
3421 gen_insn(INSN_MOVSX, OP_SIZE_4, 0, 0);
3422 gen_one(FR_SCRATCH_1);
3423 gen_one(FR_SCRATCH_1);
3425 int_op_size = OP_SIZE_8;
3426 }
3427 #endif
3428 gen_insn(int_op_size == OP_SIZE_4 ? INSN_FP_FROM_INT32 : INSN_FP_FROM_INT64, op_size, 0, 0);
3429 gen_one(target);
3430 gen_one(FR_SCRATCH_1);
3432 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
3433 return true;
3434 #elif defined(ARCH_IA64)
3435 g(gen_frame_get(ctx, OP_SIZE_INT, sign_x, slot_1, R_SCRATCH_1, ®1));
3436 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_1);
3438 g(gen_mov(ctx, OP_SIZE_NATIVE, target, reg1));
3440 gen_insn(INSN_FP_FROM_INT64, op_size, 0, 0);
3441 gen_one(target);
3442 gen_one(target);
3444 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
3445 return true;
3446 #else
3447 g(gen_frame_get(ctx, OP_SIZE_INT, garbage, slot_1, R_SCRATCH_1, ®1));
3448 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_1);
3450 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_FROM_INT32 : INSN_FP_FROM_INT64, op_size, 0, 0);
3451 gen_one(target);
3452 gen_one(reg1);
3454 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
3455 return true;
3456 #endif
3457 }
3458 #ifdef SUPPORTED_FP_X87
3459 if ((SUPPORTED_FP_X87 >> real_type) & 1) {
3460 if (slot_is_register(ctx, slot_1))
3461 g(spill(ctx, slot_1));
3462 g(gen_frame_load_x87(ctx, INSN_X87_FILD, OP_SIZE_INT, 0, slot_1));
3463 g(gen_frame_store_x87(ctx, INSN_X87_FSTP, op_size, slot_r));
3464 return true;
3465 }
3466 #endif
3467 #ifdef SUPPORTED_FP_HALF_CVT
3468 if ((SUPPORTED_FP_HALF_CVT >> real_type) & 1) {
3469 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_1);
3470 #if defined(ARCH_ARM32)
3471 g(gen_frame_get(ctx, OP_SIZE_INT, zero_x, slot_1, FR_SCRATCH_1, ®1));
3473 gen_insn(INSN_FP_FROM_INT32, OP_SIZE_4, 0, 0);
3474 gen_one(target);
3475 gen_one(reg1);
3476 #else
3477 g(gen_frame_get(ctx, OP_SIZE_INT, garbage, slot_1, R_SCRATCH_1, ®1));
3478 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_FROM_INT32 : INSN_FP_FROM_INT64, OP_SIZE_4, 0, 0);
3479 gen_one(target);
3480 gen_one(reg1);
3481 #endif
3482 gen_insn(INSN_FP_CVT, OP_SIZE_4, op_size, 0);
3483 gen_one(target);
3484 gen_one(target);
3485 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
3486 return true;
3487 }
3488 #endif
3489 goto do_upcall;
3491 do_is_exception:
3492 if ((SUPPORTED_FP >> real_type) & 1) {
3493 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, ®1));
3494 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
3495 #if defined(ARCH_ALPHA)
3496 gen_insn(INSN_FP_CMP_UNORDERED_DEST_REG, op_size, 0, 0);
3497 gen_one(FR_SCRATCH_2);
3498 gen_one(FR_SCRATCH_1);
3499 gen_one(reg1);
3501 if (!cpu_test_feature(CPU_FEATURE_fix)) {
3502 g(gen_frame_store_raw(ctx, OP_SIZE_4, slot_r, 0, FR_SCRATCH_2));
3503 g(gen_frame_load_raw(ctx, OP_SIZE_4, sign_x, slot_r, 0, false, target));
3504 } else {
3505 g(gen_mov(ctx, OP_SIZE_4, target, FR_SCRATCH_2));
3506 }
3508 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SHR, target, target, 30, 0));
3510 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
3512 return true;
3513 #elif defined(ARCH_IA64)
3514 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_P, 0);
3515 gen_one(R_CMP_RESULT);
3516 gen_one(reg1);
3517 gen_one(reg1);
3519 g(gen_mov(ctx, OP_SIZE_NATIVE, target, R_CMP_RESULT));
3521 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
3522 #elif defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS) || defined(ARCH_PARISC)
3523 gen_insn(INSN_FP_CMP_COND, op_size, FP_COND_P, 1);
3524 gen_one(reg1);
3525 gen_one(reg1);
3527 gen_insn(INSN_FP_TEST_REG, OP_SIZE_NATIVE, FP_COND_P, 0);
3528 gen_one(target);
3530 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
3531 #elif defined(ARCH_RISCV64)
3532 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_E, 0);
3533 gen_one(target);
3534 gen_one(reg1);
3535 gen_one(reg1);
3537 g(gen_imm(ctx, 1, IMM_PURPOSE_XOR, OP_SIZE_NATIVE));
3538 gen_insn(INSN_ALU, OP_SIZE_NATIVE, ALU_XOR, ALU_WRITES_FLAGS(OP_SIZE_NATIVE, ALU_XOR, false, is_imm(), ctx->const_imm));
3539 gen_one(target);
3540 gen_one(target);
3541 gen_imm_offset();
3543 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
3544 #else
3545 gen_insn(INSN_FP_CMP, op_size, 0, 1);
3546 gen_one(reg1);
3547 gen_one(reg1);
3548 #if defined(ARCH_ARM32)
3549 gen_insn(INSN_FP_TO_INT_FLAGS, 0, 0, 1);
3550 #endif
3551 g(gen_frame_set_cond(ctx, op_size, false, FP_COND_P, slot_r));
3552 #endif
3553 return true;
3554 }
3555 #ifdef SUPPORTED_FP_X87
3556 if ((SUPPORTED_FP_X87 >> real_type) & 1) {
3557 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
3558 if (likely(cpu_test_feature(CPU_FEATURE_cmov))) {
3559 gen_insn(INSN_X87_FCOMIP, op_size, 0, 0);
3560 gen_one(R_ST0);
3562 g(gen_frame_set_cond(ctx, op_size, false, COND_P, slot_r));
3563 return true;
3564 }
3566 gen_insn(INSN_X87_FCOMP, op_size, 0, 0);
3567 gen_one(R_ST0);
3569 gen_insn(INSN_X87_FNSTSW, 0, 0, 0);
3570 gen_one(R_AX);
3571 gen_one(R_AX);
3573 gen_insn(INSN_TEST, OP_SIZE_2, 0, 1);
3574 gen_one(R_AX);
3575 gen_one(ARG_IMM);
3576 gen_eight(0x0400);
3578 g(gen_frame_set_cond(ctx, op_size, false, COND_NE, slot_r));
3580 return true;
3581 }
3582 #endif
3583 #ifdef SUPPORTED_FP_HALF_CVT
3584 if ((SUPPORTED_FP_HALF_CVT >> real_type) & 1) {
3585 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, ®1));
3586 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
3587 gen_one(FR_SCRATCH_1);
3588 gen_one(reg1);
3589 gen_insn(INSN_FP_CMP, OP_SIZE_4, 0, 1);
3590 gen_one(FR_SCRATCH_1);
3591 gen_one(FR_SCRATCH_1);
3592 #if defined(ARCH_ARM32)
3593 gen_insn(INSN_FP_TO_INT_FLAGS, 0, 0, 1);
3594 #endif
3595 g(gen_frame_set_cond(ctx, op_size, false, FP_COND_P, slot_r));
3596 return true;
3597 }
3598 #endif
3600 do_upcall:
3601 g(gen_alu_typed_upcall(ctx, upc, real_type, slot_1, NO_FRAME_T, slot_r, label_ovf));
3602 return true;
3603 }
3605 static bool attr_w gen_is_exception(struct codegen_context *ctx, frame_t slot_1, frame_t slot_r)
3606 {
3607 uint32_t no_ex_label, escape_label;
3608 const struct type *type = get_type_of_local(ctx, slot_1);
3610 no_ex_label = alloc_label(ctx);
3611 if (unlikely(!no_ex_label))
3612 return false;
3613 escape_label = alloc_escape_label(ctx);
3614 if (unlikely(!escape_label))
3615 return false;
3617 if (TYPE_IS_FLAT(type))
3618 g(gen_test_1_jz_cached(ctx, slot_1, no_ex_label));
3620 g(gen_frame_load(ctx, OP_SIZE_SLOT, zero_x, slot_1, 0, false, R_SCRATCH_1));
3621 g(gen_ptr_is_thunk(ctx, R_SCRATCH_1, slot_1, escape_label));
3623 if (!TYPE_IS_FLAT(type)) {
3624 g(gen_compare_da_tag(ctx, R_SCRATCH_1, DATA_TAG_flat, COND_E, escape_label, R_SCRATCH_1));
3625 }
3627 gen_label(no_ex_label);
3628 g(gen_frame_clear(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r));
3630 flag_set(ctx, slot_r, false);
3632 return true;
3633 }
3635 static bool attr_w gen_system_property(struct codegen_context *ctx, frame_t slot_1, frame_t slot_r)
3636 {
3637 uint32_t escape_label;
3639 escape_label = alloc_escape_label(ctx);
3640 if (unlikely(!escape_label))
3641 return false;
3643 g(gen_test_1_cached(ctx, slot_1, escape_label));
3645 g(gen_upcall_start(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_ipret_system_property), 1));
3647 g(gen_frame_load(ctx, OP_SIZE_INT, garbage, slot_1, 0, false, R_ARG0));
3648 g(gen_upcall_argument(ctx, 0));
3650 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_ipret_system_property), 1));
3652 g(gen_frame_store(ctx, OP_SIZE_INT, slot_r, 0, R_RET0));
3654 flag_set(ctx, slot_1, false);
3655 flag_set(ctx, slot_r, false);
3657 return true;
3658 }
3660 static bool attr_w gen_alu_jmp(struct codegen_context *ctx, unsigned mode, unsigned op_size, unsigned op, frame_t slot_1, frame_t slot_2, int32_t jmp_offset, bool *failed)
3661 {
3662 unsigned alu;
3663 enum extend ex;
3664 unsigned reg1;
3665 unsigned attr_unused reg2;
3666 #if 0
3667 *failed = true; return true;
3668 #endif
3669 switch (mode) {
3670 case MODE_FIXED: switch (op) {
3671 case OPCODE_FIXED_OP_equal: alu = COND_E; goto do_compare;
3672 case OPCODE_FIXED_OP_not_equal: alu = COND_NE; goto do_compare;
3673 case OPCODE_FIXED_OP_less: alu = COND_L; goto do_compare;
3674 case OPCODE_FIXED_OP_less_equal: alu = COND_LE; goto do_compare;
3675 case OPCODE_FIXED_OP_greater: alu = COND_G; goto do_compare;
3676 case OPCODE_FIXED_OP_greater_equal: alu = COND_GE; goto do_compare;
3677 case OPCODE_FIXED_OP_uless: alu = COND_B; goto do_compare;
3678 case OPCODE_FIXED_OP_uless_equal: alu = COND_BE; goto do_compare;
3679 case OPCODE_FIXED_OP_ugreater: alu = COND_A; goto do_compare;
3680 case OPCODE_FIXED_OP_ugreater_equal: alu = COND_AE; goto do_compare;
3681 case OPCODE_FIXED_OP_bt: *failed = true; return true;
3682 default: internal(file_line, "gen_alu_jmp: unsupported fixed operation %u", op);
3683 }
3684 case MODE_INT: switch (op) {
3685 case OPCODE_INT_OP_equal: alu = COND_E; goto do_compare;
3686 case OPCODE_INT_OP_not_equal: alu = COND_NE; goto do_compare;
3687 case OPCODE_INT_OP_less: alu = COND_L; goto do_compare;
3688 case OPCODE_INT_OP_less_equal: alu = COND_LE; goto do_compare;
3689 case OPCODE_INT_OP_greater: alu = COND_G; goto do_compare;
3690 case OPCODE_INT_OP_greater_equal: alu = COND_GE; goto do_compare;
3691 case OPCODE_INT_OP_bt: *failed = true; return true;
3692 default: internal(file_line, "gen_alu_jmp: unsupported int operation %u", op);
3693 }
3694 case MODE_BOOL: switch (op) {
3695 case OPCODE_BOOL_OP_and: alu = ALU_AND; mode = MODE_FIXED; goto do_alu;
3696 case OPCODE_BOOL_OP_or: alu = ALU_OR; mode = MODE_FIXED; goto do_alu;
3697 case OPCODE_BOOL_OP_equal: alu = COND_E; mode = MODE_FIXED; goto do_compare;
3698 case OPCODE_BOOL_OP_not_equal: alu = COND_NE; mode = MODE_FIXED; goto do_compare;
3699 case OPCODE_BOOL_OP_less: alu = COND_L; mode = MODE_FIXED; goto do_compare;
3700 case OPCODE_BOOL_OP_less_equal: alu = COND_LE; mode = MODE_FIXED; goto do_compare;
3701 case OPCODE_BOOL_OP_greater: alu = COND_G; mode = MODE_FIXED; goto do_compare;
3702 case OPCODE_BOOL_OP_greater_equal: alu = COND_GE; mode = MODE_FIXED; goto do_compare;
3703 default: internal(file_line, "gen_alu_jmp: unsupported bool operation %u", op);
3704 }
3705 }
3706 internal(file_line, "gen_alu_jmp: unsupported mode %u", mode);
3707 do_compare: {
3708 bool attr_unused logical;
3709 if (unlikely(op_size > OP_SIZE_NATIVE)) {
3710 *failed = true;
3711 return true;
3712 }
3713 if (slot_is_register(ctx, slot_2) && !slot_is_register(ctx, slot_1)) {
3714 frame_t s = slot_1;
3715 slot_1 = slot_2;
3716 slot_2 = s;
3717 switch (alu) {
3718 case COND_L: alu = COND_G; break;
3719 case COND_LE: alu = COND_GE; break;
3720 case COND_G: alu = COND_L; break;
3721 case COND_GE: alu = COND_LE; break;
3722 case COND_B: alu = COND_A; break;
3723 case COND_BE: alu = COND_AE; break;
3724 case COND_A: alu = COND_B; break;
3725 case COND_AE: alu = COND_BE; break;
3726 }
3727 }
3728 ex = op_size == i_size_cmp(op_size) + (unsigned)zero ? garbage : alu == COND_L || alu == COND_LE || alu == COND_G || alu == COND_GE || ARCH_PREFERS_SX(op_size) ? sign_x : zero_x;
3729 g(gen_frame_get(ctx, op_size, ex, slot_1, R_SCRATCH_1, ®1));
3730 if (ARCH_HAS_JMP_2REGS(alu)) {
3731 g(gen_frame_get(ctx, op_size, ex, slot_2, R_SCRATCH_2, ®2));
3732 g(gen_jump(ctx, jmp_offset, i_size_cmp(op_size), alu ^ 1, reg1, reg2));
3733 return true;
3734 }
3735 #if ARCH_HAS_FLAGS
3736 logical = COND_IS_LOGICAL(alu ^ 1);
3737 g(gen_frame_load_cmp(ctx, op_size, logical, ex, false, slot_2, 0, false, reg1));
3738 g(gen_jump(ctx, jmp_offset, op_size, alu ^ 1, -1U, -1U));
3739 #else
3740 g(gen_frame_get(ctx, op_size, ex, slot_2, R_SCRATCH_2, ®2));
3741 g(gen_cmp_dest_reg(ctx, op_size, reg1, reg2, R_CMP_RESULT, 0, alu));
3742 g(gen_jump(ctx, jmp_offset, OP_SIZE_NATIVE, COND_E, R_CMP_RESULT, -1U));
3743 #endif
3744 return true;
3745 }
3746 do_alu: {
3747 if (slot_is_register(ctx, slot_2) && !slot_is_register(ctx, slot_1)) {
3748 frame_t s = slot_1;
3749 slot_1 = slot_2;
3750 slot_2 = s;
3751 }
3752 ex = op_size == i_size(op_size) + (unsigned)zero ? garbage : ARCH_PREFERS_SX(op_size) ? sign_x : zero_x;
3753 g(gen_frame_get(ctx, op_size, ex, slot_1, R_SCRATCH_1, ®1));
3754 #if defined(ARCH_X86)
3755 if (alu == ALU_AND && !slot_is_register(ctx, slot_2)) {
3756 g(gen_address(ctx, R_FRAME, (size_t)slot_2 * slot_size, IMM_PURPOSE_LDR_OFFSET, op_size));
3757 gen_insn(INSN_TEST, i_size(op_size), 0, 1);
3758 gen_one(reg1);
3759 gen_address_offset();
3760 g(gen_jump(ctx, jmp_offset, op_size, COND_E, -1U, -1U));
3761 return true;
3762 }
3763 #endif
3764 g(gen_frame_get(ctx, op_size, ex, slot_2, R_SCRATCH_2, ®2));
3765 #if ARCH_HAS_FLAGS && !defined(ARCH_S390)
3766 if (alu == ALU_AND) {
3767 gen_insn(INSN_TEST, i_size(op_size), 0, 1);
3768 gen_one(reg1);
3769 gen_one(reg2);
3770 g(gen_jump(ctx, jmp_offset, op_size, COND_E, -1U, -1U));
3771 return true;
3772 }
3773 #endif
3774 #if defined(ARCH_ARM64)
3775 if (alu == ALU_OR)
3776 goto skip_flags;
3777 #endif
3778 #if ARCH_HAS_FLAGS
3779 g(gen_3address_alu(ctx, i_size(op_size), alu, R_SCRATCH_1, reg1, reg2, 1));
3780 g(gen_jump(ctx, jmp_offset, i_size(op_size), COND_E, -1U, -1U));
3781 return true;
3782 #endif
3783 goto skip_flags;
3784 skip_flags:
3785 g(gen_3address_alu(ctx, i_size(op_size), alu, R_SCRATCH_1, reg1, reg2, 0));
3786 g(gen_jump(ctx, jmp_offset, i_size(op_size), COND_E, R_SCRATCH_1, -1U));
3787 return true;
3788 }
3789 }
3791 static bool attr_w gen_fp_alu_jmp(struct codegen_context *ctx, unsigned real_type, unsigned op, uint32_t label_ovf, frame_t slot_1, frame_t slot_2, int32_t jmp_offset, bool *failed)
3792 {
3793 unsigned attr_unused fp_alu;
3794 unsigned attr_unused op_size = real_type_to_op_size(real_type);
3795 unsigned reg1, reg2;
3796 unsigned attr_unused target;
3797 switch (op) {
3798 case OPCODE_REAL_OP_equal:
3799 case OPCODE_REAL_OP_equal_alt1:
3800 case OPCODE_REAL_OP_equal_alt2: fp_alu = FP_COND_E; goto do_cmp;
3801 case OPCODE_REAL_OP_not_equal:
3802 case OPCODE_REAL_OP_not_equal_alt1:
3803 case OPCODE_REAL_OP_not_equal_alt2: fp_alu = FP_COND_NE; goto do_cmp;
3804 case OPCODE_REAL_OP_less:
3805 case OPCODE_REAL_OP_less_alt1:
3806 case OPCODE_REAL_OP_less_alt2: fp_alu = FP_COND_B; goto do_cmp;
3807 case OPCODE_REAL_OP_less_equal:
3808 case OPCODE_REAL_OP_less_equal_alt1:
3809 case OPCODE_REAL_OP_less_equal_alt2: fp_alu = FP_COND_BE; goto do_cmp;
3810 case OPCODE_REAL_OP_greater:
3811 case OPCODE_REAL_OP_greater_alt1:
3812 case OPCODE_REAL_OP_greater_alt2: fp_alu = FP_COND_A; goto do_cmp;
3813 case OPCODE_REAL_OP_greater_equal:
3814 case OPCODE_REAL_OP_greater_equal_alt1:
3815 case OPCODE_REAL_OP_greater_equal_alt2: fp_alu = FP_COND_AE; goto do_cmp;
3816 default: internal(file_line, "gen_fp_alu_jmp: unsupported operation %u", op);
3817 }
3819 do_cmp:
3820 if ((SUPPORTED_FP >> real_type) & 1
3821 #if defined(ARCH_ALPHA)
3822 && OS_SUPPORTS_TRAPS && cpu_test_feature(CPU_FEATURE_fix)
3823 #endif
3824 ) {
3825 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, ®1));
3826 g(gen_frame_get(ctx, op_size, zero_x, slot_2, FR_SCRATCH_2, ®2));
3827 target = R_SCRATCH_1;
3828 #if defined(ARCH_ALPHA)
3829 gen_insn(INSN_FP_CMP_DEST_REG_TRAP, op_size, fp_alu == FP_COND_NE ? FP_COND_E : fp_alu, 0);
3830 gen_one(FR_SCRATCH_3);
3831 gen_one(reg1);
3832 gen_one(reg2);
3833 gen_four(label_ovf);
3835 g(gen_mov(ctx, OP_SIZE_4, target, FR_SCRATCH_3));
3837 if (fp_alu == FP_COND_NE) {
3838 g(gen_jump(ctx, jmp_offset, OP_SIZE_NATIVE, COND_NE, target, -1U));
3839 } else {
3840 g(gen_jump(ctx, jmp_offset, OP_SIZE_NATIVE, COND_E, target, -1U));
3841 }
3843 return true;
3844 #elif defined(ARCH_IA64)
3845 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_P, 0);
3846 gen_one(R_CMP_RESULT);
3847 gen_one(reg1);
3848 gen_one(reg2);
3850 gen_insn(INSN_JMP_REG, OP_SIZE_NATIVE, COND_NE, 0);
3851 gen_one(R_CMP_RESULT);
3852 gen_four(label_ovf);
3854 gen_insn(INSN_FP_CMP_DEST_REG, op_size, fp_alu, 0);
3855 gen_one(R_CMP_RESULT);
3856 gen_one(reg1);
3857 gen_one(reg2);
3859 g(gen_jump(ctx, jmp_offset, OP_SIZE_NATIVE, COND_E, R_CMP_RESULT, -1U));
3861 return true;
3862 #elif defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS) || defined(ARCH_PARISC)
3863 gen_insn(INSN_FP_CMP_COND, op_size, FP_COND_P, 1);
3864 gen_one(reg1);
3865 gen_one(reg2);
3867 gen_insn(INSN_JMP_FP_TEST, 0, FP_COND_P, 0);
3868 gen_four(label_ovf);
3870 gen_insn(INSN_FP_CMP_COND, op_size, fp_alu ^ 1, 1);
3871 gen_one(reg1);
3872 gen_one(reg2);
3874 g(gen_jump(ctx, jmp_offset, OP_SIZE_NATIVE, fp_alu ^ 1, -1U, -1U));
3876 return true;
3877 #elif defined(ARCH_RISCV64)
3878 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_E, 0);
3879 gen_one(R_SCRATCH_1);
3880 gen_one(reg1);
3881 gen_one(reg1);
3883 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_E, 0);
3884 gen_one(R_SCRATCH_2);
3885 gen_one(reg2);
3886 gen_one(reg2);
3888 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_AND, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_2, 0));
3890 g(gen_jmp_on_zero(ctx, OP_SIZE_NATIVE, R_SCRATCH_1, COND_E, label_ovf));
3892 gen_insn(INSN_FP_CMP_DEST_REG, op_size, fp_alu == FP_COND_NE ? FP_COND_E : fp_alu, 0);
3893 gen_one(target);
3894 gen_one(reg1);
3895 gen_one(reg2);
3897 if (fp_alu == FP_COND_NE) {
3898 g(gen_jump(ctx, jmp_offset, OP_SIZE_NATIVE, COND_NE, target, -1U));
3899 } else {
3900 g(gen_jump(ctx, jmp_offset, OP_SIZE_NATIVE, COND_E, target, -1U));
3901 }
3902 return true;
3903 #else
3904 gen_insn(INSN_FP_CMP, op_size, 0, 1);
3905 gen_one(reg1);
3906 gen_one(reg2);
3907 #if defined(ARCH_ARM32)
3908 gen_insn(INSN_FP_TO_INT_FLAGS, 0, 0, 1);
3909 #endif
3910 gen_insn(INSN_JMP_COND, op_size, FP_COND_P, 0);
3911 gen_four(label_ovf);
3912 g(gen_jump(ctx, jmp_offset, op_size, fp_alu ^ 1, -1U, -1U));
3913 return true;
3914 #endif
3915 }
3916 *failed = true;
3917 return true;
3918 }