cg-alu.inc

   1 /*
   2  * Copyright (C) 2024 Mikulas Patocka
   3  *
   4  * This file is part of Ajla.
   5  *
   6  * Ajla is free software: you can redistribute it and/or modify it under the
   7  * terms of the GNU General Public License as published by the Free Software
   8  * Foundation, either version 3 of the License, or (at your option) any later
   9  * version.
  10  *
  11  * Ajla is distributed in the hope that it will be useful, but WITHOUT ANY
  12  * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
  13  * A PARTICULAR PURPOSE. See the GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along with
  16  * Ajla. If not, see <https://www.gnu.org/licenses/>.
  17  */
  18
  19 static bool attr_w gen_jump(struct codegen_context *ctx, int32_t jmp_offset, unsigned op_size, unsigned cond, unsigned reg1, unsigned reg2);
  20
  21 static bool attr_w gen_alu_upcall(struct codegen_context *ctx, size_t upcall, unsigned op_size, frame_t slot_1, frame_t slot_2, frame_t slot_r, uint32_t label_ovf)
  22 {
  23         if (slot_is_register(ctx, slot_1))
  24                 g(spill(ctx, slot_1));
  25         if (slot_2 != NO_FRAME_T && slot_is_register(ctx, slot_2))
  26                 g(spill(ctx, slot_2));
  27         g(gen_upcall_start(ctx, frame_t_is_const(slot_2) ? 4 : slot_2 != NO_FRAME_T ? 3 : 2));
  28         g(gen_frame_address(ctx, slot_1, 0, R_ARG0));
  29         g(gen_upcall_argument(ctx, 0));
  30         if (frame_t_is_const(slot_2)) {
  31                 size_t x_offs;
  32                 g(gen_load_constant(ctx, R_ARG1, frame_t_get_const(slot_2)));
  33                 g(gen_upcall_argument(ctx, 1));
  34                 g(gen_frame_address(ctx, slot_r, 0, R_ARG2));
  35                 g(gen_upcall_argument(ctx, 2));
  36                 g(gen_get_upcall_pointer(ctx, upcall, R_ARG3));
  37                 g(gen_upcall_argument(ctx, 3));
  38                 x_offs = offsetof(struct cg_upcall_vector_s, INT_binary_const_int8_t) + op_size * sizeof(void (*)(void));
  39                 g(gen_upcall(ctx, x_offs, 4));
  40         } else if (slot_2 != NO_FRAME_T) {
  41                 g(gen_frame_address(ctx, slot_2, 0, R_ARG1));
  42                 g(gen_upcall_argument(ctx, 1));
  43                 g(gen_frame_address(ctx, slot_r, 0, R_ARG2));
  44                 g(gen_upcall_argument(ctx, 2));
  45                 g(gen_upcall(ctx, upcall, 3));
  46         } else {
  47                 g(gen_frame_address(ctx, slot_r, 0, R_ARG1));
  48                 g(gen_upcall_argument(ctx, 1));
  49                 g(gen_upcall(ctx, upcall, 2));
  50         }
  51         if (slot_is_register(ctx, slot_r))
  52                 g(unspill(ctx, slot_r));
  53         if (label_ovf)
  54                 g(gen_jmp_on_zero(ctx, OP_SIZE_1, R_RET0, COND_E, label_ovf));
  55         return true;
  56 }
  57
  58 static bool attr_w gen_alu_typed_upcall(struct codegen_context *ctx, size_t upcall, unsigned op_size, frame_t slot_1, frame_t slot_2, frame_t slot_r, uint32_t label_ovf)
  59 {
  60         upcall += op_size * sizeof(void (*)(void));
  61         return gen_alu_upcall(ctx, upcall, op_size, slot_1, slot_2, slot_r, label_ovf);
  62 }
  63
  64 #define MODE_FIXED              0
  65 #define MODE_INT                1
  66 #define MODE_REAL               2
  67 #define MODE_BOOL               3
  68 #define MODE_ARRAY_LEN_GT       4
  69
  70 static bool attr_w gen_alu(struct codegen_context *ctx, unsigned mode, unsigned op_size, unsigned op, uint32_t label_ovf, frame_t slot_1, frame_t slot_2, frame_t slot_r)
  71 {
  72         unsigned alu;
  73         bool sgn, mod;
  74         unsigned reg1, reg2, reg3, target;
  75         switch (mode) {
  76                 case MODE_FIXED: switch (op) {
  77                         case OPCODE_FIXED_OP_add:               alu = ALU_ADD; goto do_alu;
  78                         case OPCODE_FIXED_OP_subtract:          alu = ALU_SUB; goto do_alu;
  79                         case OPCODE_FIXED_OP_multiply:          goto do_multiply;
  80                         case OPCODE_FIXED_OP_divide:
  81                         case OPCODE_FIXED_OP_divide_alt1:       sgn = true; mod = false; goto do_divide;
  82                         case OPCODE_FIXED_OP_udivide:
  83                         case OPCODE_FIXED_OP_udivide_alt1:      sgn = false; mod = false; goto do_divide;
  84                         case OPCODE_FIXED_OP_modulo:
  85                         case OPCODE_FIXED_OP_modulo_alt1:       sgn = true; mod = true; goto do_divide;
  86                         case OPCODE_FIXED_OP_umodulo:
  87                         case OPCODE_FIXED_OP_umodulo_alt1:      sgn = false; mod = true; goto do_divide;
  88                         case OPCODE_FIXED_OP_power:             return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_binary_power_int8_t), op_size, slot_1, slot_2, slot_r, 0);
  89                         case OPCODE_FIXED_OP_and:               alu = ALU_AND; goto do_alu;
  90                         case OPCODE_FIXED_OP_or:                alu = ALU_OR; goto do_alu;
  91                         case OPCODE_FIXED_OP_xor:               alu = ALU_XOR; goto do_alu;
  92                         case OPCODE_FIXED_OP_shl:               alu = ROT_SHL; goto do_shift;
  93                         case OPCODE_FIXED_OP_shr:               alu = ROT_SAR; goto do_shift;
  94                         case OPCODE_FIXED_OP_ushr:              alu = ROT_SHR; goto do_shift;
  95                         case OPCODE_FIXED_OP_rol:               alu = ROT_ROL; goto do_shift;
  96                         case OPCODE_FIXED_OP_ror:               alu = ROT_ROR; goto do_shift;
  97                         case OPCODE_FIXED_OP_bts:               alu = BTX_BTS; goto do_bt;
  98                         case OPCODE_FIXED_OP_btr:               alu = BTX_BTR; goto do_bt;
  99                         case OPCODE_FIXED_OP_btc:               alu = BTX_BTC; goto do_bt;
 100                         case OPCODE_FIXED_OP_equal:             alu = COND_E; goto do_compare;
 101                         case OPCODE_FIXED_OP_not_equal:         alu = COND_NE; goto do_compare;
 102                         case OPCODE_FIXED_OP_less:              alu = COND_L; goto do_compare;
 103                         case OPCODE_FIXED_OP_less_equal:        alu = COND_LE; goto do_compare;
 104                         case OPCODE_FIXED_OP_greater:           alu = COND_G; goto do_compare;
 105                         case OPCODE_FIXED_OP_greater_equal:     alu = COND_GE; goto do_compare;
 106                         case OPCODE_FIXED_OP_uless:             alu = COND_B; goto do_compare;
 107                         case OPCODE_FIXED_OP_uless_equal:       alu = COND_BE; goto do_compare;
 108                         case OPCODE_FIXED_OP_ugreater:          alu = COND_A; goto do_compare;
 109                         case OPCODE_FIXED_OP_ugreater_equal:    alu = COND_AE; goto do_compare;
 110                         case OPCODE_FIXED_OP_bt:                alu = BTX_BT; goto do_bt;
 111                         default:                                internal(file_line, "gen_alu: unsupported fixed operation %u", op);
 112                 }
 113                 case MODE_INT: switch (op) {
 114                         case OPCODE_INT_OP_add:                 alu = ALU_ADD; goto do_alu;
 115                         case OPCODE_INT_OP_subtract:            alu = ALU_SUB; goto do_alu;
 116                         case OPCODE_INT_OP_multiply:            goto do_multiply;
 117                         case OPCODE_INT_OP_divide:
 118                         case OPCODE_INT_OP_divide_alt1:         sgn = true; mod = false; goto do_divide;
 119                         case OPCODE_INT_OP_modulo:
 120                         case OPCODE_INT_OP_modulo_alt1:         sgn = true; mod = true; goto do_divide;
 121                         case OPCODE_INT_OP_power:               return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, INT_binary_power_int8_t), op_size, slot_1, slot_2, slot_r, label_ovf);
 122                         case OPCODE_INT_OP_and:                 alu = ALU_AND; mode = MODE_FIXED; goto do_alu;
 123                         case OPCODE_INT_OP_or:                  alu = ALU_OR; mode = MODE_FIXED; goto do_alu;
 124                         case OPCODE_INT_OP_xor:                 alu = ALU_XOR; mode = MODE_FIXED; goto do_alu;
 125                         case OPCODE_INT_OP_shl:                 alu = ROT_SHL; goto do_shift;
 126                         case OPCODE_INT_OP_shr:                 alu = ROT_SAR; goto do_shift;
 127                         case OPCODE_INT_OP_bts:                 alu = BTX_BTS; goto do_bt;
 128                         case OPCODE_INT_OP_btr:                 alu = BTX_BTR; goto do_bt;
 129                         case OPCODE_INT_OP_btc:                 alu = BTX_BTC; goto do_bt;
 130                         case OPCODE_INT_OP_equal:               alu = COND_E; goto do_compare;
 131                         case OPCODE_INT_OP_not_equal:           alu = COND_NE; goto do_compare;
 132                         case OPCODE_INT_OP_less:                alu = COND_L; goto do_compare;
 133                         case OPCODE_INT_OP_less_equal:          alu = COND_LE; goto do_compare;
 134                         case OPCODE_INT_OP_greater:             alu = COND_G; goto do_compare;
 135                         case OPCODE_INT_OP_greater_equal:       alu = COND_GE; goto do_compare;
 136                         case OPCODE_INT_OP_bt:                  alu = BTX_BT; goto do_bt;
 137                         default:                                internal(file_line, "gen_alu: unsupported int operation %u", op);
 138                 }
 139                 case MODE_BOOL: switch (op) {
 140                         case OPCODE_BOOL_OP_and:                alu = ALU_AND; mode = MODE_FIXED; goto do_alu;
 141                         case OPCODE_BOOL_OP_or:                 alu = ALU_OR; mode = MODE_FIXED; goto do_alu;
 142                         case OPCODE_BOOL_OP_equal:              alu = COND_E; goto do_compare;
 143                         case OPCODE_BOOL_OP_not_equal:          alu = ALU_XOR; mode = MODE_FIXED; goto do_alu;
 144                         case OPCODE_BOOL_OP_less:               alu = COND_L; goto do_compare;
 145                         case OPCODE_BOOL_OP_less_equal:         alu = COND_LE; goto do_compare;
 146                         case OPCODE_BOOL_OP_greater:            alu = COND_G; goto do_compare;
 147                         case OPCODE_BOOL_OP_greater_equal:      alu = COND_GE; goto do_compare;
 148                         default:                                internal(file_line, "gen_alu: unsupported bool operation %u", op);
 149                 }
 150         }
 151         internal(file_line, "gen_alu: unsupported mode %u", mode);
 152
 153         /*******
 154          * ALU *
 155          *******/
 156 do_alu: {
 157                 size_t attr_unused offset;
 158                 uint8_t attr_unused long_imm;
 159                 unsigned first_flags;
 160                 unsigned second_flags;
 161                 unsigned second_alu;
 162                 unsigned attr_unused op_size_flags;
 163                 bool c;
 164                 if (unlikely(op_size > OP_SIZE_NATIVE)) {
 165 #if !defined(ARCH_X86) && !defined(ARCH_ARM) && !defined(ARCH_PARISC) && !defined(ARCH_POWER) && !defined(ARCH_SPARC32)
 166                         if (mode == MODE_FIXED) {
 167                                 if (alu == ALU_ADD) {
 168                                         g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_add_,TYPE_INT_MAX)), op_size, slot_1, slot_2, slot_r, 0));
 169                                         return true;
 170                                 } else if (alu == ALU_SUB) {
 171                                         g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_subtract_,TYPE_INT_MAX)), op_size, slot_1, slot_2, slot_r, 0));
 172                                         return true;
 173                                 }
 174                         } else if (mode == MODE_INT) {
 175                                 if (alu == ALU_ADD) {
 176                                         g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(INT_binary_add_,TYPE_INT_MAX)), op_size, slot_1, slot_2, slot_r, label_ovf));
 177                                         return true;
 178                                 } else if (alu == ALU_SUB) {
 179                                         g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(INT_binary_subtract_,TYPE_INT_MAX)), op_size, slot_1, slot_2, slot_r, label_ovf));
 180                                         return true;
 181                                 }
 182                         }
 183 #endif
 184                         first_flags = alu == ALU_ADD || alu == ALU_SUB ? 2 : 0;
 185                         second_flags = mode == MODE_INT ? 1 : 0;
 186                         second_alu = alu == ALU_ADD ? ALU_ADC : alu == ALU_SUB ? ALU_SBB : alu;
 187                         g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
 188 #if defined(ARCH_X86)
 189                         g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, alu, first_flags, slot_2, lo_word(OP_SIZE_NATIVE), true, R_SCRATCH_1));
 190                         g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, second_alu, second_flags, slot_2, hi_word(OP_SIZE_NATIVE), true, R_SCRATCH_2));
 191 #else
 192                         g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_2, 0, R_SCRATCH_3, R_SCRATCH_4));
 193                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, alu, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_3, first_flags));
 194 #if defined(ARCH_PARISC)
 195                         if (mode == MODE_INT) {
 196                                 gen_insn(INSN_ALU_FLAGS_TRAP, OP_SIZE_NATIVE, second_alu, ALU_WRITES_FLAGS(OP_SIZE_NATIVE, second_alu, false, false, 0));
 197                                 gen_one(R_SCRATCH_2);
 198                                 gen_one(R_SCRATCH_2);
 199                                 gen_one(R_SCRATCH_4);
 200                                 gen_four(label_ovf);
 201                         } else
 202 #endif
 203                         {
 204                                 gen_insn(first_flags ? INSN_ALU_FLAGS : INSN_ALU, OP_SIZE_NATIVE, second_alu, second_flags | ALU_WRITES_FLAGS(OP_SIZE_NATIVE, second_alu, false, false, 0));
 205                                 gen_one(R_SCRATCH_2);
 206                                 gen_one(R_SCRATCH_2);
 207                                 gen_one(R_SCRATCH_4);
 208                         }
 209 #endif
 210 #if !defined(ARCH_PARISC)
 211                         if (mode == MODE_INT) {
 212                                 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_O, 0);
 213                                 gen_four(label_ovf);
 214                         }
 215 #endif
 216                         g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_1, R_SCRATCH_2));
 217                         return true;
 218                 }
 219
 220                 if (((ARCH_HAS_FLAGS && i_size_cmp(op_size) == op_size + zero) || ARCH_SUPPORTS_TRAPS(op_size)) && slot_2 == slot_r && slot_1 != slot_2 && alu_is_commutative(alu)) {
 221                         frame_t x = slot_1;
 222                         slot_1 = slot_2;
 223                         slot_2 = x;
 224                 }
 225                 if (((ARCH_HAS_FLAGS && i_size_cmp(op_size) == op_size + zero) || ARCH_SUPPORTS_TRAPS(op_size)) && slot_1 == slot_r && (slot_1 != slot_2 || mode != MODE_INT)
 226 #if defined(ARCH_POWER)
 227                         && op_size == OP_SIZE_NATIVE
 228 #endif
 229                     ) {
 230                         struct cg_exit *ce;
 231                         unsigned undo_alu = alu == ALU_ADD ? ALU_SUB : ALU_ADD;
 232                         if (slot_is_register(ctx, slot_1)) {
 233                                 unsigned reg1 = ctx->registers[slot_1];
 234                                 if (slot_is_register(ctx, slot_2)
 235 #if !defined(ARCH_POWER)
 236                                     || frame_t_is_const(slot_2)
 237 #endif
 238                                     ) {
 239                                         unsigned reg2 = frame_t_is_const(slot_2) ? 0xff /* avoid warning */ : ctx->registers[slot_2];
 240                                         if (mode == MODE_INT && ARCH_SUPPORTS_TRAPS(op_size)) {
 241                                                 if (frame_t_is_const(slot_2))
 242                                                         g(gen_imm(ctx, frame_t_get_const(slot_2), alu_trap_purpose(alu) | (alu_purpose(undo_alu) << 8), i_size(op_size)));
 243                                                 gen_insn(INSN_ALU_TRAP, op_size, alu, ALU_WRITES_FLAGS(op_size, alu, false, frame_t_is_const(slot_2) && is_imm(), ctx->const_imm));
 244                                                 gen_one(reg1);
 245                                                 gen_one(reg1);
 246                                                 if (frame_t_is_const(slot_2))
 247                                                         gen_imm_offset();
 248                                                 else
 249                                                         gen_one(reg2);
 250                                                 if (ARCH_TRAP_BEFORE) {
 251                                                         gen_four(label_ovf);
 252                                                         return true;
 253                                                 } else {
 254                                                         ce = alloc_undo_label(ctx);
 255                                                         if (unlikely(!ce))
 256                                                                 return false;
 257                                                         gen_four(ce->undo_label);
 258                                                         goto do_undo_opcode;
 259                                                 }
 260                                         }
 261                                         if (frame_t_is_const(slot_2))
 262                                                 g(gen_imm(ctx, frame_t_get_const(slot_2), alu_purpose(alu) | (alu_purpose(undo_alu) << 8), i_size(op_size)));
 263                                         gen_insn(INSN_ALU + ARCH_PARTIAL_ALU(i_size(op_size)), i_size(op_size), alu, ALU_WRITES_FLAGS(i_size(op_size), alu, false, frame_t_is_const(slot_2) && is_imm(), ctx->const_imm));
 264                                         gen_one(reg1);
 265                                         gen_one(reg1);
 266                                         if (frame_t_is_const(slot_2))
 267                                                 gen_imm_offset();
 268                                         else
 269                                                 gen_one(reg2);
 270                                         if (mode == MODE_INT) {
 271                                                 size_t m;
 272                                                 ce = alloc_undo_label(ctx);
 273                                                 if (unlikely(!ce))
 274                                                         return false;
 275                                                 gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
 276                                                 gen_four(ce->undo_label);
 277 do_undo_opcode:
 278                                                 ce->undo_opcode = INSN_ALU + ARCH_PARTIAL_ALU(i_size(op_size));
 279                                                 ce->undo_op_size = i_size(op_size);
 280                                                 ce->undo_aux = undo_alu;
 281                                                 ce->undo_writes_flags = ALU_WRITES_FLAGS(i_size(op_size), undo_alu, false, frame_t_is_const(slot_2) && is_imm(), ctx->const_imm);
 282                                                 m = mark_params(ctx);
 283                                                 gen_one(reg1);
 284                                                 gen_one(reg1);
 285                                                 if (frame_t_is_const(slot_2))
 286                                                         gen_imm_offset();
 287                                                 else
 288                                                         gen_one(reg2);
 289                                                 copy_params(ctx, ce, m);
 290                                         }
 291                                         return true;
 292                                 }
 293 #if defined(ARCH_S390) || defined(ARCH_X86)
 294                                 else if (!frame_t_is_const(slot_2)) {
 295                                         size_t m;
 296                                         int64_t offset = (size_t)slot_2 * slot_size;
 297                                         g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDR_OFFSET, i_size(op_size)));
 298                                         gen_insn(INSN_ALU + ARCH_PARTIAL_ALU(i_size(op_size)), i_size(op_size), alu, ALU_WRITES_FLAGS(i_size(op_size), alu, true, false, 0) | 1);
 299                                         gen_one(reg1);
 300                                         gen_one(reg1);
 301                                         gen_address_offset();
 302                                         if (mode == MODE_INT) {
 303                                                 ce = alloc_undo_label(ctx);
 304                                                 if (unlikely(!ce))
 305                                                         return false;
 306                                                 ce->undo_opcode = INSN_ALU + ARCH_PARTIAL_ALU(i_size(op_size));
 307                                                 ce->undo_op_size = i_size(op_size);
 308                                                 ce->undo_aux = undo_alu;
 309                                                 ce->undo_writes_flags = ALU_WRITES_FLAGS(i_size(op_size), undo_alu, true, false, 0);
 310                                                 m = mark_params(ctx);
 311                                                 gen_one(reg1);
 312                                                 gen_one(reg1);
 313                                                 gen_address_offset();
 314                                                 copy_params(ctx, ce, m);
 315                                                 gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
 316                                                 gen_four(ce->undo_label);
 317                                         }
 318                                         return true;
 319                                 }
 320 #endif
 321                         }
 322 #if defined(ARCH_X86)
 323                         else {
 324                                 unsigned reg2;
 325                                 size_t m;
 326                                 int64_t offset = (size_t)slot_1 * slot_size;
 327                                 if (!frame_t_is_const(slot_2))
 328                                         g(gen_frame_get(ctx, op_size, garbage, slot_2, R_SCRATCH_1, &reg2));
 329                                 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDR_OFFSET, i_size(op_size)));
 330                                 if (frame_t_is_const(slot_2))
 331                                         g(gen_imm(ctx, frame_t_get_const(slot_2), alu_purpose(alu) | (alu_purpose(undo_alu) << 8), i_size(op_size)));
 332                                 gen_insn(INSN_ALU + ARCH_PARTIAL_ALU(op_size), i_size(op_size), alu, ALU_WRITES_FLAGS(i_size(op_size), undo_alu, true, frame_t_is_const(slot_2) && is_imm(), ctx->const_imm) | 1);
 333                                 gen_address_offset();
 334                                 gen_address_offset();
 335                                 if (frame_t_is_const(slot_2))
 336                                         gen_imm_offset();
 337                                 else
 338                                         gen_one(reg2);
 339                                 if (mode == MODE_INT) {
 340                                         ce = alloc_undo_label(ctx);
 341                                         if (unlikely(!ce))
 342                                                 return false;
 343                                         ce->undo_opcode = INSN_ALU + ARCH_PARTIAL_ALU(op_size);
 344                                         ce->undo_op_size = i_size(op_size);
 345                                         ce->undo_aux = undo_alu;
 346                                         ce->undo_writes_flags = ALU_WRITES_FLAGS(i_size(op_size), undo_alu, true, false, 0);
 347                                         m = mark_params(ctx);
 348                                         gen_address_offset();
 349                                         gen_address_offset();
 350                                         if (frame_t_is_const(slot_2))
 351                                                 gen_imm_offset();
 352                                         else
 353                                                 gen_one(reg2);
 354                                         copy_params(ctx, ce, m);
 355                                         gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
 356                                         gen_four(ce->undo_label);
 357                                 }
 358                                 return true;
 359                         }
 360 #endif
 361                 }
 362
 363 #if defined(ARCH_X86)
 364                 if (1)
 365 #elif defined(ARCH_S390)
 366                 if (op_size >= OP_SIZE_4)
 367 #elif ARCH_HAS_FLAGS && !defined(ARCH_POWER)
 368                 if (op_size == i_size(op_size) + (unsigned)zero && frame_t_is_const(slot_2))
 369 #else
 370                 if (mode != MODE_INT && op_size == i_size(op_size) + (unsigned)zero && frame_t_is_const(slot_2))
 371 #endif
 372                 {
 373                         if (mode == MODE_INT) {
 374                                 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
 375                         } else {
 376                                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, slot_2, R_SCRATCH_1);
 377                         }
 378                         g(gen_frame_load(ctx, op_size, garbage, slot_1, 0, false, target));
 379                         g(gen_frame_load_op(ctx, op_size, garbage, alu, mode == MODE_INT, slot_2, 0, false, target));
 380                         goto check_ovf_store;
 381                 }
 382                 op_size_flags = !ARCH_HAS_FLAGS && !ARCH_SUPPORTS_TRAPS(op_size) ? OP_SIZE_NATIVE : OP_SIZE_4;
 383 #if defined(ARCH_POWER)
 384                 op_size_flags = OP_SIZE_NATIVE;
 385 #endif
 386                 g(gen_frame_get(ctx, op_size, mode == MODE_INT && (op_size < op_size_flags || ARCH_SUPPORTS_TRAPS(op_size)) ? sign_x : garbage, slot_1, R_SCRATCH_1, &reg1));
 387                 if (frame_t_is_const(slot_2)
 388 #if defined(ARCH_POWER)
 389                         && 0
 390 #endif
 391                         ) {
 392                         reg2 = 0xff;
 393                         c = true;
 394                 } else {
 395                         g(gen_frame_get(ctx, op_size, mode == MODE_INT && (op_size < op_size_flags || ARCH_SUPPORTS_TRAPS(op_size)) ? sign_x : garbage, slot_2, R_SCRATCH_2, &reg2));
 396                         c = false;
 397                 }
 398 #if !ARCH_HAS_FLAGS
 399                 if (mode == MODE_INT && op_size >= OP_SIZE_4) {
 400                         if (ARCH_SUPPORTS_TRAPS(op_size)) {
 401                                 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
 402                                 if (c)
 403                                         g(gen_imm(ctx, frame_t_get_const(slot_2), alu_trap_purpose(alu), op_size));
 404                                 gen_insn(INSN_ALU_TRAP, op_size, alu, ALU_WRITES_FLAGS(op_size, alu, false, c && is_imm(), ctx->const_imm));
 405                                 gen_one(target);
 406                                 gen_one(reg1);
 407                                 if (c)
 408                                         gen_imm_offset();
 409                                 else
 410                                         gen_one(reg2);
 411                                 gen_four(label_ovf);
 412                                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
 413                                 return true;
 414                         }
 415                         if (op_size >= OP_SIZE_NATIVE) {
 416                                 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_3);
 417                                 if (c)
 418                                         g(gen_3address_alu_imm(ctx, i_size(op_size), alu, target, reg1, frame_t_get_const(slot_2), 0));
 419                                 else
 420                                         g(gen_3address_alu(ctx, i_size(op_size), alu, target, reg1, reg2, 0));
 421 #if defined(ARCH_IA64)
 422                                 if (c) {
 423                                         g(gen_3address_alu_imm(ctx, i_size(op_size), ALU_XOR, R_SCRATCH_1, reg1, frame_t_get_const(slot_2), 0));
 424                                         g(gen_3address_alu_imm(ctx, i_size(op_size), ALU_XOR, R_SCRATCH_2, target, frame_t_get_const(slot_2), 0));
 425                                 } else {
 426                                         g(gen_3address_alu(ctx, i_size(op_size), ALU_XOR, R_SCRATCH_1, reg1, reg2, 0));
 427                                         g(gen_3address_alu(ctx, i_size(op_size), ALU_XOR, R_SCRATCH_2, target, reg2, 0));
 428                                 }
 429                                 if (alu == ALU_ADD) {
 430                                         g(gen_3address_alu(ctx, i_size(op_size), ALU_ANDN, R_SCRATCH_1, R_SCRATCH_2, R_SCRATCH_1, 0));
 431                                 } else {
 432                                         g(gen_3address_alu(ctx, i_size(op_size), ALU_ANDN, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_2, 0));
 433                                 }
 434                                 g(gen_cmp_test_jmp(ctx, INSN_TEST, i_size(op_size), R_SCRATCH_1, R_SCRATCH_1, COND_S, label_ovf));
 435 #else
 436                                 if (c) {
 437                                         g(gen_cmp_test_jmp(ctx, INSN_CMP, i_size(op_size), reg1, target, (frame_t_get_const(slot_2) >= 0) ^ (alu != ALU_ADD) ? COND_G : COND_L, label_ovf));
 438                                 } else {
 439                                         gen_insn(INSN_CMP_DEST_REG, i_size(op_size), COND_L, 0);
 440                                         gen_one(R_SCRATCH_1);
 441                                         if (alu == ALU_ADD) {
 442                                                 gen_one(target);
 443                                                 gen_one(reg1);
 444                                         } else {
 445                                                 gen_one(reg1);
 446                                                 gen_one(target);
 447                                         }
 448
 449                                         g(gen_imm(ctx, 0, IMM_PURPOSE_CMP, i_size(op_size)));
 450                                         gen_insn(INSN_CMP_DEST_REG, i_size(op_size), COND_L, 0);
 451                                         gen_one(R_SCRATCH_2);
 452                                         gen_one(reg2);
 453                                         gen_imm_offset();
 454
 455                                         g(gen_cmp_test_jmp(ctx, INSN_CMP, i_size(op_size), R_SCRATCH_1, R_SCRATCH_2, COND_NE, label_ovf));
 456                                 }
 457 #endif
 458                                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
 459                                 return true;
 460                         }
 461                 }
 462 #endif
 463                 if (mode == MODE_INT) {
 464                         target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
 465                 } else if (!ARCH_IS_3ADDRESS(alu, mode == MODE_INT && op_size >= op_size_flags) && !alu_is_commutative(alu)) {
 466                         target = gen_frame_target(ctx, slot_r, NO_FRAME_T, slot_2, R_SCRATCH_1);
 467                 } else {
 468                         target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
 469                 }
 470                 if (c) {
 471                         g(gen_3address_alu_imm(ctx, i_size(op_size), alu, target, reg1, frame_t_get_const(slot_2), mode == MODE_INT && op_size >= op_size_flags));
 472                 } else {
 473                         g(gen_3address_alu(ctx, i_size(op_size), alu, target, reg1, reg2, mode == MODE_INT && op_size >= op_size_flags));
 474                 }
 475
 476                 if (mode == MODE_INT && unlikely(op_size < op_size_flags)) {
 477                         g(gen_cmp_extended(ctx, op_size_flags, op_size, target, R_SCRATCH_2, label_ovf));
 478                 } else
 479 check_ovf_store:
 480                 if (mode == MODE_INT) {
 481                         gen_insn(INSN_JMP_COND, op_size, COND_O, 0);
 482                         gen_four(label_ovf);
 483                 }
 484                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
 485                 return true;
 486         }
 487
 488         /************
 489          * MULTIPLY *
 490          ************/
 491 do_multiply: {
 492                 size_t attr_unused offset;
 493                 uint8_t attr_unused long_imm;
 494                 if (unlikely(op_size > OP_SIZE_NATIVE) || unlikely(!ARCH_HAS_MUL)) {
 495                         if (mode == MODE_INT) {
 496                                 g(gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, INT_binary_multiply_int8_t), op_size, slot_1, slot_2, slot_r, label_ovf));
 497                                 return true;
 498                         }
 499 #if defined(ARCH_X86)
 500                         g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_1, hi_word(OP_SIZE_NATIVE), true, R_CX));
 501                         g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_2, hi_word(OP_SIZE_NATIVE), true, R_AX));
 502                         g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, ALU_MUL, true, slot_2, lo_word(OP_SIZE_NATIVE), true, R_CX));
 503                         g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, ALU_MUL, true, slot_1, lo_word(OP_SIZE_NATIVE), true, R_AX));
 504                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_ADD, R_CX, R_CX, R_AX, 0));
 505                         g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_2, lo_word(OP_SIZE_NATIVE), true, R_AX));
 506
 507                         offset = (size_t)slot_1 * slot_size + lo_word(OP_SIZE_NATIVE);
 508                         g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
 509                         gen_insn(INSN_MUL_L, OP_SIZE_NATIVE, 0, 1);
 510                         gen_one(R_AX);
 511                         gen_one(R_DX);
 512                         gen_one(R_AX);
 513                         gen_address_offset();
 514
 515                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_ADD, R_DX, R_DX, R_CX, 0));
 516
 517                         g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_AX, R_DX));
 518
 519                         return true;
 520 #elif defined(ARCH_ARM32)
 521                         g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
 522                         g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_2, 0, R_SCRATCH_3, R_SCRATCH_4));
 523
 524                         g(gen_mov(ctx, OP_SIZE_NATIVE, R_SCRATCH_NA_1, R_SCRATCH_1));
 525
 526                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_MUL, R_SCRATCH_4, R_SCRATCH_1, R_SCRATCH_4, 0));
 527
 528                         gen_insn(INSN_MADD, OP_SIZE_NATIVE, 0, 0);
 529                         gen_one(R_SCRATCH_2);
 530                         gen_one(R_SCRATCH_3);
 531                         gen_one(R_SCRATCH_2);
 532                         gen_one(R_SCRATCH_4);
 533
 534                         gen_insn(INSN_MUL_L, OP_SIZE_NATIVE, 0, 0);
 535                         gen_one(R_SCRATCH_1);
 536                         gen_one(R_SCRATCH_4);
 537                         gen_one(R_SCRATCH_NA_1);
 538                         gen_one(R_SCRATCH_3);
 539
 540                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_ADD, R_SCRATCH_2, R_SCRATCH_2, R_SCRATCH_4, 0));
 541
 542                         g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_1, R_SCRATCH_2));
 543
 544                         return true;
 545 #elif defined(ARCH_ARM64)
 546                         g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
 547                         g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_2, 0, R_SCRATCH_3, R_SCRATCH_4));
 548
 549                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_UMULH, R_SCRATCH_NA_1, R_SCRATCH_1, R_SCRATCH_3, 0));
 550
 551                         gen_insn(INSN_MADD, OP_SIZE_NATIVE, 0, 0);
 552                         gen_one(R_SCRATCH_NA_1);
 553                         gen_one(R_SCRATCH_2);
 554                         gen_one(R_SCRATCH_3);
 555                         gen_one(R_SCRATCH_NA_1);
 556
 557                         gen_insn(INSN_MADD, OP_SIZE_NATIVE, 0, 0);
 558                         gen_one(R_SCRATCH_2);
 559                         gen_one(R_SCRATCH_1);
 560                         gen_one(R_SCRATCH_4);
 561                         gen_one(R_SCRATCH_NA_1);
 562
 563                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_MUL, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_3, 0));
 564
 565                         g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_1, R_SCRATCH_2));
 566
 567                         return true;
 568 #else
 569                         g(gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_binary_multiply_int8_t), op_size, slot_1, slot_2, slot_r, 0));
 570                         return true;
 571 #endif
 572                 }
 573
 574 #if defined(ARCH_X86)
 575                 if (mode == MODE_INT) {
 576                         if (op_size != OP_SIZE_1 && slot_r == slot_1 && slot_is_register(ctx, slot_1)) {
 577                                 struct cg_exit *ce;
 578                                 target = ctx->registers[slot_1];
 579                                 g(gen_mov(ctx, op_size, R_SCRATCH_1, target));
 580                                 g(gen_frame_load_op(ctx, op_size, garbage, ALU_MUL, mode == MODE_INT, slot_2, 0, false, target));
 581                                 ce = alloc_undo_label(ctx);
 582                                 if (unlikely(!ce))
 583                                         return false;
 584                                 ce->undo_opcode = INSN_MOV;
 585                                 ce->undo_op_size = op_size;
 586                                 ce->undo_aux = 0;
 587                                 ce->undo_writes_flags = 0;
 588                                 ce->undo_parameters[0] = target;
 589                                 ce->undo_parameters[1] = R_SCRATCH_1;
 590                                 ce->undo_parameters_len = 2;
 591                                 gen_insn(INSN_JMP_COND, op_size, COND_O, 0);
 592                                 gen_four(ce->undo_label);
 593                                 return true;
 594                         }
 595                         target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
 596                 } else {
 597                         target = gen_frame_target(ctx, slot_r, NO_FRAME_T, slot_2, R_SCRATCH_1);
 598                 }
 599                 if (op_size == OP_SIZE_1)
 600                         target = R_SCRATCH_1;
 601                 g(gen_frame_load(ctx, op_size, garbage, slot_1, 0, false, target));
 602                 if (op_size == OP_SIZE_1 && frame_t_is_const(slot_2)) {
 603                         g(gen_load_constant(ctx, R_SCRATCH_3, frame_t_get_const(slot_2)));
 604                         gen_insn(INSN_ALU + ARCH_PARTIAL_ALU(op_size), op_size, ALU_MUL, 1);
 605                         gen_one(target);
 606                         gen_one(target);
 607                         gen_one(R_SCRATCH_3);
 608                 } else {
 609                         g(gen_frame_load_op(ctx, op_size, garbage, ALU_MUL, mode == MODE_INT, slot_2, 0, false, target));
 610                 }
 611                 if (mode == MODE_INT) {
 612                         gen_insn(INSN_JMP_COND, op_size, COND_O, 0);
 613                         gen_four(label_ovf);
 614                 }
 615                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
 616                 return true;
 617 #endif
 618 #if defined(ARCH_ALPHA)
 619                 if (mode == MODE_INT && ARCH_SUPPORTS_TRAPS(op_size)) {
 620                         target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
 621                         g(gen_frame_get(ctx, op_size, garbage, slot_1, R_SCRATCH_1, &reg1));
 622                         g(gen_frame_get(ctx, op_size, garbage, slot_2, R_SCRATCH_2, &reg2));
 623
 624                         gen_insn(INSN_ALU_TRAP, op_size, ALU_MUL, ALU_WRITES_FLAGS(op_size, ALU_MUL, false, false, 0));
 625                         gen_one(target);
 626                         gen_one(reg1);
 627                         gen_one(reg2);
 628                         gen_four(label_ovf);
 629                         g(gen_frame_store(ctx, op_size, slot_r, 0, target));
 630
 631                         return true;
 632                 }
 633 #endif
 634 #if defined(ARCH_ARM32)
 635                 if (mode == MODE_INT && op_size == OP_SIZE_4) {
 636                         target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_3);
 637                         g(gen_frame_get(ctx, op_size, garbage, slot_1, R_SCRATCH_1, &reg1));
 638                         g(gen_frame_get(ctx, op_size, garbage, slot_2, R_SCRATCH_2, &reg2));
 639
 640                         gen_insn(INSN_MUL_L, OP_SIZE_NATIVE, 0, 0);
 641                         gen_one(target);
 642                         gen_one(R_SCRATCH_4);
 643                         gen_one(reg1);
 644                         gen_one(reg2);
 645
 646                         gen_insn(INSN_CMP, OP_SIZE_NATIVE, 0, 1);
 647                         gen_one(R_SCRATCH_4);
 648                         gen_one(ARG_SHIFTED_REGISTER);
 649                         gen_one(ARG_SHIFT_ASR | 0x1f);
 650                         gen_one(target);
 651
 652                         gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_NE, 0);
 653                         gen_four(label_ovf);
 654
 655                         g(gen_frame_store(ctx, op_size, slot_r, 0, target));
 656
 657                         return true;
 658                 }
 659 #endif
 660 #if defined(ARCH_ARM64)
 661                 if (mode == MODE_INT && op_size == OP_SIZE_4) {
 662                         target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
 663                         g(gen_frame_get(ctx, op_size, op_size < OP_SIZE_4 ? sign_x : garbage, slot_1, R_SCRATCH_1, &reg1));
 664                         g(gen_frame_get(ctx, op_size, op_size < OP_SIZE_4 ? sign_x : garbage, slot_2, R_SCRATCH_2, &reg2));
 665                         gen_insn(INSN_ALU, OP_SIZE_8, ALU_MUL, ALU_WRITES_FLAGS(OP_SIZE_8, ALU_MUL, false, false, 0));
 666                         gen_one(target);
 667                         gen_one(ARG_EXTENDED_REGISTER);
 668                         gen_one(ARG_EXTEND_SXTW);
 669                         gen_one(reg1);
 670                         gen_one(ARG_EXTENDED_REGISTER);
 671                         gen_one(ARG_EXTEND_SXTW);
 672                         gen_one(reg2);
 673
 674                         gen_insn(INSN_CMP, OP_SIZE_8, 0, 1);
 675                         gen_one(target);
 676                         gen_one(ARG_EXTENDED_REGISTER);
 677                         gen_one(ARG_EXTEND_SXTW);
 678                         gen_one(target);
 679
 680                         gen_insn(INSN_JMP_COND, OP_SIZE_8, COND_NE, 0);
 681                         gen_four(label_ovf);
 682
 683                         g(gen_frame_store(ctx, op_size, slot_r, 0, target));
 684
 685                         return true;
 686                 }
 687                 if (mode == MODE_INT && op_size == OP_SIZE_8) {
 688                         target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
 689                         g(gen_frame_get(ctx, op_size, garbage, slot_1, R_SCRATCH_1, &reg1));
 690                         g(gen_frame_get(ctx, op_size, garbage, slot_2, R_SCRATCH_2, &reg2));
 691                         g(gen_3address_alu(ctx, OP_SIZE_8, ALU_SMULH, R_SCRATCH_3, reg1, reg2, 0));
 692
 693                         g(gen_3address_alu(ctx, OP_SIZE_8, ALU_MUL, target, reg1, reg2, 0));
 694
 695                         gen_insn(INSN_CMP, OP_SIZE_8, 0, 1);
 696                         gen_one(R_SCRATCH_3);
 697                         gen_one(ARG_SHIFTED_REGISTER);
 698                         gen_one(ARG_SHIFT_ASR | 0x3f);
 699                         gen_one(target);
 700
 701                         gen_insn(INSN_JMP_COND, OP_SIZE_8, COND_NE, 0);
 702                         gen_four(label_ovf);
 703
 704                         g(gen_frame_store(ctx, op_size, slot_r, 0, target));
 705
 706                         return true;
 707                 }
 708 #endif
 709 #if defined(ARCH_POWER)
 710                 if (mode == MODE_INT && op_size >= OP_SIZE_4) {
 711                         target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
 712                         g(gen_frame_get(ctx, op_size, garbage, slot_1, R_SCRATCH_1, &reg1));
 713                         g(gen_frame_get(ctx, op_size, garbage, slot_2, R_SCRATCH_2, &reg2));
 714
 715                         g(gen_3address_alu(ctx, op_size, ALU_MUL, target, reg1, reg2, 1));
 716
 717                         gen_insn(INSN_JMP_COND, op_size, COND_O, 0);
 718                         gen_four(label_ovf);
 719
 720                         g(gen_frame_store(ctx, op_size, slot_r, 0, target));
 721
 722                         return true;
 723                 }
 724 #endif
 725 #if defined(ARCH_LOONGARCH64) || (defined(ARCH_MIPS) && MIPS_R6) || defined(ARCH_RISCV64)
 726                 if (mode == MODE_INT && op_size == OP_SIZE_NATIVE) {
 727                         target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
 728                         g(gen_frame_get(ctx, op_size, garbage, slot_1, R_SCRATCH_1, &reg1));
 729                         g(gen_frame_get(ctx, op_size, garbage, slot_2, R_SCRATCH_2, &reg2));
 730
 731                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_SMULH, R_SCRATCH_3, reg1, reg2, 0));
 732
 733                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_MUL, target, reg1, reg2, 0));
 734
 735                         g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_4, target, (8U << OP_SIZE_NATIVE) - 1, 0));
 736
 737                         g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, R_SCRATCH_3, R_SCRATCH_4, COND_NE, label_ovf));
 738
 739                         g(gen_frame_store(ctx, OP_SIZE_NATIVE, slot_r, 0, target));
 740
 741                         return true;
 742                 }
 743 #endif
 744 #if defined(ARCH_S390)
 745                 if (mode == MODE_INT && op_size >= OP_SIZE_4 && likely(cpu_test_feature(CPU_FEATURE_misc_insn_ext_2))) {
 746                         target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
 747                         g(gen_frame_load(ctx, op_size, sign_x, slot_1, 0, false, target));
 748                         g(gen_frame_load_op(ctx, op_size, sign_x, ALU_MUL, 1, slot_2, 0, false, target));
 749
 750                         gen_insn(INSN_JMP_COND, op_size, COND_O, 0);
 751                         gen_four(label_ovf);
 752
 753                         g(gen_frame_store(ctx, op_size, slot_r, 0, target));
 754                         return true;
 755                 }
 756 #endif
 757 #if (defined(ARCH_MIPS) && !MIPS_R6) || defined(ARCH_S390)
 758 #if defined(ARCH_MIPS)
 759                 if (mode == MODE_INT && op_size >= OP_SIZE_4)
 760 #endif
 761 #if defined(ARCH_S390)
 762                 if (mode == MODE_INT && op_size == OP_SIZE_4)
 763 #endif
 764                 {
 765 #if defined(ARCH_S390)
 766                         target = R_SCRATCH_1;
 767 #else
 768                         target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
 769 #endif
 770                         g(gen_frame_get(ctx, op_size, sign_x, slot_1, R_SCRATCH_1, &reg1));
 771                         g(gen_frame_get(ctx, op_size, sign_x, slot_2, R_SCRATCH_3, &reg2));
 772
 773                         gen_insn(INSN_MUL_L, op_size, 0, 0);
 774                         gen_one(target);
 775                         gen_one(R_SCRATCH_2);
 776                         gen_one(reg1);
 777                         gen_one(reg2);
 778
 779                         g(gen_3address_rot_imm(ctx, op_size, ROT_SAR, R_SCRATCH_4, target, (8U << op_size) - 1, false));
 780
 781                         g(gen_cmp_test_jmp(ctx, INSN_CMP, op_size, R_SCRATCH_2, R_SCRATCH_4, COND_NE, label_ovf));
 782
 783                         g(gen_frame_store(ctx, op_size, slot_r, 0, target));
 784                         return true;
 785                 }
 786 #endif
 787                 if (mode == MODE_INT && op_size == OP_SIZE_NATIVE) {
 788                         g(gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, INT_binary_multiply_int8_t), op_size, slot_1, slot_2, slot_r, label_ovf));
 789                         return true;
 790                 }
 791
 792                 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
 793                 if (op_size < OP_SIZE_NATIVE && mode == MODE_INT) {
 794                         g(gen_frame_get(ctx, op_size, sign_x, slot_1, R_SCRATCH_1, &reg1));
 795                         g(gen_frame_get(ctx, op_size, sign_x, slot_2, R_SCRATCH_2, &reg2));
 796
 797                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_MUL, target, reg1, reg2, 0));
 798                 } else {
 799                         g(gen_frame_load(ctx, op_size, sign_x, slot_1, 0, false, target));
 800                         g(gen_frame_load_op(ctx, op_size, sign_x, ALU_MUL, 0, slot_2, 0, false, target));
 801                 }
 802
 803                 if (mode == MODE_INT) {
 804                         g(gen_cmp_extended(ctx, OP_SIZE_NATIVE, op_size, target, R_SCRATCH_2, label_ovf));
 805                 }
 806
 807                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
 808
 809                 return true;
 810         }
 811
 812         /**********
 813          * DIVIDE *
 814          **********/
 815 do_divide: {
 816                 uint32_t attr_unused label_skip = 0;    /* avoid warning */
 817                 uint32_t attr_unused label_skip2 = 0;   /* avoid warning */
 818                 uint32_t attr_unused label_end = 0;     /* avoid warning */
 819                 uint32_t attr_unused label_div_0 = 0;   /* avoid warning */
 820                 unsigned attr_unused divide_alu = 0;    /* avoid warning */
 821                 bool attr_unused have_mod = false;
 822                 bool attr_unused force_sx = false;
 823                 unsigned attr_unused div_op_size = i_size(op_size);
 824                 if (unlikely(op_size > OP_SIZE_NATIVE) || unlikely(!ARCH_HAS_DIV)
 825 #if defined(ARCH_S390)
 826                         || !(Z || (op_size <= OP_SIZE_4 && sgn))
 827 #endif
 828                    ) {
 829                         size_t upcall;
 830                         if (mode == MODE_INT) {
 831                                 upcall = !mod ? offsetof(struct cg_upcall_vector_s, INT_binary_divide_int8_t) : offsetof(struct cg_upcall_vector_s, INT_binary_modulo_int8_t);
 832                         } else if (sgn) {
 833                                 upcall = !mod ? offsetof(struct cg_upcall_vector_s, FIXED_binary_divide_int8_t) : offsetof(struct cg_upcall_vector_s, FIXED_binary_modulo_int8_t);
 834                         } else {
 835                                 upcall = !mod ? offsetof(struct cg_upcall_vector_s, FIXED_binary_udivide_int8_t) : offsetof(struct cg_upcall_vector_s, FIXED_binary_umodulo_int8_t);
 836                         }
 837                         g(gen_alu_typed_upcall(ctx, upcall, op_size, slot_1, slot_2, slot_r, mode == MODE_INT ? label_ovf : 0));
 838                         return true;
 839                 }
 840 #if defined(ARCH_X86) || defined(ARCH_S390)
 841                 if (mode == MODE_FIXED) {
 842                         label_skip = alloc_label(ctx);
 843                         if (unlikely(!label_skip))
 844                                 return false;
 845                         label_end = alloc_label(ctx);
 846                         if (unlikely(!label_end))
 847                                 return false;
 848                         if (sgn) {
 849                                 label_skip2 = alloc_label(ctx);
 850                                 if (unlikely(!label_skip2))
 851                                         return false;
 852                         }
 853                 }
 854 #if defined(ARCH_X86)
 855                 if (R_SCRATCH_1 != R_AX || R_SCRATCH_2 != R_DX || R_SCRATCH_3 != R_CX)
 856                         internal(file_line, "gen_alu: bad scratch registers");
 857 #endif
 858                 g(gen_frame_load(ctx, op_size, sgn ? sign_x : zero_x, slot_1, 0, false, R_SCRATCH_1));
 859                 g(gen_frame_load(ctx, op_size, sgn ? sign_x : zero_x, slot_2, 0, false, R_SCRATCH_3));
 860
 861                 g(gen_jmp_on_zero(ctx, i_size(op_size), R_SCRATCH_3, COND_E, mode == MODE_INT ? label_ovf : label_skip));
 862
 863                 if (sgn) {
 864                         uint64_t val;
 865                         uint32_t label_not_minus_1;
 866                         label_not_minus_1 = alloc_label(ctx);
 867                         if (unlikely(!label_not_minus_1))
 868                                 return false;
 869
 870                         g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, op_size, R_SCRATCH_3, -1, COND_NE, label_not_minus_1));
 871
 872                         val = -(uint64_t)0x80 << (((1 << op_size) - 1) * 8);
 873                         g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, op_size, R_SCRATCH_1, val, COND_E, mode == MODE_INT ? label_ovf : label_skip2));
 874
 875                         gen_label(label_not_minus_1);
 876                 }
 877
 878 #if defined(ARCH_X86)
 879                 if (op_size >= OP_SIZE_2) {
 880                         if (sgn) {
 881                                 gen_insn(INSN_CWD + ARCH_PARTIAL_ALU(op_size), op_size, 0, 0);
 882                                 gen_one(R_SCRATCH_2);
 883                                 gen_one(R_SCRATCH_1);
 884                                 if (op_size == OP_SIZE_2)
 885                                         gen_one(R_SCRATCH_2);
 886                         } else {
 887                                 g(gen_3address_alu(ctx, OP_SIZE_4, ALU_XOR, R_SCRATCH_2, R_SCRATCH_2, R_SCRATCH_2, 0));
 888                         }
 889                 }
 890                 gen_insn(INSN_DIV_L, op_size, sgn, 1);
 891                 gen_one(R_SCRATCH_1);
 892                 gen_one(i_size(op_size) == OP_SIZE_1 ? R_SCRATCH_1 : R_SCRATCH_2);
 893                 gen_one(R_SCRATCH_1);
 894                 gen_one(i_size(op_size) == OP_SIZE_1 ? R_SCRATCH_1 : R_SCRATCH_2);
 895                 gen_one(R_SCRATCH_3);
 896 #else
 897                 if (!sgn) {
 898                         g(gen_load_constant(ctx, R_SCRATCH_2, 0));
 899                 } else if (op_size <= OP_SIZE_4) {
 900                         g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_2, R_SCRATCH_1, (1U << (OP_SIZE_NATIVE + 3)) - 1, false));
 901                 }
 902                 gen_insn(INSN_DIV_L, i_size(op_size), sgn, 1);
 903                 gen_one(R_SCRATCH_2);
 904                 gen_one(R_SCRATCH_1);
 905                 gen_one(R_SCRATCH_2);
 906                 gen_one(R_SCRATCH_1);
 907                 gen_one(R_SCRATCH_3);
 908 #endif
 909                 if (mod && i_size(op_size) == OP_SIZE_1) {
 910                         g(gen_3address_rot_imm(ctx, OP_SIZE_2, ROT_SHR, R_SCRATCH_1, R_SCRATCH_1, 8, 0));
 911                         g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
 912                 } else if (mod) {
 913                         g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_2));
 914                 } else {
 915                         g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
 916                 }
 917                 if (mode == MODE_FIXED) {
 918                         gen_insn(INSN_JMP, 0, 0, 0);
 919                         gen_four(label_end);
 920
 921                         if (sgn) {
 922                                 gen_label(label_skip2);
 923
 924                                 if (mod)
 925                                         g(gen_frame_clear(ctx, op_size, slot_r));
 926                                 else
 927                                         g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
 928
 929                                 gen_insn(INSN_JMP, 0, 0, 0);
 930                                 gen_four(label_end);
 931                         }
 932
 933                         gen_label(label_skip);
 934                         if (!mod)
 935                                 g(gen_frame_clear(ctx, op_size, slot_r));
 936                         else
 937                                 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
 938                         gen_label(label_end);
 939                 }
 940                 return true;
 941 #else
 942 #if defined(ARCH_MIPS)
 943                 have_mod = true;
 944                 div_op_size = maximum(op_size, OP_SIZE_4);
 945                 if (op_size == OP_SIZE_4)
 946                         force_sx = true;
 947 #endif
 948 #if defined(ARCH_POWER)
 949                 have_mod = cpu_test_feature(CPU_FEATURE_v30);
 950                 div_op_size = maximum(op_size, OP_SIZE_4);
 951 #endif
 952 #if defined(ARCH_LOONGARCH64) || defined(ARCH_RISCV64)
 953                 have_mod = true;
 954                 div_op_size = maximum(op_size, OP_SIZE_4);
 955 #endif
 956                 label_end = alloc_label(ctx);
 957                 if (unlikely(!label_end))
 958                         return false;
 959
 960                 g(gen_frame_get(ctx, op_size, (sgn && op_size < i_size(op_size)) || force_sx ? sign_x : zero_x, slot_1, R_SCRATCH_1, &reg1));
 961                 g(gen_frame_get(ctx, op_size, (sgn && op_size < i_size(op_size)) || force_sx ? sign_x : zero_x, slot_2, R_SCRATCH_2, &reg2));
 962                 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_3);
 963
 964                 if (ARCH_PREFERS_SX(op_size) && !sgn && op_size < i_size(op_size)) {
 965                         g(gen_extend(ctx, op_size, zero_x, R_SCRATCH_1, reg1));
 966                         reg1 = R_SCRATCH_1;
 967                         g(gen_extend(ctx, op_size, zero_x, R_SCRATCH_2, reg2));
 968                         reg2 = R_SCRATCH_2;
 969                 }
 970
 971                 if (mode == MODE_INT) {
 972                         g(gen_jmp_on_zero(ctx, i_size(op_size), reg2, COND_E, label_ovf));
 973                         if (sgn) {
 974                                 uint64_t val;
 975                                 uint32_t label_not_minus_1;
 976                                 label_not_minus_1 = alloc_label(ctx);
 977                                 if (unlikely(!label_not_minus_1))
 978                                         return false;
 979
 980                                 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, i_size(op_size), reg2, -1, COND_NE, label_not_minus_1));
 981
 982                                 val = 0xFFFFFFFFFFFFFF80ULL << (((1 << op_size) - 1) * 8);
 983                                 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, i_size(op_size), reg1, val, COND_E, label_ovf));
 984
 985                                 gen_label(label_not_minus_1);
 986                         }
 987                 } else {
 988 #if !(defined(ARCH_ARM) && ARM_ASM_DIV_NO_TRAP)
 989                         if (!mod) {
 990                                 g(gen_load_constant(ctx, target, 0));
 991                         } else {
 992                                 g(gen_mov(ctx, OP_SIZE_NATIVE, target, reg1));
 993                         }
 994                         g(gen_jmp_on_zero(ctx, i_size(op_size), reg2, COND_E, label_end));
 995                         if (sgn) {
 996                                 uint64_t val;
 997                                 uint32_t label_not_minus_1;
 998                                 label_not_minus_1 = alloc_label(ctx);
 999                                 if (unlikely(!label_not_minus_1))
1000                                         return false;
1001
1002                                 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, i_size(op_size), reg2, -1, COND_NE, label_not_minus_1));
1003
1004                                 if (!mod) {
1005                                         g(gen_mov(ctx, OP_SIZE_NATIVE, target, reg1));
1006                                 } else {
1007                                         g(gen_load_constant(ctx, target, 0));
1008                                 }
1009
1010                                 val = 0xFFFFFFFFFFFFFF80ULL << (((1 << op_size) - 1) * 8);
1011                                 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, i_size(op_size), reg1, val, COND_E, label_end));
1012
1013                                 gen_label(label_not_minus_1);
1014                         }
1015 #endif
1016                 }
1017                 if (mod && have_mod) {
1018                         g(gen_3address_alu(ctx, div_op_size, sgn ? ALU_SREM : ALU_UREM, target, reg1, reg2, 0));
1019                 } else {
1020                         g(gen_3address_alu(ctx, div_op_size, sgn ? ALU_SDIV : ALU_UDIV, target, reg1, reg2, 0));
1021                 }
1022
1023                 if (mod && !have_mod) {
1024 #if defined(ARCH_ARM)
1025                         gen_insn(INSN_MADD, i_size(op_size), 1, 0);
1026                         gen_one(target);
1027                         gen_one(target);
1028                         gen_one(reg2);
1029                         gen_one(reg1);
1030 #else
1031                         g(gen_3address_alu(ctx, i_size(op_size), ALU_MUL, R_SCRATCH_2, reg2, target, 0));
1032                         g(gen_3address_alu(ctx, i_size(op_size), ALU_SUB, target, reg1, R_SCRATCH_2, 0));
1033 #endif
1034                 }
1035
1036                 gen_label(label_end);
1037                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1038                 return true;
1039 #endif
1040         }
1041         /*********
1042          * SHIFT *
1043          *********/
1044 do_shift: {
1045                 bool sx;
1046                 bool must_mask;
1047                 unsigned op_s;
1048                 int64_t cnst = 0;       /* avoid warning */
1049                 bool c = frame_t_is_const(slot_2);
1050                 if (unlikely(op_size > OP_SIZE_NATIVE)) {
1051                         size_t upcall;
1052                         if (mode == MODE_FIXED) {
1053                                 switch (alu) {
1054                                         case ROT_SHL:   upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_shl_,TYPE_INT_MAX));
1055                                                         break;
1056                                         case ROT_SAR:   upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_shr_,TYPE_INT_MAX));
1057                                                         break;
1058                                         case ROT_SHR:   upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_ushr_,TYPE_INT_MAX));
1059                                                         break;
1060                                         case ROT_ROL:   upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_rol_,TYPE_INT_MAX));
1061                                                         break;
1062                                         case ROT_ROR:   upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_ror_,TYPE_INT_MAX));
1063                                                         break;
1064                                         default:        internal(file_line, "do_alu: invalid shift %u", alu);
1065                                 }
1066                         } else {
1067                                 switch (alu) {
1068                                         case ROT_SHL:   upcall = offsetof(struct cg_upcall_vector_s, cat(INT_binary_shl_,TYPE_INT_MAX));
1069                                                         break;
1070                                         case ROT_SAR:   upcall = offsetof(struct cg_upcall_vector_s, cat(INT_binary_shr_,TYPE_INT_MAX));
1071                                                         break;
1072                                         default:        internal(file_line, "do_alu: invalid shift %u", alu);
1073                                 }
1074                         }
1075                         g(gen_alu_upcall(ctx, upcall, op_size, slot_1, slot_2, slot_r, mode == MODE_INT ? label_ovf : 0));
1076                         return true;
1077                 }
1078                 op_s = i_size_rot(op_size);
1079 #if defined(ARCH_X86)
1080                 if (slot_1 == slot_r && !slot_is_register(ctx, slot_1) && !(mode == MODE_INT && alu == ROT_SHL)) {
1081                         int64_t offset = (size_t)slot_1 * slot_size;
1082                         if (c) {
1083                                 cnst = frame_t_get_const(slot_2);
1084                                 if (mode == MODE_INT) {
1085                                         if ((uint64_t)cnst > (8U << op_size) - 1) {
1086                                                 gen_insn(INSN_JMP, 0, 0, 0);
1087                                                 gen_four(label_ovf);
1088                                                 return true;
1089                                         }
1090                                 } else {
1091                                         cnst &= (8U << op_size) - 1;
1092                                 }
1093                                 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDR_OFFSET, op_size));
1094                                 gen_insn(INSN_ROT + ARCH_PARTIAL_ALU(op_size), op_size, alu, 1);
1095                                 gen_address_offset();
1096                                 gen_address_offset();
1097                                 gen_one(ARG_IMM);
1098                                 gen_eight(cnst);
1099                         } else {
1100                                 g(gen_frame_load(ctx, op_size, garbage, slot_2, 0, false, R_SCRATCH_3));
1101                                 if (mode == MODE_INT) {
1102                                         int64_t imm = (8U << op_size) - 1;
1103                                         g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, op_size, R_SCRATCH_3, imm, COND_A, label_ovf));
1104                                 } else if ((alu != ROT_ROL && alu != ROT_ROR) && op_size < OP_SIZE_4) {
1105                                         g(gen_3address_alu_imm(ctx, OP_SIZE_1, ALU_AND, R_SCRATCH_3, R_SCRATCH_3, (8U << op_size) - 1, 0));
1106                                 }
1107                                 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDR_OFFSET, op_size));
1108                                 gen_insn(INSN_ROT + ARCH_PARTIAL_ALU(op_size), op_size, alu, 1);
1109                                 gen_address_offset();
1110                                 gen_address_offset();
1111                                 gen_one(R_SCRATCH_3);
1112                         }
1113                         return true;
1114                 }
1115                 if (mode == MODE_INT && alu == ROT_SHL && op_size < OP_SIZE_NATIVE)
1116                         op_s = op_size + 1;
1117 #endif
1118                 must_mask = op_size < ARCH_SHIFT_SIZE;
1119                 sx = (alu == ROT_SAR && op_size < op_s) || (alu == ROT_SHL && op_size < OP_SIZE_NATIVE && mode == MODE_INT);
1120 #if defined(ARCH_MIPS)
1121                 sx |= op_size == OP_SIZE_4;
1122 #endif
1123                 g(gen_frame_get(ctx, op_size, sx ? sign_x : zero_x, slot_1, R_SCRATCH_1, &reg1));
1124                 if (c) {
1125                         reg3 = 0xff;    /* avoid warning */
1126                         cnst = frame_t_get_const(slot_2);
1127                 } else
1128 #if defined(ARCH_X86)
1129                 if (!ARCH_IS_3ADDRESS_ROT(alu, op_size)) {
1130                         g(gen_frame_load(ctx, op_size, garbage, slot_2, 0, false, R_SCRATCH_3));
1131                         reg3 = R_SCRATCH_3;
1132                 } else
1133 #endif
1134                 g(gen_frame_get(ctx, op_size, garbage, slot_2, R_SCRATCH_3, &reg3));
1135
1136                 if (ARCH_PREFERS_SX(op_size) && !sx && op_size < op_s) {
1137                         g(gen_extend(ctx, op_size, zero_x, R_SCRATCH_1, reg1));
1138                         reg1 = R_SCRATCH_1;
1139                 }
1140
1141                 if (mode == MODE_INT) {
1142                         int64_t imm = (8U << op_size) - 1;
1143                         if (c) {
1144                                 if ((uint64_t)cnst > (uint64_t)imm) {
1145                                         gen_insn(INSN_JMP, 0, 0, 0);
1146                                         gen_four(label_ovf);
1147                                         return true;
1148                                 }
1149                         } else {
1150                                 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, i_size_cmp(op_size), reg3, imm, COND_A, label_ovf));
1151                         }
1152                 } else {
1153 #if defined(ARCH_ARM)
1154                         if (alu == ROT_ROL) {
1155                                 if (c) {
1156                                         cnst = -(uint64_t)cnst;
1157                                 } else {
1158                                         g(gen_2address_alu1(ctx, OP_SIZE_4, ALU1_NEG, R_SCRATCH_3, reg3, 0));
1159                                         reg3 = R_SCRATCH_3;
1160                                 }
1161                                 alu = ROT_ROR;
1162                         }
1163 #endif
1164 #if defined(ARCH_LOONGARCH64)
1165                         if (alu == ROT_ROL && op_size >= OP_SIZE_4) {
1166                                 if (c) {
1167                                         cnst = -(uint64_t)cnst;
1168                                 } else {
1169                                         g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NEG, R_SCRATCH_3, reg3, 0));
1170                                         reg3 = R_SCRATCH_3;
1171                                 }
1172                                 alu = ROT_ROR;
1173                         }
1174 #endif
1175 #if defined(ARCH_MIPS)
1176                         if (MIPS_HAS_ROT && alu == ROT_ROL && op_size >= OP_SIZE_4) {
1177                                 if (c) {
1178                                         cnst = -(uint64_t)cnst;
1179                                 } else {
1180                                         g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NEG, R_SCRATCH_3, reg3, 0));
1181                                         reg3 = R_SCRATCH_3;
1182                                 }
1183                                 alu = ROT_ROR;
1184                         }
1185 #endif
1186 #if defined(ARCH_POWER)
1187                         if (alu == ROT_ROR && op_size >= OP_SIZE_4) {
1188                                 if (c) {
1189                                         cnst = -(uint64_t)cnst;
1190                                 } else {
1191                                         g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NEG, R_SCRATCH_3, reg3, 0));
1192                                         reg3 = R_SCRATCH_3;
1193                                 }
1194                                 alu = ROT_ROL;
1195                         }
1196 #endif
1197 #if defined(ARCH_S390)
1198                         if (Z && alu == ROT_ROR && op_size >= OP_SIZE_4) {
1199                                 if (c) {
1200                                         cnst = -(uint64_t)cnst;
1201                                 } else {
1202                                         g(gen_2address_alu1(ctx, OP_SIZE_4, ALU1_NEG, R_SCRATCH_3, reg3, 0));
1203                                         reg3 = R_SCRATCH_3;
1204                                 }
1205                                 alu = ROT_ROL;
1206                         }
1207 #endif
1208                         if (c) {
1209                                 cnst &= (8U << op_size) - 1;
1210                         } else if (must_mask) {
1211                                 g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_4), ALU_AND, R_SCRATCH_3, reg3, (8U << op_size) - 1, 0));
1212                                 reg3 = R_SCRATCH_3;
1213                         }
1214                 }
1215
1216 #if defined(ARCH_X86)
1217                 if (mode == MODE_INT && alu == ROT_SHL) {
1218                         target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_2);
1219                 } else {
1220                         target = gen_frame_target(ctx, slot_r, NO_FRAME_T, slot_2, R_SCRATCH_2);
1221                 }
1222                 if (c) {
1223                         g(gen_3address_rot_imm(ctx, op_s, alu, target, reg1, cnst, 0));
1224                 } else  {
1225                         g(gen_3address_rot(ctx, op_s, alu, target, reg1, reg3));
1226                 }
1227
1228                 if (mode == MODE_INT && alu == ROT_SHL) {
1229                         if (op_size < OP_SIZE_NATIVE) {
1230                                 gen_insn(INSN_MOVSX, op_size, 0, 0);
1231                                 gen_one(R_SCRATCH_4);
1232                                 gen_one(target);
1233
1234                                 g(gen_cmp_test_jmp(ctx, INSN_CMP, op_s, target, R_SCRATCH_4, COND_NE, label_ovf));
1235                         } else {
1236                                 if (c) {
1237                                         g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_4, target, cnst, 0));
1238                                 } else {
1239                                         g(gen_3address_rot(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_4, target, reg3));
1240                                 }
1241
1242                                 g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, reg1, R_SCRATCH_4, COND_NE, label_ovf));
1243                         }
1244                 }
1245                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1246                 return true;
1247 #endif
1248 #if defined(ARCH_ARM)
1249                 if (op_size <= OP_SIZE_2 && alu == ROT_ROR) {
1250                         gen_insn(INSN_ALU, OP_SIZE_4, ALU_OR, ALU_WRITES_FLAGS(OP_SIZE_4, ALU_OR, false, false, 0));
1251                         gen_one(R_SCRATCH_1);
1252                         gen_one(reg1);
1253                         gen_one(ARG_SHIFTED_REGISTER);
1254                         gen_one(ARG_SHIFT_LSL | (8U << op_size));
1255                         gen_one(reg1);
1256                         if (op_size == OP_SIZE_1)
1257                                 alu = ROT_SHR;
1258                         reg1 = R_SCRATCH_1;
1259                 }
1260                 goto do_generic_shift;
1261 #endif
1262 #if defined(ARCH_LOONGARCH64)
1263                 if (alu == ROT_ROR && op_size >= OP_SIZE_4)
1264                         goto do_generic_shift;
1265 #endif
1266 #if defined(ARCH_MIPS)
1267                 if (MIPS_HAS_ROT && alu == ROT_ROR && op_size >= OP_SIZE_4)
1268                         goto do_generic_shift;
1269 #endif
1270 #if defined(ARCH_POWER)
1271                 if (alu == ROT_ROL && op_size >= OP_SIZE_4)
1272                         goto do_generic_shift;
1273 #endif
1274 #if defined(ARCH_RISCV64)
1275                 if ((alu == ROT_ROL || alu == ROT_ROR) && likely(cpu_test_feature(CPU_FEATURE_zbb))) {
1276                         if (likely(op_size >= OP_SIZE_4)) {
1277                                 goto do_generic_shift;
1278                         }
1279                 }
1280 #endif
1281 #if defined(ARCH_S390)
1282                 if (Z && alu == ROT_ROL && op_size >= OP_SIZE_4)
1283                         goto do_generic_shift;
1284 #endif
1285                 if (alu == ROT_ROL || alu == ROT_ROR) {
1286                         target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
1287                         if (c) {
1288                                 g(gen_3address_rot_imm(ctx, op_s, alu == ROT_ROL ? ROT_SHL : ROT_SHR, R_SCRATCH_2, reg1, cnst, 0));
1289                                 g(gen_3address_rot_imm(ctx, op_s, alu == ROT_ROL ? ROT_SHR : ROT_SHL, target, reg1, -(uint64_t)cnst & ((8U << op_size) - 1), 0));
1290                         } else {
1291                                 g(gen_3address_rot(ctx, op_s, alu == ROT_ROL ? ROT_SHL : ROT_SHR, R_SCRATCH_2, reg1, reg3));
1292                                 g(gen_2address_alu1(ctx, i_size(OP_SIZE_4), ALU1_NEG, R_SCRATCH_3, reg3, 0));
1293                                 if (must_mask) {
1294                                         g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_4), ALU_AND, R_SCRATCH_3, R_SCRATCH_3, (8U << op_size) - 1, 0));
1295                                 }
1296                                 g(gen_3address_rot(ctx, op_s, alu == ROT_ROL ? ROT_SHR : ROT_SHL, target, reg1, R_SCRATCH_3));
1297                         }
1298                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_OR, target, target, R_SCRATCH_2, 0));
1299                         g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1300                         return true;
1301                 }
1302
1303                 goto do_generic_shift;
1304 do_generic_shift:
1305                 if (mode == MODE_INT && alu == ROT_SHL) {
1306                         target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
1307 #if defined(ARCH_S390)
1308                         if (op_size >= OP_SIZE_4) {
1309                                 if (c)
1310                                         g(gen_3address_rot_imm(ctx, op_size, ROT_SAL, target, reg1, cnst, 0));
1311                                 else
1312                                         g(gen_3address_rot(ctx, op_size, ROT_SAL, target, reg1, reg3));
1313
1314                                 gen_insn(INSN_JMP_COND, op_size, COND_O, 0);
1315                                 gen_four(label_ovf);
1316                         } else
1317 #endif
1318                         if (op_size <= OP_SIZE_NATIVE - 1) {
1319                                 if (c)
1320                                         g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, alu, target, reg1, cnst, 0));
1321                                 else
1322                                         g(gen_3address_rot(ctx, OP_SIZE_NATIVE, alu, target, reg1, reg3));
1323
1324                                 g(gen_cmp_extended(ctx, OP_SIZE_NATIVE, op_size, target, R_SCRATCH_2, label_ovf));
1325                         } else {
1326                                 if (c) {
1327                                         g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, alu, R_SCRATCH_2, reg1, cnst, 0));
1328                                         g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_4, R_SCRATCH_2, cnst, 0));
1329                                 } else {
1330                                         g(gen_3address_rot(ctx, OP_SIZE_NATIVE, alu, R_SCRATCH_2, reg1, reg3));
1331                                         g(gen_3address_rot(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_4, R_SCRATCH_2, reg3));
1332                                 }
1333
1334                                 g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, reg1, R_SCRATCH_4, COND_NE, label_ovf));
1335
1336                                 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_2));
1337
1338                                 return true;
1339                         }
1340                 } else {
1341                         target = gen_frame_target(ctx, slot_r, NO_FRAME_T, slot_2, R_SCRATCH_1);
1342                         if (c)
1343                                 g(gen_3address_rot_imm(ctx, op_s, alu, target, reg1, cnst, 0));
1344                         else
1345                                 g(gen_3address_rot(ctx, op_s, alu, target, reg1, reg3));
1346                 }
1347
1348                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1349                 return true;
1350         }
1351         /******
1352          * BT *
1353          ******/
1354 do_bt: {
1355                 unsigned attr_unused op_s;
1356                 bool need_mask;
1357                 bool c = frame_t_is_const(slot_2);
1358                 int64_t cnst = !c ? 0 : frame_t_get_const(slot_2);
1359                 int64_t max_imm = (8U << op_size) - 1;
1360                 if (c) {
1361                         if (mode == MODE_INT) {
1362                                 if (alu == BTX_BT ? (uint64_t)cnst > (uint64_t)max_imm : (uint64_t)cnst >= (uint64_t)max_imm) {
1363                                         gen_insn(INSN_JMP, 0, 0, 0);
1364                                         gen_four(label_ovf);
1365                                         return true;
1366                                 }
1367                         }
1368                         cnst &= max_imm;
1369                 }
1370 #if defined(ARCH_X86)
1371                 if ((alu == BTX_BT || slot_1 == slot_r) && !slot_is_register(ctx, slot_1)) {
1372                         int64_t offset;
1373                         unsigned n_op_size = minimum(op_size, OP_SIZE_NATIVE);
1374                         g(gen_frame_get(ctx, n_op_size, garbage, slot_2, R_SCRATCH_1, &reg2));
1375                         if (mode == MODE_INT) {
1376                                 if (!c) {
1377                                         g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, n_op_size, reg2, max_imm, alu == BTX_BT ? COND_A : COND_AE, label_ovf));
1378                                         if (unlikely(op_size > OP_SIZE_NATIVE)) {
1379                                                 g(gen_address(ctx, R_FRAME, (size_t)slot_2 * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_STR_OFFSET, OP_SIZE_NATIVE));
1380                                                 gen_insn(INSN_CMP, OP_SIZE_NATIVE, 0, 1);
1381                                                 gen_address_offset();
1382                                                 gen_one(ARG_IMM);
1383                                                 gen_eight(0);
1384                                                 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_NE, 0);
1385                                                 gen_four(label_ovf);
1386                                         }
1387                                 }
1388                         } else {
1389                                 if (!c) {
1390                                         g(gen_3address_alu_imm(ctx, OP_SIZE_4, ALU_AND, R_SCRATCH_1, reg2, (8U << op_size) - 1, 0));
1391                                         reg2 = R_SCRATCH_1;
1392                                 }
1393                         }
1394                         offset = (size_t)slot_1 * slot_size;
1395                         if (c && cnst >= 8U << OP_SIZE_NATIVE) {
1396                                 offset += 1U << OP_SIZE_NATIVE;
1397                                 cnst -= 8U << OP_SIZE_NATIVE;
1398                         }
1399                         g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_STR_OFFSET, maximum(n_op_size, OP_SIZE_2)));
1400                         if (alu == BTX_BT) {
1401                                 gen_insn(INSN_BT, maximum(n_op_size, OP_SIZE_2), 0, 1);
1402                                 gen_address_offset();
1403                                 if (c) {
1404                                         gen_one(ARG_IMM);
1405                                         gen_eight(cnst);
1406                                 } else {
1407                                         gen_one(reg2);
1408                                 }
1409                                 g(gen_frame_set_cond(ctx, maximum(n_op_size, OP_SIZE_2), false, COND_B, slot_r));
1410                         } else {
1411                                 gen_insn(INSN_BTX, maximum(n_op_size, OP_SIZE_2), alu, 1);
1412                                 gen_address_offset();
1413                                 gen_address_offset();
1414                                 if (c) {
1415                                         gen_one(ARG_IMM);
1416                                         gen_eight(cnst);
1417                                 } else {
1418                                         gen_one(reg2);
1419                                 }
1420                         }
1421                         return true;
1422                 }
1423 #endif
1424                 if (unlikely(op_size > OP_SIZE_NATIVE)) {
1425                         size_t upcall;
1426                         if (mode == MODE_FIXED) {
1427                                 switch (alu) {
1428                                         case BTX_BTS:   upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_bts_,TYPE_INT_MAX));
1429                                                         break;
1430                                         case BTX_BTR:   upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_btr_,TYPE_INT_MAX));
1431                                                         break;
1432                                         case BTX_BTC:   upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_btc_,TYPE_INT_MAX));
1433                                                         break;
1434                                         case BTX_BT:    upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_bt_,TYPE_INT_MAX));
1435                                                         break;
1436                                         default:        internal(file_line, "do_alu: invalid bit test %u", alu);
1437                                 }
1438                         } else {
1439                                 switch (alu) {
1440                                         case BTX_BTS:   upcall = offsetof(struct cg_upcall_vector_s, cat(INT_binary_bts_,TYPE_INT_MAX));
1441                                                         break;
1442                                         case BTX_BTR:   upcall = offsetof(struct cg_upcall_vector_s, cat(INT_binary_btr_,TYPE_INT_MAX));
1443                                                         break;
1444                                         case BTX_BTC:   upcall = offsetof(struct cg_upcall_vector_s, cat(INT_binary_btc_,TYPE_INT_MAX));
1445                                                         break;
1446                                         case BTX_BT:    upcall = offsetof(struct cg_upcall_vector_s, cat(INT_binary_bt_,TYPE_INT_MAX));
1447                                                         break;
1448                                         default:        internal(file_line, "do_alu: invalid bit test %u", alu);
1449                                 }
1450                         }
1451                         g(gen_alu_upcall(ctx, upcall, op_size, slot_1, slot_2, slot_r, label_ovf));
1452                         return true;
1453                 }
1454                 op_s = minimum(OP_SIZE_NATIVE, ARCH_SHIFT_SIZE);
1455                 op_s = maximum(op_s, op_size);
1456                 g(gen_frame_get(ctx, op_size, zero_x, slot_1, R_SCRATCH_1, &reg1));
1457                 if (c) {
1458                         target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1459                         if (ARCH_HAS_BTX(alu == BTX_BT ? BTX_BTEXT : alu, OP_SIZE_NATIVE, true)) {
1460 #ifdef ARCH_X86
1461                                 g(gen_mov(ctx, OP_SIZE_NATIVE, target, reg1));
1462                                 reg1 = target;
1463                                 gen_insn(INSN_BTX, OP_SIZE_NATIVE, alu == BTX_BT ? BTX_BTEXT : alu, 1);
1464 #else
1465                                 gen_insn(INSN_BTX, OP_SIZE_NATIVE, alu == BTX_BT ? BTX_BTEXT : alu, 0);
1466 #endif
1467                                 gen_one(target);
1468                                 gen_one(reg1);
1469                                 gen_one(ARG_IMM);
1470                                 gen_eight(cnst);
1471                         } else switch (alu) {
1472                                 case BTX_BTS:
1473                                         g(gen_3address_alu_imm(ctx, i_size(op_size), ALU_OR, target, reg1, 1ULL << cnst, 0));
1474                                         break;
1475                                 case BTX_BTR:
1476                                         g(gen_3address_alu_imm(ctx, i_size(op_size), ALU_AND, target, reg1, ~(1ULL << cnst), 0));
1477                                         break;
1478                                 case BTX_BTC:
1479                                         g(gen_3address_alu_imm(ctx, i_size(op_size), ALU_XOR, target, reg1, 1ULL << cnst, 0));
1480                                         break;
1481                                 case BTX_BT:
1482                                         if (cnst) {
1483                                                 g(gen_3address_rot_imm(ctx, i_size(op_size), ROT_SHR, target, reg1, cnst, 0));
1484                                         }
1485                                         g(gen_3address_alu_imm(ctx, i_size(op_size), ALU_AND, target, target, 1, 0));
1486                                         break;
1487                                 default:
1488                                         internal(file_line, "do_alu: invalid bit test %u", alu);
1489                         }
1490                         if (alu == BTX_BT)
1491                                 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
1492                         else
1493                                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1494                         return true;
1495                 }
1496                 g(gen_frame_get(ctx, op_size, garbage, slot_2, R_SCRATCH_2, &reg2));
1497                 if (mode == MODE_INT) {
1498                         if (!c)
1499                                 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, i_size_cmp(op_size), reg2, max_imm, alu == BTX_BT ? COND_A : COND_AE, label_ovf));
1500                 }
1501                 if (alu != BTX_BT) {
1502                         if (!ARCH_HAS_BTX(alu, OP_SIZE_NATIVE, false))
1503                                 goto do_generic_bt;
1504                         need_mask = !ARCH_HAS_BTX(alu, op_size, false);
1505                 } else {
1506 #if defined(ARCH_X86)
1507                         need_mask = op_size < OP_SIZE_2;
1508 #else
1509                         if (!ARCH_HAS_BTX(BTX_BTEXT, OP_SIZE_NATIVE, false))
1510                                 goto do_generic_bt;
1511                         need_mask = !ARCH_HAS_BTX(BTX_BTEXT, op_size, false);
1512 #endif
1513                 }
1514                 if (need_mask) {
1515                         g(gen_3address_alu_imm(ctx, OP_SIZE_4, ALU_AND, R_SCRATCH_2, reg2, (8U << op_size) - 1, 0));
1516                         reg2 = R_SCRATCH_2;
1517                 }
1518                 if (alu == BTX_BT) {
1519 #if defined(ARCH_X86)
1520                         gen_insn(INSN_BT, maximum(op_size, OP_SIZE_2), 0, 1);
1521                         gen_one(reg1);
1522                         gen_one(reg2);
1523
1524                         g(gen_frame_set_cond(ctx, maximum(op_size, OP_SIZE_2), false, COND_B, slot_r));
1525 #else
1526                         target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1527                         gen_insn(INSN_BTX, need_mask ? OP_SIZE_NATIVE : op_size, BTX_BTEXT, 0);
1528                         gen_one(target);
1529                         gen_one(reg1);
1530                         gen_one(reg2);
1531
1532                         g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
1533 #endif
1534                 } else {
1535                         target = gen_frame_target(ctx, slot_r, NO_FRAME_T, slot_2, R_SCRATCH_1);
1536 #if defined(ARCH_X86)
1537                         if (target == reg2)
1538                                 target = R_SCRATCH_1;
1539                         if (target != reg1) {
1540                                 g(gen_mov(ctx, op_size, target, reg1));
1541                                 reg1 = target;
1542                         }
1543                         gen_insn(INSN_BTX, maximum(op_size, OP_SIZE_2), alu, 1);
1544 #else
1545                         gen_insn(INSN_BTX, need_mask ? OP_SIZE_NATIVE : op_size, alu, 0);
1546 #endif
1547                         gen_one(target);
1548                         gen_one(reg1);
1549                         gen_one(reg2);
1550
1551                         g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1552                 }
1553                 return true;
1554
1555                 goto do_generic_bt;
1556 do_generic_bt:
1557                 if (mode == MODE_FIXED && op_size < ARCH_SHIFT_SIZE) {
1558                         g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_4), ALU_AND, R_SCRATCH_2, reg2, (8U << op_size) - 1, 0));
1559                         reg2 = R_SCRATCH_2;
1560                 }
1561                 g(gen_load_constant(ctx, R_SCRATCH_3, 1));
1562
1563                 g(gen_3address_rot(ctx, op_s, ROT_SHL, R_SCRATCH_3, R_SCRATCH_3, reg2));
1564
1565                 switch (alu) {
1566                         case BTX_BT:
1567 #if ARCH_HAS_FLAGS
1568 #if defined(ARCH_S390) || defined(ARCH_POWER)
1569                                 g(gen_3address_alu(ctx, i_size(op_size), ALU_AND, R_SCRATCH_1, reg1, R_SCRATCH_3, 1));
1570 #else
1571                                 gen_insn(INSN_TEST, i_size(op_size), 0, 1);
1572                                 gen_one(reg1);
1573                                 gen_one(R_SCRATCH_3);
1574 #endif
1575                                 g(gen_frame_set_cond(ctx, i_size_cmp(op_size), false, COND_NE, slot_r));
1576 #else
1577                                 g(gen_3address_alu(ctx, i_size(op_size), ALU_AND, R_SCRATCH_1, reg1, R_SCRATCH_3, 0));
1578                                 g(gen_frame_cmp_imm_set_cond_reg(ctx, i_size(op_size), R_SCRATCH_1, 0, COND_NE, slot_r));
1579 #endif
1580                                 return true;
1581                         case BTX_BTS:
1582                                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1583                                 g(gen_3address_alu(ctx, i_size(op_size), ALU_OR, target, reg1, R_SCRATCH_3, 0));
1584                                 break;
1585                         case BTX_BTR:
1586                                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1587                                 if (!ARCH_HAS_ANDN) {
1588                                         g(gen_3address_alu_imm(ctx, i_size(op_size), ALU_XOR, R_SCRATCH_3, R_SCRATCH_3, -1, 0));
1589
1590                                         g(gen_3address_alu(ctx, i_size(op_size), ALU_AND, target, reg1, R_SCRATCH_3, 0));
1591                                         break;
1592                                 }
1593                                 g(gen_3address_alu(ctx, i_size(op_size), ALU_ANDN, target, reg1, R_SCRATCH_3, 0));
1594                                 break;
1595                         case BTX_BTC:
1596                                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1597                                 g(gen_3address_alu(ctx, i_size(op_size), ALU_XOR, target, reg1, R_SCRATCH_3, 0));
1598                                 break;
1599                         default:
1600                                 internal(file_line, "gen_alu: unsupported bit test %u", alu);
1601                 }
1602
1603                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1604
1605                 return true;
1606         }
1607         /***********
1608          * COMPARE *
1609          ***********/
1610 do_compare: {
1611                 if (unlikely(op_size > OP_SIZE_NATIVE)) {
1612                         size_t attr_unused upcall;
1613                         frame_t attr_unused swap;
1614                         switch (alu) {
1615                                 case COND_E:
1616                                 case COND_NE:
1617                                         g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
1618                                         g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, ALU_XOR, 0, slot_2, lo_word(OP_SIZE_NATIVE), true, R_SCRATCH_1));
1619                                         g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, ALU_XOR, 0, slot_2, hi_word(OP_SIZE_NATIVE), true, R_SCRATCH_2));
1620 #if defined(ARCH_ARM64)
1621                                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_OR, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_2, 0));
1622
1623                                         gen_insn(INSN_CMP, OP_SIZE_NATIVE, 0, 1);
1624                                         gen_one(R_SCRATCH_1);
1625                                         gen_one(ARG_IMM);
1626                                         gen_eight(0);
1627 #else
1628                                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_OR, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_2, ARCH_HAS_FLAGS));
1629 #endif
1630 #if ARCH_HAS_FLAGS
1631                                         g(gen_frame_set_cond(ctx, OP_SIZE_NATIVE, false, alu, slot_r));
1632 #else
1633                                         g(gen_frame_cmp_imm_set_cond_reg(ctx, OP_SIZE_NATIVE, R_SCRATCH_1, 0, alu, slot_r));
1634 #endif
1635                                         return true;
1636 #if defined(ARCH_X86_64) || defined(ARCH_X86_X32) || defined(ARCH_ARM)
1637                                 case COND_G:
1638                                 case COND_A:
1639                                         swap = slot_1; slot_1 = slot_2; slot_2 = swap;
1640                                         alu = alu == COND_G ? COND_L : COND_B;
1641                                         /*-fallthrough*/
1642                                 case COND_L:
1643                                 case COND_B:
1644                                         g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_2, lo_word(OP_SIZE_NATIVE), true, R_SCRATCH_2));
1645                                         g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_1, hi_word(OP_SIZE_NATIVE), true, R_SCRATCH_1));
1646                                         g(gen_frame_load_cmp(ctx, OP_SIZE_NATIVE, false, garbage, true, slot_1, lo_word(OP_SIZE_NATIVE), true, R_SCRATCH_2));
1647                                         g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, ALU_SBB, 1, slot_2, hi_word(OP_SIZE_NATIVE), true, R_SCRATCH_1));
1648                                         g(gen_frame_set_cond(ctx, OP_SIZE_NATIVE, false, alu, slot_r));
1649                                         return true;
1650                                 case COND_GE:
1651                                 case COND_AE:
1652                                         swap = slot_1; slot_1 = slot_2; slot_2 = swap;
1653                                         alu = alu == COND_GE ? COND_LE : COND_BE;
1654                                         /*-fallthrough*/
1655                                 case COND_LE:
1656                                 case COND_BE:
1657                                         g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_1, lo_word(OP_SIZE_NATIVE), true, R_SCRATCH_2));
1658                                         g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_2, hi_word(OP_SIZE_NATIVE), true, R_SCRATCH_1));
1659                                         g(gen_frame_load_cmp(ctx, OP_SIZE_NATIVE, false, garbage, true, slot_2, lo_word(OP_SIZE_NATIVE), true, R_SCRATCH_2));
1660                                         g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, ALU_SBB, 1, slot_1, hi_word(OP_SIZE_NATIVE), true, R_SCRATCH_1));
1661                                         g(gen_frame_set_cond(ctx, OP_SIZE_NATIVE, false, alu == COND_LE ? COND_GE : COND_AE, slot_r));
1662                                         return true;
1663 #else
1664                                 case COND_L:    upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_less_,TYPE_INT_MAX)); goto do_upcall;
1665                                 case COND_LE:   upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_less_equal_,TYPE_INT_MAX)); goto do_upcall;
1666                                 case COND_G:    upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_greater_,TYPE_INT_MAX)); goto do_upcall;
1667                                 case COND_GE:   upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_greater_equal_,TYPE_INT_MAX)); goto do_upcall;
1668                                 case COND_B:    upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_uless_,TYPE_INT_MAX)); goto do_upcall;
1669                                 case COND_BE:   upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_uless_equal_,TYPE_INT_MAX)); goto do_upcall;
1670                                 case COND_A:    upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_ugreater_,TYPE_INT_MAX)); goto do_upcall;
1671                                 case COND_AE:   upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_ugreater_equal_,TYPE_INT_MAX)); goto do_upcall;
1672                                 do_upcall:      g(gen_alu_upcall(ctx, upcall, op_size, slot_1, slot_2, slot_r, 0));
1673                                                 return true;
1674 #endif
1675                                 default:
1676                                         internal(file_line, "gen_alu: unsupported condition %u", alu);
1677                         }
1678                         return false;
1679                 }
1680 #if defined(ARCH_X86)
1681                 g(gen_frame_get(ctx, op_size, garbage, slot_1, R_SCRATCH_1, &reg1));
1682                 g(gen_frame_load_cmp_set_cond(ctx, op_size, garbage, slot_2, reg1, alu, slot_r));
1683 #else
1684                 g(gen_frame_get(ctx, op_size, op_size == i_size_cmp(op_size) + (unsigned)zero ? garbage : alu == COND_L || alu == COND_LE || alu == COND_G || alu == COND_GE || ARCH_PREFERS_SX(op_size) ? sign_x : zero_x, slot_1, R_SCRATCH_1, &reg1));
1685                 g(gen_frame_load_cmp_set_cond(ctx, op_size, alu == COND_L || alu == COND_LE || alu == COND_G || alu == COND_GE || ARCH_PREFERS_SX(op_size) ? sign_x : zero_x, slot_2, reg1, alu, slot_r));
1686 #endif
1687                 return true;
1688         }
1689 }
1690
1691 static bool attr_w gen_alu1(struct codegen_context *ctx, unsigned mode, unsigned op_size, unsigned op, uint32_t label_ovf, frame_t slot_1, frame_t slot_r)
1692 {
1693         unsigned alu;
1694         unsigned reg1, target;
1695         switch (mode) {
1696                 case MODE_FIXED: switch (op) {
1697                         case OPCODE_FIXED_OP_not:               alu = ALU1_NOT; goto do_alu;
1698                         case OPCODE_FIXED_OP_neg:               alu = ALU1_NEG; goto do_alu;
1699                         case OPCODE_FIXED_OP_bswap:
1700                         case OPCODE_FIXED_OP_bswap_alt1:        alu = ALU1_BSWAP; goto do_bswap;
1701                         case OPCODE_FIXED_OP_brev:
1702                         case OPCODE_FIXED_OP_brev_alt1:         alu = ALU1_BREV; goto do_brev;
1703                         case OPCODE_FIXED_OP_bsf:
1704                         case OPCODE_FIXED_OP_bsf_alt1:          alu = ALU1_BSF; goto do_bsf_bsr_popcnt;
1705                         case OPCODE_FIXED_OP_bsr:
1706                         case OPCODE_FIXED_OP_bsr_alt1:          alu = ALU1_BSR; goto do_bsf_bsr_popcnt;
1707                         case OPCODE_FIXED_OP_popcnt:
1708                         case OPCODE_FIXED_OP_popcnt_alt1:       alu = ALU1_POPCNT; goto do_bsf_bsr_popcnt;
1709                         case OPCODE_FIXED_OP_to_int:            goto do_fixed_conv;
1710                         case OPCODE_FIXED_OP_from_int:          goto do_fixed_conv;
1711                         case OPCODE_FIXED_OP_uto_int:           goto conv_uto_int;
1712                         case OPCODE_FIXED_OP_ufrom_int:         goto conv_ufrom_int;
1713                         default:                                internal(file_line, "gen_alu1: unsupported fixed operation %u", op);
1714                 }
1715                 case MODE_INT: switch (op) {
1716                         case OPCODE_INT_OP_not:                 alu = ALU1_NOT; mode = MODE_FIXED; goto do_alu;
1717                         case OPCODE_INT_OP_neg:                 alu = ALU1_NEG; goto do_alu;
1718                         case OPCODE_INT_OP_bsf:                 alu = ALU1_BSF; goto do_bsf_bsr_popcnt;
1719                         case OPCODE_INT_OP_bsr:                 alu = ALU1_BSR; goto do_bsf_bsr_popcnt;
1720                         case OPCODE_INT_OP_popcnt:
1721                         case OPCODE_INT_OP_popcnt_alt1:         alu = ALU1_POPCNT; goto do_bsf_bsr_popcnt;
1722                         case OPCODE_INT_OP_to_int:              goto do_conv;
1723                         case OPCODE_INT_OP_from_int:            goto do_conv;
1724                         default:                                internal(file_line, "gen_alu1: unsupported int operation %u", op);
1725                 }
1726                 case MODE_BOOL: switch (op) {
1727                         case OPCODE_BOOL_OP_not:                goto do_bool_not;
1728                         default:                                internal(file_line, "gen_alu1: unsupported bool operation %u", op);
1729                 }
1730         }
1731         internal(file_line, "gen_alu1: unsupported mode %u", mode);
1732
1733         /*******
1734          * ALU *
1735          *******/
1736 do_alu: {
1737                 bool arch_use_flags = ARCH_HAS_FLAGS;
1738                 enum extend ex;
1739 #if defined(ARCH_POWER)
1740                 arch_use_flags = false;
1741 #endif
1742                 if (op_size > OP_SIZE_NATIVE) {
1743 #if !defined(ARCH_X86) && !defined(ARCH_ARM) && !defined(ARCH_POWER)
1744                         if (alu == ALU1_NEG) {
1745                                 if (mode == MODE_FIXED)
1746                                         g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(FIXED_unary_neg_,TYPE_INT_MAX)), op_size, slot_1, NO_FRAME_T, slot_r, 0));
1747                                 else
1748                                         g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(INT_unary_neg_,TYPE_INT_MAX)), op_size, slot_1, NO_FRAME_T, slot_r, label_ovf));
1749                                 return true;
1750                         }
1751 #endif
1752                         g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
1753 #if defined(ARCH_S390)
1754                         if (alu == ALU1_NOT) {
1755                                 g(gen_load_constant(ctx, R_SCRATCH_3, -1));
1756
1757                                 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_XOR, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_3, 0));
1758                                 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_XOR, R_SCRATCH_2, R_SCRATCH_2, R_SCRATCH_3, 0));
1759
1760                                 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_1, R_SCRATCH_2));
1761                                 return true;
1762                         }
1763 #endif
1764                         g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, alu, R_SCRATCH_1, R_SCRATCH_1, alu == ALU1_NEG ? 2 : 0));
1765                         if (alu == ALU1_NOT) {
1766                                 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NOT, R_SCRATCH_2, R_SCRATCH_2, 0));
1767                         } else {
1768 #if defined(ARCH_X86)
1769                                 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NOT, R_SCRATCH_2, R_SCRATCH_2, 0));
1770
1771                                 g(gen_imm(ctx, -1, IMM_PURPOSE_SUB, OP_SIZE_NATIVE));
1772                                 gen_insn(INSN_ALU, OP_SIZE_NATIVE, ALU_SBB, ALU_WRITES_FLAGS(OP_SIZE_NATIVE, ALU_SBB, false, is_imm(), ctx->const_imm));
1773                                 gen_one(R_SCRATCH_2);
1774                                 gen_one(R_SCRATCH_2);
1775                                 gen_imm_offset();
1776 #else
1777                                 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NGC, R_SCRATCH_2, R_SCRATCH_2, (mode == MODE_INT)));
1778 #endif
1779                         }
1780                         if (mode == MODE_INT) {
1781                                 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_O, 0);
1782                                 gen_four(label_ovf);
1783                         }
1784                         g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_1, R_SCRATCH_2));
1785                         return true;
1786                 }
1787                 if ((arch_use_flags || ARCH_SUPPORTS_TRAPS(op_size)) && slot_1 == slot_r && i_size_cmp(op_size) == op_size + zero) {
1788                         struct cg_exit *ce;
1789                         unsigned undo_alu = alu;
1790                         if (slot_is_register(ctx, slot_1)) {
1791                                 unsigned reg = ctx->registers[slot_1];
1792                                 if (mode == MODE_INT && ARCH_SUPPORTS_TRAPS(op_size)) {
1793                                         gen_insn(INSN_ALU1_TRAP, op_size, alu, ALU1_WRITES_FLAGS(alu));
1794                                         gen_one(reg);
1795                                         gen_one(reg);
1796                                         if (ARCH_TRAP_BEFORE || alu == undo_alu) {
1797                                                 gen_four(label_ovf);
1798                                                 return true;
1799                                         } else {
1800                                                 ce = alloc_undo_label(ctx);
1801                                                 if (unlikely(!ce))
1802                                                         return false;
1803                                                 gen_four(ce->undo_label);
1804                                                 goto do_undo_opcode;
1805                                         }
1806                                 }
1807                                 g(gen_2address_alu1(ctx, i_size(op_size), alu, reg, reg, mode == MODE_INT));
1808                                 if (mode == MODE_INT) {
1809                                         if (alu != undo_alu) {
1810                                                 ce = alloc_undo_label(ctx);
1811                                                 if (unlikely(!ce))
1812                                                         return false;
1813                                                 gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
1814                                                 gen_four(ce->undo_label);
1815 do_undo_opcode:
1816                                                 ce->undo_opcode = INSN_ALU1 + ARCH_PARTIAL_ALU(op_size);
1817                                                 ce->undo_op_size = i_size(op_size);
1818                                                 ce->undo_aux = undo_alu;
1819                                                 ce->undo_writes_flags = ALU1_WRITES_FLAGS(undo_alu);
1820                                                 ce->undo_parameters[0] = reg;
1821                                                 ce->undo_parameters[1] = reg;
1822                                                 ce->undo_parameters_len = 2;
1823                                         } else {
1824                                                 gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
1825                                                 gen_four(label_ovf);
1826                                         }
1827                                 }
1828                                 return true;
1829                         }
1830 #if defined(ARCH_X86)
1831                         else {
1832                                 size_t m;
1833                                 int64_t offset = (size_t)slot_1 * slot_size;
1834                                 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDR_OFFSET, i_size(op_size)));
1835                                 gen_insn(INSN_ALU1 + ARCH_PARTIAL_ALU(op_size), i_size(op_size), alu, ALU1_WRITES_FLAGS(alu) | (mode == MODE_INT));
1836                                 gen_address_offset();
1837                                 gen_address_offset();
1838                                 if (mode == MODE_INT) {
1839                                         if (alu != undo_alu) {
1840                                                 ce = alloc_undo_label(ctx);
1841                                                 if (unlikely(!ce))
1842                                                         return false;
1843                                                 ce->undo_opcode = INSN_ALU1 + ARCH_PARTIAL_ALU(op_size);
1844                                                 ce->undo_op_size = i_size(op_size);
1845                                                 ce->undo_aux = undo_alu;
1846                                                 ce->undo_writes_flags = ALU1_WRITES_FLAGS(undo_alu);
1847                                                 m = mark_params(ctx);
1848                                                 gen_address_offset();
1849                                                 gen_address_offset();
1850                                                 copy_params(ctx, ce, m);
1851                                                 gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
1852                                                 gen_four(ce->undo_label);
1853                                         } else {
1854                                                 gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
1855                                                 gen_four(label_ovf);
1856                                         }
1857                                 }
1858                                 return true;
1859                         }
1860 #endif
1861                 }
1862                 target = gen_frame_target(ctx, slot_r, mode == MODE_INT ? slot_1 : NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1863                 if (mode == MODE_FIXED) {
1864                         ex = garbage;
1865                 } else {
1866                         ex = ARCH_PREFERS_SX(op_size) ? sign_x : zero_x;
1867                         if (ARCH_SUPPORTS_TRAPS(op_size))
1868                                 ex = garbage;
1869                         if (op_size == i_size(op_size) + (unsigned)zero)
1870                                 ex = garbage;
1871                 }
1872                 g(gen_frame_get(ctx, op_size, ex, slot_1, mode == MODE_INT ? R_SCRATCH_2 : target, &reg1));
1873 #if defined(ARCH_S390)
1874                 if (alu == ALU1_NOT) {
1875                         g(gen_3address_alu_imm(ctx, i_size(op_size), ALU_XOR, target, reg1, -1, 0));
1876
1877                         g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1878                         return true;
1879                 }
1880 #endif
1881 #if defined(ARCH_X86)
1882                 g(gen_2address_alu1(ctx, op_size, alu, target, reg1, mode == MODE_INT));
1883 #else
1884                 if (mode == MODE_INT) {
1885 #if defined(ARCH_POWER)
1886                         if (op_size == OP_SIZE_NATIVE) {
1887                                 g(gen_2address_alu1(ctx, i_size(op_size), alu, target, reg1, 0));
1888                                 if (alu == ALU1_NEG) {
1889                                         g(gen_3address_alu(ctx, i_size(op_size), ALU_AND, R_CG_SCRATCH, target, reg1, 1));
1890                                 }
1891                                 gen_insn(INSN_JMP_COND, op_size, COND_L, 0);
1892                                 gen_four(label_ovf);
1893
1894                                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1895
1896                                 return true;
1897                         }
1898 #endif
1899                         if (!arch_use_flags && !ARCH_SUPPORTS_TRAPS(op_size) && ARCH_HAS_ANDN && op_size >= OP_SIZE_4) {
1900                                 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, alu, target, reg1, 0));
1901
1902                                 if (alu == ALU1_NEG) {
1903                                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_AND, R_SCRATCH_3, target, reg1, 0));
1904                                 }
1905
1906                                 if (op_size < OP_SIZE_NATIVE)
1907                                         g(gen_extend(ctx, op_size, sign_x, R_SCRATCH_3, R_SCRATCH_3));
1908
1909                                 g(gen_jmp_on_zero(ctx, OP_SIZE_NATIVE, R_SCRATCH_3, COND_S, label_ovf));
1910
1911                                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1912
1913                                 return true;
1914                         }
1915                         if (op_size <= OP_SIZE_2 || (!arch_use_flags && !ARCH_SUPPORTS_TRAPS(op_size))) {
1916                                 int64_t imm = (ARCH_PREFERS_SX(op_size) ? -0x80ULL : 0x80ULL) << (((1 << op_size) - 1) * 8);
1917
1918                                 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, i_size_cmp(op_size), reg1, imm, COND_E, label_ovf));
1919
1920                                 mode = MODE_FIXED;
1921
1922                                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1923                         }
1924                 }
1925 #if !ARCH_HAS_FLAGS
1926                 if (mode == MODE_INT) {
1927                         gen_insn(INSN_ALU1_TRAP, op_size, alu, ALU1_WRITES_FLAGS(alu));
1928                         gen_one(target);
1929                         gen_one(reg1);
1930                         gen_four(label_ovf);
1931                         g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1932                         return true;
1933                 }
1934 #endif
1935                 g(gen_2address_alu1(ctx, i_size(op_size), alu, target, reg1, mode == MODE_INT));
1936 #endif
1937                 if (mode == MODE_INT) {
1938                         gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
1939                         gen_four(label_ovf);
1940                 }
1941                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1942                 return true;
1943         }
1944
1945         /*******
1946          * NOT *
1947          *******/
1948 do_bool_not: {
1949                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1950                 g(gen_frame_get(ctx, op_size, garbage, slot_1, target, &reg1));
1951
1952                 g(gen_3address_alu_imm(ctx, i_size(op_size), ALU_XOR, target, reg1, 1, 0));
1953
1954                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1955                 return true;
1956         }
1957
1958         /*********
1959          * BSWAP *
1960          *********/
1961 do_bswap: {
1962                 bool attr_unused sx = false;
1963 #if defined(ARCH_X86) || defined(ARCH_ARM) || defined(ARCH_IA64) || defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS) || defined(ARCH_RISCV64) || defined(ARCH_S390)
1964 #if defined(ARCH_ARM32)
1965                 if (unlikely(!cpu_test_feature(CPU_FEATURE_armv6)))
1966                         goto do_generic_bswap;
1967 #endif
1968 #if defined(ARCH_MIPS)
1969                 if (unlikely(!MIPS_HAS_ROT))
1970                         goto do_generic_bswap;
1971                 sx = op_size == OP_SIZE_4;
1972 #endif
1973 #if defined(ARCH_RISCV64)
1974                 if (unlikely(!cpu_test_feature(CPU_FEATURE_zbb)))
1975                         goto do_generic_bswap;
1976 #endif
1977 #if defined(ARCH_S390)
1978                 if (op_size == OP_SIZE_2)
1979                         goto do_generic_bswap;
1980 #endif
1981 #if defined(ARCH_X86)
1982                 if (op_size >= OP_SIZE_4 && !cpu_test_feature(CPU_FEATURE_bswap))
1983                         goto do_generic_bswap;
1984 #endif
1985                 if (op_size > OP_SIZE_NATIVE) {
1986                         g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
1987                         reg1 = R_SCRATCH_1;
1988                         target = R_SCRATCH_1;
1989                 } else {
1990                         target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1991                         g(gen_frame_get(ctx, op_size, sx ? sign_x : garbage, slot_1, target, &reg1));
1992                 }
1993
1994                 if (op_size == OP_SIZE_1) {
1995 #if defined(ARCH_IA64) || defined(ARCH_RISCV64)
1996                 } else if (op_size == OP_SIZE_2 || op_size == OP_SIZE_4) {
1997                         g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_BSWAP, target, reg1, 0));
1998
1999                         g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SAR, target, target, op_size == OP_SIZE_2 ? 48 : 32, 0));
2000 #endif
2001                 } else if (op_size == OP_SIZE_2) {
2002 #if defined(ARCH_X86)
2003                         g(gen_3address_rot_imm(ctx, OP_SIZE_2, ROT_ROR, target, reg1, 8, 0));
2004 #else
2005                         g(gen_2address_alu1(ctx, OP_SIZE_4, ALU1_BSWAP16, target, reg1, 0));
2006 #endif
2007                 } else {
2008                         g(gen_2address_alu1(ctx, minimum(op_size, OP_SIZE_NATIVE), ALU1_BSWAP, target, reg1, 0));
2009                 }
2010                 if (op_size > OP_SIZE_NATIVE) {
2011                         g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_BSWAP, R_SCRATCH_2, R_SCRATCH_2, 0));
2012                 }
2013
2014                 if (op_size > OP_SIZE_NATIVE)
2015                         g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_2, R_SCRATCH_1));
2016                 else
2017                         g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2018                 return true;
2019 #endif
2020                 goto do_generic_bswap;
2021 do_generic_bswap:
2022                 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_unary_bswap_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, 0);
2023         }
2024         /********
2025          * BREV *
2026          ********/
2027 do_brev: {
2028 #if defined(ARCH_ARM) || defined(ARCH_LOONGARCH64) || (defined(ARCH_MIPS) && MIPS_R6)
2029 #if defined(ARCH_ARM32)
2030                 if (unlikely(!cpu_test_feature(CPU_FEATURE_armv6t2)))
2031                         goto do_generic_brev;
2032 #endif
2033                 if (op_size > OP_SIZE_NATIVE) {
2034                         g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
2035                         reg1 = R_SCRATCH_1;
2036                         target = R_SCRATCH_1;
2037                 } else {
2038                         target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
2039                         g(gen_frame_get(ctx, op_size, garbage, slot_1, target, &reg1));
2040                 }
2041
2042                 g(gen_2address_alu1(ctx, minimum(maximum(OP_SIZE_4, op_size), OP_SIZE_NATIVE), ALU1_BREV, target, reg1, 0));
2043                 if (op_size <= OP_SIZE_2) {
2044                         g(gen_3address_rot_imm(ctx, OP_SIZE_4, ROT_SHR, target, target, op_size == OP_SIZE_1 ? 24 : 16, 0));
2045                 }
2046                 if (op_size > OP_SIZE_NATIVE) {
2047                         g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_BREV, R_SCRATCH_2, R_SCRATCH_2, 0));
2048                 }
2049
2050                 if (op_size > OP_SIZE_NATIVE)
2051                         g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_2, R_SCRATCH_1));
2052                 else
2053                         g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2054                 return true;
2055 #endif
2056                 goto do_generic_brev;
2057 do_generic_brev:
2058                 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_unary_brev_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, 0);
2059         }
2060         /******************
2061          * BSF/BSR/POPCNT *
2062          ******************/
2063 do_bsf_bsr_popcnt: {
2064                 if (op_size > OP_SIZE_NATIVE) {
2065 #if defined(ARCH_X86)
2066                         uint32_t label_finish = 0;      /* avoid warning */
2067                         if (alu == ALU1_POPCNT && unlikely(!cpu_test_feature(CPU_FEATURE_popcnt)))
2068                                 goto do_generic_bsf_bsr_popcnt;
2069                         if (alu == ALU1_BSR || alu == ALU1_POPCNT) {
2070                                 if (mode == MODE_INT) {
2071                                         g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
2072                                         g(gen_imm(ctx, 0, IMM_PURPOSE_STORE_VALUE, OP_SIZE_NATIVE));
2073                                         gen_insn(INSN_CMP, OP_SIZE_NATIVE, 0, 1);
2074                                         gen_address_offset();
2075                                         gen_imm_offset();
2076
2077                                         gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_S, 0);
2078                                         gen_four(label_ovf);
2079                                 }
2080                         }
2081                         if (alu == ALU1_BSF) {
2082                                 label_finish = alloc_label(ctx);
2083                                 if (unlikely(!label_finish))
2084                                         return false;
2085
2086                                 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + lo_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
2087                                 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_BSF, ALU1_WRITES_FLAGS(ALU1_BSF));
2088                                 gen_one(R_SCRATCH_1);
2089                                 gen_address_offset();
2090
2091                                 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_NE, 0);
2092                                 gen_four(label_finish);
2093
2094                                 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
2095                                 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_BSF, ALU1_WRITES_FLAGS(ALU1_BSF));
2096                                 gen_one(R_SCRATCH_1);
2097                                 gen_address_offset();
2098
2099                                 g(gen_3address_alu_imm(ctx, OP_SIZE_4, ALU_ADD, R_SCRATCH_1, R_SCRATCH_1, 8U << OP_SIZE_NATIVE, 0));
2100                         }
2101                         if (alu == ALU1_BSR) {
2102                                 label_finish = alloc_label(ctx);
2103                                 if (unlikely(!label_finish))
2104                                         return false;
2105
2106                                 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
2107                                 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_BSR, ALU1_WRITES_FLAGS(ALU1_BSR));
2108                                 gen_one(R_SCRATCH_1);
2109                                 gen_address_offset();
2110
2111                                 g(gen_3address_alu_imm(ctx, OP_SIZE_4, ALU_ADD, R_SCRATCH_1, R_SCRATCH_1, 8U << OP_SIZE_NATIVE, 0));
2112
2113                                 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_NE, 0);
2114                                 gen_four(label_finish);
2115
2116                                 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + lo_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
2117                                 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_BSR, ALU1_WRITES_FLAGS(ALU1_BSR));
2118                                 gen_one(R_SCRATCH_1);
2119                                 gen_address_offset();
2120                         }
2121                         if (alu == ALU1_BSF || alu == ALU1_BSR) {
2122                                 if (mode == MODE_INT) {
2123                                         gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_E, 0);
2124                                         gen_four(label_ovf);
2125                                 } else {
2126                                         gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_NE, 0);
2127                                         gen_four(label_finish);
2128
2129                                         g(gen_load_constant(ctx, R_SCRATCH_1, -1));
2130                                 }
2131
2132                                 gen_label(label_finish);
2133
2134                                 if (mode == MODE_INT)
2135                                         goto write_result;
2136
2137                                 if (R_SCRATCH_1 != R_AX || R_SCRATCH_2 != R_DX)
2138                                         internal(file_line, "gen_alu1: bad scratch registers");
2139                                 gen_insn(INSN_CWD, OP_SIZE_NATIVE, 0, 0);
2140                                 gen_one(R_DX);
2141                                 gen_one(R_AX);
2142
2143                                 g(gen_address(ctx, R_FRAME, (size_t)slot_r * slot_size + lo_word(OP_SIZE_NATIVE), IMM_PURPOSE_STR_OFFSET, OP_SIZE_NATIVE));
2144                                 gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
2145                                 gen_address_offset();
2146                                 gen_one(R_SCRATCH_1);
2147
2148                                 g(gen_address(ctx, R_FRAME, (size_t)slot_r * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_STR_OFFSET, OP_SIZE_NATIVE));
2149                                 gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
2150                                 gen_address_offset();
2151                                 gen_one(R_SCRATCH_2);
2152
2153                                 return true;
2154                         }
2155                         if (alu == ALU1_POPCNT) {
2156                                 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + lo_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
2157                                 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_POPCNT, ALU1_WRITES_FLAGS(ALU1_POPCNT));
2158                                 gen_one(R_SCRATCH_1);
2159                                 gen_address_offset();
2160
2161                                 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
2162                                 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_POPCNT, ALU1_WRITES_FLAGS(ALU1_POPCNT));
2163                                 gen_one(R_SCRATCH_2);
2164                                 gen_address_offset();
2165
2166                                 g(gen_3address_alu(ctx, OP_SIZE_4, ALU_ADD, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_2, 1));
2167 write_result:
2168                                 g(gen_address(ctx, R_FRAME, (size_t)slot_r * slot_size + lo_word(OP_SIZE_NATIVE), IMM_PURPOSE_STR_OFFSET, OP_SIZE_NATIVE));
2169                                 gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
2170                                 gen_address_offset();
2171                                 gen_one(R_SCRATCH_1);
2172
2173                                 g(gen_address(ctx, R_FRAME, (size_t)slot_r * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_STR_OFFSET, OP_SIZE_NATIVE));
2174                                 g(gen_imm(ctx, 0, IMM_PURPOSE_STORE_VALUE, OP_SIZE_NATIVE));
2175                                 gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
2176                                 gen_address_offset();
2177                                 gen_imm_offset();
2178
2179                                 return true;
2180                         }
2181 #endif
2182                         goto do_generic_bsf_bsr_popcnt;
2183                 }
2184 #if defined(ARCH_X86)
2185                 if (alu == ALU1_POPCNT && unlikely(!cpu_test_feature(CPU_FEATURE_popcnt)))
2186                         goto do_generic_bsf_bsr_popcnt;
2187                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
2188                 if (op_size == OP_SIZE_1 || ((alu == ALU1_BSR || alu == ALU1_POPCNT) && mode == MODE_INT)) {
2189                         g(gen_frame_get(ctx, op_size, zero_x, slot_1, target, &reg1));
2190                         if ((alu == ALU1_BSR || alu == ALU1_POPCNT) && mode == MODE_INT) {
2191                                 g(gen_cmp_test_jmp(ctx, INSN_TEST, op_size, reg1, reg1, alu == ALU1_BSR ? COND_LE : COND_S, label_ovf));
2192                         }
2193                         g(gen_2address_alu1(ctx, maximum(op_size, OP_SIZE_2), alu, target, reg1, 1));
2194                         if ((alu == ALU1_BSR || alu == ALU1_POPCNT) && mode == MODE_INT)
2195                                 goto x86_bsf_bsr_popcnt_finish;
2196                 } else {
2197                         g(gen_frame_load_op1(ctx, op_size, alu, 1, slot_1, target));
2198                 }
2199                 if (alu == ALU1_POPCNT)
2200                         goto x86_bsf_bsr_popcnt_finish;
2201                 if (mode == MODE_FIXED) {
2202                         uint32_t cmov_label;
2203                         gen_insn(INSN_MOV, maximum(op_size, OP_SIZE_4), 0, 0);
2204                         gen_one(R_SCRATCH_2);
2205                         gen_one(ARG_IMM);
2206                         gen_eight(-1);
2207                         g(gen_cmov(ctx, maximum(op_size, OP_SIZE_4), COND_E, target, &cmov_label));
2208                         gen_one(R_SCRATCH_2);
2209                         if (cmov_label)
2210                                 gen_label(cmov_label);
2211
2212                 } else {
2213                         gen_insn(INSN_JMP_COND, maximum(op_size, OP_SIZE_2), COND_E, 0);
2214                         gen_four(label_ovf);
2215                 }
2216 x86_bsf_bsr_popcnt_finish:
2217                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2218                 return true;
2219 #endif
2220 #if defined(ARCH_ARM)
2221 #if defined(ARCH_ARM32)
2222                 if (alu == ALU1_BSR && unlikely(!cpu_test_feature(CPU_FEATURE_armv6)))
2223                         goto do_generic_bsf_bsr_popcnt;
2224                 if (alu == ALU1_BSF && unlikely(!cpu_test_feature(CPU_FEATURE_armv6t2)))
2225                         goto do_generic_bsf_bsr_popcnt;
2226 #endif
2227                 if (alu == ALU1_POPCNT && unlikely(!cpu_test_feature(CPU_FEATURE_neon)))
2228                         goto do_generic_bsf_bsr_popcnt;
2229                 g(gen_frame_get(ctx, op_size, mode == MODE_INT ? sign_x : zero_x, slot_1, R_SCRATCH_1, &reg1));
2230                 if (mode == MODE_INT) {
2231                         g(gen_cmp_test_jmp(ctx, INSN_TEST, i_size(op_size), reg1, reg1, alu == ALU1_BSR ? COND_LE : alu == ALU1_BSF ? COND_E : COND_S, label_ovf));
2232                 }
2233
2234                 if (alu == ALU1_POPCNT) {
2235                         g(gen_mov(ctx, OP_SIZE_NATIVE, FR_SCRATCH_1, reg1));
2236                         gen_insn(INSN_FP_ALU1, OP_SIZE_NATIVE, FP_ALU1_VCNT8, 0);
2237                         gen_one(FR_SCRATCH_1);
2238                         gen_one(FR_SCRATCH_1);
2239 #if defined(ARCH_ARM32)
2240                         if (op_size > OP_SIZE_1) {
2241                                 gen_insn(INSN_FP_ALU1, OP_SIZE_1, FP_ALU1_VPADDL, 0);
2242                                 gen_one(FR_SCRATCH_1);
2243                                 gen_one(FR_SCRATCH_1);
2244                         }
2245                         if (op_size > OP_SIZE_2) {
2246                                 gen_insn(INSN_FP_ALU1, OP_SIZE_2, FP_ALU1_VPADDL, 0);
2247                                 gen_one(FR_SCRATCH_1);
2248                                 gen_one(FR_SCRATCH_1);
2249                         }
2250 #else
2251                         if (op_size > OP_SIZE_1) {
2252                                 gen_insn(INSN_FP_ALU1, OP_SIZE_1, FP_ALU1_ADDV, 0);
2253                                 gen_one(FR_SCRATCH_1);
2254                                 gen_one(FR_SCRATCH_1);
2255                         }
2256 #endif
2257                         g(gen_frame_store(ctx, op_size, slot_r, 0, FR_SCRATCH_1));
2258                         if (slot_is_register(ctx, slot_r))
2259                                 g(unspill(ctx, slot_r));
2260                         return true;
2261                 }
2262
2263                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
2264                 if (mode == MODE_FIXED && alu == ALU1_BSF) {
2265                         gen_insn(INSN_TEST, i_size(op_size), 0, 1);
2266                         gen_one(target);
2267                         gen_one(reg1);
2268                         reg1 = target;
2269                 }
2270
2271                 if (alu == ALU1_BSF) {
2272                         g(gen_2address_alu1(ctx, i_size(op_size), ALU1_BREV, target, reg1, 0));
2273                         reg1 = target;
2274                 }
2275
2276                 g(gen_2address_alu1(ctx, i_size(op_size), ALU1_LZCNT, target, reg1, 0));
2277
2278                 if (alu == ALU1_BSR) {
2279                         g(gen_load_constant(ctx, R_SCRATCH_2, op_size == OP_SIZE_8 ? 63 : 31));
2280                         g(gen_3address_alu(ctx, i_size(op_size), ALU_SUB, target, R_SCRATCH_2, target, 0));
2281                 }
2282
2283                 if (mode == MODE_FIXED && alu == ALU1_BSF) {
2284 #if defined(ARCH_ARM32)
2285                         g(gen_imm(ctx, -1, IMM_PURPOSE_CMOV, OP_SIZE_NATIVE));
2286                         gen_insn(INSN_CMOV, OP_SIZE_NATIVE, COND_E, 0);
2287                         gen_one(target);
2288                         gen_one(target);
2289                         gen_imm_offset();
2290 #else
2291                         gen_insn(INSN_CSEL_INV, i_size(op_size), COND_NE, 0);
2292                         gen_one(target);
2293                         gen_one(ARG_IMM);
2294                         gen_eight(0);
2295                         gen_one(target);
2296 #endif
2297                 }
2298
2299                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2300                 return true;
2301 #endif
2302 #if defined(ARCH_ALPHA)
2303                 if (likely(cpu_test_feature(CPU_FEATURE_cix))) {
2304                         g(gen_frame_get(ctx, op_size, mode == MODE_INT ? sign_x : zero_x, slot_1, R_SCRATCH_1, &reg1));
2305                         target = gen_frame_target(ctx, slot_r, slot_1, NO_FRAME_T, R_SCRATCH_2);
2306                         if (mode == MODE_INT) {
2307                                 g(gen_cmp_test_jmp(ctx, INSN_TEST, OP_SIZE_NATIVE, reg1, reg1, alu == ALU1_BSR ? COND_LE : alu == ALU1_BSF ? COND_E : COND_S, label_ovf));
2308                         }
2309                         if (alu == ALU1_POPCNT) {
2310                                 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_POPCNT, target, reg1, 0));
2311                         }
2312                         if (alu == ALU1_BSF) {
2313                                 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_BSF, target, reg1, 0));
2314
2315                                 if (mode == MODE_FIXED) {
2316                                         g(gen_imm(ctx, -1, IMM_PURPOSE_MOVR, OP_SIZE_INT));
2317                                         gen_insn(INSN_MOVR, OP_SIZE_NATIVE, COND_E, 0);
2318                                         gen_one(target);
2319                                         gen_one(target);
2320                                         gen_one(reg1);
2321                                         gen_imm_offset();
2322                                 }
2323                         }
2324                         if (alu == ALU1_BSR) {
2325                                 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_LZCNT, target, reg1, 0));
2326
2327                                 g(gen_load_constant(ctx, R_SCRATCH_3, OP_SIZE_NATIVE == OP_SIZE_8 ? 63 : 31));
2328
2329                                 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_SUB, target, R_SCRATCH_3, target, 0));
2330                         }
2331                         g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2332                         return true;
2333                 }
2334 #endif
2335 #if defined(ARCH_MIPS)
2336                 if (MIPS_HAS_CLZ && alu != ALU1_POPCNT) {
2337                         g(gen_frame_get(ctx, op_size, mode == MODE_INT ? sign_x : zero_x, slot_1, R_SCRATCH_1, &reg1));
2338                         target = gen_frame_target(ctx, slot_r, slot_1, NO_FRAME_T, R_SCRATCH_2);
2339                         if (mode == MODE_INT) {
2340                                 g(gen_cmp_test_jmp(ctx, INSN_TEST, OP_SIZE_NATIVE, reg1, reg1, alu == ALU1_BSR ? COND_LE : alu == ALU1_BSF ? COND_E : COND_S, label_ovf));
2341                         }
2342                         if (alu == ALU1_BSF) {
2343                                 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NEG, target, reg1, 0));
2344
2345                                 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_AND, R_SCRATCH_1, reg1, target, 0));
2346                                 reg1 = R_SCRATCH_1;
2347                         }
2348                         g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_LZCNT, target, reg1, 0));
2349
2350                         g(gen_load_constant(ctx, R_SCRATCH_3, OP_SIZE_NATIVE == OP_SIZE_8 ? 63 : 31));
2351
2352                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_SUB, target, R_SCRATCH_3, target, 0));
2353
2354                         g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2355                         return true;
2356                 }
2357 #endif
2358 #if defined(ARCH_POWER)
2359                 if (alu == ALU1_BSF && (unlikely(!cpu_test_feature(CPU_FEATURE_v203)) || unlikely(!cpu_test_feature(CPU_FEATURE_v30))))
2360                         goto do_generic_bsf_bsr_popcnt;
2361                 if (alu == ALU1_POPCNT && unlikely(!cpu_test_feature(CPU_FEATURE_v206)))
2362                         goto do_generic_bsf_bsr_popcnt;
2363                 g(gen_frame_get(ctx, op_size, mode == MODE_INT ? sign_x : zero_x, slot_1, R_SCRATCH_1, &reg1));
2364                 target = gen_frame_target(ctx, slot_r, slot_1, NO_FRAME_T, R_SCRATCH_2);
2365                 if (mode == MODE_INT) {
2366                         g(gen_cmp_test_jmp(ctx, INSN_TEST, i_size(op_size), reg1, reg1, alu == ALU1_BSR ? COND_LE : alu == ALU1_BSF ? COND_E : COND_S, label_ovf));
2367                 }
2368                 if (alu == ALU1_POPCNT) {
2369                         g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_POPCNT, target, reg1, 0));
2370                 }
2371                 if (alu == ALU1_BSF) {
2372                         g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_BSF, target, reg1, 0));
2373
2374                         if (mode == MODE_FIXED) {
2375                                 g(gen_3address_alu(ctx, i_size(op_size), ALU_AND, R_SCRATCH_3, reg1, reg1, 1));
2376
2377                                 g(gen_imm(ctx, -1, IMM_PURPOSE_CMOV, OP_SIZE_NATIVE));
2378                                 gen_insn(INSN_CMOV, OP_SIZE_NATIVE, COND_E, 0);
2379                                 gen_one(target);
2380                                 gen_one(target);
2381                                 gen_imm_offset();
2382                         }
2383                 }
2384                 if (alu == ALU1_BSR) {
2385                         g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_LZCNT, target, reg1, 0));
2386
2387                         g(gen_load_constant(ctx, R_SCRATCH_3, OP_SIZE_NATIVE == OP_SIZE_8 ? 63 : 31));
2388
2389                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_SUB, target, R_SCRATCH_3, target, 0));
2390                 }
2391                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2392                 return true;
2393 #endif
2394 #if defined(ARCH_LOONGARCH64) || defined(ARCH_RISCV64)
2395 #if defined(ARCH_LOONGARCH64)
2396                 if (alu == ALU1_POPCNT)
2397                         goto do_generic_bsf_bsr_popcnt;
2398 #endif
2399 #if defined(ARCH_RISCV64)
2400                 if (unlikely(!cpu_test_feature(CPU_FEATURE_zbb)))
2401                         goto do_generic_bsf_bsr_popcnt;
2402 #endif
2403                 g(gen_frame_get(ctx, op_size, mode == MODE_INT ? sign_x : zero_x, slot_1, R_SCRATCH_1, &reg1));
2404                 target = gen_frame_target(ctx, slot_r, slot_1, NO_FRAME_T, R_SCRATCH_2);
2405                 if (mode == MODE_INT) {
2406                         g(gen_cmp_test_jmp(ctx, INSN_TEST, OP_SIZE_NATIVE, reg1, reg1, alu == ALU1_BSR ? COND_LE : alu == ALU1_BSF ? COND_E : COND_S, label_ovf));
2407                 }
2408                 if (alu == ALU1_POPCNT) {
2409                         g(gen_2address_alu1(ctx, maximum(OP_SIZE_4, op_size), ALU1_POPCNT, target, reg1, 0));
2410                 }
2411                 if (alu == ALU1_BSF) {
2412                         g(gen_2address_alu1(ctx, maximum(OP_SIZE_4, op_size), ALU1_BSF, target, reg1, 0));
2413
2414                         if (mode == MODE_FIXED) {
2415                                 g(gen_imm(ctx, 1, IMM_PURPOSE_CMP, OP_SIZE_NATIVE));
2416                                 gen_insn(INSN_CMP_DEST_REG, OP_SIZE_NATIVE, COND_B, 0);
2417                                 gen_one(R_SCRATCH_3);
2418                                 gen_one(reg1);
2419                                 gen_imm_offset();
2420
2421                                 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NEG, R_SCRATCH_3, R_SCRATCH_3, 0));
2422
2423                                 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_OR, target, target, R_SCRATCH_3, 0));
2424                         }
2425                 }
2426                 if (alu == ALU1_BSR) {
2427                         g(gen_2address_alu1(ctx, maximum(OP_SIZE_4, op_size), ALU1_LZCNT, target, reg1, 0));
2428
2429                         g(gen_load_constant(ctx, R_SCRATCH_3, op_size <= OP_SIZE_4 ? 31 : 63));
2430
2431                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_SUB, target, R_SCRATCH_3, target, 0));
2432                 }
2433                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2434                 return true;
2435 #endif
2436 #if defined(ARCH_IA64) || defined(ARCH_S390) || defined(ARCH_SPARC)
2437                 if (alu == ALU1_BSF && !ARCH_HAS_ANDN)
2438                         goto do_generic_bsf_bsr_popcnt;
2439 #if defined(ARCH_S390)
2440                 if (!cpu_test_feature(CPU_FEATURE_misc_45) || !cpu_test_feature(CPU_FEATURE_misc_insn_ext_3))
2441                         goto do_generic_bsf_bsr_popcnt;
2442 #endif
2443 #if defined(ARCH_SPARC)
2444                 if (!SPARC_9)
2445                         goto do_generic_bsf_bsr_popcnt;
2446 #endif
2447                 g(gen_frame_get(ctx, op_size, mode == MODE_INT ? sign_x : zero_x, slot_1, R_SCRATCH_1, &reg1));
2448                 target = gen_frame_target(ctx, slot_r, slot_1, NO_FRAME_T, R_SCRATCH_2);
2449                 if (mode == MODE_INT) {
2450                         g(gen_cmp_test_jmp(ctx, INSN_TEST, maximum(op_size, OP_SIZE_4), reg1, reg1, alu == ALU1_BSR ? COND_LE : alu == ALU1_BSF ? COND_E : COND_S, label_ovf));
2451                 } else {
2452                         if (ARCH_PREFERS_SX(op_size) && alu == ALU1_POPCNT && op_size < OP_SIZE_NATIVE) {
2453                                 g(gen_extend(ctx, op_size, zero_x, R_SCRATCH_1, reg1));
2454                                 reg1 = R_SCRATCH_1;
2455                         }
2456                 }
2457                 if (alu == ALU1_POPCNT) {
2458                         g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_POPCNT, R_SCRATCH_1, reg1, 0));
2459                         g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
2460                         return true;
2461                 }
2462                 if (alu == ALU1_BSF) {
2463                         g(gen_3address_alu_imm(ctx, OP_SIZE_NATIVE, ALU_SUB, target, reg1, 1, 0));
2464
2465                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_ANDN, target, target, reg1, 0));
2466
2467                         g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_POPCNT, target, target, 0));
2468
2469                         if (mode == MODE_FIXED) {
2470                                 unsigned attr_unused test_reg = R_SCRATCH_1;
2471 #if defined(ARCH_S390)
2472                                 g(gen_imm(ctx, 0, COND_IS_LOGICAL(COND_E) ? IMM_PURPOSE_CMP_LOGICAL : IMM_PURPOSE_CMP, OP_SIZE_NATIVE));
2473                                 gen_insn(INSN_CMP, OP_SIZE_NATIVE, 0, 1 + COND_IS_LOGICAL(COND_E));
2474                                 gen_one(reg1);
2475                                 gen_imm_offset();
2476
2477                                 g(gen_imm(ctx, -1, IMM_PURPOSE_CMOV, OP_SIZE_NATIVE));
2478                                 gen_insn(INSN_CMOV, OP_SIZE_NATIVE, COND_E, 0);
2479                                 gen_one(target);
2480                                 gen_one(target);
2481                                 gen_imm_offset();
2482 #else
2483 #if defined(ARCH_IA64)
2484                                 g(gen_cmp_dest_reg(ctx, OP_SIZE_NATIVE, reg1, (unsigned)-1, R_CMP_RESULT, 0, COND_NE));
2485                                 test_reg = R_CMP_RESULT;
2486 #endif
2487                                 g(gen_imm(ctx, -1, IMM_PURPOSE_MOVR, OP_SIZE_NATIVE));
2488                                 gen_insn(INSN_MOVR, OP_SIZE_NATIVE, COND_E, 0);
2489                                 gen_one(target);
2490                                 gen_one(target);
2491                                 gen_one(test_reg);
2492                                 gen_imm_offset();
2493 #endif
2494                         }
2495
2496                         g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2497                         return true;
2498                 }
2499 #endif
2500 do_generic_bsf_bsr_popcnt:
2501                 if (alu == ALU1_BSF) {
2502                         if (mode == MODE_FIXED)
2503                                 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_unary_bsf_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, 0);
2504                         else
2505                                 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, INT_unary_bsf_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, label_ovf);
2506                 }
2507                 if (alu == ALU1_BSR) {
2508                         if (mode == MODE_FIXED)
2509                                 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_unary_bsr_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, 0);
2510                         else
2511                                 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, INT_unary_bsr_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, label_ovf);
2512                 }
2513                 if (alu == ALU1_POPCNT) {
2514                         if (mode == MODE_FIXED)
2515                                 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_unary_popcnt_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, 0);
2516                         else
2517                                 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, INT_unary_popcnt_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, label_ovf);
2518                 }
2519         }
2520         /**************
2521          * CONVERSION *
2522          **************/
2523 do_fixed_conv:
2524 do_conv: {
2525                 unsigned src_op_size, dest_op_size;
2526                 const struct type *src_type, *dest_type;
2527                 src_type = get_type_of_local(ctx, slot_1);
2528                 dest_type = get_type_of_local(ctx, slot_r);
2529
2530                 if (TYPE_TAG_IS_FIXED(src_type->tag)) {
2531                         src_op_size = TYPE_TAG_IDX_FIXED(src_type->tag) >> 1;
2532                 } else {
2533                         src_op_size = TYPE_TAG_IDX_INT(src_type->tag);
2534                 }
2535
2536                 if (TYPE_TAG_IS_FIXED(dest_type->tag)) {
2537                         dest_op_size = TYPE_TAG_IDX_FIXED(dest_type->tag) >> 1;
2538                 } else {
2539                         dest_op_size = TYPE_TAG_IDX_INT(dest_type->tag);
2540                 }
2541
2542                 if (src_op_size <= OP_SIZE_NATIVE) {
2543                         g(gen_frame_get(ctx, src_op_size, sign_x, slot_1, R_SCRATCH_1, &reg1));
2544                 } else {
2545 #if defined(ARCH_X86)
2546                         if (dest_op_size < src_op_size)
2547                                 g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_1, 0, false, R_SCRATCH_1));
2548                         else
2549 #endif
2550                                 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_3));
2551                         reg1 = R_SCRATCH_1;
2552                 }
2553
2554                 if (dest_op_size >= src_op_size) {
2555                         if (dest_op_size <= OP_SIZE_NATIVE) {
2556                                 g(gen_frame_store(ctx, dest_op_size, slot_r, 0, reg1));
2557                         } else {
2558                                 if (src_op_size <= OP_SIZE_NATIVE) {
2559 #if defined(ARCH_X86)
2560                                         if (R_SCRATCH_1 != R_AX || R_SCRATCH_2 != R_DX)
2561                                                 internal(file_line, "gen_alu1: bad scratch registers");
2562                                         if (reg1 == R_SCRATCH_1) {
2563                                                 gen_insn(INSN_CWD, OP_SIZE_NATIVE, 0, 0);
2564                                                 gen_one(R_DX);
2565                                                 gen_one(R_AX);
2566                                         } else
2567 #endif
2568                                         g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_2, reg1, (1U << (OP_SIZE_NATIVE + 3)) - 1, false));
2569                                         g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, reg1, R_SCRATCH_2));
2570                                 } else {
2571                                         g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, reg1, R_SCRATCH_3));
2572                                 }
2573                         }
2574                         return true;
2575                 } else {
2576                         if (src_op_size > OP_SIZE_NATIVE) {
2577 #if defined(ARCH_ARM)
2578                                 gen_insn(INSN_CMP, OP_SIZE_NATIVE, 0, 1);
2579                                 gen_one(R_SCRATCH_3);
2580                                 gen_one(ARG_SHIFTED_REGISTER);
2581                                 gen_one(ARG_SHIFT_ASR | ((1U << (OP_SIZE_NATIVE + 3)) - 1));
2582                                 gen_one(R_SCRATCH_1);
2583
2584                                 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_NE, 0);
2585                                 gen_four(label_ovf);
2586 #elif defined(ARCH_X86)
2587                                 if (R_SCRATCH_1 != R_AX || R_SCRATCH_2 != R_DX)
2588                                         internal(file_line, "gen_alu1: bad scratch registers");
2589                                 gen_insn(INSN_CWD, OP_SIZE_NATIVE, 0, 0);
2590                                 gen_one(R_DX);
2591                                 gen_one(R_AX);
2592
2593                                 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
2594                                 gen_insn(INSN_CMP, OP_SIZE_NATIVE, 0, 1);
2595                                 gen_one(R_SCRATCH_2);
2596                                 gen_address_offset();
2597
2598                                 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_NE, 0);
2599                                 gen_four(label_ovf);
2600 #else
2601                                 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_2, R_SCRATCH_1, (1U << (OP_SIZE_NATIVE + 3)) - 1, 0));
2602
2603                                 g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, R_SCRATCH_3, R_SCRATCH_2, COND_NE, label_ovf));
2604 #endif
2605
2606                                 src_op_size = OP_SIZE_NATIVE;
2607                         }
2608                         if (src_op_size > dest_op_size) {
2609                                 g(gen_cmp_extended(ctx, OP_SIZE_NATIVE, dest_op_size, reg1, R_SCRATCH_2, label_ovf));
2610                         }
2611                         g(gen_frame_store(ctx, dest_op_size, slot_r, 0, reg1));
2612                         return true;
2613                 }
2614         }
2615
2616 conv_uto_int: {
2617                 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_uto_int_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, label_ovf);
2618         }
2619
2620 conv_ufrom_int: {
2621                 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_ufrom_int_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, label_ovf);
2622         }
2623 }
2624
2625 static bool attr_w gen_constant(struct codegen_context *ctx, bool real, unsigned op_size, bool shrt, frame_t slot_r)
2626 {
2627         uintbig_t c;
2628         if (shrt) {
2629                 c = (int16_t)get_unaligned_16(ctx->current_position);
2630         } else switch (op_size) {
2631 #define fx(n, type, utype, sz, bits)                                    \
2632                 case n:                                                 \
2633                         c = (type)cat(get_unaligned_,bits)(ctx->current_position);\
2634                         break;
2635                 for_all_fixed(fx);
2636 #undef fx
2637                 default:
2638                         internal(file_line, "gen_constant: invalid type %u", op_size);
2639         }
2640         if (op_size > OP_SIZE_NATIVE) {
2641                 unsigned shift = (8U << OP_SIZE_NATIVE) - 1;
2642                 g(gen_frame_store_imm_raw(ctx, OP_SIZE_NATIVE, slot_r, lo_word(OP_SIZE_NATIVE), c & ((2ULL << shift) - 1)));
2643                 g(gen_frame_store_imm_raw(ctx, OP_SIZE_NATIVE, slot_r, hi_word(OP_SIZE_NATIVE), c >> 1 >> shift));
2644                 if (real && slot_is_register(ctx, slot_r))
2645                         g(unspill(ctx, slot_r));
2646                 return true;
2647         } else if (real && slot_is_register(ctx, slot_r)) {
2648                 if (ARCH_HAS_FP_GP_MOV) {
2649                         g(gen_load_constant(ctx, R_SCRATCH_1, c));
2650                         g(gen_mov(ctx, op_size, ctx->registers[slot_r], R_SCRATCH_1));
2651                 } else {
2652                         g(gen_frame_store_imm_raw(ctx, op_size, slot_r, 0, c));
2653                         g(unspill(ctx, slot_r));
2654                 }
2655         } else {
2656                 g(gen_frame_store_imm(ctx, op_size, slot_r, 0, c));
2657         }
2658         return true;
2659 }
2660
2661 static bool attr_w gen_real_constant(struct codegen_context *ctx, const struct type *t, frame_t slot_r)
2662 {
2663         int64_t offset;
2664         if (is_power_of_2(t->size) && t->size <= sizeof(uintbig_t))
2665                 return gen_constant(ctx, true, log_2(t->size), false, slot_r);
2666
2667         g(load_function_offset(ctx, R_SCRATCH_3, offsetof(struct data, u_.function.code)));
2668
2669         offset = (ctx->current_position - da(ctx->fn,function)->code) * sizeof(code_t);
2670
2671         g(gen_memcpy_raw(ctx, R_FRAME, (size_t)slot_r * slot_size, R_SCRATCH_3, offset, t->size, minimum(t->align, sizeof(code_t))));
2672         if (slot_is_register(ctx, slot_r))
2673                 g(unspill(ctx, slot_r));
2674
2675         return true;
2676 }
2677
2678 static bool attr_w gen_copy(struct codegen_context *ctx, unsigned op_size, frame_t slot_1, frame_t slot_r)
2679 {
2680         unsigned reg1;
2681         if (unlikely(op_size > OP_SIZE_NATIVE)) {
2682                 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
2683                 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_1, R_SCRATCH_2));
2684                 return true;
2685         } else {
2686                 unsigned target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
2687                 g(gen_frame_get(ctx, op_size, garbage, slot_1, target, &reg1));
2688                 g(gen_frame_store(ctx, op_size, slot_r, 0, reg1));
2689                 return true;
2690         }
2691 }
2692
2693 static bool attr_w gen_fp_alu(struct codegen_context *ctx, unsigned real_type, unsigned op, uint32_t label_ovf, frame_t slot_1, frame_t slot_2, frame_t slot_r)
2694 {
2695         unsigned attr_unused fp_alu;
2696         size_t upc;
2697         unsigned attr_unused op_size = real_type_to_op_size(real_type);
2698         unsigned reg1, reg2, target;
2699         switch (op) {
2700                 case OPCODE_REAL_OP_add:
2701                 case OPCODE_REAL_OP_add_alt1:
2702                 case OPCODE_REAL_OP_add_alt2: fp_alu = FP_ALU_ADD; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_add_real16_t); label_ovf = 0; goto do_alu;
2703                 case OPCODE_REAL_OP_subtract:
2704                 case OPCODE_REAL_OP_subtract_alt1:
2705                 case OPCODE_REAL_OP_subtract_alt2: fp_alu = FP_ALU_SUB; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_subtract_real16_t); label_ovf = 0; goto do_alu;
2706                 case OPCODE_REAL_OP_multiply:
2707                 case OPCODE_REAL_OP_multiply_alt1:
2708                 case OPCODE_REAL_OP_multiply_alt2: fp_alu = FP_ALU_MUL; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_multiply_real16_t); label_ovf = 0; goto do_alu;
2709                 case OPCODE_REAL_OP_divide:
2710                 case OPCODE_REAL_OP_divide_alt1:
2711                 case OPCODE_REAL_OP_divide_alt2: fp_alu = FP_ALU_DIV; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_divide_real16_t); label_ovf = 0; goto do_alu;
2712                 case OPCODE_REAL_OP_modulo:
2713                 case OPCODE_REAL_OP_power:
2714                 case OPCODE_REAL_OP_ldexp:
2715                 case OPCODE_REAL_OP_atan2: upc = offsetof(struct cg_upcall_vector_s, REAL_binary_modulo_real16_t) + (op - OPCODE_REAL_OP_modulo) * TYPE_REAL_N * sizeof(void (*)(void)); goto do_upcall;
2716                 case OPCODE_REAL_OP_equal:
2717                 case OPCODE_REAL_OP_equal_alt1:
2718                 case OPCODE_REAL_OP_equal_alt2: fp_alu = FP_COND_E; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_equal_real16_t); goto do_cmp;
2719                 case OPCODE_REAL_OP_not_equal:
2720                 case OPCODE_REAL_OP_not_equal_alt1:
2721                 case OPCODE_REAL_OP_not_equal_alt2: fp_alu = FP_COND_NE; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_not_equal_real16_t); goto do_cmp;
2722                 case OPCODE_REAL_OP_less:
2723                 case OPCODE_REAL_OP_less_alt1:
2724                 case OPCODE_REAL_OP_less_alt2: fp_alu = FP_COND_B; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_less_real16_t); goto do_cmp;
2725                 case OPCODE_REAL_OP_less_equal:
2726                 case OPCODE_REAL_OP_less_equal_alt1:
2727                 case OPCODE_REAL_OP_less_equal_alt2: fp_alu = FP_COND_BE; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_less_equal_real16_t); goto do_cmp;
2728                 case OPCODE_REAL_OP_greater:
2729                 case OPCODE_REAL_OP_greater_alt1:
2730                 case OPCODE_REAL_OP_greater_alt2: fp_alu = FP_COND_A; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_greater_real16_t); goto do_cmp;
2731                 case OPCODE_REAL_OP_greater_equal:
2732                 case OPCODE_REAL_OP_greater_equal_alt1:
2733                 case OPCODE_REAL_OP_greater_equal_alt2: fp_alu = FP_COND_AE; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_greater_equal_real16_t); goto do_cmp;
2734                 default: internal(file_line, "gen_fp_alu: unsupported operation %u", op);
2735         }
2736
2737 do_alu:
2738         if ((SUPPORTED_FP >> real_type) & 1) {
2739 #if defined(ARCH_IA64)
2740                 if (unlikely(fp_alu == FP_ALU_DIV))
2741                         goto do_upcall;
2742 #endif
2743 #if defined(ARCH_X86)
2744                 if (1)
2745 #elif defined(ARCH_S390)
2746                 if ((op_size <= OP_SIZE_8 && (size_t)slot_2 * slot_size < 4096) || slot_is_register(ctx, slot_2))
2747 #else
2748                 if (slot_is_register(ctx, slot_2))
2749 #endif
2750                 {
2751                         target = gen_frame_target(ctx, slot_r, NO_FRAME_T, slot_2, FR_SCRATCH_1);
2752                         g(gen_frame_get(ctx, op_size, garbage, slot_1, FR_SCRATCH_1, &reg1));
2753                         if (slot_is_register(ctx, slot_2)) {
2754                                 g(gen_3address_fp_alu(ctx, op_size, fp_alu, target, reg1, ctx->registers[slot_2]));
2755                         } else {
2756                                 if (target != reg1 && !ARCH_IS_3ADDRESS_FP) {
2757                                         g(gen_mov(ctx, op_size, target, reg1));
2758                                         reg1 = target;
2759                                 }
2760                                 g(gen_address(ctx, R_FRAME, (size_t)slot_2 * slot_size, IMM_PURPOSE_VLDR_VSTR_OFFSET, op_size));
2761                                 gen_insn(INSN_FP_ALU, op_size, fp_alu, 0);
2762                                 gen_one(target);
2763                                 gen_one(reg1);
2764                                 gen_address_offset();
2765                         }
2766                         g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2767                         return true;
2768                 }
2769 #if defined(ARCH_ALPHA)
2770                 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, &reg1));
2771                 g(gen_frame_get(ctx, op_size, zero_x, slot_2, FR_SCRATCH_2, &reg2));
2772                 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, FR_SCRATCH_3);
2773                 g(gen_3address_fp_alu(ctx, op_size, fp_alu, target, reg1, reg2));
2774                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2775 #else
2776                 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, &reg1));
2777                 g(gen_frame_get(ctx, op_size, zero_x, slot_2, FR_SCRATCH_2, &reg2));
2778                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_1);
2779                 g(gen_3address_fp_alu(ctx, op_size, fp_alu, target, reg1, reg2));
2780                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2781 #endif
2782                 return true;
2783         }
2784 #ifdef SUPPORTED_FP_X87
2785         if ((SUPPORTED_FP_X87 >> real_type) & 1) {
2786                 if (real_type != 3) {
2787                         g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_2));
2788                         g(gen_frame_load_x87(ctx, INSN_X87_ALU, op_size, fp_alu, slot_1));
2789                 } else {
2790                         g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
2791                         g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_2));
2792                         gen_insn(INSN_X87_ALUP, op_size, fp_alu, 0);
2793                         gen_one(R_ST1);
2794                 }
2795                 g(gen_frame_store_x87(ctx, INSN_X87_FSTP, op_size, slot_r));
2796                 return true;
2797         }
2798 #endif
2799 #ifdef SUPPORTED_FP_HALF_CVT
2800         if ((SUPPORTED_FP_HALF_CVT >> real_type) & 1) {
2801                 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, &reg1));
2802                 g(gen_frame_get(ctx, op_size, zero_x, slot_2, FR_SCRATCH_2, &reg2));
2803                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_1);
2804                 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
2805                 gen_one(FR_SCRATCH_1);
2806                 gen_one(reg1);
2807                 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
2808                 gen_one(FR_SCRATCH_2);
2809                 gen_one(reg2);
2810                 gen_insn(INSN_FP_ALU, OP_SIZE_4, fp_alu, 0);
2811                 gen_one(FR_SCRATCH_1);
2812                 gen_one(FR_SCRATCH_1);
2813                 gen_one(FR_SCRATCH_2);
2814                 gen_insn(INSN_FP_CVT, OP_SIZE_4, op_size, 0);
2815                 gen_one(target);
2816                 gen_one(FR_SCRATCH_1);
2817                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2818                 return true;
2819         }
2820 #endif
2821         goto do_upcall;
2822
2823 do_cmp:
2824         if ((SUPPORTED_FP >> real_type) & 1
2825 #if defined(ARCH_ALPHA)
2826                 && OS_SUPPORTS_TRAPS
2827 #endif
2828         ) {
2829                 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, &reg1));
2830                 g(gen_frame_get(ctx, op_size, zero_x, slot_2, FR_SCRATCH_2, &reg2));
2831                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
2832 #if defined(ARCH_ALPHA)
2833                 gen_insn(INSN_FP_CMP_DEST_REG_TRAP, op_size, fp_alu == FP_COND_NE ? FP_COND_E : fp_alu, 0);
2834                 gen_one(FR_SCRATCH_3);
2835                 gen_one(reg1);
2836                 gen_one(reg2);
2837                 gen_four(label_ovf);
2838
2839                 if (!ARCH_HAS_FP_GP_MOV) {
2840                         g(gen_frame_store_raw(ctx, OP_SIZE_4, slot_r, 0, FR_SCRATCH_3));
2841                         g(gen_frame_load_raw(ctx, OP_SIZE_4, sign_x, slot_r, 0, false, target));
2842                 } else {
2843                         g(gen_mov(ctx, OP_SIZE_4, target, FR_SCRATCH_3));
2844                 }
2845
2846                 if (fp_alu == FP_COND_NE) {
2847                         g(gen_imm(ctx, 0, IMM_PURPOSE_CMP, OP_SIZE_NATIVE));
2848                         gen_insn(INSN_CMP_DEST_REG, OP_SIZE_NATIVE, COND_E, 0);
2849                         gen_one(target);
2850                         gen_one(target);
2851                         gen_imm_offset();
2852                 } else {
2853                         g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SHR, target, target, 30, 0));
2854                 }
2855
2856                 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
2857
2858                 return true;
2859 #elif defined(ARCH_IA64)
2860                 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_P, 0);
2861                 gen_one(R_CMP_RESULT);
2862                 gen_one(reg1);
2863                 gen_one(reg2);
2864
2865                 gen_insn(INSN_JMP_REG, OP_SIZE_NATIVE, COND_NE, 0);
2866                 gen_one(R_CMP_RESULT);
2867                 gen_four(label_ovf);
2868
2869                 gen_insn(INSN_FP_CMP_DEST_REG, op_size, fp_alu, 0);
2870                 gen_one(R_CMP_RESULT);
2871                 gen_one(reg1);
2872                 gen_one(reg2);
2873
2874                 g(gen_mov(ctx, OP_SIZE_NATIVE, target, R_CMP_RESULT));
2875
2876                 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
2877
2878                 return true;
2879 #elif defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS) || defined(ARCH_PARISC)
2880                 gen_insn(INSN_FP_CMP_COND, op_size, FP_COND_P, 1);
2881                 gen_one(reg1);
2882                 gen_one(reg2);
2883
2884                 gen_insn(INSN_JMP_FP_TEST, 0, FP_COND_P, 0);
2885                 gen_four(label_ovf);
2886
2887                 gen_insn(INSN_FP_CMP_COND, op_size, fp_alu, 1);
2888                 gen_one(reg1);
2889                 gen_one(reg2);
2890
2891                 gen_insn(INSN_FP_TEST_REG, OP_SIZE_NATIVE, fp_alu, 0);
2892                 gen_one(target);
2893
2894                 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
2895
2896                 return true;
2897 #elif defined(ARCH_RISCV64)
2898                 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_E, 0);
2899                 gen_one(R_SCRATCH_1);
2900                 gen_one(reg1);
2901                 gen_one(reg1);
2902
2903                 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_E, 0);
2904                 gen_one(R_SCRATCH_2);
2905                 gen_one(reg2);
2906                 gen_one(reg2);
2907
2908                 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_AND, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_2, 0));
2909
2910                 g(gen_jmp_on_zero(ctx, OP_SIZE_NATIVE, R_SCRATCH_1, COND_E, label_ovf));
2911
2912                 gen_insn(INSN_FP_CMP_DEST_REG, op_size, fp_alu == FP_COND_NE ? FP_COND_E : fp_alu, 0);
2913                 gen_one(target);
2914                 gen_one(reg1);
2915                 gen_one(reg2);
2916
2917                 if (fp_alu == FP_COND_NE) {
2918                         g(gen_imm(ctx, 1, IMM_PURPOSE_XOR, OP_SIZE_NATIVE));
2919                         gen_insn(INSN_ALU, OP_SIZE_NATIVE, ALU_XOR, ALU_WRITES_FLAGS(OP_SIZE_NATIVE, ALU_XOR, false, is_imm(), ctx->const_imm));
2920                         gen_one(target);
2921                         gen_one(target);
2922                         gen_imm_offset();
2923                 }
2924
2925                 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
2926                 return true;
2927 #else
2928                 gen_insn(INSN_FP_CMP, op_size, 0, 1);
2929                 gen_one(reg1);
2930                 gen_one(reg2);
2931 #if defined(ARCH_ARM32)
2932                 gen_insn(INSN_FP_TO_INT_FLAGS, 0, 0, 1);
2933 #endif
2934                 gen_insn(INSN_JMP_COND, op_size, FP_COND_P, 0);
2935                 gen_four(label_ovf);
2936                 g(gen_frame_set_cond(ctx, op_size, false, fp_alu, slot_r));
2937                 return true;
2938 #endif
2939         }
2940 #ifdef SUPPORTED_FP_X87
2941         if ((SUPPORTED_FP_X87 >> real_type) & 1) {
2942                 if (likely(cpu_test_feature(CPU_FEATURE_cmov))) {
2943                         g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_2));
2944                         g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
2945                         gen_insn(INSN_X87_FCOMIP, op_size, 0, 0);
2946                         gen_one(R_ST1);
2947                         gen_insn(INSN_X87_FSTP, op_size, 0, 0);
2948                         gen_one(R_ST0);
2949                         gen_insn(INSN_JMP_COND, op_size, COND_P, 0);
2950                         gen_four(label_ovf);
2951                         g(gen_frame_set_cond(ctx, op_size, false, fp_alu & 0xf, slot_r));
2952                         return true;
2953                 }
2954
2955                 if (real_type != 3) {
2956                         g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
2957                         g(gen_frame_load_x87(ctx, INSN_X87_FCOMP, op_size, 0, slot_2));
2958                 } else {
2959                         g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_2));
2960                         g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
2961                         gen_insn(INSN_X87_FCOMPP, op_size, 0, 0);
2962                 }
2963
2964                 gen_insn(INSN_X87_FNSTSW, 0, 0, 0);
2965                 gen_one(R_AX);
2966                 gen_one(R_AX);
2967
2968                 gen_insn(INSN_TEST, OP_SIZE_2, 0, 1);
2969                 gen_one(R_AX);
2970                 gen_one(ARG_IMM);
2971                 gen_eight(0x0400);
2972
2973                 gen_insn(INSN_JMP_COND, OP_SIZE_2, COND_NE, 0);
2974                 gen_four(label_ovf);
2975
2976                 switch (fp_alu) {
2977                         case FP_COND_E:
2978                                 gen_insn(INSN_TEST, OP_SIZE_2, 0, 1);
2979                                 gen_one(R_AX);
2980                                 gen_one(ARG_IMM);
2981                                 gen_eight(0x4000);
2982                                 g(gen_frame_set_cond(ctx, OP_SIZE_2, false, COND_NE, slot_r));
2983                                 break;
2984                         case FP_COND_NE:
2985                                 gen_insn(INSN_TEST, OP_SIZE_2, 0, 1);
2986                                 gen_one(R_AX);
2987                                 gen_one(ARG_IMM);
2988                                 gen_eight(0x4000);
2989                                 g(gen_frame_set_cond(ctx, OP_SIZE_2, false, COND_E, slot_r));
2990                                 break;
2991                         case FP_COND_B:
2992                                 gen_insn(INSN_TEST, OP_SIZE_2, 0, 1);
2993                                 gen_one(R_AX);
2994                                 gen_one(ARG_IMM);
2995                                 gen_eight(0x0100);
2996                                 g(gen_frame_set_cond(ctx, OP_SIZE_2, false, COND_NE, slot_r));
2997                                 break;
2998                         case FP_COND_BE:
2999                                 gen_insn(INSN_TEST, OP_SIZE_2, 0, 1);
3000                                 gen_one(R_AX);
3001                                 gen_one(ARG_IMM);
3002                                 gen_eight(0x4100);
3003                                 g(gen_frame_set_cond(ctx, OP_SIZE_2, false, COND_NE, slot_r));
3004                                 break;
3005                         default:
3006                                 internal(file_line, "gen_fp_alu: invalid condition %u", fp_alu);
3007                 }
3008                 return true;
3009         }
3010 #endif
3011 #ifdef SUPPORTED_FP_HALF_CVT
3012         if ((SUPPORTED_FP_HALF_CVT >> real_type) & 1) {
3013                 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, &reg1));
3014                 g(gen_frame_get(ctx, op_size, zero_x, slot_2, FR_SCRATCH_2, &reg2));
3015                 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
3016                 gen_one(FR_SCRATCH_1);
3017                 gen_one(reg1);
3018                 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
3019                 gen_one(FR_SCRATCH_2);
3020                 gen_one(reg2);
3021                 gen_insn(INSN_FP_CMP, OP_SIZE_4, 0, 1);
3022                 gen_one(FR_SCRATCH_1);
3023                 gen_one(FR_SCRATCH_2);
3024 #if defined(ARCH_ARM32)
3025                 gen_insn(INSN_FP_TO_INT_FLAGS, 0, 0, 1);
3026 #endif
3027                 gen_insn(INSN_JMP_COND, op_size, FP_COND_P, 0);
3028                 gen_four(label_ovf);
3029                 g(gen_frame_set_cond(ctx, op_size, false, fp_alu, slot_r));
3030                 return true;
3031         }
3032 #endif
3033
3034 do_upcall:
3035         return gen_alu_typed_upcall(ctx, upc, real_type, slot_1, slot_2, slot_r, label_ovf);
3036 }
3037
3038 #define OP_IS_ROUND(alu)        ((alu) == FP_ALU1_ROUND || (alu) == FP_ALU1_FLOOR || (alu) == FP_ALU1_CEIL || (alu) == FP_ALU1_TRUNC)
3039
3040 static bool attr_w gen_fp_alu1(struct codegen_context *ctx, unsigned real_type, unsigned op, uint32_t label_ovf, frame_t slot_1, frame_t slot_r)
3041 {
3042         unsigned attr_unused fp_alu;
3043         size_t upc;
3044         unsigned attr_unused op_size = real_type_to_op_size(real_type);
3045         unsigned reg1, target;
3046         switch (op) {
3047                 case OPCODE_REAL_OP_neg:
3048                 case OPCODE_REAL_OP_neg_alt1:
3049                 case OPCODE_REAL_OP_neg_alt2: fp_alu = FP_ALU1_NEG; upc = offsetof(struct cg_upcall_vector_s, REAL_unary_neg_real16_t); label_ovf = 0; goto do_alu;
3050                 case OPCODE_REAL_OP_sqrt:
3051                 case OPCODE_REAL_OP_sqrt_alt1:
3052                 case OPCODE_REAL_OP_sqrt_alt2: fp_alu = FP_ALU1_SQRT; upc = offsetof(struct cg_upcall_vector_s, REAL_unary_sqrt_real16_t); label_ovf = 0; goto do_alu;
3053                 case OPCODE_REAL_OP_round: fp_alu = FP_ALU1_ROUND; upc = offsetof(struct cg_upcall_vector_s, REAL_unary_round_real16_t); label_ovf = 0; goto do_alu;
3054                 case OPCODE_REAL_OP_floor: fp_alu = FP_ALU1_FLOOR; upc = offsetof(struct cg_upcall_vector_s, REAL_unary_floor_real16_t); label_ovf = 0; goto do_alu;
3055                 case OPCODE_REAL_OP_ceil: fp_alu = FP_ALU1_CEIL; upc = offsetof(struct cg_upcall_vector_s, REAL_unary_ceil_real16_t); label_ovf = 0; goto do_alu;
3056                 case OPCODE_REAL_OP_trunc: fp_alu = FP_ALU1_TRUNC; upc = offsetof(struct cg_upcall_vector_s, REAL_unary_trunc_real16_t); label_ovf = 0; goto do_alu;
3057                 case OPCODE_REAL_OP_to_int:
3058                 case OPCODE_REAL_OP_to_int_alt1:
3059                 case OPCODE_REAL_OP_to_int_alt2: upc = offsetof(struct cg_upcall_vector_s, REAL_unary_to_int_real16_t); goto do_to_int;
3060                 case OPCODE_REAL_OP_from_int:
3061                 case OPCODE_REAL_OP_from_int_alt1:
3062                 case OPCODE_REAL_OP_from_int_alt2: upc = offsetof(struct cg_upcall_vector_s, REAL_unary_from_int_real16_t); label_ovf = 0; goto do_from_int;
3063                 case OPCODE_REAL_OP_is_exception:
3064                 case OPCODE_REAL_OP_is_exception_alt1:
3065                 case OPCODE_REAL_OP_is_exception_alt2: upc = offsetof(struct cg_upcall_vector_s, REAL_unary_is_exception_real16_t); label_ovf = 0; goto do_is_exception;
3066                 default: upc = offsetof(struct cg_upcall_vector_s, REAL_unary_cbrt_real16_t) + (op - OPCODE_REAL_OP_cbrt) * TYPE_REAL_N * sizeof(void (*)(void)); label_ovf = 0; goto do_upcall;
3067         }
3068
3069 do_alu:
3070         if ((SUPPORTED_FP >> real_type) & 1 && (
3071 #if defined(ARCH_ALPHA)
3072                 fp_alu == FP_ALU1_NEG ||
3073                 (fp_alu == FP_ALU1_SQRT && cpu_test_feature(CPU_FEATURE_fix)) ||
3074 #elif defined(ARCH_ARM32)
3075                 fp_alu == FP_ALU1_NEG ||
3076                 fp_alu == FP_ALU1_SQRT ||
3077 #elif defined(ARCH_ARM64)
3078                 true ||
3079 #elif defined(ARCH_IA64)
3080                 fp_alu == FP_ALU1_NEG ||
3081 #elif defined(ARCH_LOONGARCH64)
3082                 fp_alu == FP_ALU1_NEG ||
3083                 fp_alu == FP_ALU1_SQRT ||
3084                 fp_alu == FP_ALU1_ROUND ||
3085 #elif defined(ARCH_MIPS)
3086                 fp_alu == FP_ALU1_NEG ||
3087                 (fp_alu == FP_ALU1_SQRT && MIPS_HAS_SQRT) ||
3088 #elif defined(ARCH_PARISC)
3089                 (fp_alu == FP_ALU1_NEG && PA_20) ||
3090                 fp_alu == FP_ALU1_SQRT ||
3091 #elif defined(ARCH_POWER)
3092                 fp_alu == FP_ALU1_NEG ||
3093                 (fp_alu == FP_ALU1_SQRT && cpu_test_feature(CPU_FEATURE_p2) && real_type != 4) ||
3094 #elif defined(ARCH_S390)
3095                 true ||
3096 #elif defined(ARCH_SPARC)
3097                 fp_alu == FP_ALU1_NEG ||
3098                 fp_alu == FP_ALU1_SQRT ||
3099 #elif defined(ARCH_RISCV64)
3100                 fp_alu == FP_ALU1_NEG ||
3101                 fp_alu == FP_ALU1_SQRT ||
3102 #elif defined(ARCH_X86)
3103                 fp_alu == FP_ALU1_SQRT ||
3104                 (OP_IS_ROUND(fp_alu) && cpu_test_feature(CPU_FEATURE_sse41)) ||
3105 #endif
3106                 false)) {
3107 #if defined(ARCH_S390)
3108                 if (op_size <= OP_SIZE_8 && (size_t)slot_1 * slot_size < 4096 && fp_alu == FP_ALU1_SQRT) {
3109                         target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_1);
3110                         if (slot_is_register(ctx, slot_1)) {
3111                                 gen_insn(INSN_FP_ALU1, op_size, fp_alu, 0);
3112                                 gen_one(target);
3113                                 gen_one(ctx->registers[slot_1]);
3114                                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
3115                         } else {
3116                                 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size, IMM_PURPOSE_VLDR_VSTR_OFFSET, op_size));
3117                                 gen_insn(INSN_FP_ALU1, op_size, fp_alu, 0);
3118                                 gen_one(target);
3119                                 gen_address_offset();
3120                                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
3121                         }
3122                         return true;
3123                 }
3124 #endif
3125                 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, &reg1));
3126                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_2);
3127                 gen_insn(INSN_FP_ALU1, op_size, fp_alu, 0);
3128                 gen_one(target);
3129                 gen_one(reg1);
3130                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
3131                 return true;
3132         }
3133 #ifdef SUPPORTED_FP_X87
3134         if ((SUPPORTED_FP_X87 >> real_type) & 1) {
3135                 if (fp_alu == FP_ALU1_NEG) {
3136                         g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
3137                         gen_insn(INSN_X87_FCHS, op_size, 0, 0);
3138                         g(gen_frame_store_x87(ctx, INSN_X87_FSTP, op_size, slot_r));
3139                         return true;
3140                 } else if (fp_alu == FP_ALU1_SQRT) {
3141                         g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
3142                         gen_insn(INSN_X87_FSQRT, op_size, 0, 0);
3143                         g(gen_frame_store_x87(ctx, INSN_X87_FSTP, op_size, slot_r));
3144                         return true;
3145                 } else if (fp_alu == FP_ALU1_ROUND) {
3146                         g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
3147                         gen_insn(INSN_X87_FRNDINT, op_size, 0, 0);
3148                         g(gen_frame_store_x87(ctx, INSN_X87_FSTP, op_size, slot_r));
3149                         return true;
3150                 }
3151         }
3152 #endif
3153 #ifdef SUPPORTED_FP_HALF_CVT
3154         if ((SUPPORTED_FP_HALF_CVT >> real_type) & 1 && (
3155 #if defined(ARCH_ARM32)
3156                 fp_alu == FP_ALU1_NEG ||
3157                 fp_alu == FP_ALU1_SQRT ||
3158 #elif defined(ARCH_ARM64)
3159                 true ||
3160 #elif defined(ARCH_X86)
3161                 fp_alu == FP_ALU1_SQRT ||
3162                 (OP_IS_ROUND(fp_alu) && cpu_test_feature(CPU_FEATURE_sse41)) ||
3163 #endif
3164                 false)) {
3165                 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, &reg1));
3166                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_1);
3167                 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
3168                 gen_one(target);
3169                 gen_one(reg1);
3170                 gen_insn(INSN_FP_ALU1, OP_SIZE_4, fp_alu, 0);
3171                 gen_one(target);
3172                 gen_one(target);
3173                 gen_insn(INSN_FP_CVT, OP_SIZE_4, op_size, 0);
3174                 gen_one(target);
3175                 gen_one(target);
3176                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
3177                 return true;
3178         }
3179 #endif
3180         goto do_upcall;
3181
3182 do_to_int:
3183         if ((SUPPORTED_FP >> real_type) & 1
3184 #if defined(ARCH_ALPHA)
3185                 && OS_SUPPORTS_TRAPS
3186 #endif
3187 #if defined(ARCH_MIPS)
3188                 && MIPS_HAS_TRUNC
3189 #endif
3190         ) {
3191                 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, &reg1));
3192                 goto do_cvt_to_int;
3193 do_cvt_to_int:
3194                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
3195 #if defined(ARCH_X86)
3196                 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_TO_INT32 : INSN_FP_TO_INT64, op_size, 0, 0);
3197                 gen_one(target);
3198                 gen_one(reg1);
3199
3200                 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, OP_SIZE_INT, target, sign_bit(uint_default_t), COND_E, label_ovf));
3201
3202                 g(gen_frame_store(ctx, OP_SIZE_INT, slot_r, 0, target));
3203                 return true;
3204 #endif
3205 #if defined(ARCH_ARM) || defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS)
3206 #if defined(ARCH_ARM)
3207                 gen_insn(INSN_FP_CMP, op_size, 0, 1);
3208                 gen_one(reg1);
3209                 gen_one(reg1);
3210 #if defined(ARCH_ARM32)
3211                 gen_insn(INSN_FP_TO_INT_FLAGS, 0, 0, 1);
3212 #endif
3213                 gen_insn(INSN_JMP_COND, op_size, FP_COND_P, 0);
3214                 gen_four(label_ovf);
3215 #else
3216                 gen_insn(INSN_FP_CMP_COND, op_size, FP_COND_P, 1);
3217                 gen_one(reg1);
3218                 gen_one(reg1);
3219
3220                 gen_insn(INSN_JMP_FP_TEST, 0, FP_COND_P, 0);
3221                 gen_four(label_ovf);
3222 #endif
3223 #if defined(ARCH_ARM32) || defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS)
3224                 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_TO_INT32 : INSN_FP_TO_INT64, op_size, 0, 0);
3225                 gen_one(FR_SCRATCH_1);
3226                 gen_one(reg1);
3227
3228                 g(gen_mov(ctx, OP_SIZE_INT, target, FR_SCRATCH_1));
3229 #else
3230                 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_TO_INT32 : INSN_FP_TO_INT64, op_size, 0, 0);
3231                 gen_one(target);
3232                 gen_one(reg1);
3233 #endif
3234                 g(gen_imm(ctx, (int_default_t)(sign_bit(uint_default_t) + 1), IMM_PURPOSE_ADD, OP_SIZE_INT));
3235                 gen_insn(INSN_ALU, OP_SIZE_INT, ALU_ADD, ALU_WRITES_FLAGS(OP_SIZE_INT, ALU_ADD, false, is_imm(), ctx->const_imm));
3236                 gen_one(R_SCRATCH_2);
3237                 gen_one(target);
3238                 gen_imm_offset();
3239
3240                 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, OP_SIZE_INT, R_SCRATCH_2, 1, COND_BE, label_ovf));
3241
3242                 g(gen_frame_store(ctx, OP_SIZE_INT, slot_r, 0, target));
3243                 return true;
3244 #endif
3245 #if defined(ARCH_IA64)
3246                 gen_insn(INSN_FP_TO_INT64, op_size, 0, 0);
3247                 gen_one(FR_SCRATCH_1);
3248                 gen_one(reg1);
3249
3250                 g(gen_mov(ctx, OP_SIZE_NATIVE, target, FR_SCRATCH_1));
3251
3252                 if (OP_SIZE_INT == OP_SIZE_4) {
3253                         g(gen_extend(ctx, OP_SIZE_4, sign_x, R_SCRATCH_2, target));
3254                         g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, target, R_SCRATCH_2, COND_NE, label_ovf));
3255                 } else {
3256                         g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, target, sign_bit(uint64_t), COND_E, label_ovf));
3257                 }
3258
3259                 g(gen_frame_store(ctx, OP_SIZE_INT, slot_r, 0, target));
3260                 return true;
3261 #endif
3262 #if defined(ARCH_PARISC) || defined(ARCH_POWER) || defined(ARCH_SPARC)
3263 #if defined(ARCH_POWER)
3264                 if (!cpu_test_feature(CPU_FEATURE_ppc))
3265                         goto do_upcall;
3266                 if (OP_SIZE_INT == OP_SIZE_4)
3267                         goto do_upcall;
3268 #endif
3269                 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_TO_INT32 : INSN_FP_TO_INT64, op_size, 0, 0);
3270                 gen_one(FR_SCRATCH_1);
3271                 gen_one(reg1);
3272
3273                 g(gen_frame_store_raw(ctx, OP_SIZE_INT, slot_r, 0, FR_SCRATCH_1));
3274                 if (slot_is_register(ctx, slot_r))
3275                         g(unspill(ctx, slot_r));
3276                 g(gen_frame_load(ctx, OP_SIZE_INT, garbage, slot_r, 0, false, target));
3277
3278                 g(gen_imm(ctx, sign_bit(uint_default_t) + 1, IMM_PURPOSE_ADD, OP_SIZE_INT));
3279                 gen_insn(INSN_ALU, i_size(OP_SIZE_INT), ALU_ADD, ALU_WRITES_FLAGS(i_size(OP_SIZE_INT), ALU_ADD, false, is_imm(), ctx->const_imm));
3280                 gen_one(R_SCRATCH_2);
3281                 gen_one(target);
3282                 gen_imm_offset();
3283
3284                 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, OP_SIZE_INT, R_SCRATCH_2, 1, COND_BE, label_ovf));
3285
3286                 return true;
3287 #endif
3288 #if defined(ARCH_ALPHA)
3289                 gen_insn(INSN_FP_TO_INT64_TRAP, op_size, 0, 0);
3290                 gen_one(FR_SCRATCH_2);
3291                 gen_one(reg1);
3292                 gen_four(label_ovf);
3293
3294                 if (OP_SIZE_INT == OP_SIZE_4) {
3295                         gen_insn(INSN_FP_INT64_TO_INT32_TRAP, 0, 0, 0);
3296                         gen_one(FR_SCRATCH_3);
3297                         gen_one(FR_SCRATCH_2);
3298                         gen_four(label_ovf);
3299                         g(gen_frame_store_raw(ctx, OP_SIZE_INT, slot_r, 0, FR_SCRATCH_3));
3300                 } else {
3301                         g(gen_frame_store_raw(ctx, OP_SIZE_INT, slot_r, 0, FR_SCRATCH_2));
3302                 }
3303                 if (slot_is_register(ctx, slot_r))
3304                         g(unspill(ctx, slot_r));
3305                 return true;
3306 #endif
3307 #if defined(ARCH_S390)
3308                 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_TO_INT32 : INSN_FP_TO_INT64, op_size, 0, 1);
3309                 gen_one(target);
3310                 gen_one(reg1);
3311
3312                 gen_insn(INSN_JMP_COND, op_size, FP_COND_P, 0);
3313                 gen_four(label_ovf);
3314
3315                 g(gen_frame_store(ctx, OP_SIZE_INT, slot_r, 0, target));
3316                 return true;
3317 #endif
3318 #if defined(ARCH_RISCV64)
3319                 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_TO_INT32 : INSN_FP_TO_INT64, op_size, 0, 0);
3320                 gen_one(target);
3321                 gen_one(reg1);
3322
3323                 g(gen_load_constant(ctx, R_SCRATCH_2, sign_bit(int_default_t)));
3324
3325                 g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, target, R_SCRATCH_2, COND_E, label_ovf));
3326
3327                 g(gen_imm(ctx, -1, IMM_PURPOSE_XOR, i_size(size)));
3328                 gen_insn(INSN_ALU, OP_SIZE_NATIVE, ALU_XOR, ALU_WRITES_FLAGS(OP_SIZE_NATIVE, ALU_XOR, false, is_imm(), ctx->const_imm));
3329                 gen_one(R_SCRATCH_2);
3330                 gen_one(R_SCRATCH_2);
3331                 gen_imm_offset();
3332
3333                 g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, target, R_SCRATCH_2, COND_E, label_ovf));
3334
3335                 g(gen_frame_store(ctx, OP_SIZE_INT, slot_r, 0, target));
3336                 return true;
3337 #endif
3338         }
3339 #ifdef SUPPORTED_FP_X87
3340         if ((SUPPORTED_FP_X87 >> real_type) & 1) {
3341                 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
3342
3343                 if (likely(cpu_test_feature(CPU_FEATURE_sse3))) {
3344                         g(gen_frame_store_x87(ctx, INSN_X87_FISTTP, OP_SIZE_INT, slot_r));
3345                 } else {
3346                         gen_insn(INSN_PUSH, OP_SIZE_NATIVE, 0, 0);
3347                         gen_one(ARG_IMM);
3348                         gen_eight(0x0f7f);
3349
3350                         gen_insn(INSN_X87_FLDCW, 0, 0, 0);
3351                         gen_one(ARG_ADDRESS_1);
3352                         gen_one(R_SP);
3353                         gen_eight(0);
3354
3355                         g(gen_frame_store_x87(ctx, INSN_X87_FISTP, OP_SIZE_INT, slot_r));
3356
3357                         gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
3358                         gen_one(ARG_ADDRESS_1);
3359                         gen_one(R_SP);
3360                         gen_eight(0);
3361                         gen_one(ARG_IMM);
3362                         gen_eight(0x037f);
3363
3364                         gen_insn(INSN_X87_FLDCW, 0, 0, 0);
3365                         gen_one(ARG_ADDRESS_1);
3366                         gen_one(R_SP);
3367                         gen_eight(0);
3368
3369                         gen_insn(INSN_ALU, i_size(OP_SIZE_ADDRESS), ALU_ADD, 1);
3370                         gen_one(R_SP);
3371                         gen_one(R_SP);
3372                         gen_one(ARG_IMM);
3373                         gen_eight(1 << OP_SIZE_NATIVE);
3374                 }
3375                 if (slot_is_register(ctx, slot_r))
3376                         g(unspill(ctx, slot_r));
3377                 g(gen_frame_load(ctx, OP_SIZE_INT, garbage, slot_r, 0, false, R_SCRATCH_1));
3378
3379                 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, OP_SIZE_INT, R_SCRATCH_1, sign_bit(int_default_t), COND_E, label_ovf));
3380
3381                 return true;
3382         }
3383 #endif
3384 #ifdef SUPPORTED_FP_HALF_CVT
3385         if ((SUPPORTED_FP_HALF_CVT >> real_type) & 1) {
3386                 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, &reg1));
3387                 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
3388                 gen_one(FR_SCRATCH_1);
3389                 gen_one(reg1);
3390                 reg1 = FR_SCRATCH_1;
3391                 real_type = 1;
3392                 op_size = real_type_to_op_size(real_type);
3393                 goto do_cvt_to_int;
3394         }
3395 #endif
3396         goto do_upcall;
3397
3398 do_from_int:
3399         if ((SUPPORTED_FP >> real_type) & 1) {
3400 #if defined(ARCH_ALPHA) || defined(ARCH_ARM32) || defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS) || defined(ARCH_PARISC) || defined(ARCH_POWER) || defined(ARCH_SPARC)
3401                 int int_op_size = OP_SIZE_INT;
3402 #if defined(ARCH_POWER)
3403                 if (int_op_size == OP_SIZE_4)
3404                         goto do_upcall;
3405                 if (op_size == OP_SIZE_4 && !cpu_test_feature(CPU_FEATURE_v206))
3406                         goto do_upcall;
3407                 if (op_size == OP_SIZE_8 && !cpu_test_feature(CPU_FEATURE_ppc))
3408                         goto do_upcall;
3409 #endif
3410                 if (slot_is_register(ctx, slot_1))
3411                         g(spill(ctx, slot_1));
3412                 g(gen_frame_load_raw(ctx, int_op_size, zero_x, slot_1, 0, false, FR_SCRATCH_1));
3413                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_2);
3414 #if defined(ARCH_ALPHA)
3415                 if (OP_SIZE_INT == OP_SIZE_4) {
3416                         gen_insn(INSN_MOVSX, OP_SIZE_4, 0, 0);
3417                         gen_one(FR_SCRATCH_1);
3418                         gen_one(FR_SCRATCH_1);
3419
3420                         int_op_size = OP_SIZE_8;
3421                 }
3422 #endif
3423                 gen_insn(int_op_size == OP_SIZE_4 ? INSN_FP_FROM_INT32 : INSN_FP_FROM_INT64, op_size, 0, 0);
3424                 gen_one(target);
3425                 gen_one(FR_SCRATCH_1);
3426
3427                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
3428                 return true;
3429 #elif defined(ARCH_IA64)
3430                 g(gen_frame_get(ctx, OP_SIZE_INT, sign_x, slot_1, R_SCRATCH_1, &reg1));
3431                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_1);
3432
3433                 g(gen_mov(ctx, OP_SIZE_NATIVE, target, reg1));
3434
3435                 gen_insn(INSN_FP_FROM_INT64, op_size, 0, 0);
3436                 gen_one(target);
3437                 gen_one(target);
3438
3439                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
3440                 return true;
3441 #else
3442                 g(gen_frame_get(ctx, OP_SIZE_INT, garbage, slot_1, R_SCRATCH_1, &reg1));
3443                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_1);
3444
3445                 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_FROM_INT32 : INSN_FP_FROM_INT64, op_size, 0, 0);
3446                 gen_one(target);
3447                 gen_one(reg1);
3448
3449                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
3450                 return true;
3451 #endif
3452         }
3453 #ifdef SUPPORTED_FP_X87
3454         if ((SUPPORTED_FP_X87 >> real_type) & 1) {
3455                 if (slot_is_register(ctx, slot_1))
3456                         g(spill(ctx, slot_1));
3457                 g(gen_frame_load_x87(ctx, INSN_X87_FILD, OP_SIZE_INT, 0, slot_1));
3458                 g(gen_frame_store_x87(ctx, INSN_X87_FSTP, op_size, slot_r));
3459                 return true;
3460         }
3461 #endif
3462 #ifdef SUPPORTED_FP_HALF_CVT
3463         if ((SUPPORTED_FP_HALF_CVT >> real_type) & 1) {
3464                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_1);
3465 #if defined(ARCH_ARM32)
3466                 g(gen_frame_get(ctx, OP_SIZE_INT, zero_x, slot_1, FR_SCRATCH_1, &reg1));
3467
3468                 gen_insn(INSN_FP_FROM_INT32, OP_SIZE_4, 0, 0);
3469                 gen_one(target);
3470                 gen_one(reg1);
3471 #else
3472                 g(gen_frame_get(ctx, OP_SIZE_INT, garbage, slot_1, R_SCRATCH_1, &reg1));
3473                 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_FROM_INT32 : INSN_FP_FROM_INT64, OP_SIZE_4, 0, 0);
3474                 gen_one(target);
3475                 gen_one(reg1);
3476 #endif
3477                 gen_insn(INSN_FP_CVT, OP_SIZE_4, op_size, 0);
3478                 gen_one(target);
3479                 gen_one(target);
3480                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
3481                 return true;
3482         }
3483 #endif
3484         goto do_upcall;
3485
3486 do_is_exception:
3487         if ((SUPPORTED_FP >> real_type) & 1) {
3488                 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, &reg1));
3489                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
3490 #if defined(ARCH_ALPHA)
3491                 gen_insn(INSN_FP_CMP_UNORDERED_DEST_REG, op_size, 0, 0);
3492                 gen_one(FR_SCRATCH_2);
3493                 gen_one(FR_SCRATCH_1);
3494                 gen_one(reg1);
3495
3496                 if (!cpu_test_feature(CPU_FEATURE_fix)) {
3497                         g(gen_frame_store_raw(ctx, OP_SIZE_4, slot_r, 0, FR_SCRATCH_2));
3498                         g(gen_frame_load_raw(ctx, OP_SIZE_4, sign_x, slot_r, 0, false, target));
3499                 } else {
3500                         g(gen_mov(ctx, OP_SIZE_4, target, FR_SCRATCH_2));
3501                 }
3502
3503                 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SHR, target, target, 30, 0));
3504
3505                 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
3506
3507                 return true;
3508 #elif defined(ARCH_IA64)
3509                 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_P, 0);
3510                 gen_one(R_CMP_RESULT);
3511                 gen_one(reg1);
3512                 gen_one(reg1);
3513
3514                 g(gen_mov(ctx, OP_SIZE_NATIVE, target, R_CMP_RESULT));
3515
3516                 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
3517 #elif defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS) || defined(ARCH_PARISC)
3518                 gen_insn(INSN_FP_CMP_COND, op_size, FP_COND_P, 1);
3519                 gen_one(reg1);
3520                 gen_one(reg1);
3521
3522                 gen_insn(INSN_FP_TEST_REG, OP_SIZE_NATIVE, FP_COND_P, 0);
3523                 gen_one(target);
3524
3525                 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
3526 #elif defined(ARCH_RISCV64)
3527                 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_E, 0);
3528                 gen_one(target);
3529                 gen_one(reg1);
3530                 gen_one(reg1);
3531
3532                 g(gen_imm(ctx, 1, IMM_PURPOSE_XOR, OP_SIZE_NATIVE));
3533                 gen_insn(INSN_ALU, OP_SIZE_NATIVE, ALU_XOR, ALU_WRITES_FLAGS(OP_SIZE_NATIVE, ALU_XOR, false, is_imm(), ctx->const_imm));
3534                 gen_one(target);
3535                 gen_one(target);
3536                 gen_imm_offset();
3537
3538                 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
3539 #else
3540                 gen_insn(INSN_FP_CMP, op_size, 0, 1);
3541                 gen_one(reg1);
3542                 gen_one(reg1);
3543 #if defined(ARCH_ARM32)
3544                 gen_insn(INSN_FP_TO_INT_FLAGS, 0, 0, 1);
3545 #endif
3546                 g(gen_frame_set_cond(ctx, op_size, false, FP_COND_P, slot_r));
3547 #endif
3548                 return true;
3549         }
3550 #ifdef SUPPORTED_FP_X87
3551         if ((SUPPORTED_FP_X87 >> real_type) & 1) {
3552                 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
3553                 if (likely(cpu_test_feature(CPU_FEATURE_cmov))) {
3554                         gen_insn(INSN_X87_FCOMIP, op_size, 0, 0);
3555                         gen_one(R_ST0);
3556
3557                         g(gen_frame_set_cond(ctx, op_size, false, COND_P, slot_r));
3558                         return true;
3559                 }
3560
3561                 gen_insn(INSN_X87_FCOMP, op_size, 0, 0);
3562                 gen_one(R_ST0);
3563
3564                 gen_insn(INSN_X87_FNSTSW, 0, 0, 0);
3565                 gen_one(R_AX);
3566                 gen_one(R_AX);
3567
3568                 gen_insn(INSN_TEST, OP_SIZE_2, 0, 1);
3569                 gen_one(R_AX);
3570                 gen_one(ARG_IMM);
3571                 gen_eight(0x0400);
3572
3573                 g(gen_frame_set_cond(ctx, op_size, false, COND_NE, slot_r));
3574
3575                 return true;
3576         }
3577 #endif
3578 #ifdef SUPPORTED_FP_HALF_CVT
3579         if ((SUPPORTED_FP_HALF_CVT >> real_type) & 1) {
3580                 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, &reg1));
3581                 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
3582                 gen_one(FR_SCRATCH_1);
3583                 gen_one(reg1);
3584                 gen_insn(INSN_FP_CMP, OP_SIZE_4, 0, 1);
3585                 gen_one(FR_SCRATCH_1);
3586                 gen_one(FR_SCRATCH_1);
3587 #if defined(ARCH_ARM32)
3588                 gen_insn(INSN_FP_TO_INT_FLAGS, 0, 0, 1);
3589 #endif
3590                 g(gen_frame_set_cond(ctx, op_size, false, FP_COND_P, slot_r));
3591                 return true;
3592         }
3593 #endif
3594
3595 do_upcall:
3596         g(gen_alu_typed_upcall(ctx, upc, real_type, slot_1, NO_FRAME_T, slot_r, label_ovf));
3597         return true;
3598 }
3599
3600 static bool attr_w gen_is_exception(struct codegen_context *ctx, frame_t slot_1, frame_t slot_r)
3601 {
3602         uint32_t no_ex_label, escape_label;
3603         const struct type *type = get_type_of_local(ctx, slot_1);
3604
3605         no_ex_label = alloc_label(ctx);
3606         if (unlikely(!no_ex_label))
3607                 return false;
3608         escape_label = alloc_escape_label(ctx);
3609         if (unlikely(!escape_label))
3610                 return false;
3611
3612         if (TYPE_IS_FLAT(type))
3613                 g(gen_test_1_jz_cached(ctx, slot_1, no_ex_label));
3614
3615         g(gen_frame_load(ctx, OP_SIZE_SLOT, zero_x, slot_1, 0, false, R_SCRATCH_1));
3616         g(gen_ptr_is_thunk(ctx, R_SCRATCH_1, slot_1, escape_label));
3617
3618         if (!TYPE_IS_FLAT(type)) {
3619                 g(gen_compare_da_tag(ctx, R_SCRATCH_1, DATA_TAG_flat, COND_E, escape_label, R_SCRATCH_1));
3620         }
3621
3622         gen_label(no_ex_label);
3623         g(gen_frame_clear(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r));
3624
3625         flag_set(ctx, slot_r, false);
3626
3627         return true;
3628 }
3629
3630 static bool attr_w gen_system_property(struct codegen_context *ctx, frame_t slot_1, frame_t slot_r)
3631 {
3632         uint32_t escape_label;
3633
3634         escape_label = alloc_escape_label(ctx);
3635         if (unlikely(!escape_label))
3636                 return false;
3637
3638         g(gen_test_1_cached(ctx, slot_1, escape_label));
3639
3640         g(gen_upcall_start(ctx, 1));
3641
3642         g(gen_frame_load(ctx, OP_SIZE_INT, garbage, slot_1, 0, false, R_ARG0));
3643         g(gen_upcall_argument(ctx, 0));
3644
3645         g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_ipret_system_property), 1));
3646
3647         g(gen_frame_store(ctx, OP_SIZE_INT, slot_r, 0, R_RET0));
3648
3649         flag_set(ctx, slot_1, false);
3650         flag_set(ctx, slot_r, false);
3651
3652         return true;
3653 }
3654
3655 static bool attr_w gen_alu_jmp(struct codegen_context *ctx, unsigned mode, unsigned op_size, unsigned op, frame_t slot_1, frame_t slot_2, int32_t jmp_offset, bool *failed)
3656 {
3657         unsigned alu;
3658         enum extend ex;
3659         unsigned reg1;
3660         unsigned attr_unused reg2;
3661 #if 0
3662         *failed = true; return true;
3663 #endif
3664         switch (mode) {
3665                 case MODE_FIXED: switch (op) {
3666                         case OPCODE_FIXED_OP_equal:             alu = COND_E; goto do_compare;
3667                         case OPCODE_FIXED_OP_not_equal:         alu = COND_NE; goto do_compare;
3668                         case OPCODE_FIXED_OP_less:              alu = COND_L; goto do_compare;
3669                         case OPCODE_FIXED_OP_less_equal:        alu = COND_LE; goto do_compare;
3670                         case OPCODE_FIXED_OP_greater:           alu = COND_G; goto do_compare;
3671                         case OPCODE_FIXED_OP_greater_equal:     alu = COND_GE; goto do_compare;
3672                         case OPCODE_FIXED_OP_uless:             alu = COND_B; goto do_compare;
3673                         case OPCODE_FIXED_OP_uless_equal:       alu = COND_BE; goto do_compare;
3674                         case OPCODE_FIXED_OP_ugreater:          alu = COND_A; goto do_compare;
3675                         case OPCODE_FIXED_OP_ugreater_equal:    alu = COND_AE; goto do_compare;
3676                         case OPCODE_FIXED_OP_bt:                *failed = true; return true;
3677                         default:                                internal(file_line, "gen_alu_jmp: unsupported fixed operation %u", op);
3678                 }
3679                 case MODE_INT: switch (op) {
3680                         case OPCODE_INT_OP_equal:               alu = COND_E; goto do_compare;
3681                         case OPCODE_INT_OP_not_equal:           alu = COND_NE; goto do_compare;
3682                         case OPCODE_INT_OP_less:                alu = COND_L; goto do_compare;
3683                         case OPCODE_INT_OP_less_equal:          alu = COND_LE; goto do_compare;
3684                         case OPCODE_INT_OP_greater:             alu = COND_G; goto do_compare;
3685                         case OPCODE_INT_OP_greater_equal:       alu = COND_GE; goto do_compare;
3686                         case OPCODE_INT_OP_bt:                  *failed = true; return true;
3687                         default:                                internal(file_line, "gen_alu_jmp: unsupported int operation %u", op);
3688                 }
3689                 case MODE_BOOL: switch (op) {
3690                         case OPCODE_BOOL_OP_and:                alu = ALU_AND; mode = MODE_FIXED; goto do_alu;
3691                         case OPCODE_BOOL_OP_or:                 alu = ALU_OR; mode = MODE_FIXED; goto do_alu;
3692                         case OPCODE_BOOL_OP_equal:              alu = COND_E; mode = MODE_FIXED; goto do_compare;
3693                         case OPCODE_BOOL_OP_not_equal:          alu = COND_NE; mode = MODE_FIXED; goto do_compare;
3694                         case OPCODE_BOOL_OP_less:               alu = COND_L; mode = MODE_FIXED; goto do_compare;
3695                         case OPCODE_BOOL_OP_less_equal:         alu = COND_LE; mode = MODE_FIXED; goto do_compare;
3696                         case OPCODE_BOOL_OP_greater:            alu = COND_G; mode = MODE_FIXED; goto do_compare;
3697                         case OPCODE_BOOL_OP_greater_equal:      alu = COND_GE; mode = MODE_FIXED; goto do_compare;
3698                         default:                                internal(file_line, "gen_alu_jmp: unsupported bool operation %u", op);
3699                 }
3700         }
3701         internal(file_line, "gen_alu_jmp: unsupported mode %u", mode);
3702 do_compare: {
3703                 bool attr_unused logical;
3704                 if (unlikely(op_size > OP_SIZE_NATIVE)) {
3705                         *failed = true;
3706                         return true;
3707                 }
3708                 if (slot_is_register(ctx, slot_2) && !slot_is_register(ctx, slot_1)) {
3709                         frame_t s = slot_1;
3710                         slot_1 = slot_2;
3711                         slot_2 = s;
3712                         switch (alu) {
3713                                 case COND_L:    alu = COND_G; break;
3714                                 case COND_LE:   alu = COND_GE; break;
3715                                 case COND_G:    alu = COND_L; break;
3716                                 case COND_GE:   alu = COND_LE; break;
3717                                 case COND_B:    alu = COND_A; break;
3718                                 case COND_BE:   alu = COND_AE; break;
3719                                 case COND_A:    alu = COND_B; break;
3720                                 case COND_AE:   alu = COND_BE; break;
3721                         }
3722                 }
3723                 ex = op_size == i_size_cmp(op_size) + (unsigned)zero ? garbage : alu == COND_L || alu == COND_LE || alu == COND_G || alu == COND_GE || ARCH_PREFERS_SX(op_size) ? sign_x : zero_x;
3724                 g(gen_frame_get(ctx, op_size, ex, slot_1, R_SCRATCH_1, &reg1));
3725                 if (ARCH_HAS_JMP_2REGS(alu)) {
3726                         g(gen_frame_get(ctx, op_size, ex, slot_2, R_SCRATCH_2, &reg2));
3727                         g(gen_jump(ctx, jmp_offset, i_size_cmp(op_size), alu ^ 1, reg1, reg2));
3728                         return true;
3729                 }
3730 #if ARCH_HAS_FLAGS
3731                 logical = COND_IS_LOGICAL(alu ^ 1);
3732                 g(gen_frame_load_cmp(ctx, op_size, logical, ex, false, slot_2, 0, false, reg1));
3733                 g(gen_jump(ctx, jmp_offset, op_size, alu ^ 1, -1U, -1U));
3734 #else
3735                 g(gen_frame_get(ctx, op_size, ex, slot_2, R_SCRATCH_2, &reg2));
3736                 g(gen_cmp_dest_reg(ctx, op_size, reg1, reg2, R_CMP_RESULT, 0, alu));
3737                 g(gen_jump(ctx, jmp_offset, OP_SIZE_NATIVE, COND_E, R_CMP_RESULT, -1U));
3738 #endif
3739                 return true;
3740         }
3741 do_alu: {
3742                 if (slot_is_register(ctx, slot_2) && !slot_is_register(ctx, slot_1)) {
3743                         frame_t s = slot_1;
3744                         slot_1 = slot_2;
3745                         slot_2 = s;
3746                 }
3747                 ex = op_size == i_size(op_size) + (unsigned)zero ? garbage : ARCH_PREFERS_SX(op_size) ? sign_x : zero_x;
3748                 g(gen_frame_get(ctx, op_size, ex, slot_1, R_SCRATCH_1, &reg1));
3749 #if defined(ARCH_X86)
3750                 if (alu == ALU_AND && !slot_is_register(ctx, slot_2)) {
3751                         g(gen_address(ctx, R_FRAME, (size_t)slot_2 * slot_size, IMM_PURPOSE_LDR_OFFSET, op_size));
3752                         gen_insn(INSN_TEST, i_size(op_size), 0, 1);
3753                         gen_one(reg1);
3754                         gen_address_offset();
3755                         g(gen_jump(ctx, jmp_offset, op_size, COND_E, -1U, -1U));
3756                         return true;
3757                 }
3758 #endif
3759                 g(gen_frame_get(ctx, op_size, ex, slot_2, R_SCRATCH_2, &reg2));
3760 #if ARCH_HAS_FLAGS && !defined(ARCH_S390)
3761                 if (alu == ALU_AND) {
3762                         gen_insn(INSN_TEST, i_size(op_size), 0, 1);
3763                         gen_one(reg1);
3764                         gen_one(reg2);
3765                         g(gen_jump(ctx, jmp_offset, op_size, COND_E, -1U, -1U));
3766                         return true;
3767                 }
3768 #endif
3769 #if defined(ARCH_ARM64)
3770                 if (alu == ALU_OR)
3771                         goto skip_flags;
3772 #endif
3773 #if ARCH_HAS_FLAGS
3774                 g(gen_3address_alu(ctx, i_size(op_size), alu, R_SCRATCH_1, reg1, reg2, 1));
3775                 g(gen_jump(ctx, jmp_offset, i_size(op_size), COND_E, -1U, -1U));
3776                 return true;
3777 #endif
3778                 goto skip_flags;
3779 skip_flags:
3780                 g(gen_3address_alu(ctx, i_size(op_size), alu, R_SCRATCH_1, reg1, reg2, 0));
3781                 g(gen_jump(ctx, jmp_offset, i_size(op_size), COND_E, R_SCRATCH_1, -1U));
3782                 return true;
3783         }
3784 }
3785
3786 static bool attr_w gen_fp_alu_jmp(struct codegen_context *ctx, unsigned real_type, unsigned op, uint32_t label_ovf, frame_t slot_1, frame_t slot_2, int32_t jmp_offset, bool *failed)
3787 {
3788         unsigned attr_unused fp_alu;
3789         unsigned attr_unused op_size = real_type_to_op_size(real_type);
3790         unsigned reg1, reg2;
3791         unsigned attr_unused target;
3792         switch (op) {
3793                 case OPCODE_REAL_OP_equal:
3794                 case OPCODE_REAL_OP_equal_alt1:
3795                 case OPCODE_REAL_OP_equal_alt2: fp_alu = FP_COND_E; goto do_cmp;
3796                 case OPCODE_REAL_OP_not_equal:
3797                 case OPCODE_REAL_OP_not_equal_alt1:
3798                 case OPCODE_REAL_OP_not_equal_alt2: fp_alu = FP_COND_NE; goto do_cmp;
3799                 case OPCODE_REAL_OP_less:
3800                 case OPCODE_REAL_OP_less_alt1:
3801                 case OPCODE_REAL_OP_less_alt2: fp_alu = FP_COND_B; goto do_cmp;
3802                 case OPCODE_REAL_OP_less_equal:
3803                 case OPCODE_REAL_OP_less_equal_alt1:
3804                 case OPCODE_REAL_OP_less_equal_alt2: fp_alu = FP_COND_BE; goto do_cmp;
3805                 case OPCODE_REAL_OP_greater:
3806                 case OPCODE_REAL_OP_greater_alt1:
3807                 case OPCODE_REAL_OP_greater_alt2: fp_alu = FP_COND_A; goto do_cmp;
3808                 case OPCODE_REAL_OP_greater_equal:
3809                 case OPCODE_REAL_OP_greater_equal_alt1:
3810                 case OPCODE_REAL_OP_greater_equal_alt2: fp_alu = FP_COND_AE; goto do_cmp;
3811                 default: internal(file_line, "gen_fp_alu_jmp: unsupported operation %u", op);
3812         }
3813
3814 do_cmp:
3815         if ((SUPPORTED_FP >> real_type) & 1
3816 #if defined(ARCH_ALPHA)
3817                 && OS_SUPPORTS_TRAPS && cpu_test_feature(CPU_FEATURE_fix)
3818 #endif
3819         ) {
3820                 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, &reg1));
3821                 g(gen_frame_get(ctx, op_size, zero_x, slot_2, FR_SCRATCH_2, &reg2));
3822                 target = R_SCRATCH_1;
3823 #if defined(ARCH_ALPHA)
3824                 gen_insn(INSN_FP_CMP_DEST_REG_TRAP, op_size, fp_alu == FP_COND_NE ? FP_COND_E : fp_alu, 0);
3825                 gen_one(FR_SCRATCH_3);
3826                 gen_one(reg1);
3827                 gen_one(reg2);
3828                 gen_four(label_ovf);
3829
3830                 g(gen_mov(ctx, OP_SIZE_4, target, FR_SCRATCH_3));
3831
3832                 if (fp_alu == FP_COND_NE) {
3833                         g(gen_jump(ctx, jmp_offset, OP_SIZE_NATIVE, COND_NE, target, -1U));
3834                 } else {
3835                         g(gen_jump(ctx, jmp_offset, OP_SIZE_NATIVE, COND_E, target, -1U));
3836                 }
3837
3838                 return true;
3839 #elif defined(ARCH_IA64)
3840                 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_P, 0);
3841                 gen_one(R_CMP_RESULT);
3842                 gen_one(reg1);
3843                 gen_one(reg2);
3844
3845                 gen_insn(INSN_JMP_REG, OP_SIZE_NATIVE, COND_NE, 0);
3846                 gen_one(R_CMP_RESULT);
3847                 gen_four(label_ovf);
3848
3849                 gen_insn(INSN_FP_CMP_DEST_REG, op_size, fp_alu, 0);
3850                 gen_one(R_CMP_RESULT);
3851                 gen_one(reg1);
3852                 gen_one(reg2);
3853
3854                 g(gen_jump(ctx, jmp_offset, OP_SIZE_NATIVE, COND_E, R_CMP_RESULT, -1U));
3855
3856                 return true;
3857 #elif defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS) || defined(ARCH_PARISC)
3858                 gen_insn(INSN_FP_CMP_COND, op_size, FP_COND_P, 1);
3859                 gen_one(reg1);
3860                 gen_one(reg2);
3861
3862                 gen_insn(INSN_JMP_FP_TEST, 0, FP_COND_P, 0);
3863                 gen_four(label_ovf);
3864
3865                 gen_insn(INSN_FP_CMP_COND, op_size, fp_alu ^ 1, 1);
3866                 gen_one(reg1);
3867                 gen_one(reg2);
3868
3869                 g(gen_jump(ctx, jmp_offset, OP_SIZE_NATIVE, fp_alu ^ 1, -1U, -1U));
3870
3871                 return true;
3872 #elif defined(ARCH_RISCV64)
3873                 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_E, 0);
3874                 gen_one(R_SCRATCH_1);
3875                 gen_one(reg1);
3876                 gen_one(reg1);
3877
3878                 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_E, 0);
3879                 gen_one(R_SCRATCH_2);
3880                 gen_one(reg2);
3881                 gen_one(reg2);
3882
3883                 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_AND, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_2, 0));
3884
3885                 g(gen_jmp_on_zero(ctx, OP_SIZE_NATIVE, R_SCRATCH_1, COND_E, label_ovf));
3886
3887                 gen_insn(INSN_FP_CMP_DEST_REG, op_size, fp_alu == FP_COND_NE ? FP_COND_E : fp_alu, 0);
3888                 gen_one(target);
3889                 gen_one(reg1);
3890                 gen_one(reg2);
3891
3892                 if (fp_alu == FP_COND_NE) {
3893                         g(gen_jump(ctx, jmp_offset, OP_SIZE_NATIVE, COND_NE, target, -1U));
3894                 } else {
3895                         g(gen_jump(ctx, jmp_offset, OP_SIZE_NATIVE, COND_E, target, -1U));
3896                 }
3897                 return true;
3898 #else
3899                 gen_insn(INSN_FP_CMP, op_size, 0, 1);
3900                 gen_one(reg1);
3901                 gen_one(reg2);
3902 #if defined(ARCH_ARM32)
3903                 gen_insn(INSN_FP_TO_INT_FLAGS, 0, 0, 1);
3904 #endif
3905                 gen_insn(INSN_JMP_COND, op_size, FP_COND_P, 0);
3906                 gen_four(label_ovf);
3907                 g(gen_jump(ctx, jmp_offset, op_size, fp_alu ^ 1, -1U, -1U));
3908                 return true;
3909 #endif
3910         }
3911         *failed = true;
3912         return true;
3913 }