cg-alu.inc

   1 /*
   2  * Copyright (C) 2024 Mikulas Patocka
   3  *
   4  * This file is part of Ajla.
   5  *
   6  * Ajla is free software: you can redistribute it and/or modify it under the
   7  * terms of the GNU General Public License as published by the Free Software
   8  * Foundation, either version 3 of the License, or (at your option) any later
   9  * version.
  10  *
  11  * Ajla is distributed in the hope that it will be useful, but WITHOUT ANY
  12  * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
  13  * A PARTICULAR PURPOSE. See the GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along with
  16  * Ajla. If not, see <https://www.gnu.org/licenses/>.
  17  */
  18
  19 static bool attr_w gen_jump(struct codegen_context *ctx, int32_t jmp_offset, unsigned op_size, unsigned cond, unsigned reg1, unsigned reg2);
  20
  21 static bool attr_w gen_alu_upcall(struct codegen_context *ctx, size_t upcall, unsigned op_size, frame_t slot_1, frame_t slot_2, frame_t slot_r, uint32_t label_ovf)
  22 {
  23         if (slot_is_register(ctx, slot_1))
  24                 g(spill(ctx, slot_1));
  25         if (slot_2 != NO_FRAME_T && slot_is_register(ctx, slot_2))
  26                 g(spill(ctx, slot_2));
  27         g(gen_upcall_start(ctx, frame_t_is_const(slot_2) ? 4 : slot_2 != NO_FRAME_T ? 3 : 2));
  28         g(gen_frame_address(ctx, slot_1, 0, R_ARG0));
  29         g(gen_upcall_argument(ctx, 0));
  30         if (frame_t_is_const(slot_2)) {
  31                 size_t x_offs;
  32                 g(gen_load_constant(ctx, R_ARG1, frame_t_get_const(slot_2)));
  33                 g(gen_upcall_argument(ctx, 1));
  34                 g(gen_frame_address(ctx, slot_r, 0, R_ARG2));
  35                 g(gen_upcall_argument(ctx, 2));
  36                 g(gen_get_upcall_pointer(ctx, upcall, R_ARG3));
  37                 g(gen_upcall_argument(ctx, 3));
  38                 x_offs = offsetof(struct cg_upcall_vector_s, INT_binary_const_int8_t) + op_size * sizeof(void (*)(void));
  39                 g(gen_upcall(ctx, x_offs, 4));
  40         } else if (slot_2 != NO_FRAME_T) {
  41                 g(gen_frame_address(ctx, slot_2, 0, R_ARG1));
  42                 g(gen_upcall_argument(ctx, 1));
  43                 g(gen_frame_address(ctx, slot_r, 0, R_ARG2));
  44                 g(gen_upcall_argument(ctx, 2));
  45                 g(gen_upcall(ctx, upcall, 3));
  46         } else {
  47                 g(gen_frame_address(ctx, slot_r, 0, R_ARG1));
  48                 g(gen_upcall_argument(ctx, 1));
  49                 g(gen_upcall(ctx, upcall, 2));
  50         }
  51         if (slot_is_register(ctx, slot_r))
  52                 g(unspill(ctx, slot_r));
  53         if (label_ovf)
  54                 g(gen_jmp_on_zero(ctx, OP_SIZE_1, R_RET0, COND_E, label_ovf));
  55         return true;
  56 }
  57
  58 static bool attr_w gen_alu_typed_upcall(struct codegen_context *ctx, size_t upcall, unsigned op_size, frame_t slot_1, frame_t slot_2, frame_t slot_r, uint32_t label_ovf)
  59 {
  60         upcall += op_size * sizeof(void (*)(void));
  61         return gen_alu_upcall(ctx, upcall, op_size, slot_1, slot_2, slot_r, label_ovf);
  62 }
  63
  64 #define MODE_FIXED              0
  65 #define MODE_INT                1
  66 #define MODE_REAL               2
  67 #define MODE_BOOL               3
  68 #define MODE_ARRAY_LEN_GT       4
  69
  70 static bool attr_w gen_alu(struct codegen_context *ctx, unsigned mode, unsigned op_size, unsigned op, uint32_t label_ovf, frame_t slot_1, frame_t slot_2, frame_t slot_r)
  71 {
  72         unsigned alu;
  73         bool sgn, mod;
  74         unsigned reg1, reg2, reg3, target;
  75         switch (mode) {
  76                 case MODE_FIXED: switch (op) {
  77                         case OPCODE_FIXED_OP_add:               alu = ALU_ADD; goto do_alu;
  78                         case OPCODE_FIXED_OP_subtract:          alu = ALU_SUB; goto do_alu;
  79                         case OPCODE_FIXED_OP_multiply:          goto do_multiply;
  80                         case OPCODE_FIXED_OP_divide:
  81                         case OPCODE_FIXED_OP_divide_alt1:       sgn = true; mod = false; goto do_divide;
  82                         case OPCODE_FIXED_OP_udivide:
  83                         case OPCODE_FIXED_OP_udivide_alt1:      sgn = false; mod = false; goto do_divide;
  84                         case OPCODE_FIXED_OP_modulo:
  85                         case OPCODE_FIXED_OP_modulo_alt1:       sgn = true; mod = true; goto do_divide;
  86                         case OPCODE_FIXED_OP_umodulo:
  87                         case OPCODE_FIXED_OP_umodulo_alt1:      sgn = false; mod = true; goto do_divide;
  88                         case OPCODE_FIXED_OP_power:             return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_binary_power_int8_t), op_size, slot_1, slot_2, slot_r, 0);
  89                         case OPCODE_FIXED_OP_and:               alu = ALU_AND; goto do_alu;
  90                         case OPCODE_FIXED_OP_or:                alu = ALU_OR; goto do_alu;
  91                         case OPCODE_FIXED_OP_xor:               alu = ALU_XOR; goto do_alu;
  92                         case OPCODE_FIXED_OP_shl:               alu = ROT_SHL; goto do_shift;
  93                         case OPCODE_FIXED_OP_shr:               alu = ROT_SAR; goto do_shift;
  94                         case OPCODE_FIXED_OP_ushr:              alu = ROT_SHR; goto do_shift;
  95                         case OPCODE_FIXED_OP_rol:               alu = ROT_ROL; goto do_shift;
  96                         case OPCODE_FIXED_OP_ror:               alu = ROT_ROR; goto do_shift;
  97                         case OPCODE_FIXED_OP_bts:               alu = BTX_BTS; goto do_bt;
  98                         case OPCODE_FIXED_OP_btr:               alu = BTX_BTR; goto do_bt;
  99                         case OPCODE_FIXED_OP_btc:               alu = BTX_BTC; goto do_bt;
 100                         case OPCODE_FIXED_OP_equal:             alu = COND_E; goto do_compare;
 101                         case OPCODE_FIXED_OP_not_equal:         alu = COND_NE; goto do_compare;
 102                         case OPCODE_FIXED_OP_less:              alu = COND_L; goto do_compare;
 103                         case OPCODE_FIXED_OP_less_equal:        alu = COND_LE; goto do_compare;
 104                         case OPCODE_FIXED_OP_uless:             alu = COND_B; goto do_compare;
 105                         case OPCODE_FIXED_OP_uless_equal:       alu = COND_BE; goto do_compare;
 106                         case OPCODE_FIXED_OP_bt:                alu = BTX_BT; goto do_bt;
 107                         default:                                internal(file_line, "gen_alu: unsupported fixed operation %u", op);
 108                 }
 109                 case MODE_INT: switch (op) {
 110                         case OPCODE_INT_OP_add:                 alu = ALU_ADD; goto do_alu;
 111                         case OPCODE_INT_OP_subtract:            alu = ALU_SUB; goto do_alu;
 112                         case OPCODE_INT_OP_multiply:            goto do_multiply;
 113                         case OPCODE_INT_OP_divide:
 114                         case OPCODE_INT_OP_divide_alt1:         sgn = true; mod = false; goto do_divide;
 115                         case OPCODE_INT_OP_modulo:
 116                         case OPCODE_INT_OP_modulo_alt1:         sgn = true; mod = true; goto do_divide;
 117                         case OPCODE_INT_OP_power:               return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, INT_binary_power_int8_t), op_size, slot_1, slot_2, slot_r, label_ovf);
 118                         case OPCODE_INT_OP_and:                 alu = ALU_AND; mode = MODE_FIXED; goto do_alu;
 119                         case OPCODE_INT_OP_or:                  alu = ALU_OR; mode = MODE_FIXED; goto do_alu;
 120                         case OPCODE_INT_OP_xor:                 alu = ALU_XOR; mode = MODE_FIXED; goto do_alu;
 121                         case OPCODE_INT_OP_shl:                 alu = ROT_SHL; goto do_shift;
 122                         case OPCODE_INT_OP_shr:                 alu = ROT_SAR; goto do_shift;
 123                         case OPCODE_INT_OP_bts:                 alu = BTX_BTS; goto do_bt;
 124                         case OPCODE_INT_OP_btr:                 alu = BTX_BTR; goto do_bt;
 125                         case OPCODE_INT_OP_btc:                 alu = BTX_BTC; goto do_bt;
 126                         case OPCODE_INT_OP_equal:               alu = COND_E; goto do_compare;
 127                         case OPCODE_INT_OP_not_equal:           alu = COND_NE; goto do_compare;
 128                         case OPCODE_INT_OP_less:                alu = COND_L; goto do_compare;
 129                         case OPCODE_INT_OP_less_equal:          alu = COND_LE; goto do_compare;
 130                         case OPCODE_INT_OP_bt:                  alu = BTX_BT; goto do_bt;
 131                         default:                                internal(file_line, "gen_alu: unsupported int operation %u", op);
 132                 }
 133                 case MODE_BOOL: switch (op) {
 134                         case OPCODE_BOOL_OP_and:                alu = ALU_AND; mode = MODE_FIXED; goto do_alu;
 135                         case OPCODE_BOOL_OP_or:                 alu = ALU_OR; mode = MODE_FIXED; goto do_alu;
 136                         case OPCODE_BOOL_OP_equal:              alu = COND_E; goto do_compare;
 137                         case OPCODE_BOOL_OP_not_equal:          alu = ALU_XOR; mode = MODE_FIXED; goto do_alu;
 138                         case OPCODE_BOOL_OP_less:               alu = COND_L; goto do_compare;
 139                         case OPCODE_BOOL_OP_less_equal:         alu = COND_LE; goto do_compare;
 140                         default:                                internal(file_line, "gen_alu: unsupported bool operation %u", op);
 141                 }
 142         }
 143         internal(file_line, "gen_alu: unsupported mode %u", mode);
 144
 145         /*******
 146          * ALU *
 147          *******/
 148 do_alu: {
 149                 size_t attr_unused offset;
 150                 uint8_t attr_unused long_imm;
 151                 unsigned first_flags;
 152                 unsigned second_flags;
 153                 unsigned second_alu;
 154                 unsigned attr_unused op_size_flags;
 155                 if (unlikely(op_size > OP_SIZE_NATIVE)) {
 156 #if !defined(ARCH_X86) && !defined(ARCH_ARM) && !defined(ARCH_PARISC) && !defined(ARCH_POWER) && !defined(ARCH_SPARC32)
 157                         if (mode == MODE_FIXED) {
 158                                 if (alu == ALU_ADD) {
 159                                         g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_add_,TYPE_INT_MAX)), op_size, slot_1, slot_2, slot_r, 0));
 160                                         return true;
 161                                 } else if (alu == ALU_SUB) {
 162                                         g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_subtract_,TYPE_INT_MAX)), op_size, slot_1, slot_2, slot_r, 0));
 163                                         return true;
 164                                 }
 165                         } else if (mode == MODE_INT) {
 166                                 if (alu == ALU_ADD) {
 167                                         g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(INT_binary_add_,TYPE_INT_MAX)), op_size, slot_1, slot_2, slot_r, label_ovf));
 168                                         return true;
 169                                 } else if (alu == ALU_SUB) {
 170                                         g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(INT_binary_subtract_,TYPE_INT_MAX)), op_size, slot_1, slot_2, slot_r, label_ovf));
 171                                         return true;
 172                                 }
 173                         }
 174 #endif
 175                         first_flags = alu == ALU_ADD || alu == ALU_SUB ? 2 : 0;
 176                         second_flags = mode == MODE_INT ? 1 : 0;
 177                         second_alu = alu == ALU_ADD ? ALU_ADC : alu == ALU_SUB ? ALU_SBB : alu;
 178                         g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
 179 #if defined(ARCH_X86)
 180                         g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, alu, first_flags, slot_2, lo_word(OP_SIZE_NATIVE), true, R_SCRATCH_1));
 181                         g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, second_alu, second_flags, slot_2, hi_word(OP_SIZE_NATIVE), true, R_SCRATCH_2));
 182 #else
 183                         g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_2, 0, R_SCRATCH_3, R_SCRATCH_4));
 184                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, alu, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_3, first_flags));
 185 #if defined(ARCH_PARISC)
 186                         if (mode == MODE_INT) {
 187                                 gen_insn(INSN_ALU_FLAGS_TRAP, OP_SIZE_NATIVE, second_alu, ALU_WRITES_FLAGS(second_alu, false));
 188                                 gen_one(R_SCRATCH_2);
 189                                 gen_one(R_SCRATCH_2);
 190                                 gen_one(R_SCRATCH_4);
 191                                 gen_four(label_ovf);
 192                         } else
 193 #endif
 194                         {
 195                                 gen_insn(first_flags ? INSN_ALU_FLAGS : INSN_ALU, OP_SIZE_NATIVE, second_alu, second_flags | ALU_WRITES_FLAGS(second_alu, false));
 196                                 gen_one(R_SCRATCH_2);
 197                                 gen_one(R_SCRATCH_2);
 198                                 gen_one(R_SCRATCH_4);
 199                         }
 200 #endif
 201 #if !defined(ARCH_PARISC)
 202                         if (mode == MODE_INT) {
 203                                 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_O, 0);
 204                                 gen_four(label_ovf);
 205                         }
 206 #endif
 207                         g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_1, R_SCRATCH_2));
 208                         return true;
 209                 }
 210
 211                 if ((ARCH_HAS_FLAGS || ARCH_SUPPORTS_TRAPS) && slot_2 == slot_r && slot_1 != slot_2 && alu_is_commutative(alu)) {
 212                         frame_t x = slot_1;
 213                         slot_1 = slot_2;
 214                         slot_2 = x;
 215                 }
 216                 if ((ARCH_HAS_FLAGS || ARCH_SUPPORTS_TRAPS) && slot_1 == slot_r && (slot_1 != slot_2 || mode != MODE_INT) && i_size_cmp(op_size) == op_size + zero
 217 #if defined(ARCH_POWER)
 218                         && op_size == OP_SIZE_NATIVE
 219 #endif
 220                     ) {
 221                         struct cg_exit *ce;
 222                         unsigned undo_alu = alu == ALU_ADD ? ALU_SUB : ALU_ADD;
 223                         if (slot_is_register(ctx, slot_1)) {
 224                                 unsigned reg1 = ctx->registers[slot_1];
 225                                 if (slot_is_register(ctx, slot_2) || frame_t_is_const(slot_2)) {
 226                                         unsigned reg2 = frame_t_is_const(slot_2) ? 0xff /* avoid warning */ : ctx->registers[slot_2];
 227                                         if (mode == MODE_INT && ARCH_SUPPORTS_TRAPS) {
 228                                                 if (frame_t_is_const(slot_2))
 229                                                         g(gen_imm(ctx, frame_t_get_const(slot_2), alu_trap_purpose(alu), i_size(op_size)));
 230                                                 gen_insn(INSN_ALU_TRAP, op_size, alu, ALU_WRITES_FLAGS(alu, frame_t_is_const(slot_2) && is_imm()));
 231                                                 gen_one(reg1);
 232                                                 gen_one(reg1);
 233                                                 if (frame_t_is_const(slot_2))
 234                                                         gen_imm_offset();
 235                                                 else
 236                                                         gen_one(reg2);
 237                                                 if (ARCH_TRAP_BEFORE) {
 238                                                         gen_four(label_ovf);
 239                                                         return true;
 240                                                 } else {
 241                                                         ce = alloc_undo_label(ctx);
 242                                                         if (unlikely(!ce))
 243                                                                 return false;
 244                                                         gen_four(ce->undo_label);
 245                                                         goto do_undo_opcode;
 246                                                 }
 247                                         }
 248                                         if (frame_t_is_const(slot_2))
 249                                                 g(gen_3address_alu_imm(ctx, i_size(op_size), alu, reg1, reg1, frame_t_get_const(slot_2), mode == MODE_INT));
 250                                         else
 251                                                 g(gen_3address_alu(ctx, i_size(op_size), alu, reg1, reg1, reg2, mode == MODE_INT));
 252                                         if (mode == MODE_INT) {
 253                                                 size_t m;
 254                                                 ce = alloc_undo_label(ctx);
 255                                                 if (unlikely(!ce))
 256                                                         return false;
 257                                                 gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
 258                                                 gen_four(ce->undo_label);
 259 do_undo_opcode:
 260                                                 ce->undo_opcode = INSN_ALU + ARCH_PARTIAL_ALU(op_size);
 261                                                 ce->undo_op_size = i_size(op_size);
 262                                                 ce->undo_aux = undo_alu;
 263                                                 ce->undo_writes_flags = ALU_WRITES_FLAGS(undo_alu, frame_t_is_const(slot_2) && is_imm());
 264                                                 m = mark_params(ctx);
 265                                                 gen_one(reg1);
 266                                                 gen_one(reg1);
 267                                                 if (frame_t_is_const(slot_2))
 268                                                         gen_imm_offset();
 269                                                 else
 270                                                         gen_one(reg2);
 271                                                 copy_params(ctx, ce, m);
 272                                         }
 273                                         return true;
 274                                 }
 275 #if defined(ARCH_S390) || defined(ARCH_X86)
 276                                 else if (!frame_t_is_const(slot_2)) {
 277                                         size_t m;
 278                                         int64_t offset = (size_t)slot_2 * slot_size;
 279                                         g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDR_OFFSET, i_size(op_size)));
 280                                         gen_insn(INSN_ALU + ARCH_PARTIAL_ALU(op_size), i_size(op_size), alu, 1);
 281                                         gen_one(reg1);
 282                                         gen_one(reg1);
 283                                         gen_address_offset();
 284                                         if (mode == MODE_INT) {
 285                                                 ce = alloc_undo_label(ctx);
 286                                                 if (unlikely(!ce))
 287                                                         return false;
 288                                                 ce->undo_opcode = INSN_ALU + ARCH_PARTIAL_ALU(op_size);
 289                                                 ce->undo_op_size = i_size(op_size);
 290                                                 ce->undo_aux = undo_alu;
 291                                                 ce->undo_writes_flags = ARCH_HAS_FLAGS;
 292                                                 m = mark_params(ctx);
 293                                                 gen_one(reg1);
 294                                                 gen_one(reg1);
 295                                                 gen_address_offset();
 296                                                 copy_params(ctx, ce, m);
 297                                                 gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
 298                                                 gen_four(ce->undo_label);
 299                                         }
 300                                         return true;
 301                                 }
 302 #endif
 303                         }
 304 #if defined(ARCH_X86)
 305                         else {
 306                                 unsigned reg2;
 307                                 size_t m;
 308                                 int64_t offset = (size_t)slot_1 * slot_size;
 309                                 if (!frame_t_is_const(slot_2))
 310                                         g(gen_frame_get(ctx, op_size, garbage, slot_2, R_SCRATCH_1, &reg2));
 311                                 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDR_OFFSET, i_size(op_size)));
 312                                 if (frame_t_is_const(slot_2))
 313                                         g(gen_imm(ctx, frame_t_get_const(slot_2), alu_purpose(alu), i_size(op_size)));
 314                                 gen_insn(INSN_ALU + ARCH_PARTIAL_ALU(op_size), i_size(op_size), alu, 1);
 315                                 gen_address_offset();
 316                                 gen_address_offset();
 317                                 if (frame_t_is_const(slot_2))
 318                                         gen_imm_offset();
 319                                 else
 320                                         gen_one(reg2);
 321                                 if (mode == MODE_INT) {
 322                                         ce = alloc_undo_label(ctx);
 323                                         if (unlikely(!ce))
 324                                                 return false;
 325                                         ce->undo_opcode = INSN_ALU + ARCH_PARTIAL_ALU(op_size);
 326                                         ce->undo_op_size = i_size(op_size);
 327                                         ce->undo_aux = undo_alu;
 328                                         ce->undo_writes_flags = ARCH_HAS_FLAGS;
 329                                         m = mark_params(ctx);
 330                                         gen_address_offset();
 331                                         gen_address_offset();
 332                                         if (frame_t_is_const(slot_2))
 333                                                 gen_imm_offset();
 334                                         else
 335                                                 gen_one(reg2);
 336                                         copy_params(ctx, ce, m);
 337                                         gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
 338                                         gen_four(ce->undo_label);
 339                                 }
 340                                 return true;
 341                         }
 342 #endif
 343                 }
 344
 345 #if defined(ARCH_X86)
 346                 if (1)
 347 #elif defined(ARCH_S390)
 348                 if (op_size >= OP_SIZE_4)
 349 #elif ARCH_HAS_FLAGS && !defined(ARCH_POWER)
 350                 if (op_size == i_size(op_size) + (unsigned)zero && frame_t_is_const(slot_2))
 351 #else
 352                 if (mode != MODE_INT && op_size == i_size(op_size) + (unsigned)zero && frame_t_is_const(slot_2))
 353 #endif
 354                 {
 355                         if (mode == MODE_INT) {
 356                                 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
 357                         } else {
 358                                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, slot_2, R_SCRATCH_1);
 359                         }
 360                         g(gen_frame_load(ctx, op_size, garbage, slot_1, 0, false, target));
 361                         g(gen_frame_load_op(ctx, op_size, garbage, alu, mode == MODE_INT, slot_2, 0, false, target));
 362                         goto check_ovf_store;
 363                 }
 364                 op_size_flags = !ARCH_HAS_FLAGS && !ARCH_SUPPORTS_TRAPS ? OP_SIZE_NATIVE : OP_SIZE_4;
 365 #if defined(ARCH_POWER)
 366                 op_size_flags = OP_SIZE_NATIVE;
 367 #endif
 368                 g(gen_frame_get(ctx, op_size, mode == MODE_INT && (op_size < op_size_flags || ARCH_SUPPORTS_TRAPS) ? sign_x : garbage, slot_1, R_SCRATCH_1, &reg1));
 369                 g(gen_frame_get(ctx, op_size, mode == MODE_INT && (op_size < op_size_flags || ARCH_SUPPORTS_TRAPS) ? sign_x : garbage, slot_2, R_SCRATCH_2, &reg2));
 370 #if !ARCH_HAS_FLAGS
 371                 if (mode == MODE_INT && op_size >= OP_SIZE_4) {
 372                         if (ARCH_SUPPORTS_TRAPS) {
 373                                 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
 374                                 gen_insn(INSN_ALU_TRAP, op_size, alu, ALU_WRITES_FLAGS(alu, false));
 375                                 gen_one(target);
 376                                 gen_one(reg1);
 377                                 gen_one(reg2);
 378                                 gen_four(label_ovf);
 379                                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
 380                                 return true;
 381                         }
 382                         if (op_size >= OP_SIZE_NATIVE) {
 383                                 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_3);
 384                                 g(gen_3address_alu(ctx, i_size(op_size), alu, target, reg1, reg2, 0));
 385 #if defined(ARCH_IA64)
 386                                 g(gen_3address_alu(ctx, i_size(op_size), ALU_XOR, R_SCRATCH_1, reg1, reg2, 0));
 387                                 g(gen_3address_alu(ctx, i_size(op_size), ALU_XOR, R_SCRATCH_2, reg2, target, 0));
 388                                 if (alu == ALU_ADD) {
 389                                         g(gen_3address_alu(ctx, i_size(op_size), ALU_ANDN, R_SCRATCH_1, R_SCRATCH_2, R_SCRATCH_1, 0));
 390                                 } else {
 391                                         g(gen_3address_alu(ctx, i_size(op_size), ALU_ANDN, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_2, 0));
 392                                 }
 393                                 g(gen_cmp_test_jmp(ctx, INSN_TEST, i_size(op_size), R_SCRATCH_1, R_SCRATCH_1, COND_S, label_ovf));
 394 #else
 395                                 gen_insn(INSN_CMP_DEST_REG, i_size(op_size), COND_L, 0);
 396                                 gen_one(R_SCRATCH_1);
 397                                 if (alu == ALU_ADD) {
 398                                         gen_one(target);
 399                                         gen_one(reg1);
 400                                 } else {
 401                                         gen_one(reg1);
 402                                         gen_one(target);
 403                                 }
 404
 405                                 g(gen_imm(ctx, 0, IMM_PURPOSE_CMP, i_size(op_size)));
 406                                 gen_insn(INSN_CMP_DEST_REG, i_size(op_size), COND_L, 0);
 407                                 gen_one(R_SCRATCH_2);
 408                                 gen_one(reg2);
 409                                 gen_imm_offset();
 410
 411                                 g(gen_cmp_test_jmp(ctx, INSN_CMP, i_size(op_size), R_SCRATCH_1, R_SCRATCH_2, COND_NE, label_ovf));
 412 #endif
 413                                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
 414                                 return true;
 415                         }
 416                 }
 417 #endif
 418                 if (mode == MODE_INT) {
 419                         target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
 420                 } else if (!ARCH_IS_3ADDRESS(alu, mode == MODE_INT && op_size >= op_size_flags) && !alu_is_commutative(alu)) {
 421                         target = gen_frame_target(ctx, slot_r, NO_FRAME_T, slot_2, R_SCRATCH_1);
 422                 } else {
 423                         target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
 424                 }
 425                 g(gen_3address_alu(ctx, i_size(op_size), alu, target, reg1, reg2, mode == MODE_INT && op_size >= op_size_flags));
 426
 427                 if (mode == MODE_INT && unlikely(op_size < op_size_flags)) {
 428                         g(gen_cmp_extended(ctx, op_size_flags, op_size, target, R_SCRATCH_2, label_ovf));
 429                 } else
 430 check_ovf_store:
 431                 if (mode == MODE_INT) {
 432                         gen_insn(INSN_JMP_COND, op_size, COND_O, 0);
 433                         gen_four(label_ovf);
 434                 }
 435                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
 436                 return true;
 437         }
 438
 439         /************
 440          * MULTIPLY *
 441          ************/
 442 do_multiply: {
 443                 size_t attr_unused offset;
 444                 uint8_t attr_unused long_imm;
 445                 if (unlikely(op_size > OP_SIZE_NATIVE) || unlikely(!ARCH_HAS_MUL)) {
 446                         if (mode == MODE_INT) {
 447                                 g(gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, INT_binary_multiply_int8_t), op_size, slot_1, slot_2, slot_r, label_ovf));
 448                                 return true;
 449                         }
 450 #if defined(ARCH_X86)
 451                         g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_1, hi_word(OP_SIZE_NATIVE), true, R_CX));
 452                         g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_2, hi_word(OP_SIZE_NATIVE), true, R_AX));
 453                         g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, ALU_MUL, true, slot_2, lo_word(OP_SIZE_NATIVE), true, R_CX));
 454                         g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, ALU_MUL, true, slot_1, lo_word(OP_SIZE_NATIVE), true, R_AX));
 455                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_ADD, R_CX, R_CX, R_AX, 0));
 456                         g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_2, lo_word(OP_SIZE_NATIVE), true, R_AX));
 457
 458                         offset = (size_t)slot_1 * slot_size + lo_word(OP_SIZE_NATIVE);
 459                         g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
 460                         gen_insn(INSN_MUL_L, OP_SIZE_NATIVE, 0, 1);
 461                         gen_one(R_AX);
 462                         gen_one(R_DX);
 463                         gen_one(R_AX);
 464                         gen_address_offset();
 465
 466                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_ADD, R_DX, R_DX, R_CX, 0));
 467
 468                         g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_AX, R_DX));
 469
 470                         return true;
 471 #elif defined(ARCH_ARM32)
 472                         g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
 473                         g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_2, 0, R_SCRATCH_3, R_SCRATCH_4));
 474
 475                         g(gen_mov(ctx, OP_SIZE_NATIVE, R_SCRATCH_NA_1, R_SCRATCH_1));
 476
 477                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_MUL, R_SCRATCH_4, R_SCRATCH_1, R_SCRATCH_4, 0));
 478
 479                         gen_insn(INSN_MADD, OP_SIZE_NATIVE, 0, 0);
 480                         gen_one(R_SCRATCH_2);
 481                         gen_one(R_SCRATCH_3);
 482                         gen_one(R_SCRATCH_2);
 483                         gen_one(R_SCRATCH_4);
 484
 485                         gen_insn(INSN_MUL_L, OP_SIZE_NATIVE, 0, 0);
 486                         gen_one(R_SCRATCH_1);
 487                         gen_one(R_SCRATCH_4);
 488                         gen_one(R_SCRATCH_NA_1);
 489                         gen_one(R_SCRATCH_3);
 490
 491                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_ADD, R_SCRATCH_2, R_SCRATCH_2, R_SCRATCH_4, 0));
 492
 493                         g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_1, R_SCRATCH_2));
 494
 495                         return true;
 496 #elif defined(ARCH_ARM64)
 497                         g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
 498                         g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_2, 0, R_SCRATCH_3, R_SCRATCH_4));
 499
 500                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_UMULH, R_SCRATCH_NA_1, R_SCRATCH_1, R_SCRATCH_3, 0));
 501
 502                         gen_insn(INSN_MADD, OP_SIZE_NATIVE, 0, 0);
 503                         gen_one(R_SCRATCH_NA_1);
 504                         gen_one(R_SCRATCH_2);
 505                         gen_one(R_SCRATCH_3);
 506                         gen_one(R_SCRATCH_NA_1);
 507
 508                         gen_insn(INSN_MADD, OP_SIZE_NATIVE, 0, 0);
 509                         gen_one(R_SCRATCH_2);
 510                         gen_one(R_SCRATCH_1);
 511                         gen_one(R_SCRATCH_4);
 512                         gen_one(R_SCRATCH_NA_1);
 513
 514                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_MUL, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_3, 0));
 515
 516                         g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_1, R_SCRATCH_2));
 517
 518                         return true;
 519 #else
 520                         g(gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_binary_multiply_int8_t), op_size, slot_1, slot_2, slot_r, 0));
 521                         return true;
 522 #endif
 523                 }
 524
 525 #if defined(ARCH_X86)
 526                 if (mode == MODE_INT) {
 527                         if (op_size != OP_SIZE_1 && slot_r == slot_1 && slot_is_register(ctx, slot_1)) {
 528                                 struct cg_exit *ce;
 529                                 target = ctx->registers[slot_1];
 530                                 g(gen_mov(ctx, op_size, R_SCRATCH_1, target));
 531                                 g(gen_frame_load_op(ctx, op_size, garbage, ALU_MUL, mode == MODE_INT, slot_2, 0, false, target));
 532                                 ce = alloc_undo_label(ctx);
 533                                 if (unlikely(!ce))
 534                                         return false;
 535                                 ce->undo_opcode = INSN_MOV;
 536                                 ce->undo_op_size = op_size;
 537                                 ce->undo_aux = 0;
 538                                 ce->undo_writes_flags = 0;
 539                                 ce->undo_parameters[0] = target;
 540                                 ce->undo_parameters[1] = R_SCRATCH_1;
 541                                 ce->undo_parameters_len = 2;
 542                                 gen_insn(INSN_JMP_COND, op_size, COND_O, 0);
 543                                 gen_four(ce->undo_label);
 544                                 return true;
 545                         }
 546                         target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
 547                 } else {
 548                         target = gen_frame_target(ctx, slot_r, NO_FRAME_T, slot_2, R_SCRATCH_1);
 549                 }
 550                 if (op_size == OP_SIZE_1)
 551                         target = R_SCRATCH_1;
 552                 g(gen_frame_load(ctx, op_size, garbage, slot_1, 0, false, target));
 553                 g(gen_frame_load_op(ctx, op_size, garbage, ALU_MUL, mode == MODE_INT, slot_2, 0, false, target));
 554                 if (mode == MODE_INT) {
 555                         gen_insn(INSN_JMP_COND, op_size, COND_O, 0);
 556                         gen_four(label_ovf);
 557                 }
 558                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
 559                 return true;
 560 #endif
 561 #if defined(ARCH_ALPHA)
 562                 if (mode == MODE_INT && op_size >= OP_SIZE_4 && ARCH_SUPPORTS_TRAPS) {
 563                         target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
 564                         g(gen_frame_get(ctx, op_size, garbage, slot_1, R_SCRATCH_1, &reg1));
 565                         g(gen_frame_get(ctx, op_size, garbage, slot_2, R_SCRATCH_2, &reg2));
 566
 567                         gen_insn(INSN_ALU_TRAP, op_size, ALU_MUL, ALU_WRITES_FLAGS(ALU_MUL, false));
 568                         gen_one(target);
 569                         gen_one(reg1);
 570                         gen_one(reg2);
 571                         gen_four(label_ovf);
 572                         g(gen_frame_store(ctx, op_size, slot_r, 0, target));
 573
 574                         return true;
 575                 }
 576 #endif
 577 #if defined(ARCH_ARM32)
 578                 if (mode == MODE_INT && op_size == OP_SIZE_4) {
 579                         target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_3);
 580                         g(gen_frame_get(ctx, op_size, garbage, slot_1, R_SCRATCH_1, &reg1));
 581                         g(gen_frame_get(ctx, op_size, garbage, slot_2, R_SCRATCH_2, &reg2));
 582
 583                         gen_insn(INSN_MUL_L, OP_SIZE_NATIVE, 0, 0);
 584                         gen_one(target);
 585                         gen_one(R_SCRATCH_4);
 586                         gen_one(reg1);
 587                         gen_one(reg2);
 588
 589                         gen_insn(INSN_CMP, OP_SIZE_NATIVE, 0, 1);
 590                         gen_one(R_SCRATCH_4);
 591                         gen_one(ARG_SHIFTED_REGISTER);
 592                         gen_one(ARG_SHIFT_ASR | 0x1f);
 593                         gen_one(target);
 594
 595                         gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_NE, 0);
 596                         gen_four(label_ovf);
 597
 598                         g(gen_frame_store(ctx, op_size, slot_r, 0, target));
 599
 600                         return true;
 601                 }
 602 #endif
 603 #if defined(ARCH_ARM64)
 604                 if (mode == MODE_INT && op_size == OP_SIZE_4) {
 605                         target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
 606                         g(gen_frame_get(ctx, op_size, op_size < OP_SIZE_4 ? sign_x : garbage, slot_1, R_SCRATCH_1, &reg1));
 607                         g(gen_frame_get(ctx, op_size, op_size < OP_SIZE_4 ? sign_x : garbage, slot_2, R_SCRATCH_2, &reg2));
 608                         gen_insn(INSN_ALU, OP_SIZE_8, ALU_MUL, ALU_WRITES_FLAGS(ALU_MUL, false));
 609                         gen_one(target);
 610                         gen_one(ARG_EXTENDED_REGISTER);
 611                         gen_one(ARG_EXTEND_SXTW);
 612                         gen_one(reg1);
 613                         gen_one(ARG_EXTENDED_REGISTER);
 614                         gen_one(ARG_EXTEND_SXTW);
 615                         gen_one(reg2);
 616
 617                         gen_insn(INSN_CMP, OP_SIZE_8, 0, 1);
 618                         gen_one(target);
 619                         gen_one(ARG_EXTENDED_REGISTER);
 620                         gen_one(ARG_EXTEND_SXTW);
 621                         gen_one(target);
 622
 623                         gen_insn(INSN_JMP_COND, OP_SIZE_8, COND_NE, 0);
 624                         gen_four(label_ovf);
 625
 626                         g(gen_frame_store(ctx, op_size, slot_r, 0, target));
 627
 628                         return true;
 629                 }
 630                 if (mode == MODE_INT && op_size == OP_SIZE_8) {
 631                         target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
 632                         g(gen_frame_get(ctx, op_size, garbage, slot_1, R_SCRATCH_1, &reg1));
 633                         g(gen_frame_get(ctx, op_size, garbage, slot_2, R_SCRATCH_2, &reg2));
 634                         g(gen_3address_alu(ctx, OP_SIZE_8, ALU_SMULH, R_SCRATCH_3, reg1, reg2, 0));
 635
 636                         g(gen_3address_alu(ctx, OP_SIZE_8, ALU_MUL, target, reg1, reg2, 0));
 637
 638                         gen_insn(INSN_CMP, OP_SIZE_8, 0, 1);
 639                         gen_one(R_SCRATCH_3);
 640                         gen_one(ARG_SHIFTED_REGISTER);
 641                         gen_one(ARG_SHIFT_ASR | 0x3f);
 642                         gen_one(target);
 643
 644                         gen_insn(INSN_JMP_COND, OP_SIZE_8, COND_NE, 0);
 645                         gen_four(label_ovf);
 646
 647                         g(gen_frame_store(ctx, op_size, slot_r, 0, target));
 648
 649                         return true;
 650                 }
 651 #endif
 652 #if defined(ARCH_POWER)
 653                 if (mode == MODE_INT && op_size >= OP_SIZE_4) {
 654                         target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
 655                         g(gen_frame_get(ctx, op_size, garbage, slot_1, R_SCRATCH_1, &reg1));
 656                         g(gen_frame_get(ctx, op_size, garbage, slot_2, R_SCRATCH_2, &reg2));
 657
 658                         g(gen_3address_alu(ctx, op_size, ALU_MUL, target, reg1, reg2, 1));
 659
 660                         gen_insn(INSN_JMP_COND, op_size, COND_O, 0);
 661                         gen_four(label_ovf);
 662
 663                         g(gen_frame_store(ctx, op_size, slot_r, 0, target));
 664
 665                         return true;
 666                 }
 667 #endif
 668 #if defined(ARCH_LOONGARCH64) || (defined(ARCH_MIPS) && MIPS_R6) || defined(ARCH_RISCV64)
 669                 if (mode == MODE_INT && op_size == OP_SIZE_NATIVE) {
 670                         target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
 671                         g(gen_frame_get(ctx, op_size, garbage, slot_1, R_SCRATCH_1, &reg1));
 672                         g(gen_frame_get(ctx, op_size, garbage, slot_2, R_SCRATCH_2, &reg2));
 673
 674                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_SMULH, R_SCRATCH_3, reg1, reg2, 0));
 675
 676                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_MUL, target, reg1, reg2, 0));
 677
 678                         g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_4, target, (8U << OP_SIZE_NATIVE) - 1, 0));
 679
 680                         g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, R_SCRATCH_3, R_SCRATCH_4, COND_NE, label_ovf));
 681
 682                         g(gen_frame_store(ctx, OP_SIZE_NATIVE, slot_r, 0, target));
 683
 684                         return true;
 685                 }
 686 #endif
 687 #if defined(ARCH_S390)
 688                 if (mode == MODE_INT && op_size >= OP_SIZE_4 && likely(cpu_test_feature(CPU_FEATURE_misc_insn_ext_2))) {
 689                         target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
 690                         g(gen_frame_load(ctx, op_size, sign_x, slot_1, 0, false, target));
 691                         g(gen_frame_load_op(ctx, op_size, sign_x, ALU_MUL, 1, slot_2, 0, false, target));
 692
 693                         gen_insn(INSN_JMP_COND, op_size, COND_O, 0);
 694                         gen_four(label_ovf);
 695
 696                         g(gen_frame_store(ctx, op_size, slot_r, 0, target));
 697                         return true;
 698                 }
 699 #endif
 700 #if (defined(ARCH_MIPS) && !MIPS_R6) || defined(ARCH_S390)
 701 #if defined(ARCH_MIPS)
 702                 if (mode == MODE_INT && op_size >= OP_SIZE_4)
 703 #endif
 704 #if defined(ARCH_S390)
 705                 if (mode == MODE_INT && op_size == OP_SIZE_4)
 706 #endif
 707                 {
 708 #if defined(ARCH_S390)
 709                         target = R_SCRATCH_1;
 710 #else
 711                         target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
 712 #endif
 713                         g(gen_frame_get(ctx, op_size, sign_x, slot_1, R_SCRATCH_1, &reg1));
 714                         g(gen_frame_get(ctx, op_size, sign_x, slot_2, R_SCRATCH_3, &reg2));
 715
 716                         gen_insn(INSN_MUL_L, op_size, 0, 0);
 717                         gen_one(target);
 718                         gen_one(R_SCRATCH_2);
 719                         gen_one(reg1);
 720                         gen_one(reg2);
 721
 722                         g(gen_3address_rot_imm(ctx, op_size, ROT_SAR, R_SCRATCH_4, target, (8U << op_size) - 1, false));
 723
 724                         g(gen_cmp_test_jmp(ctx, INSN_CMP, op_size, R_SCRATCH_2, R_SCRATCH_4, COND_NE, label_ovf));
 725
 726                         g(gen_frame_store(ctx, op_size, slot_r, 0, target));
 727                         return true;
 728                 }
 729 #endif
 730                 if (mode == MODE_INT && op_size == OP_SIZE_NATIVE) {
 731                         g(gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, INT_binary_multiply_int8_t), op_size, slot_1, slot_2, slot_r, label_ovf));
 732                         return true;
 733                 }
 734
 735                 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
 736                 if (op_size < OP_SIZE_NATIVE && mode == MODE_INT) {
 737                         g(gen_frame_get(ctx, op_size, sign_x, slot_1, R_SCRATCH_1, &reg1));
 738                         g(gen_frame_get(ctx, op_size, sign_x, slot_2, R_SCRATCH_2, &reg2));
 739
 740                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_MUL, target, reg1, reg2, 0));
 741                 } else {
 742                         g(gen_frame_load(ctx, op_size, sign_x, slot_1, 0, false, target));
 743                         g(gen_frame_load_op(ctx, op_size, sign_x, ALU_MUL, 0, slot_2, 0, false, target));
 744                 }
 745
 746                 if (mode == MODE_INT) {
 747                         g(gen_cmp_extended(ctx, OP_SIZE_NATIVE, op_size, target, R_SCRATCH_2, label_ovf));
 748                 }
 749
 750                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
 751
 752                 return true;
 753         }
 754
 755         /**********
 756          * DIVIDE *
 757          **********/
 758 do_divide: {
 759                 uint32_t attr_unused label_skip = 0;    /* avoid warning */
 760                 uint32_t attr_unused label_skip2 = 0;   /* avoid warning */
 761                 uint32_t attr_unused label_end = 0;     /* avoid warning */
 762                 uint32_t attr_unused label_div_0 = 0;   /* avoid warning */
 763                 unsigned attr_unused divide_alu = 0;    /* avoid warning */
 764                 bool attr_unused have_mod = false;
 765                 bool attr_unused force_sx = false;
 766                 unsigned attr_unused div_op_size = i_size(op_size);
 767                 if (unlikely(op_size > OP_SIZE_NATIVE) || unlikely(!ARCH_HAS_DIV)
 768 #if defined(ARCH_S390)
 769                         || !(Z || (op_size <= OP_SIZE_4 && sgn))
 770 #endif
 771                    ) {
 772                         size_t upcall;
 773                         if (mode == MODE_INT) {
 774                                 upcall = !mod ? offsetof(struct cg_upcall_vector_s, INT_binary_divide_int8_t) : offsetof(struct cg_upcall_vector_s, INT_binary_modulo_int8_t);
 775                         } else if (sgn) {
 776                                 upcall = !mod ? offsetof(struct cg_upcall_vector_s, FIXED_binary_divide_int8_t) : offsetof(struct cg_upcall_vector_s, FIXED_binary_modulo_int8_t);
 777                         } else {
 778                                 upcall = !mod ? offsetof(struct cg_upcall_vector_s, FIXED_binary_udivide_int8_t) : offsetof(struct cg_upcall_vector_s, FIXED_binary_umodulo_int8_t);
 779                         }
 780                         g(gen_alu_typed_upcall(ctx, upcall, op_size, slot_1, slot_2, slot_r, mode == MODE_INT ? label_ovf : 0));
 781                         return true;
 782                 }
 783 #if defined(ARCH_X86) || defined(ARCH_S390)
 784                 if (mode == MODE_FIXED) {
 785                         label_skip = alloc_label(ctx);
 786                         if (unlikely(!label_skip))
 787                                 return false;
 788                         label_end = alloc_label(ctx);
 789                         if (unlikely(!label_end))
 790                                 return false;
 791                         if (sgn) {
 792                                 label_skip2 = alloc_label(ctx);
 793                                 if (unlikely(!label_skip2))
 794                                         return false;
 795                         }
 796                 }
 797 #if defined(ARCH_X86)
 798                 if (R_SCRATCH_1 != R_AX || R_SCRATCH_2 != R_DX || R_SCRATCH_3 != R_CX)
 799                         internal(file_line, "gen_alu: bad scratch registers");
 800 #endif
 801                 g(gen_frame_load(ctx, op_size, sgn ? sign_x : zero_x, slot_1, 0, false, R_SCRATCH_1));
 802                 g(gen_frame_load(ctx, op_size, sgn ? sign_x : zero_x, slot_2, 0, false, R_SCRATCH_3));
 803
 804                 g(gen_jmp_on_zero(ctx, i_size(op_size), R_SCRATCH_3, COND_E, mode == MODE_INT ? label_ovf : label_skip));
 805
 806                 if (sgn) {
 807                         uint64_t val;
 808                         uint32_t label_not_minus_1;
 809                         label_not_minus_1 = alloc_label(ctx);
 810                         if (unlikely(!label_not_minus_1))
 811                                 return false;
 812
 813                         g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, op_size, R_SCRATCH_3, -1, COND_NE, label_not_minus_1));
 814
 815                         val = -(uint64_t)0x80 << (((1 << op_size) - 1) * 8);
 816                         g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, op_size, R_SCRATCH_1, val, COND_E, mode == MODE_INT ? label_ovf : label_skip2));
 817
 818                         gen_label(label_not_minus_1);
 819                 }
 820
 821 #if defined(ARCH_X86)
 822                 if (op_size >= OP_SIZE_2) {
 823                         if (sgn) {
 824                                 gen_insn(INSN_CWD + ARCH_PARTIAL_ALU(op_size), op_size, 0, 0);
 825                                 gen_one(R_SCRATCH_2);
 826                                 gen_one(R_SCRATCH_1);
 827                                 if (op_size == OP_SIZE_2)
 828                                         gen_one(R_SCRATCH_2);
 829                         } else {
 830                                 g(gen_3address_alu(ctx, OP_SIZE_4, ALU_XOR, R_SCRATCH_2, R_SCRATCH_2, R_SCRATCH_2, 0));
 831                         }
 832                 }
 833                 gen_insn(INSN_DIV_L, op_size, sgn, 1);
 834                 gen_one(R_SCRATCH_1);
 835                 gen_one(i_size(op_size) == OP_SIZE_1 ? R_SCRATCH_1 : R_SCRATCH_2);
 836                 gen_one(R_SCRATCH_1);
 837                 gen_one(i_size(op_size) == OP_SIZE_1 ? R_SCRATCH_1 : R_SCRATCH_2);
 838                 gen_one(R_SCRATCH_3);
 839 #else
 840                 if (!sgn) {
 841                         g(gen_load_constant(ctx, R_SCRATCH_2, 0));
 842                 } else if (op_size <= OP_SIZE_4) {
 843                         g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_2, R_SCRATCH_1, (1U << (OP_SIZE_NATIVE + 3)) - 1, false));
 844                 }
 845                 gen_insn(INSN_DIV_L, i_size(op_size), sgn, 1);
 846                 gen_one(R_SCRATCH_2);
 847                 gen_one(R_SCRATCH_1);
 848                 gen_one(R_SCRATCH_2);
 849                 gen_one(R_SCRATCH_1);
 850                 gen_one(R_SCRATCH_3);
 851 #endif
 852                 if (mod && i_size(op_size) == OP_SIZE_1) {
 853                         g(gen_3address_rot_imm(ctx, OP_SIZE_2, ROT_SHR, R_SCRATCH_1, R_SCRATCH_1, 8, 0));
 854                         g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
 855                 } else if (mod) {
 856                         g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_2));
 857                 } else {
 858                         g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
 859                 }
 860                 if (mode == MODE_FIXED) {
 861                         gen_insn(INSN_JMP, 0, 0, 0);
 862                         gen_four(label_end);
 863
 864                         if (sgn) {
 865                                 gen_label(label_skip2);
 866
 867                                 if (mod)
 868                                         g(gen_frame_clear(ctx, op_size, slot_r));
 869                                 else
 870                                         g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
 871
 872                                 gen_insn(INSN_JMP, 0, 0, 0);
 873                                 gen_four(label_end);
 874                         }
 875
 876                         gen_label(label_skip);
 877                         if (!mod)
 878                                 g(gen_frame_clear(ctx, op_size, slot_r));
 879                         else
 880                                 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
 881                         gen_label(label_end);
 882                 }
 883                 return true;
 884 #else
 885 #if defined(ARCH_MIPS)
 886                 have_mod = true;
 887                 div_op_size = maximum(op_size, OP_SIZE_4);
 888                 if (op_size == OP_SIZE_4)
 889                         force_sx = true;
 890 #endif
 891 #if defined(ARCH_POWER)
 892                 have_mod = cpu_test_feature(CPU_FEATURE_v30);
 893                 div_op_size = maximum(op_size, OP_SIZE_4);
 894 #endif
 895 #if defined(ARCH_LOONGARCH64) || defined(ARCH_RISCV64)
 896                 have_mod = true;
 897                 div_op_size = maximum(op_size, OP_SIZE_4);
 898 #endif
 899                 label_end = alloc_label(ctx);
 900                 if (unlikely(!label_end))
 901                         return false;
 902
 903                 g(gen_frame_get(ctx, op_size, (sgn && op_size < i_size(op_size)) || force_sx ? sign_x : zero_x, slot_1, R_SCRATCH_1, &reg1));
 904                 g(gen_frame_get(ctx, op_size, (sgn && op_size < i_size(op_size)) || force_sx ? sign_x : zero_x, slot_2, R_SCRATCH_2, &reg2));
 905                 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_3);
 906
 907                 if (ARCH_PREFERS_SX(op_size) && !sgn && op_size < i_size(op_size)) {
 908                         g(gen_extend(ctx, op_size, zero_x, R_SCRATCH_1, reg1));
 909                         reg1 = R_SCRATCH_1;
 910                         g(gen_extend(ctx, op_size, zero_x, R_SCRATCH_2, reg2));
 911                         reg2 = R_SCRATCH_2;
 912                 }
 913
 914                 if (mode == MODE_INT) {
 915                         g(gen_jmp_on_zero(ctx, i_size(op_size), reg2, COND_E, label_ovf));
 916                         if (sgn) {
 917                                 uint64_t val;
 918                                 uint32_t label_not_minus_1;
 919                                 label_not_minus_1 = alloc_label(ctx);
 920                                 if (unlikely(!label_not_minus_1))
 921                                         return false;
 922
 923                                 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, i_size(op_size), reg2, -1, COND_NE, label_not_minus_1));
 924
 925                                 val = 0xFFFFFFFFFFFFFF80ULL << (((1 << op_size) - 1) * 8);
 926                                 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, i_size(op_size), reg1, val, COND_E, label_ovf));
 927
 928                                 gen_label(label_not_minus_1);
 929                         }
 930                 } else {
 931 #if !(defined(ARCH_ARM) && ARM_ASM_DIV_NO_TRAP)
 932                         if (!mod) {
 933                                 g(gen_load_constant(ctx, target, 0));
 934                         } else {
 935                                 g(gen_mov(ctx, OP_SIZE_NATIVE, target, reg1));
 936                         }
 937                         g(gen_jmp_on_zero(ctx, i_size(op_size), reg2, COND_E, label_end));
 938                         if (sgn) {
 939                                 uint64_t val;
 940                                 uint32_t label_not_minus_1;
 941                                 label_not_minus_1 = alloc_label(ctx);
 942                                 if (unlikely(!label_not_minus_1))
 943                                         return false;
 944
 945                                 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, i_size(op_size), reg2, -1, COND_NE, label_not_minus_1));
 946
 947                                 if (!mod) {
 948                                         g(gen_mov(ctx, OP_SIZE_NATIVE, target, reg1));
 949                                 } else {
 950                                         g(gen_load_constant(ctx, target, 0));
 951                                 }
 952
 953                                 val = 0xFFFFFFFFFFFFFF80ULL << (((1 << op_size) - 1) * 8);
 954                                 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, i_size(op_size), reg1, val, COND_E, label_end));
 955
 956                                 gen_label(label_not_minus_1);
 957                         }
 958 #endif
 959                 }
 960                 if (mod && have_mod) {
 961                         g(gen_3address_alu(ctx, div_op_size, sgn ? ALU_SREM : ALU_UREM, target, reg1, reg2, 0));
 962                 } else {
 963                         g(gen_3address_alu(ctx, div_op_size, sgn ? ALU_SDIV : ALU_UDIV, target, reg1, reg2, 0));
 964                 }
 965
 966                 if (mod && !have_mod) {
 967 #if defined(ARCH_ARM)
 968                         gen_insn(INSN_MADD, i_size(op_size), 1, 0);
 969                         gen_one(target);
 970                         gen_one(target);
 971                         gen_one(reg2);
 972                         gen_one(reg1);
 973 #else
 974                         g(gen_3address_alu(ctx, i_size(op_size), ALU_MUL, R_SCRATCH_2, reg2, target, 0));
 975                         g(gen_3address_alu(ctx, i_size(op_size), ALU_SUB, target, reg1, R_SCRATCH_2, 0));
 976 #endif
 977                 }
 978
 979                 gen_label(label_end);
 980                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
 981                 return true;
 982 #endif
 983         }
 984         /*********
 985          * SHIFT *
 986          *********/
 987 do_shift: {
 988                 bool sx;
 989                 bool must_mask;
 990                 unsigned op_s;
 991                 int64_t cnst = 0;       /* avoid warning */
 992                 bool c = frame_t_is_const(slot_2);
 993                 if (unlikely(op_size > OP_SIZE_NATIVE)) {
 994                         size_t upcall;
 995                         if (mode == MODE_FIXED) {
 996                                 switch (alu) {
 997                                         case ROT_SHL:   upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_shl_,TYPE_INT_MAX));
 998                                                         break;
 999                                         case ROT_SAR:   upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_shr_,TYPE_INT_MAX));
1000                                                         break;
1001                                         case ROT_SHR:   upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_ushr_,TYPE_INT_MAX));
1002                                                         break;
1003                                         case ROT_ROL:   upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_rol_,TYPE_INT_MAX));
1004                                                         break;
1005                                         case ROT_ROR:   upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_ror_,TYPE_INT_MAX));
1006                                                         break;
1007                                         default:        internal(file_line, "do_alu: invalid shift %u", alu);
1008                                 }
1009                         } else {
1010                                 switch (alu) {
1011                                         case ROT_SHL:   upcall = offsetof(struct cg_upcall_vector_s, cat(INT_binary_shl_,TYPE_INT_MAX));
1012                                                         break;
1013                                         case ROT_SAR:   upcall = offsetof(struct cg_upcall_vector_s, cat(INT_binary_shr_,TYPE_INT_MAX));
1014                                                         break;
1015                                         default:        internal(file_line, "do_alu: invalid shift %u", alu);
1016                                 }
1017                         }
1018                         g(gen_alu_upcall(ctx, upcall, op_size, slot_1, slot_2, slot_r, mode == MODE_INT ? label_ovf : 0));
1019                         return true;
1020                 }
1021                 op_s = i_size_rot(op_size);
1022 #if defined(ARCH_X86)
1023                 if (slot_1 == slot_r && !slot_is_register(ctx, slot_1) && !(mode == MODE_INT && alu == ROT_SHL)) {
1024                         int64_t offset = (size_t)slot_1 * slot_size;
1025                         if (c) {
1026                                 cnst = frame_t_get_const(slot_2);
1027                                 if (mode == MODE_INT) {
1028                                         if ((uint64_t)cnst > (8U << op_size) - 1) {
1029                                                 gen_insn(INSN_JMP, 0, 0, 0);
1030                                                 gen_four(label_ovf);
1031                                                 return true;
1032                                         }
1033                                 } else {
1034                                         cnst &= (8U << op_size) - 1;
1035                                 }
1036                                 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDR_OFFSET, op_size));
1037                                 gen_insn(INSN_ROT + ARCH_PARTIAL_ALU(op_size), op_size, alu, 1);
1038                                 gen_address_offset();
1039                                 gen_address_offset();
1040                                 gen_one(ARG_IMM);
1041                                 gen_eight(cnst);
1042                         } else {
1043                                 g(gen_frame_load(ctx, op_size, garbage, slot_2, 0, false, R_SCRATCH_3));
1044                                 if (mode == MODE_INT) {
1045                                         int64_t imm = (8U << op_size) - 1;
1046                                         g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, op_size, R_SCRATCH_3, imm, COND_A, label_ovf));
1047                                 } else if ((alu != ROT_ROL && alu != ROT_ROR) && op_size < OP_SIZE_4) {
1048                                         g(gen_3address_alu_imm(ctx, OP_SIZE_1, ALU_AND, R_SCRATCH_3, R_SCRATCH_3, (8U << op_size) - 1, 0));
1049                                 }
1050                                 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDR_OFFSET, op_size));
1051                                 gen_insn(INSN_ROT + ARCH_PARTIAL_ALU(op_size), op_size, alu, 1);
1052                                 gen_address_offset();
1053                                 gen_address_offset();
1054                                 gen_one(R_SCRATCH_3);
1055                         }
1056                         return true;
1057                 }
1058                 if (mode == MODE_INT && alu == ROT_SHL && op_size < OP_SIZE_NATIVE)
1059                         op_s = op_size + 1;
1060 #endif
1061                 must_mask = op_size < ARCH_SHIFT_SIZE;
1062                 sx = (alu == ROT_SAR && op_size < op_s) || (alu == ROT_SHL && op_size < OP_SIZE_NATIVE && mode == MODE_INT);
1063 #if defined(ARCH_MIPS)
1064                 sx |= op_size == OP_SIZE_4;
1065 #endif
1066                 g(gen_frame_get(ctx, op_size, sx ? sign_x : zero_x, slot_1, R_SCRATCH_1, &reg1));
1067                 if (c) {
1068                         reg3 = 0xff;    /* avoid warning */
1069                         cnst = frame_t_get_const(slot_2);
1070                 } else
1071 #if defined(ARCH_X86)
1072                 if (!ARCH_IS_3ADDRESS_ROT(alu, op_size)) {
1073                         g(gen_frame_load(ctx, op_size, garbage, slot_2, 0, false, R_SCRATCH_3));
1074                         reg3 = R_SCRATCH_3;
1075                 } else
1076 #endif
1077                 g(gen_frame_get(ctx, op_size, garbage, slot_2, R_SCRATCH_3, &reg3));
1078
1079                 if (ARCH_PREFERS_SX(op_size) && !sx && op_size < op_s) {
1080                         g(gen_extend(ctx, op_size, zero_x, R_SCRATCH_1, reg1));
1081                         reg1 = R_SCRATCH_1;
1082                 }
1083
1084                 if (mode == MODE_INT) {
1085                         int64_t imm = (8U << op_size) - 1;
1086                         if (c) {
1087                                 if ((uint64_t)cnst > (uint64_t)imm) {
1088                                         gen_insn(INSN_JMP, 0, 0, 0);
1089                                         gen_four(label_ovf);
1090                                         return true;
1091                                 }
1092                         } else {
1093                                 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, i_size_cmp(op_size), reg3, imm, COND_A, label_ovf));
1094                         }
1095                 } else {
1096 #if defined(ARCH_ARM)
1097                         if (alu == ROT_ROL) {
1098                                 if (c) {
1099                                         cnst = -(uint64_t)cnst;
1100                                 } else {
1101                                         g(gen_2address_alu1(ctx, OP_SIZE_4, ALU1_NEG, R_SCRATCH_3, reg3, 0));
1102                                         reg3 = R_SCRATCH_3;
1103                                 }
1104                                 alu = ROT_ROR;
1105                         }
1106 #endif
1107 #if defined(ARCH_LOONGARCH64)
1108                         if (alu == ROT_ROL && op_size >= OP_SIZE_4) {
1109                                 if (c) {
1110                                         cnst = -(uint64_t)cnst;
1111                                 } else {
1112                                         g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NEG, R_SCRATCH_3, reg3, 0));
1113                                         reg3 = R_SCRATCH_3;
1114                                 }
1115                                 alu = ROT_ROR;
1116                         }
1117 #endif
1118 #if defined(ARCH_MIPS)
1119                         if (MIPS_HAS_ROT && alu == ROT_ROL && op_size >= OP_SIZE_4) {
1120                                 if (c) {
1121                                         cnst = -(uint64_t)cnst;
1122                                 } else {
1123                                         g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NEG, R_SCRATCH_3, reg3, 0));
1124                                         reg3 = R_SCRATCH_3;
1125                                 }
1126                                 alu = ROT_ROR;
1127                         }
1128 #endif
1129 #if defined(ARCH_POWER)
1130                         if (alu == ROT_ROR && op_size >= OP_SIZE_4) {
1131                                 if (c) {
1132                                         cnst = -(uint64_t)cnst;
1133                                 } else {
1134                                         g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NEG, R_SCRATCH_3, reg3, 0));
1135                                         reg3 = R_SCRATCH_3;
1136                                 }
1137                                 alu = ROT_ROL;
1138                         }
1139 #endif
1140 #if defined(ARCH_S390)
1141                         if (Z && alu == ROT_ROR && op_size >= OP_SIZE_4) {
1142                                 if (c) {
1143                                         cnst = -(uint64_t)cnst;
1144                                 } else {
1145                                         g(gen_2address_alu1(ctx, OP_SIZE_4, ALU1_NEG, R_SCRATCH_3, reg3, 0));
1146                                         reg3 = R_SCRATCH_3;
1147                                 }
1148                                 alu = ROT_ROL;
1149                         }
1150 #endif
1151                         if (c) {
1152                                 cnst &= (8U << op_size) - 1;
1153                         } else if (must_mask) {
1154                                 g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_4), ALU_AND, R_SCRATCH_3, reg3, (8U << op_size) - 1, 0));
1155                                 reg3 = R_SCRATCH_3;
1156                         }
1157                 }
1158
1159 #if defined(ARCH_X86)
1160                 if (mode == MODE_INT && alu == ROT_SHL) {
1161                         target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_2);
1162                 } else {
1163                         target = gen_frame_target(ctx, slot_r, NO_FRAME_T, slot_2, R_SCRATCH_2);
1164                 }
1165                 if (c) {
1166                         g(gen_3address_rot_imm(ctx, op_s, alu, target, reg1, cnst, 0));
1167                 } else  {
1168                         g(gen_3address_rot(ctx, op_s, alu, target, reg1, reg3));
1169                 }
1170
1171                 if (mode == MODE_INT && alu == ROT_SHL) {
1172                         if (op_size < OP_SIZE_NATIVE) {
1173                                 gen_insn(INSN_MOVSX, op_size, 0, 0);
1174                                 gen_one(R_SCRATCH_4);
1175                                 gen_one(target);
1176
1177                                 g(gen_cmp_test_jmp(ctx, INSN_CMP, op_s, target, R_SCRATCH_4, COND_NE, label_ovf));
1178                         } else {
1179                                 if (c) {
1180                                         g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_4, target, cnst, 0));
1181                                 } else {
1182                                         g(gen_3address_rot(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_4, target, reg3));
1183                                 }
1184
1185                                 g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, reg1, R_SCRATCH_4, COND_NE, label_ovf));
1186                         }
1187                 }
1188                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1189                 return true;
1190 #endif
1191 #if defined(ARCH_ARM)
1192                 if (op_size <= OP_SIZE_2 && alu == ROT_ROR) {
1193                         gen_insn(INSN_ALU, OP_SIZE_4, ALU_OR, ALU_WRITES_FLAGS(ALU_OR, false));
1194                         gen_one(R_SCRATCH_1);
1195                         gen_one(reg1);
1196                         gen_one(ARG_SHIFTED_REGISTER);
1197                         gen_one(ARG_SHIFT_LSL | (8U << op_size));
1198                         gen_one(reg1);
1199                         if (op_size == OP_SIZE_1)
1200                                 alu = ROT_SHR;
1201                         reg1 = R_SCRATCH_1;
1202                 }
1203                 goto do_generic_shift;
1204 #endif
1205 #if defined(ARCH_LOONGARCH64)
1206                 if (alu == ROT_ROR && op_size >= OP_SIZE_4)
1207                         goto do_generic_shift;
1208 #endif
1209 #if defined(ARCH_MIPS)
1210                 if (MIPS_HAS_ROT && alu == ROT_ROR && op_size >= OP_SIZE_4)
1211                         goto do_generic_shift;
1212 #endif
1213 #if defined(ARCH_POWER)
1214                 if (alu == ROT_ROL && op_size >= OP_SIZE_4)
1215                         goto do_generic_shift;
1216 #endif
1217 #if defined(ARCH_RISCV64)
1218                 if ((alu == ROT_ROL || alu == ROT_ROR) && likely(cpu_test_feature(CPU_FEATURE_zbb))) {
1219                         if (likely(op_size >= OP_SIZE_4)) {
1220                                 goto do_generic_shift;
1221                         }
1222                 }
1223 #endif
1224 #if defined(ARCH_S390)
1225                 if (Z && alu == ROT_ROL && op_size >= OP_SIZE_4)
1226                         goto do_generic_shift;
1227 #endif
1228                 if (alu == ROT_ROL || alu == ROT_ROR) {
1229                         target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
1230                         if (c) {
1231                                 g(gen_3address_rot_imm(ctx, op_s, alu == ROT_ROL ? ROT_SHL : ROT_SHR, R_SCRATCH_2, reg1, cnst, 0));
1232                                 g(gen_3address_rot_imm(ctx, op_s, alu == ROT_ROL ? ROT_SHR : ROT_SHL, target, reg1, -(uint64_t)cnst & ((8U << op_size) - 1), 0));
1233                         } else {
1234                                 g(gen_3address_rot(ctx, op_s, alu == ROT_ROL ? ROT_SHL : ROT_SHR, R_SCRATCH_2, reg1, reg3));
1235                                 g(gen_2address_alu1(ctx, i_size(OP_SIZE_4), ALU1_NEG, R_SCRATCH_3, reg3, 0));
1236                                 if (must_mask) {
1237                                         g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_4), ALU_AND, R_SCRATCH_3, R_SCRATCH_3, (8U << op_size) - 1, 0));
1238                                 }
1239                                 g(gen_3address_rot(ctx, op_s, alu == ROT_ROL ? ROT_SHR : ROT_SHL, target, reg1, R_SCRATCH_3));
1240                         }
1241                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_OR, target, target, R_SCRATCH_2, 0));
1242                         g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1243                         return true;
1244                 }
1245
1246                 goto do_generic_shift;
1247 do_generic_shift:
1248                 if (mode == MODE_INT && alu == ROT_SHL) {
1249                         target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
1250 #if defined(ARCH_S390)
1251                         if (op_size >= OP_SIZE_4) {
1252                                 if (c)
1253                                         g(gen_3address_rot_imm(ctx, op_size, ROT_SAL, target, reg1, cnst, 0));
1254                                 else
1255                                         g(gen_3address_rot(ctx, op_size, ROT_SAL, target, reg1, reg3));
1256
1257                                 gen_insn(INSN_JMP_COND, op_size, COND_O, 0);
1258                                 gen_four(label_ovf);
1259                         } else
1260 #endif
1261                         if (op_size <= OP_SIZE_NATIVE - 1) {
1262                                 if (c)
1263                                         g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, alu, target, reg1, cnst, 0));
1264                                 else
1265                                         g(gen_3address_rot(ctx, OP_SIZE_NATIVE, alu, target, reg1, reg3));
1266
1267                                 g(gen_cmp_extended(ctx, OP_SIZE_NATIVE, op_size, target, R_SCRATCH_2, label_ovf));
1268                         } else {
1269                                 if (c) {
1270                                         g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, alu, R_SCRATCH_2, reg1, cnst, 0));
1271                                         g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_4, R_SCRATCH_2, cnst, 0));
1272                                 } else {
1273                                         g(gen_3address_rot(ctx, OP_SIZE_NATIVE, alu, R_SCRATCH_2, reg1, reg3));
1274                                         g(gen_3address_rot(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_4, R_SCRATCH_2, reg3));
1275                                 }
1276
1277                                 g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, reg1, R_SCRATCH_4, COND_NE, label_ovf));
1278
1279                                 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_2));
1280
1281                                 return true;
1282                         }
1283                 } else {
1284                         target = gen_frame_target(ctx, slot_r, NO_FRAME_T, slot_2, R_SCRATCH_1);
1285                         if (c)
1286                                 g(gen_3address_rot_imm(ctx, op_s, alu, target, reg1, cnst, 0));
1287                         else
1288                                 g(gen_3address_rot(ctx, op_s, alu, target, reg1, reg3));
1289                 }
1290
1291                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1292                 return true;
1293         }
1294         /******
1295          * BT *
1296          ******/
1297 do_bt: {
1298                 unsigned attr_unused op_s;
1299                 bool need_mask;
1300 #if defined(ARCH_X86)
1301                 if ((alu == BTX_BT || slot_1 == slot_r) && !slot_is_register(ctx, slot_1)) {
1302                         unsigned n_op_size = minimum(op_size, OP_SIZE_NATIVE);
1303                         g(gen_frame_get(ctx, n_op_size, garbage, slot_2, R_SCRATCH_1, &reg2));
1304                         if (mode == MODE_INT) {
1305                                 int64_t imm = (8U << op_size) - 1;
1306                                 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, n_op_size, reg2, imm, alu == BTX_BT ? COND_A : COND_AE, label_ovf));
1307                                 if (unlikely(op_size > OP_SIZE_NATIVE)) {
1308                                         g(gen_address(ctx, R_FRAME, (size_t)slot_2 * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_STR_OFFSET, OP_SIZE_NATIVE));
1309                                         gen_insn(INSN_CMP, OP_SIZE_NATIVE, 0, 1);
1310                                         gen_address_offset();
1311                                         gen_one(ARG_IMM);
1312                                         gen_eight(0);
1313                                         gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_NE, 0);
1314                                         gen_four(label_ovf);
1315                                 }
1316                         } else {
1317                                 g(gen_3address_alu_imm(ctx, OP_SIZE_4, ALU_AND, R_SCRATCH_1, reg2, (8U << op_size) - 1, 0));
1318                                 reg2 = R_SCRATCH_1;
1319                         }
1320                         g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size, IMM_PURPOSE_STR_OFFSET, maximum(n_op_size, OP_SIZE_2)));
1321                         if (alu == BTX_BT) {
1322                                 gen_insn(INSN_BT, maximum(n_op_size, OP_SIZE_2), 0, 1);
1323                                 gen_address_offset();
1324                                 gen_one(reg2);
1325                                 g(gen_frame_set_cond(ctx, maximum(n_op_size, OP_SIZE_2), false, COND_B, slot_r));
1326                         } else {
1327                                 gen_insn(INSN_BTX, maximum(n_op_size, OP_SIZE_2), alu, 1);
1328                                 gen_address_offset();
1329                                 gen_address_offset();
1330                                 gen_one(reg2);
1331                         }
1332                         return true;
1333                 }
1334 #endif
1335                 if (unlikely(op_size > OP_SIZE_NATIVE)) {
1336                         size_t upcall;
1337                         if (mode == MODE_FIXED) {
1338                                 switch (alu) {
1339                                         case BTX_BTS:   upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_bts_,TYPE_INT_MAX));
1340                                                         break;
1341                                         case BTX_BTR:   upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_btr_,TYPE_INT_MAX));
1342                                                         break;
1343                                         case BTX_BTC:   upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_btc_,TYPE_INT_MAX));
1344                                                         break;
1345                                         case BTX_BT:    upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_bt_,TYPE_INT_MAX));
1346                                                         break;
1347                                         default:        internal(file_line, "do_alu: invalid bit test %u", alu);
1348                                 }
1349                         } else {
1350                                 switch (alu) {
1351                                         case BTX_BTS:   upcall = offsetof(struct cg_upcall_vector_s, cat(INT_binary_bts_,TYPE_INT_MAX));
1352                                                         break;
1353                                         case BTX_BTR:   upcall = offsetof(struct cg_upcall_vector_s, cat(INT_binary_btr_,TYPE_INT_MAX));
1354                                                         break;
1355                                         case BTX_BTC:   upcall = offsetof(struct cg_upcall_vector_s, cat(INT_binary_btc_,TYPE_INT_MAX));
1356                                                         break;
1357                                         case BTX_BT:    upcall = offsetof(struct cg_upcall_vector_s, cat(INT_binary_bt_,TYPE_INT_MAX));
1358                                                         break;
1359                                         default:        internal(file_line, "do_alu: invalid bit test %u", alu);
1360                                 }
1361                         }
1362                         g(gen_alu_upcall(ctx, upcall, op_size, slot_1, slot_2, slot_r, label_ovf));
1363                         return true;
1364                 }
1365                 op_s = minimum(OP_SIZE_NATIVE, ARCH_SHIFT_SIZE);
1366                 op_s = maximum(op_s, op_size);
1367                 g(gen_frame_get(ctx, op_size, zero_x, slot_1, R_SCRATCH_1, &reg1));
1368                 g(gen_frame_get(ctx, op_size, garbage, slot_2, R_SCRATCH_2, &reg2));
1369                 if (mode == MODE_INT) {
1370                         int64_t imm = (8U << op_size) - 1;
1371                         g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, i_size_cmp(op_size), reg2, imm, alu == BTX_BT ? COND_A : COND_AE, label_ovf));
1372                 }
1373                 if (alu != BTX_BT) {
1374                         if (!ARCH_HAS_BTX(alu, OP_SIZE_NATIVE, false))
1375                                 goto do_generic_bt;
1376                         need_mask = !ARCH_HAS_BTX(alu, op_size, false);
1377                 } else {
1378 #if defined(ARCH_X86)
1379                         need_mask = op_size < OP_SIZE_2;
1380 #else
1381                         if (!ARCH_HAS_BTX(BTX_BTEXT, OP_SIZE_NATIVE, false))
1382                                 goto do_generic_bt;
1383                         need_mask = !ARCH_HAS_BTX(BTX_BTEXT, op_size, false);
1384 #endif
1385                 }
1386                 if (need_mask) {
1387                         g(gen_3address_alu_imm(ctx, OP_SIZE_4, ALU_AND, R_SCRATCH_2, reg2, (8U << op_size) - 1, 0));
1388                         reg2 = R_SCRATCH_2;
1389                 }
1390                 if (alu == BTX_BT) {
1391 #if defined(ARCH_X86)
1392                         gen_insn(INSN_BT, maximum(op_size, OP_SIZE_2), 0, 1);
1393                         gen_one(reg1);
1394                         gen_one(reg2);
1395
1396                         g(gen_frame_set_cond(ctx, maximum(op_size, OP_SIZE_2), false, COND_B, slot_r));
1397 #else
1398                         target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1399                         gen_insn(INSN_BTX, need_mask ? OP_SIZE_NATIVE : op_size, BTX_BTEXT, 0);
1400                         gen_one(target);
1401                         gen_one(reg1);
1402                         gen_one(reg2);
1403
1404                         g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
1405 #endif
1406                 } else {
1407                         target = gen_frame_target(ctx, slot_r, NO_FRAME_T, slot_2, R_SCRATCH_1);
1408 #if defined(ARCH_X86)
1409                         if (target == reg2)
1410                                 target = R_SCRATCH_1;
1411                         if (target != reg1) {
1412                                 g(gen_mov(ctx, op_size, target, reg1));
1413                                 reg1 = target;
1414                         }
1415                         gen_insn(INSN_BTX, maximum(op_size, OP_SIZE_2), alu, 1);
1416 #else
1417                         gen_insn(INSN_BTX, need_mask ? OP_SIZE_NATIVE : op_size, alu, 0);
1418 #endif
1419                         gen_one(target);
1420                         gen_one(reg1);
1421                         gen_one(reg2);
1422
1423                         g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1424                 }
1425                 return true;
1426
1427                 goto do_generic_bt;
1428 do_generic_bt:
1429                 if (mode == MODE_FIXED && op_size < ARCH_SHIFT_SIZE) {
1430                         g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_4), ALU_AND, R_SCRATCH_2, reg2, (8U << op_size) - 1, 0));
1431                         reg2 = R_SCRATCH_2;
1432                 }
1433                 g(gen_load_constant(ctx, R_SCRATCH_3, 1));
1434
1435                 g(gen_3address_rot(ctx, op_s, ROT_SHL, R_SCRATCH_3, R_SCRATCH_3, reg2));
1436
1437                 switch (alu) {
1438                         case BTX_BT:
1439 #if ARCH_HAS_FLAGS
1440 #if defined(ARCH_S390) || defined(ARCH_POWER)
1441                                 g(gen_3address_alu(ctx, i_size(op_size), ALU_AND, R_SCRATCH_1, reg1, R_SCRATCH_3, 1));
1442 #else
1443                                 gen_insn(INSN_TEST, i_size(op_size), 0, 1);
1444                                 gen_one(reg1);
1445                                 gen_one(R_SCRATCH_3);
1446 #endif
1447                                 g(gen_frame_set_cond(ctx, i_size_cmp(op_size), false, COND_NE, slot_r));
1448 #else
1449                                 g(gen_3address_alu(ctx, i_size(op_size), ALU_AND, R_SCRATCH_1, reg1, R_SCRATCH_3, 0));
1450                                 g(gen_frame_cmp_imm_set_cond_reg(ctx, i_size(op_size), R_SCRATCH_1, 0, COND_NE, slot_r));
1451 #endif
1452                                 return true;
1453                         case BTX_BTS:
1454                                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1455                                 g(gen_3address_alu(ctx, i_size(op_size), ALU_OR, target, reg1, R_SCRATCH_3, 0));
1456                                 break;
1457                         case BTX_BTR:
1458                                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1459                                 if (!ARCH_HAS_ANDN) {
1460                                         g(gen_3address_alu_imm(ctx, i_size(op_size), ALU_XOR, R_SCRATCH_3, R_SCRATCH_3, -1, 0));
1461
1462                                         g(gen_3address_alu(ctx, i_size(op_size), ALU_AND, target, reg1, R_SCRATCH_3, 0));
1463                                         break;
1464                                 }
1465                                 g(gen_3address_alu(ctx, i_size(op_size), ALU_ANDN, target, reg1, R_SCRATCH_3, 0));
1466                                 break;
1467                         case BTX_BTC:
1468                                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1469                                 g(gen_3address_alu(ctx, i_size(op_size), ALU_XOR, target, reg1, R_SCRATCH_3, 0));
1470                                 break;
1471                         default:
1472                                 internal(file_line, "gen_alu: unsupported bit test %u", alu);
1473                 }
1474
1475                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1476
1477                 return true;
1478         }
1479         /***********
1480          * COMPARE *
1481          ***********/
1482 do_compare: {
1483                 if (unlikely(op_size > OP_SIZE_NATIVE)) {
1484                         size_t attr_unused upcall;
1485                         switch (alu) {
1486                                 case COND_E:
1487                                 case COND_NE:
1488                                         g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
1489                                         g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, ALU_XOR, 0, slot_2, lo_word(OP_SIZE_NATIVE), true, R_SCRATCH_1));
1490                                         g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, ALU_XOR, 0, slot_2, hi_word(OP_SIZE_NATIVE), true, R_SCRATCH_2));
1491 #if defined(ARCH_ARM64)
1492                                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_OR, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_2, 0));
1493
1494                                         gen_insn(INSN_CMP, OP_SIZE_NATIVE, 0, 1);
1495                                         gen_one(R_SCRATCH_1);
1496                                         gen_one(ARG_IMM);
1497                                         gen_eight(0);
1498 #else
1499                                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_OR, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_2, ARCH_HAS_FLAGS));
1500 #endif
1501 #if ARCH_HAS_FLAGS
1502                                         g(gen_frame_set_cond(ctx, OP_SIZE_NATIVE, false, alu, slot_r));
1503 #else
1504                                         g(gen_frame_cmp_imm_set_cond_reg(ctx, OP_SIZE_NATIVE, R_SCRATCH_1, 0, alu, slot_r));
1505 #endif
1506                                         return true;
1507 #if defined(ARCH_X86_64) || defined(ARCH_X86_X32) || defined(ARCH_ARM)
1508                                 case COND_L:
1509                                 case COND_B:
1510                                         g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_2, lo_word(OP_SIZE_NATIVE), true, R_SCRATCH_2));
1511                                         g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_1, hi_word(OP_SIZE_NATIVE), true, R_SCRATCH_1));
1512                                         g(gen_frame_load_cmp(ctx, OP_SIZE_NATIVE, false, garbage, true, slot_1, lo_word(OP_SIZE_NATIVE), true, R_SCRATCH_2));
1513                                         g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, ALU_SBB, 1, slot_2, hi_word(OP_SIZE_NATIVE), true, R_SCRATCH_1));
1514                                         g(gen_frame_set_cond(ctx, OP_SIZE_NATIVE, false, alu, slot_r));
1515                                         return true;
1516                                 case COND_LE:
1517                                 case COND_BE:
1518                                         g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_1, lo_word(OP_SIZE_NATIVE), true, R_SCRATCH_2));
1519                                         g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_2, hi_word(OP_SIZE_NATIVE), true, R_SCRATCH_1));
1520                                         g(gen_frame_load_cmp(ctx, OP_SIZE_NATIVE, false, garbage, true, slot_2, lo_word(OP_SIZE_NATIVE), true, R_SCRATCH_2));
1521                                         g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, ALU_SBB, 1, slot_1, hi_word(OP_SIZE_NATIVE), true, R_SCRATCH_1));
1522                                         g(gen_frame_set_cond(ctx, OP_SIZE_NATIVE, false, alu == COND_LE ? COND_GE : COND_AE, slot_r));
1523                                         return true;
1524 #else
1525                                 case COND_L:    upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_less_,TYPE_INT_MAX)); goto do_upcall;
1526                                 case COND_B:    upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_uless_,TYPE_INT_MAX)); goto do_upcall;
1527                                 case COND_LE:   upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_less_equal_,TYPE_INT_MAX)); goto do_upcall;
1528                                 case COND_BE:   upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_uless_equal_,TYPE_INT_MAX)); goto do_upcall;
1529                                 do_upcall:      g(gen_alu_upcall(ctx, upcall, op_size, slot_1, slot_2, slot_r, 0));
1530                                                 return true;
1531 #endif
1532                                 default:
1533                                         internal(file_line, "gen_alu: unsupported condition %u", alu);
1534                         }
1535                         return false;
1536                 }
1537 #if defined(ARCH_X86)
1538                 g(gen_frame_get(ctx, op_size, garbage, slot_1, R_SCRATCH_1, &reg1));
1539                 g(gen_frame_load_cmp_set_cond(ctx, op_size, garbage, slot_2, reg1, alu, slot_r));
1540 #else
1541                 g(gen_frame_get(ctx, op_size, op_size == i_size_cmp(op_size) + (unsigned)zero ? garbage : alu == COND_L || alu == COND_LE || ARCH_PREFERS_SX(op_size) ? sign_x : zero_x, slot_1, R_SCRATCH_1, &reg1));
1542                 g(gen_frame_load_cmp_set_cond(ctx, op_size, alu == COND_L || alu == COND_LE || ARCH_PREFERS_SX(op_size) ? sign_x : zero_x, slot_2, reg1, alu, slot_r));
1543 #endif
1544                 return true;
1545         }
1546 }
1547
1548 static bool attr_w gen_alu1(struct codegen_context *ctx, unsigned mode, unsigned op_size, unsigned op, uint32_t label_ovf, frame_t slot_1, frame_t slot_r)
1549 {
1550         unsigned alu;
1551         unsigned reg1, target;
1552         switch (mode) {
1553                 case MODE_FIXED: switch (op) {
1554                         case OPCODE_FIXED_OP_not:               alu = ALU1_NOT; goto do_alu;
1555                         case OPCODE_FIXED_OP_neg:               alu = ALU1_NEG; goto do_alu;
1556                         case OPCODE_FIXED_OP_inc:               alu = ALU1_INC; goto do_alu;
1557                         case OPCODE_FIXED_OP_dec:               alu = ALU1_DEC; goto do_alu;
1558                         case OPCODE_FIXED_OP_bswap:
1559                         case OPCODE_FIXED_OP_bswap_alt1:        alu = ALU1_BSWAP; goto do_bswap;
1560                         case OPCODE_FIXED_OP_brev:
1561                         case OPCODE_FIXED_OP_brev_alt1:         alu = ALU1_BREV; goto do_brev;
1562                         case OPCODE_FIXED_OP_bsf:
1563                         case OPCODE_FIXED_OP_bsf_alt1:          alu = ALU1_BSF; goto do_bsf_bsr_popcnt;
1564                         case OPCODE_FIXED_OP_bsr:
1565                         case OPCODE_FIXED_OP_bsr_alt1:          alu = ALU1_BSR; goto do_bsf_bsr_popcnt;
1566                         case OPCODE_FIXED_OP_popcnt:
1567                         case OPCODE_FIXED_OP_popcnt_alt1:       alu = ALU1_POPCNT; goto do_bsf_bsr_popcnt;
1568                         case OPCODE_FIXED_OP_to_int:            goto do_fixed_conv;
1569                         case OPCODE_FIXED_OP_from_int:          goto do_fixed_conv;
1570                         case OPCODE_FIXED_OP_uto_int:           goto conv_uto_int;
1571                         case OPCODE_FIXED_OP_ufrom_int:         goto conv_ufrom_int;
1572                         default:                                internal(file_line, "gen_alu1: unsupported fixed operation %u", op);
1573                 }
1574                 case MODE_INT: switch (op) {
1575                         case OPCODE_INT_OP_not:                 alu = ALU1_NOT; mode = MODE_FIXED; goto do_alu;
1576                         case OPCODE_INT_OP_neg:                 alu = ALU1_NEG; goto do_alu;
1577                         case OPCODE_INT_OP_inc:                 alu = ALU1_INC; goto do_alu;
1578                         case OPCODE_INT_OP_dec:                 alu = ALU1_DEC; goto do_alu;
1579                         case OPCODE_INT_OP_bsf:                 alu = ALU1_BSF; goto do_bsf_bsr_popcnt;
1580                         case OPCODE_INT_OP_bsr:                 alu = ALU1_BSR; goto do_bsf_bsr_popcnt;
1581                         case OPCODE_INT_OP_popcnt:
1582                         case OPCODE_INT_OP_popcnt_alt1:         alu = ALU1_POPCNT; goto do_bsf_bsr_popcnt;
1583                         case OPCODE_INT_OP_to_int:              goto do_conv;
1584                         case OPCODE_INT_OP_from_int:            goto do_conv;
1585                         default:                                internal(file_line, "gen_alu1: unsupported int operation %u", op);
1586                 }
1587                 case MODE_BOOL: switch (op) {
1588                         case OPCODE_BOOL_OP_not:                goto do_bool_not;
1589                         default:                                internal(file_line, "gen_alu1: unsupported bool operation %u", op);
1590                 }
1591         }
1592         internal(file_line, "gen_alu1: unsupported mode %u", mode);
1593
1594         /*******
1595          * ALU *
1596          *******/
1597 do_alu: {
1598                 bool arch_use_flags = ARCH_HAS_FLAGS;
1599                 enum extend ex;
1600 #if defined(ARCH_POWER)
1601                 arch_use_flags = false;
1602 #endif
1603                 if (op_size > OP_SIZE_NATIVE) {
1604 #if !defined(ARCH_X86) && !defined(ARCH_ARM) && !defined(ARCH_POWER)
1605                         if (alu == ALU1_NEG) {
1606                                 if (mode == MODE_FIXED)
1607                                         g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(FIXED_unary_neg_,TYPE_INT_MAX)), op_size, slot_1, NO_FRAME_T, slot_r, 0));
1608                                 else
1609                                         g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(INT_unary_neg_,TYPE_INT_MAX)), op_size, slot_1, NO_FRAME_T, slot_r, label_ovf));
1610                                 return true;
1611                         }
1612                         if (alu == ALU1_DEC) {
1613                                 if (mode == MODE_FIXED)
1614                                         g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(FIXED_unary_dec_,TYPE_INT_MAX)), op_size, slot_1, NO_FRAME_T, slot_r, 0));
1615                                 else
1616                                         g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(INT_unary_dec_,TYPE_INT_MAX)), op_size, slot_1, NO_FRAME_T, slot_r, label_ovf));
1617                                 return true;
1618                         }
1619                         if (alu == ALU1_INC) {
1620                                 if (mode == MODE_FIXED)
1621                                         g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(FIXED_unary_inc_,TYPE_INT_MAX)), op_size, slot_1, NO_FRAME_T, slot_r, 0));
1622                                 else
1623                                         g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(INT_unary_inc_,TYPE_INT_MAX)), op_size, slot_1, NO_FRAME_T, slot_r, label_ovf));
1624                                 return true;
1625                         }
1626 #endif
1627                         g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
1628 #if defined(ARCH_S390)
1629                         if (alu == ALU1_NOT) {
1630                                 g(gen_load_constant(ctx, R_SCRATCH_3, -1));
1631
1632                                 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_XOR, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_3, 0));
1633                                 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_XOR, R_SCRATCH_2, R_SCRATCH_2, R_SCRATCH_3, 0));
1634
1635                                 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_1, R_SCRATCH_2));
1636                                 return true;
1637                         }
1638 #endif
1639                         g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, alu, R_SCRATCH_1, R_SCRATCH_1, alu == ALU1_INC || alu == ALU1_DEC || alu == ALU1_NEG ? 2 : 0));
1640                         if (alu == ALU1_NOT) {
1641                                 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NOT, R_SCRATCH_2, R_SCRATCH_2, 0));
1642                         } else if (alu == ALU1_INC || alu == ALU1_DEC) {
1643                                 g(gen_imm(ctx, 0, alu == ALU1_INC ? IMM_PURPOSE_ADD : IMM_PURPOSE_SUB, OP_SIZE_NATIVE));
1644                                 gen_insn(INSN_ALU, OP_SIZE_NATIVE, alu == ALU1_INC ? ALU_ADC : ALU_SBB, (mode == MODE_INT) | ALU_WRITES_FLAGS(alu == ALU1_INC ? ALU_ADC : ALU_SBB, is_imm()));
1645                                 gen_one(R_SCRATCH_2);
1646                                 gen_one(R_SCRATCH_2);
1647                                 gen_imm_offset();
1648                         } else {
1649 #if defined(ARCH_X86)
1650                                 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NOT, R_SCRATCH_2, R_SCRATCH_2, 0));
1651
1652                                 g(gen_imm(ctx, -1, IMM_PURPOSE_SUB, OP_SIZE_NATIVE));
1653                                 gen_insn(INSN_ALU, OP_SIZE_NATIVE, ALU_SBB, ALU_WRITES_FLAGS(ALU_SBB, is_imm()));
1654                                 gen_one(R_SCRATCH_2);
1655                                 gen_one(R_SCRATCH_2);
1656                                 gen_imm_offset();
1657 #else
1658                                 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NGC, R_SCRATCH_2, R_SCRATCH_2, (mode == MODE_INT)));
1659 #endif
1660                         }
1661                         if (mode == MODE_INT) {
1662                                 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_O, 0);
1663                                 gen_four(label_ovf);
1664                         }
1665                         g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_1, R_SCRATCH_2));
1666                         return true;
1667                 }
1668                 if ((arch_use_flags || ARCH_SUPPORTS_TRAPS) && slot_1 == slot_r && i_size_cmp(op_size) == op_size + zero) {
1669                         struct cg_exit *ce;
1670                         unsigned undo_alu = alu == ALU1_INC ? ALU1_DEC : alu == ALU1_DEC ? ALU1_INC : alu;
1671                         if (slot_is_register(ctx, slot_1)) {
1672                                 unsigned reg = ctx->registers[slot_1];
1673                                 if (mode == MODE_INT && ARCH_SUPPORTS_TRAPS) {
1674                                         gen_insn(INSN_ALU1_TRAP, op_size, alu, ALU1_WRITES_FLAGS(alu));
1675                                         gen_one(reg);
1676                                         gen_one(reg);
1677                                         if (ARCH_TRAP_BEFORE || alu == undo_alu) {
1678                                                 gen_four(label_ovf);
1679                                                 return true;
1680                                         } else {
1681                                                 ce = alloc_undo_label(ctx);
1682                                                 if (unlikely(!ce))
1683                                                         return false;
1684                                                 gen_four(ce->undo_label);
1685                                                 goto do_undo_opcode;
1686                                         }
1687                                 }
1688                                 g(gen_2address_alu1(ctx, i_size(op_size), alu, reg, reg, mode == MODE_INT));
1689                                 if (mode == MODE_INT) {
1690                                         if (alu != undo_alu) {
1691                                                 ce = alloc_undo_label(ctx);
1692                                                 if (unlikely(!ce))
1693                                                         return false;
1694                                                 gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
1695                                                 gen_four(ce->undo_label);
1696 do_undo_opcode:
1697                                                 ce->undo_opcode = INSN_ALU1 + ARCH_PARTIAL_ALU(op_size);
1698                                                 ce->undo_op_size = i_size(op_size);
1699                                                 ce->undo_aux = undo_alu;
1700                                                 ce->undo_writes_flags = ALU1_WRITES_FLAGS(undo_alu);
1701                                                 ce->undo_parameters[0] = reg;
1702                                                 ce->undo_parameters[1] = reg;
1703                                                 ce->undo_parameters_len = 2;
1704                                         } else {
1705                                                 gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
1706                                                 gen_four(label_ovf);
1707                                         }
1708                                 }
1709                                 return true;
1710                         }
1711 #if defined(ARCH_X86)
1712                         else {
1713                                 size_t m;
1714                                 int64_t offset = (size_t)slot_1 * slot_size;
1715                                 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDR_OFFSET, i_size(op_size)));
1716                                 gen_insn(INSN_ALU1 + ARCH_PARTIAL_ALU(op_size), i_size(op_size), alu, ALU1_WRITES_FLAGS(alu) | (mode == MODE_INT));
1717                                 gen_address_offset();
1718                                 gen_address_offset();
1719                                 if (mode == MODE_INT) {
1720                                         if (alu != undo_alu) {
1721                                                 ce = alloc_undo_label(ctx);
1722                                                 if (unlikely(!ce))
1723                                                         return false;
1724                                                 ce->undo_opcode = INSN_ALU1 + ARCH_PARTIAL_ALU(op_size);
1725                                                 ce->undo_op_size = i_size(op_size);
1726                                                 ce->undo_aux = undo_alu;
1727                                                 ce->undo_writes_flags = ALU1_WRITES_FLAGS(undo_alu);
1728                                                 m = mark_params(ctx);
1729                                                 gen_address_offset();
1730                                                 gen_address_offset();
1731                                                 copy_params(ctx, ce, m);
1732                                                 gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
1733                                                 gen_four(ce->undo_label);
1734                                         } else {
1735                                                 gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
1736                                                 gen_four(label_ovf);
1737                                         }
1738                                 }
1739                                 return true;
1740                         }
1741 #endif
1742                 }
1743                 target = gen_frame_target(ctx, slot_r, mode == MODE_INT ? slot_1 : NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1744                 if (mode == MODE_FIXED) {
1745                         ex = garbage;
1746                 } else {
1747                         ex = ARCH_PREFERS_SX(op_size) ? sign_x : zero_x;
1748                         if (ARCH_SUPPORTS_TRAPS && op_size >= OP_SIZE_4)
1749                                 ex = garbage;
1750                         if (op_size == i_size(op_size) + (unsigned)zero)
1751                                 ex = garbage;
1752                 }
1753                 g(gen_frame_get(ctx, op_size, ex, slot_1, mode == MODE_INT ? R_SCRATCH_2 : target, &reg1));
1754 #if defined(ARCH_S390)
1755                 if (alu == ALU1_NOT) {
1756                         g(gen_3address_alu_imm(ctx, i_size(op_size), ALU_XOR, target, reg1, -1, 0));
1757
1758                         g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1759                         return true;
1760                 }
1761 #endif
1762 #if defined(ARCH_X86)
1763                 g(gen_2address_alu1(ctx, op_size, alu, target, reg1, mode == MODE_INT));
1764 #else
1765                 if (mode == MODE_INT) {
1766 #if defined(ARCH_POWER)
1767                         if (op_size == OP_SIZE_NATIVE) {
1768                                 g(gen_2address_alu1(ctx, i_size(op_size), alu, target, reg1, 0));
1769                                 if (alu == ALU1_NEG) {
1770                                         g(gen_3address_alu(ctx, i_size(op_size), ALU_AND, R_CG_SCRATCH, target, reg1, 1));
1771                                 } else if (alu == ALU1_INC) {
1772                                         g(gen_3address_alu(ctx, i_size(op_size), ALU_ANDN, R_CG_SCRATCH, target, reg1, 1));
1773                                 } else if (alu == ALU1_DEC) {
1774                                         g(gen_3address_alu(ctx, i_size(op_size), ALU_ANDN, R_CG_SCRATCH, reg1, target, 1));
1775                                 }
1776                                 gen_insn(INSN_JMP_COND, op_size, COND_L, 0);
1777                                 gen_four(label_ovf);
1778
1779                                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1780
1781                                 return true;
1782                         }
1783 #endif
1784                         if (ARCH_HAS_JMP_2REGS(COND_L) && ARCH_HAS_JMP_2REGS(COND_G) && (alu == ALU1_INC || alu == ALU1_DEC) && op_size == i_size_cmp(op_size) + (unsigned)zero) {
1785                                 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, alu, target, reg1, 0));
1786
1787                                 g(gen_cmp_test_jmp(ctx, INSN_CMP, i_size_cmp(op_size), target, reg1, alu == ALU1_INC ? COND_L : COND_G, label_ovf));
1788
1789                                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1790
1791                                 return true;
1792                         }
1793                         if (!arch_use_flags && !ARCH_SUPPORTS_TRAPS && ARCH_HAS_ANDN && op_size >= OP_SIZE_4) {
1794                                 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, alu, target, reg1, 0));
1795
1796                                 if (alu == ALU1_NEG) {
1797                                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_AND, R_SCRATCH_3, target, reg1, 0));
1798                                 } else if (alu == ALU1_INC) {
1799                                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_ANDN, R_SCRATCH_3, target, reg1, 0));
1800                                 } else if (alu == ALU1_DEC) {
1801                                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_ANDN, R_SCRATCH_3, reg1, target, 0));
1802                                 }
1803
1804                                 if (op_size < OP_SIZE_NATIVE)
1805                                         g(gen_extend(ctx, op_size, sign_x, R_SCRATCH_3, R_SCRATCH_3));
1806
1807                                 g(gen_jmp_on_zero(ctx, OP_SIZE_NATIVE, R_SCRATCH_3, COND_S, label_ovf));
1808
1809                                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1810
1811                                 return true;
1812                         }
1813                         if (op_size <= OP_SIZE_2 || (!arch_use_flags && !ARCH_SUPPORTS_TRAPS)) {
1814                                 int64_t imm = ((alu != ALU1_INC && ARCH_PREFERS_SX(op_size) ? -0x80ULL : 0x80ULL) << (((1 << op_size) - 1) * 8)) - (alu == ALU1_INC);
1815
1816                                 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, i_size_cmp(op_size), reg1, imm, COND_E, label_ovf));
1817
1818                                 mode = MODE_FIXED;
1819
1820                                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1821                         }
1822                 }
1823 #if !ARCH_HAS_FLAGS
1824                 if (mode == MODE_INT) {
1825                         gen_insn(INSN_ALU1_TRAP, op_size, alu, ALU1_WRITES_FLAGS(alu));
1826                         gen_one(target);
1827                         gen_one(reg1);
1828                         gen_four(label_ovf);
1829                         g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1830                         return true;
1831                 }
1832 #endif
1833                 g(gen_2address_alu1(ctx, i_size(op_size), alu, target, reg1, mode == MODE_INT));
1834 #endif
1835                 if (mode == MODE_INT) {
1836                         gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
1837                         gen_four(label_ovf);
1838                 }
1839                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1840                 return true;
1841         }
1842
1843         /*******
1844          * NOT *
1845          *******/
1846 do_bool_not: {
1847                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1848                 g(gen_frame_get(ctx, op_size, garbage, slot_1, target, &reg1));
1849
1850                 g(gen_3address_alu_imm(ctx, i_size(op_size), ALU_XOR, target, reg1, 1, 0));
1851
1852                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1853                 return true;
1854         }
1855
1856         /*********
1857          * BSWAP *
1858          *********/
1859 do_bswap: {
1860                 bool attr_unused sx = false;
1861 #if defined(ARCH_X86) || defined(ARCH_ARM) || defined(ARCH_IA64) || defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS) || defined(ARCH_RISCV64) || defined(ARCH_S390)
1862 #if defined(ARCH_ARM32)
1863                 if (unlikely(!cpu_test_feature(CPU_FEATURE_armv6)))
1864                         goto do_generic_bswap;
1865 #endif
1866 #if defined(ARCH_MIPS)
1867                 if (unlikely(!MIPS_HAS_ROT))
1868                         goto do_generic_bswap;
1869                 sx = op_size == OP_SIZE_4;
1870 #endif
1871 #if defined(ARCH_RISCV64)
1872                 if (unlikely(!cpu_test_feature(CPU_FEATURE_zbb)))
1873                         goto do_generic_bswap;
1874 #endif
1875 #if defined(ARCH_S390)
1876                 if (op_size == OP_SIZE_2)
1877                         goto do_generic_bswap;
1878 #endif
1879 #if defined(ARCH_X86)
1880                 if (op_size >= OP_SIZE_4 && !cpu_test_feature(CPU_FEATURE_bswap))
1881                         goto do_generic_bswap;
1882 #endif
1883                 if (op_size > OP_SIZE_NATIVE) {
1884                         g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
1885                         reg1 = R_SCRATCH_1;
1886                         target = R_SCRATCH_1;
1887                 } else {
1888                         target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1889                         g(gen_frame_get(ctx, op_size, sx ? sign_x : garbage, slot_1, target, &reg1));
1890                 }
1891
1892                 if (op_size == OP_SIZE_1) {
1893 #if defined(ARCH_IA64) || defined(ARCH_RISCV64)
1894                 } else if (op_size == OP_SIZE_2 || op_size == OP_SIZE_4) {
1895                         g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_BSWAP, target, reg1, 0));
1896
1897                         g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SAR, target, target, op_size == OP_SIZE_2 ? 48 : 32, 0));
1898 #endif
1899                 } else if (op_size == OP_SIZE_2) {
1900 #if defined(ARCH_X86)
1901                         g(gen_3address_rot_imm(ctx, OP_SIZE_2, ROT_ROR, target, reg1, 8, 0));
1902 #else
1903                         g(gen_2address_alu1(ctx, OP_SIZE_4, ALU1_BSWAP16, target, reg1, 0));
1904 #endif
1905                 } else {
1906                         g(gen_2address_alu1(ctx, minimum(op_size, OP_SIZE_NATIVE), ALU1_BSWAP, target, reg1, 0));
1907                 }
1908                 if (op_size > OP_SIZE_NATIVE) {
1909                         g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_BSWAP, R_SCRATCH_2, R_SCRATCH_2, 0));
1910                 }
1911
1912                 if (op_size > OP_SIZE_NATIVE)
1913                         g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_2, R_SCRATCH_1));
1914                 else
1915                         g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1916                 return true;
1917 #endif
1918                 goto do_generic_bswap;
1919 do_generic_bswap:
1920                 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_unary_bswap_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, 0);
1921         }
1922         /********
1923          * BREV *
1924          ********/
1925 do_brev: {
1926 #if defined(ARCH_ARM) || defined(ARCH_LOONGARCH64) || (defined(ARCH_MIPS) && MIPS_R6)
1927 #if defined(ARCH_ARM32)
1928                 if (unlikely(!cpu_test_feature(CPU_FEATURE_armv6t2)))
1929                         goto do_generic_brev;
1930 #endif
1931                 if (op_size > OP_SIZE_NATIVE) {
1932                         g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
1933                         reg1 = R_SCRATCH_1;
1934                         target = R_SCRATCH_1;
1935                 } else {
1936                         target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1937                         g(gen_frame_get(ctx, op_size, garbage, slot_1, target, &reg1));
1938                 }
1939
1940                 g(gen_2address_alu1(ctx, minimum(maximum(OP_SIZE_4, op_size), OP_SIZE_NATIVE), ALU1_BREV, target, reg1, 0));
1941                 if (op_size <= OP_SIZE_2) {
1942                         g(gen_3address_rot_imm(ctx, OP_SIZE_4, ROT_SHR, target, target, op_size == OP_SIZE_1 ? 24 : 16, 0));
1943                 }
1944                 if (op_size > OP_SIZE_NATIVE) {
1945                         g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_BREV, R_SCRATCH_2, R_SCRATCH_2, 0));
1946                 }
1947
1948                 if (op_size > OP_SIZE_NATIVE)
1949                         g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_2, R_SCRATCH_1));
1950                 else
1951                         g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1952                 return true;
1953 #endif
1954                 goto do_generic_brev;
1955 do_generic_brev:
1956                 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_unary_brev_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, 0);
1957         }
1958         /******************
1959          * BSF/BSR/POPCNT *
1960          ******************/
1961 do_bsf_bsr_popcnt: {
1962                 if (op_size > OP_SIZE_NATIVE) {
1963 #if defined(ARCH_X86)
1964                         uint32_t label_finish = 0;      /* avoid warning */
1965                         if (alu == ALU1_POPCNT && unlikely(!cpu_test_feature(CPU_FEATURE_popcnt)))
1966                                 goto do_generic_bsf_bsr_popcnt;
1967                         if (alu == ALU1_BSR || alu == ALU1_POPCNT) {
1968                                 if (mode == MODE_INT) {
1969                                         g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
1970                                         g(gen_imm(ctx, 0, IMM_PURPOSE_STORE_VALUE, OP_SIZE_NATIVE));
1971                                         gen_insn(INSN_CMP, OP_SIZE_NATIVE, 0, 1);
1972                                         gen_address_offset();
1973                                         gen_imm_offset();
1974
1975                                         gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_S, 0);
1976                                         gen_four(label_ovf);
1977                                 }
1978                         }
1979                         if (alu == ALU1_BSF) {
1980                                 label_finish = alloc_label(ctx);
1981                                 if (unlikely(!label_finish))
1982                                         return false;
1983
1984                                 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + lo_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
1985                                 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_BSF, ALU1_WRITES_FLAGS(ALU1_BSF));
1986                                 gen_one(R_SCRATCH_1);
1987                                 gen_address_offset();
1988
1989                                 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_NE, 0);
1990                                 gen_four(label_finish);
1991
1992                                 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
1993                                 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_BSF, ALU1_WRITES_FLAGS(ALU1_BSF));
1994                                 gen_one(R_SCRATCH_1);
1995                                 gen_address_offset();
1996
1997                                 g(gen_3address_alu_imm(ctx, OP_SIZE_4, ALU_ADD, R_SCRATCH_1, R_SCRATCH_1, 8U << OP_SIZE_NATIVE, 0));
1998                         }
1999                         if (alu == ALU1_BSR) {
2000                                 label_finish = alloc_label(ctx);
2001                                 if (unlikely(!label_finish))
2002                                         return false;
2003
2004                                 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
2005                                 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_BSR, ALU1_WRITES_FLAGS(ALU1_BSR));
2006                                 gen_one(R_SCRATCH_1);
2007                                 gen_address_offset();
2008
2009                                 g(gen_3address_alu_imm(ctx, OP_SIZE_4, ALU_ADD, R_SCRATCH_1, R_SCRATCH_1, 8U << OP_SIZE_NATIVE, 0));
2010
2011                                 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_NE, 0);
2012                                 gen_four(label_finish);
2013
2014                                 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + lo_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
2015                                 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_BSR, ALU1_WRITES_FLAGS(ALU1_BSR));
2016                                 gen_one(R_SCRATCH_1);
2017                                 gen_address_offset();
2018                         }
2019                         if (alu == ALU1_BSF || alu == ALU1_BSR) {
2020                                 if (mode == MODE_INT) {
2021                                         gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_E, 0);
2022                                         gen_four(label_ovf);
2023                                 } else {
2024                                         gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_NE, 0);
2025                                         gen_four(label_finish);
2026
2027                                         g(gen_load_constant(ctx, R_SCRATCH_1, -1));
2028                                 }
2029
2030                                 gen_label(label_finish);
2031
2032                                 if (mode == MODE_INT)
2033                                         goto write_result;
2034
2035                                 if (R_SCRATCH_1 != R_AX || R_SCRATCH_2 != R_DX)
2036                                         internal(file_line, "gen_alu1: bad scratch registers");
2037                                 gen_insn(INSN_CWD, OP_SIZE_NATIVE, 0, 0);
2038                                 gen_one(R_DX);
2039                                 gen_one(R_AX);
2040
2041                                 g(gen_address(ctx, R_FRAME, (size_t)slot_r * slot_size + lo_word(OP_SIZE_NATIVE), IMM_PURPOSE_STR_OFFSET, OP_SIZE_NATIVE));
2042                                 gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
2043                                 gen_address_offset();
2044                                 gen_one(R_SCRATCH_1);
2045
2046                                 g(gen_address(ctx, R_FRAME, (size_t)slot_r * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_STR_OFFSET, OP_SIZE_NATIVE));
2047                                 gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
2048                                 gen_address_offset();
2049                                 gen_one(R_SCRATCH_2);
2050
2051                                 return true;
2052                         }
2053                         if (alu == ALU1_POPCNT) {
2054                                 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + lo_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
2055                                 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_POPCNT, ALU1_WRITES_FLAGS(ALU1_POPCNT));
2056                                 gen_one(R_SCRATCH_1);
2057                                 gen_address_offset();
2058
2059                                 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
2060                                 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_POPCNT, ALU1_WRITES_FLAGS(ALU1_POPCNT));
2061                                 gen_one(R_SCRATCH_2);
2062                                 gen_address_offset();
2063
2064                                 g(gen_3address_alu(ctx, OP_SIZE_4, ALU_ADD, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_2, 1));
2065 write_result:
2066                                 g(gen_address(ctx, R_FRAME, (size_t)slot_r * slot_size + lo_word(OP_SIZE_NATIVE), IMM_PURPOSE_STR_OFFSET, OP_SIZE_NATIVE));
2067                                 gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
2068                                 gen_address_offset();
2069                                 gen_one(R_SCRATCH_1);
2070
2071                                 g(gen_address(ctx, R_FRAME, (size_t)slot_r * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_STR_OFFSET, OP_SIZE_NATIVE));
2072                                 g(gen_imm(ctx, 0, IMM_PURPOSE_STORE_VALUE, OP_SIZE_NATIVE));
2073                                 gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
2074                                 gen_address_offset();
2075                                 gen_imm_offset();
2076
2077                                 return true;
2078                         }
2079 #endif
2080                         goto do_generic_bsf_bsr_popcnt;
2081                 }
2082 #if defined(ARCH_X86)
2083                 if (alu == ALU1_POPCNT && unlikely(!cpu_test_feature(CPU_FEATURE_popcnt)))
2084                         goto do_generic_bsf_bsr_popcnt;
2085                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
2086                 if (op_size == OP_SIZE_1 || ((alu == ALU1_BSR || alu == ALU1_POPCNT) && mode == MODE_INT)) {
2087                         g(gen_frame_get(ctx, op_size, zero_x, slot_1, target, &reg1));
2088                         if ((alu == ALU1_BSR || alu == ALU1_POPCNT) && mode == MODE_INT) {
2089                                 g(gen_cmp_test_jmp(ctx, INSN_TEST, op_size, reg1, reg1, alu == ALU1_BSR ? COND_LE : COND_S, label_ovf));
2090                         }
2091                         g(gen_2address_alu1(ctx, maximum(op_size, OP_SIZE_2), alu, target, reg1, 1));
2092                         if ((alu == ALU1_BSR || alu == ALU1_POPCNT) && mode == MODE_INT)
2093                                 goto x86_bsf_bsr_popcnt_finish;
2094                 } else {
2095                         g(gen_frame_load_op1(ctx, op_size, alu, 1, slot_1, target));
2096                 }
2097                 if (alu == ALU1_POPCNT)
2098                         goto x86_bsf_bsr_popcnt_finish;
2099                 if (mode == MODE_FIXED) {
2100                         uint32_t cmov_label;
2101                         gen_insn(INSN_MOV, maximum(op_size, OP_SIZE_4), 0, 0);
2102                         gen_one(R_SCRATCH_2);
2103                         gen_one(ARG_IMM);
2104                         gen_eight(-1);
2105                         g(gen_cmov(ctx, maximum(op_size, OP_SIZE_4), COND_E, target, &cmov_label));
2106                         gen_one(R_SCRATCH_2);
2107                         if (cmov_label)
2108                                 gen_label(cmov_label);
2109
2110                 } else {
2111                         gen_insn(INSN_JMP_COND, maximum(op_size, OP_SIZE_2), COND_E, 0);
2112                         gen_four(label_ovf);
2113                 }
2114 x86_bsf_bsr_popcnt_finish:
2115                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2116                 return true;
2117 #endif
2118 #if defined(ARCH_ARM)
2119 #if defined(ARCH_ARM32)
2120                 if (alu == ALU1_BSR && unlikely(!cpu_test_feature(CPU_FEATURE_armv6)))
2121                         goto do_generic_bsf_bsr_popcnt;
2122                 if (alu == ALU1_BSF && unlikely(!cpu_test_feature(CPU_FEATURE_armv6t2)))
2123                         goto do_generic_bsf_bsr_popcnt;
2124 #endif
2125                 if (alu == ALU1_POPCNT && unlikely(!cpu_test_feature(CPU_FEATURE_neon)))
2126                         goto do_generic_bsf_bsr_popcnt;
2127                 g(gen_frame_get(ctx, op_size, mode == MODE_INT ? sign_x : zero_x, slot_1, R_SCRATCH_1, &reg1));
2128                 if (mode == MODE_INT) {
2129                         g(gen_cmp_test_jmp(ctx, INSN_TEST, i_size(op_size), reg1, reg1, alu == ALU1_BSR ? COND_LE : alu == ALU1_BSF ? COND_E : COND_S, label_ovf));
2130                 }
2131
2132                 if (alu == ALU1_POPCNT) {
2133                         g(gen_mov(ctx, OP_SIZE_NATIVE, FR_SCRATCH_1, reg1));
2134                         gen_insn(INSN_FP_ALU1, OP_SIZE_NATIVE, FP_ALU1_VCNT8, 0);
2135                         gen_one(FR_SCRATCH_1);
2136                         gen_one(FR_SCRATCH_1);
2137 #if defined(ARCH_ARM32)
2138                         if (op_size > OP_SIZE_1) {
2139                                 gen_insn(INSN_FP_ALU1, OP_SIZE_1, FP_ALU1_VPADDL, 0);
2140                                 gen_one(FR_SCRATCH_1);
2141                                 gen_one(FR_SCRATCH_1);
2142                         }
2143                         if (op_size > OP_SIZE_2) {
2144                                 gen_insn(INSN_FP_ALU1, OP_SIZE_2, FP_ALU1_VPADDL, 0);
2145                                 gen_one(FR_SCRATCH_1);
2146                                 gen_one(FR_SCRATCH_1);
2147                         }
2148 #else
2149                         if (op_size > OP_SIZE_1) {
2150                                 gen_insn(INSN_FP_ALU1, OP_SIZE_1, FP_ALU1_ADDV, 0);
2151                                 gen_one(FR_SCRATCH_1);
2152                                 gen_one(FR_SCRATCH_1);
2153                         }
2154 #endif
2155                         g(gen_frame_store(ctx, op_size, slot_r, 0, FR_SCRATCH_1));
2156                         if (slot_is_register(ctx, slot_r))
2157                                 g(unspill(ctx, slot_r));
2158                         return true;
2159                 }
2160
2161                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
2162                 if (mode == MODE_FIXED && alu == ALU1_BSF) {
2163                         gen_insn(INSN_TEST, i_size(op_size), 0, 1);
2164                         gen_one(target);
2165                         gen_one(reg1);
2166                         reg1 = target;
2167                 }
2168
2169                 if (alu == ALU1_BSF) {
2170                         g(gen_2address_alu1(ctx, i_size(op_size), ALU1_BREV, target, reg1, 0));
2171                         reg1 = target;
2172                 }
2173
2174                 g(gen_2address_alu1(ctx, i_size(op_size), ALU1_LZCNT, target, reg1, 0));
2175
2176                 if (alu == ALU1_BSR) {
2177                         g(gen_load_constant(ctx, R_SCRATCH_2, op_size == OP_SIZE_8 ? 63 : 31));
2178                         g(gen_3address_alu(ctx, i_size(op_size), ALU_SUB, target, R_SCRATCH_2, target, 0));
2179                 }
2180
2181                 if (mode == MODE_FIXED && alu == ALU1_BSF) {
2182 #if defined(ARCH_ARM32)
2183                         g(gen_imm(ctx, -1, IMM_PURPOSE_CMOV, OP_SIZE_NATIVE));
2184                         gen_insn(INSN_CMOV, OP_SIZE_NATIVE, COND_E, 0);
2185                         gen_one(target);
2186                         gen_one(target);
2187                         gen_imm_offset();
2188 #else
2189                         gen_insn(INSN_CSEL_INV, i_size(op_size), COND_NE, 0);
2190                         gen_one(target);
2191                         gen_one(ARG_IMM);
2192                         gen_eight(0);
2193                         gen_one(target);
2194 #endif
2195                 }
2196
2197                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2198                 return true;
2199 #endif
2200 #if defined(ARCH_ALPHA)
2201                 if (likely(cpu_test_feature(CPU_FEATURE_cix))) {
2202                         g(gen_frame_get(ctx, op_size, mode == MODE_INT ? sign_x : zero_x, slot_1, R_SCRATCH_1, &reg1));
2203                         target = gen_frame_target(ctx, slot_r, slot_1, NO_FRAME_T, R_SCRATCH_2);
2204                         if (mode == MODE_INT) {
2205                                 g(gen_cmp_test_jmp(ctx, INSN_TEST, OP_SIZE_NATIVE, reg1, reg1, alu == ALU1_BSR ? COND_LE : alu == ALU1_BSF ? COND_E : COND_S, label_ovf));
2206                         }
2207                         if (alu == ALU1_POPCNT) {
2208                                 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_POPCNT, target, reg1, 0));
2209                         }
2210                         if (alu == ALU1_BSF) {
2211                                 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_BSF, target, reg1, 0));
2212
2213                                 if (mode == MODE_FIXED) {
2214                                         g(gen_imm(ctx, -1, IMM_PURPOSE_MOVR, OP_SIZE_INT));
2215                                         gen_insn(INSN_MOVR, OP_SIZE_NATIVE, COND_E, 0);
2216                                         gen_one(target);
2217                                         gen_one(target);
2218                                         gen_one(reg1);
2219                                         gen_imm_offset();
2220                                 }
2221                         }
2222                         if (alu == ALU1_BSR) {
2223                                 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_LZCNT, target, reg1, 0));
2224
2225                                 g(gen_load_constant(ctx, R_SCRATCH_3, OP_SIZE_NATIVE == OP_SIZE_8 ? 63 : 31));
2226
2227                                 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_SUB, target, R_SCRATCH_3, target, 0));
2228                         }
2229                         g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2230                         return true;
2231                 }
2232 #endif
2233 #if defined(ARCH_MIPS)
2234                 if (MIPS_HAS_CLZ && alu != ALU1_POPCNT) {
2235                         g(gen_frame_get(ctx, op_size, mode == MODE_INT ? sign_x : zero_x, slot_1, R_SCRATCH_1, &reg1));
2236                         target = gen_frame_target(ctx, slot_r, slot_1, NO_FRAME_T, R_SCRATCH_2);
2237                         if (mode == MODE_INT) {
2238                                 g(gen_cmp_test_jmp(ctx, INSN_TEST, OP_SIZE_NATIVE, reg1, reg1, alu == ALU1_BSR ? COND_LE : alu == ALU1_BSF ? COND_E : COND_S, label_ovf));
2239                         }
2240                         if (alu == ALU1_BSF) {
2241                                 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NEG, target, reg1, 0));
2242
2243                                 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_AND, R_SCRATCH_1, reg1, target, 0));
2244                                 reg1 = R_SCRATCH_1;
2245                         }
2246                         g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_LZCNT, target, reg1, 0));
2247
2248                         g(gen_load_constant(ctx, R_SCRATCH_3, OP_SIZE_NATIVE == OP_SIZE_8 ? 63 : 31));
2249
2250                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_SUB, target, R_SCRATCH_3, target, 0));
2251
2252                         g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2253                         return true;
2254                 }
2255 #endif
2256 #if defined(ARCH_POWER)
2257                 if (alu == ALU1_BSF && (unlikely(!cpu_test_feature(CPU_FEATURE_v203)) || unlikely(!cpu_test_feature(CPU_FEATURE_v30))))
2258                         goto do_generic_bsf_bsr_popcnt;
2259                 if (alu == ALU1_POPCNT && unlikely(!cpu_test_feature(CPU_FEATURE_v206)))
2260                         goto do_generic_bsf_bsr_popcnt;
2261                 g(gen_frame_get(ctx, op_size, mode == MODE_INT ? sign_x : zero_x, slot_1, R_SCRATCH_1, &reg1));
2262                 target = gen_frame_target(ctx, slot_r, slot_1, NO_FRAME_T, R_SCRATCH_2);
2263                 if (mode == MODE_INT) {
2264                         g(gen_cmp_test_jmp(ctx, INSN_TEST, i_size(op_size), reg1, reg1, alu == ALU1_BSR ? COND_LE : alu == ALU1_BSF ? COND_E : COND_S, label_ovf));
2265                 }
2266                 if (alu == ALU1_POPCNT) {
2267                         g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_POPCNT, target, reg1, 0));
2268                 }
2269                 if (alu == ALU1_BSF) {
2270                         g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_BSF, target, reg1, 0));
2271
2272                         if (mode == MODE_FIXED) {
2273                                 g(gen_3address_alu(ctx, i_size(op_size), ALU_AND, R_SCRATCH_3, reg1, reg1, 1));
2274
2275                                 g(gen_imm(ctx, -1, IMM_PURPOSE_CMOV, OP_SIZE_NATIVE));
2276                                 gen_insn(INSN_CMOV, OP_SIZE_NATIVE, COND_E, 0);
2277                                 gen_one(target);
2278                                 gen_one(target);
2279                                 gen_imm_offset();
2280                         }
2281                 }
2282                 if (alu == ALU1_BSR) {
2283                         g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_LZCNT, target, reg1, 0));
2284
2285                         g(gen_load_constant(ctx, R_SCRATCH_3, OP_SIZE_NATIVE == OP_SIZE_8 ? 63 : 31));
2286
2287                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_SUB, target, R_SCRATCH_3, target, 0));
2288                 }
2289                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2290                 return true;
2291 #endif
2292 #if defined(ARCH_LOONGARCH64) || defined(ARCH_RISCV64)
2293 #if defined(ARCH_LOONGARCH64)
2294                 if (alu == ALU1_POPCNT)
2295                         goto do_generic_bsf_bsr_popcnt;
2296 #endif
2297 #if defined(ARCH_RISCV64)
2298                 if (unlikely(!cpu_test_feature(CPU_FEATURE_zbb)))
2299                         goto do_generic_bsf_bsr_popcnt;
2300 #endif
2301                 g(gen_frame_get(ctx, op_size, mode == MODE_INT ? sign_x : zero_x, slot_1, R_SCRATCH_1, &reg1));
2302                 target = gen_frame_target(ctx, slot_r, slot_1, NO_FRAME_T, R_SCRATCH_2);
2303                 if (mode == MODE_INT) {
2304                         g(gen_cmp_test_jmp(ctx, INSN_TEST, OP_SIZE_NATIVE, reg1, reg1, alu == ALU1_BSR ? COND_LE : alu == ALU1_BSF ? COND_E : COND_S, label_ovf));
2305                 }
2306                 if (alu == ALU1_POPCNT) {
2307                         g(gen_2address_alu1(ctx, maximum(OP_SIZE_4, op_size), ALU1_POPCNT, target, reg1, 0));
2308                 }
2309                 if (alu == ALU1_BSF) {
2310                         g(gen_2address_alu1(ctx, maximum(OP_SIZE_4, op_size), ALU1_BSF, target, reg1, 0));
2311
2312                         if (mode == MODE_FIXED) {
2313                                 g(gen_imm(ctx, 1, IMM_PURPOSE_CMP, OP_SIZE_NATIVE));
2314                                 gen_insn(INSN_CMP_DEST_REG, OP_SIZE_NATIVE, COND_B, 0);
2315                                 gen_one(R_SCRATCH_3);
2316                                 gen_one(reg1);
2317                                 gen_imm_offset();
2318
2319                                 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NEG, R_SCRATCH_3, R_SCRATCH_3, 0));
2320
2321                                 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_OR, target, target, R_SCRATCH_3, 0));
2322                         }
2323                 }
2324                 if (alu == ALU1_BSR) {
2325                         g(gen_2address_alu1(ctx, maximum(OP_SIZE_4, op_size), ALU1_LZCNT, target, reg1, 0));
2326
2327                         g(gen_load_constant(ctx, R_SCRATCH_3, op_size <= OP_SIZE_4 ? 31 : 63));
2328
2329                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_SUB, target, R_SCRATCH_3, target, 0));
2330                 }
2331                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2332                 return true;
2333 #endif
2334 #if defined(ARCH_IA64) || defined(ARCH_S390) || defined(ARCH_SPARC)
2335                 if (alu == ALU1_BSF && !ARCH_HAS_ANDN)
2336                         goto do_generic_bsf_bsr_popcnt;
2337 #if defined(ARCH_S390)
2338                 if (!cpu_test_feature(CPU_FEATURE_misc_45) || !cpu_test_feature(CPU_FEATURE_misc_insn_ext_3))
2339                         goto do_generic_bsf_bsr_popcnt;
2340 #endif
2341 #if defined(ARCH_SPARC)
2342                 if (!SPARC_9)
2343                         goto do_generic_bsf_bsr_popcnt;
2344 #endif
2345                 g(gen_frame_get(ctx, op_size, mode == MODE_INT ? sign_x : zero_x, slot_1, R_SCRATCH_1, &reg1));
2346                 target = gen_frame_target(ctx, slot_r, slot_1, NO_FRAME_T, R_SCRATCH_2);
2347                 if (mode == MODE_INT) {
2348                         g(gen_cmp_test_jmp(ctx, INSN_TEST, maximum(op_size, OP_SIZE_4), reg1, reg1, alu == ALU1_BSR ? COND_LE : alu == ALU1_BSF ? COND_E : COND_S, label_ovf));
2349                 } else {
2350                         if (ARCH_PREFERS_SX(op_size) && alu == ALU1_POPCNT && op_size < OP_SIZE_NATIVE) {
2351                                 g(gen_extend(ctx, op_size, zero_x, R_SCRATCH_1, reg1));
2352                                 reg1 = R_SCRATCH_1;
2353                         }
2354                 }
2355                 if (alu == ALU1_POPCNT) {
2356                         g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_POPCNT, R_SCRATCH_1, reg1, 0));
2357                         g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
2358                         return true;
2359                 }
2360                 if (alu == ALU1_BSF) {
2361                         g(gen_3address_alu_imm(ctx, OP_SIZE_NATIVE, ALU_SUB, target, reg1, 1, 0));
2362
2363                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_ANDN, target, target, reg1, 0));
2364
2365                         g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_POPCNT, target, target, 0));
2366
2367                         if (mode == MODE_FIXED) {
2368                                 unsigned attr_unused test_reg = R_SCRATCH_1;
2369 #if defined(ARCH_S390)
2370                                 g(gen_imm(ctx, 0, COND_IS_LOGICAL(COND_E) ? IMM_PURPOSE_CMP_LOGICAL : IMM_PURPOSE_CMP, OP_SIZE_NATIVE));
2371                                 gen_insn(INSN_CMP, OP_SIZE_NATIVE, 0, 1 + COND_IS_LOGICAL(COND_E));
2372                                 gen_one(reg1);
2373                                 gen_imm_offset();
2374
2375                                 g(gen_imm(ctx, -1, IMM_PURPOSE_CMOV, OP_SIZE_NATIVE));
2376                                 gen_insn(INSN_CMOV, OP_SIZE_NATIVE, COND_E, 0);
2377                                 gen_one(target);
2378                                 gen_one(target);
2379                                 gen_imm_offset();
2380 #else
2381 #if defined(ARCH_IA64)
2382                                 g(gen_cmp_dest_reg(ctx, OP_SIZE_NATIVE, reg1, (unsigned)-1, R_CMP_RESULT, 0, COND_NE));
2383                                 test_reg = R_CMP_RESULT;
2384 #endif
2385                                 g(gen_imm(ctx, -1, IMM_PURPOSE_MOVR, OP_SIZE_NATIVE));
2386                                 gen_insn(INSN_MOVR, OP_SIZE_NATIVE, COND_E, 0);
2387                                 gen_one(target);
2388                                 gen_one(target);
2389                                 gen_one(test_reg);
2390                                 gen_imm_offset();
2391 #endif
2392                         }
2393
2394                         g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2395                         return true;
2396                 }
2397 #endif
2398 do_generic_bsf_bsr_popcnt:
2399                 if (alu == ALU1_BSF) {
2400                         if (mode == MODE_FIXED)
2401                                 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_unary_bsf_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, 0);
2402                         else
2403                                 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, INT_unary_bsf_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, label_ovf);
2404                 }
2405                 if (alu == ALU1_BSR) {
2406                         if (mode == MODE_FIXED)
2407                                 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_unary_bsr_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, 0);
2408                         else
2409                                 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, INT_unary_bsr_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, label_ovf);
2410                 }
2411                 if (alu == ALU1_POPCNT) {
2412                         if (mode == MODE_FIXED)
2413                                 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_unary_popcnt_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, 0);
2414                         else
2415                                 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, INT_unary_popcnt_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, label_ovf);
2416                 }
2417         }
2418         /**************
2419          * CONVERSION *
2420          **************/
2421 do_fixed_conv:
2422 do_conv: {
2423                 unsigned src_op_size, dest_op_size;
2424                 const struct type *src_type, *dest_type;
2425                 src_type = get_type_of_local(ctx, slot_1);
2426                 dest_type = get_type_of_local(ctx, slot_r);
2427
2428                 if (TYPE_TAG_IS_FIXED(src_type->tag)) {
2429                         src_op_size = TYPE_TAG_IDX_FIXED(src_type->tag) >> 1;
2430                 } else {
2431                         src_op_size = TYPE_TAG_IDX_INT(src_type->tag);
2432                 }
2433
2434                 if (TYPE_TAG_IS_FIXED(dest_type->tag)) {
2435                         dest_op_size = TYPE_TAG_IDX_FIXED(dest_type->tag) >> 1;
2436                 } else {
2437                         dest_op_size = TYPE_TAG_IDX_INT(dest_type->tag);
2438                 }
2439
2440                 if (src_op_size <= OP_SIZE_NATIVE) {
2441                         g(gen_frame_get(ctx, src_op_size, sign_x, slot_1, R_SCRATCH_1, &reg1));
2442                 } else {
2443 #if defined(ARCH_X86)
2444                         if (dest_op_size < src_op_size)
2445                                 g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_1, 0, false, R_SCRATCH_1));
2446                         else
2447 #endif
2448                                 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_3));
2449                         reg1 = R_SCRATCH_1;
2450                 }
2451
2452                 if (dest_op_size >= src_op_size) {
2453                         if (dest_op_size <= OP_SIZE_NATIVE) {
2454                                 g(gen_frame_store(ctx, dest_op_size, slot_r, 0, reg1));
2455                         } else {
2456                                 if (src_op_size <= OP_SIZE_NATIVE) {
2457 #if defined(ARCH_X86)
2458                                         if (R_SCRATCH_1 != R_AX || R_SCRATCH_2 != R_DX)
2459                                                 internal(file_line, "gen_alu1: bad scratch registers");
2460                                         if (reg1 == R_SCRATCH_1) {
2461                                                 gen_insn(INSN_CWD, OP_SIZE_NATIVE, 0, 0);
2462                                                 gen_one(R_DX);
2463                                                 gen_one(R_AX);
2464                                         } else
2465 #endif
2466                                         g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_2, reg1, (1U << (OP_SIZE_NATIVE + 3)) - 1, false));
2467                                         g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, reg1, R_SCRATCH_2));
2468                                 } else {
2469                                         g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, reg1, R_SCRATCH_3));
2470                                 }
2471                         }
2472                         return true;
2473                 } else {
2474                         if (src_op_size > OP_SIZE_NATIVE) {
2475 #if defined(ARCH_ARM)
2476                                 gen_insn(INSN_CMP, OP_SIZE_NATIVE, 0, 1);
2477                                 gen_one(R_SCRATCH_3);
2478                                 gen_one(ARG_SHIFTED_REGISTER);
2479                                 gen_one(ARG_SHIFT_ASR | ((1U << (OP_SIZE_NATIVE + 3)) - 1));
2480                                 gen_one(R_SCRATCH_1);
2481
2482                                 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_NE, 0);
2483                                 gen_four(label_ovf);
2484 #elif defined(ARCH_X86)
2485                                 if (R_SCRATCH_1 != R_AX || R_SCRATCH_2 != R_DX)
2486                                         internal(file_line, "gen_alu1: bad scratch registers");
2487                                 gen_insn(INSN_CWD, OP_SIZE_NATIVE, 0, 0);
2488                                 gen_one(R_DX);
2489                                 gen_one(R_AX);
2490
2491                                 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
2492                                 gen_insn(INSN_CMP, OP_SIZE_NATIVE, 0, 1);
2493                                 gen_one(R_SCRATCH_2);
2494                                 gen_address_offset();
2495
2496                                 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_NE, 0);
2497                                 gen_four(label_ovf);
2498 #else
2499                                 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_2, R_SCRATCH_1, (1U << (OP_SIZE_NATIVE + 3)) - 1, 0));
2500
2501                                 g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, R_SCRATCH_3, R_SCRATCH_2, COND_NE, label_ovf));
2502 #endif
2503
2504                                 src_op_size = OP_SIZE_NATIVE;
2505                         }
2506                         if (src_op_size > dest_op_size) {
2507                                 g(gen_cmp_extended(ctx, OP_SIZE_NATIVE, dest_op_size, reg1, R_SCRATCH_2, label_ovf));
2508                         }
2509                         g(gen_frame_store(ctx, dest_op_size, slot_r, 0, reg1));
2510                         return true;
2511                 }
2512         }
2513
2514 conv_uto_int: {
2515                 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_uto_int_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, label_ovf);
2516         }
2517
2518 conv_ufrom_int: {
2519                 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_ufrom_int_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, label_ovf);
2520         }
2521 }
2522
2523 static bool attr_w gen_constant(struct codegen_context *ctx, bool real, unsigned op_size, bool shrt, frame_t slot_r)
2524 {
2525         uintbig_t c;
2526         if (shrt) {
2527                 c = (int16_t)get_unaligned_16(ctx->current_position);
2528         } else switch (op_size) {
2529 #define fx(n, type, utype, sz, bits)                                    \
2530                 case n:                                                 \
2531                         c = (type)cat(get_unaligned_,bits)(ctx->current_position);\
2532                         break;
2533                 for_all_fixed(fx);
2534 #undef fx
2535                 default:
2536                         internal(file_line, "gen_constant: invalid type %u", op_size);
2537         }
2538         if (op_size > OP_SIZE_NATIVE) {
2539                 unsigned shift = (8U << OP_SIZE_NATIVE) - 1;
2540                 g(gen_frame_store_imm_raw(ctx, OP_SIZE_NATIVE, slot_r, lo_word(OP_SIZE_NATIVE), c & ((2ULL << shift) - 1)));
2541                 g(gen_frame_store_imm_raw(ctx, OP_SIZE_NATIVE, slot_r, hi_word(OP_SIZE_NATIVE), c >> 1 >> shift));
2542                 if (real && slot_is_register(ctx, slot_r))
2543                         g(unspill(ctx, slot_r));
2544                 return true;
2545         } else if (real && slot_is_register(ctx, slot_r)) {
2546                 if (ARCH_HAS_FP_GP_MOV) {
2547                         g(gen_load_constant(ctx, R_SCRATCH_1, c));
2548                         g(gen_mov(ctx, op_size, ctx->registers[slot_r], R_SCRATCH_1));
2549                 } else {
2550                         g(gen_frame_store_imm_raw(ctx, op_size, slot_r, 0, c));
2551                         g(unspill(ctx, slot_r));
2552                 }
2553         } else {
2554                 g(gen_frame_store_imm(ctx, op_size, slot_r, 0, c));
2555         }
2556         return true;
2557 }
2558
2559 static bool attr_w gen_real_constant(struct codegen_context *ctx, const struct type *t, frame_t slot_r)
2560 {
2561         int64_t offset;
2562         if (is_power_of_2(t->size) && t->size <= sizeof(uintbig_t))
2563                 return gen_constant(ctx, true, log_2(t->size), false, slot_r);
2564
2565         g(load_function_offset(ctx, R_SCRATCH_3, offsetof(struct data, u_.function.code)));
2566
2567         offset = (ctx->current_position - da(ctx->fn,function)->code) * sizeof(code_t);
2568
2569         g(gen_memcpy_raw(ctx, R_FRAME, (size_t)slot_r * slot_size, R_SCRATCH_3, offset, t->size, minimum(t->align, sizeof(code_t))));
2570         if (slot_is_register(ctx, slot_r))
2571                 g(unspill(ctx, slot_r));
2572
2573         return true;
2574 }
2575
2576 static bool attr_w gen_copy(struct codegen_context *ctx, unsigned op_size, frame_t slot_1, frame_t slot_r)
2577 {
2578         unsigned reg1;
2579         if (unlikely(op_size > OP_SIZE_NATIVE)) {
2580                 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
2581                 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_1, R_SCRATCH_2));
2582                 return true;
2583         } else {
2584                 unsigned target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
2585                 g(gen_frame_get(ctx, op_size, garbage, slot_1, target, &reg1));
2586                 g(gen_frame_store(ctx, op_size, slot_r, 0, reg1));
2587                 return true;
2588         }
2589 }
2590
2591 static bool attr_w gen_fp_alu(struct codegen_context *ctx, unsigned real_type, unsigned op, uint32_t label_ovf, frame_t slot_1, frame_t slot_2, frame_t slot_r)
2592 {
2593         unsigned attr_unused fp_alu;
2594         size_t upc;
2595         unsigned attr_unused op_size = real_type_to_op_size(real_type);
2596         unsigned reg1, reg2, target;
2597         switch (op) {
2598                 case OPCODE_REAL_OP_add:
2599                 case OPCODE_REAL_OP_add_alt1:
2600                 case OPCODE_REAL_OP_add_alt2: fp_alu = FP_ALU_ADD; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_add_real16_t); label_ovf = 0; goto do_alu;
2601                 case OPCODE_REAL_OP_subtract:
2602                 case OPCODE_REAL_OP_subtract_alt1:
2603                 case OPCODE_REAL_OP_subtract_alt2: fp_alu = FP_ALU_SUB; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_subtract_real16_t); label_ovf = 0; goto do_alu;
2604                 case OPCODE_REAL_OP_multiply:
2605                 case OPCODE_REAL_OP_multiply_alt1:
2606                 case OPCODE_REAL_OP_multiply_alt2: fp_alu = FP_ALU_MUL; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_multiply_real16_t); label_ovf = 0; goto do_alu;
2607                 case OPCODE_REAL_OP_divide:
2608                 case OPCODE_REAL_OP_divide_alt1:
2609                 case OPCODE_REAL_OP_divide_alt2: fp_alu = FP_ALU_DIV; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_divide_real16_t); label_ovf = 0; goto do_alu;
2610                 case OPCODE_REAL_OP_modulo:
2611                 case OPCODE_REAL_OP_power:
2612                 case OPCODE_REAL_OP_ldexp:
2613                 case OPCODE_REAL_OP_atan2: upc = offsetof(struct cg_upcall_vector_s, REAL_binary_modulo_real16_t) + (op - OPCODE_REAL_OP_modulo) * TYPE_REAL_N * sizeof(void (*)(void)); goto do_upcall;
2614                 case OPCODE_REAL_OP_equal:
2615                 case OPCODE_REAL_OP_equal_alt1:
2616                 case OPCODE_REAL_OP_equal_alt2: fp_alu = FP_COND_E; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_equal_real16_t); goto do_cmp;
2617                 case OPCODE_REAL_OP_not_equal:
2618                 case OPCODE_REAL_OP_not_equal_alt1:
2619                 case OPCODE_REAL_OP_not_equal_alt2: fp_alu = FP_COND_NE; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_not_equal_real16_t); goto do_cmp;
2620                 case OPCODE_REAL_OP_less:
2621                 case OPCODE_REAL_OP_less_alt1:
2622                 case OPCODE_REAL_OP_less_alt2: fp_alu = FP_COND_B; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_less_real16_t); goto do_cmp;
2623                 case OPCODE_REAL_OP_less_equal:
2624                 case OPCODE_REAL_OP_less_equal_alt1:
2625                 case OPCODE_REAL_OP_less_equal_alt2: fp_alu = FP_COND_BE; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_less_equal_real16_t); goto do_cmp;
2626                 default: internal(file_line, "gen_fp_alu: unsupported operation %u", op);
2627         }
2628
2629 do_alu:
2630         if ((SUPPORTED_FP >> real_type) & 1) {
2631 #if defined(ARCH_IA64)
2632                 if (unlikely(fp_alu == FP_ALU_DIV))
2633                         goto do_upcall;
2634 #endif
2635 #if defined(ARCH_X86)
2636                 if (1)
2637 #elif defined(ARCH_S390)
2638                 if ((op_size <= OP_SIZE_8 && (size_t)slot_2 * slot_size < 4096) || slot_is_register(ctx, slot_2))
2639 #else
2640                 if (slot_is_register(ctx, slot_2))
2641 #endif
2642                 {
2643                         target = gen_frame_target(ctx, slot_r, NO_FRAME_T, slot_2, FR_SCRATCH_1);
2644                         g(gen_frame_get(ctx, op_size, garbage, slot_1, FR_SCRATCH_1, &reg1));
2645                         if (slot_is_register(ctx, slot_2)) {
2646                                 g(gen_3address_fp_alu(ctx, op_size, fp_alu, target, reg1, ctx->registers[slot_2]));
2647                         } else {
2648                                 if (target != reg1 && !ARCH_IS_3ADDRESS_FP) {
2649                                         g(gen_mov(ctx, op_size, target, reg1));
2650                                         reg1 = target;
2651                                 }
2652                                 g(gen_address(ctx, R_FRAME, (size_t)slot_2 * slot_size, IMM_PURPOSE_VLDR_VSTR_OFFSET, op_size));
2653                                 gen_insn(INSN_FP_ALU, op_size, fp_alu, 0);
2654                                 gen_one(target);
2655                                 gen_one(reg1);
2656                                 gen_address_offset();
2657                         }
2658                         g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2659                         return true;
2660                 }
2661 #if defined(ARCH_ALPHA)
2662                 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, &reg1));
2663                 g(gen_frame_get(ctx, op_size, zero_x, slot_2, FR_SCRATCH_2, &reg2));
2664                 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, FR_SCRATCH_3);
2665                 g(gen_3address_fp_alu(ctx, op_size, fp_alu, target, reg1, reg2));
2666                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2667 #else
2668                 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, &reg1));
2669                 g(gen_frame_get(ctx, op_size, zero_x, slot_2, FR_SCRATCH_2, &reg2));
2670                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_1);
2671                 g(gen_3address_fp_alu(ctx, op_size, fp_alu, target, reg1, reg2));
2672                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2673 #endif
2674                 return true;
2675         }
2676 #ifdef SUPPORTED_FP_X87
2677         if ((SUPPORTED_FP_X87 >> real_type) & 1) {
2678                 if (real_type != 3) {
2679                         g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_2));
2680                         g(gen_frame_load_x87(ctx, INSN_X87_ALU, op_size, fp_alu, slot_1));
2681                 } else {
2682                         g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
2683                         g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_2));
2684                         gen_insn(INSN_X87_ALUP, op_size, fp_alu, 0);
2685                         gen_one(R_ST1);
2686                 }
2687                 g(gen_frame_store_x87(ctx, INSN_X87_FSTP, op_size, slot_r));
2688                 return true;
2689         }
2690 #endif
2691 #ifdef SUPPORTED_FP_HALF_CVT
2692         if ((SUPPORTED_FP_HALF_CVT >> real_type) & 1) {
2693                 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, &reg1));
2694                 g(gen_frame_get(ctx, op_size, zero_x, slot_2, FR_SCRATCH_2, &reg2));
2695                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_1);
2696                 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
2697                 gen_one(FR_SCRATCH_1);
2698                 gen_one(reg1);
2699                 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
2700                 gen_one(FR_SCRATCH_2);
2701                 gen_one(reg2);
2702                 gen_insn(INSN_FP_ALU, OP_SIZE_4, fp_alu, 0);
2703                 gen_one(FR_SCRATCH_1);
2704                 gen_one(FR_SCRATCH_1);
2705                 gen_one(FR_SCRATCH_2);
2706                 gen_insn(INSN_FP_CVT, OP_SIZE_4, op_size, 0);
2707                 gen_one(target);
2708                 gen_one(FR_SCRATCH_1);
2709                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2710                 return true;
2711         }
2712 #endif
2713         goto do_upcall;
2714
2715 do_cmp:
2716         if ((SUPPORTED_FP >> real_type) & 1
2717 #if defined(ARCH_ALPHA)
2718                 && ARCH_SUPPORTS_TRAPS
2719 #endif
2720         ) {
2721                 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, &reg1));
2722                 g(gen_frame_get(ctx, op_size, zero_x, slot_2, FR_SCRATCH_2, &reg2));
2723                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
2724 #if defined(ARCH_ALPHA)
2725                 gen_insn(INSN_FP_CMP_DEST_REG_TRAP, op_size, fp_alu == FP_COND_NE ? FP_COND_E : fp_alu, 0);
2726                 gen_one(FR_SCRATCH_3);
2727                 gen_one(reg1);
2728                 gen_one(reg2);
2729                 gen_four(label_ovf);
2730
2731                 if (!ARCH_HAS_FP_GP_MOV) {
2732                         g(gen_frame_store_raw(ctx, OP_SIZE_4, slot_r, 0, FR_SCRATCH_3));
2733                         g(gen_frame_load_raw(ctx, OP_SIZE_4, sign_x, slot_r, 0, false, target));
2734                 } else {
2735                         g(gen_mov(ctx, OP_SIZE_4, target, FR_SCRATCH_3));
2736                 }
2737
2738                 if (fp_alu == FP_COND_NE) {
2739                         g(gen_imm(ctx, 0, IMM_PURPOSE_CMP, OP_SIZE_NATIVE));
2740                         gen_insn(INSN_CMP_DEST_REG, OP_SIZE_NATIVE, COND_E, 0);
2741                         gen_one(target);
2742                         gen_one(target);
2743                         gen_imm_offset();
2744                 } else {
2745                         g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SHR, target, target, 30, 0));
2746                 }
2747
2748                 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
2749
2750                 return true;
2751 #elif defined(ARCH_IA64)
2752                 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_P, 0);
2753                 gen_one(R_CMP_RESULT);
2754                 gen_one(reg1);
2755                 gen_one(reg2);
2756
2757                 gen_insn(INSN_JMP_REG, OP_SIZE_NATIVE, COND_NE, 0);
2758                 gen_one(R_CMP_RESULT);
2759                 gen_four(label_ovf);
2760
2761                 gen_insn(INSN_FP_CMP_DEST_REG, op_size, fp_alu, 0);
2762                 gen_one(R_CMP_RESULT);
2763                 gen_one(reg1);
2764                 gen_one(reg2);
2765
2766                 g(gen_mov(ctx, OP_SIZE_NATIVE, target, R_CMP_RESULT));
2767
2768                 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
2769
2770                 return true;
2771 #elif defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS) || defined(ARCH_PARISC)
2772                 gen_insn(INSN_FP_CMP_COND, op_size, FP_COND_P, 1);
2773                 gen_one(reg1);
2774                 gen_one(reg2);
2775
2776                 gen_insn(INSN_JMP_FP_TEST, 0, FP_COND_P, 0);
2777                 gen_four(label_ovf);
2778
2779                 gen_insn(INSN_FP_CMP_COND, op_size, fp_alu, 1);
2780                 gen_one(reg1);
2781                 gen_one(reg2);
2782
2783                 gen_insn(INSN_FP_TEST_REG, OP_SIZE_NATIVE, fp_alu, 0);
2784                 gen_one(target);
2785
2786                 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
2787
2788                 return true;
2789 #elif defined(ARCH_RISCV64)
2790                 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_E, 0);
2791                 gen_one(R_SCRATCH_1);
2792                 gen_one(reg1);
2793                 gen_one(reg1);
2794
2795                 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_E, 0);
2796                 gen_one(R_SCRATCH_2);
2797                 gen_one(reg2);
2798                 gen_one(reg2);
2799
2800                 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_AND, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_2, 0));
2801
2802                 g(gen_jmp_on_zero(ctx, OP_SIZE_NATIVE, R_SCRATCH_1, COND_E, label_ovf));
2803
2804                 gen_insn(INSN_FP_CMP_DEST_REG, op_size, fp_alu == FP_COND_NE ? FP_COND_E : fp_alu, 0);
2805                 gen_one(target);
2806                 gen_one(reg1);
2807                 gen_one(reg2);
2808
2809                 if (fp_alu == FP_COND_NE) {
2810                         g(gen_imm(ctx, 1, IMM_PURPOSE_XOR, OP_SIZE_NATIVE));
2811                         gen_insn(INSN_ALU, OP_SIZE_NATIVE, ALU_XOR, ALU_WRITES_FLAGS(ALU_AND, false));
2812                         gen_one(target);
2813                         gen_one(target);
2814                         gen_imm_offset();
2815                 }
2816
2817                 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
2818                 return true;
2819 #else
2820                 gen_insn(INSN_FP_CMP, op_size, 0, 1);
2821                 gen_one(reg1);
2822                 gen_one(reg2);
2823 #if defined(ARCH_ARM32)
2824                 gen_insn(INSN_FP_TO_INT_FLAGS, 0, 0, 1);
2825 #endif
2826                 gen_insn(INSN_JMP_COND, op_size, FP_COND_P, 0);
2827                 gen_four(label_ovf);
2828                 g(gen_frame_set_cond(ctx, op_size, false, fp_alu, slot_r));
2829                 return true;
2830 #endif
2831         }
2832 #ifdef SUPPORTED_FP_X87
2833         if ((SUPPORTED_FP_X87 >> real_type) & 1) {
2834                 if (likely(cpu_test_feature(CPU_FEATURE_cmov))) {
2835                         g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_2));
2836                         g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
2837                         gen_insn(INSN_X87_FCOMIP, op_size, 0, 0);
2838                         gen_one(R_ST1);
2839                         gen_insn(INSN_X87_FSTP, op_size, 0, 0);
2840                         gen_one(R_ST0);
2841                         gen_insn(INSN_JMP_COND, op_size, COND_P, 0);
2842                         gen_four(label_ovf);
2843                         g(gen_frame_set_cond(ctx, op_size, false, fp_alu & 0xf, slot_r));
2844                         return true;
2845                 }
2846
2847                 if (real_type != 3) {
2848                         g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
2849                         g(gen_frame_load_x87(ctx, INSN_X87_FCOMP, op_size, 0, slot_2));
2850                 } else {
2851                         g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_2));
2852                         g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
2853                         gen_insn(INSN_X87_FCOMPP, op_size, 0, 0);
2854                 }
2855
2856                 gen_insn(INSN_X87_FNSTSW, 0, 0, 0);
2857                 gen_one(R_AX);
2858                 gen_one(R_AX);
2859
2860                 gen_insn(INSN_TEST, OP_SIZE_2, 0, 1);
2861                 gen_one(R_AX);
2862                 gen_one(ARG_IMM);
2863                 gen_eight(0x0400);
2864
2865                 gen_insn(INSN_JMP_COND, OP_SIZE_2, COND_NE, 0);
2866                 gen_four(label_ovf);
2867
2868                 switch (fp_alu) {
2869                         case FP_COND_E:
2870                                 gen_insn(INSN_TEST, OP_SIZE_2, 0, 1);
2871                                 gen_one(R_AX);
2872                                 gen_one(ARG_IMM);
2873                                 gen_eight(0x4000);
2874                                 g(gen_frame_set_cond(ctx, OP_SIZE_2, false, COND_NE, slot_r));
2875                                 break;
2876                         case FP_COND_NE:
2877                                 gen_insn(INSN_TEST, OP_SIZE_2, 0, 1);
2878                                 gen_one(R_AX);
2879                                 gen_one(ARG_IMM);
2880                                 gen_eight(0x4000);
2881                                 g(gen_frame_set_cond(ctx, OP_SIZE_2, false, COND_E, slot_r));
2882                                 break;
2883                         case FP_COND_B:
2884                                 gen_insn(INSN_TEST, OP_SIZE_2, 0, 1);
2885                                 gen_one(R_AX);
2886                                 gen_one(ARG_IMM);
2887                                 gen_eight(0x0100);
2888                                 g(gen_frame_set_cond(ctx, OP_SIZE_2, false, COND_NE, slot_r));
2889                                 break;
2890                         case FP_COND_BE:
2891                                 gen_insn(INSN_TEST, OP_SIZE_2, 0, 1);
2892                                 gen_one(R_AX);
2893                                 gen_one(ARG_IMM);
2894                                 gen_eight(0x4100);
2895                                 g(gen_frame_set_cond(ctx, OP_SIZE_2, false, COND_NE, slot_r));
2896                                 break;
2897                         default:
2898                                 internal(file_line, "gen_fp_alu: invalid condition %u", fp_alu);
2899                 }
2900                 return true;
2901         }
2902 #endif
2903 #ifdef SUPPORTED_FP_HALF_CVT
2904         if ((SUPPORTED_FP_HALF_CVT >> real_type) & 1) {
2905                 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, &reg1));
2906                 g(gen_frame_get(ctx, op_size, zero_x, slot_2, FR_SCRATCH_2, &reg2));
2907                 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
2908                 gen_one(FR_SCRATCH_1);
2909                 gen_one(reg1);
2910                 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
2911                 gen_one(FR_SCRATCH_2);
2912                 gen_one(reg2);
2913                 gen_insn(INSN_FP_CMP, OP_SIZE_4, 0, 1);
2914                 gen_one(FR_SCRATCH_1);
2915                 gen_one(FR_SCRATCH_2);
2916 #if defined(ARCH_ARM32)
2917                 gen_insn(INSN_FP_TO_INT_FLAGS, 0, 0, 1);
2918 #endif
2919                 gen_insn(INSN_JMP_COND, op_size, FP_COND_P, 0);
2920                 gen_four(label_ovf);
2921                 g(gen_frame_set_cond(ctx, op_size, false, fp_alu, slot_r));
2922                 return true;
2923         }
2924 #endif
2925
2926 do_upcall:
2927         return gen_alu_typed_upcall(ctx, upc, real_type, slot_1, slot_2, slot_r, label_ovf);
2928 }
2929
2930 #define OP_IS_ROUND(alu)        ((alu) == FP_ALU1_ROUND || (alu) == FP_ALU1_FLOOR || (alu) == FP_ALU1_CEIL || (alu) == FP_ALU1_TRUNC)
2931
2932 static bool attr_w gen_fp_alu1(struct codegen_context *ctx, unsigned real_type, unsigned op, uint32_t label_ovf, frame_t slot_1, frame_t slot_r)
2933 {
2934         unsigned attr_unused fp_alu;
2935         size_t upc;
2936         unsigned attr_unused op_size = real_type_to_op_size(real_type);
2937         unsigned reg1, target;
2938         switch (op) {
2939                 case OPCODE_REAL_OP_neg:
2940                 case OPCODE_REAL_OP_neg_alt1:
2941                 case OPCODE_REAL_OP_neg_alt2: fp_alu = FP_ALU1_NEG; upc = offsetof(struct cg_upcall_vector_s, REAL_unary_neg_real16_t); label_ovf = 0; goto do_alu;
2942                 case OPCODE_REAL_OP_sqrt:
2943                 case OPCODE_REAL_OP_sqrt_alt1:
2944                 case OPCODE_REAL_OP_sqrt_alt2: fp_alu = FP_ALU1_SQRT; upc = offsetof(struct cg_upcall_vector_s, REAL_unary_sqrt_real16_t); label_ovf = 0; goto do_alu;
2945                 case OPCODE_REAL_OP_round: fp_alu = FP_ALU1_ROUND; upc = offsetof(struct cg_upcall_vector_s, REAL_unary_round_real16_t); label_ovf = 0; goto do_alu;
2946                 case OPCODE_REAL_OP_floor: fp_alu = FP_ALU1_FLOOR; upc = offsetof(struct cg_upcall_vector_s, REAL_unary_floor_real16_t); label_ovf = 0; goto do_alu;
2947                 case OPCODE_REAL_OP_ceil: fp_alu = FP_ALU1_CEIL; upc = offsetof(struct cg_upcall_vector_s, REAL_unary_ceil_real16_t); label_ovf = 0; goto do_alu;
2948                 case OPCODE_REAL_OP_trunc: fp_alu = FP_ALU1_TRUNC; upc = offsetof(struct cg_upcall_vector_s, REAL_unary_trunc_real16_t); label_ovf = 0; goto do_alu;
2949                 case OPCODE_REAL_OP_to_int:
2950                 case OPCODE_REAL_OP_to_int_alt1:
2951                 case OPCODE_REAL_OP_to_int_alt2: upc = offsetof(struct cg_upcall_vector_s, REAL_unary_to_int_real16_t); goto do_to_int;
2952                 case OPCODE_REAL_OP_from_int:
2953                 case OPCODE_REAL_OP_from_int_alt1:
2954                 case OPCODE_REAL_OP_from_int_alt2: upc = offsetof(struct cg_upcall_vector_s, REAL_unary_from_int_real16_t); label_ovf = 0; goto do_from_int;
2955                 case OPCODE_REAL_OP_is_exception:
2956                 case OPCODE_REAL_OP_is_exception_alt1:
2957                 case OPCODE_REAL_OP_is_exception_alt2: upc = offsetof(struct cg_upcall_vector_s, REAL_unary_is_exception_real16_t); label_ovf = 0; goto do_is_exception;
2958                 default: upc = offsetof(struct cg_upcall_vector_s, REAL_unary_cbrt_real16_t) + (op - OPCODE_REAL_OP_cbrt) * TYPE_REAL_N * sizeof(void (*)(void)); label_ovf = 0; goto do_upcall;
2959         }
2960
2961 do_alu:
2962         if ((SUPPORTED_FP >> real_type) & 1 && (
2963 #if defined(ARCH_ALPHA)
2964                 fp_alu == FP_ALU1_NEG ||
2965                 (fp_alu == FP_ALU1_SQRT && cpu_test_feature(CPU_FEATURE_fix)) ||
2966 #elif defined(ARCH_ARM32)
2967                 fp_alu == FP_ALU1_NEG ||
2968                 fp_alu == FP_ALU1_SQRT ||
2969 #elif defined(ARCH_ARM64)
2970                 true ||
2971 #elif defined(ARCH_IA64)
2972                 fp_alu == FP_ALU1_NEG ||
2973 #elif defined(ARCH_LOONGARCH64)
2974                 fp_alu == FP_ALU1_NEG ||
2975                 fp_alu == FP_ALU1_SQRT ||
2976                 fp_alu == FP_ALU1_ROUND ||
2977 #elif defined(ARCH_MIPS)
2978                 fp_alu == FP_ALU1_NEG ||
2979                 (fp_alu == FP_ALU1_SQRT && MIPS_HAS_SQRT) ||
2980 #elif defined(ARCH_PARISC)
2981                 (fp_alu == FP_ALU1_NEG && PA_20) ||
2982                 fp_alu == FP_ALU1_SQRT ||
2983 #elif defined(ARCH_POWER)
2984                 fp_alu == FP_ALU1_NEG ||
2985                 (fp_alu == FP_ALU1_SQRT && cpu_test_feature(CPU_FEATURE_p2) && real_type != 4) ||
2986 #elif defined(ARCH_S390)
2987                 true ||
2988 #elif defined(ARCH_SPARC)
2989                 fp_alu == FP_ALU1_NEG ||
2990                 fp_alu == FP_ALU1_SQRT ||
2991 #elif defined(ARCH_RISCV64)
2992                 fp_alu == FP_ALU1_NEG ||
2993                 fp_alu == FP_ALU1_SQRT ||
2994 #elif defined(ARCH_X86)
2995                 fp_alu == FP_ALU1_SQRT ||
2996                 (OP_IS_ROUND(fp_alu) && cpu_test_feature(CPU_FEATURE_sse41)) ||
2997 #endif
2998                 false)) {
2999 #if defined(ARCH_S390)
3000                 if (op_size <= OP_SIZE_8 && (size_t)slot_1 * slot_size < 4096 && fp_alu == FP_ALU1_SQRT) {
3001                         target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_1);
3002                         if (slot_is_register(ctx, slot_1)) {
3003                                 gen_insn(INSN_FP_ALU1, op_size, fp_alu, 0);
3004                                 gen_one(target);
3005                                 gen_one(ctx->registers[slot_1]);
3006                                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
3007                         } else {
3008                                 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size, IMM_PURPOSE_VLDR_VSTR_OFFSET, op_size));
3009                                 gen_insn(INSN_FP_ALU1, op_size, fp_alu, 0);
3010                                 gen_one(target);
3011                                 gen_address_offset();
3012                                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
3013                         }
3014                         return true;
3015                 }
3016 #endif
3017                 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, &reg1));
3018                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_2);
3019                 gen_insn(INSN_FP_ALU1, op_size, fp_alu, 0);
3020                 gen_one(target);
3021                 gen_one(reg1);
3022                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
3023                 return true;
3024         }
3025 #ifdef SUPPORTED_FP_X87
3026         if ((SUPPORTED_FP_X87 >> real_type) & 1) {
3027                 if (fp_alu == FP_ALU1_NEG) {
3028                         g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
3029                         gen_insn(INSN_X87_FCHS, op_size, 0, 0);
3030                         g(gen_frame_store_x87(ctx, INSN_X87_FSTP, op_size, slot_r));
3031                         return true;
3032                 } else if (fp_alu == FP_ALU1_SQRT) {
3033                         g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
3034                         gen_insn(INSN_X87_FSQRT, op_size, 0, 0);
3035                         g(gen_frame_store_x87(ctx, INSN_X87_FSTP, op_size, slot_r));
3036                         return true;
3037                 } else if (fp_alu == FP_ALU1_ROUND) {
3038                         g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
3039                         gen_insn(INSN_X87_FRNDINT, op_size, 0, 0);
3040                         g(gen_frame_store_x87(ctx, INSN_X87_FSTP, op_size, slot_r));
3041                         return true;
3042                 }
3043         }
3044 #endif
3045 #ifdef SUPPORTED_FP_HALF_CVT
3046         if ((SUPPORTED_FP_HALF_CVT >> real_type) & 1 && (
3047 #if defined(ARCH_ARM32)
3048                 fp_alu == FP_ALU1_NEG ||
3049                 fp_alu == FP_ALU1_SQRT ||
3050 #elif defined(ARCH_ARM64)
3051                 true ||
3052 #elif defined(ARCH_X86)
3053                 fp_alu == FP_ALU1_SQRT ||
3054                 (OP_IS_ROUND(fp_alu) && cpu_test_feature(CPU_FEATURE_sse41)) ||
3055 #endif
3056                 false)) {
3057                 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, &reg1));
3058                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_1);
3059                 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
3060                 gen_one(target);
3061                 gen_one(reg1);
3062                 gen_insn(INSN_FP_ALU1, OP_SIZE_4, fp_alu, 0);
3063                 gen_one(target);
3064                 gen_one(target);
3065                 gen_insn(INSN_FP_CVT, OP_SIZE_4, op_size, 0);
3066                 gen_one(target);
3067                 gen_one(target);
3068                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
3069                 return true;
3070         }
3071 #endif
3072         goto do_upcall;
3073
3074 do_to_int:
3075         if ((SUPPORTED_FP >> real_type) & 1
3076 #if defined(ARCH_ALPHA)
3077                 && ARCH_SUPPORTS_TRAPS
3078 #endif
3079 #if defined(ARCH_MIPS)
3080                 && MIPS_HAS_TRUNC
3081 #endif
3082         ) {
3083                 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, &reg1));
3084                 goto do_cvt_to_int;
3085 do_cvt_to_int:
3086                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
3087 #if defined(ARCH_X86)
3088                 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_TO_INT32 : INSN_FP_TO_INT64, op_size, 0, 0);
3089                 gen_one(target);
3090                 gen_one(reg1);
3091
3092                 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, OP_SIZE_INT, target, sign_bit(uint_default_t), COND_E, label_ovf));
3093
3094                 g(gen_frame_store(ctx, OP_SIZE_INT, slot_r, 0, target));
3095                 return true;
3096 #endif
3097 #if defined(ARCH_ARM) || defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS)
3098 #if defined(ARCH_ARM)
3099                 gen_insn(INSN_FP_CMP, op_size, 0, 1);
3100                 gen_one(reg1);
3101                 gen_one(reg1);
3102 #if defined(ARCH_ARM32)
3103                 gen_insn(INSN_FP_TO_INT_FLAGS, 0, 0, 1);
3104 #endif
3105                 gen_insn(INSN_JMP_COND, op_size, FP_COND_P, 0);
3106                 gen_four(label_ovf);
3107 #else
3108                 gen_insn(INSN_FP_CMP_COND, op_size, FP_COND_P, 1);
3109                 gen_one(reg1);
3110                 gen_one(reg1);
3111
3112                 gen_insn(INSN_JMP_FP_TEST, 0, FP_COND_P, 0);
3113                 gen_four(label_ovf);
3114 #endif
3115 #if defined(ARCH_ARM32) || defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS)
3116                 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_TO_INT32 : INSN_FP_TO_INT64, op_size, 0, 0);
3117                 gen_one(FR_SCRATCH_1);
3118                 gen_one(reg1);
3119
3120                 g(gen_mov(ctx, OP_SIZE_INT, target, FR_SCRATCH_1));
3121 #else
3122                 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_TO_INT32 : INSN_FP_TO_INT64, op_size, 0, 0);
3123                 gen_one(target);
3124                 gen_one(reg1);
3125 #endif
3126                 g(gen_imm(ctx, (int_default_t)(sign_bit(uint_default_t) + 1), IMM_PURPOSE_ADD, OP_SIZE_INT));
3127                 gen_insn(INSN_ALU, OP_SIZE_INT, ALU_ADD, ALU_WRITES_FLAGS(ALU_ADD, is_imm()));
3128                 gen_one(R_SCRATCH_2);
3129                 gen_one(target);
3130                 gen_imm_offset();
3131
3132                 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, OP_SIZE_INT, R_SCRATCH_2, 1, COND_BE, label_ovf));
3133
3134                 g(gen_frame_store(ctx, OP_SIZE_INT, slot_r, 0, target));
3135                 return true;
3136 #endif
3137 #if defined(ARCH_IA64)
3138                 gen_insn(INSN_FP_TO_INT64, op_size, 0, 0);
3139                 gen_one(FR_SCRATCH_1);
3140                 gen_one(reg1);
3141
3142                 g(gen_mov(ctx, OP_SIZE_NATIVE, target, FR_SCRATCH_1));
3143
3144                 if (OP_SIZE_INT == OP_SIZE_4) {
3145                         g(gen_extend(ctx, OP_SIZE_4, sign_x, R_SCRATCH_2, target));
3146                         g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, target, R_SCRATCH_2, COND_NE, label_ovf));
3147                 } else {
3148                         g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, target, sign_bit(uint64_t), COND_E, label_ovf));
3149                 }
3150
3151                 g(gen_frame_store(ctx, OP_SIZE_INT, slot_r, 0, target));
3152                 return true;
3153 #endif
3154 #if defined(ARCH_PARISC) || defined(ARCH_POWER) || defined(ARCH_SPARC)
3155 #if defined(ARCH_POWER)
3156                 if (!cpu_test_feature(CPU_FEATURE_ppc))
3157                         goto do_upcall;
3158                 if (OP_SIZE_INT == OP_SIZE_4)
3159                         goto do_upcall;
3160 #endif
3161                 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_TO_INT32 : INSN_FP_TO_INT64, op_size, 0, 0);
3162                 gen_one(FR_SCRATCH_1);
3163                 gen_one(reg1);
3164
3165                 g(gen_frame_store_raw(ctx, OP_SIZE_INT, slot_r, 0, FR_SCRATCH_1));
3166                 if (slot_is_register(ctx, slot_r))
3167                         g(unspill(ctx, slot_r));
3168                 g(gen_frame_load(ctx, OP_SIZE_INT, garbage, slot_r, 0, false, target));
3169
3170                 g(gen_imm(ctx, sign_bit(uint_default_t) + 1, IMM_PURPOSE_ADD, OP_SIZE_INT));
3171                 gen_insn(INSN_ALU, i_size(OP_SIZE_INT), ALU_ADD, ALU_WRITES_FLAGS(ALU_ADD, is_imm()));
3172                 gen_one(R_SCRATCH_2);
3173                 gen_one(target);
3174                 gen_imm_offset();
3175
3176                 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, OP_SIZE_INT, R_SCRATCH_2, 1, COND_BE, label_ovf));
3177
3178                 return true;
3179 #endif
3180 #if defined(ARCH_ALPHA)
3181                 gen_insn(INSN_FP_TO_INT64_TRAP, op_size, 0, 0);
3182                 gen_one(FR_SCRATCH_2);
3183                 gen_one(reg1);
3184                 gen_four(label_ovf);
3185
3186                 if (OP_SIZE_INT == OP_SIZE_4) {
3187                         gen_insn(INSN_FP_INT64_TO_INT32_TRAP, 0, 0, 0);
3188                         gen_one(FR_SCRATCH_3);
3189                         gen_one(FR_SCRATCH_2);
3190                         gen_four(label_ovf);
3191                         g(gen_frame_store_raw(ctx, OP_SIZE_INT, slot_r, 0, FR_SCRATCH_3));
3192                 } else {
3193                         g(gen_frame_store_raw(ctx, OP_SIZE_INT, slot_r, 0, FR_SCRATCH_2));
3194                 }
3195                 if (slot_is_register(ctx, slot_r))
3196                         g(unspill(ctx, slot_r));
3197                 return true;
3198 #endif
3199 #if defined(ARCH_S390)
3200                 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_TO_INT32 : INSN_FP_TO_INT64, op_size, 0, 1);
3201                 gen_one(target);
3202                 gen_one(reg1);
3203
3204                 gen_insn(INSN_JMP_COND, op_size, FP_COND_P, 0);
3205                 gen_four(label_ovf);
3206
3207                 g(gen_frame_store(ctx, OP_SIZE_INT, slot_r, 0, target));
3208                 return true;
3209 #endif
3210 #if defined(ARCH_RISCV64)
3211                 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_TO_INT32 : INSN_FP_TO_INT64, op_size, 0, 0);
3212                 gen_one(target);
3213                 gen_one(reg1);
3214
3215                 g(gen_load_constant(ctx, R_SCRATCH_2, sign_bit(int_default_t)));
3216
3217                 g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, target, R_SCRATCH_2, COND_E, label_ovf));
3218
3219                 g(gen_imm(ctx, -1, IMM_PURPOSE_XOR, i_size(size)));
3220                 gen_insn(INSN_ALU, OP_SIZE_NATIVE, ALU_XOR, ALU_WRITES_FLAGS(ALU_XOR, is_imm()));
3221                 gen_one(R_SCRATCH_2);
3222                 gen_one(R_SCRATCH_2);
3223                 gen_imm_offset();
3224
3225                 g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, target, R_SCRATCH_2, COND_E, label_ovf));
3226
3227                 g(gen_frame_store(ctx, OP_SIZE_INT, slot_r, 0, target));
3228                 return true;
3229 #endif
3230         }
3231 #ifdef SUPPORTED_FP_X87
3232         if ((SUPPORTED_FP_X87 >> real_type) & 1) {
3233                 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
3234
3235                 if (likely(cpu_test_feature(CPU_FEATURE_sse3))) {
3236                         g(gen_frame_store_x87(ctx, INSN_X87_FISTTP, OP_SIZE_INT, slot_r));
3237                 } else {
3238                         gen_insn(INSN_PUSH, OP_SIZE_NATIVE, 0, 0);
3239                         gen_one(ARG_IMM);
3240                         gen_eight(0x0f7f);
3241
3242                         gen_insn(INSN_X87_FLDCW, 0, 0, 0);
3243                         gen_one(ARG_ADDRESS_1);
3244                         gen_one(R_SP);
3245                         gen_eight(0);
3246
3247                         g(gen_frame_store_x87(ctx, INSN_X87_FISTP, OP_SIZE_INT, slot_r));
3248
3249                         gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
3250                         gen_one(ARG_ADDRESS_1);
3251                         gen_one(R_SP);
3252                         gen_eight(0);
3253                         gen_one(ARG_IMM);
3254                         gen_eight(0x037f);
3255
3256                         gen_insn(INSN_X87_FLDCW, 0, 0, 0);
3257                         gen_one(ARG_ADDRESS_1);
3258                         gen_one(R_SP);
3259                         gen_eight(0);
3260
3261                         gen_insn(INSN_ALU, i_size(OP_SIZE_ADDRESS), ALU_ADD, 1);
3262                         gen_one(R_SP);
3263                         gen_one(R_SP);
3264                         gen_one(ARG_IMM);
3265                         gen_eight(1 << OP_SIZE_NATIVE);
3266                 }
3267                 if (slot_is_register(ctx, slot_r))
3268                         g(unspill(ctx, slot_r));
3269                 g(gen_frame_load(ctx, OP_SIZE_INT, garbage, slot_r, 0, false, R_SCRATCH_1));
3270
3271                 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, OP_SIZE_INT, R_SCRATCH_1, sign_bit(int_default_t), COND_E, label_ovf));
3272
3273                 return true;
3274         }
3275 #endif
3276 #ifdef SUPPORTED_FP_HALF_CVT
3277         if ((SUPPORTED_FP_HALF_CVT >> real_type) & 1) {
3278                 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, &reg1));
3279                 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
3280                 gen_one(FR_SCRATCH_1);
3281                 gen_one(reg1);
3282                 reg1 = FR_SCRATCH_1;
3283                 real_type = 1;
3284                 op_size = real_type_to_op_size(real_type);
3285                 goto do_cvt_to_int;
3286         }
3287 #endif
3288         goto do_upcall;
3289
3290 do_from_int:
3291         if ((SUPPORTED_FP >> real_type) & 1) {
3292 #if defined(ARCH_ALPHA) || defined(ARCH_ARM32) || defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS) || defined(ARCH_PARISC) || defined(ARCH_POWER) || defined(ARCH_SPARC)
3293                 int int_op_size = OP_SIZE_INT;
3294 #if defined(ARCH_POWER)
3295                 if (int_op_size == OP_SIZE_4)
3296                         goto do_upcall;
3297                 if (op_size == OP_SIZE_4 && !cpu_test_feature(CPU_FEATURE_v206))
3298                         goto do_upcall;
3299                 if (op_size == OP_SIZE_8 && !cpu_test_feature(CPU_FEATURE_ppc))
3300                         goto do_upcall;
3301 #endif
3302                 if (slot_is_register(ctx, slot_1))
3303                         g(spill(ctx, slot_1));
3304                 g(gen_frame_load_raw(ctx, int_op_size, zero_x, slot_1, 0, false, FR_SCRATCH_1));
3305                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_2);
3306 #if defined(ARCH_ALPHA)
3307                 if (OP_SIZE_INT == OP_SIZE_4) {
3308                         gen_insn(INSN_MOVSX, OP_SIZE_4, 0, 0);
3309                         gen_one(FR_SCRATCH_1);
3310                         gen_one(FR_SCRATCH_1);
3311
3312                         int_op_size = OP_SIZE_8;
3313                 }
3314 #endif
3315                 gen_insn(int_op_size == OP_SIZE_4 ? INSN_FP_FROM_INT32 : INSN_FP_FROM_INT64, op_size, 0, 0);
3316                 gen_one(target);
3317                 gen_one(FR_SCRATCH_1);
3318
3319                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
3320                 return true;
3321 #elif defined(ARCH_IA64)
3322                 g(gen_frame_get(ctx, OP_SIZE_INT, sign_x, slot_1, R_SCRATCH_1, &reg1));
3323                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_1);
3324
3325                 g(gen_mov(ctx, OP_SIZE_NATIVE, target, reg1));
3326
3327                 gen_insn(INSN_FP_FROM_INT64, op_size, 0, 0);
3328                 gen_one(target);
3329                 gen_one(target);
3330
3331                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
3332                 return true;
3333 #else
3334                 g(gen_frame_get(ctx, OP_SIZE_INT, garbage, slot_1, R_SCRATCH_1, &reg1));
3335                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_1);
3336
3337                 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_FROM_INT32 : INSN_FP_FROM_INT64, op_size, 0, 0);
3338                 gen_one(target);
3339                 gen_one(reg1);
3340
3341                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
3342                 return true;
3343 #endif
3344         }
3345 #ifdef SUPPORTED_FP_X87
3346         if ((SUPPORTED_FP_X87 >> real_type) & 1) {
3347                 if (slot_is_register(ctx, slot_1))
3348                         g(spill(ctx, slot_1));
3349                 g(gen_frame_load_x87(ctx, INSN_X87_FILD, OP_SIZE_INT, 0, slot_1));
3350                 g(gen_frame_store_x87(ctx, INSN_X87_FSTP, op_size, slot_r));
3351                 return true;
3352         }
3353 #endif
3354 #ifdef SUPPORTED_FP_HALF_CVT
3355         if ((SUPPORTED_FP_HALF_CVT >> real_type) & 1) {
3356                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_1);
3357 #if defined(ARCH_ARM32)
3358                 g(gen_frame_get(ctx, OP_SIZE_INT, zero_x, slot_1, FR_SCRATCH_1, &reg1));
3359
3360                 gen_insn(INSN_FP_FROM_INT32, OP_SIZE_4, 0, 0);
3361                 gen_one(target);
3362                 gen_one(reg1);
3363 #else
3364                 g(gen_frame_get(ctx, OP_SIZE_INT, garbage, slot_1, R_SCRATCH_1, &reg1));
3365                 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_FROM_INT32 : INSN_FP_FROM_INT64, OP_SIZE_4, 0, 0);
3366                 gen_one(target);
3367                 gen_one(reg1);
3368 #endif
3369                 gen_insn(INSN_FP_CVT, OP_SIZE_4, op_size, 0);
3370                 gen_one(target);
3371                 gen_one(target);
3372                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
3373                 return true;
3374         }
3375 #endif
3376         goto do_upcall;
3377
3378 do_is_exception:
3379         if ((SUPPORTED_FP >> real_type) & 1) {
3380                 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, &reg1));
3381                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
3382 #if defined(ARCH_ALPHA)
3383                 gen_insn(INSN_FP_CMP_UNORDERED_DEST_REG, op_size, 0, 0);
3384                 gen_one(FR_SCRATCH_2);
3385                 gen_one(FR_SCRATCH_1);
3386                 gen_one(reg1);
3387
3388                 if (!cpu_test_feature(CPU_FEATURE_fix)) {
3389                         g(gen_frame_store_raw(ctx, OP_SIZE_4, slot_r, 0, FR_SCRATCH_2));
3390                         g(gen_frame_load_raw(ctx, OP_SIZE_4, sign_x, slot_r, 0, false, target));
3391                 } else {
3392                         g(gen_mov(ctx, OP_SIZE_4, target, FR_SCRATCH_2));
3393                 }
3394
3395                 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SHR, target, target, 30, 0));
3396
3397                 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
3398
3399                 return true;
3400 #elif defined(ARCH_IA64)
3401                 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_P, 0);
3402                 gen_one(R_CMP_RESULT);
3403                 gen_one(reg1);
3404                 gen_one(reg1);
3405
3406                 g(gen_mov(ctx, OP_SIZE_NATIVE, target, R_CMP_RESULT));
3407
3408                 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
3409 #elif defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS) || defined(ARCH_PARISC)
3410                 gen_insn(INSN_FP_CMP_COND, op_size, FP_COND_P, 1);
3411                 gen_one(reg1);
3412                 gen_one(reg1);
3413
3414                 gen_insn(INSN_FP_TEST_REG, OP_SIZE_NATIVE, FP_COND_P, 0);
3415                 gen_one(target);
3416
3417                 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
3418 #elif defined(ARCH_RISCV64)
3419                 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_E, 0);
3420                 gen_one(target);
3421                 gen_one(reg1);
3422                 gen_one(reg1);
3423
3424                 g(gen_imm(ctx, 1, IMM_PURPOSE_XOR, OP_SIZE_NATIVE));
3425                 gen_insn(INSN_ALU, OP_SIZE_NATIVE, ALU_XOR, ALU_WRITES_FLAGS(ALU_XOR, is_imm()));
3426                 gen_one(target);
3427                 gen_one(target);
3428                 gen_imm_offset();
3429
3430                 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
3431 #else
3432                 gen_insn(INSN_FP_CMP, op_size, 0, 1);
3433                 gen_one(reg1);
3434                 gen_one(reg1);
3435 #if defined(ARCH_ARM32)
3436                 gen_insn(INSN_FP_TO_INT_FLAGS, 0, 0, 1);
3437 #endif
3438                 g(gen_frame_set_cond(ctx, op_size, false, FP_COND_P, slot_r));
3439 #endif
3440                 return true;
3441         }
3442 #ifdef SUPPORTED_FP_X87
3443         if ((SUPPORTED_FP_X87 >> real_type) & 1) {
3444                 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
3445                 if (likely(cpu_test_feature(CPU_FEATURE_cmov))) {
3446                         gen_insn(INSN_X87_FCOMIP, op_size, 0, 0);
3447                         gen_one(R_ST0);
3448
3449                         g(gen_frame_set_cond(ctx, op_size, false, COND_P, slot_r));
3450                         return true;
3451                 }
3452
3453                 gen_insn(INSN_X87_FCOMP, op_size, 0, 0);
3454                 gen_one(R_ST0);
3455
3456                 gen_insn(INSN_X87_FNSTSW, 0, 0, 0);
3457                 gen_one(R_AX);
3458                 gen_one(R_AX);
3459
3460                 gen_insn(INSN_TEST, OP_SIZE_2, 0, 1);
3461                 gen_one(R_AX);
3462                 gen_one(ARG_IMM);
3463                 gen_eight(0x0400);
3464
3465                 g(gen_frame_set_cond(ctx, op_size, false, COND_NE, slot_r));
3466
3467                 return true;
3468         }
3469 #endif
3470 #ifdef SUPPORTED_FP_HALF_CVT
3471         if ((SUPPORTED_FP_HALF_CVT >> real_type) & 1) {
3472                 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, &reg1));
3473                 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
3474                 gen_one(FR_SCRATCH_1);
3475                 gen_one(reg1);
3476                 gen_insn(INSN_FP_CMP, OP_SIZE_4, 0, 1);
3477                 gen_one(FR_SCRATCH_1);
3478                 gen_one(FR_SCRATCH_1);
3479 #if defined(ARCH_ARM32)
3480                 gen_insn(INSN_FP_TO_INT_FLAGS, 0, 0, 1);
3481 #endif
3482                 g(gen_frame_set_cond(ctx, op_size, false, FP_COND_P, slot_r));
3483                 return true;
3484         }
3485 #endif
3486
3487 do_upcall:
3488         g(gen_alu_typed_upcall(ctx, upc, real_type, slot_1, NO_FRAME_T, slot_r, label_ovf));
3489         return true;
3490 }
3491
3492 static bool attr_w gen_is_exception(struct codegen_context *ctx, frame_t slot_1, frame_t slot_r)
3493 {
3494         uint32_t no_ex_label, escape_label;
3495         const struct type *type = get_type_of_local(ctx, slot_1);
3496
3497         no_ex_label = alloc_label(ctx);
3498         if (unlikely(!no_ex_label))
3499                 return false;
3500         escape_label = alloc_escape_label(ctx);
3501         if (unlikely(!escape_label))
3502                 return false;
3503
3504         if (TYPE_IS_FLAT(type))
3505                 g(gen_test_1_jz_cached(ctx, slot_1, no_ex_label));
3506
3507         g(gen_frame_load(ctx, OP_SIZE_SLOT, zero_x, slot_1, 0, false, R_SCRATCH_1));
3508         g(gen_ptr_is_thunk(ctx, R_SCRATCH_1, slot_1, escape_label));
3509
3510         if (!TYPE_IS_FLAT(type)) {
3511                 g(gen_compare_da_tag(ctx, R_SCRATCH_1, DATA_TAG_flat, COND_E, escape_label, R_SCRATCH_1));
3512         }
3513
3514         gen_label(no_ex_label);
3515         g(gen_frame_clear(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r));
3516
3517         flag_set(ctx, slot_r, false);
3518
3519         return true;
3520 }
3521
3522 static bool attr_w gen_system_property(struct codegen_context *ctx, frame_t slot_1, frame_t slot_r)
3523 {
3524         uint32_t escape_label;
3525
3526         escape_label = alloc_escape_label(ctx);
3527         if (unlikely(!escape_label))
3528                 return false;
3529
3530         g(gen_test_1_cached(ctx, slot_1, escape_label));
3531
3532         g(gen_upcall_start(ctx, 1));
3533
3534         g(gen_frame_load(ctx, OP_SIZE_INT, garbage, slot_1, 0, false, R_ARG0));
3535         g(gen_upcall_argument(ctx, 0));
3536
3537         g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_ipret_system_property), 1));
3538
3539         g(gen_frame_store(ctx, OP_SIZE_INT, slot_r, 0, R_RET0));
3540
3541         flag_set(ctx, slot_1, false);
3542         flag_set(ctx, slot_r, false);
3543
3544         return true;
3545 }
3546
3547 static bool attr_w gen_alu_jmp(struct codegen_context *ctx, unsigned mode, unsigned op_size, unsigned op, frame_t slot_1, frame_t slot_2, int32_t jmp_offset, bool *failed)
3548 {
3549         unsigned alu;
3550         enum extend ex;
3551         unsigned reg1;
3552         unsigned attr_unused reg2;
3553 #if 0
3554         *failed = true; return true;
3555 #endif
3556         switch (mode) {
3557                 case MODE_FIXED: switch (op) {
3558                         case OPCODE_FIXED_OP_equal:             alu = COND_E; goto do_compare;
3559                         case OPCODE_FIXED_OP_not_equal:         alu = COND_NE; goto do_compare;
3560                         case OPCODE_FIXED_OP_less:              alu = COND_L; goto do_compare;
3561                         case OPCODE_FIXED_OP_less_equal:        alu = COND_LE; goto do_compare;
3562                         case OPCODE_FIXED_OP_uless:             alu = COND_B; goto do_compare;
3563                         case OPCODE_FIXED_OP_uless_equal:       alu = COND_BE; goto do_compare;
3564                         case OPCODE_FIXED_OP_bt:                *failed = true; return true;
3565                         default:                                internal(file_line, "gen_alu_jmp: unsupported fixed operation %u", op);
3566                 }
3567                 case MODE_INT: switch (op) {
3568                         case OPCODE_INT_OP_equal:               alu = COND_E; goto do_compare;
3569                         case OPCODE_INT_OP_not_equal:           alu = COND_NE; goto do_compare;
3570                         case OPCODE_INT_OP_less:                alu = COND_L; goto do_compare;
3571                         case OPCODE_INT_OP_less_equal:          alu = COND_LE; goto do_compare;
3572                         case OPCODE_INT_OP_bt:                  *failed = true; return true;
3573                         default:                                internal(file_line, "gen_alu_jmp: unsupported int operation %u", op);
3574                 }
3575                 case MODE_BOOL: switch (op) {
3576                         case OPCODE_BOOL_OP_and:                alu = ALU_AND; mode = MODE_FIXED; goto do_alu;
3577                         case OPCODE_BOOL_OP_or:                 alu = ALU_OR; mode = MODE_FIXED; goto do_alu;
3578                         case OPCODE_BOOL_OP_equal:              alu = COND_E; mode = MODE_FIXED; goto do_compare;
3579                         case OPCODE_BOOL_OP_not_equal:          alu = COND_NE; mode = MODE_FIXED; goto do_compare;
3580                         case OPCODE_BOOL_OP_less:               alu = COND_L; mode = MODE_FIXED; goto do_compare;
3581                         case OPCODE_BOOL_OP_less_equal:         alu = COND_LE; mode = MODE_FIXED; goto do_compare;
3582                         default:                                internal(file_line, "gen_alu_jmp: unsupported bool operation %u", op);
3583                 }
3584         }
3585         internal(file_line, "gen_alu_jmp: unsupported mode %u", mode);
3586 do_compare: {
3587                 bool attr_unused logical;
3588                 if (unlikely(op_size > OP_SIZE_NATIVE)) {
3589                         *failed = true;
3590                         return true;
3591                 }
3592                 if (slot_is_register(ctx, slot_2) && !slot_is_register(ctx, slot_1)) {
3593                         frame_t s = slot_1;
3594                         slot_1 = slot_2;
3595                         slot_2 = s;
3596                         switch (alu) {
3597                                 case COND_L:    alu = COND_G; break;
3598                                 case COND_LE:   alu = COND_GE; break;
3599                                 case COND_B:    alu = COND_A; break;
3600                                 case COND_BE:   alu = COND_AE; break;
3601                         }
3602                 }
3603                 ex = op_size == i_size_cmp(op_size) + (unsigned)zero ? garbage : alu == COND_L || alu == COND_LE || alu == COND_G || alu == COND_GE || ARCH_PREFERS_SX(op_size) ? sign_x : zero_x;
3604                 g(gen_frame_get(ctx, op_size, ex, slot_1, R_SCRATCH_1, &reg1));
3605                 if (ARCH_HAS_JMP_2REGS(alu)) {
3606                         g(gen_frame_get(ctx, op_size, ex, slot_2, R_SCRATCH_2, &reg2));
3607                         g(gen_jump(ctx, jmp_offset, i_size_cmp(op_size), alu ^ 1, reg1, reg2));
3608                         return true;
3609                 }
3610 #if ARCH_HAS_FLAGS
3611                 logical = COND_IS_LOGICAL(alu ^ 1);
3612                 g(gen_frame_load_cmp(ctx, op_size, logical, ex, false, slot_2, 0, false, reg1));
3613                 g(gen_jump(ctx, jmp_offset, op_size, alu ^ 1, -1U, -1U));
3614 #else
3615                 g(gen_frame_get(ctx, op_size, ex, slot_2, R_SCRATCH_2, &reg2));
3616                 g(gen_cmp_dest_reg(ctx, op_size, reg1, reg2, R_CMP_RESULT, 0, alu));
3617                 g(gen_jump(ctx, jmp_offset, OP_SIZE_NATIVE, COND_E, R_CMP_RESULT, -1U));
3618 #endif
3619                 return true;
3620         }
3621 do_alu: {
3622                 if (slot_is_register(ctx, slot_2) && !slot_is_register(ctx, slot_1)) {
3623                         frame_t s = slot_1;
3624                         slot_1 = slot_2;
3625                         slot_2 = s;
3626                 }
3627                 ex = op_size == i_size(op_size) + (unsigned)zero ? garbage : ARCH_PREFERS_SX(op_size) ? sign_x : zero_x;
3628                 g(gen_frame_get(ctx, op_size, ex, slot_1, R_SCRATCH_1, &reg1));
3629 #if defined(ARCH_X86)
3630                 if (alu == ALU_AND && !slot_is_register(ctx, slot_2)) {
3631                         g(gen_address(ctx, R_FRAME, (size_t)slot_2 * slot_size, IMM_PURPOSE_LDR_OFFSET, op_size));
3632                         gen_insn(INSN_TEST, i_size(op_size), 0, 1);
3633                         gen_one(reg1);
3634                         gen_address_offset();
3635                         g(gen_jump(ctx, jmp_offset, op_size, COND_E, -1U, -1U));
3636                         return true;
3637                 }
3638 #endif
3639                 g(gen_frame_get(ctx, op_size, ex, slot_2, R_SCRATCH_2, &reg2));
3640 #if ARCH_HAS_FLAGS && !defined(ARCH_S390)
3641                 if (alu == ALU_AND) {
3642                         gen_insn(INSN_TEST, i_size(op_size), 0, 1);
3643                         gen_one(reg1);
3644                         gen_one(reg2);
3645                         g(gen_jump(ctx, jmp_offset, op_size, COND_E, -1U, -1U));
3646                         return true;
3647                 }
3648 #endif
3649 #if defined(ARCH_ARM64)
3650                 if (alu == ALU_OR)
3651                         goto skip_flags;
3652 #endif
3653 #if ARCH_HAS_FLAGS
3654                 g(gen_3address_alu(ctx, i_size(op_size), alu, R_SCRATCH_1, reg1, reg2, 1));
3655                 g(gen_jump(ctx, jmp_offset, i_size(op_size), COND_E, -1U, -1U));
3656                 return true;
3657 #endif
3658                 goto skip_flags;
3659 skip_flags:
3660                 g(gen_3address_alu(ctx, i_size(op_size), alu, R_SCRATCH_1, reg1, reg2, 0));
3661                 g(gen_jump(ctx, jmp_offset, i_size(op_size), COND_E, R_SCRATCH_1, -1U));
3662                 return true;
3663         }
3664 }
3665
3666 static bool attr_w gen_fp_alu_jmp(struct codegen_context *ctx, unsigned real_type, unsigned op, uint32_t label_ovf, frame_t slot_1, frame_t slot_2, int32_t jmp_offset, bool *failed)
3667 {
3668         unsigned attr_unused fp_alu;
3669         unsigned attr_unused op_size = real_type_to_op_size(real_type);
3670         unsigned reg1, reg2;
3671         unsigned attr_unused target;
3672         switch (op) {
3673                 case OPCODE_REAL_OP_equal:
3674                 case OPCODE_REAL_OP_equal_alt1:
3675                 case OPCODE_REAL_OP_equal_alt2: fp_alu = FP_COND_E; goto do_cmp;
3676                 case OPCODE_REAL_OP_not_equal:
3677                 case OPCODE_REAL_OP_not_equal_alt1:
3678                 case OPCODE_REAL_OP_not_equal_alt2: fp_alu = FP_COND_NE; goto do_cmp;
3679                 case OPCODE_REAL_OP_less:
3680                 case OPCODE_REAL_OP_less_alt1:
3681                 case OPCODE_REAL_OP_less_alt2: fp_alu = FP_COND_B; goto do_cmp;
3682                 case OPCODE_REAL_OP_less_equal:
3683                 case OPCODE_REAL_OP_less_equal_alt1:
3684                 case OPCODE_REAL_OP_less_equal_alt2: fp_alu = FP_COND_BE; goto do_cmp;
3685                 default: internal(file_line, "gen_fp_alu_jmp: unsupported operation %u", op);
3686         }
3687
3688 do_cmp:
3689         if ((SUPPORTED_FP >> real_type) & 1
3690 #if defined(ARCH_ALPHA)
3691                 && ARCH_SUPPORTS_TRAPS && cpu_test_feature(CPU_FEATURE_fix)
3692 #endif
3693         ) {
3694                 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, &reg1));
3695                 g(gen_frame_get(ctx, op_size, zero_x, slot_2, FR_SCRATCH_2, &reg2));
3696                 target = R_SCRATCH_1;
3697 #if defined(ARCH_ALPHA)
3698                 gen_insn(INSN_FP_CMP_DEST_REG_TRAP, op_size, fp_alu == FP_COND_NE ? FP_COND_E : fp_alu, 0);
3699                 gen_one(FR_SCRATCH_3);
3700                 gen_one(reg1);
3701                 gen_one(reg2);
3702                 gen_four(label_ovf);
3703
3704                 g(gen_mov(ctx, OP_SIZE_4, target, FR_SCRATCH_3));
3705
3706                 if (fp_alu == FP_COND_NE) {
3707                         g(gen_jump(ctx, jmp_offset, OP_SIZE_NATIVE, COND_NE, target, -1U));
3708                 } else {
3709                         g(gen_jump(ctx, jmp_offset, OP_SIZE_NATIVE, COND_E, target, -1U));
3710                 }
3711
3712                 return true;
3713 #elif defined(ARCH_IA64)
3714                 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_P, 0);
3715                 gen_one(R_CMP_RESULT);
3716                 gen_one(reg1);
3717                 gen_one(reg2);
3718
3719                 gen_insn(INSN_JMP_REG, OP_SIZE_NATIVE, COND_NE, 0);
3720                 gen_one(R_CMP_RESULT);
3721                 gen_four(label_ovf);
3722
3723                 gen_insn(INSN_FP_CMP_DEST_REG, op_size, fp_alu, 0);
3724                 gen_one(R_CMP_RESULT);
3725                 gen_one(reg1);
3726                 gen_one(reg2);
3727
3728                 g(gen_jump(ctx, jmp_offset, OP_SIZE_NATIVE, COND_E, R_CMP_RESULT, -1U);
3729
3730                 return true;
3731 #elif defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS) || defined(ARCH_PARISC)
3732                 gen_insn(INSN_FP_CMP_COND, op_size, FP_COND_P, 1);
3733                 gen_one(reg1);
3734                 gen_one(reg2);
3735
3736                 gen_insn(INSN_JMP_FP_TEST, 0, FP_COND_P, 0);
3737                 gen_four(label_ovf);
3738
3739                 gen_insn(INSN_FP_CMP_COND, op_size, fp_alu ^ 1, 1);
3740                 gen_one(reg1);
3741                 gen_one(reg2);
3742
3743                 g(gen_jump(ctx, jmp_offset, OP_SIZE_NATIVE, fp_alu ^ 1, -1U, -1U));
3744
3745                 return true;
3746 #elif defined(ARCH_RISCV64)
3747                 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_E, 0);
3748                 gen_one(R_SCRATCH_1);
3749                 gen_one(reg1);
3750                 gen_one(reg1);
3751
3752                 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_E, 0);
3753                 gen_one(R_SCRATCH_2);
3754                 gen_one(reg2);
3755                 gen_one(reg2);
3756
3757                 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_AND, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_2, 0));
3758
3759                 g(gen_jmp_on_zero(ctx, OP_SIZE_NATIVE, R_SCRATCH_1, COND_E, label_ovf));
3760
3761                 gen_insn(INSN_FP_CMP_DEST_REG, op_size, fp_alu == FP_COND_NE ? FP_COND_E : fp_alu, 0);
3762                 gen_one(target);
3763                 gen_one(reg1);
3764                 gen_one(reg2);
3765
3766                 if (fp_alu == FP_COND_NE) {
3767                         g(gen_jump(ctx, jmp_offset, OP_SIZE_NATIVE, COND_NE, target, -1U));
3768                 } else {
3769                         g(gen_jump(ctx, jmp_offset, OP_SIZE_NATIVE, COND_E, target, -1U));
3770                 }
3771                 return true;
3772 #else
3773                 gen_insn(INSN_FP_CMP, op_size, 0, 1);
3774                 gen_one(reg1);
3775                 gen_one(reg2);
3776 #if defined(ARCH_ARM32)
3777                 gen_insn(INSN_FP_TO_INT_FLAGS, 0, 0, 1);
3778 #endif
3779                 gen_insn(INSN_JMP_COND, op_size, FP_COND_P, 0);
3780                 gen_four(label_ovf);
3781                 g(gen_jump(ctx, jmp_offset, op_size, fp_alu ^ 1, -1U, -1U));
3782                 return true;
3783 #endif
3784         }
3785         *failed = true;
3786         return true;
3787 }