cg-alu.inc

   1 /*
   2  * Copyright (C) 2024 Mikulas Patocka
   3  *
   4  * This file is part of Ajla.
   5  *
   6  * Ajla is free software: you can redistribute it and/or modify it under the
   7  * terms of the GNU General Public License as published by the Free Software
   8  * Foundation, either version 3 of the License, or (at your option) any later
   9  * version.
  10  *
  11  * Ajla is distributed in the hope that it will be useful, but WITHOUT ANY
  12  * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
  13  * A PARTICULAR PURPOSE. See the GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along with
  16  * Ajla. If not, see <https://www.gnu.org/licenses/>.
  17  */
  18
  19 static bool attr_w gen_jump(struct codegen_context *ctx, int32_t jmp_offset, unsigned op_size, unsigned cond, unsigned reg1, unsigned reg2);
  20
  21 static bool attr_w gen_alu_upcall(struct codegen_context *ctx, size_t upcall, unsigned op_size, frame_t slot_1, frame_t slot_2, frame_t slot_r, uint32_t label_ovf)
  22 {
  23         if (slot_is_register(ctx, slot_1))
  24                 g(spill(ctx, slot_1));
  25         if (slot_2 != NO_FRAME_T && slot_is_register(ctx, slot_2))
  26                 g(spill(ctx, slot_2));
  27         g(gen_upcall_start(ctx, frame_t_is_const(slot_2) ? 4 : slot_2 != NO_FRAME_T ? 3 : 2));
  28         g(gen_frame_address(ctx, slot_1, 0, R_ARG0));
  29         g(gen_upcall_argument(ctx, 0));
  30         if (frame_t_is_const(slot_2)) {
  31                 size_t x_offs;
  32                 g(gen_load_constant(ctx, R_ARG1, frame_t_get_const(slot_2)));
  33                 g(gen_upcall_argument(ctx, 1));
  34                 g(gen_frame_address(ctx, slot_r, 0, R_ARG2));
  35                 g(gen_upcall_argument(ctx, 2));
  36                 g(gen_get_upcall_pointer(ctx, upcall, R_ARG3));
  37                 g(gen_upcall_argument(ctx, 3));
  38                 x_offs = offsetof(struct cg_upcall_vector_s, INT_binary_const_int8_t) + op_size * sizeof(void (*)(void));
  39                 g(gen_upcall(ctx, x_offs, 4));
  40         } else if (slot_2 != NO_FRAME_T) {
  41                 g(gen_frame_address(ctx, slot_2, 0, R_ARG1));
  42                 g(gen_upcall_argument(ctx, 1));
  43                 g(gen_frame_address(ctx, slot_r, 0, R_ARG2));
  44                 g(gen_upcall_argument(ctx, 2));
  45                 g(gen_upcall(ctx, upcall, 3));
  46         } else {
  47                 g(gen_frame_address(ctx, slot_r, 0, R_ARG1));
  48                 g(gen_upcall_argument(ctx, 1));
  49                 g(gen_upcall(ctx, upcall, 2));
  50         }
  51         if (slot_is_register(ctx, slot_r))
  52                 g(unspill(ctx, slot_r));
  53         if (label_ovf)
  54                 g(gen_jmp_on_zero(ctx, OP_SIZE_1, R_RET0, COND_E, label_ovf));
  55         return true;
  56 }
  57
  58 static bool attr_w gen_alu_typed_upcall(struct codegen_context *ctx, size_t upcall, unsigned op_size, frame_t slot_1, frame_t slot_2, frame_t slot_r, uint32_t label_ovf)
  59 {
  60         upcall += op_size * sizeof(void (*)(void));
  61         return gen_alu_upcall(ctx, upcall, op_size, slot_1, slot_2, slot_r, label_ovf);
  62 }
  63
  64 #define MODE_FIXED              0
  65 #define MODE_INT                1
  66 #define MODE_REAL               2
  67 #define MODE_BOOL               3
  68 #define MODE_ARRAY_LEN_GT       4
  69
  70 static bool attr_w gen_alu(struct codegen_context *ctx, unsigned mode, unsigned op_size, unsigned op, uint32_t label_ovf, frame_t slot_1, frame_t slot_2, frame_t slot_r)
  71 {
  72         unsigned alu;
  73         bool sgn, mod;
  74         unsigned reg1, reg2, reg3, target;
  75         switch (mode) {
  76                 case MODE_FIXED: switch (op) {
  77                         case OPCODE_FIXED_OP_add:               alu = ALU_ADD; goto do_alu;
  78                         case OPCODE_FIXED_OP_subtract:          alu = ALU_SUB; goto do_alu;
  79                         case OPCODE_FIXED_OP_multiply:          goto do_multiply;
  80                         case OPCODE_FIXED_OP_divide:
  81                         case OPCODE_FIXED_OP_divide_alt1:       sgn = true; mod = false; goto do_divide;
  82                         case OPCODE_FIXED_OP_udivide:
  83                         case OPCODE_FIXED_OP_udivide_alt1:      sgn = false; mod = false; goto do_divide;
  84                         case OPCODE_FIXED_OP_modulo:
  85                         case OPCODE_FIXED_OP_modulo_alt1:       sgn = true; mod = true; goto do_divide;
  86                         case OPCODE_FIXED_OP_umodulo:
  87                         case OPCODE_FIXED_OP_umodulo_alt1:      sgn = false; mod = true; goto do_divide;
  88                         case OPCODE_FIXED_OP_power:             return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_binary_power_int8_t), op_size, slot_1, slot_2, slot_r, 0);
  89                         case OPCODE_FIXED_OP_and:               alu = ALU_AND; goto do_alu;
  90                         case OPCODE_FIXED_OP_or:                alu = ALU_OR; goto do_alu;
  91                         case OPCODE_FIXED_OP_xor:               alu = ALU_XOR; goto do_alu;
  92                         case OPCODE_FIXED_OP_shl:               alu = ROT_SHL; goto do_shift;
  93                         case OPCODE_FIXED_OP_shr:               alu = ROT_SAR; goto do_shift;
  94                         case OPCODE_FIXED_OP_ushr:              alu = ROT_SHR; goto do_shift;
  95                         case OPCODE_FIXED_OP_rol:               alu = ROT_ROL; goto do_shift;
  96                         case OPCODE_FIXED_OP_ror:               alu = ROT_ROR; goto do_shift;
  97                         case OPCODE_FIXED_OP_bts:               alu = BTX_BTS; goto do_bt;
  98                         case OPCODE_FIXED_OP_btr:               alu = BTX_BTR; goto do_bt;
  99                         case OPCODE_FIXED_OP_btc:               alu = BTX_BTC; goto do_bt;
 100                         case OPCODE_FIXED_OP_equal:             alu = COND_E; goto do_compare;
 101                         case OPCODE_FIXED_OP_not_equal:         alu = COND_NE; goto do_compare;
 102                         case OPCODE_FIXED_OP_less:              alu = COND_L; goto do_compare;
 103                         case OPCODE_FIXED_OP_less_equal:        alu = COND_LE; goto do_compare;
 104                         case OPCODE_FIXED_OP_greater:           alu = COND_G; goto do_compare;
 105                         case OPCODE_FIXED_OP_greater_equal:     alu = COND_GE; goto do_compare;
 106                         case OPCODE_FIXED_OP_uless:             alu = COND_B; goto do_compare;
 107                         case OPCODE_FIXED_OP_uless_equal:       alu = COND_BE; goto do_compare;
 108                         case OPCODE_FIXED_OP_ugreater:          alu = COND_A; goto do_compare;
 109                         case OPCODE_FIXED_OP_ugreater_equal:    alu = COND_AE; goto do_compare;
 110                         case OPCODE_FIXED_OP_bt:                alu = BTX_BT; goto do_bt;
 111                         default:                                internal(file_line, "gen_alu: unsupported fixed operation %u", op);
 112                 }
 113                 case MODE_INT: switch (op) {
 114                         case OPCODE_INT_OP_add:                 alu = ALU_ADD; goto do_alu;
 115                         case OPCODE_INT_OP_subtract:            alu = ALU_SUB; goto do_alu;
 116                         case OPCODE_INT_OP_multiply:            goto do_multiply;
 117                         case OPCODE_INT_OP_divide:
 118                         case OPCODE_INT_OP_divide_alt1:         sgn = true; mod = false; goto do_divide;
 119                         case OPCODE_INT_OP_modulo:
 120                         case OPCODE_INT_OP_modulo_alt1:         sgn = true; mod = true; goto do_divide;
 121                         case OPCODE_INT_OP_power:               return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, INT_binary_power_int8_t), op_size, slot_1, slot_2, slot_r, label_ovf);
 122                         case OPCODE_INT_OP_and:                 alu = ALU_AND; mode = MODE_FIXED; goto do_alu;
 123                         case OPCODE_INT_OP_or:                  alu = ALU_OR; mode = MODE_FIXED; goto do_alu;
 124                         case OPCODE_INT_OP_xor:                 alu = ALU_XOR; mode = MODE_FIXED; goto do_alu;
 125                         case OPCODE_INT_OP_shl:                 alu = ROT_SHL; goto do_shift;
 126                         case OPCODE_INT_OP_shr:                 alu = ROT_SAR; goto do_shift;
 127                         case OPCODE_INT_OP_bts:                 alu = BTX_BTS; goto do_bt;
 128                         case OPCODE_INT_OP_btr:                 alu = BTX_BTR; goto do_bt;
 129                         case OPCODE_INT_OP_btc:                 alu = BTX_BTC; goto do_bt;
 130                         case OPCODE_INT_OP_equal:               alu = COND_E; goto do_compare;
 131                         case OPCODE_INT_OP_not_equal:           alu = COND_NE; goto do_compare;
 132                         case OPCODE_INT_OP_less:                alu = COND_L; goto do_compare;
 133                         case OPCODE_INT_OP_less_equal:          alu = COND_LE; goto do_compare;
 134                         case OPCODE_INT_OP_greater:             alu = COND_G; goto do_compare;
 135                         case OPCODE_INT_OP_greater_equal:       alu = COND_GE; goto do_compare;
 136                         case OPCODE_INT_OP_bt:                  alu = BTX_BT; goto do_bt;
 137                         default:                                internal(file_line, "gen_alu: unsupported int operation %u", op);
 138                 }
 139                 case MODE_BOOL: switch (op) {
 140                         case OPCODE_BOOL_OP_and:                alu = ALU_AND; mode = MODE_FIXED; goto do_alu;
 141                         case OPCODE_BOOL_OP_or:                 alu = ALU_OR; mode = MODE_FIXED; goto do_alu;
 142                         case OPCODE_BOOL_OP_equal:              alu = COND_E; goto do_compare;
 143                         case OPCODE_BOOL_OP_not_equal:          alu = ALU_XOR; mode = MODE_FIXED; goto do_alu;
 144                         case OPCODE_BOOL_OP_less:               alu = COND_L; goto do_compare;
 145                         case OPCODE_BOOL_OP_less_equal:         alu = COND_LE; goto do_compare;
 146                         case OPCODE_BOOL_OP_greater:            alu = COND_G; goto do_compare;
 147                         case OPCODE_BOOL_OP_greater_equal:      alu = COND_GE; goto do_compare;
 148                         default:                                internal(file_line, "gen_alu: unsupported bool operation %u", op);
 149                 }
 150         }
 151         internal(file_line, "gen_alu: unsupported mode %u", mode);
 152
 153         /*******
 154          * ALU *
 155          *******/
 156 do_alu: {
 157                 size_t attr_unused offset;
 158                 uint8_t attr_unused long_imm;
 159                 unsigned first_flags;
 160                 unsigned second_flags;
 161                 unsigned second_alu;
 162                 unsigned attr_unused op_size_flags;
 163                 bool c;
 164                 if (unlikely(op_size > OP_SIZE_NATIVE)) {
 165 #if !defined(ARCH_X86) && !defined(ARCH_ARM) && !defined(ARCH_PARISC) && !defined(ARCH_POWER) && !defined(ARCH_SPARC32)
 166                         if (mode == MODE_FIXED) {
 167                                 if (alu == ALU_ADD) {
 168                                         g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_add_,TYPE_INT_MAX)), op_size, slot_1, slot_2, slot_r, 0));
 169                                         return true;
 170                                 } else if (alu == ALU_SUB) {
 171                                         g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_subtract_,TYPE_INT_MAX)), op_size, slot_1, slot_2, slot_r, 0));
 172                                         return true;
 173                                 }
 174                         } else if (mode == MODE_INT) {
 175                                 if (alu == ALU_ADD) {
 176                                         g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(INT_binary_add_,TYPE_INT_MAX)), op_size, slot_1, slot_2, slot_r, label_ovf));
 177                                         return true;
 178                                 } else if (alu == ALU_SUB) {
 179                                         g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(INT_binary_subtract_,TYPE_INT_MAX)), op_size, slot_1, slot_2, slot_r, label_ovf));
 180                                         return true;
 181                                 }
 182                         }
 183 #endif
 184                         first_flags = alu == ALU_ADD || alu == ALU_SUB ? 2 : 0;
 185                         second_flags = mode == MODE_INT ? 1 : 0;
 186                         second_alu = alu == ALU_ADD ? ALU_ADC : alu == ALU_SUB ? ALU_SBB : alu;
 187                         g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
 188 #if defined(ARCH_X86)
 189                         g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, alu, first_flags, slot_2, lo_word(OP_SIZE_NATIVE), true, R_SCRATCH_1));
 190                         g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, second_alu, second_flags, slot_2, hi_word(OP_SIZE_NATIVE), true, R_SCRATCH_2));
 191 #else
 192                         g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_2, 0, R_SCRATCH_3, R_SCRATCH_4));
 193                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, alu, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_3, first_flags));
 194 #if defined(ARCH_PARISC)
 195                         if (mode == MODE_INT) {
 196                                 gen_insn(INSN_ALU_FLAGS_TRAP, OP_SIZE_NATIVE, second_alu, ALU_WRITES_FLAGS(second_alu, false));
 197                                 gen_one(R_SCRATCH_2);
 198                                 gen_one(R_SCRATCH_2);
 199                                 gen_one(R_SCRATCH_4);
 200                                 gen_four(label_ovf);
 201                         } else
 202 #endif
 203                         {
 204                                 gen_insn(first_flags ? INSN_ALU_FLAGS : INSN_ALU, OP_SIZE_NATIVE, second_alu, second_flags | ALU_WRITES_FLAGS(second_alu, false));
 205                                 gen_one(R_SCRATCH_2);
 206                                 gen_one(R_SCRATCH_2);
 207                                 gen_one(R_SCRATCH_4);
 208                         }
 209 #endif
 210 #if !defined(ARCH_PARISC)
 211                         if (mode == MODE_INT) {
 212                                 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_O, 0);
 213                                 gen_four(label_ovf);
 214                         }
 215 #endif
 216                         g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_1, R_SCRATCH_2));
 217                         return true;
 218                 }
 219
 220                 if ((ARCH_HAS_FLAGS || ARCH_SUPPORTS_TRAPS) && slot_2 == slot_r && slot_1 != slot_2 && alu_is_commutative(alu)) {
 221                         frame_t x = slot_1;
 222                         slot_1 = slot_2;
 223                         slot_2 = x;
 224                 }
 225                 if ((ARCH_HAS_FLAGS || ARCH_SUPPORTS_TRAPS) && slot_1 == slot_r && (slot_1 != slot_2 || mode != MODE_INT) && i_size_cmp(op_size) == op_size + zero
 226 #if defined(ARCH_POWER)
 227                         && op_size == OP_SIZE_NATIVE
 228 #endif
 229                     ) {
 230                         struct cg_exit *ce;
 231                         unsigned undo_alu = alu == ALU_ADD ? ALU_SUB : ALU_ADD;
 232                         if (slot_is_register(ctx, slot_1)) {
 233                                 unsigned reg1 = ctx->registers[slot_1];
 234                                 if (slot_is_register(ctx, slot_2)
 235 #if !defined(ARCH_POWER)
 236                                     || frame_t_is_const(slot_2)
 237 #endif
 238                                     ) {
 239                                         unsigned reg2 = frame_t_is_const(slot_2) ? 0xff /* avoid warning */ : ctx->registers[slot_2];
 240                                         if (mode == MODE_INT && ARCH_SUPPORTS_TRAPS) {
 241                                                 if (frame_t_is_const(slot_2))
 242                                                         g(gen_imm(ctx, frame_t_get_const(slot_2), alu_trap_purpose(alu), i_size(op_size)));
 243                                                 gen_insn(INSN_ALU_TRAP, op_size, alu, ALU_WRITES_FLAGS(alu, frame_t_is_const(slot_2) && is_imm()));
 244                                                 gen_one(reg1);
 245                                                 gen_one(reg1);
 246                                                 if (frame_t_is_const(slot_2))
 247                                                         gen_imm_offset();
 248                                                 else
 249                                                         gen_one(reg2);
 250                                                 if (ARCH_TRAP_BEFORE) {
 251                                                         gen_four(label_ovf);
 252                                                         return true;
 253                                                 } else {
 254                                                         ce = alloc_undo_label(ctx);
 255                                                         if (unlikely(!ce))
 256                                                                 return false;
 257                                                         gen_four(ce->undo_label);
 258                                                         goto do_undo_opcode;
 259                                                 }
 260                                         }
 261                                         if (frame_t_is_const(slot_2))
 262                                                 g(gen_3address_alu_imm(ctx, i_size(op_size), alu, reg1, reg1, frame_t_get_const(slot_2), mode == MODE_INT));
 263                                         else
 264                                                 g(gen_3address_alu(ctx, i_size(op_size), alu, reg1, reg1, reg2, mode == MODE_INT));
 265                                         if (mode == MODE_INT) {
 266                                                 size_t m;
 267                                                 ce = alloc_undo_label(ctx);
 268                                                 if (unlikely(!ce))
 269                                                         return false;
 270                                                 gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
 271                                                 gen_four(ce->undo_label);
 272 do_undo_opcode:
 273                                                 ce->undo_opcode = INSN_ALU + ARCH_PARTIAL_ALU(op_size);
 274                                                 ce->undo_op_size = i_size(op_size);
 275                                                 ce->undo_aux = undo_alu;
 276                                                 ce->undo_writes_flags = ALU_WRITES_FLAGS(undo_alu, frame_t_is_const(slot_2) && is_imm());
 277                                                 m = mark_params(ctx);
 278                                                 gen_one(reg1);
 279                                                 gen_one(reg1);
 280                                                 if (frame_t_is_const(slot_2))
 281                                                         gen_imm_offset();
 282                                                 else
 283                                                         gen_one(reg2);
 284                                                 copy_params(ctx, ce, m);
 285                                         }
 286                                         return true;
 287                                 }
 288 #if defined(ARCH_S390) || defined(ARCH_X86)
 289                                 else if (!frame_t_is_const(slot_2)) {
 290                                         size_t m;
 291                                         int64_t offset = (size_t)slot_2 * slot_size;
 292                                         g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDR_OFFSET, i_size(op_size)));
 293                                         gen_insn(INSN_ALU + ARCH_PARTIAL_ALU(op_size), i_size(op_size), alu, 1);
 294                                         gen_one(reg1);
 295                                         gen_one(reg1);
 296                                         gen_address_offset();
 297                                         if (mode == MODE_INT) {
 298                                                 ce = alloc_undo_label(ctx);
 299                                                 if (unlikely(!ce))
 300                                                         return false;
 301                                                 ce->undo_opcode = INSN_ALU + ARCH_PARTIAL_ALU(op_size);
 302                                                 ce->undo_op_size = i_size(op_size);
 303                                                 ce->undo_aux = undo_alu;
 304                                                 ce->undo_writes_flags = ARCH_HAS_FLAGS;
 305                                                 m = mark_params(ctx);
 306                                                 gen_one(reg1);
 307                                                 gen_one(reg1);
 308                                                 gen_address_offset();
 309                                                 copy_params(ctx, ce, m);
 310                                                 gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
 311                                                 gen_four(ce->undo_label);
 312                                         }
 313                                         return true;
 314                                 }
 315 #endif
 316                         }
 317 #if defined(ARCH_X86)
 318                         else {
 319                                 unsigned reg2;
 320                                 size_t m;
 321                                 int64_t offset = (size_t)slot_1 * slot_size;
 322                                 if (!frame_t_is_const(slot_2))
 323                                         g(gen_frame_get(ctx, op_size, garbage, slot_2, R_SCRATCH_1, &reg2));
 324                                 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDR_OFFSET, i_size(op_size)));
 325                                 if (frame_t_is_const(slot_2))
 326                                         g(gen_imm(ctx, frame_t_get_const(slot_2), alu_purpose(alu), i_size(op_size)));
 327                                 gen_insn(INSN_ALU + ARCH_PARTIAL_ALU(op_size), i_size(op_size), alu, 1);
 328                                 gen_address_offset();
 329                                 gen_address_offset();
 330                                 if (frame_t_is_const(slot_2))
 331                                         gen_imm_offset();
 332                                 else
 333                                         gen_one(reg2);
 334                                 if (mode == MODE_INT) {
 335                                         ce = alloc_undo_label(ctx);
 336                                         if (unlikely(!ce))
 337                                                 return false;
 338                                         ce->undo_opcode = INSN_ALU + ARCH_PARTIAL_ALU(op_size);
 339                                         ce->undo_op_size = i_size(op_size);
 340                                         ce->undo_aux = undo_alu;
 341                                         ce->undo_writes_flags = ARCH_HAS_FLAGS;
 342                                         m = mark_params(ctx);
 343                                         gen_address_offset();
 344                                         gen_address_offset();
 345                                         if (frame_t_is_const(slot_2))
 346                                                 gen_imm_offset();
 347                                         else
 348                                                 gen_one(reg2);
 349                                         copy_params(ctx, ce, m);
 350                                         gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
 351                                         gen_four(ce->undo_label);
 352                                 }
 353                                 return true;
 354                         }
 355 #endif
 356                 }
 357
 358 #if defined(ARCH_X86)
 359                 if (1)
 360 #elif defined(ARCH_S390)
 361                 if (op_size >= OP_SIZE_4)
 362 #elif ARCH_HAS_FLAGS && !defined(ARCH_POWER)
 363                 if (op_size == i_size(op_size) + (unsigned)zero && frame_t_is_const(slot_2))
 364 #else
 365                 if (mode != MODE_INT && op_size == i_size(op_size) + (unsigned)zero && frame_t_is_const(slot_2))
 366 #endif
 367                 {
 368                         if (mode == MODE_INT) {
 369                                 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
 370                         } else {
 371                                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, slot_2, R_SCRATCH_1);
 372                         }
 373                         g(gen_frame_load(ctx, op_size, garbage, slot_1, 0, false, target));
 374                         g(gen_frame_load_op(ctx, op_size, garbage, alu, mode == MODE_INT, slot_2, 0, false, target));
 375                         goto check_ovf_store;
 376                 }
 377                 op_size_flags = !ARCH_HAS_FLAGS && !ARCH_SUPPORTS_TRAPS ? OP_SIZE_NATIVE : OP_SIZE_4;
 378 #if defined(ARCH_POWER)
 379                 op_size_flags = OP_SIZE_NATIVE;
 380 #endif
 381                 g(gen_frame_get(ctx, op_size, mode == MODE_INT && (op_size < op_size_flags || ARCH_SUPPORTS_TRAPS) ? sign_x : garbage, slot_1, R_SCRATCH_1, &reg1));
 382                 if (frame_t_is_const(slot_2)
 383 #if defined(ARCH_POWER)
 384                         && 0
 385 #endif
 386                         ) {
 387                         reg2 = 0xff;
 388                         c = true;
 389                 } else {
 390                         g(gen_frame_get(ctx, op_size, mode == MODE_INT && (op_size < op_size_flags || ARCH_SUPPORTS_TRAPS) ? sign_x : garbage, slot_2, R_SCRATCH_2, &reg2));
 391                         c = false;
 392                 }
 393 #if !ARCH_HAS_FLAGS
 394                 if (mode == MODE_INT && op_size >= OP_SIZE_4) {
 395                         if (ARCH_SUPPORTS_TRAPS) {
 396                                 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
 397                                 if (c)
 398                                         g(gen_imm(ctx, frame_t_get_const(slot_2), alu_trap_purpose(alu), op_size));
 399                                 gen_insn(INSN_ALU_TRAP, op_size, alu, ALU_WRITES_FLAGS(alu, c && is_imm()));
 400                                 gen_one(target);
 401                                 gen_one(reg1);
 402                                 if (c)
 403                                         gen_imm_offset();
 404                                 else
 405                                         gen_one(reg2);
 406                                 gen_four(label_ovf);
 407                                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
 408                                 return true;
 409                         }
 410                         if (op_size >= OP_SIZE_NATIVE) {
 411                                 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_3);
 412                                 if (c)
 413                                         g(gen_3address_alu_imm(ctx, i_size(op_size), alu, target, reg1, frame_t_get_const(slot_2), 0));
 414                                 else
 415                                         g(gen_3address_alu(ctx, i_size(op_size), alu, target, reg1, reg2, 0));
 416 #if defined(ARCH_IA64)
 417                                 if (c) {
 418                                         g(gen_3address_alu_imm(ctx, i_size(op_size), ALU_XOR, R_SCRATCH_1, reg1, frame_t_get_const(slot_2), 0));
 419                                         g(gen_3address_alu_imm(ctx, i_size(op_size), ALU_XOR, R_SCRATCH_2, target, frame_t_get_const(slot_2), 0));
 420                                 } else {
 421                                         g(gen_3address_alu(ctx, i_size(op_size), ALU_XOR, R_SCRATCH_1, reg1, reg2, 0));
 422                                         g(gen_3address_alu(ctx, i_size(op_size), ALU_XOR, R_SCRATCH_2, target, reg2, 0));
 423                                 }
 424                                 if (alu == ALU_ADD) {
 425                                         g(gen_3address_alu(ctx, i_size(op_size), ALU_ANDN, R_SCRATCH_1, R_SCRATCH_2, R_SCRATCH_1, 0));
 426                                 } else {
 427                                         g(gen_3address_alu(ctx, i_size(op_size), ALU_ANDN, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_2, 0));
 428                                 }
 429                                 g(gen_cmp_test_jmp(ctx, INSN_TEST, i_size(op_size), R_SCRATCH_1, R_SCRATCH_1, COND_S, label_ovf));
 430 #else
 431                                 if (c) {
 432                                         g(gen_cmp_test_jmp(ctx, INSN_CMP, i_size(op_size), reg1, target, (frame_t_get_const(slot_2) >= 0) ^ (alu != ALU_ADD) ? COND_G : COND_L, label_ovf));
 433                                 } else {
 434                                         gen_insn(INSN_CMP_DEST_REG, i_size(op_size), COND_L, 0);
 435                                         gen_one(R_SCRATCH_1);
 436                                         if (alu == ALU_ADD) {
 437                                                 gen_one(target);
 438                                                 gen_one(reg1);
 439                                         } else {
 440                                                 gen_one(reg1);
 441                                                 gen_one(target);
 442                                         }
 443
 444                                         g(gen_imm(ctx, 0, IMM_PURPOSE_CMP, i_size(op_size)));
 445                                         gen_insn(INSN_CMP_DEST_REG, i_size(op_size), COND_L, 0);
 446                                         gen_one(R_SCRATCH_2);
 447                                         gen_one(reg2);
 448                                         gen_imm_offset();
 449
 450                                         g(gen_cmp_test_jmp(ctx, INSN_CMP, i_size(op_size), R_SCRATCH_1, R_SCRATCH_2, COND_NE, label_ovf));
 451                                 }
 452 #endif
 453                                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
 454                                 return true;
 455                         }
 456                 }
 457 #endif
 458                 if (mode == MODE_INT) {
 459                         target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
 460                 } else if (!ARCH_IS_3ADDRESS(alu, mode == MODE_INT && op_size >= op_size_flags) && !alu_is_commutative(alu)) {
 461                         target = gen_frame_target(ctx, slot_r, NO_FRAME_T, slot_2, R_SCRATCH_1);
 462                 } else {
 463                         target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
 464                 }
 465                 if (c) {
 466                         g(gen_3address_alu_imm(ctx, i_size(op_size), alu, target, reg1, frame_t_get_const(slot_2), mode == MODE_INT && op_size >= op_size_flags));
 467                 } else {
 468                         g(gen_3address_alu(ctx, i_size(op_size), alu, target, reg1, reg2, mode == MODE_INT && op_size >= op_size_flags));
 469                 }
 470
 471                 if (mode == MODE_INT && unlikely(op_size < op_size_flags)) {
 472                         g(gen_cmp_extended(ctx, op_size_flags, op_size, target, R_SCRATCH_2, label_ovf));
 473                 } else
 474 check_ovf_store:
 475                 if (mode == MODE_INT) {
 476                         gen_insn(INSN_JMP_COND, op_size, COND_O, 0);
 477                         gen_four(label_ovf);
 478                 }
 479                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
 480                 return true;
 481         }
 482
 483         /************
 484          * MULTIPLY *
 485          ************/
 486 do_multiply: {
 487                 size_t attr_unused offset;
 488                 uint8_t attr_unused long_imm;
 489                 if (unlikely(op_size > OP_SIZE_NATIVE) || unlikely(!ARCH_HAS_MUL)) {
 490                         if (mode == MODE_INT) {
 491                                 g(gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, INT_binary_multiply_int8_t), op_size, slot_1, slot_2, slot_r, label_ovf));
 492                                 return true;
 493                         }
 494 #if defined(ARCH_X86)
 495                         g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_1, hi_word(OP_SIZE_NATIVE), true, R_CX));
 496                         g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_2, hi_word(OP_SIZE_NATIVE), true, R_AX));
 497                         g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, ALU_MUL, true, slot_2, lo_word(OP_SIZE_NATIVE), true, R_CX));
 498                         g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, ALU_MUL, true, slot_1, lo_word(OP_SIZE_NATIVE), true, R_AX));
 499                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_ADD, R_CX, R_CX, R_AX, 0));
 500                         g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_2, lo_word(OP_SIZE_NATIVE), true, R_AX));
 501
 502                         offset = (size_t)slot_1 * slot_size + lo_word(OP_SIZE_NATIVE);
 503                         g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
 504                         gen_insn(INSN_MUL_L, OP_SIZE_NATIVE, 0, 1);
 505                         gen_one(R_AX);
 506                         gen_one(R_DX);
 507                         gen_one(R_AX);
 508                         gen_address_offset();
 509
 510                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_ADD, R_DX, R_DX, R_CX, 0));
 511
 512                         g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_AX, R_DX));
 513
 514                         return true;
 515 #elif defined(ARCH_ARM32)
 516                         g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
 517                         g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_2, 0, R_SCRATCH_3, R_SCRATCH_4));
 518
 519                         g(gen_mov(ctx, OP_SIZE_NATIVE, R_SCRATCH_NA_1, R_SCRATCH_1));
 520
 521                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_MUL, R_SCRATCH_4, R_SCRATCH_1, R_SCRATCH_4, 0));
 522
 523                         gen_insn(INSN_MADD, OP_SIZE_NATIVE, 0, 0);
 524                         gen_one(R_SCRATCH_2);
 525                         gen_one(R_SCRATCH_3);
 526                         gen_one(R_SCRATCH_2);
 527                         gen_one(R_SCRATCH_4);
 528
 529                         gen_insn(INSN_MUL_L, OP_SIZE_NATIVE, 0, 0);
 530                         gen_one(R_SCRATCH_1);
 531                         gen_one(R_SCRATCH_4);
 532                         gen_one(R_SCRATCH_NA_1);
 533                         gen_one(R_SCRATCH_3);
 534
 535                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_ADD, R_SCRATCH_2, R_SCRATCH_2, R_SCRATCH_4, 0));
 536
 537                         g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_1, R_SCRATCH_2));
 538
 539                         return true;
 540 #elif defined(ARCH_ARM64)
 541                         g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
 542                         g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_2, 0, R_SCRATCH_3, R_SCRATCH_4));
 543
 544                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_UMULH, R_SCRATCH_NA_1, R_SCRATCH_1, R_SCRATCH_3, 0));
 545
 546                         gen_insn(INSN_MADD, OP_SIZE_NATIVE, 0, 0);
 547                         gen_one(R_SCRATCH_NA_1);
 548                         gen_one(R_SCRATCH_2);
 549                         gen_one(R_SCRATCH_3);
 550                         gen_one(R_SCRATCH_NA_1);
 551
 552                         gen_insn(INSN_MADD, OP_SIZE_NATIVE, 0, 0);
 553                         gen_one(R_SCRATCH_2);
 554                         gen_one(R_SCRATCH_1);
 555                         gen_one(R_SCRATCH_4);
 556                         gen_one(R_SCRATCH_NA_1);
 557
 558                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_MUL, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_3, 0));
 559
 560                         g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_1, R_SCRATCH_2));
 561
 562                         return true;
 563 #else
 564                         g(gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_binary_multiply_int8_t), op_size, slot_1, slot_2, slot_r, 0));
 565                         return true;
 566 #endif
 567                 }
 568
 569 #if defined(ARCH_X86)
 570                 if (mode == MODE_INT) {
 571                         if (op_size != OP_SIZE_1 && slot_r == slot_1 && slot_is_register(ctx, slot_1)) {
 572                                 struct cg_exit *ce;
 573                                 target = ctx->registers[slot_1];
 574                                 g(gen_mov(ctx, op_size, R_SCRATCH_1, target));
 575                                 g(gen_frame_load_op(ctx, op_size, garbage, ALU_MUL, mode == MODE_INT, slot_2, 0, false, target));
 576                                 ce = alloc_undo_label(ctx);
 577                                 if (unlikely(!ce))
 578                                         return false;
 579                                 ce->undo_opcode = INSN_MOV;
 580                                 ce->undo_op_size = op_size;
 581                                 ce->undo_aux = 0;
 582                                 ce->undo_writes_flags = 0;
 583                                 ce->undo_parameters[0] = target;
 584                                 ce->undo_parameters[1] = R_SCRATCH_1;
 585                                 ce->undo_parameters_len = 2;
 586                                 gen_insn(INSN_JMP_COND, op_size, COND_O, 0);
 587                                 gen_four(ce->undo_label);
 588                                 return true;
 589                         }
 590                         target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
 591                 } else {
 592                         target = gen_frame_target(ctx, slot_r, NO_FRAME_T, slot_2, R_SCRATCH_1);
 593                 }
 594                 if (op_size == OP_SIZE_1)
 595                         target = R_SCRATCH_1;
 596                 g(gen_frame_load(ctx, op_size, garbage, slot_1, 0, false, target));
 597                 if (op_size == OP_SIZE_1 && frame_t_is_const(slot_2)) {
 598                         g(gen_load_constant(ctx, R_SCRATCH_3, frame_t_get_const(slot_2)));
 599                         gen_insn(INSN_ALU + ARCH_PARTIAL_ALU(op_size), op_size, ALU_MUL, 1);
 600                         gen_one(target);
 601                         gen_one(target);
 602                         gen_one(R_SCRATCH_3);
 603                 } else {
 604                         g(gen_frame_load_op(ctx, op_size, garbage, ALU_MUL, mode == MODE_INT, slot_2, 0, false, target));
 605                 }
 606                 if (mode == MODE_INT) {
 607                         gen_insn(INSN_JMP_COND, op_size, COND_O, 0);
 608                         gen_four(label_ovf);
 609                 }
 610                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
 611                 return true;
 612 #endif
 613 #if defined(ARCH_ALPHA)
 614                 if (mode == MODE_INT && op_size >= OP_SIZE_4 && ARCH_SUPPORTS_TRAPS) {
 615                         target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
 616                         g(gen_frame_get(ctx, op_size, garbage, slot_1, R_SCRATCH_1, &reg1));
 617                         g(gen_frame_get(ctx, op_size, garbage, slot_2, R_SCRATCH_2, &reg2));
 618
 619                         gen_insn(INSN_ALU_TRAP, op_size, ALU_MUL, ALU_WRITES_FLAGS(ALU_MUL, false));
 620                         gen_one(target);
 621                         gen_one(reg1);
 622                         gen_one(reg2);
 623                         gen_four(label_ovf);
 624                         g(gen_frame_store(ctx, op_size, slot_r, 0, target));
 625
 626                         return true;
 627                 }
 628 #endif
 629 #if defined(ARCH_ARM32)
 630                 if (mode == MODE_INT && op_size == OP_SIZE_4) {
 631                         target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_3);
 632                         g(gen_frame_get(ctx, op_size, garbage, slot_1, R_SCRATCH_1, &reg1));
 633                         g(gen_frame_get(ctx, op_size, garbage, slot_2, R_SCRATCH_2, &reg2));
 634
 635                         gen_insn(INSN_MUL_L, OP_SIZE_NATIVE, 0, 0);
 636                         gen_one(target);
 637                         gen_one(R_SCRATCH_4);
 638                         gen_one(reg1);
 639                         gen_one(reg2);
 640
 641                         gen_insn(INSN_CMP, OP_SIZE_NATIVE, 0, 1);
 642                         gen_one(R_SCRATCH_4);
 643                         gen_one(ARG_SHIFTED_REGISTER);
 644                         gen_one(ARG_SHIFT_ASR | 0x1f);
 645                         gen_one(target);
 646
 647                         gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_NE, 0);
 648                         gen_four(label_ovf);
 649
 650                         g(gen_frame_store(ctx, op_size, slot_r, 0, target));
 651
 652                         return true;
 653                 }
 654 #endif
 655 #if defined(ARCH_ARM64)
 656                 if (mode == MODE_INT && op_size == OP_SIZE_4) {
 657                         target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
 658                         g(gen_frame_get(ctx, op_size, op_size < OP_SIZE_4 ? sign_x : garbage, slot_1, R_SCRATCH_1, &reg1));
 659                         g(gen_frame_get(ctx, op_size, op_size < OP_SIZE_4 ? sign_x : garbage, slot_2, R_SCRATCH_2, &reg2));
 660                         gen_insn(INSN_ALU, OP_SIZE_8, ALU_MUL, ALU_WRITES_FLAGS(ALU_MUL, false));
 661                         gen_one(target);
 662                         gen_one(ARG_EXTENDED_REGISTER);
 663                         gen_one(ARG_EXTEND_SXTW);
 664                         gen_one(reg1);
 665                         gen_one(ARG_EXTENDED_REGISTER);
 666                         gen_one(ARG_EXTEND_SXTW);
 667                         gen_one(reg2);
 668
 669                         gen_insn(INSN_CMP, OP_SIZE_8, 0, 1);
 670                         gen_one(target);
 671                         gen_one(ARG_EXTENDED_REGISTER);
 672                         gen_one(ARG_EXTEND_SXTW);
 673                         gen_one(target);
 674
 675                         gen_insn(INSN_JMP_COND, OP_SIZE_8, COND_NE, 0);
 676                         gen_four(label_ovf);
 677
 678                         g(gen_frame_store(ctx, op_size, slot_r, 0, target));
 679
 680                         return true;
 681                 }
 682                 if (mode == MODE_INT && op_size == OP_SIZE_8) {
 683                         target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
 684                         g(gen_frame_get(ctx, op_size, garbage, slot_1, R_SCRATCH_1, &reg1));
 685                         g(gen_frame_get(ctx, op_size, garbage, slot_2, R_SCRATCH_2, &reg2));
 686                         g(gen_3address_alu(ctx, OP_SIZE_8, ALU_SMULH, R_SCRATCH_3, reg1, reg2, 0));
 687
 688                         g(gen_3address_alu(ctx, OP_SIZE_8, ALU_MUL, target, reg1, reg2, 0));
 689
 690                         gen_insn(INSN_CMP, OP_SIZE_8, 0, 1);
 691                         gen_one(R_SCRATCH_3);
 692                         gen_one(ARG_SHIFTED_REGISTER);
 693                         gen_one(ARG_SHIFT_ASR | 0x3f);
 694                         gen_one(target);
 695
 696                         gen_insn(INSN_JMP_COND, OP_SIZE_8, COND_NE, 0);
 697                         gen_four(label_ovf);
 698
 699                         g(gen_frame_store(ctx, op_size, slot_r, 0, target));
 700
 701                         return true;
 702                 }
 703 #endif
 704 #if defined(ARCH_POWER)
 705                 if (mode == MODE_INT && op_size >= OP_SIZE_4) {
 706                         target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
 707                         g(gen_frame_get(ctx, op_size, garbage, slot_1, R_SCRATCH_1, &reg1));
 708                         g(gen_frame_get(ctx, op_size, garbage, slot_2, R_SCRATCH_2, &reg2));
 709
 710                         g(gen_3address_alu(ctx, op_size, ALU_MUL, target, reg1, reg2, 1));
 711
 712                         gen_insn(INSN_JMP_COND, op_size, COND_O, 0);
 713                         gen_four(label_ovf);
 714
 715                         g(gen_frame_store(ctx, op_size, slot_r, 0, target));
 716
 717                         return true;
 718                 }
 719 #endif
 720 #if defined(ARCH_LOONGARCH64) || (defined(ARCH_MIPS) && MIPS_R6) || defined(ARCH_RISCV64)
 721                 if (mode == MODE_INT && op_size == OP_SIZE_NATIVE) {
 722                         target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
 723                         g(gen_frame_get(ctx, op_size, garbage, slot_1, R_SCRATCH_1, &reg1));
 724                         g(gen_frame_get(ctx, op_size, garbage, slot_2, R_SCRATCH_2, &reg2));
 725
 726                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_SMULH, R_SCRATCH_3, reg1, reg2, 0));
 727
 728                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_MUL, target, reg1, reg2, 0));
 729
 730                         g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_4, target, (8U << OP_SIZE_NATIVE) - 1, 0));
 731
 732                         g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, R_SCRATCH_3, R_SCRATCH_4, COND_NE, label_ovf));
 733
 734                         g(gen_frame_store(ctx, OP_SIZE_NATIVE, slot_r, 0, target));
 735
 736                         return true;
 737                 }
 738 #endif
 739 #if defined(ARCH_S390)
 740                 if (mode == MODE_INT && op_size >= OP_SIZE_4 && likely(cpu_test_feature(CPU_FEATURE_misc_insn_ext_2))) {
 741                         target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
 742                         g(gen_frame_load(ctx, op_size, sign_x, slot_1, 0, false, target));
 743                         g(gen_frame_load_op(ctx, op_size, sign_x, ALU_MUL, 1, slot_2, 0, false, target));
 744
 745                         gen_insn(INSN_JMP_COND, op_size, COND_O, 0);
 746                         gen_four(label_ovf);
 747
 748                         g(gen_frame_store(ctx, op_size, slot_r, 0, target));
 749                         return true;
 750                 }
 751 #endif
 752 #if (defined(ARCH_MIPS) && !MIPS_R6) || defined(ARCH_S390)
 753 #if defined(ARCH_MIPS)
 754                 if (mode == MODE_INT && op_size >= OP_SIZE_4)
 755 #endif
 756 #if defined(ARCH_S390)
 757                 if (mode == MODE_INT && op_size == OP_SIZE_4)
 758 #endif
 759                 {
 760 #if defined(ARCH_S390)
 761                         target = R_SCRATCH_1;
 762 #else
 763                         target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
 764 #endif
 765                         g(gen_frame_get(ctx, op_size, sign_x, slot_1, R_SCRATCH_1, &reg1));
 766                         g(gen_frame_get(ctx, op_size, sign_x, slot_2, R_SCRATCH_3, &reg2));
 767
 768                         gen_insn(INSN_MUL_L, op_size, 0, 0);
 769                         gen_one(target);
 770                         gen_one(R_SCRATCH_2);
 771                         gen_one(reg1);
 772                         gen_one(reg2);
 773
 774                         g(gen_3address_rot_imm(ctx, op_size, ROT_SAR, R_SCRATCH_4, target, (8U << op_size) - 1, false));
 775
 776                         g(gen_cmp_test_jmp(ctx, INSN_CMP, op_size, R_SCRATCH_2, R_SCRATCH_4, COND_NE, label_ovf));
 777
 778                         g(gen_frame_store(ctx, op_size, slot_r, 0, target));
 779                         return true;
 780                 }
 781 #endif
 782                 if (mode == MODE_INT && op_size == OP_SIZE_NATIVE) {
 783                         g(gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, INT_binary_multiply_int8_t), op_size, slot_1, slot_2, slot_r, label_ovf));
 784                         return true;
 785                 }
 786
 787                 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
 788                 if (op_size < OP_SIZE_NATIVE && mode == MODE_INT) {
 789                         g(gen_frame_get(ctx, op_size, sign_x, slot_1, R_SCRATCH_1, &reg1));
 790                         g(gen_frame_get(ctx, op_size, sign_x, slot_2, R_SCRATCH_2, &reg2));
 791
 792                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_MUL, target, reg1, reg2, 0));
 793                 } else {
 794                         g(gen_frame_load(ctx, op_size, sign_x, slot_1, 0, false, target));
 795                         g(gen_frame_load_op(ctx, op_size, sign_x, ALU_MUL, 0, slot_2, 0, false, target));
 796                 }
 797
 798                 if (mode == MODE_INT) {
 799                         g(gen_cmp_extended(ctx, OP_SIZE_NATIVE, op_size, target, R_SCRATCH_2, label_ovf));
 800                 }
 801
 802                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
 803
 804                 return true;
 805         }
 806
 807         /**********
 808          * DIVIDE *
 809          **********/
 810 do_divide: {
 811                 uint32_t attr_unused label_skip = 0;    /* avoid warning */
 812                 uint32_t attr_unused label_skip2 = 0;   /* avoid warning */
 813                 uint32_t attr_unused label_end = 0;     /* avoid warning */
 814                 uint32_t attr_unused label_div_0 = 0;   /* avoid warning */
 815                 unsigned attr_unused divide_alu = 0;    /* avoid warning */
 816                 bool attr_unused have_mod = false;
 817                 bool attr_unused force_sx = false;
 818                 unsigned attr_unused div_op_size = i_size(op_size);
 819                 if (unlikely(op_size > OP_SIZE_NATIVE) || unlikely(!ARCH_HAS_DIV)
 820 #if defined(ARCH_S390)
 821                         || !(Z || (op_size <= OP_SIZE_4 && sgn))
 822 #endif
 823                    ) {
 824                         size_t upcall;
 825                         if (mode == MODE_INT) {
 826                                 upcall = !mod ? offsetof(struct cg_upcall_vector_s, INT_binary_divide_int8_t) : offsetof(struct cg_upcall_vector_s, INT_binary_modulo_int8_t);
 827                         } else if (sgn) {
 828                                 upcall = !mod ? offsetof(struct cg_upcall_vector_s, FIXED_binary_divide_int8_t) : offsetof(struct cg_upcall_vector_s, FIXED_binary_modulo_int8_t);
 829                         } else {
 830                                 upcall = !mod ? offsetof(struct cg_upcall_vector_s, FIXED_binary_udivide_int8_t) : offsetof(struct cg_upcall_vector_s, FIXED_binary_umodulo_int8_t);
 831                         }
 832                         g(gen_alu_typed_upcall(ctx, upcall, op_size, slot_1, slot_2, slot_r, mode == MODE_INT ? label_ovf : 0));
 833                         return true;
 834                 }
 835 #if defined(ARCH_X86) || defined(ARCH_S390)
 836                 if (mode == MODE_FIXED) {
 837                         label_skip = alloc_label(ctx);
 838                         if (unlikely(!label_skip))
 839                                 return false;
 840                         label_end = alloc_label(ctx);
 841                         if (unlikely(!label_end))
 842                                 return false;
 843                         if (sgn) {
 844                                 label_skip2 = alloc_label(ctx);
 845                                 if (unlikely(!label_skip2))
 846                                         return false;
 847                         }
 848                 }
 849 #if defined(ARCH_X86)
 850                 if (R_SCRATCH_1 != R_AX || R_SCRATCH_2 != R_DX || R_SCRATCH_3 != R_CX)
 851                         internal(file_line, "gen_alu: bad scratch registers");
 852 #endif
 853                 g(gen_frame_load(ctx, op_size, sgn ? sign_x : zero_x, slot_1, 0, false, R_SCRATCH_1));
 854                 g(gen_frame_load(ctx, op_size, sgn ? sign_x : zero_x, slot_2, 0, false, R_SCRATCH_3));
 855
 856                 g(gen_jmp_on_zero(ctx, i_size(op_size), R_SCRATCH_3, COND_E, mode == MODE_INT ? label_ovf : label_skip));
 857
 858                 if (sgn) {
 859                         uint64_t val;
 860                         uint32_t label_not_minus_1;
 861                         label_not_minus_1 = alloc_label(ctx);
 862                         if (unlikely(!label_not_minus_1))
 863                                 return false;
 864
 865                         g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, op_size, R_SCRATCH_3, -1, COND_NE, label_not_minus_1));
 866
 867                         val = -(uint64_t)0x80 << (((1 << op_size) - 1) * 8);
 868                         g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, op_size, R_SCRATCH_1, val, COND_E, mode == MODE_INT ? label_ovf : label_skip2));
 869
 870                         gen_label(label_not_minus_1);
 871                 }
 872
 873 #if defined(ARCH_X86)
 874                 if (op_size >= OP_SIZE_2) {
 875                         if (sgn) {
 876                                 gen_insn(INSN_CWD + ARCH_PARTIAL_ALU(op_size), op_size, 0, 0);
 877                                 gen_one(R_SCRATCH_2);
 878                                 gen_one(R_SCRATCH_1);
 879                                 if (op_size == OP_SIZE_2)
 880                                         gen_one(R_SCRATCH_2);
 881                         } else {
 882                                 g(gen_3address_alu(ctx, OP_SIZE_4, ALU_XOR, R_SCRATCH_2, R_SCRATCH_2, R_SCRATCH_2, 0));
 883                         }
 884                 }
 885                 gen_insn(INSN_DIV_L, op_size, sgn, 1);
 886                 gen_one(R_SCRATCH_1);
 887                 gen_one(i_size(op_size) == OP_SIZE_1 ? R_SCRATCH_1 : R_SCRATCH_2);
 888                 gen_one(R_SCRATCH_1);
 889                 gen_one(i_size(op_size) == OP_SIZE_1 ? R_SCRATCH_1 : R_SCRATCH_2);
 890                 gen_one(R_SCRATCH_3);
 891 #else
 892                 if (!sgn) {
 893                         g(gen_load_constant(ctx, R_SCRATCH_2, 0));
 894                 } else if (op_size <= OP_SIZE_4) {
 895                         g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_2, R_SCRATCH_1, (1U << (OP_SIZE_NATIVE + 3)) - 1, false));
 896                 }
 897                 gen_insn(INSN_DIV_L, i_size(op_size), sgn, 1);
 898                 gen_one(R_SCRATCH_2);
 899                 gen_one(R_SCRATCH_1);
 900                 gen_one(R_SCRATCH_2);
 901                 gen_one(R_SCRATCH_1);
 902                 gen_one(R_SCRATCH_3);
 903 #endif
 904                 if (mod && i_size(op_size) == OP_SIZE_1) {
 905                         g(gen_3address_rot_imm(ctx, OP_SIZE_2, ROT_SHR, R_SCRATCH_1, R_SCRATCH_1, 8, 0));
 906                         g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
 907                 } else if (mod) {
 908                         g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_2));
 909                 } else {
 910                         g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
 911                 }
 912                 if (mode == MODE_FIXED) {
 913                         gen_insn(INSN_JMP, 0, 0, 0);
 914                         gen_four(label_end);
 915
 916                         if (sgn) {
 917                                 gen_label(label_skip2);
 918
 919                                 if (mod)
 920                                         g(gen_frame_clear(ctx, op_size, slot_r));
 921                                 else
 922                                         g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
 923
 924                                 gen_insn(INSN_JMP, 0, 0, 0);
 925                                 gen_four(label_end);
 926                         }
 927
 928                         gen_label(label_skip);
 929                         if (!mod)
 930                                 g(gen_frame_clear(ctx, op_size, slot_r));
 931                         else
 932                                 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
 933                         gen_label(label_end);
 934                 }
 935                 return true;
 936 #else
 937 #if defined(ARCH_MIPS)
 938                 have_mod = true;
 939                 div_op_size = maximum(op_size, OP_SIZE_4);
 940                 if (op_size == OP_SIZE_4)
 941                         force_sx = true;
 942 #endif
 943 #if defined(ARCH_POWER)
 944                 have_mod = cpu_test_feature(CPU_FEATURE_v30);
 945                 div_op_size = maximum(op_size, OP_SIZE_4);
 946 #endif
 947 #if defined(ARCH_LOONGARCH64) || defined(ARCH_RISCV64)
 948                 have_mod = true;
 949                 div_op_size = maximum(op_size, OP_SIZE_4);
 950 #endif
 951                 label_end = alloc_label(ctx);
 952                 if (unlikely(!label_end))
 953                         return false;
 954
 955                 g(gen_frame_get(ctx, op_size, (sgn && op_size < i_size(op_size)) || force_sx ? sign_x : zero_x, slot_1, R_SCRATCH_1, &reg1));
 956                 g(gen_frame_get(ctx, op_size, (sgn && op_size < i_size(op_size)) || force_sx ? sign_x : zero_x, slot_2, R_SCRATCH_2, &reg2));
 957                 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_3);
 958
 959                 if (ARCH_PREFERS_SX(op_size) && !sgn && op_size < i_size(op_size)) {
 960                         g(gen_extend(ctx, op_size, zero_x, R_SCRATCH_1, reg1));
 961                         reg1 = R_SCRATCH_1;
 962                         g(gen_extend(ctx, op_size, zero_x, R_SCRATCH_2, reg2));
 963                         reg2 = R_SCRATCH_2;
 964                 }
 965
 966                 if (mode == MODE_INT) {
 967                         g(gen_jmp_on_zero(ctx, i_size(op_size), reg2, COND_E, label_ovf));
 968                         if (sgn) {
 969                                 uint64_t val;
 970                                 uint32_t label_not_minus_1;
 971                                 label_not_minus_1 = alloc_label(ctx);
 972                                 if (unlikely(!label_not_minus_1))
 973                                         return false;
 974
 975                                 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, i_size(op_size), reg2, -1, COND_NE, label_not_minus_1));
 976
 977                                 val = 0xFFFFFFFFFFFFFF80ULL << (((1 << op_size) - 1) * 8);
 978                                 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, i_size(op_size), reg1, val, COND_E, label_ovf));
 979
 980                                 gen_label(label_not_minus_1);
 981                         }
 982                 } else {
 983 #if !(defined(ARCH_ARM) && ARM_ASM_DIV_NO_TRAP)
 984                         if (!mod) {
 985                                 g(gen_load_constant(ctx, target, 0));
 986                         } else {
 987                                 g(gen_mov(ctx, OP_SIZE_NATIVE, target, reg1));
 988                         }
 989                         g(gen_jmp_on_zero(ctx, i_size(op_size), reg2, COND_E, label_end));
 990                         if (sgn) {
 991                                 uint64_t val;
 992                                 uint32_t label_not_minus_1;
 993                                 label_not_minus_1 = alloc_label(ctx);
 994                                 if (unlikely(!label_not_minus_1))
 995                                         return false;
 996
 997                                 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, i_size(op_size), reg2, -1, COND_NE, label_not_minus_1));
 998
 999                                 if (!mod) {
1000                                         g(gen_mov(ctx, OP_SIZE_NATIVE, target, reg1));
1001                                 } else {
1002                                         g(gen_load_constant(ctx, target, 0));
1003                                 }
1004
1005                                 val = 0xFFFFFFFFFFFFFF80ULL << (((1 << op_size) - 1) * 8);
1006                                 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, i_size(op_size), reg1, val, COND_E, label_end));
1007
1008                                 gen_label(label_not_minus_1);
1009                         }
1010 #endif
1011                 }
1012                 if (mod && have_mod) {
1013                         g(gen_3address_alu(ctx, div_op_size, sgn ? ALU_SREM : ALU_UREM, target, reg1, reg2, 0));
1014                 } else {
1015                         g(gen_3address_alu(ctx, div_op_size, sgn ? ALU_SDIV : ALU_UDIV, target, reg1, reg2, 0));
1016                 }
1017
1018                 if (mod && !have_mod) {
1019 #if defined(ARCH_ARM)
1020                         gen_insn(INSN_MADD, i_size(op_size), 1, 0);
1021                         gen_one(target);
1022                         gen_one(target);
1023                         gen_one(reg2);
1024                         gen_one(reg1);
1025 #else
1026                         g(gen_3address_alu(ctx, i_size(op_size), ALU_MUL, R_SCRATCH_2, reg2, target, 0));
1027                         g(gen_3address_alu(ctx, i_size(op_size), ALU_SUB, target, reg1, R_SCRATCH_2, 0));
1028 #endif
1029                 }
1030
1031                 gen_label(label_end);
1032                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1033                 return true;
1034 #endif
1035         }
1036         /*********
1037          * SHIFT *
1038          *********/
1039 do_shift: {
1040                 bool sx;
1041                 bool must_mask;
1042                 unsigned op_s;
1043                 int64_t cnst = 0;       /* avoid warning */
1044                 bool c = frame_t_is_const(slot_2);
1045                 if (unlikely(op_size > OP_SIZE_NATIVE)) {
1046                         size_t upcall;
1047                         if (mode == MODE_FIXED) {
1048                                 switch (alu) {
1049                                         case ROT_SHL:   upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_shl_,TYPE_INT_MAX));
1050                                                         break;
1051                                         case ROT_SAR:   upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_shr_,TYPE_INT_MAX));
1052                                                         break;
1053                                         case ROT_SHR:   upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_ushr_,TYPE_INT_MAX));
1054                                                         break;
1055                                         case ROT_ROL:   upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_rol_,TYPE_INT_MAX));
1056                                                         break;
1057                                         case ROT_ROR:   upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_ror_,TYPE_INT_MAX));
1058                                                         break;
1059                                         default:        internal(file_line, "do_alu: invalid shift %u", alu);
1060                                 }
1061                         } else {
1062                                 switch (alu) {
1063                                         case ROT_SHL:   upcall = offsetof(struct cg_upcall_vector_s, cat(INT_binary_shl_,TYPE_INT_MAX));
1064                                                         break;
1065                                         case ROT_SAR:   upcall = offsetof(struct cg_upcall_vector_s, cat(INT_binary_shr_,TYPE_INT_MAX));
1066                                                         break;
1067                                         default:        internal(file_line, "do_alu: invalid shift %u", alu);
1068                                 }
1069                         }
1070                         g(gen_alu_upcall(ctx, upcall, op_size, slot_1, slot_2, slot_r, mode == MODE_INT ? label_ovf : 0));
1071                         return true;
1072                 }
1073                 op_s = i_size_rot(op_size);
1074 #if defined(ARCH_X86)
1075                 if (slot_1 == slot_r && !slot_is_register(ctx, slot_1) && !(mode == MODE_INT && alu == ROT_SHL)) {
1076                         int64_t offset = (size_t)slot_1 * slot_size;
1077                         if (c) {
1078                                 cnst = frame_t_get_const(slot_2);
1079                                 if (mode == MODE_INT) {
1080                                         if ((uint64_t)cnst > (8U << op_size) - 1) {
1081                                                 gen_insn(INSN_JMP, 0, 0, 0);
1082                                                 gen_four(label_ovf);
1083                                                 return true;
1084                                         }
1085                                 } else {
1086                                         cnst &= (8U << op_size) - 1;
1087                                 }
1088                                 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDR_OFFSET, op_size));
1089                                 gen_insn(INSN_ROT + ARCH_PARTIAL_ALU(op_size), op_size, alu, 1);
1090                                 gen_address_offset();
1091                                 gen_address_offset();
1092                                 gen_one(ARG_IMM);
1093                                 gen_eight(cnst);
1094                         } else {
1095                                 g(gen_frame_load(ctx, op_size, garbage, slot_2, 0, false, R_SCRATCH_3));
1096                                 if (mode == MODE_INT) {
1097                                         int64_t imm = (8U << op_size) - 1;
1098                                         g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, op_size, R_SCRATCH_3, imm, COND_A, label_ovf));
1099                                 } else if ((alu != ROT_ROL && alu != ROT_ROR) && op_size < OP_SIZE_4) {
1100                                         g(gen_3address_alu_imm(ctx, OP_SIZE_1, ALU_AND, R_SCRATCH_3, R_SCRATCH_3, (8U << op_size) - 1, 0));
1101                                 }
1102                                 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDR_OFFSET, op_size));
1103                                 gen_insn(INSN_ROT + ARCH_PARTIAL_ALU(op_size), op_size, alu, 1);
1104                                 gen_address_offset();
1105                                 gen_address_offset();
1106                                 gen_one(R_SCRATCH_3);
1107                         }
1108                         return true;
1109                 }
1110                 if (mode == MODE_INT && alu == ROT_SHL && op_size < OP_SIZE_NATIVE)
1111                         op_s = op_size + 1;
1112 #endif
1113                 must_mask = op_size < ARCH_SHIFT_SIZE;
1114                 sx = (alu == ROT_SAR && op_size < op_s) || (alu == ROT_SHL && op_size < OP_SIZE_NATIVE && mode == MODE_INT);
1115 #if defined(ARCH_MIPS)
1116                 sx |= op_size == OP_SIZE_4;
1117 #endif
1118                 g(gen_frame_get(ctx, op_size, sx ? sign_x : zero_x, slot_1, R_SCRATCH_1, &reg1));
1119                 if (c) {
1120                         reg3 = 0xff;    /* avoid warning */
1121                         cnst = frame_t_get_const(slot_2);
1122                 } else
1123 #if defined(ARCH_X86)
1124                 if (!ARCH_IS_3ADDRESS_ROT(alu, op_size)) {
1125                         g(gen_frame_load(ctx, op_size, garbage, slot_2, 0, false, R_SCRATCH_3));
1126                         reg3 = R_SCRATCH_3;
1127                 } else
1128 #endif
1129                 g(gen_frame_get(ctx, op_size, garbage, slot_2, R_SCRATCH_3, &reg3));
1130
1131                 if (ARCH_PREFERS_SX(op_size) && !sx && op_size < op_s) {
1132                         g(gen_extend(ctx, op_size, zero_x, R_SCRATCH_1, reg1));
1133                         reg1 = R_SCRATCH_1;
1134                 }
1135
1136                 if (mode == MODE_INT) {
1137                         int64_t imm = (8U << op_size) - 1;
1138                         if (c) {
1139                                 if ((uint64_t)cnst > (uint64_t)imm) {
1140                                         gen_insn(INSN_JMP, 0, 0, 0);
1141                                         gen_four(label_ovf);
1142                                         return true;
1143                                 }
1144                         } else {
1145                                 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, i_size_cmp(op_size), reg3, imm, COND_A, label_ovf));
1146                         }
1147                 } else {
1148 #if defined(ARCH_ARM)
1149                         if (alu == ROT_ROL) {
1150                                 if (c) {
1151                                         cnst = -(uint64_t)cnst;
1152                                 } else {
1153                                         g(gen_2address_alu1(ctx, OP_SIZE_4, ALU1_NEG, R_SCRATCH_3, reg3, 0));
1154                                         reg3 = R_SCRATCH_3;
1155                                 }
1156                                 alu = ROT_ROR;
1157                         }
1158 #endif
1159 #if defined(ARCH_LOONGARCH64)
1160                         if (alu == ROT_ROL && op_size >= OP_SIZE_4) {
1161                                 if (c) {
1162                                         cnst = -(uint64_t)cnst;
1163                                 } else {
1164                                         g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NEG, R_SCRATCH_3, reg3, 0));
1165                                         reg3 = R_SCRATCH_3;
1166                                 }
1167                                 alu = ROT_ROR;
1168                         }
1169 #endif
1170 #if defined(ARCH_MIPS)
1171                         if (MIPS_HAS_ROT && alu == ROT_ROL && op_size >= OP_SIZE_4) {
1172                                 if (c) {
1173                                         cnst = -(uint64_t)cnst;
1174                                 } else {
1175                                         g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NEG, R_SCRATCH_3, reg3, 0));
1176                                         reg3 = R_SCRATCH_3;
1177                                 }
1178                                 alu = ROT_ROR;
1179                         }
1180 #endif
1181 #if defined(ARCH_POWER)
1182                         if (alu == ROT_ROR && op_size >= OP_SIZE_4) {
1183                                 if (c) {
1184                                         cnst = -(uint64_t)cnst;
1185                                 } else {
1186                                         g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NEG, R_SCRATCH_3, reg3, 0));
1187                                         reg3 = R_SCRATCH_3;
1188                                 }
1189                                 alu = ROT_ROL;
1190                         }
1191 #endif
1192 #if defined(ARCH_S390)
1193                         if (Z && alu == ROT_ROR && op_size >= OP_SIZE_4) {
1194                                 if (c) {
1195                                         cnst = -(uint64_t)cnst;
1196                                 } else {
1197                                         g(gen_2address_alu1(ctx, OP_SIZE_4, ALU1_NEG, R_SCRATCH_3, reg3, 0));
1198                                         reg3 = R_SCRATCH_3;
1199                                 }
1200                                 alu = ROT_ROL;
1201                         }
1202 #endif
1203                         if (c) {
1204                                 cnst &= (8U << op_size) - 1;
1205                         } else if (must_mask) {
1206                                 g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_4), ALU_AND, R_SCRATCH_3, reg3, (8U << op_size) - 1, 0));
1207                                 reg3 = R_SCRATCH_3;
1208                         }
1209                 }
1210
1211 #if defined(ARCH_X86)
1212                 if (mode == MODE_INT && alu == ROT_SHL) {
1213                         target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_2);
1214                 } else {
1215                         target = gen_frame_target(ctx, slot_r, NO_FRAME_T, slot_2, R_SCRATCH_2);
1216                 }
1217                 if (c) {
1218                         g(gen_3address_rot_imm(ctx, op_s, alu, target, reg1, cnst, 0));
1219                 } else  {
1220                         g(gen_3address_rot(ctx, op_s, alu, target, reg1, reg3));
1221                 }
1222
1223                 if (mode == MODE_INT && alu == ROT_SHL) {
1224                         if (op_size < OP_SIZE_NATIVE) {
1225                                 gen_insn(INSN_MOVSX, op_size, 0, 0);
1226                                 gen_one(R_SCRATCH_4);
1227                                 gen_one(target);
1228
1229                                 g(gen_cmp_test_jmp(ctx, INSN_CMP, op_s, target, R_SCRATCH_4, COND_NE, label_ovf));
1230                         } else {
1231                                 if (c) {
1232                                         g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_4, target, cnst, 0));
1233                                 } else {
1234                                         g(gen_3address_rot(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_4, target, reg3));
1235                                 }
1236
1237                                 g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, reg1, R_SCRATCH_4, COND_NE, label_ovf));
1238                         }
1239                 }
1240                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1241                 return true;
1242 #endif
1243 #if defined(ARCH_ARM)
1244                 if (op_size <= OP_SIZE_2 && alu == ROT_ROR) {
1245                         gen_insn(INSN_ALU, OP_SIZE_4, ALU_OR, ALU_WRITES_FLAGS(ALU_OR, false));
1246                         gen_one(R_SCRATCH_1);
1247                         gen_one(reg1);
1248                         gen_one(ARG_SHIFTED_REGISTER);
1249                         gen_one(ARG_SHIFT_LSL | (8U << op_size));
1250                         gen_one(reg1);
1251                         if (op_size == OP_SIZE_1)
1252                                 alu = ROT_SHR;
1253                         reg1 = R_SCRATCH_1;
1254                 }
1255                 goto do_generic_shift;
1256 #endif
1257 #if defined(ARCH_LOONGARCH64)
1258                 if (alu == ROT_ROR && op_size >= OP_SIZE_4)
1259                         goto do_generic_shift;
1260 #endif
1261 #if defined(ARCH_MIPS)
1262                 if (MIPS_HAS_ROT && alu == ROT_ROR && op_size >= OP_SIZE_4)
1263                         goto do_generic_shift;
1264 #endif
1265 #if defined(ARCH_POWER)
1266                 if (alu == ROT_ROL && op_size >= OP_SIZE_4)
1267                         goto do_generic_shift;
1268 #endif
1269 #if defined(ARCH_RISCV64)
1270                 if ((alu == ROT_ROL || alu == ROT_ROR) && likely(cpu_test_feature(CPU_FEATURE_zbb))) {
1271                         if (likely(op_size >= OP_SIZE_4)) {
1272                                 goto do_generic_shift;
1273                         }
1274                 }
1275 #endif
1276 #if defined(ARCH_S390)
1277                 if (Z && alu == ROT_ROL && op_size >= OP_SIZE_4)
1278                         goto do_generic_shift;
1279 #endif
1280                 if (alu == ROT_ROL || alu == ROT_ROR) {
1281                         target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
1282                         if (c) {
1283                                 g(gen_3address_rot_imm(ctx, op_s, alu == ROT_ROL ? ROT_SHL : ROT_SHR, R_SCRATCH_2, reg1, cnst, 0));
1284                                 g(gen_3address_rot_imm(ctx, op_s, alu == ROT_ROL ? ROT_SHR : ROT_SHL, target, reg1, -(uint64_t)cnst & ((8U << op_size) - 1), 0));
1285                         } else {
1286                                 g(gen_3address_rot(ctx, op_s, alu == ROT_ROL ? ROT_SHL : ROT_SHR, R_SCRATCH_2, reg1, reg3));
1287                                 g(gen_2address_alu1(ctx, i_size(OP_SIZE_4), ALU1_NEG, R_SCRATCH_3, reg3, 0));
1288                                 if (must_mask) {
1289                                         g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_4), ALU_AND, R_SCRATCH_3, R_SCRATCH_3, (8U << op_size) - 1, 0));
1290                                 }
1291                                 g(gen_3address_rot(ctx, op_s, alu == ROT_ROL ? ROT_SHR : ROT_SHL, target, reg1, R_SCRATCH_3));
1292                         }
1293                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_OR, target, target, R_SCRATCH_2, 0));
1294                         g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1295                         return true;
1296                 }
1297
1298                 goto do_generic_shift;
1299 do_generic_shift:
1300                 if (mode == MODE_INT && alu == ROT_SHL) {
1301                         target = gen_frame_target(ctx, slot_r, slot_1, slot_2, R_SCRATCH_1);
1302 #if defined(ARCH_S390)
1303                         if (op_size >= OP_SIZE_4) {
1304                                 if (c)
1305                                         g(gen_3address_rot_imm(ctx, op_size, ROT_SAL, target, reg1, cnst, 0));
1306                                 else
1307                                         g(gen_3address_rot(ctx, op_size, ROT_SAL, target, reg1, reg3));
1308
1309                                 gen_insn(INSN_JMP_COND, op_size, COND_O, 0);
1310                                 gen_four(label_ovf);
1311                         } else
1312 #endif
1313                         if (op_size <= OP_SIZE_NATIVE - 1) {
1314                                 if (c)
1315                                         g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, alu, target, reg1, cnst, 0));
1316                                 else
1317                                         g(gen_3address_rot(ctx, OP_SIZE_NATIVE, alu, target, reg1, reg3));
1318
1319                                 g(gen_cmp_extended(ctx, OP_SIZE_NATIVE, op_size, target, R_SCRATCH_2, label_ovf));
1320                         } else {
1321                                 if (c) {
1322                                         g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, alu, R_SCRATCH_2, reg1, cnst, 0));
1323                                         g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_4, R_SCRATCH_2, cnst, 0));
1324                                 } else {
1325                                         g(gen_3address_rot(ctx, OP_SIZE_NATIVE, alu, R_SCRATCH_2, reg1, reg3));
1326                                         g(gen_3address_rot(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_4, R_SCRATCH_2, reg3));
1327                                 }
1328
1329                                 g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, reg1, R_SCRATCH_4, COND_NE, label_ovf));
1330
1331                                 g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_2));
1332
1333                                 return true;
1334                         }
1335                 } else {
1336                         target = gen_frame_target(ctx, slot_r, NO_FRAME_T, slot_2, R_SCRATCH_1);
1337                         if (c)
1338                                 g(gen_3address_rot_imm(ctx, op_s, alu, target, reg1, cnst, 0));
1339                         else
1340                                 g(gen_3address_rot(ctx, op_s, alu, target, reg1, reg3));
1341                 }
1342
1343                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1344                 return true;
1345         }
1346         /******
1347          * BT *
1348          ******/
1349 do_bt: {
1350                 unsigned attr_unused op_s;
1351                 bool need_mask;
1352                 bool c = frame_t_is_const(slot_2);
1353                 int64_t cnst = !c ? 0 : frame_t_get_const(slot_2);
1354                 int64_t max_imm = (8U << op_size) - 1;
1355                 if (c) {
1356                         if (mode == MODE_INT) {
1357                                 if (alu == BTX_BT ? (uint64_t)cnst > (uint64_t)max_imm : (uint64_t)cnst >= (uint64_t)max_imm) {
1358                                         gen_insn(INSN_JMP, 0, 0, 0);
1359                                         gen_four(label_ovf);
1360                                         return true;
1361                                 }
1362                         }
1363                         cnst &= max_imm;
1364                 }
1365 #if defined(ARCH_X86)
1366                 if ((alu == BTX_BT || slot_1 == slot_r) && !slot_is_register(ctx, slot_1)) {
1367                         int64_t offset;
1368                         unsigned n_op_size = minimum(op_size, OP_SIZE_NATIVE);
1369                         g(gen_frame_get(ctx, n_op_size, garbage, slot_2, R_SCRATCH_1, &reg2));
1370                         if (mode == MODE_INT) {
1371                                 if (!c) {
1372                                         g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, n_op_size, reg2, max_imm, alu == BTX_BT ? COND_A : COND_AE, label_ovf));
1373                                         if (unlikely(op_size > OP_SIZE_NATIVE)) {
1374                                                 g(gen_address(ctx, R_FRAME, (size_t)slot_2 * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_STR_OFFSET, OP_SIZE_NATIVE));
1375                                                 gen_insn(INSN_CMP, OP_SIZE_NATIVE, 0, 1);
1376                                                 gen_address_offset();
1377                                                 gen_one(ARG_IMM);
1378                                                 gen_eight(0);
1379                                                 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_NE, 0);
1380                                                 gen_four(label_ovf);
1381                                         }
1382                                 }
1383                         } else {
1384                                 if (!c) {
1385                                         g(gen_3address_alu_imm(ctx, OP_SIZE_4, ALU_AND, R_SCRATCH_1, reg2, (8U << op_size) - 1, 0));
1386                                         reg2 = R_SCRATCH_1;
1387                                 }
1388                         }
1389                         offset = (size_t)slot_1 * slot_size;
1390                         if (c && cnst >= 8U << OP_SIZE_NATIVE) {
1391                                 offset += 1U << OP_SIZE_NATIVE;
1392                                 cnst -= 8U << OP_SIZE_NATIVE;
1393                         }
1394                         g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_STR_OFFSET, maximum(n_op_size, OP_SIZE_2)));
1395                         if (alu == BTX_BT) {
1396                                 gen_insn(INSN_BT, maximum(n_op_size, OP_SIZE_2), 0, 1);
1397                                 gen_address_offset();
1398                                 if (c) {
1399                                         gen_one(ARG_IMM);
1400                                         gen_eight(cnst);
1401                                 } else {
1402                                         gen_one(reg2);
1403                                 }
1404                                 g(gen_frame_set_cond(ctx, maximum(n_op_size, OP_SIZE_2), false, COND_B, slot_r));
1405                         } else {
1406                                 gen_insn(INSN_BTX, maximum(n_op_size, OP_SIZE_2), alu, 1);
1407                                 gen_address_offset();
1408                                 gen_address_offset();
1409                                 if (c) {
1410                                         gen_one(ARG_IMM);
1411                                         gen_eight(cnst);
1412                                 } else {
1413                                         gen_one(reg2);
1414                                 }
1415                         }
1416                         return true;
1417                 }
1418 #endif
1419                 if (unlikely(op_size > OP_SIZE_NATIVE)) {
1420                         size_t upcall;
1421                         if (mode == MODE_FIXED) {
1422                                 switch (alu) {
1423                                         case BTX_BTS:   upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_bts_,TYPE_INT_MAX));
1424                                                         break;
1425                                         case BTX_BTR:   upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_btr_,TYPE_INT_MAX));
1426                                                         break;
1427                                         case BTX_BTC:   upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_btc_,TYPE_INT_MAX));
1428                                                         break;
1429                                         case BTX_BT:    upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_bt_,TYPE_INT_MAX));
1430                                                         break;
1431                                         default:        internal(file_line, "do_alu: invalid bit test %u", alu);
1432                                 }
1433                         } else {
1434                                 switch (alu) {
1435                                         case BTX_BTS:   upcall = offsetof(struct cg_upcall_vector_s, cat(INT_binary_bts_,TYPE_INT_MAX));
1436                                                         break;
1437                                         case BTX_BTR:   upcall = offsetof(struct cg_upcall_vector_s, cat(INT_binary_btr_,TYPE_INT_MAX));
1438                                                         break;
1439                                         case BTX_BTC:   upcall = offsetof(struct cg_upcall_vector_s, cat(INT_binary_btc_,TYPE_INT_MAX));
1440                                                         break;
1441                                         case BTX_BT:    upcall = offsetof(struct cg_upcall_vector_s, cat(INT_binary_bt_,TYPE_INT_MAX));
1442                                                         break;
1443                                         default:        internal(file_line, "do_alu: invalid bit test %u", alu);
1444                                 }
1445                         }
1446                         g(gen_alu_upcall(ctx, upcall, op_size, slot_1, slot_2, slot_r, label_ovf));
1447                         return true;
1448                 }
1449                 op_s = minimum(OP_SIZE_NATIVE, ARCH_SHIFT_SIZE);
1450                 op_s = maximum(op_s, op_size);
1451                 g(gen_frame_get(ctx, op_size, zero_x, slot_1, R_SCRATCH_1, &reg1));
1452                 if (c) {
1453                         target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1454                         if (ARCH_HAS_BTX(alu == BTX_BT ? BTX_BTEXT : alu, OP_SIZE_NATIVE, true)) {
1455 #ifdef ARCH_X86
1456                                 g(gen_mov(ctx, OP_SIZE_NATIVE, target, reg1));
1457                                 reg1 = target;
1458                                 gen_insn(INSN_BTX, OP_SIZE_NATIVE, alu == BTX_BT ? BTX_BTEXT : alu, 1);
1459 #else
1460                                 gen_insn(INSN_BTX, OP_SIZE_NATIVE, alu == BTX_BT ? BTX_BTEXT : alu, 0);
1461 #endif
1462                                 gen_one(target);
1463                                 gen_one(reg1);
1464                                 gen_one(ARG_IMM);
1465                                 gen_eight(cnst);
1466                         } else switch (alu) {
1467                                 case BTX_BTS:
1468                                         g(gen_3address_alu_imm(ctx, i_size(op_size), ALU_OR, target, reg1, 1ULL << cnst, 0));
1469                                         break;
1470                                 case BTX_BTR:
1471                                         g(gen_3address_alu_imm(ctx, i_size(op_size), ALU_AND, target, reg1, ~(1ULL << cnst), 0));
1472                                         break;
1473                                 case BTX_BTC:
1474                                         g(gen_3address_alu_imm(ctx, i_size(op_size), ALU_XOR, target, reg1, 1ULL << cnst, 0));
1475                                         break;
1476                                 case BTX_BT:
1477                                         if (cnst) {
1478                                                 g(gen_3address_rot_imm(ctx, i_size(op_size), ROT_SHR, target, reg1, cnst, 0));
1479                                         }
1480                                         g(gen_3address_alu_imm(ctx, i_size(op_size), ALU_AND, target, target, 1, 0));
1481                                         break;
1482                                 default:
1483                                         internal(file_line, "do_alu: invalid bit test %u", alu);
1484                         }
1485                         if (alu == BTX_BT)
1486                                 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
1487                         else
1488                                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1489                         return true;
1490                 }
1491                 g(gen_frame_get(ctx, op_size, garbage, slot_2, R_SCRATCH_2, &reg2));
1492                 if (mode == MODE_INT) {
1493                         if (!c)
1494                                 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, i_size_cmp(op_size), reg2, max_imm, alu == BTX_BT ? COND_A : COND_AE, label_ovf));
1495                 }
1496                 if (alu != BTX_BT) {
1497                         if (!ARCH_HAS_BTX(alu, OP_SIZE_NATIVE, false))
1498                                 goto do_generic_bt;
1499                         need_mask = !ARCH_HAS_BTX(alu, op_size, false);
1500                 } else {
1501 #if defined(ARCH_X86)
1502                         need_mask = op_size < OP_SIZE_2;
1503 #else
1504                         if (!ARCH_HAS_BTX(BTX_BTEXT, OP_SIZE_NATIVE, false))
1505                                 goto do_generic_bt;
1506                         need_mask = !ARCH_HAS_BTX(BTX_BTEXT, op_size, false);
1507 #endif
1508                 }
1509                 if (need_mask) {
1510                         g(gen_3address_alu_imm(ctx, OP_SIZE_4, ALU_AND, R_SCRATCH_2, reg2, (8U << op_size) - 1, 0));
1511                         reg2 = R_SCRATCH_2;
1512                 }
1513                 if (alu == BTX_BT) {
1514 #if defined(ARCH_X86)
1515                         gen_insn(INSN_BT, maximum(op_size, OP_SIZE_2), 0, 1);
1516                         gen_one(reg1);
1517                         gen_one(reg2);
1518
1519                         g(gen_frame_set_cond(ctx, maximum(op_size, OP_SIZE_2), false, COND_B, slot_r));
1520 #else
1521                         target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1522                         gen_insn(INSN_BTX, need_mask ? OP_SIZE_NATIVE : op_size, BTX_BTEXT, 0);
1523                         gen_one(target);
1524                         gen_one(reg1);
1525                         gen_one(reg2);
1526
1527                         g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
1528 #endif
1529                 } else {
1530                         target = gen_frame_target(ctx, slot_r, NO_FRAME_T, slot_2, R_SCRATCH_1);
1531 #if defined(ARCH_X86)
1532                         if (target == reg2)
1533                                 target = R_SCRATCH_1;
1534                         if (target != reg1) {
1535                                 g(gen_mov(ctx, op_size, target, reg1));
1536                                 reg1 = target;
1537                         }
1538                         gen_insn(INSN_BTX, maximum(op_size, OP_SIZE_2), alu, 1);
1539 #else
1540                         gen_insn(INSN_BTX, need_mask ? OP_SIZE_NATIVE : op_size, alu, 0);
1541 #endif
1542                         gen_one(target);
1543                         gen_one(reg1);
1544                         gen_one(reg2);
1545
1546                         g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1547                 }
1548                 return true;
1549
1550                 goto do_generic_bt;
1551 do_generic_bt:
1552                 if (mode == MODE_FIXED && op_size < ARCH_SHIFT_SIZE) {
1553                         g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_4), ALU_AND, R_SCRATCH_2, reg2, (8U << op_size) - 1, 0));
1554                         reg2 = R_SCRATCH_2;
1555                 }
1556                 g(gen_load_constant(ctx, R_SCRATCH_3, 1));
1557
1558                 g(gen_3address_rot(ctx, op_s, ROT_SHL, R_SCRATCH_3, R_SCRATCH_3, reg2));
1559
1560                 switch (alu) {
1561                         case BTX_BT:
1562 #if ARCH_HAS_FLAGS
1563 #if defined(ARCH_S390) || defined(ARCH_POWER)
1564                                 g(gen_3address_alu(ctx, i_size(op_size), ALU_AND, R_SCRATCH_1, reg1, R_SCRATCH_3, 1));
1565 #else
1566                                 gen_insn(INSN_TEST, i_size(op_size), 0, 1);
1567                                 gen_one(reg1);
1568                                 gen_one(R_SCRATCH_3);
1569 #endif
1570                                 g(gen_frame_set_cond(ctx, i_size_cmp(op_size), false, COND_NE, slot_r));
1571 #else
1572                                 g(gen_3address_alu(ctx, i_size(op_size), ALU_AND, R_SCRATCH_1, reg1, R_SCRATCH_3, 0));
1573                                 g(gen_frame_cmp_imm_set_cond_reg(ctx, i_size(op_size), R_SCRATCH_1, 0, COND_NE, slot_r));
1574 #endif
1575                                 return true;
1576                         case BTX_BTS:
1577                                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1578                                 g(gen_3address_alu(ctx, i_size(op_size), ALU_OR, target, reg1, R_SCRATCH_3, 0));
1579                                 break;
1580                         case BTX_BTR:
1581                                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1582                                 if (!ARCH_HAS_ANDN) {
1583                                         g(gen_3address_alu_imm(ctx, i_size(op_size), ALU_XOR, R_SCRATCH_3, R_SCRATCH_3, -1, 0));
1584
1585                                         g(gen_3address_alu(ctx, i_size(op_size), ALU_AND, target, reg1, R_SCRATCH_3, 0));
1586                                         break;
1587                                 }
1588                                 g(gen_3address_alu(ctx, i_size(op_size), ALU_ANDN, target, reg1, R_SCRATCH_3, 0));
1589                                 break;
1590                         case BTX_BTC:
1591                                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1592                                 g(gen_3address_alu(ctx, i_size(op_size), ALU_XOR, target, reg1, R_SCRATCH_3, 0));
1593                                 break;
1594                         default:
1595                                 internal(file_line, "gen_alu: unsupported bit test %u", alu);
1596                 }
1597
1598                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1599
1600                 return true;
1601         }
1602         /***********
1603          * COMPARE *
1604          ***********/
1605 do_compare: {
1606                 if (unlikely(op_size > OP_SIZE_NATIVE)) {
1607                         size_t attr_unused upcall;
1608                         frame_t attr_unused swap;
1609                         switch (alu) {
1610                                 case COND_E:
1611                                 case COND_NE:
1612                                         g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
1613                                         g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, ALU_XOR, 0, slot_2, lo_word(OP_SIZE_NATIVE), true, R_SCRATCH_1));
1614                                         g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, ALU_XOR, 0, slot_2, hi_word(OP_SIZE_NATIVE), true, R_SCRATCH_2));
1615 #if defined(ARCH_ARM64)
1616                                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_OR, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_2, 0));
1617
1618                                         gen_insn(INSN_CMP, OP_SIZE_NATIVE, 0, 1);
1619                                         gen_one(R_SCRATCH_1);
1620                                         gen_one(ARG_IMM);
1621                                         gen_eight(0);
1622 #else
1623                                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_OR, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_2, ARCH_HAS_FLAGS));
1624 #endif
1625 #if ARCH_HAS_FLAGS
1626                                         g(gen_frame_set_cond(ctx, OP_SIZE_NATIVE, false, alu, slot_r));
1627 #else
1628                                         g(gen_frame_cmp_imm_set_cond_reg(ctx, OP_SIZE_NATIVE, R_SCRATCH_1, 0, alu, slot_r));
1629 #endif
1630                                         return true;
1631 #if defined(ARCH_X86_64) || defined(ARCH_X86_X32) || defined(ARCH_ARM)
1632                                 case COND_G:
1633                                 case COND_A:
1634                                         swap = slot_1; slot_1 = slot_2; slot_2 = swap;
1635                                         alu = alu == COND_G ? COND_L : COND_B;
1636                                         /*-fallthrough*/
1637                                 case COND_L:
1638                                 case COND_B:
1639                                         g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_2, lo_word(OP_SIZE_NATIVE), true, R_SCRATCH_2));
1640                                         g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_1, hi_word(OP_SIZE_NATIVE), true, R_SCRATCH_1));
1641                                         g(gen_frame_load_cmp(ctx, OP_SIZE_NATIVE, false, garbage, true, slot_1, lo_word(OP_SIZE_NATIVE), true, R_SCRATCH_2));
1642                                         g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, ALU_SBB, 1, slot_2, hi_word(OP_SIZE_NATIVE), true, R_SCRATCH_1));
1643                                         g(gen_frame_set_cond(ctx, OP_SIZE_NATIVE, false, alu, slot_r));
1644                                         return true;
1645                                 case COND_GE:
1646                                 case COND_AE:
1647                                         swap = slot_1; slot_1 = slot_2; slot_2 = swap;
1648                                         alu = alu == COND_GE ? COND_LE : COND_BE;
1649                                         /*-fallthrough*/
1650                                 case COND_LE:
1651                                 case COND_BE:
1652                                         g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_1, lo_word(OP_SIZE_NATIVE), true, R_SCRATCH_2));
1653                                         g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_2, hi_word(OP_SIZE_NATIVE), true, R_SCRATCH_1));
1654                                         g(gen_frame_load_cmp(ctx, OP_SIZE_NATIVE, false, garbage, true, slot_2, lo_word(OP_SIZE_NATIVE), true, R_SCRATCH_2));
1655                                         g(gen_frame_load_op(ctx, OP_SIZE_NATIVE, garbage, ALU_SBB, 1, slot_1, hi_word(OP_SIZE_NATIVE), true, R_SCRATCH_1));
1656                                         g(gen_frame_set_cond(ctx, OP_SIZE_NATIVE, false, alu == COND_LE ? COND_GE : COND_AE, slot_r));
1657                                         return true;
1658 #else
1659                                 case COND_L:    upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_less_,TYPE_INT_MAX)); goto do_upcall;
1660                                 case COND_LE:   upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_less_equal_,TYPE_INT_MAX)); goto do_upcall;
1661                                 case COND_G:    upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_greater_,TYPE_INT_MAX)); goto do_upcall;
1662                                 case COND_GE:   upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_greater_equal_,TYPE_INT_MAX)); goto do_upcall;
1663                                 case COND_B:    upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_uless_,TYPE_INT_MAX)); goto do_upcall;
1664                                 case COND_BE:   upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_uless_equal_,TYPE_INT_MAX)); goto do_upcall;
1665                                 case COND_A:    upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_ugreater_,TYPE_INT_MAX)); goto do_upcall;
1666                                 case COND_AE:   upcall = offsetof(struct cg_upcall_vector_s, cat(FIXED_binary_ugreater_equal_,TYPE_INT_MAX)); goto do_upcall;
1667                                 do_upcall:      g(gen_alu_upcall(ctx, upcall, op_size, slot_1, slot_2, slot_r, 0));
1668                                                 return true;
1669 #endif
1670                                 default:
1671                                         internal(file_line, "gen_alu: unsupported condition %u", alu);
1672                         }
1673                         return false;
1674                 }
1675 #if defined(ARCH_X86)
1676                 g(gen_frame_get(ctx, op_size, garbage, slot_1, R_SCRATCH_1, &reg1));
1677                 g(gen_frame_load_cmp_set_cond(ctx, op_size, garbage, slot_2, reg1, alu, slot_r));
1678 #else
1679                 g(gen_frame_get(ctx, op_size, op_size == i_size_cmp(op_size) + (unsigned)zero ? garbage : alu == COND_L || alu == COND_LE || alu == COND_G || alu == COND_GE || ARCH_PREFERS_SX(op_size) ? sign_x : zero_x, slot_1, R_SCRATCH_1, &reg1));
1680                 g(gen_frame_load_cmp_set_cond(ctx, op_size, alu == COND_L || alu == COND_LE || alu == COND_G || alu == COND_GE || ARCH_PREFERS_SX(op_size) ? sign_x : zero_x, slot_2, reg1, alu, slot_r));
1681 #endif
1682                 return true;
1683         }
1684 }
1685
1686 static bool attr_w gen_alu1(struct codegen_context *ctx, unsigned mode, unsigned op_size, unsigned op, uint32_t label_ovf, frame_t slot_1, frame_t slot_r)
1687 {
1688         unsigned alu;
1689         unsigned reg1, target;
1690         switch (mode) {
1691                 case MODE_FIXED: switch (op) {
1692                         case OPCODE_FIXED_OP_not:               alu = ALU1_NOT; goto do_alu;
1693                         case OPCODE_FIXED_OP_neg:               alu = ALU1_NEG; goto do_alu;
1694                         case OPCODE_FIXED_OP_bswap:
1695                         case OPCODE_FIXED_OP_bswap_alt1:        alu = ALU1_BSWAP; goto do_bswap;
1696                         case OPCODE_FIXED_OP_brev:
1697                         case OPCODE_FIXED_OP_brev_alt1:         alu = ALU1_BREV; goto do_brev;
1698                         case OPCODE_FIXED_OP_bsf:
1699                         case OPCODE_FIXED_OP_bsf_alt1:          alu = ALU1_BSF; goto do_bsf_bsr_popcnt;
1700                         case OPCODE_FIXED_OP_bsr:
1701                         case OPCODE_FIXED_OP_bsr_alt1:          alu = ALU1_BSR; goto do_bsf_bsr_popcnt;
1702                         case OPCODE_FIXED_OP_popcnt:
1703                         case OPCODE_FIXED_OP_popcnt_alt1:       alu = ALU1_POPCNT; goto do_bsf_bsr_popcnt;
1704                         case OPCODE_FIXED_OP_to_int:            goto do_fixed_conv;
1705                         case OPCODE_FIXED_OP_from_int:          goto do_fixed_conv;
1706                         case OPCODE_FIXED_OP_uto_int:           goto conv_uto_int;
1707                         case OPCODE_FIXED_OP_ufrom_int:         goto conv_ufrom_int;
1708                         default:                                internal(file_line, "gen_alu1: unsupported fixed operation %u", op);
1709                 }
1710                 case MODE_INT: switch (op) {
1711                         case OPCODE_INT_OP_not:                 alu = ALU1_NOT; mode = MODE_FIXED; goto do_alu;
1712                         case OPCODE_INT_OP_neg:                 alu = ALU1_NEG; goto do_alu;
1713                         case OPCODE_INT_OP_bsf:                 alu = ALU1_BSF; goto do_bsf_bsr_popcnt;
1714                         case OPCODE_INT_OP_bsr:                 alu = ALU1_BSR; goto do_bsf_bsr_popcnt;
1715                         case OPCODE_INT_OP_popcnt:
1716                         case OPCODE_INT_OP_popcnt_alt1:         alu = ALU1_POPCNT; goto do_bsf_bsr_popcnt;
1717                         case OPCODE_INT_OP_to_int:              goto do_conv;
1718                         case OPCODE_INT_OP_from_int:            goto do_conv;
1719                         default:                                internal(file_line, "gen_alu1: unsupported int operation %u", op);
1720                 }
1721                 case MODE_BOOL: switch (op) {
1722                         case OPCODE_BOOL_OP_not:                goto do_bool_not;
1723                         default:                                internal(file_line, "gen_alu1: unsupported bool operation %u", op);
1724                 }
1725         }
1726         internal(file_line, "gen_alu1: unsupported mode %u", mode);
1727
1728         /*******
1729          * ALU *
1730          *******/
1731 do_alu: {
1732                 bool arch_use_flags = ARCH_HAS_FLAGS;
1733                 enum extend ex;
1734 #if defined(ARCH_POWER)
1735                 arch_use_flags = false;
1736 #endif
1737                 if (op_size > OP_SIZE_NATIVE) {
1738 #if !defined(ARCH_X86) && !defined(ARCH_ARM) && !defined(ARCH_POWER)
1739                         if (alu == ALU1_NEG) {
1740                                 if (mode == MODE_FIXED)
1741                                         g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(FIXED_unary_neg_,TYPE_INT_MAX)), op_size, slot_1, NO_FRAME_T, slot_r, 0));
1742                                 else
1743                                         g(gen_alu_upcall(ctx, offsetof(struct cg_upcall_vector_s, cat(INT_unary_neg_,TYPE_INT_MAX)), op_size, slot_1, NO_FRAME_T, slot_r, label_ovf));
1744                                 return true;
1745                         }
1746 #endif
1747                         g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
1748 #if defined(ARCH_S390)
1749                         if (alu == ALU1_NOT) {
1750                                 g(gen_load_constant(ctx, R_SCRATCH_3, -1));
1751
1752                                 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_XOR, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_3, 0));
1753                                 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_XOR, R_SCRATCH_2, R_SCRATCH_2, R_SCRATCH_3, 0));
1754
1755                                 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_1, R_SCRATCH_2));
1756                                 return true;
1757                         }
1758 #endif
1759                         g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, alu, R_SCRATCH_1, R_SCRATCH_1, alu == ALU1_NEG ? 2 : 0));
1760                         if (alu == ALU1_NOT) {
1761                                 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NOT, R_SCRATCH_2, R_SCRATCH_2, 0));
1762                         } else {
1763 #if defined(ARCH_X86)
1764                                 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NOT, R_SCRATCH_2, R_SCRATCH_2, 0));
1765
1766                                 g(gen_imm(ctx, -1, IMM_PURPOSE_SUB, OP_SIZE_NATIVE));
1767                                 gen_insn(INSN_ALU, OP_SIZE_NATIVE, ALU_SBB, ALU_WRITES_FLAGS(ALU_SBB, is_imm()));
1768                                 gen_one(R_SCRATCH_2);
1769                                 gen_one(R_SCRATCH_2);
1770                                 gen_imm_offset();
1771 #else
1772                                 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NGC, R_SCRATCH_2, R_SCRATCH_2, (mode == MODE_INT)));
1773 #endif
1774                         }
1775                         if (mode == MODE_INT) {
1776                                 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_O, 0);
1777                                 gen_four(label_ovf);
1778                         }
1779                         g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_1, R_SCRATCH_2));
1780                         return true;
1781                 }
1782                 if ((arch_use_flags || ARCH_SUPPORTS_TRAPS) && slot_1 == slot_r && i_size_cmp(op_size) == op_size + zero) {
1783                         struct cg_exit *ce;
1784                         unsigned undo_alu = alu;
1785                         if (slot_is_register(ctx, slot_1)) {
1786                                 unsigned reg = ctx->registers[slot_1];
1787                                 if (mode == MODE_INT && ARCH_SUPPORTS_TRAPS) {
1788                                         gen_insn(INSN_ALU1_TRAP, op_size, alu, ALU1_WRITES_FLAGS(alu));
1789                                         gen_one(reg);
1790                                         gen_one(reg);
1791                                         if (ARCH_TRAP_BEFORE || alu == undo_alu) {
1792                                                 gen_four(label_ovf);
1793                                                 return true;
1794                                         } else {
1795                                                 ce = alloc_undo_label(ctx);
1796                                                 if (unlikely(!ce))
1797                                                         return false;
1798                                                 gen_four(ce->undo_label);
1799                                                 goto do_undo_opcode;
1800                                         }
1801                                 }
1802                                 g(gen_2address_alu1(ctx, i_size(op_size), alu, reg, reg, mode == MODE_INT));
1803                                 if (mode == MODE_INT) {
1804                                         if (alu != undo_alu) {
1805                                                 ce = alloc_undo_label(ctx);
1806                                                 if (unlikely(!ce))
1807                                                         return false;
1808                                                 gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
1809                                                 gen_four(ce->undo_label);
1810 do_undo_opcode:
1811                                                 ce->undo_opcode = INSN_ALU1 + ARCH_PARTIAL_ALU(op_size);
1812                                                 ce->undo_op_size = i_size(op_size);
1813                                                 ce->undo_aux = undo_alu;
1814                                                 ce->undo_writes_flags = ALU1_WRITES_FLAGS(undo_alu);
1815                                                 ce->undo_parameters[0] = reg;
1816                                                 ce->undo_parameters[1] = reg;
1817                                                 ce->undo_parameters_len = 2;
1818                                         } else {
1819                                                 gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
1820                                                 gen_four(label_ovf);
1821                                         }
1822                                 }
1823                                 return true;
1824                         }
1825 #if defined(ARCH_X86)
1826                         else {
1827                                 size_t m;
1828                                 int64_t offset = (size_t)slot_1 * slot_size;
1829                                 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDR_OFFSET, i_size(op_size)));
1830                                 gen_insn(INSN_ALU1 + ARCH_PARTIAL_ALU(op_size), i_size(op_size), alu, ALU1_WRITES_FLAGS(alu) | (mode == MODE_INT));
1831                                 gen_address_offset();
1832                                 gen_address_offset();
1833                                 if (mode == MODE_INT) {
1834                                         if (alu != undo_alu) {
1835                                                 ce = alloc_undo_label(ctx);
1836                                                 if (unlikely(!ce))
1837                                                         return false;
1838                                                 ce->undo_opcode = INSN_ALU1 + ARCH_PARTIAL_ALU(op_size);
1839                                                 ce->undo_op_size = i_size(op_size);
1840                                                 ce->undo_aux = undo_alu;
1841                                                 ce->undo_writes_flags = ALU1_WRITES_FLAGS(undo_alu);
1842                                                 m = mark_params(ctx);
1843                                                 gen_address_offset();
1844                                                 gen_address_offset();
1845                                                 copy_params(ctx, ce, m);
1846                                                 gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
1847                                                 gen_four(ce->undo_label);
1848                                         } else {
1849                                                 gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
1850                                                 gen_four(label_ovf);
1851                                         }
1852                                 }
1853                                 return true;
1854                         }
1855 #endif
1856                 }
1857                 target = gen_frame_target(ctx, slot_r, mode == MODE_INT ? slot_1 : NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1858                 if (mode == MODE_FIXED) {
1859                         ex = garbage;
1860                 } else {
1861                         ex = ARCH_PREFERS_SX(op_size) ? sign_x : zero_x;
1862                         if (ARCH_SUPPORTS_TRAPS && op_size >= OP_SIZE_4)
1863                                 ex = garbage;
1864                         if (op_size == i_size(op_size) + (unsigned)zero)
1865                                 ex = garbage;
1866                 }
1867                 g(gen_frame_get(ctx, op_size, ex, slot_1, mode == MODE_INT ? R_SCRATCH_2 : target, &reg1));
1868 #if defined(ARCH_S390)
1869                 if (alu == ALU1_NOT) {
1870                         g(gen_3address_alu_imm(ctx, i_size(op_size), ALU_XOR, target, reg1, -1, 0));
1871
1872                         g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1873                         return true;
1874                 }
1875 #endif
1876 #if defined(ARCH_X86)
1877                 g(gen_2address_alu1(ctx, op_size, alu, target, reg1, mode == MODE_INT));
1878 #else
1879                 if (mode == MODE_INT) {
1880 #if defined(ARCH_POWER)
1881                         if (op_size == OP_SIZE_NATIVE) {
1882                                 g(gen_2address_alu1(ctx, i_size(op_size), alu, target, reg1, 0));
1883                                 if (alu == ALU1_NEG) {
1884                                         g(gen_3address_alu(ctx, i_size(op_size), ALU_AND, R_CG_SCRATCH, target, reg1, 1));
1885                                 }
1886                                 gen_insn(INSN_JMP_COND, op_size, COND_L, 0);
1887                                 gen_four(label_ovf);
1888
1889                                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1890
1891                                 return true;
1892                         }
1893 #endif
1894                         if (!arch_use_flags && !ARCH_SUPPORTS_TRAPS && ARCH_HAS_ANDN && op_size >= OP_SIZE_4) {
1895                                 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, alu, target, reg1, 0));
1896
1897                                 if (alu == ALU1_NEG) {
1898                                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_AND, R_SCRATCH_3, target, reg1, 0));
1899                                 }
1900
1901                                 if (op_size < OP_SIZE_NATIVE)
1902                                         g(gen_extend(ctx, op_size, sign_x, R_SCRATCH_3, R_SCRATCH_3));
1903
1904                                 g(gen_jmp_on_zero(ctx, OP_SIZE_NATIVE, R_SCRATCH_3, COND_S, label_ovf));
1905
1906                                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1907
1908                                 return true;
1909                         }
1910                         if (op_size <= OP_SIZE_2 || (!arch_use_flags && !ARCH_SUPPORTS_TRAPS)) {
1911                                 int64_t imm = (ARCH_PREFERS_SX(op_size) ? -0x80ULL : 0x80ULL) << (((1 << op_size) - 1) * 8);
1912
1913                                 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, i_size_cmp(op_size), reg1, imm, COND_E, label_ovf));
1914
1915                                 mode = MODE_FIXED;
1916
1917                                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1918                         }
1919                 }
1920 #if !ARCH_HAS_FLAGS
1921                 if (mode == MODE_INT) {
1922                         gen_insn(INSN_ALU1_TRAP, op_size, alu, ALU1_WRITES_FLAGS(alu));
1923                         gen_one(target);
1924                         gen_one(reg1);
1925                         gen_four(label_ovf);
1926                         g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1927                         return true;
1928                 }
1929 #endif
1930                 g(gen_2address_alu1(ctx, i_size(op_size), alu, target, reg1, mode == MODE_INT));
1931 #endif
1932                 if (mode == MODE_INT) {
1933                         gen_insn(INSN_JMP_COND, i_size_cmp(op_size), COND_O, 0);
1934                         gen_four(label_ovf);
1935                 }
1936                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1937                 return true;
1938         }
1939
1940         /*******
1941          * NOT *
1942          *******/
1943 do_bool_not: {
1944                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1945                 g(gen_frame_get(ctx, op_size, garbage, slot_1, target, &reg1));
1946
1947                 g(gen_3address_alu_imm(ctx, i_size(op_size), ALU_XOR, target, reg1, 1, 0));
1948
1949                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
1950                 return true;
1951         }
1952
1953         /*********
1954          * BSWAP *
1955          *********/
1956 do_bswap: {
1957                 bool attr_unused sx = false;
1958 #if defined(ARCH_X86) || defined(ARCH_ARM) || defined(ARCH_IA64) || defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS) || defined(ARCH_RISCV64) || defined(ARCH_S390)
1959 #if defined(ARCH_ARM32)
1960                 if (unlikely(!cpu_test_feature(CPU_FEATURE_armv6)))
1961                         goto do_generic_bswap;
1962 #endif
1963 #if defined(ARCH_MIPS)
1964                 if (unlikely(!MIPS_HAS_ROT))
1965                         goto do_generic_bswap;
1966                 sx = op_size == OP_SIZE_4;
1967 #endif
1968 #if defined(ARCH_RISCV64)
1969                 if (unlikely(!cpu_test_feature(CPU_FEATURE_zbb)))
1970                         goto do_generic_bswap;
1971 #endif
1972 #if defined(ARCH_S390)
1973                 if (op_size == OP_SIZE_2)
1974                         goto do_generic_bswap;
1975 #endif
1976 #if defined(ARCH_X86)
1977                 if (op_size >= OP_SIZE_4 && !cpu_test_feature(CPU_FEATURE_bswap))
1978                         goto do_generic_bswap;
1979 #endif
1980                 if (op_size > OP_SIZE_NATIVE) {
1981                         g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
1982                         reg1 = R_SCRATCH_1;
1983                         target = R_SCRATCH_1;
1984                 } else {
1985                         target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
1986                         g(gen_frame_get(ctx, op_size, sx ? sign_x : garbage, slot_1, target, &reg1));
1987                 }
1988
1989                 if (op_size == OP_SIZE_1) {
1990 #if defined(ARCH_IA64) || defined(ARCH_RISCV64)
1991                 } else if (op_size == OP_SIZE_2 || op_size == OP_SIZE_4) {
1992                         g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_BSWAP, target, reg1, 0));
1993
1994                         g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SAR, target, target, op_size == OP_SIZE_2 ? 48 : 32, 0));
1995 #endif
1996                 } else if (op_size == OP_SIZE_2) {
1997 #if defined(ARCH_X86)
1998                         g(gen_3address_rot_imm(ctx, OP_SIZE_2, ROT_ROR, target, reg1, 8, 0));
1999 #else
2000                         g(gen_2address_alu1(ctx, OP_SIZE_4, ALU1_BSWAP16, target, reg1, 0));
2001 #endif
2002                 } else {
2003                         g(gen_2address_alu1(ctx, minimum(op_size, OP_SIZE_NATIVE), ALU1_BSWAP, target, reg1, 0));
2004                 }
2005                 if (op_size > OP_SIZE_NATIVE) {
2006                         g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_BSWAP, R_SCRATCH_2, R_SCRATCH_2, 0));
2007                 }
2008
2009                 if (op_size > OP_SIZE_NATIVE)
2010                         g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_2, R_SCRATCH_1));
2011                 else
2012                         g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2013                 return true;
2014 #endif
2015                 goto do_generic_bswap;
2016 do_generic_bswap:
2017                 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_unary_bswap_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, 0);
2018         }
2019         /********
2020          * BREV *
2021          ********/
2022 do_brev: {
2023 #if defined(ARCH_ARM) || defined(ARCH_LOONGARCH64) || (defined(ARCH_MIPS) && MIPS_R6)
2024 #if defined(ARCH_ARM32)
2025                 if (unlikely(!cpu_test_feature(CPU_FEATURE_armv6t2)))
2026                         goto do_generic_brev;
2027 #endif
2028                 if (op_size > OP_SIZE_NATIVE) {
2029                         g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
2030                         reg1 = R_SCRATCH_1;
2031                         target = R_SCRATCH_1;
2032                 } else {
2033                         target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
2034                         g(gen_frame_get(ctx, op_size, garbage, slot_1, target, &reg1));
2035                 }
2036
2037                 g(gen_2address_alu1(ctx, minimum(maximum(OP_SIZE_4, op_size), OP_SIZE_NATIVE), ALU1_BREV, target, reg1, 0));
2038                 if (op_size <= OP_SIZE_2) {
2039                         g(gen_3address_rot_imm(ctx, OP_SIZE_4, ROT_SHR, target, target, op_size == OP_SIZE_1 ? 24 : 16, 0));
2040                 }
2041                 if (op_size > OP_SIZE_NATIVE) {
2042                         g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_BREV, R_SCRATCH_2, R_SCRATCH_2, 0));
2043                 }
2044
2045                 if (op_size > OP_SIZE_NATIVE)
2046                         g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_2, R_SCRATCH_1));
2047                 else
2048                         g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2049                 return true;
2050 #endif
2051                 goto do_generic_brev;
2052 do_generic_brev:
2053                 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_unary_brev_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, 0);
2054         }
2055         /******************
2056          * BSF/BSR/POPCNT *
2057          ******************/
2058 do_bsf_bsr_popcnt: {
2059                 if (op_size > OP_SIZE_NATIVE) {
2060 #if defined(ARCH_X86)
2061                         uint32_t label_finish = 0;      /* avoid warning */
2062                         if (alu == ALU1_POPCNT && unlikely(!cpu_test_feature(CPU_FEATURE_popcnt)))
2063                                 goto do_generic_bsf_bsr_popcnt;
2064                         if (alu == ALU1_BSR || alu == ALU1_POPCNT) {
2065                                 if (mode == MODE_INT) {
2066                                         g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
2067                                         g(gen_imm(ctx, 0, IMM_PURPOSE_STORE_VALUE, OP_SIZE_NATIVE));
2068                                         gen_insn(INSN_CMP, OP_SIZE_NATIVE, 0, 1);
2069                                         gen_address_offset();
2070                                         gen_imm_offset();
2071
2072                                         gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_S, 0);
2073                                         gen_four(label_ovf);
2074                                 }
2075                         }
2076                         if (alu == ALU1_BSF) {
2077                                 label_finish = alloc_label(ctx);
2078                                 if (unlikely(!label_finish))
2079                                         return false;
2080
2081                                 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + lo_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
2082                                 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_BSF, ALU1_WRITES_FLAGS(ALU1_BSF));
2083                                 gen_one(R_SCRATCH_1);
2084                                 gen_address_offset();
2085
2086                                 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_NE, 0);
2087                                 gen_four(label_finish);
2088
2089                                 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
2090                                 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_BSF, ALU1_WRITES_FLAGS(ALU1_BSF));
2091                                 gen_one(R_SCRATCH_1);
2092                                 gen_address_offset();
2093
2094                                 g(gen_3address_alu_imm(ctx, OP_SIZE_4, ALU_ADD, R_SCRATCH_1, R_SCRATCH_1, 8U << OP_SIZE_NATIVE, 0));
2095                         }
2096                         if (alu == ALU1_BSR) {
2097                                 label_finish = alloc_label(ctx);
2098                                 if (unlikely(!label_finish))
2099                                         return false;
2100
2101                                 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
2102                                 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_BSR, ALU1_WRITES_FLAGS(ALU1_BSR));
2103                                 gen_one(R_SCRATCH_1);
2104                                 gen_address_offset();
2105
2106                                 g(gen_3address_alu_imm(ctx, OP_SIZE_4, ALU_ADD, R_SCRATCH_1, R_SCRATCH_1, 8U << OP_SIZE_NATIVE, 0));
2107
2108                                 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_NE, 0);
2109                                 gen_four(label_finish);
2110
2111                                 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + lo_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
2112                                 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_BSR, ALU1_WRITES_FLAGS(ALU1_BSR));
2113                                 gen_one(R_SCRATCH_1);
2114                                 gen_address_offset();
2115                         }
2116                         if (alu == ALU1_BSF || alu == ALU1_BSR) {
2117                                 if (mode == MODE_INT) {
2118                                         gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_E, 0);
2119                                         gen_four(label_ovf);
2120                                 } else {
2121                                         gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_NE, 0);
2122                                         gen_four(label_finish);
2123
2124                                         g(gen_load_constant(ctx, R_SCRATCH_1, -1));
2125                                 }
2126
2127                                 gen_label(label_finish);
2128
2129                                 if (mode == MODE_INT)
2130                                         goto write_result;
2131
2132                                 if (R_SCRATCH_1 != R_AX || R_SCRATCH_2 != R_DX)
2133                                         internal(file_line, "gen_alu1: bad scratch registers");
2134                                 gen_insn(INSN_CWD, OP_SIZE_NATIVE, 0, 0);
2135                                 gen_one(R_DX);
2136                                 gen_one(R_AX);
2137
2138                                 g(gen_address(ctx, R_FRAME, (size_t)slot_r * slot_size + lo_word(OP_SIZE_NATIVE), IMM_PURPOSE_STR_OFFSET, OP_SIZE_NATIVE));
2139                                 gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
2140                                 gen_address_offset();
2141                                 gen_one(R_SCRATCH_1);
2142
2143                                 g(gen_address(ctx, R_FRAME, (size_t)slot_r * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_STR_OFFSET, OP_SIZE_NATIVE));
2144                                 gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
2145                                 gen_address_offset();
2146                                 gen_one(R_SCRATCH_2);
2147
2148                                 return true;
2149                         }
2150                         if (alu == ALU1_POPCNT) {
2151                                 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + lo_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
2152                                 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_POPCNT, ALU1_WRITES_FLAGS(ALU1_POPCNT));
2153                                 gen_one(R_SCRATCH_1);
2154                                 gen_address_offset();
2155
2156                                 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
2157                                 gen_insn(INSN_ALU1, OP_SIZE_NATIVE, ALU1_POPCNT, ALU1_WRITES_FLAGS(ALU1_POPCNT));
2158                                 gen_one(R_SCRATCH_2);
2159                                 gen_address_offset();
2160
2161                                 g(gen_3address_alu(ctx, OP_SIZE_4, ALU_ADD, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_2, 1));
2162 write_result:
2163                                 g(gen_address(ctx, R_FRAME, (size_t)slot_r * slot_size + lo_word(OP_SIZE_NATIVE), IMM_PURPOSE_STR_OFFSET, OP_SIZE_NATIVE));
2164                                 gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
2165                                 gen_address_offset();
2166                                 gen_one(R_SCRATCH_1);
2167
2168                                 g(gen_address(ctx, R_FRAME, (size_t)slot_r * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_STR_OFFSET, OP_SIZE_NATIVE));
2169                                 g(gen_imm(ctx, 0, IMM_PURPOSE_STORE_VALUE, OP_SIZE_NATIVE));
2170                                 gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
2171                                 gen_address_offset();
2172                                 gen_imm_offset();
2173
2174                                 return true;
2175                         }
2176 #endif
2177                         goto do_generic_bsf_bsr_popcnt;
2178                 }
2179 #if defined(ARCH_X86)
2180                 if (alu == ALU1_POPCNT && unlikely(!cpu_test_feature(CPU_FEATURE_popcnt)))
2181                         goto do_generic_bsf_bsr_popcnt;
2182                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
2183                 if (op_size == OP_SIZE_1 || ((alu == ALU1_BSR || alu == ALU1_POPCNT) && mode == MODE_INT)) {
2184                         g(gen_frame_get(ctx, op_size, zero_x, slot_1, target, &reg1));
2185                         if ((alu == ALU1_BSR || alu == ALU1_POPCNT) && mode == MODE_INT) {
2186                                 g(gen_cmp_test_jmp(ctx, INSN_TEST, op_size, reg1, reg1, alu == ALU1_BSR ? COND_LE : COND_S, label_ovf));
2187                         }
2188                         g(gen_2address_alu1(ctx, maximum(op_size, OP_SIZE_2), alu, target, reg1, 1));
2189                         if ((alu == ALU1_BSR || alu == ALU1_POPCNT) && mode == MODE_INT)
2190                                 goto x86_bsf_bsr_popcnt_finish;
2191                 } else {
2192                         g(gen_frame_load_op1(ctx, op_size, alu, 1, slot_1, target));
2193                 }
2194                 if (alu == ALU1_POPCNT)
2195                         goto x86_bsf_bsr_popcnt_finish;
2196                 if (mode == MODE_FIXED) {
2197                         uint32_t cmov_label;
2198                         gen_insn(INSN_MOV, maximum(op_size, OP_SIZE_4), 0, 0);
2199                         gen_one(R_SCRATCH_2);
2200                         gen_one(ARG_IMM);
2201                         gen_eight(-1);
2202                         g(gen_cmov(ctx, maximum(op_size, OP_SIZE_4), COND_E, target, &cmov_label));
2203                         gen_one(R_SCRATCH_2);
2204                         if (cmov_label)
2205                                 gen_label(cmov_label);
2206
2207                 } else {
2208                         gen_insn(INSN_JMP_COND, maximum(op_size, OP_SIZE_2), COND_E, 0);
2209                         gen_four(label_ovf);
2210                 }
2211 x86_bsf_bsr_popcnt_finish:
2212                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2213                 return true;
2214 #endif
2215 #if defined(ARCH_ARM)
2216 #if defined(ARCH_ARM32)
2217                 if (alu == ALU1_BSR && unlikely(!cpu_test_feature(CPU_FEATURE_armv6)))
2218                         goto do_generic_bsf_bsr_popcnt;
2219                 if (alu == ALU1_BSF && unlikely(!cpu_test_feature(CPU_FEATURE_armv6t2)))
2220                         goto do_generic_bsf_bsr_popcnt;
2221 #endif
2222                 if (alu == ALU1_POPCNT && unlikely(!cpu_test_feature(CPU_FEATURE_neon)))
2223                         goto do_generic_bsf_bsr_popcnt;
2224                 g(gen_frame_get(ctx, op_size, mode == MODE_INT ? sign_x : zero_x, slot_1, R_SCRATCH_1, &reg1));
2225                 if (mode == MODE_INT) {
2226                         g(gen_cmp_test_jmp(ctx, INSN_TEST, i_size(op_size), reg1, reg1, alu == ALU1_BSR ? COND_LE : alu == ALU1_BSF ? COND_E : COND_S, label_ovf));
2227                 }
2228
2229                 if (alu == ALU1_POPCNT) {
2230                         g(gen_mov(ctx, OP_SIZE_NATIVE, FR_SCRATCH_1, reg1));
2231                         gen_insn(INSN_FP_ALU1, OP_SIZE_NATIVE, FP_ALU1_VCNT8, 0);
2232                         gen_one(FR_SCRATCH_1);
2233                         gen_one(FR_SCRATCH_1);
2234 #if defined(ARCH_ARM32)
2235                         if (op_size > OP_SIZE_1) {
2236                                 gen_insn(INSN_FP_ALU1, OP_SIZE_1, FP_ALU1_VPADDL, 0);
2237                                 gen_one(FR_SCRATCH_1);
2238                                 gen_one(FR_SCRATCH_1);
2239                         }
2240                         if (op_size > OP_SIZE_2) {
2241                                 gen_insn(INSN_FP_ALU1, OP_SIZE_2, FP_ALU1_VPADDL, 0);
2242                                 gen_one(FR_SCRATCH_1);
2243                                 gen_one(FR_SCRATCH_1);
2244                         }
2245 #else
2246                         if (op_size > OP_SIZE_1) {
2247                                 gen_insn(INSN_FP_ALU1, OP_SIZE_1, FP_ALU1_ADDV, 0);
2248                                 gen_one(FR_SCRATCH_1);
2249                                 gen_one(FR_SCRATCH_1);
2250                         }
2251 #endif
2252                         g(gen_frame_store(ctx, op_size, slot_r, 0, FR_SCRATCH_1));
2253                         if (slot_is_register(ctx, slot_r))
2254                                 g(unspill(ctx, slot_r));
2255                         return true;
2256                 }
2257
2258                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
2259                 if (mode == MODE_FIXED && alu == ALU1_BSF) {
2260                         gen_insn(INSN_TEST, i_size(op_size), 0, 1);
2261                         gen_one(target);
2262                         gen_one(reg1);
2263                         reg1 = target;
2264                 }
2265
2266                 if (alu == ALU1_BSF) {
2267                         g(gen_2address_alu1(ctx, i_size(op_size), ALU1_BREV, target, reg1, 0));
2268                         reg1 = target;
2269                 }
2270
2271                 g(gen_2address_alu1(ctx, i_size(op_size), ALU1_LZCNT, target, reg1, 0));
2272
2273                 if (alu == ALU1_BSR) {
2274                         g(gen_load_constant(ctx, R_SCRATCH_2, op_size == OP_SIZE_8 ? 63 : 31));
2275                         g(gen_3address_alu(ctx, i_size(op_size), ALU_SUB, target, R_SCRATCH_2, target, 0));
2276                 }
2277
2278                 if (mode == MODE_FIXED && alu == ALU1_BSF) {
2279 #if defined(ARCH_ARM32)
2280                         g(gen_imm(ctx, -1, IMM_PURPOSE_CMOV, OP_SIZE_NATIVE));
2281                         gen_insn(INSN_CMOV, OP_SIZE_NATIVE, COND_E, 0);
2282                         gen_one(target);
2283                         gen_one(target);
2284                         gen_imm_offset();
2285 #else
2286                         gen_insn(INSN_CSEL_INV, i_size(op_size), COND_NE, 0);
2287                         gen_one(target);
2288                         gen_one(ARG_IMM);
2289                         gen_eight(0);
2290                         gen_one(target);
2291 #endif
2292                 }
2293
2294                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2295                 return true;
2296 #endif
2297 #if defined(ARCH_ALPHA)
2298                 if (likely(cpu_test_feature(CPU_FEATURE_cix))) {
2299                         g(gen_frame_get(ctx, op_size, mode == MODE_INT ? sign_x : zero_x, slot_1, R_SCRATCH_1, &reg1));
2300                         target = gen_frame_target(ctx, slot_r, slot_1, NO_FRAME_T, R_SCRATCH_2);
2301                         if (mode == MODE_INT) {
2302                                 g(gen_cmp_test_jmp(ctx, INSN_TEST, OP_SIZE_NATIVE, reg1, reg1, alu == ALU1_BSR ? COND_LE : alu == ALU1_BSF ? COND_E : COND_S, label_ovf));
2303                         }
2304                         if (alu == ALU1_POPCNT) {
2305                                 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_POPCNT, target, reg1, 0));
2306                         }
2307                         if (alu == ALU1_BSF) {
2308                                 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_BSF, target, reg1, 0));
2309
2310                                 if (mode == MODE_FIXED) {
2311                                         g(gen_imm(ctx, -1, IMM_PURPOSE_MOVR, OP_SIZE_INT));
2312                                         gen_insn(INSN_MOVR, OP_SIZE_NATIVE, COND_E, 0);
2313                                         gen_one(target);
2314                                         gen_one(target);
2315                                         gen_one(reg1);
2316                                         gen_imm_offset();
2317                                 }
2318                         }
2319                         if (alu == ALU1_BSR) {
2320                                 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_LZCNT, target, reg1, 0));
2321
2322                                 g(gen_load_constant(ctx, R_SCRATCH_3, OP_SIZE_NATIVE == OP_SIZE_8 ? 63 : 31));
2323
2324                                 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_SUB, target, R_SCRATCH_3, target, 0));
2325                         }
2326                         g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2327                         return true;
2328                 }
2329 #endif
2330 #if defined(ARCH_MIPS)
2331                 if (MIPS_HAS_CLZ && alu != ALU1_POPCNT) {
2332                         g(gen_frame_get(ctx, op_size, mode == MODE_INT ? sign_x : zero_x, slot_1, R_SCRATCH_1, &reg1));
2333                         target = gen_frame_target(ctx, slot_r, slot_1, NO_FRAME_T, R_SCRATCH_2);
2334                         if (mode == MODE_INT) {
2335                                 g(gen_cmp_test_jmp(ctx, INSN_TEST, OP_SIZE_NATIVE, reg1, reg1, alu == ALU1_BSR ? COND_LE : alu == ALU1_BSF ? COND_E : COND_S, label_ovf));
2336                         }
2337                         if (alu == ALU1_BSF) {
2338                                 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NEG, target, reg1, 0));
2339
2340                                 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_AND, R_SCRATCH_1, reg1, target, 0));
2341                                 reg1 = R_SCRATCH_1;
2342                         }
2343                         g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_LZCNT, target, reg1, 0));
2344
2345                         g(gen_load_constant(ctx, R_SCRATCH_3, OP_SIZE_NATIVE == OP_SIZE_8 ? 63 : 31));
2346
2347                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_SUB, target, R_SCRATCH_3, target, 0));
2348
2349                         g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2350                         return true;
2351                 }
2352 #endif
2353 #if defined(ARCH_POWER)
2354                 if (alu == ALU1_BSF && (unlikely(!cpu_test_feature(CPU_FEATURE_v203)) || unlikely(!cpu_test_feature(CPU_FEATURE_v30))))
2355                         goto do_generic_bsf_bsr_popcnt;
2356                 if (alu == ALU1_POPCNT && unlikely(!cpu_test_feature(CPU_FEATURE_v206)))
2357                         goto do_generic_bsf_bsr_popcnt;
2358                 g(gen_frame_get(ctx, op_size, mode == MODE_INT ? sign_x : zero_x, slot_1, R_SCRATCH_1, &reg1));
2359                 target = gen_frame_target(ctx, slot_r, slot_1, NO_FRAME_T, R_SCRATCH_2);
2360                 if (mode == MODE_INT) {
2361                         g(gen_cmp_test_jmp(ctx, INSN_TEST, i_size(op_size), reg1, reg1, alu == ALU1_BSR ? COND_LE : alu == ALU1_BSF ? COND_E : COND_S, label_ovf));
2362                 }
2363                 if (alu == ALU1_POPCNT) {
2364                         g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_POPCNT, target, reg1, 0));
2365                 }
2366                 if (alu == ALU1_BSF) {
2367                         g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_BSF, target, reg1, 0));
2368
2369                         if (mode == MODE_FIXED) {
2370                                 g(gen_3address_alu(ctx, i_size(op_size), ALU_AND, R_SCRATCH_3, reg1, reg1, 1));
2371
2372                                 g(gen_imm(ctx, -1, IMM_PURPOSE_CMOV, OP_SIZE_NATIVE));
2373                                 gen_insn(INSN_CMOV, OP_SIZE_NATIVE, COND_E, 0);
2374                                 gen_one(target);
2375                                 gen_one(target);
2376                                 gen_imm_offset();
2377                         }
2378                 }
2379                 if (alu == ALU1_BSR) {
2380                         g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_LZCNT, target, reg1, 0));
2381
2382                         g(gen_load_constant(ctx, R_SCRATCH_3, OP_SIZE_NATIVE == OP_SIZE_8 ? 63 : 31));
2383
2384                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_SUB, target, R_SCRATCH_3, target, 0));
2385                 }
2386                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2387                 return true;
2388 #endif
2389 #if defined(ARCH_LOONGARCH64) || defined(ARCH_RISCV64)
2390 #if defined(ARCH_LOONGARCH64)
2391                 if (alu == ALU1_POPCNT)
2392                         goto do_generic_bsf_bsr_popcnt;
2393 #endif
2394 #if defined(ARCH_RISCV64)
2395                 if (unlikely(!cpu_test_feature(CPU_FEATURE_zbb)))
2396                         goto do_generic_bsf_bsr_popcnt;
2397 #endif
2398                 g(gen_frame_get(ctx, op_size, mode == MODE_INT ? sign_x : zero_x, slot_1, R_SCRATCH_1, &reg1));
2399                 target = gen_frame_target(ctx, slot_r, slot_1, NO_FRAME_T, R_SCRATCH_2);
2400                 if (mode == MODE_INT) {
2401                         g(gen_cmp_test_jmp(ctx, INSN_TEST, OP_SIZE_NATIVE, reg1, reg1, alu == ALU1_BSR ? COND_LE : alu == ALU1_BSF ? COND_E : COND_S, label_ovf));
2402                 }
2403                 if (alu == ALU1_POPCNT) {
2404                         g(gen_2address_alu1(ctx, maximum(OP_SIZE_4, op_size), ALU1_POPCNT, target, reg1, 0));
2405                 }
2406                 if (alu == ALU1_BSF) {
2407                         g(gen_2address_alu1(ctx, maximum(OP_SIZE_4, op_size), ALU1_BSF, target, reg1, 0));
2408
2409                         if (mode == MODE_FIXED) {
2410                                 g(gen_imm(ctx, 1, IMM_PURPOSE_CMP, OP_SIZE_NATIVE));
2411                                 gen_insn(INSN_CMP_DEST_REG, OP_SIZE_NATIVE, COND_B, 0);
2412                                 gen_one(R_SCRATCH_3);
2413                                 gen_one(reg1);
2414                                 gen_imm_offset();
2415
2416                                 g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_NEG, R_SCRATCH_3, R_SCRATCH_3, 0));
2417
2418                                 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_OR, target, target, R_SCRATCH_3, 0));
2419                         }
2420                 }
2421                 if (alu == ALU1_BSR) {
2422                         g(gen_2address_alu1(ctx, maximum(OP_SIZE_4, op_size), ALU1_LZCNT, target, reg1, 0));
2423
2424                         g(gen_load_constant(ctx, R_SCRATCH_3, op_size <= OP_SIZE_4 ? 31 : 63));
2425
2426                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_SUB, target, R_SCRATCH_3, target, 0));
2427                 }
2428                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2429                 return true;
2430 #endif
2431 #if defined(ARCH_IA64) || defined(ARCH_S390) || defined(ARCH_SPARC)
2432                 if (alu == ALU1_BSF && !ARCH_HAS_ANDN)
2433                         goto do_generic_bsf_bsr_popcnt;
2434 #if defined(ARCH_S390)
2435                 if (!cpu_test_feature(CPU_FEATURE_misc_45) || !cpu_test_feature(CPU_FEATURE_misc_insn_ext_3))
2436                         goto do_generic_bsf_bsr_popcnt;
2437 #endif
2438 #if defined(ARCH_SPARC)
2439                 if (!SPARC_9)
2440                         goto do_generic_bsf_bsr_popcnt;
2441 #endif
2442                 g(gen_frame_get(ctx, op_size, mode == MODE_INT ? sign_x : zero_x, slot_1, R_SCRATCH_1, &reg1));
2443                 target = gen_frame_target(ctx, slot_r, slot_1, NO_FRAME_T, R_SCRATCH_2);
2444                 if (mode == MODE_INT) {
2445                         g(gen_cmp_test_jmp(ctx, INSN_TEST, maximum(op_size, OP_SIZE_4), reg1, reg1, alu == ALU1_BSR ? COND_LE : alu == ALU1_BSF ? COND_E : COND_S, label_ovf));
2446                 } else {
2447                         if (ARCH_PREFERS_SX(op_size) && alu == ALU1_POPCNT && op_size < OP_SIZE_NATIVE) {
2448                                 g(gen_extend(ctx, op_size, zero_x, R_SCRATCH_1, reg1));
2449                                 reg1 = R_SCRATCH_1;
2450                         }
2451                 }
2452                 if (alu == ALU1_POPCNT) {
2453                         g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_POPCNT, R_SCRATCH_1, reg1, 0));
2454                         g(gen_frame_store(ctx, op_size, slot_r, 0, R_SCRATCH_1));
2455                         return true;
2456                 }
2457                 if (alu == ALU1_BSF) {
2458                         g(gen_3address_alu_imm(ctx, OP_SIZE_NATIVE, ALU_SUB, target, reg1, 1, 0));
2459
2460                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_ANDN, target, target, reg1, 0));
2461
2462                         g(gen_2address_alu1(ctx, OP_SIZE_NATIVE, ALU1_POPCNT, target, target, 0));
2463
2464                         if (mode == MODE_FIXED) {
2465                                 unsigned attr_unused test_reg = R_SCRATCH_1;
2466 #if defined(ARCH_S390)
2467                                 g(gen_imm(ctx, 0, COND_IS_LOGICAL(COND_E) ? IMM_PURPOSE_CMP_LOGICAL : IMM_PURPOSE_CMP, OP_SIZE_NATIVE));
2468                                 gen_insn(INSN_CMP, OP_SIZE_NATIVE, 0, 1 + COND_IS_LOGICAL(COND_E));
2469                                 gen_one(reg1);
2470                                 gen_imm_offset();
2471
2472                                 g(gen_imm(ctx, -1, IMM_PURPOSE_CMOV, OP_SIZE_NATIVE));
2473                                 gen_insn(INSN_CMOV, OP_SIZE_NATIVE, COND_E, 0);
2474                                 gen_one(target);
2475                                 gen_one(target);
2476                                 gen_imm_offset();
2477 #else
2478 #if defined(ARCH_IA64)
2479                                 g(gen_cmp_dest_reg(ctx, OP_SIZE_NATIVE, reg1, (unsigned)-1, R_CMP_RESULT, 0, COND_NE));
2480                                 test_reg = R_CMP_RESULT;
2481 #endif
2482                                 g(gen_imm(ctx, -1, IMM_PURPOSE_MOVR, OP_SIZE_NATIVE));
2483                                 gen_insn(INSN_MOVR, OP_SIZE_NATIVE, COND_E, 0);
2484                                 gen_one(target);
2485                                 gen_one(target);
2486                                 gen_one(test_reg);
2487                                 gen_imm_offset();
2488 #endif
2489                         }
2490
2491                         g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2492                         return true;
2493                 }
2494 #endif
2495 do_generic_bsf_bsr_popcnt:
2496                 if (alu == ALU1_BSF) {
2497                         if (mode == MODE_FIXED)
2498                                 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_unary_bsf_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, 0);
2499                         else
2500                                 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, INT_unary_bsf_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, label_ovf);
2501                 }
2502                 if (alu == ALU1_BSR) {
2503                         if (mode == MODE_FIXED)
2504                                 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_unary_bsr_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, 0);
2505                         else
2506                                 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, INT_unary_bsr_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, label_ovf);
2507                 }
2508                 if (alu == ALU1_POPCNT) {
2509                         if (mode == MODE_FIXED)
2510                                 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_unary_popcnt_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, 0);
2511                         else
2512                                 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, INT_unary_popcnt_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, label_ovf);
2513                 }
2514         }
2515         /**************
2516          * CONVERSION *
2517          **************/
2518 do_fixed_conv:
2519 do_conv: {
2520                 unsigned src_op_size, dest_op_size;
2521                 const struct type *src_type, *dest_type;
2522                 src_type = get_type_of_local(ctx, slot_1);
2523                 dest_type = get_type_of_local(ctx, slot_r);
2524
2525                 if (TYPE_TAG_IS_FIXED(src_type->tag)) {
2526                         src_op_size = TYPE_TAG_IDX_FIXED(src_type->tag) >> 1;
2527                 } else {
2528                         src_op_size = TYPE_TAG_IDX_INT(src_type->tag);
2529                 }
2530
2531                 if (TYPE_TAG_IS_FIXED(dest_type->tag)) {
2532                         dest_op_size = TYPE_TAG_IDX_FIXED(dest_type->tag) >> 1;
2533                 } else {
2534                         dest_op_size = TYPE_TAG_IDX_INT(dest_type->tag);
2535                 }
2536
2537                 if (src_op_size <= OP_SIZE_NATIVE) {
2538                         g(gen_frame_get(ctx, src_op_size, sign_x, slot_1, R_SCRATCH_1, &reg1));
2539                 } else {
2540 #if defined(ARCH_X86)
2541                         if (dest_op_size < src_op_size)
2542                                 g(gen_frame_load(ctx, OP_SIZE_NATIVE, garbage, slot_1, 0, false, R_SCRATCH_1));
2543                         else
2544 #endif
2545                                 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_3));
2546                         reg1 = R_SCRATCH_1;
2547                 }
2548
2549                 if (dest_op_size >= src_op_size) {
2550                         if (dest_op_size <= OP_SIZE_NATIVE) {
2551                                 g(gen_frame_store(ctx, dest_op_size, slot_r, 0, reg1));
2552                         } else {
2553                                 if (src_op_size <= OP_SIZE_NATIVE) {
2554 #if defined(ARCH_X86)
2555                                         if (R_SCRATCH_1 != R_AX || R_SCRATCH_2 != R_DX)
2556                                                 internal(file_line, "gen_alu1: bad scratch registers");
2557                                         if (reg1 == R_SCRATCH_1) {
2558                                                 gen_insn(INSN_CWD, OP_SIZE_NATIVE, 0, 0);
2559                                                 gen_one(R_DX);
2560                                                 gen_one(R_AX);
2561                                         } else
2562 #endif
2563                                         g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_2, reg1, (1U << (OP_SIZE_NATIVE + 3)) - 1, false));
2564                                         g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, reg1, R_SCRATCH_2));
2565                                 } else {
2566                                         g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, reg1, R_SCRATCH_3));
2567                                 }
2568                         }
2569                         return true;
2570                 } else {
2571                         if (src_op_size > OP_SIZE_NATIVE) {
2572 #if defined(ARCH_ARM)
2573                                 gen_insn(INSN_CMP, OP_SIZE_NATIVE, 0, 1);
2574                                 gen_one(R_SCRATCH_3);
2575                                 gen_one(ARG_SHIFTED_REGISTER);
2576                                 gen_one(ARG_SHIFT_ASR | ((1U << (OP_SIZE_NATIVE + 3)) - 1));
2577                                 gen_one(R_SCRATCH_1);
2578
2579                                 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_NE, 0);
2580                                 gen_four(label_ovf);
2581 #elif defined(ARCH_X86)
2582                                 if (R_SCRATCH_1 != R_AX || R_SCRATCH_2 != R_DX)
2583                                         internal(file_line, "gen_alu1: bad scratch registers");
2584                                 gen_insn(INSN_CWD, OP_SIZE_NATIVE, 0, 0);
2585                                 gen_one(R_DX);
2586                                 gen_one(R_AX);
2587
2588                                 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size + hi_word(OP_SIZE_NATIVE), IMM_PURPOSE_LDR_OFFSET, OP_SIZE_NATIVE));
2589                                 gen_insn(INSN_CMP, OP_SIZE_NATIVE, 0, 1);
2590                                 gen_one(R_SCRATCH_2);
2591                                 gen_address_offset();
2592
2593                                 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, COND_NE, 0);
2594                                 gen_four(label_ovf);
2595 #else
2596                                 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SAR, R_SCRATCH_2, R_SCRATCH_1, (1U << (OP_SIZE_NATIVE + 3)) - 1, 0));
2597
2598                                 g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, R_SCRATCH_3, R_SCRATCH_2, COND_NE, label_ovf));
2599 #endif
2600
2601                                 src_op_size = OP_SIZE_NATIVE;
2602                         }
2603                         if (src_op_size > dest_op_size) {
2604                                 g(gen_cmp_extended(ctx, OP_SIZE_NATIVE, dest_op_size, reg1, R_SCRATCH_2, label_ovf));
2605                         }
2606                         g(gen_frame_store(ctx, dest_op_size, slot_r, 0, reg1));
2607                         return true;
2608                 }
2609         }
2610
2611 conv_uto_int: {
2612                 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_uto_int_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, label_ovf);
2613         }
2614
2615 conv_ufrom_int: {
2616                 return gen_alu_typed_upcall(ctx, offsetof(struct cg_upcall_vector_s, FIXED_ufrom_int_int8_t), op_size, slot_1, NO_FRAME_T, slot_r, label_ovf);
2617         }
2618 }
2619
2620 static bool attr_w gen_constant(struct codegen_context *ctx, bool real, unsigned op_size, bool shrt, frame_t slot_r)
2621 {
2622         uintbig_t c;
2623         if (shrt) {
2624                 c = (int16_t)get_unaligned_16(ctx->current_position);
2625         } else switch (op_size) {
2626 #define fx(n, type, utype, sz, bits)                                    \
2627                 case n:                                                 \
2628                         c = (type)cat(get_unaligned_,bits)(ctx->current_position);\
2629                         break;
2630                 for_all_fixed(fx);
2631 #undef fx
2632                 default:
2633                         internal(file_line, "gen_constant: invalid type %u", op_size);
2634         }
2635         if (op_size > OP_SIZE_NATIVE) {
2636                 unsigned shift = (8U << OP_SIZE_NATIVE) - 1;
2637                 g(gen_frame_store_imm_raw(ctx, OP_SIZE_NATIVE, slot_r, lo_word(OP_SIZE_NATIVE), c & ((2ULL << shift) - 1)));
2638                 g(gen_frame_store_imm_raw(ctx, OP_SIZE_NATIVE, slot_r, hi_word(OP_SIZE_NATIVE), c >> 1 >> shift));
2639                 if (real && slot_is_register(ctx, slot_r))
2640                         g(unspill(ctx, slot_r));
2641                 return true;
2642         } else if (real && slot_is_register(ctx, slot_r)) {
2643                 if (ARCH_HAS_FP_GP_MOV) {
2644                         g(gen_load_constant(ctx, R_SCRATCH_1, c));
2645                         g(gen_mov(ctx, op_size, ctx->registers[slot_r], R_SCRATCH_1));
2646                 } else {
2647                         g(gen_frame_store_imm_raw(ctx, op_size, slot_r, 0, c));
2648                         g(unspill(ctx, slot_r));
2649                 }
2650         } else {
2651                 g(gen_frame_store_imm(ctx, op_size, slot_r, 0, c));
2652         }
2653         return true;
2654 }
2655
2656 static bool attr_w gen_real_constant(struct codegen_context *ctx, const struct type *t, frame_t slot_r)
2657 {
2658         int64_t offset;
2659         if (is_power_of_2(t->size) && t->size <= sizeof(uintbig_t))
2660                 return gen_constant(ctx, true, log_2(t->size), false, slot_r);
2661
2662         g(load_function_offset(ctx, R_SCRATCH_3, offsetof(struct data, u_.function.code)));
2663
2664         offset = (ctx->current_position - da(ctx->fn,function)->code) * sizeof(code_t);
2665
2666         g(gen_memcpy_raw(ctx, R_FRAME, (size_t)slot_r * slot_size, R_SCRATCH_3, offset, t->size, minimum(t->align, sizeof(code_t))));
2667         if (slot_is_register(ctx, slot_r))
2668                 g(unspill(ctx, slot_r));
2669
2670         return true;
2671 }
2672
2673 static bool attr_w gen_copy(struct codegen_context *ctx, unsigned op_size, frame_t slot_1, frame_t slot_r)
2674 {
2675         unsigned reg1;
2676         if (unlikely(op_size > OP_SIZE_NATIVE)) {
2677                 g(gen_frame_load_2(ctx, OP_SIZE_NATIVE, slot_1, 0, R_SCRATCH_1, R_SCRATCH_2));
2678                 g(gen_frame_store_2(ctx, OP_SIZE_NATIVE, slot_r, 0, R_SCRATCH_1, R_SCRATCH_2));
2679                 return true;
2680         } else {
2681                 unsigned target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
2682                 g(gen_frame_get(ctx, op_size, garbage, slot_1, target, &reg1));
2683                 g(gen_frame_store(ctx, op_size, slot_r, 0, reg1));
2684                 return true;
2685         }
2686 }
2687
2688 static bool attr_w gen_fp_alu(struct codegen_context *ctx, unsigned real_type, unsigned op, uint32_t label_ovf, frame_t slot_1, frame_t slot_2, frame_t slot_r)
2689 {
2690         unsigned attr_unused fp_alu;
2691         size_t upc;
2692         unsigned attr_unused op_size = real_type_to_op_size(real_type);
2693         unsigned reg1, reg2, target;
2694         switch (op) {
2695                 case OPCODE_REAL_OP_add:
2696                 case OPCODE_REAL_OP_add_alt1:
2697                 case OPCODE_REAL_OP_add_alt2: fp_alu = FP_ALU_ADD; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_add_real16_t); label_ovf = 0; goto do_alu;
2698                 case OPCODE_REAL_OP_subtract:
2699                 case OPCODE_REAL_OP_subtract_alt1:
2700                 case OPCODE_REAL_OP_subtract_alt2: fp_alu = FP_ALU_SUB; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_subtract_real16_t); label_ovf = 0; goto do_alu;
2701                 case OPCODE_REAL_OP_multiply:
2702                 case OPCODE_REAL_OP_multiply_alt1:
2703                 case OPCODE_REAL_OP_multiply_alt2: fp_alu = FP_ALU_MUL; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_multiply_real16_t); label_ovf = 0; goto do_alu;
2704                 case OPCODE_REAL_OP_divide:
2705                 case OPCODE_REAL_OP_divide_alt1:
2706                 case OPCODE_REAL_OP_divide_alt2: fp_alu = FP_ALU_DIV; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_divide_real16_t); label_ovf = 0; goto do_alu;
2707                 case OPCODE_REAL_OP_modulo:
2708                 case OPCODE_REAL_OP_power:
2709                 case OPCODE_REAL_OP_ldexp:
2710                 case OPCODE_REAL_OP_atan2: upc = offsetof(struct cg_upcall_vector_s, REAL_binary_modulo_real16_t) + (op - OPCODE_REAL_OP_modulo) * TYPE_REAL_N * sizeof(void (*)(void)); goto do_upcall;
2711                 case OPCODE_REAL_OP_equal:
2712                 case OPCODE_REAL_OP_equal_alt1:
2713                 case OPCODE_REAL_OP_equal_alt2: fp_alu = FP_COND_E; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_equal_real16_t); goto do_cmp;
2714                 case OPCODE_REAL_OP_not_equal:
2715                 case OPCODE_REAL_OP_not_equal_alt1:
2716                 case OPCODE_REAL_OP_not_equal_alt2: fp_alu = FP_COND_NE; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_not_equal_real16_t); goto do_cmp;
2717                 case OPCODE_REAL_OP_less:
2718                 case OPCODE_REAL_OP_less_alt1:
2719                 case OPCODE_REAL_OP_less_alt2: fp_alu = FP_COND_B; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_less_real16_t); goto do_cmp;
2720                 case OPCODE_REAL_OP_less_equal:
2721                 case OPCODE_REAL_OP_less_equal_alt1:
2722                 case OPCODE_REAL_OP_less_equal_alt2: fp_alu = FP_COND_BE; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_less_equal_real16_t); goto do_cmp;
2723                 case OPCODE_REAL_OP_greater:
2724                 case OPCODE_REAL_OP_greater_alt1:
2725                 case OPCODE_REAL_OP_greater_alt2: fp_alu = FP_COND_A; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_greater_real16_t); goto do_cmp;
2726                 case OPCODE_REAL_OP_greater_equal:
2727                 case OPCODE_REAL_OP_greater_equal_alt1:
2728                 case OPCODE_REAL_OP_greater_equal_alt2: fp_alu = FP_COND_AE; upc = offsetof(struct cg_upcall_vector_s, REAL_binary_greater_equal_real16_t); goto do_cmp;
2729                 default: internal(file_line, "gen_fp_alu: unsupported operation %u", op);
2730         }
2731
2732 do_alu:
2733         if ((SUPPORTED_FP >> real_type) & 1) {
2734 #if defined(ARCH_IA64)
2735                 if (unlikely(fp_alu == FP_ALU_DIV))
2736                         goto do_upcall;
2737 #endif
2738 #if defined(ARCH_X86)
2739                 if (1)
2740 #elif defined(ARCH_S390)
2741                 if ((op_size <= OP_SIZE_8 && (size_t)slot_2 * slot_size < 4096) || slot_is_register(ctx, slot_2))
2742 #else
2743                 if (slot_is_register(ctx, slot_2))
2744 #endif
2745                 {
2746                         target = gen_frame_target(ctx, slot_r, NO_FRAME_T, slot_2, FR_SCRATCH_1);
2747                         g(gen_frame_get(ctx, op_size, garbage, slot_1, FR_SCRATCH_1, &reg1));
2748                         if (slot_is_register(ctx, slot_2)) {
2749                                 g(gen_3address_fp_alu(ctx, op_size, fp_alu, target, reg1, ctx->registers[slot_2]));
2750                         } else {
2751                                 if (target != reg1 && !ARCH_IS_3ADDRESS_FP) {
2752                                         g(gen_mov(ctx, op_size, target, reg1));
2753                                         reg1 = target;
2754                                 }
2755                                 g(gen_address(ctx, R_FRAME, (size_t)slot_2 * slot_size, IMM_PURPOSE_VLDR_VSTR_OFFSET, op_size));
2756                                 gen_insn(INSN_FP_ALU, op_size, fp_alu, 0);
2757                                 gen_one(target);
2758                                 gen_one(reg1);
2759                                 gen_address_offset();
2760                         }
2761                         g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2762                         return true;
2763                 }
2764 #if defined(ARCH_ALPHA)
2765                 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, &reg1));
2766                 g(gen_frame_get(ctx, op_size, zero_x, slot_2, FR_SCRATCH_2, &reg2));
2767                 target = gen_frame_target(ctx, slot_r, slot_1, slot_2, FR_SCRATCH_3);
2768                 g(gen_3address_fp_alu(ctx, op_size, fp_alu, target, reg1, reg2));
2769                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2770 #else
2771                 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, &reg1));
2772                 g(gen_frame_get(ctx, op_size, zero_x, slot_2, FR_SCRATCH_2, &reg2));
2773                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_1);
2774                 g(gen_3address_fp_alu(ctx, op_size, fp_alu, target, reg1, reg2));
2775                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2776 #endif
2777                 return true;
2778         }
2779 #ifdef SUPPORTED_FP_X87
2780         if ((SUPPORTED_FP_X87 >> real_type) & 1) {
2781                 if (real_type != 3) {
2782                         g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_2));
2783                         g(gen_frame_load_x87(ctx, INSN_X87_ALU, op_size, fp_alu, slot_1));
2784                 } else {
2785                         g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
2786                         g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_2));
2787                         gen_insn(INSN_X87_ALUP, op_size, fp_alu, 0);
2788                         gen_one(R_ST1);
2789                 }
2790                 g(gen_frame_store_x87(ctx, INSN_X87_FSTP, op_size, slot_r));
2791                 return true;
2792         }
2793 #endif
2794 #ifdef SUPPORTED_FP_HALF_CVT
2795         if ((SUPPORTED_FP_HALF_CVT >> real_type) & 1) {
2796                 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, &reg1));
2797                 g(gen_frame_get(ctx, op_size, zero_x, slot_2, FR_SCRATCH_2, &reg2));
2798                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_1);
2799                 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
2800                 gen_one(FR_SCRATCH_1);
2801                 gen_one(reg1);
2802                 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
2803                 gen_one(FR_SCRATCH_2);
2804                 gen_one(reg2);
2805                 gen_insn(INSN_FP_ALU, OP_SIZE_4, fp_alu, 0);
2806                 gen_one(FR_SCRATCH_1);
2807                 gen_one(FR_SCRATCH_1);
2808                 gen_one(FR_SCRATCH_2);
2809                 gen_insn(INSN_FP_CVT, OP_SIZE_4, op_size, 0);
2810                 gen_one(target);
2811                 gen_one(FR_SCRATCH_1);
2812                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
2813                 return true;
2814         }
2815 #endif
2816         goto do_upcall;
2817
2818 do_cmp:
2819         if ((SUPPORTED_FP >> real_type) & 1
2820 #if defined(ARCH_ALPHA)
2821                 && ARCH_SUPPORTS_TRAPS
2822 #endif
2823         ) {
2824                 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, &reg1));
2825                 g(gen_frame_get(ctx, op_size, zero_x, slot_2, FR_SCRATCH_2, &reg2));
2826                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
2827 #if defined(ARCH_ALPHA)
2828                 gen_insn(INSN_FP_CMP_DEST_REG_TRAP, op_size, fp_alu == FP_COND_NE ? FP_COND_E : fp_alu, 0);
2829                 gen_one(FR_SCRATCH_3);
2830                 gen_one(reg1);
2831                 gen_one(reg2);
2832                 gen_four(label_ovf);
2833
2834                 if (!ARCH_HAS_FP_GP_MOV) {
2835                         g(gen_frame_store_raw(ctx, OP_SIZE_4, slot_r, 0, FR_SCRATCH_3));
2836                         g(gen_frame_load_raw(ctx, OP_SIZE_4, sign_x, slot_r, 0, false, target));
2837                 } else {
2838                         g(gen_mov(ctx, OP_SIZE_4, target, FR_SCRATCH_3));
2839                 }
2840
2841                 if (fp_alu == FP_COND_NE) {
2842                         g(gen_imm(ctx, 0, IMM_PURPOSE_CMP, OP_SIZE_NATIVE));
2843                         gen_insn(INSN_CMP_DEST_REG, OP_SIZE_NATIVE, COND_E, 0);
2844                         gen_one(target);
2845                         gen_one(target);
2846                         gen_imm_offset();
2847                 } else {
2848                         g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SHR, target, target, 30, 0));
2849                 }
2850
2851                 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
2852
2853                 return true;
2854 #elif defined(ARCH_IA64)
2855                 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_P, 0);
2856                 gen_one(R_CMP_RESULT);
2857                 gen_one(reg1);
2858                 gen_one(reg2);
2859
2860                 gen_insn(INSN_JMP_REG, OP_SIZE_NATIVE, COND_NE, 0);
2861                 gen_one(R_CMP_RESULT);
2862                 gen_four(label_ovf);
2863
2864                 gen_insn(INSN_FP_CMP_DEST_REG, op_size, fp_alu, 0);
2865                 gen_one(R_CMP_RESULT);
2866                 gen_one(reg1);
2867                 gen_one(reg2);
2868
2869                 g(gen_mov(ctx, OP_SIZE_NATIVE, target, R_CMP_RESULT));
2870
2871                 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
2872
2873                 return true;
2874 #elif defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS) || defined(ARCH_PARISC)
2875                 gen_insn(INSN_FP_CMP_COND, op_size, FP_COND_P, 1);
2876                 gen_one(reg1);
2877                 gen_one(reg2);
2878
2879                 gen_insn(INSN_JMP_FP_TEST, 0, FP_COND_P, 0);
2880                 gen_four(label_ovf);
2881
2882                 gen_insn(INSN_FP_CMP_COND, op_size, fp_alu, 1);
2883                 gen_one(reg1);
2884                 gen_one(reg2);
2885
2886                 gen_insn(INSN_FP_TEST_REG, OP_SIZE_NATIVE, fp_alu, 0);
2887                 gen_one(target);
2888
2889                 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
2890
2891                 return true;
2892 #elif defined(ARCH_RISCV64)
2893                 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_E, 0);
2894                 gen_one(R_SCRATCH_1);
2895                 gen_one(reg1);
2896                 gen_one(reg1);
2897
2898                 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_E, 0);
2899                 gen_one(R_SCRATCH_2);
2900                 gen_one(reg2);
2901                 gen_one(reg2);
2902
2903                 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_AND, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_2, 0));
2904
2905                 g(gen_jmp_on_zero(ctx, OP_SIZE_NATIVE, R_SCRATCH_1, COND_E, label_ovf));
2906
2907                 gen_insn(INSN_FP_CMP_DEST_REG, op_size, fp_alu == FP_COND_NE ? FP_COND_E : fp_alu, 0);
2908                 gen_one(target);
2909                 gen_one(reg1);
2910                 gen_one(reg2);
2911
2912                 if (fp_alu == FP_COND_NE) {
2913                         g(gen_imm(ctx, 1, IMM_PURPOSE_XOR, OP_SIZE_NATIVE));
2914                         gen_insn(INSN_ALU, OP_SIZE_NATIVE, ALU_XOR, ALU_WRITES_FLAGS(ALU_AND, false));
2915                         gen_one(target);
2916                         gen_one(target);
2917                         gen_imm_offset();
2918                 }
2919
2920                 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
2921                 return true;
2922 #else
2923                 gen_insn(INSN_FP_CMP, op_size, 0, 1);
2924                 gen_one(reg1);
2925                 gen_one(reg2);
2926 #if defined(ARCH_ARM32)
2927                 gen_insn(INSN_FP_TO_INT_FLAGS, 0, 0, 1);
2928 #endif
2929                 gen_insn(INSN_JMP_COND, op_size, FP_COND_P, 0);
2930                 gen_four(label_ovf);
2931                 g(gen_frame_set_cond(ctx, op_size, false, fp_alu, slot_r));
2932                 return true;
2933 #endif
2934         }
2935 #ifdef SUPPORTED_FP_X87
2936         if ((SUPPORTED_FP_X87 >> real_type) & 1) {
2937                 if (likely(cpu_test_feature(CPU_FEATURE_cmov))) {
2938                         g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_2));
2939                         g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
2940                         gen_insn(INSN_X87_FCOMIP, op_size, 0, 0);
2941                         gen_one(R_ST1);
2942                         gen_insn(INSN_X87_FSTP, op_size, 0, 0);
2943                         gen_one(R_ST0);
2944                         gen_insn(INSN_JMP_COND, op_size, COND_P, 0);
2945                         gen_four(label_ovf);
2946                         g(gen_frame_set_cond(ctx, op_size, false, fp_alu & 0xf, slot_r));
2947                         return true;
2948                 }
2949
2950                 if (real_type != 3) {
2951                         g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
2952                         g(gen_frame_load_x87(ctx, INSN_X87_FCOMP, op_size, 0, slot_2));
2953                 } else {
2954                         g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_2));
2955                         g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
2956                         gen_insn(INSN_X87_FCOMPP, op_size, 0, 0);
2957                 }
2958
2959                 gen_insn(INSN_X87_FNSTSW, 0, 0, 0);
2960                 gen_one(R_AX);
2961                 gen_one(R_AX);
2962
2963                 gen_insn(INSN_TEST, OP_SIZE_2, 0, 1);
2964                 gen_one(R_AX);
2965                 gen_one(ARG_IMM);
2966                 gen_eight(0x0400);
2967
2968                 gen_insn(INSN_JMP_COND, OP_SIZE_2, COND_NE, 0);
2969                 gen_four(label_ovf);
2970
2971                 switch (fp_alu) {
2972                         case FP_COND_E:
2973                                 gen_insn(INSN_TEST, OP_SIZE_2, 0, 1);
2974                                 gen_one(R_AX);
2975                                 gen_one(ARG_IMM);
2976                                 gen_eight(0x4000);
2977                                 g(gen_frame_set_cond(ctx, OP_SIZE_2, false, COND_NE, slot_r));
2978                                 break;
2979                         case FP_COND_NE:
2980                                 gen_insn(INSN_TEST, OP_SIZE_2, 0, 1);
2981                                 gen_one(R_AX);
2982                                 gen_one(ARG_IMM);
2983                                 gen_eight(0x4000);
2984                                 g(gen_frame_set_cond(ctx, OP_SIZE_2, false, COND_E, slot_r));
2985                                 break;
2986                         case FP_COND_B:
2987                                 gen_insn(INSN_TEST, OP_SIZE_2, 0, 1);
2988                                 gen_one(R_AX);
2989                                 gen_one(ARG_IMM);
2990                                 gen_eight(0x0100);
2991                                 g(gen_frame_set_cond(ctx, OP_SIZE_2, false, COND_NE, slot_r));
2992                                 break;
2993                         case FP_COND_BE:
2994                                 gen_insn(INSN_TEST, OP_SIZE_2, 0, 1);
2995                                 gen_one(R_AX);
2996                                 gen_one(ARG_IMM);
2997                                 gen_eight(0x4100);
2998                                 g(gen_frame_set_cond(ctx, OP_SIZE_2, false, COND_NE, slot_r));
2999                                 break;
3000                         default:
3001                                 internal(file_line, "gen_fp_alu: invalid condition %u", fp_alu);
3002                 }
3003                 return true;
3004         }
3005 #endif
3006 #ifdef SUPPORTED_FP_HALF_CVT
3007         if ((SUPPORTED_FP_HALF_CVT >> real_type) & 1) {
3008                 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, &reg1));
3009                 g(gen_frame_get(ctx, op_size, zero_x, slot_2, FR_SCRATCH_2, &reg2));
3010                 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
3011                 gen_one(FR_SCRATCH_1);
3012                 gen_one(reg1);
3013                 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
3014                 gen_one(FR_SCRATCH_2);
3015                 gen_one(reg2);
3016                 gen_insn(INSN_FP_CMP, OP_SIZE_4, 0, 1);
3017                 gen_one(FR_SCRATCH_1);
3018                 gen_one(FR_SCRATCH_2);
3019 #if defined(ARCH_ARM32)
3020                 gen_insn(INSN_FP_TO_INT_FLAGS, 0, 0, 1);
3021 #endif
3022                 gen_insn(INSN_JMP_COND, op_size, FP_COND_P, 0);
3023                 gen_four(label_ovf);
3024                 g(gen_frame_set_cond(ctx, op_size, false, fp_alu, slot_r));
3025                 return true;
3026         }
3027 #endif
3028
3029 do_upcall:
3030         return gen_alu_typed_upcall(ctx, upc, real_type, slot_1, slot_2, slot_r, label_ovf);
3031 }
3032
3033 #define OP_IS_ROUND(alu)        ((alu) == FP_ALU1_ROUND || (alu) == FP_ALU1_FLOOR || (alu) == FP_ALU1_CEIL || (alu) == FP_ALU1_TRUNC)
3034
3035 static bool attr_w gen_fp_alu1(struct codegen_context *ctx, unsigned real_type, unsigned op, uint32_t label_ovf, frame_t slot_1, frame_t slot_r)
3036 {
3037         unsigned attr_unused fp_alu;
3038         size_t upc;
3039         unsigned attr_unused op_size = real_type_to_op_size(real_type);
3040         unsigned reg1, target;
3041         switch (op) {
3042                 case OPCODE_REAL_OP_neg:
3043                 case OPCODE_REAL_OP_neg_alt1:
3044                 case OPCODE_REAL_OP_neg_alt2: fp_alu = FP_ALU1_NEG; upc = offsetof(struct cg_upcall_vector_s, REAL_unary_neg_real16_t); label_ovf = 0; goto do_alu;
3045                 case OPCODE_REAL_OP_sqrt:
3046                 case OPCODE_REAL_OP_sqrt_alt1:
3047                 case OPCODE_REAL_OP_sqrt_alt2: fp_alu = FP_ALU1_SQRT; upc = offsetof(struct cg_upcall_vector_s, REAL_unary_sqrt_real16_t); label_ovf = 0; goto do_alu;
3048                 case OPCODE_REAL_OP_round: fp_alu = FP_ALU1_ROUND; upc = offsetof(struct cg_upcall_vector_s, REAL_unary_round_real16_t); label_ovf = 0; goto do_alu;
3049                 case OPCODE_REAL_OP_floor: fp_alu = FP_ALU1_FLOOR; upc = offsetof(struct cg_upcall_vector_s, REAL_unary_floor_real16_t); label_ovf = 0; goto do_alu;
3050                 case OPCODE_REAL_OP_ceil: fp_alu = FP_ALU1_CEIL; upc = offsetof(struct cg_upcall_vector_s, REAL_unary_ceil_real16_t); label_ovf = 0; goto do_alu;
3051                 case OPCODE_REAL_OP_trunc: fp_alu = FP_ALU1_TRUNC; upc = offsetof(struct cg_upcall_vector_s, REAL_unary_trunc_real16_t); label_ovf = 0; goto do_alu;
3052                 case OPCODE_REAL_OP_to_int:
3053                 case OPCODE_REAL_OP_to_int_alt1:
3054                 case OPCODE_REAL_OP_to_int_alt2: upc = offsetof(struct cg_upcall_vector_s, REAL_unary_to_int_real16_t); goto do_to_int;
3055                 case OPCODE_REAL_OP_from_int:
3056                 case OPCODE_REAL_OP_from_int_alt1:
3057                 case OPCODE_REAL_OP_from_int_alt2: upc = offsetof(struct cg_upcall_vector_s, REAL_unary_from_int_real16_t); label_ovf = 0; goto do_from_int;
3058                 case OPCODE_REAL_OP_is_exception:
3059                 case OPCODE_REAL_OP_is_exception_alt1:
3060                 case OPCODE_REAL_OP_is_exception_alt2: upc = offsetof(struct cg_upcall_vector_s, REAL_unary_is_exception_real16_t); label_ovf = 0; goto do_is_exception;
3061                 default: upc = offsetof(struct cg_upcall_vector_s, REAL_unary_cbrt_real16_t) + (op - OPCODE_REAL_OP_cbrt) * TYPE_REAL_N * sizeof(void (*)(void)); label_ovf = 0; goto do_upcall;
3062         }
3063
3064 do_alu:
3065         if ((SUPPORTED_FP >> real_type) & 1 && (
3066 #if defined(ARCH_ALPHA)
3067                 fp_alu == FP_ALU1_NEG ||
3068                 (fp_alu == FP_ALU1_SQRT && cpu_test_feature(CPU_FEATURE_fix)) ||
3069 #elif defined(ARCH_ARM32)
3070                 fp_alu == FP_ALU1_NEG ||
3071                 fp_alu == FP_ALU1_SQRT ||
3072 #elif defined(ARCH_ARM64)
3073                 true ||
3074 #elif defined(ARCH_IA64)
3075                 fp_alu == FP_ALU1_NEG ||
3076 #elif defined(ARCH_LOONGARCH64)
3077                 fp_alu == FP_ALU1_NEG ||
3078                 fp_alu == FP_ALU1_SQRT ||
3079                 fp_alu == FP_ALU1_ROUND ||
3080 #elif defined(ARCH_MIPS)
3081                 fp_alu == FP_ALU1_NEG ||
3082                 (fp_alu == FP_ALU1_SQRT && MIPS_HAS_SQRT) ||
3083 #elif defined(ARCH_PARISC)
3084                 (fp_alu == FP_ALU1_NEG && PA_20) ||
3085                 fp_alu == FP_ALU1_SQRT ||
3086 #elif defined(ARCH_POWER)
3087                 fp_alu == FP_ALU1_NEG ||
3088                 (fp_alu == FP_ALU1_SQRT && cpu_test_feature(CPU_FEATURE_p2) && real_type != 4) ||
3089 #elif defined(ARCH_S390)
3090                 true ||
3091 #elif defined(ARCH_SPARC)
3092                 fp_alu == FP_ALU1_NEG ||
3093                 fp_alu == FP_ALU1_SQRT ||
3094 #elif defined(ARCH_RISCV64)
3095                 fp_alu == FP_ALU1_NEG ||
3096                 fp_alu == FP_ALU1_SQRT ||
3097 #elif defined(ARCH_X86)
3098                 fp_alu == FP_ALU1_SQRT ||
3099                 (OP_IS_ROUND(fp_alu) && cpu_test_feature(CPU_FEATURE_sse41)) ||
3100 #endif
3101                 false)) {
3102 #if defined(ARCH_S390)
3103                 if (op_size <= OP_SIZE_8 && (size_t)slot_1 * slot_size < 4096 && fp_alu == FP_ALU1_SQRT) {
3104                         target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_1);
3105                         if (slot_is_register(ctx, slot_1)) {
3106                                 gen_insn(INSN_FP_ALU1, op_size, fp_alu, 0);
3107                                 gen_one(target);
3108                                 gen_one(ctx->registers[slot_1]);
3109                                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
3110                         } else {
3111                                 g(gen_address(ctx, R_FRAME, (size_t)slot_1 * slot_size, IMM_PURPOSE_VLDR_VSTR_OFFSET, op_size));
3112                                 gen_insn(INSN_FP_ALU1, op_size, fp_alu, 0);
3113                                 gen_one(target);
3114                                 gen_address_offset();
3115                                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
3116                         }
3117                         return true;
3118                 }
3119 #endif
3120                 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, &reg1));
3121                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_2);
3122                 gen_insn(INSN_FP_ALU1, op_size, fp_alu, 0);
3123                 gen_one(target);
3124                 gen_one(reg1);
3125                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
3126                 return true;
3127         }
3128 #ifdef SUPPORTED_FP_X87
3129         if ((SUPPORTED_FP_X87 >> real_type) & 1) {
3130                 if (fp_alu == FP_ALU1_NEG) {
3131                         g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
3132                         gen_insn(INSN_X87_FCHS, op_size, 0, 0);
3133                         g(gen_frame_store_x87(ctx, INSN_X87_FSTP, op_size, slot_r));
3134                         return true;
3135                 } else if (fp_alu == FP_ALU1_SQRT) {
3136                         g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
3137                         gen_insn(INSN_X87_FSQRT, op_size, 0, 0);
3138                         g(gen_frame_store_x87(ctx, INSN_X87_FSTP, op_size, slot_r));
3139                         return true;
3140                 } else if (fp_alu == FP_ALU1_ROUND) {
3141                         g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
3142                         gen_insn(INSN_X87_FRNDINT, op_size, 0, 0);
3143                         g(gen_frame_store_x87(ctx, INSN_X87_FSTP, op_size, slot_r));
3144                         return true;
3145                 }
3146         }
3147 #endif
3148 #ifdef SUPPORTED_FP_HALF_CVT
3149         if ((SUPPORTED_FP_HALF_CVT >> real_type) & 1 && (
3150 #if defined(ARCH_ARM32)
3151                 fp_alu == FP_ALU1_NEG ||
3152                 fp_alu == FP_ALU1_SQRT ||
3153 #elif defined(ARCH_ARM64)
3154                 true ||
3155 #elif defined(ARCH_X86)
3156                 fp_alu == FP_ALU1_SQRT ||
3157                 (OP_IS_ROUND(fp_alu) && cpu_test_feature(CPU_FEATURE_sse41)) ||
3158 #endif
3159                 false)) {
3160                 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, &reg1));
3161                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_1);
3162                 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
3163                 gen_one(target);
3164                 gen_one(reg1);
3165                 gen_insn(INSN_FP_ALU1, OP_SIZE_4, fp_alu, 0);
3166                 gen_one(target);
3167                 gen_one(target);
3168                 gen_insn(INSN_FP_CVT, OP_SIZE_4, op_size, 0);
3169                 gen_one(target);
3170                 gen_one(target);
3171                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
3172                 return true;
3173         }
3174 #endif
3175         goto do_upcall;
3176
3177 do_to_int:
3178         if ((SUPPORTED_FP >> real_type) & 1
3179 #if defined(ARCH_ALPHA)
3180                 && ARCH_SUPPORTS_TRAPS
3181 #endif
3182 #if defined(ARCH_MIPS)
3183                 && MIPS_HAS_TRUNC
3184 #endif
3185         ) {
3186                 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, &reg1));
3187                 goto do_cvt_to_int;
3188 do_cvt_to_int:
3189                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
3190 #if defined(ARCH_X86)
3191                 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_TO_INT32 : INSN_FP_TO_INT64, op_size, 0, 0);
3192                 gen_one(target);
3193                 gen_one(reg1);
3194
3195                 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, OP_SIZE_INT, target, sign_bit(uint_default_t), COND_E, label_ovf));
3196
3197                 g(gen_frame_store(ctx, OP_SIZE_INT, slot_r, 0, target));
3198                 return true;
3199 #endif
3200 #if defined(ARCH_ARM) || defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS)
3201 #if defined(ARCH_ARM)
3202                 gen_insn(INSN_FP_CMP, op_size, 0, 1);
3203                 gen_one(reg1);
3204                 gen_one(reg1);
3205 #if defined(ARCH_ARM32)
3206                 gen_insn(INSN_FP_TO_INT_FLAGS, 0, 0, 1);
3207 #endif
3208                 gen_insn(INSN_JMP_COND, op_size, FP_COND_P, 0);
3209                 gen_four(label_ovf);
3210 #else
3211                 gen_insn(INSN_FP_CMP_COND, op_size, FP_COND_P, 1);
3212                 gen_one(reg1);
3213                 gen_one(reg1);
3214
3215                 gen_insn(INSN_JMP_FP_TEST, 0, FP_COND_P, 0);
3216                 gen_four(label_ovf);
3217 #endif
3218 #if defined(ARCH_ARM32) || defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS)
3219                 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_TO_INT32 : INSN_FP_TO_INT64, op_size, 0, 0);
3220                 gen_one(FR_SCRATCH_1);
3221                 gen_one(reg1);
3222
3223                 g(gen_mov(ctx, OP_SIZE_INT, target, FR_SCRATCH_1));
3224 #else
3225                 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_TO_INT32 : INSN_FP_TO_INT64, op_size, 0, 0);
3226                 gen_one(target);
3227                 gen_one(reg1);
3228 #endif
3229                 g(gen_imm(ctx, (int_default_t)(sign_bit(uint_default_t) + 1), IMM_PURPOSE_ADD, OP_SIZE_INT));
3230                 gen_insn(INSN_ALU, OP_SIZE_INT, ALU_ADD, ALU_WRITES_FLAGS(ALU_ADD, is_imm()));
3231                 gen_one(R_SCRATCH_2);
3232                 gen_one(target);
3233                 gen_imm_offset();
3234
3235                 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, OP_SIZE_INT, R_SCRATCH_2, 1, COND_BE, label_ovf));
3236
3237                 g(gen_frame_store(ctx, OP_SIZE_INT, slot_r, 0, target));
3238                 return true;
3239 #endif
3240 #if defined(ARCH_IA64)
3241                 gen_insn(INSN_FP_TO_INT64, op_size, 0, 0);
3242                 gen_one(FR_SCRATCH_1);
3243                 gen_one(reg1);
3244
3245                 g(gen_mov(ctx, OP_SIZE_NATIVE, target, FR_SCRATCH_1));
3246
3247                 if (OP_SIZE_INT == OP_SIZE_4) {
3248                         g(gen_extend(ctx, OP_SIZE_4, sign_x, R_SCRATCH_2, target));
3249                         g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, target, R_SCRATCH_2, COND_NE, label_ovf));
3250                 } else {
3251                         g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, target, sign_bit(uint64_t), COND_E, label_ovf));
3252                 }
3253
3254                 g(gen_frame_store(ctx, OP_SIZE_INT, slot_r, 0, target));
3255                 return true;
3256 #endif
3257 #if defined(ARCH_PARISC) || defined(ARCH_POWER) || defined(ARCH_SPARC)
3258 #if defined(ARCH_POWER)
3259                 if (!cpu_test_feature(CPU_FEATURE_ppc))
3260                         goto do_upcall;
3261                 if (OP_SIZE_INT == OP_SIZE_4)
3262                         goto do_upcall;
3263 #endif
3264                 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_TO_INT32 : INSN_FP_TO_INT64, op_size, 0, 0);
3265                 gen_one(FR_SCRATCH_1);
3266                 gen_one(reg1);
3267
3268                 g(gen_frame_store_raw(ctx, OP_SIZE_INT, slot_r, 0, FR_SCRATCH_1));
3269                 if (slot_is_register(ctx, slot_r))
3270                         g(unspill(ctx, slot_r));
3271                 g(gen_frame_load(ctx, OP_SIZE_INT, garbage, slot_r, 0, false, target));
3272
3273                 g(gen_imm(ctx, sign_bit(uint_default_t) + 1, IMM_PURPOSE_ADD, OP_SIZE_INT));
3274                 gen_insn(INSN_ALU, i_size(OP_SIZE_INT), ALU_ADD, ALU_WRITES_FLAGS(ALU_ADD, is_imm()));
3275                 gen_one(R_SCRATCH_2);
3276                 gen_one(target);
3277                 gen_imm_offset();
3278
3279                 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, OP_SIZE_INT, R_SCRATCH_2, 1, COND_BE, label_ovf));
3280
3281                 return true;
3282 #endif
3283 #if defined(ARCH_ALPHA)
3284                 gen_insn(INSN_FP_TO_INT64_TRAP, op_size, 0, 0);
3285                 gen_one(FR_SCRATCH_2);
3286                 gen_one(reg1);
3287                 gen_four(label_ovf);
3288
3289                 if (OP_SIZE_INT == OP_SIZE_4) {
3290                         gen_insn(INSN_FP_INT64_TO_INT32_TRAP, 0, 0, 0);
3291                         gen_one(FR_SCRATCH_3);
3292                         gen_one(FR_SCRATCH_2);
3293                         gen_four(label_ovf);
3294                         g(gen_frame_store_raw(ctx, OP_SIZE_INT, slot_r, 0, FR_SCRATCH_3));
3295                 } else {
3296                         g(gen_frame_store_raw(ctx, OP_SIZE_INT, slot_r, 0, FR_SCRATCH_2));
3297                 }
3298                 if (slot_is_register(ctx, slot_r))
3299                         g(unspill(ctx, slot_r));
3300                 return true;
3301 #endif
3302 #if defined(ARCH_S390)
3303                 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_TO_INT32 : INSN_FP_TO_INT64, op_size, 0, 1);
3304                 gen_one(target);
3305                 gen_one(reg1);
3306
3307                 gen_insn(INSN_JMP_COND, op_size, FP_COND_P, 0);
3308                 gen_four(label_ovf);
3309
3310                 g(gen_frame_store(ctx, OP_SIZE_INT, slot_r, 0, target));
3311                 return true;
3312 #endif
3313 #if defined(ARCH_RISCV64)
3314                 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_TO_INT32 : INSN_FP_TO_INT64, op_size, 0, 0);
3315                 gen_one(target);
3316                 gen_one(reg1);
3317
3318                 g(gen_load_constant(ctx, R_SCRATCH_2, sign_bit(int_default_t)));
3319
3320                 g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, target, R_SCRATCH_2, COND_E, label_ovf));
3321
3322                 g(gen_imm(ctx, -1, IMM_PURPOSE_XOR, i_size(size)));
3323                 gen_insn(INSN_ALU, OP_SIZE_NATIVE, ALU_XOR, ALU_WRITES_FLAGS(ALU_XOR, is_imm()));
3324                 gen_one(R_SCRATCH_2);
3325                 gen_one(R_SCRATCH_2);
3326                 gen_imm_offset();
3327
3328                 g(gen_cmp_test_jmp(ctx, INSN_CMP, OP_SIZE_NATIVE, target, R_SCRATCH_2, COND_E, label_ovf));
3329
3330                 g(gen_frame_store(ctx, OP_SIZE_INT, slot_r, 0, target));
3331                 return true;
3332 #endif
3333         }
3334 #ifdef SUPPORTED_FP_X87
3335         if ((SUPPORTED_FP_X87 >> real_type) & 1) {
3336                 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
3337
3338                 if (likely(cpu_test_feature(CPU_FEATURE_sse3))) {
3339                         g(gen_frame_store_x87(ctx, INSN_X87_FISTTP, OP_SIZE_INT, slot_r));
3340                 } else {
3341                         gen_insn(INSN_PUSH, OP_SIZE_NATIVE, 0, 0);
3342                         gen_one(ARG_IMM);
3343                         gen_eight(0x0f7f);
3344
3345                         gen_insn(INSN_X87_FLDCW, 0, 0, 0);
3346                         gen_one(ARG_ADDRESS_1);
3347                         gen_one(R_SP);
3348                         gen_eight(0);
3349
3350                         g(gen_frame_store_x87(ctx, INSN_X87_FISTP, OP_SIZE_INT, slot_r));
3351
3352                         gen_insn(INSN_MOV, OP_SIZE_NATIVE, 0, 0);
3353                         gen_one(ARG_ADDRESS_1);
3354                         gen_one(R_SP);
3355                         gen_eight(0);
3356                         gen_one(ARG_IMM);
3357                         gen_eight(0x037f);
3358
3359                         gen_insn(INSN_X87_FLDCW, 0, 0, 0);
3360                         gen_one(ARG_ADDRESS_1);
3361                         gen_one(R_SP);
3362                         gen_eight(0);
3363
3364                         gen_insn(INSN_ALU, i_size(OP_SIZE_ADDRESS), ALU_ADD, 1);
3365                         gen_one(R_SP);
3366                         gen_one(R_SP);
3367                         gen_one(ARG_IMM);
3368                         gen_eight(1 << OP_SIZE_NATIVE);
3369                 }
3370                 if (slot_is_register(ctx, slot_r))
3371                         g(unspill(ctx, slot_r));
3372                 g(gen_frame_load(ctx, OP_SIZE_INT, garbage, slot_r, 0, false, R_SCRATCH_1));
3373
3374                 g(gen_cmp_test_imm_jmp(ctx, INSN_CMP, OP_SIZE_INT, R_SCRATCH_1, sign_bit(int_default_t), COND_E, label_ovf));
3375
3376                 return true;
3377         }
3378 #endif
3379 #ifdef SUPPORTED_FP_HALF_CVT
3380         if ((SUPPORTED_FP_HALF_CVT >> real_type) & 1) {
3381                 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, &reg1));
3382                 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
3383                 gen_one(FR_SCRATCH_1);
3384                 gen_one(reg1);
3385                 reg1 = FR_SCRATCH_1;
3386                 real_type = 1;
3387                 op_size = real_type_to_op_size(real_type);
3388                 goto do_cvt_to_int;
3389         }
3390 #endif
3391         goto do_upcall;
3392
3393 do_from_int:
3394         if ((SUPPORTED_FP >> real_type) & 1) {
3395 #if defined(ARCH_ALPHA) || defined(ARCH_ARM32) || defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS) || defined(ARCH_PARISC) || defined(ARCH_POWER) || defined(ARCH_SPARC)
3396                 int int_op_size = OP_SIZE_INT;
3397 #if defined(ARCH_POWER)
3398                 if (int_op_size == OP_SIZE_4)
3399                         goto do_upcall;
3400                 if (op_size == OP_SIZE_4 && !cpu_test_feature(CPU_FEATURE_v206))
3401                         goto do_upcall;
3402                 if (op_size == OP_SIZE_8 && !cpu_test_feature(CPU_FEATURE_ppc))
3403                         goto do_upcall;
3404 #endif
3405                 if (slot_is_register(ctx, slot_1))
3406                         g(spill(ctx, slot_1));
3407                 g(gen_frame_load_raw(ctx, int_op_size, zero_x, slot_1, 0, false, FR_SCRATCH_1));
3408                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_2);
3409 #if defined(ARCH_ALPHA)
3410                 if (OP_SIZE_INT == OP_SIZE_4) {
3411                         gen_insn(INSN_MOVSX, OP_SIZE_4, 0, 0);
3412                         gen_one(FR_SCRATCH_1);
3413                         gen_one(FR_SCRATCH_1);
3414
3415                         int_op_size = OP_SIZE_8;
3416                 }
3417 #endif
3418                 gen_insn(int_op_size == OP_SIZE_4 ? INSN_FP_FROM_INT32 : INSN_FP_FROM_INT64, op_size, 0, 0);
3419                 gen_one(target);
3420                 gen_one(FR_SCRATCH_1);
3421
3422                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
3423                 return true;
3424 #elif defined(ARCH_IA64)
3425                 g(gen_frame_get(ctx, OP_SIZE_INT, sign_x, slot_1, R_SCRATCH_1, &reg1));
3426                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_1);
3427
3428                 g(gen_mov(ctx, OP_SIZE_NATIVE, target, reg1));
3429
3430                 gen_insn(INSN_FP_FROM_INT64, op_size, 0, 0);
3431                 gen_one(target);
3432                 gen_one(target);
3433
3434                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
3435                 return true;
3436 #else
3437                 g(gen_frame_get(ctx, OP_SIZE_INT, garbage, slot_1, R_SCRATCH_1, &reg1));
3438                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_1);
3439
3440                 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_FROM_INT32 : INSN_FP_FROM_INT64, op_size, 0, 0);
3441                 gen_one(target);
3442                 gen_one(reg1);
3443
3444                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
3445                 return true;
3446 #endif
3447         }
3448 #ifdef SUPPORTED_FP_X87
3449         if ((SUPPORTED_FP_X87 >> real_type) & 1) {
3450                 if (slot_is_register(ctx, slot_1))
3451                         g(spill(ctx, slot_1));
3452                 g(gen_frame_load_x87(ctx, INSN_X87_FILD, OP_SIZE_INT, 0, slot_1));
3453                 g(gen_frame_store_x87(ctx, INSN_X87_FSTP, op_size, slot_r));
3454                 return true;
3455         }
3456 #endif
3457 #ifdef SUPPORTED_FP_HALF_CVT
3458         if ((SUPPORTED_FP_HALF_CVT >> real_type) & 1) {
3459                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, FR_SCRATCH_1);
3460 #if defined(ARCH_ARM32)
3461                 g(gen_frame_get(ctx, OP_SIZE_INT, zero_x, slot_1, FR_SCRATCH_1, &reg1));
3462
3463                 gen_insn(INSN_FP_FROM_INT32, OP_SIZE_4, 0, 0);
3464                 gen_one(target);
3465                 gen_one(reg1);
3466 #else
3467                 g(gen_frame_get(ctx, OP_SIZE_INT, garbage, slot_1, R_SCRATCH_1, &reg1));
3468                 gen_insn(OP_SIZE_INT == OP_SIZE_4 ? INSN_FP_FROM_INT32 : INSN_FP_FROM_INT64, OP_SIZE_4, 0, 0);
3469                 gen_one(target);
3470                 gen_one(reg1);
3471 #endif
3472                 gen_insn(INSN_FP_CVT, OP_SIZE_4, op_size, 0);
3473                 gen_one(target);
3474                 gen_one(target);
3475                 g(gen_frame_store(ctx, op_size, slot_r, 0, target));
3476                 return true;
3477         }
3478 #endif
3479         goto do_upcall;
3480
3481 do_is_exception:
3482         if ((SUPPORTED_FP >> real_type) & 1) {
3483                 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, &reg1));
3484                 target = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
3485 #if defined(ARCH_ALPHA)
3486                 gen_insn(INSN_FP_CMP_UNORDERED_DEST_REG, op_size, 0, 0);
3487                 gen_one(FR_SCRATCH_2);
3488                 gen_one(FR_SCRATCH_1);
3489                 gen_one(reg1);
3490
3491                 if (!cpu_test_feature(CPU_FEATURE_fix)) {
3492                         g(gen_frame_store_raw(ctx, OP_SIZE_4, slot_r, 0, FR_SCRATCH_2));
3493                         g(gen_frame_load_raw(ctx, OP_SIZE_4, sign_x, slot_r, 0, false, target));
3494                 } else {
3495                         g(gen_mov(ctx, OP_SIZE_4, target, FR_SCRATCH_2));
3496                 }
3497
3498                 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SHR, target, target, 30, 0));
3499
3500                 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
3501
3502                 return true;
3503 #elif defined(ARCH_IA64)
3504                 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_P, 0);
3505                 gen_one(R_CMP_RESULT);
3506                 gen_one(reg1);
3507                 gen_one(reg1);
3508
3509                 g(gen_mov(ctx, OP_SIZE_NATIVE, target, R_CMP_RESULT));
3510
3511                 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
3512 #elif defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS) || defined(ARCH_PARISC)
3513                 gen_insn(INSN_FP_CMP_COND, op_size, FP_COND_P, 1);
3514                 gen_one(reg1);
3515                 gen_one(reg1);
3516
3517                 gen_insn(INSN_FP_TEST_REG, OP_SIZE_NATIVE, FP_COND_P, 0);
3518                 gen_one(target);
3519
3520                 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
3521 #elif defined(ARCH_RISCV64)
3522                 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_E, 0);
3523                 gen_one(target);
3524                 gen_one(reg1);
3525                 gen_one(reg1);
3526
3527                 g(gen_imm(ctx, 1, IMM_PURPOSE_XOR, OP_SIZE_NATIVE));
3528                 gen_insn(INSN_ALU, OP_SIZE_NATIVE, ALU_XOR, ALU_WRITES_FLAGS(ALU_XOR, is_imm()));
3529                 gen_one(target);
3530                 gen_one(target);
3531                 gen_imm_offset();
3532
3533                 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, target));
3534 #else
3535                 gen_insn(INSN_FP_CMP, op_size, 0, 1);
3536                 gen_one(reg1);
3537                 gen_one(reg1);
3538 #if defined(ARCH_ARM32)
3539                 gen_insn(INSN_FP_TO_INT_FLAGS, 0, 0, 1);
3540 #endif
3541                 g(gen_frame_set_cond(ctx, op_size, false, FP_COND_P, slot_r));
3542 #endif
3543                 return true;
3544         }
3545 #ifdef SUPPORTED_FP_X87
3546         if ((SUPPORTED_FP_X87 >> real_type) & 1) {
3547                 g(gen_frame_load_x87(ctx, INSN_X87_FLD, op_size, 0, slot_1));
3548                 if (likely(cpu_test_feature(CPU_FEATURE_cmov))) {
3549                         gen_insn(INSN_X87_FCOMIP, op_size, 0, 0);
3550                         gen_one(R_ST0);
3551
3552                         g(gen_frame_set_cond(ctx, op_size, false, COND_P, slot_r));
3553                         return true;
3554                 }
3555
3556                 gen_insn(INSN_X87_FCOMP, op_size, 0, 0);
3557                 gen_one(R_ST0);
3558
3559                 gen_insn(INSN_X87_FNSTSW, 0, 0, 0);
3560                 gen_one(R_AX);
3561                 gen_one(R_AX);
3562
3563                 gen_insn(INSN_TEST, OP_SIZE_2, 0, 1);
3564                 gen_one(R_AX);
3565                 gen_one(ARG_IMM);
3566                 gen_eight(0x0400);
3567
3568                 g(gen_frame_set_cond(ctx, op_size, false, COND_NE, slot_r));
3569
3570                 return true;
3571         }
3572 #endif
3573 #ifdef SUPPORTED_FP_HALF_CVT
3574         if ((SUPPORTED_FP_HALF_CVT >> real_type) & 1) {
3575                 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, &reg1));
3576                 gen_insn(INSN_FP_CVT, op_size, OP_SIZE_4, 0);
3577                 gen_one(FR_SCRATCH_1);
3578                 gen_one(reg1);
3579                 gen_insn(INSN_FP_CMP, OP_SIZE_4, 0, 1);
3580                 gen_one(FR_SCRATCH_1);
3581                 gen_one(FR_SCRATCH_1);
3582 #if defined(ARCH_ARM32)
3583                 gen_insn(INSN_FP_TO_INT_FLAGS, 0, 0, 1);
3584 #endif
3585                 g(gen_frame_set_cond(ctx, op_size, false, FP_COND_P, slot_r));
3586                 return true;
3587         }
3588 #endif
3589
3590 do_upcall:
3591         g(gen_alu_typed_upcall(ctx, upc, real_type, slot_1, NO_FRAME_T, slot_r, label_ovf));
3592         return true;
3593 }
3594
3595 static bool attr_w gen_is_exception(struct codegen_context *ctx, frame_t slot_1, frame_t slot_r)
3596 {
3597         uint32_t no_ex_label, escape_label;
3598         const struct type *type = get_type_of_local(ctx, slot_1);
3599
3600         no_ex_label = alloc_label(ctx);
3601         if (unlikely(!no_ex_label))
3602                 return false;
3603         escape_label = alloc_escape_label(ctx);
3604         if (unlikely(!escape_label))
3605                 return false;
3606
3607         if (TYPE_IS_FLAT(type))
3608                 g(gen_test_1_jz_cached(ctx, slot_1, no_ex_label));
3609
3610         g(gen_frame_load(ctx, OP_SIZE_SLOT, zero_x, slot_1, 0, false, R_SCRATCH_1));
3611         g(gen_ptr_is_thunk(ctx, R_SCRATCH_1, slot_1, escape_label));
3612
3613         if (!TYPE_IS_FLAT(type)) {
3614                 g(gen_compare_da_tag(ctx, R_SCRATCH_1, DATA_TAG_flat, COND_E, escape_label, R_SCRATCH_1));
3615         }
3616
3617         gen_label(no_ex_label);
3618         g(gen_frame_clear(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r));
3619
3620         flag_set(ctx, slot_r, false);
3621
3622         return true;
3623 }
3624
3625 static bool attr_w gen_system_property(struct codegen_context *ctx, frame_t slot_1, frame_t slot_r)
3626 {
3627         uint32_t escape_label;
3628
3629         escape_label = alloc_escape_label(ctx);
3630         if (unlikely(!escape_label))
3631                 return false;
3632
3633         g(gen_test_1_cached(ctx, slot_1, escape_label));
3634
3635         g(gen_upcall_start(ctx, 1));
3636
3637         g(gen_frame_load(ctx, OP_SIZE_INT, garbage, slot_1, 0, false, R_ARG0));
3638         g(gen_upcall_argument(ctx, 0));
3639
3640         g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_ipret_system_property), 1));
3641
3642         g(gen_frame_store(ctx, OP_SIZE_INT, slot_r, 0, R_RET0));
3643
3644         flag_set(ctx, slot_1, false);
3645         flag_set(ctx, slot_r, false);
3646
3647         return true;
3648 }
3649
3650 static bool attr_w gen_alu_jmp(struct codegen_context *ctx, unsigned mode, unsigned op_size, unsigned op, frame_t slot_1, frame_t slot_2, int32_t jmp_offset, bool *failed)
3651 {
3652         unsigned alu;
3653         enum extend ex;
3654         unsigned reg1;
3655         unsigned attr_unused reg2;
3656 #if 0
3657         *failed = true; return true;
3658 #endif
3659         switch (mode) {
3660                 case MODE_FIXED: switch (op) {
3661                         case OPCODE_FIXED_OP_equal:             alu = COND_E; goto do_compare;
3662                         case OPCODE_FIXED_OP_not_equal:         alu = COND_NE; goto do_compare;
3663                         case OPCODE_FIXED_OP_less:              alu = COND_L; goto do_compare;
3664                         case OPCODE_FIXED_OP_less_equal:        alu = COND_LE; goto do_compare;
3665                         case OPCODE_FIXED_OP_greater:           alu = COND_G; goto do_compare;
3666                         case OPCODE_FIXED_OP_greater_equal:     alu = COND_GE; goto do_compare;
3667                         case OPCODE_FIXED_OP_uless:             alu = COND_B; goto do_compare;
3668                         case OPCODE_FIXED_OP_uless_equal:       alu = COND_BE; goto do_compare;
3669                         case OPCODE_FIXED_OP_ugreater:          alu = COND_A; goto do_compare;
3670                         case OPCODE_FIXED_OP_ugreater_equal:    alu = COND_AE; goto do_compare;
3671                         case OPCODE_FIXED_OP_bt:                *failed = true; return true;
3672                         default:                                internal(file_line, "gen_alu_jmp: unsupported fixed operation %u", op);
3673                 }
3674                 case MODE_INT: switch (op) {
3675                         case OPCODE_INT_OP_equal:               alu = COND_E; goto do_compare;
3676                         case OPCODE_INT_OP_not_equal:           alu = COND_NE; goto do_compare;
3677                         case OPCODE_INT_OP_less:                alu = COND_L; goto do_compare;
3678                         case OPCODE_INT_OP_less_equal:          alu = COND_LE; goto do_compare;
3679                         case OPCODE_INT_OP_greater:             alu = COND_G; goto do_compare;
3680                         case OPCODE_INT_OP_greater_equal:       alu = COND_GE; goto do_compare;
3681                         case OPCODE_INT_OP_bt:                  *failed = true; return true;
3682                         default:                                internal(file_line, "gen_alu_jmp: unsupported int operation %u", op);
3683                 }
3684                 case MODE_BOOL: switch (op) {
3685                         case OPCODE_BOOL_OP_and:                alu = ALU_AND; mode = MODE_FIXED; goto do_alu;
3686                         case OPCODE_BOOL_OP_or:                 alu = ALU_OR; mode = MODE_FIXED; goto do_alu;
3687                         case OPCODE_BOOL_OP_equal:              alu = COND_E; mode = MODE_FIXED; goto do_compare;
3688                         case OPCODE_BOOL_OP_not_equal:          alu = COND_NE; mode = MODE_FIXED; goto do_compare;
3689                         case OPCODE_BOOL_OP_less:               alu = COND_L; mode = MODE_FIXED; goto do_compare;
3690                         case OPCODE_BOOL_OP_less_equal:         alu = COND_LE; mode = MODE_FIXED; goto do_compare;
3691                         case OPCODE_BOOL_OP_greater:            alu = COND_G; mode = MODE_FIXED; goto do_compare;
3692                         case OPCODE_BOOL_OP_greater_equal:      alu = COND_GE; mode = MODE_FIXED; goto do_compare;
3693                         default:                                internal(file_line, "gen_alu_jmp: unsupported bool operation %u", op);
3694                 }
3695         }
3696         internal(file_line, "gen_alu_jmp: unsupported mode %u", mode);
3697 do_compare: {
3698                 bool attr_unused logical;
3699                 if (unlikely(op_size > OP_SIZE_NATIVE)) {
3700                         *failed = true;
3701                         return true;
3702                 }
3703                 if (slot_is_register(ctx, slot_2) && !slot_is_register(ctx, slot_1)) {
3704                         frame_t s = slot_1;
3705                         slot_1 = slot_2;
3706                         slot_2 = s;
3707                         switch (alu) {
3708                                 case COND_L:    alu = COND_G; break;
3709                                 case COND_LE:   alu = COND_GE; break;
3710                                 case COND_G:    alu = COND_L; break;
3711                                 case COND_GE:   alu = COND_LE; break;
3712                                 case COND_B:    alu = COND_A; break;
3713                                 case COND_BE:   alu = COND_AE; break;
3714                                 case COND_A:    alu = COND_B; break;
3715                                 case COND_AE:   alu = COND_BE; break;
3716                         }
3717                 }
3718                 ex = op_size == i_size_cmp(op_size) + (unsigned)zero ? garbage : alu == COND_L || alu == COND_LE || alu == COND_G || alu == COND_GE || ARCH_PREFERS_SX(op_size) ? sign_x : zero_x;
3719                 g(gen_frame_get(ctx, op_size, ex, slot_1, R_SCRATCH_1, &reg1));
3720                 if (ARCH_HAS_JMP_2REGS(alu)) {
3721                         g(gen_frame_get(ctx, op_size, ex, slot_2, R_SCRATCH_2, &reg2));
3722                         g(gen_jump(ctx, jmp_offset, i_size_cmp(op_size), alu ^ 1, reg1, reg2));
3723                         return true;
3724                 }
3725 #if ARCH_HAS_FLAGS
3726                 logical = COND_IS_LOGICAL(alu ^ 1);
3727                 g(gen_frame_load_cmp(ctx, op_size, logical, ex, false, slot_2, 0, false, reg1));
3728                 g(gen_jump(ctx, jmp_offset, op_size, alu ^ 1, -1U, -1U));
3729 #else
3730                 g(gen_frame_get(ctx, op_size, ex, slot_2, R_SCRATCH_2, &reg2));
3731                 g(gen_cmp_dest_reg(ctx, op_size, reg1, reg2, R_CMP_RESULT, 0, alu));
3732                 g(gen_jump(ctx, jmp_offset, OP_SIZE_NATIVE, COND_E, R_CMP_RESULT, -1U));
3733 #endif
3734                 return true;
3735         }
3736 do_alu: {
3737                 if (slot_is_register(ctx, slot_2) && !slot_is_register(ctx, slot_1)) {
3738                         frame_t s = slot_1;
3739                         slot_1 = slot_2;
3740                         slot_2 = s;
3741                 }
3742                 ex = op_size == i_size(op_size) + (unsigned)zero ? garbage : ARCH_PREFERS_SX(op_size) ? sign_x : zero_x;
3743                 g(gen_frame_get(ctx, op_size, ex, slot_1, R_SCRATCH_1, &reg1));
3744 #if defined(ARCH_X86)
3745                 if (alu == ALU_AND && !slot_is_register(ctx, slot_2)) {
3746                         g(gen_address(ctx, R_FRAME, (size_t)slot_2 * slot_size, IMM_PURPOSE_LDR_OFFSET, op_size));
3747                         gen_insn(INSN_TEST, i_size(op_size), 0, 1);
3748                         gen_one(reg1);
3749                         gen_address_offset();
3750                         g(gen_jump(ctx, jmp_offset, op_size, COND_E, -1U, -1U));
3751                         return true;
3752                 }
3753 #endif
3754                 g(gen_frame_get(ctx, op_size, ex, slot_2, R_SCRATCH_2, &reg2));
3755 #if ARCH_HAS_FLAGS && !defined(ARCH_S390)
3756                 if (alu == ALU_AND) {
3757                         gen_insn(INSN_TEST, i_size(op_size), 0, 1);
3758                         gen_one(reg1);
3759                         gen_one(reg2);
3760                         g(gen_jump(ctx, jmp_offset, op_size, COND_E, -1U, -1U));
3761                         return true;
3762                 }
3763 #endif
3764 #if defined(ARCH_ARM64)
3765                 if (alu == ALU_OR)
3766                         goto skip_flags;
3767 #endif
3768 #if ARCH_HAS_FLAGS
3769                 g(gen_3address_alu(ctx, i_size(op_size), alu, R_SCRATCH_1, reg1, reg2, 1));
3770                 g(gen_jump(ctx, jmp_offset, i_size(op_size), COND_E, -1U, -1U));
3771                 return true;
3772 #endif
3773                 goto skip_flags;
3774 skip_flags:
3775                 g(gen_3address_alu(ctx, i_size(op_size), alu, R_SCRATCH_1, reg1, reg2, 0));
3776                 g(gen_jump(ctx, jmp_offset, i_size(op_size), COND_E, R_SCRATCH_1, -1U));
3777                 return true;
3778         }
3779 }
3780
3781 static bool attr_w gen_fp_alu_jmp(struct codegen_context *ctx, unsigned real_type, unsigned op, uint32_t label_ovf, frame_t slot_1, frame_t slot_2, int32_t jmp_offset, bool *failed)
3782 {
3783         unsigned attr_unused fp_alu;
3784         unsigned attr_unused op_size = real_type_to_op_size(real_type);
3785         unsigned reg1, reg2;
3786         unsigned attr_unused target;
3787         switch (op) {
3788                 case OPCODE_REAL_OP_equal:
3789                 case OPCODE_REAL_OP_equal_alt1:
3790                 case OPCODE_REAL_OP_equal_alt2: fp_alu = FP_COND_E; goto do_cmp;
3791                 case OPCODE_REAL_OP_not_equal:
3792                 case OPCODE_REAL_OP_not_equal_alt1:
3793                 case OPCODE_REAL_OP_not_equal_alt2: fp_alu = FP_COND_NE; goto do_cmp;
3794                 case OPCODE_REAL_OP_less:
3795                 case OPCODE_REAL_OP_less_alt1:
3796                 case OPCODE_REAL_OP_less_alt2: fp_alu = FP_COND_B; goto do_cmp;
3797                 case OPCODE_REAL_OP_less_equal:
3798                 case OPCODE_REAL_OP_less_equal_alt1:
3799                 case OPCODE_REAL_OP_less_equal_alt2: fp_alu = FP_COND_BE; goto do_cmp;
3800                 case OPCODE_REAL_OP_greater:
3801                 case OPCODE_REAL_OP_greater_alt1:
3802                 case OPCODE_REAL_OP_greater_alt2: fp_alu = FP_COND_A; goto do_cmp;
3803                 case OPCODE_REAL_OP_greater_equal:
3804                 case OPCODE_REAL_OP_greater_equal_alt1:
3805                 case OPCODE_REAL_OP_greater_equal_alt2: fp_alu = FP_COND_AE; goto do_cmp;
3806                 default: internal(file_line, "gen_fp_alu_jmp: unsupported operation %u", op);
3807         }
3808
3809 do_cmp:
3810         if ((SUPPORTED_FP >> real_type) & 1
3811 #if defined(ARCH_ALPHA)
3812                 && ARCH_SUPPORTS_TRAPS && cpu_test_feature(CPU_FEATURE_fix)
3813 #endif
3814         ) {
3815                 g(gen_frame_get(ctx, op_size, zero_x, slot_1, FR_SCRATCH_1, &reg1));
3816                 g(gen_frame_get(ctx, op_size, zero_x, slot_2, FR_SCRATCH_2, &reg2));
3817                 target = R_SCRATCH_1;
3818 #if defined(ARCH_ALPHA)
3819                 gen_insn(INSN_FP_CMP_DEST_REG_TRAP, op_size, fp_alu == FP_COND_NE ? FP_COND_E : fp_alu, 0);
3820                 gen_one(FR_SCRATCH_3);
3821                 gen_one(reg1);
3822                 gen_one(reg2);
3823                 gen_four(label_ovf);
3824
3825                 g(gen_mov(ctx, OP_SIZE_4, target, FR_SCRATCH_3));
3826
3827                 if (fp_alu == FP_COND_NE) {
3828                         g(gen_jump(ctx, jmp_offset, OP_SIZE_NATIVE, COND_NE, target, -1U));
3829                 } else {
3830                         g(gen_jump(ctx, jmp_offset, OP_SIZE_NATIVE, COND_E, target, -1U));
3831                 }
3832
3833                 return true;
3834 #elif defined(ARCH_IA64)
3835                 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_P, 0);
3836                 gen_one(R_CMP_RESULT);
3837                 gen_one(reg1);
3838                 gen_one(reg2);
3839
3840                 gen_insn(INSN_JMP_REG, OP_SIZE_NATIVE, COND_NE, 0);
3841                 gen_one(R_CMP_RESULT);
3842                 gen_four(label_ovf);
3843
3844                 gen_insn(INSN_FP_CMP_DEST_REG, op_size, fp_alu, 0);
3845                 gen_one(R_CMP_RESULT);
3846                 gen_one(reg1);
3847                 gen_one(reg2);
3848
3849                 g(gen_jump(ctx, jmp_offset, OP_SIZE_NATIVE, COND_E, R_CMP_RESULT, -1U));
3850
3851                 return true;
3852 #elif defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS) || defined(ARCH_PARISC)
3853                 gen_insn(INSN_FP_CMP_COND, op_size, FP_COND_P, 1);
3854                 gen_one(reg1);
3855                 gen_one(reg2);
3856
3857                 gen_insn(INSN_JMP_FP_TEST, 0, FP_COND_P, 0);
3858                 gen_four(label_ovf);
3859
3860                 gen_insn(INSN_FP_CMP_COND, op_size, fp_alu ^ 1, 1);
3861                 gen_one(reg1);
3862                 gen_one(reg2);
3863
3864                 g(gen_jump(ctx, jmp_offset, OP_SIZE_NATIVE, fp_alu ^ 1, -1U, -1U));
3865
3866                 return true;
3867 #elif defined(ARCH_RISCV64)
3868                 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_E, 0);
3869                 gen_one(R_SCRATCH_1);
3870                 gen_one(reg1);
3871                 gen_one(reg1);
3872
3873                 gen_insn(INSN_FP_CMP_DEST_REG, op_size, FP_COND_E, 0);
3874                 gen_one(R_SCRATCH_2);
3875                 gen_one(reg2);
3876                 gen_one(reg2);
3877
3878                 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_AND, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_2, 0));
3879
3880                 g(gen_jmp_on_zero(ctx, OP_SIZE_NATIVE, R_SCRATCH_1, COND_E, label_ovf));
3881
3882                 gen_insn(INSN_FP_CMP_DEST_REG, op_size, fp_alu == FP_COND_NE ? FP_COND_E : fp_alu, 0);
3883                 gen_one(target);
3884                 gen_one(reg1);
3885                 gen_one(reg2);
3886
3887                 if (fp_alu == FP_COND_NE) {
3888                         g(gen_jump(ctx, jmp_offset, OP_SIZE_NATIVE, COND_NE, target, -1U));
3889                 } else {
3890                         g(gen_jump(ctx, jmp_offset, OP_SIZE_NATIVE, COND_E, target, -1U));
3891                 }
3892                 return true;
3893 #else
3894                 gen_insn(INSN_FP_CMP, op_size, 0, 1);
3895                 gen_one(reg1);
3896                 gen_one(reg2);
3897 #if defined(ARCH_ARM32)
3898                 gen_insn(INSN_FP_TO_INT_FLAGS, 0, 0, 1);
3899 #endif
3900                 gen_insn(INSN_JMP_COND, op_size, FP_COND_P, 0);
3901                 gen_four(label_ovf);
3902                 g(gen_jump(ctx, jmp_offset, op_size, fp_alu ^ 1, -1U, -1U));
3903                 return true;
3904 #endif
3905         }
3906         *failed = true;
3907         return true;
3908 }