cg-util.inc

   1 /*
   2  * Copyright (C) 2024 Mikulas Patocka
   3  *
   4  * This file is part of Ajla.
   5  *
   6  * Ajla is free software: you can redistribute it and/or modify it under the
   7  * terms of the GNU General Public License as published by the Free Software
   8  * Foundation, either version 3 of the License, or (at your option) any later
   9  * version.
  10  *
  11  * Ajla is distributed in the hope that it will be useful, but WITHOUT ANY
  12  * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
  13  * A PARTICULAR PURPOSE. See the GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along with
  16  * Ajla. If not, see <https://www.gnu.org/licenses/>.
  17  */
  18
  19 static bool attr_w gen_mov(struct codegen_context *ctx, unsigned size, unsigned dest, unsigned src)
  20 {
  21         if (dest == src && (size == OP_SIZE_NATIVE || reg_is_fp(dest)))
  22                 return true;
  23
  24         gen_insn(INSN_MOV, size, 0, 0);
  25         gen_one(dest);
  26         gen_one(src);
  27
  28         return true;
  29 }
  30
  31 static bool attr_w gen_sanitize_returned_pointer(struct codegen_context attr_unused *ctx, unsigned attr_unused reg)
  32 {
  33 #if defined(ARCH_X86_X32)
  34         g(gen_mov(ctx, OP_SIZE_ADDRESS, reg, reg));
  35 #endif
  36         return true;
  37 }
  38
  39 static bool alu_is_commutative(unsigned alu)
  40 {
  41         return alu == ALU_ADD || alu == ALU_OR || alu == ALU_AND || alu == ALU_XOR || alu == ALU_MUL || alu == ALU_UMULH || alu == ALU_SMULH;
  42 }
  43
  44 static bool attr_w gen_3address_alu(struct codegen_context *ctx, unsigned size, unsigned alu, unsigned dest, unsigned src1, unsigned src2, unsigned writes_flags)
  45 {
  46         if (unlikely(dest == src2) && alu_is_commutative(alu)) {
  47                 unsigned swap = src1;
  48                 src1 = src2;
  49                 src2 = swap;
  50         }
  51         if (!ARCH_IS_3ADDRESS(alu, writes_flags) && unlikely(dest == src2) && unlikely(dest != src1)) {
  52                 internal(file_line, "gen_3address_alu: invalid registers: %u, %u, %x, %x, %x", size, alu, dest, src1, src2);
  53         }
  54         if (!ARCH_IS_3ADDRESS(alu, writes_flags) && dest != src1) {
  55                 g(gen_mov(ctx, OP_SIZE_NATIVE, dest, src1));
  56
  57                 gen_insn(INSN_ALU + ARCH_PARTIAL_ALU(size), size, alu, ALU_WRITES_FLAGS(size, alu, false, false, 0) | writes_flags);
  58                 gen_one(dest);
  59                 gen_one(dest);
  60                 gen_one(src2);
  61
  62                 return true;
  63         }
  64         gen_insn(INSN_ALU + ARCH_PARTIAL_ALU(size), size, alu, ALU_WRITES_FLAGS(size, alu, false, false, 0) | writes_flags);
  65         gen_one(dest);
  66         gen_one(src1);
  67         gen_one(src2);
  68         return true;
  69 }
  70
  71 static bool attr_w gen_3address_alu_imm(struct codegen_context *ctx, unsigned size, unsigned alu, unsigned dest, unsigned src, int64_t imm, unsigned writes_flags)
  72 {
  73         unsigned purpose = alu_purpose(alu);
  74         if (!ARCH_IS_3ADDRESS_IMM(alu, writes_flags) && dest != src) {
  75                 g(gen_mov(ctx, OP_SIZE_NATIVE, dest, src));
  76
  77                 g(gen_imm(ctx, imm, purpose, i_size(OP_SIZE_ADDRESS)));
  78                 gen_insn(INSN_ALU + ARCH_PARTIAL_ALU(size), size, alu, ALU_WRITES_FLAGS(size, alu, false, is_imm(), ctx->const_imm) | writes_flags);
  79                 gen_one(dest);
  80                 gen_one(dest);
  81                 gen_imm_offset();
  82
  83                 return true;
  84         }
  85         g(gen_imm(ctx, imm, purpose, i_size(OP_SIZE_ADDRESS)));
  86         gen_insn(INSN_ALU + ARCH_PARTIAL_ALU(size), size, alu, ALU_WRITES_FLAGS(size, alu, false, is_imm(), ctx->const_imm) | writes_flags);
  87         gen_one(dest);
  88         gen_one(src);
  89         gen_imm_offset();
  90
  91         return true;
  92 }
  93
  94 static bool attr_w attr_unused gen_3address_rot(struct codegen_context *ctx, unsigned size, unsigned alu, unsigned dest, unsigned src1, unsigned src2)
  95 {
  96 #ifdef ARCH_X86
  97         if (dest == src1 && src2 == R_CX) {
  98                 gen_insn(INSN_ROT + ARCH_PARTIAL_ALU(size), size, alu, 1);
  99                 gen_one(dest);
 100                 gen_one(src1);
 101                 gen_one(src2);
 102
 103                 return true;
 104         }
 105 #endif
 106         if (!ARCH_IS_3ADDRESS_ROT(alu, size) && dest != src1) {
 107                 if (unlikely(dest == src2))
 108                         internal(file_line, "gen_3address_rot: invalid registers: %u, %u, %x, %x, %x", size, alu, dest, src1, src2);
 109
 110                 g(gen_mov(ctx, OP_SIZE_NATIVE, dest, src1));
 111
 112                 gen_insn(INSN_ROT + ARCH_PARTIAL_ALU(size), size, alu, ROT_WRITES_FLAGS(alu, size, false));
 113                 gen_one(dest);
 114                 gen_one(dest);
 115                 gen_one(src2);
 116
 117                 return true;
 118         }
 119         gen_insn(INSN_ROT + ARCH_PARTIAL_ALU(size), size, alu, ROT_WRITES_FLAGS(alu, size, false));
 120         gen_one(dest);
 121         gen_one(src1);
 122         gen_one(src2);
 123
 124         return true;
 125 }
 126
 127 static bool attr_w gen_3address_rot_imm(struct codegen_context *ctx, unsigned size, unsigned alu, unsigned dest, unsigned src, int64_t imm, unsigned writes_flags)
 128 {
 129         if (!ARCH_IS_3ADDRESS_ROT_IMM(alu) && dest != src) {
 130                 g(gen_mov(ctx, OP_SIZE_NATIVE, dest, src));
 131
 132                 gen_insn(INSN_ROT + ARCH_PARTIAL_ALU(size), size, alu, ROT_WRITES_FLAGS(alu, size, true) | writes_flags);
 133                 gen_one(dest);
 134                 gen_one(dest);
 135                 gen_one(ARG_IMM);
 136                 gen_eight(imm);
 137
 138                 return true;
 139         }
 140         gen_insn(INSN_ROT + ARCH_PARTIAL_ALU(size), size, alu, ROT_WRITES_FLAGS(alu, size, true) | writes_flags);
 141         gen_one(dest);
 142         gen_one(src);
 143         gen_one(ARG_IMM);
 144         gen_eight(imm);
 145
 146         return true;
 147 }
 148
 149 static bool attr_w gen_2address_alu1(struct codegen_context *ctx, unsigned size, unsigned alu, unsigned dest, unsigned src, unsigned writes_flags)
 150 {
 151         if (!ARCH_IS_2ADDRESS(alu) && dest != src) {
 152                 g(gen_mov(ctx, OP_SIZE_NATIVE, dest, src));
 153
 154                 gen_insn(INSN_ALU1 + ARCH_PARTIAL_ALU(size), size, alu, ALU1_WRITES_FLAGS(alu) | writes_flags);
 155                 gen_one(dest);
 156                 gen_one(dest);
 157
 158                 return true;
 159         }
 160         gen_insn(INSN_ALU1 + ARCH_PARTIAL_ALU(size), size, alu, ALU1_WRITES_FLAGS(alu) | writes_flags);
 161         gen_one(dest);
 162         gen_one(src);
 163
 164         return true;
 165 }
 166
 167 static bool attr_w gen_3address_fp_alu(struct codegen_context *ctx, unsigned size, unsigned alu, unsigned dest, unsigned src1, unsigned src2)
 168 {
 169         if (!ARCH_IS_3ADDRESS_FP && unlikely(dest == src2) && unlikely(dest != src1)) {
 170                 internal(file_line, "gen_3address_fp_alu: invalid registers: %u, %u, %x, %x, %x", size, alu, dest, src1, src2);
 171         }
 172         if (!ARCH_IS_3ADDRESS_FP && dest != src1) {
 173                 g(gen_mov(ctx, size, dest, src1));
 174
 175                 gen_insn(INSN_FP_ALU, size, alu, 0);
 176                 gen_one(dest);
 177                 gen_one(dest);
 178                 gen_one(src2);
 179
 180                 return true;
 181         }
 182         gen_insn(INSN_FP_ALU, size, alu, 0);
 183         gen_one(dest);
 184         gen_one(src1);
 185         gen_one(src2);
 186
 187         return true;
 188 }
 189
 190
 191 static bool attr_w attr_unused gen_load_two(struct codegen_context *ctx, unsigned dest, unsigned src, int64_t offset)
 192 {
 193         if (!ARCH_HAS_BWX) {
 194                 if (!(offset & 7)) {
 195                         g(gen_address(ctx, src, offset, IMM_PURPOSE_LDR_OFFSET, OP_SIZE_8));
 196                         gen_insn(INSN_MOV_U, OP_SIZE_NATIVE, 0, 0);
 197                         gen_one(dest);
 198                         gen_address_offset();
 199
 200                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_EXTWL, dest, dest, src, 0));
 201                 } else {
 202                         g(gen_imm(ctx, offset, IMM_PURPOSE_ADD, i_size(OP_SIZE_ADDRESS)));
 203                         gen_insn(INSN_ALU, i_size(OP_SIZE_ADDRESS), ALU_ADD, ALU_WRITES_FLAGS(i_size(OP_SIZE_ADDRESS), ALU_ADD, false, is_imm(), ctx->const_imm));
 204                         gen_one(R_OFFSET_IMM);
 205                         gen_one(src);
 206                         gen_imm_offset();
 207
 208                         gen_insn(INSN_MOV_U, OP_SIZE_NATIVE, 0, 0);
 209                         gen_one(dest);
 210                         gen_one(ARG_ADDRESS_1);
 211                         gen_one(R_OFFSET_IMM);
 212                         gen_eight(0);
 213
 214                         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_EXTWL, dest, dest, R_OFFSET_IMM, 0));
 215                 }
 216 #if defined(ARCH_S390)
 217         } else if (!cpu_test_feature(CPU_FEATURE_extended_imm)) {
 218                 g(gen_address(ctx, src, offset, IMM_PURPOSE_LDR_SX_OFFSET, OP_SIZE_2));
 219                 gen_insn(INSN_MOVSX, OP_SIZE_2, 0, 0);
 220                 gen_one(dest);
 221                 gen_address_offset();
 222
 223                 g(gen_3address_alu_imm(ctx, OP_SIZE_NATIVE, ALU_AND, dest, dest, 0xffff, 0));
 224 #endif
 225         } else {
 226                 g(gen_address(ctx, src, offset, IMM_PURPOSE_LDR_OFFSET, OP_SIZE_2));
 227                 gen_insn(INSN_MOV, OP_SIZE_2, 0, 0);
 228                 gen_one(dest);
 229                 gen_address_offset();
 230         }
 231         return true;
 232 }
 233
 234 static bool attr_w gen_load_code_32(struct codegen_context *ctx, unsigned dest, unsigned src, int64_t offset)
 235 {
 236 #if ARG_MODE_N == 3 && defined(ARCH_ALPHA) && !(defined(C_BIG_ENDIAN) ^ CODE_ENDIAN)
 237         if (!ARCH_HAS_BWX && UNALIGNED_TRAP) {
 238                 if (offset & 7) {
 239                         g(gen_3address_alu_imm(ctx, OP_SIZE_NATIVE, ALU_ADD, R_OFFSET_IMM, src, offset, 0));
 240                         src = R_OFFSET_IMM;
 241                         offset = 0;
 242                 }
 243                 g(gen_address(ctx, src, offset, IMM_PURPOSE_LDR_OFFSET, OP_SIZE_8));
 244                 gen_insn(INSN_MOV_U, OP_SIZE_NATIVE, 0, 0);
 245                 gen_one(dest);
 246                 gen_address_offset();
 247
 248                 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_EXTLL, dest, dest, src, 0));
 249
 250                 g(gen_address(ctx, src, offset + 3, IMM_PURPOSE_LDR_OFFSET, OP_SIZE_8));
 251                 gen_insn(INSN_MOV_U, OP_SIZE_NATIVE, 0, 0);
 252                 gen_one(R_CONST_IMM);
 253                 gen_address_offset();
 254
 255                 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_EXTLH, R_CONST_IMM, R_CONST_IMM, src, 0));
 256
 257                 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_OR, dest, dest, R_CONST_IMM, 0));
 258
 259                 return true;
 260         }
 261 #endif
 262 #if ARG_MODE_N == 3 && defined(ARCH_MIPS) && !(defined(C_BIG_ENDIAN) ^ CODE_ENDIAN)
 263         if (!MIPS_R6 && UNALIGNED_TRAP) {
 264                 g(gen_address(ctx, src, offset, IMM_PURPOSE_LDR_OFFSET, OP_SIZE_4));
 265                 gen_insn(INSN_MOV_LR, OP_SIZE_4, !CODE_ENDIAN, 0);
 266                 gen_one(dest);
 267                 gen_one(dest);
 268                 gen_address_offset();
 269
 270                 g(gen_address(ctx, src, offset + 3, IMM_PURPOSE_LDR_OFFSET, OP_SIZE_4));
 271                 gen_insn(INSN_MOV_LR, OP_SIZE_4, CODE_ENDIAN, 0);
 272                 gen_one(dest);
 273                 gen_one(dest);
 274                 gen_address_offset();
 275
 276                 return true;
 277         }
 278 #endif
 279 #if ARG_MODE_N == 3
 280 #if !(defined(C_BIG_ENDIAN) ^ CODE_ENDIAN)
 281         if (UNALIGNED_TRAP)
 282 #endif
 283         {
 284                 g(gen_load_two(ctx, dest, src, offset));
 285                 g(gen_load_two(ctx, R_CONST_IMM, src, offset + 2));
 286 #if CODE_ENDIAN
 287                 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SHL, dest, dest, 16, false));
 288 #else
 289                 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SHL, R_CONST_IMM, R_CONST_IMM, 16, false));
 290 #endif
 291                 g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_OR, dest, dest, R_CONST_IMM, 0));
 292                 return true;
 293         }
 294 #endif
 295         g(gen_address(ctx, src, offset, IMM_PURPOSE_LDR_OFFSET, ARG_MODE_N - 1));
 296         gen_insn(INSN_MOV, ARG_MODE_N - 1, 0, 0);
 297         gen_one(dest);
 298         gen_address_offset();
 299         return true;
 300 }
 301
 302 static bool attr_w attr_unused gen_cmp_dest_reg(struct codegen_context *ctx, unsigned attr_unused size, unsigned reg1, unsigned reg2, unsigned reg_dest, int64_t imm, unsigned cond)
 303 {
 304         unsigned neg_result = false;
 305
 306 #if defined(ARCH_ALPHA)
 307         if (cond == COND_NE) {
 308                 if (reg2 == (unsigned)-1)
 309                         g(gen_imm(ctx, imm, IMM_PURPOSE_CMP, i_size_cmp(size)));
 310                 gen_insn(INSN_CMP_DEST_REG, i_size_cmp(size), COND_E, 0);
 311                 gen_one(reg_dest);
 312                 gen_one(reg1);
 313                 if (reg2 == (unsigned)-1)
 314                         gen_imm_offset();
 315                 else
 316                         gen_one(reg2);
 317                 neg_result = true;
 318                 goto done;
 319         }
 320 #endif
 321 #if defined(ARCH_LOONGARCH64) || defined(ARCH_MIPS) || defined(ARCH_RISCV64)
 322         if (cond == COND_E || cond == COND_NE) {
 323                 unsigned rx;
 324                 if (reg2 == (unsigned)-1 && !imm) {
 325                         rx = reg1;
 326                         goto skip_xor;
 327                 }
 328                 if (reg2 == (unsigned)-1)
 329                         g(gen_imm(ctx, imm, IMM_PURPOSE_XOR, i_size(size)));
 330                 gen_insn(INSN_ALU, i_size(size), ALU_XOR, ALU_WRITES_FLAGS(i_size(size), ALU_XOR, false, reg2 == (unsigned)-1 && is_imm(), ctx->const_imm));
 331                 gen_one(reg_dest);
 332                 gen_one(reg1);
 333                 if (reg2 == (unsigned)-1)
 334                         gen_imm_offset();
 335                 else
 336                         gen_one(reg2);
 337                 rx = reg_dest;
 338 skip_xor:
 339                 if (cond == COND_E) {
 340                         g(gen_imm(ctx, 1, IMM_PURPOSE_CMP, i_size_cmp(size)));
 341                         gen_insn(INSN_CMP_DEST_REG, i_size_cmp(size), COND_B, 0);
 342                         gen_one(reg_dest);
 343                         gen_one(rx);
 344                         gen_imm_offset();
 345                 } else {
 346                         gen_insn(INSN_CMP_DEST_REG, i_size_cmp(size), COND_B, 0);
 347                         gen_one(reg_dest);
 348                         gen_one(ARG_IMM);
 349                         gen_eight(0);
 350                         gen_one(rx);
 351                 }
 352                 goto done;
 353         }
 354         if (cond == COND_GE || cond == COND_LE || cond == COND_AE || cond == COND_BE) {
 355                 cond ^= 1;
 356                 neg_result = true;
 357         }
 358 #endif
 359 #if defined(ARCH_IA64)
 360         if (reg2 == (unsigned)-1)
 361                 g(gen_imm(ctx, imm, IMM_PURPOSE_CMP, i_size_cmp(size)));
 362         gen_insn(INSN_CMP_DEST_REG, i_size_cmp(size), cond, 0);
 363         gen_one(R_CMP_RESULT);
 364         gen_one(reg1);
 365         if (reg2 == (unsigned)-1)
 366                 gen_imm_offset();
 367         else
 368                 gen_one(reg2);
 369
 370         g(gen_mov(ctx, OP_SIZE_NATIVE, reg_dest, R_CMP_RESULT));
 371
 372         goto done;
 373 #endif
 374         if (reg2 == (unsigned)-1)
 375                 g(gen_imm(ctx, imm, IMM_PURPOSE_CMP, i_size_cmp(size)));
 376         gen_insn(INSN_CMP_DEST_REG, i_size_cmp(size), cond, 0);
 377         gen_one(reg_dest);
 378         gen_one(reg1);
 379         if (reg2 == (unsigned)-1)
 380                 gen_imm_offset();
 381         else
 382                 gen_one(reg2);
 383
 384         goto done;
 385 done:
 386         if (neg_result)
 387                 g(gen_3address_alu_imm(ctx, i_size(size), ALU_XOR, reg_dest, reg_dest, 1, 0));
 388
 389         return true;
 390 }
 391
 392 static bool attr_w gen_cmp_test_jmp(struct codegen_context *ctx, unsigned insn, unsigned op_size, unsigned reg1, unsigned reg2, unsigned cond, uint32_t label)
 393 {
 394         bool arch_use_flags = ARCH_HAS_FLAGS;
 395 #if defined(ARCH_ARM64)
 396         if (insn == INSN_TEST && reg1 == reg2 && (cond == COND_E || cond == COND_NE))
 397                 arch_use_flags = false;
 398 #endif
 399 #if defined(ARCH_SPARC)
 400         if (insn == INSN_TEST && reg1 == reg2)
 401                 arch_use_flags = false;
 402 #endif
 403         if (arch_use_flags) {
 404                 if (COND_IS_LOGICAL(cond)) {
 405                         gen_insn(insn, op_size, 0, 2);
 406                         gen_one(reg1);
 407                         gen_one(reg2);
 408
 409                         gen_insn(INSN_JMP_COND_LOGICAL, op_size, cond, 0);
 410                         gen_four(label);
 411
 412                         return true;
 413                 }
 414
 415                 gen_insn(insn, op_size, 0, 1);
 416                 gen_one(reg1);
 417                 gen_one(reg2);
 418
 419 #if defined(ARCH_POWER) || defined(ARCH_S390)
 420                 if (insn == INSN_TEST) {
 421                         if (cond == COND_S)
 422                                 cond = COND_L;
 423                         if (cond == COND_NS)
 424                                 cond = COND_GE;
 425                 }
 426 #endif
 427                 gen_insn(INSN_JMP_COND, op_size, cond, 0);
 428                 gen_four(label);
 429         } else {
 430                 if (insn == INSN_CMP) {
 431 #if defined(ARCH_LOONGARCH64) || defined(ARCH_PARISC) || defined(ARCH_RISCV64) || (defined(ARCH_MIPS) && MIPS_R6)
 432                         gen_insn(INSN_JMP_2REGS, op_size, cond, 0);
 433                         gen_one(reg1);
 434                         gen_one(reg2);
 435                         gen_four(label);
 436                         return true;
 437 #else
 438 #ifdef R_CMP_RESULT
 439                         unsigned jmp_cond = COND_NE;
 440 #if defined(ARCH_MIPS)
 441                         if (cond == COND_E || cond == COND_NE) {
 442                                 gen_insn(INSN_JMP_2REGS, op_size, cond, 0);
 443                                 gen_one(reg1);
 444                                 gen_one(reg2);
 445                                 gen_four(label);
 446                                 return true;
 447                         }
 448                         if (cond == COND_AE || cond == COND_BE || cond == COND_LE || cond == COND_GE) {
 449                                 cond ^= 1;
 450                                 jmp_cond ^= 1;
 451                         }
 452 #endif
 453 #if defined(ARCH_ALPHA)
 454                         if (cond == COND_NE) {
 455                                 g(gen_3address_alu(ctx, op_size, ALU_XOR, R_CMP_RESULT, reg1, reg2, 0));
 456                         } else
 457 #endif
 458                         {
 459                                 gen_insn(INSN_CMP_DEST_REG, op_size, cond, 0);
 460                                 gen_one(R_CMP_RESULT);
 461                                 gen_one(reg1);
 462                                 gen_one(reg2);
 463                         }
 464
 465                         gen_insn(INSN_JMP_REG, OP_SIZE_NATIVE, jmp_cond, 0);
 466                         gen_one(R_CMP_RESULT);
 467                         gen_four(label);
 468 #else
 469                         internal(file_line, "gen_cmp_test_jmp: R_CMP_RESULT not defined");
 470 #endif
 471 #endif
 472                 } else if (insn == INSN_TEST) {
 473                         if (reg1 != reg2) {
 474                                 internal(file_line, "gen_cmp_test_jmp: INSN_TEST with two distinct registers is unsupported");
 475                         }
 476 #if defined(ARCH_IA64)
 477                         if (cond == COND_S)
 478                                 cond = COND_L;
 479                         if (cond == COND_NS)
 480                                 cond = COND_GE;
 481                         g(gen_imm(ctx, 0, IMM_PURPOSE_CMP, OP_SIZE_NATIVE));
 482                         gen_insn(INSN_CMP_DEST_REG, OP_SIZE_NATIVE, cond, 0);
 483                         gen_one(R_CMP_RESULT);
 484                         gen_one(reg1);
 485                         gen_imm_offset();
 486
 487                         reg1 = R_CMP_RESULT;
 488                         cond = COND_NE;
 489 #endif
 490                         gen_insn(INSN_JMP_REG, OP_SIZE_NATIVE, cond, 0);
 491                         gen_one(reg1);
 492                         gen_four(label);
 493                 }
 494         }
 495         return true;
 496 }
 497
 498 static bool attr_w gen_cmp_test_imm_jmp(struct codegen_context *ctx, unsigned insn, unsigned attr_unused op_size, unsigned reg1, int64_t value, unsigned cond, uint32_t label)
 499 {
 500         if (insn == INSN_TEST && (cond == COND_E || cond == COND_NE) && is_power_of_2((uint64_t)value)) {
 501 #ifdef HAVE_BUILTIN_CTZ
 502                 unsigned attr_unused bit = __builtin_ctzll(value);
 503 #else
 504                 unsigned attr_unused bit = 0;
 505                 uint64_t v = value;
 506                 while ((v = v >> 1))
 507                         bit++;
 508 #endif
 509 #if defined(ARCH_ALPHA) || defined(ARCH_PARISC)
 510                 if (value == 1 && (cond == COND_E || cond == COND_NE)) {
 511                         gen_insn(INSN_JMP_REG, OP_SIZE_NATIVE, cond == COND_E ? COND_BLBC : COND_BLBS, 0);
 512                         gen_one(reg1);
 513                         gen_four(label);
 514                         return true;
 515                 }
 516 #endif
 517 #if defined(ARCH_ARM64) || defined(ARCH_PARISC)
 518                 gen_insn(INSN_JMP_REG_BIT, OP_SIZE_NATIVE, bit | ((cond == COND_NE) << 6), 0);
 519                 gen_one(reg1);
 520                 gen_four(label);
 521
 522                 return true;
 523 #endif
 524 #if defined(ARCH_POWER)
 525                 g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SHL, R_CONST_IMM, reg1, (8U << OP_SIZE_NATIVE) - 1 - bit, true));
 526
 527                 gen_insn(INSN_JMP_COND, OP_SIZE_NATIVE, cond == COND_E ? COND_GE : COND_L, 0);
 528                 gen_four(label);
 529
 530                 return true;
 531 #endif
 532 #if defined(ARCH_IA64)
 533                 gen_insn(INSN_TEST_DEST_REG, OP_SIZE_NATIVE, bit | ((cond == COND_NE) << 6), 0);
 534                 gen_one(R_CMP_RESULT);
 535                 gen_one(reg1);
 536
 537                 gen_insn(INSN_JMP_REG, OP_SIZE_NATIVE, COND_NE, 0);
 538                 gen_one(R_CMP_RESULT);
 539                 gen_four(label);
 540
 541                 return true;
 542 #endif
 543 #if defined(R_CMP_RESULT)
 544                 if (!is_direct_const(1ULL << bit, IMM_PURPOSE_AND, OP_SIZE_NATIVE) && ARCH_HAS_BTX(BTX_BTEXT, OP_SIZE_NATIVE, true)) {
 545                         gen_insn(INSN_BTX, OP_SIZE_NATIVE, BTX_BTEXT, 0);
 546                         gen_one(R_CMP_RESULT);
 547                         gen_one(reg1);
 548                         gen_one(ARG_IMM);
 549                         gen_eight(bit);
 550
 551                         gen_insn(INSN_JMP_REG, OP_SIZE_NATIVE, cond, 0);
 552                         gen_one(R_CMP_RESULT);
 553                         gen_four(label);
 554
 555                         return true;
 556                 }
 557 #endif
 558         }
 559 #if ARCH_HAS_FLAGS
 560         if (unlikely(insn == INSN_CMP) && COND_IS_LOGICAL(cond)) {
 561                 g(gen_imm(ctx, value, IMM_PURPOSE_CMP_LOGICAL, op_size));
 562                 gen_insn(insn, op_size, 0, 2);
 563                 gen_one(reg1);
 564                 gen_imm_offset();
 565
 566                 gen_insn(INSN_JMP_COND_LOGICAL, op_size, cond, 0);
 567                 gen_four(label);
 568
 569                 return true;
 570         }
 571         g(gen_imm(ctx, value, insn == INSN_CMP ? IMM_PURPOSE_CMP : IMM_PURPOSE_TEST, op_size));
 572         gen_insn(insn, op_size, 0, 1);
 573         gen_one(reg1);
 574         gen_imm_offset();
 575
 576         gen_insn(INSN_JMP_COND, op_size, cond, 0);
 577         gen_four(label);
 578 #else
 579         if (insn == INSN_CMP) {
 580 #if defined(ARCH_LOONGARCH64) || defined(ARCH_PARISC) || defined(ARCH_RISCV64)
 581                 g(gen_imm(ctx, value, IMM_PURPOSE_JMP_2REGS, op_size));
 582 #if defined(ARCH_PARISC)
 583                 gen_insn(INSN_JMP_2REGS, op_size, cond, 0);
 584 #else
 585                 gen_insn(INSN_JMP_2REGS, i_size_cmp(op_size), cond, 0);
 586 #endif
 587                 gen_one(reg1);
 588                 gen_imm_offset();
 589                 gen_four(label);
 590                 return true;
 591 #else
 592                 unsigned final_cond = COND_NE;
 593 #if defined(ARCH_ALPHA)
 594                 if (cond == COND_AE || cond == COND_A || cond == COND_GE || cond == COND_G) {
 595                         cond ^= 1;
 596                         final_cond ^= 1;
 597                         goto gen_const;
 598                 } else if (cond == COND_NE) {
 599                         g(gen_3address_alu_imm(ctx, OP_SIZE_NATIVE, ALU_XOR, R_CMP_RESULT, reg1, value, 0));
 600                 } else
 601 gen_const:
 602 #endif
 603 #if defined(ARCH_MIPS)
 604                 if (cond == COND_E || cond == COND_NE) {
 605                         g(gen_load_constant(ctx, R_CONST_IMM, value));
 606                         gen_insn(INSN_JMP_2REGS, OP_SIZE_NATIVE, cond, 0);
 607                         gen_one(reg1);
 608                         gen_one(R_CONST_IMM);
 609                         gen_four(label);
 610                         return true;
 611                 }
 612                 if (cond == COND_AE || cond == COND_BE || cond == COND_LE || cond == COND_GE) {
 613                         cond ^= 1;
 614                         final_cond ^= 1;
 615                 }
 616                 if (cond == COND_A || cond == COND_G) {
 617                         g(gen_load_constant(ctx, R_CONST_IMM, value));
 618                         gen_insn(INSN_CMP_DEST_REG, OP_SIZE_NATIVE, cond, 0);
 619                         gen_one(R_CMP_RESULT);
 620                         gen_one(reg1);
 621                         gen_one(R_CONST_IMM);
 622                 } else
 623 #endif
 624                 {
 625                         g(gen_imm(ctx, value, IMM_PURPOSE_CMP, OP_SIZE_NATIVE));
 626                         gen_insn(INSN_CMP_DEST_REG, OP_SIZE_NATIVE, cond, 0);
 627                         gen_one(R_CMP_RESULT);
 628                         gen_one(reg1);
 629                         gen_imm_offset();
 630                 }
 631
 632                 gen_insn(INSN_JMP_REG, OP_SIZE_NATIVE, final_cond, 0);
 633                 gen_one(R_CMP_RESULT);
 634                 gen_four(label);
 635 #endif
 636         } else if (insn == INSN_TEST) {
 637 #if defined(ARCH_IA64)
 638                 internal(file_line, "gen_cmp_test_imm_jmp: value %"PRIxMAX" not supported", (uintmax_t)value);
 639 #endif
 640                 g(gen_3address_alu_imm(ctx, OP_SIZE_NATIVE, ALU_AND, R_CMP_RESULT, reg1, value, 0));
 641
 642                 gen_insn(INSN_JMP_REG, OP_SIZE_NATIVE, cond, 0);
 643                 gen_one(R_CMP_RESULT);
 644                 gen_four(label);
 645         } else {
 646                 internal(file_line, "gen_cmp_test_imm_jmp: invalid insn");
 647         }
 648 #endif
 649         return true;
 650 }
 651
 652 static bool attr_w gen_jmp_on_zero(struct codegen_context *ctx, unsigned attr_unused op_size, unsigned reg, unsigned cond, uint32_t label)
 653 {
 654         bool jmp_reg = false;
 655 #if defined(ARCH_ALPHA) || defined(ARCH_ARM64) || defined(ARCH_LOONGARCH64) || defined(ARCH_RISCV64)
 656         jmp_reg = true;
 657 #endif
 658 #if defined(ARCH_SPARC)
 659         jmp_reg |= SPARC_9;
 660 #endif
 661         if (jmp_reg) {
 662                 gen_insn(INSN_JMP_REG, i_size(op_size), cond, 0);
 663                 gen_one(reg);
 664                 gen_four(label);
 665
 666                 return true;
 667         }
 668         g(gen_cmp_test_jmp(ctx, INSN_TEST, i_size(op_size), reg, reg, cond, label));
 669
 670         return true;
 671 }
 672
 673 static bool attr_w gen_jmp_if_negative(struct codegen_context *ctx, unsigned reg, uint32_t label)
 674 {
 675 #if defined(ARCH_ARM64) || defined(ARCH_PARISC)
 676         gen_insn(INSN_JMP_REG_BIT, OP_SIZE_NATIVE, (INT_DEFAULT_BITS - 1) | ((uint32_t)1 << 6), 0);
 677         gen_one(reg);
 678         gen_four(label);
 679 #else
 680         g(gen_jmp_on_zero(ctx, OP_SIZE_INT, reg, COND_S, label));
 681 #endif
 682         return true;
 683 }
 684
 685 #if defined(ARCH_X86)
 686 static bool attr_w gen_cmov(struct codegen_context *ctx, unsigned op_size, unsigned cond, unsigned reg, uint32_t *label)
 687 {
 688         if (unlikely(op_size < OP_SIZE_4))
 689                 internal(file_line, "gen_cmov: unsupported operand size");
 690         if (likely(cpu_test_feature(CPU_FEATURE_cmov))) {
 691                 gen_insn(INSN_CMOV, op_size, cond, 0);
 692                 gen_one(reg);
 693                 gen_one(reg);
 694                 *label = 0;
 695         } else {
 696                 *label = alloc_label(ctx);
 697                 if (unlikely(!*label))
 698                         return false;
 699                 gen_insn(INSN_JMP_COND, op_size, cond ^ 1, 0);
 700                 gen_four(*label);
 701                 gen_insn(INSN_MOV, op_size, 0, 0);
 702                 gen_one(reg);
 703         }
 704         return true;
 705 }
 706 #endif
 707
 708 enum extend {
 709         zero_x,
 710         sign_x,
 711         native,
 712         garbage,
 713 };
 714
 715 static bool attr_w gen_extend(struct codegen_context *ctx, unsigned op_size, enum extend ex, unsigned dest, unsigned src)
 716 {
 717         unsigned attr_unused shift;
 718         if (ex == native)
 719                 ex = ARCH_PREFERS_SX(op_size) ? sign_x : zero_x;
 720         ajla_assert_lo(ex == zero_x || ex == sign_x, (file_line, "gen_extend: invalid mode %u", (unsigned)ex));
 721         if (unlikely(op_size == OP_SIZE_NATIVE)) {
 722                 g(gen_mov(ctx, op_size, dest, src));
 723                 return true;
 724         }
 725         if (OP_SIZE_NATIVE == OP_SIZE_4) {
 726                 shift = op_size == OP_SIZE_1 ? 24 : 16;
 727         } else if (OP_SIZE_NATIVE == OP_SIZE_8) {
 728                 shift = op_size == OP_SIZE_1 ? 56 : op_size == OP_SIZE_2 ? 48 : 32;
 729         } else {
 730                 internal(file_line, "gen_extend: invalid OP_SIZE_NATIVE");
 731         }
 732 #if defined(ARCH_ARM) || defined(ARCH_IA64) || defined(ARCH_LOONGARCH64) || defined(ARCH_PARISC) || defined(ARCH_X86)
 733 #if defined(ARCH_ARM32)
 734         if (unlikely(!cpu_test_feature(CPU_FEATURE_armv6)))
 735                 goto default_extend;
 736 #endif
 737         gen_insn(ex == sign_x ? INSN_MOVSX : INSN_MOV, op_size, 0, 0);
 738         gen_one(dest);
 739         gen_one(src);
 740         return true;
 741 #endif
 742 #if defined(ARCH_POWER)
 743         if (ex == zero_x || op_size == OP_SIZE_2 || cpu_test_feature(CPU_FEATURE_ppc)) {
 744                 gen_insn(ex == sign_x ? INSN_MOVSX : INSN_MOV, op_size, 0, 0);
 745                 gen_one(dest);
 746                 gen_one(src);
 747                 return true;
 748         }
 749 #endif
 750 #if defined(ARCH_ALPHA)
 751         if (ex == zero_x) {
 752                 g(gen_3address_alu_imm(ctx, OP_SIZE_NATIVE, ALU_ZAPNOT, dest, src, op_size == OP_SIZE_1 ? 0x1 : op_size == OP_SIZE_2 ? 0x3 : 0xf, 0));
 753                 return true;
 754         } else if (op_size == OP_SIZE_4 || ARCH_HAS_BWX) {
 755                 gen_insn(INSN_MOVSX, op_size, 0, 0);
 756                 gen_one(dest);
 757                 gen_one(src);
 758                 return true;
 759         }
 760 #endif
 761 #if defined(ARCH_MIPS)
 762         if (ex == sign_x && shift == 32) {
 763                 g(gen_3address_rot_imm(ctx, OP_SIZE_4, ROT_SHL, dest, src, 0, 0));
 764                 return true;
 765         }
 766         if (ex == sign_x && MIPS_HAS_ROT) {
 767                 gen_insn(INSN_MOVSX, op_size, 0, 0);
 768                 gen_one(dest);
 769                 gen_one(src);
 770                 return true;
 771         }
 772 #endif
 773 #if defined(ARCH_S390)
 774         if (((op_size == OP_SIZE_1 || op_size == OP_SIZE_2) && cpu_test_feature(CPU_FEATURE_extended_imm)) || op_size == OP_SIZE_4) {
 775                 gen_insn(ex == zero_x ? INSN_MOV : INSN_MOVSX, op_size, 0, 0);
 776                 gen_one(dest);
 777                 gen_one(src);
 778                 return true;
 779         }
 780 #endif
 781 #if defined(ARCH_SPARC)
 782         if (shift == 32) {
 783                 g(gen_3address_rot_imm(ctx, OP_SIZE_4, ex == sign_x ? ROT_SAR : ROT_SHR, dest, src, 0, 0));
 784                 return true;
 785         }
 786 #endif
 787 #if defined(ARCH_RISCV64)
 788         if (ex == sign_x && (op_size == OP_SIZE_4 || likely(cpu_test_feature(CPU_FEATURE_zbb)))) {
 789                 gen_insn(INSN_MOVSX, op_size, 0, 0);
 790                 gen_one(dest);
 791                 gen_one(src);
 792                 return true;
 793         }
 794         if (ex == zero_x && ((op_size == OP_SIZE_1) ||
 795                     (op_size == OP_SIZE_2 && likely(cpu_test_feature(CPU_FEATURE_zbb))) ||
 796                     (op_size == OP_SIZE_4 && likely(cpu_test_feature(CPU_FEATURE_zba))))) {
 797                 g(gen_mov(ctx, op_size, dest, src));
 798                 return true;
 799         }
 800 #endif
 801         goto default_extend;
 802 default_extend:
 803         if (ex == zero_x && op_size <= OP_SIZE_4) {
 804                 int64_t cnst = (0x1ULL << (8U << op_size)) - 1;
 805                 if (is_direct_const(cnst, IMM_PURPOSE_AND, OP_SIZE_NATIVE)) {
 806                         g(gen_imm(ctx, 0xff, IMM_PURPOSE_AND, OP_SIZE_NATIVE));
 807                         gen_insn(INSN_ALU, OP_SIZE_NATIVE, ALU_AND, ALU_WRITES_FLAGS(OP_SIZE_NATIVE, ALU_AND, false, is_imm(), ctx->const_imm));
 808                         gen_one(dest);
 809                         gen_one(src);
 810                         gen_imm_offset();
 811                         return true;
 812                 }
 813         }
 814         g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SHL, dest, src, shift, false));
 815         g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ex == sign_x ? ROT_SAR : ROT_SHR, dest, dest, shift, false));
 816         return true;
 817 }
 818
 819 static bool attr_w gen_cmp_extended(struct codegen_context *ctx, unsigned cmp_op_size, unsigned sub_op_size, unsigned reg, unsigned attr_unused tmp_reg, uint32_t label_ovf)
 820 {
 821         if (unlikely(sub_op_size >= cmp_op_size))
 822                 return true;
 823 #if defined(ARCH_ARM64)
 824         gen_insn(INSN_CMP, cmp_op_size, 0, 1);
 825         gen_one(reg);
 826         gen_one(ARG_EXTENDED_REGISTER);
 827         gen_one(sub_op_size == OP_SIZE_1 ? ARG_EXTEND_SXTB : sub_op_size == OP_SIZE_2 ? ARG_EXTEND_SXTH : ARG_EXTEND_SXTW);
 828         gen_one(reg);
 829
 830         gen_insn(INSN_JMP_COND, cmp_op_size, COND_NE, 0);
 831         gen_four(label_ovf);
 832 #else
 833         g(gen_extend(ctx, sub_op_size, sign_x, tmp_reg, reg));
 834
 835         g(gen_cmp_test_jmp(ctx, INSN_CMP, cmp_op_size, reg, tmp_reg, COND_NE, label_ovf));
 836 #endif
 837         return true;
 838 }
 839
 840 static bool attr_w gen_lea3(struct codegen_context *ctx, unsigned dest, unsigned base, unsigned shifted, unsigned shift, int64_t offset)
 841 {
 842 #if defined(ARCH_X86)
 843         gen_insn(INSN_LEA3, i_size(OP_SIZE_ADDRESS), shift, 0);
 844         gen_one(dest);
 845         gen_one(base);
 846         gen_one(shifted);
 847         gen_one(ARG_IMM);
 848         gen_eight(likely(imm_is_32bit(offset)) ? offset : 0);
 849
 850         if (unlikely(!imm_is_32bit(offset)))
 851                 g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_ADDRESS), ALU_ADD, dest, dest, offset, 0));
 852
 853         return true;
 854 #endif
 855         if (ARCH_HAS_SHIFTED_ADD(shift)) {
 856                 gen_insn(INSN_ALU, i_size(OP_SIZE_ADDRESS), ALU_ADD, ALU_WRITES_FLAGS(i_size(OP_SIZE_ADDRESS), ALU_ADD, false, false, 0));
 857                 gen_one(dest);
 858                 gen_one(base);
 859                 gen_one(ARG_SHIFTED_REGISTER);
 860                 gen_one(ARG_SHIFT_LSL | shift);
 861                 gen_one(shifted);
 862
 863                 if (offset) {
 864                         g(gen_imm(ctx, offset, IMM_PURPOSE_ADD, i_size(OP_SIZE_ADDRESS)));
 865                         gen_insn(INSN_ALU, i_size(OP_SIZE_ADDRESS), ALU_ADD, ALU_WRITES_FLAGS(i_size(OP_SIZE_ADDRESS), ALU_ADD, false, is_imm(), ctx->const_imm));
 866                         gen_one(dest);
 867                         gen_one(dest);
 868                         gen_imm_offset();
 869                 }
 870
 871                 return true;
 872         }
 873
 874         g(gen_3address_rot_imm(ctx, OP_SIZE_NATIVE, ROT_SHL, dest, shifted, shift, false));
 875
 876         g(gen_3address_alu(ctx, OP_SIZE_NATIVE, ALU_ADD, dest, dest, base, 0));
 877
 878         if (offset)
 879                 g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_ADDRESS), ALU_ADD, dest, dest, offset, 0));
 880         return true;
 881 }
 882
 883 #if !defined(POINTER_COMPRESSION)
 884 #define gen_pointer_compression(base)           do { } while (0)
 885 #define gen_address_offset_compressed()         gen_address_offset()
 886 #elif defined(ARCH_X86)
 887 #define gen_pointer_compression(base)           do { } while (0)
 888 #define gen_address_offset_compressed()                                 \
 889 do {                                                                    \
 890         if (likely(!ctx->offset_reg)) {                                 \
 891                 gen_one(ARG_ADDRESS_1 + POINTER_COMPRESSION);           \
 892                 gen_one(ctx->base_reg);                                 \
 893                 gen_eight(ctx->offset_imm);                             \
 894         } else {                                                        \
 895                 gen_one(ARG_ADDRESS_2 + POINTER_COMPRESSION);           \
 896                 gen_one(R_OFFSET_IMM);                                  \
 897                 gen_one(ctx->base_reg);                                 \
 898                 gen_eight(0);                                           \
 899         }                                                               \
 900 } while (0)
 901 #else
 902 #define gen_pointer_compression(base)                                   \
 903 do {                                                                    \
 904         if (ARCH_PREFERS_SX(OP_SIZE_4)) {                               \
 905                 g(gen_extend(ctx, OP_SIZE_4, zero_x, base, base));      \
 906         }                                                               \
 907         g(gen_3address_rot_imm(ctx, OP_SIZE_ADDRESS, ROT_SHL, base, base, POINTER_COMPRESSION, 0));\
 908 } while (0)
 909 #define gen_address_offset_compressed()         gen_address_offset()
 910 #endif
 911
 912