sljit_src/sljitNativeS390X.c

   1 /*
   2  *    Stack-less Just-In-Time compiler
   3  *
   4  *    Copyright Zoltan Herczeg (hzmester@freemail.hu). All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without modification, are
   7  * permitted provided that the following conditions are met:
   8  *
   9  *   1. Redistributions of source code must retain the above copyright notice, this list of
  10  *      conditions and the following disclaimer.
  11  *
  12  *   2. Redistributions in binary form must reproduce the above copyright notice, this list
  13  *      of conditions and the following disclaimer in the documentation and/or other materials
  14  *      provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) AND CONTRIBUTORS ``AS IS'' AND ANY
  17  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  18  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
  19  * SHALL THE COPYRIGHT HOLDER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  20  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
  21  * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
  22  * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  23  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  24  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  */
  26
  27 #include <sys/auxv.h>
  28
  29 #ifdef __ARCH__
  30 #define ENABLE_STATIC_FACILITY_DETECTION 1
  31 #else
  32 #define ENABLE_STATIC_FACILITY_DETECTION 0
  33 #endif
  34 #define ENABLE_DYNAMIC_FACILITY_DETECTION 1
  35
  36 SLJIT_API_FUNC_ATTRIBUTE const char* sljit_get_platform_name(void)
  37 {
  38         return "s390x" SLJIT_CPUINFO;
  39 }
  40
  41 /* Instructions. */
  42 typedef sljit_uw sljit_ins;
  43
  44 /* Instruction tags (most significant halfword). */
  45 static const sljit_ins sljit_ins_const = (sljit_ins)1 << 48;
  46
  47 #define TMP_REG1        (SLJIT_NUMBER_OF_REGISTERS + 2)
  48 #define TMP_REG2        (SLJIT_NUMBER_OF_REGISTERS + 3)
  49
  50 static const sljit_u8 reg_map[SLJIT_NUMBER_OF_REGISTERS + 4] = {
  51         0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 15, 0, 1
  52 };
  53
  54 /* there are also a[2-15] available, but they are slower to access and
  55  * their use is limited as mundaym explained:
  56  *   https://github.com/zherczeg/sljit/pull/91#discussion_r486895689
  57  */
  58
  59 /* General Purpose Registers [0-15]. */
  60 typedef sljit_uw sljit_gpr;
  61
  62 /*
  63  * WARNING
  64  * the following code is non standard and should be improved for
  65  * consistency, but doesn't use SLJIT_NUMBER_OF_REGISTERS based
  66  * registers because r0 and r1 are the ABI recommended volatiles.
  67  * there is a gpr() function that maps sljit to physical register numbers
  68  * that should be used instead of the usual index into reg_map[] and
  69  * will be retired ASAP (TODO: carenas)
  70  */
  71
  72 static const sljit_gpr r0 = 0;          /* reg_map[SLJIT_NUMBER_OF_REGISTERS + 2]: 0 in address calculations; reserved */
  73 static const sljit_gpr r1 = 1;          /* reg_map[SLJIT_NUMBER_OF_REGISTERS + 3]: reserved */
  74 static const sljit_gpr r2 = 2;          /* reg_map[1]: 1st argument */
  75 static const sljit_gpr r3 = 3;          /* reg_map[2]: 2nd argument */
  76 static const sljit_gpr r4 = 4;          /* reg_map[3]: 3rd argument */
  77 static const sljit_gpr r5 = 5;          /* reg_map[4]: 4th argument */
  78 static const sljit_gpr r6 = 6;          /* reg_map[5]: 5th argument; 1st saved register */
  79 static const sljit_gpr r7 = 7;          /* reg_map[6] */
  80 static const sljit_gpr r8 = 8;          /* reg_map[7] */
  81 static const sljit_gpr r9 = 9;          /* reg_map[8] */
  82 static const sljit_gpr r10 = 10;        /* reg_map[9] */
  83 static const sljit_gpr r11 = 11;        /* reg_map[10] */
  84 static const sljit_gpr r12 = 12;        /* reg_map[11]: GOT */
  85 static const sljit_gpr r13 = 13;        /* reg_map[12]: Literal Pool pointer */
  86 static const sljit_gpr r14 = 14;        /* reg_map[0]: return address and flag register */
  87 static const sljit_gpr r15 = 15;        /* reg_map[SLJIT_NUMBER_OF_REGISTERS + 1]: stack pointer */
  88
  89 /* WARNING: r12 and r13 shouldn't be used as per ABI recommendation */
  90 /* TODO(carenas): r12 might conflict in PIC code, reserve? */
  91 /* TODO(carenas): r13 is usually pointed to "pool" per ABI, using a tmp
  92  *                like we do know might be faster though, reserve?
  93  */
  94
  95 /* TODO(carenas): should be named TMP_REG[1-2] for consistency */
  96 #define tmp0    r0
  97 #define tmp1    r1
  98
  99 /* TODO(carenas): flags should move to a different register so that
 100  *                link register doesn't need to change
 101  */
 102
 103 /* When reg cannot be unused. */
 104 #define IS_GPR_REG(reg)         ((reg > 0) && (reg) <= SLJIT_SP)
 105
 106 /* Link registers. The normal link register is r14, but since
 107    we use that for flags we need to use r0 instead to do fast
 108    calls so that flags are preserved. */
 109 static const sljit_gpr link_r = 14;     /* r14 */
 110 static const sljit_gpr fast_link_r = 0; /* r0 */
 111
 112 #define TMP_FREG1       (0)
 113
 114 static const sljit_u8 freg_map[SLJIT_NUMBER_OF_FLOAT_REGISTERS + 1] = {
 115         1, 0, 2, 4, 6, 3, 5, 7, 15, 14, 13, 12, 11, 10, 9, 8,
 116 };
 117
 118 #define R0A(r) (r)
 119 #define R4A(r) ((r) << 4)
 120 #define R8A(r) ((r) << 8)
 121 #define R12A(r) ((r) << 12)
 122 #define R16A(r) ((r) << 16)
 123 #define R20A(r) ((r) << 20)
 124 #define R28A(r) ((r) << 28)
 125 #define R32A(r) ((r) << 32)
 126 #define R36A(r) ((r) << 36)
 127
 128 #define R0(r) ((sljit_ins)reg_map[r])
 129
 130 #define F0(r) ((sljit_ins)freg_map[r])
 131 #define F4(r) (R4A((sljit_ins)freg_map[r]))
 132 #define F20(r) (R20A((sljit_ins)freg_map[r]))
 133 #define F36(r) (R36A((sljit_ins)freg_map[r]))
 134
 135 struct sljit_s390x_const {
 136         struct sljit_const const_; /* must be first */
 137         sljit_sw init_value;       /* required to build literal pool */
 138 };
 139
 140 /* Convert SLJIT register to hardware register. */
 141 static SLJIT_INLINE sljit_gpr gpr(sljit_s32 r)
 142 {
 143         SLJIT_ASSERT(r >= 0 && r < (sljit_s32)(sizeof(reg_map) / sizeof(reg_map[0])));
 144         return reg_map[r];
 145 }
 146
 147 static SLJIT_INLINE sljit_gpr fgpr(sljit_s32 r)
 148 {
 149         SLJIT_ASSERT(r >= 0 && r < (sljit_s32)(sizeof(freg_map) / sizeof(freg_map[0])));
 150         return freg_map[r];
 151 }
 152
 153 /* Size of instruction in bytes. Tags must already be cleared. */
 154 static SLJIT_INLINE sljit_uw sizeof_ins(sljit_ins ins)
 155 {
 156         /* keep faulting instructions */
 157         if (ins == 0)
 158                 return 2;
 159
 160         if ((ins & 0x00000000ffffL) == ins)
 161                 return 2;
 162         if ((ins & 0x0000ffffffffL) == ins)
 163                 return 4;
 164         if ((ins & 0xffffffffffffL) == ins)
 165                 return 6;
 166
 167         SLJIT_UNREACHABLE();
 168         return (sljit_uw)-1;
 169 }
 170
 171 static sljit_s32 push_inst(struct sljit_compiler *compiler, sljit_ins ins)
 172 {
 173         sljit_ins *ibuf = (sljit_ins *)ensure_buf(compiler, sizeof(sljit_ins));
 174         FAIL_IF(!ibuf);
 175         *ibuf = ins;
 176         compiler->size++;
 177         return SLJIT_SUCCESS;
 178 }
 179
 180 static sljit_s32 encode_inst(void **ptr, sljit_ins ins)
 181 {
 182         sljit_u16 *ibuf = (sljit_u16 *)*ptr;
 183         sljit_uw size = sizeof_ins(ins);
 184
 185         SLJIT_ASSERT((size & 6) == size);
 186         switch (size) {
 187         case 6:
 188                 *ibuf++ = (sljit_u16)(ins >> 32);
 189                 /* fallthrough */
 190         case 4:
 191                 *ibuf++ = (sljit_u16)(ins >> 16);
 192                 /* fallthrough */
 193         case 2:
 194                 *ibuf++ = (sljit_u16)(ins);
 195         }
 196         *ptr = (void*)ibuf;
 197         return SLJIT_SUCCESS;
 198 }
 199
 200 #define SLJIT_ADD_SUB_NO_COMPARE(status_flags_state) \
 201         (((status_flags_state) & (SLJIT_CURRENT_FLAGS_ADD | SLJIT_CURRENT_FLAGS_SUB)) \
 202                 && !((status_flags_state) & SLJIT_CURRENT_FLAGS_COMPARE))
 203
 204 /* Map the given type to a 4-bit condition code mask. */
 205 static SLJIT_INLINE sljit_u8 get_cc(struct sljit_compiler *compiler, sljit_s32 type) {
 206         const sljit_u8 cc0 = 1 << 3; /* equal {,to zero} */
 207         const sljit_u8 cc1 = 1 << 2; /* less than {,zero} */
 208         const sljit_u8 cc2 = 1 << 1; /* greater than {,zero} */
 209         const sljit_u8 cc3 = 1 << 0; /* {overflow,NaN} */
 210
 211         switch (type) {
 212         case SLJIT_EQUAL:
 213                 if (SLJIT_ADD_SUB_NO_COMPARE(compiler->status_flags_state)) {
 214                         sljit_s32 type = GET_FLAG_TYPE(compiler->status_flags_state);
 215                         if (type >= SLJIT_SIG_LESS && type <= SLJIT_SIG_LESS_EQUAL)
 216                                 return cc0;
 217                         if (type == SLJIT_OVERFLOW)
 218                                 return (cc0 | cc3);
 219                         return (cc0 | cc2);
 220                 }
 221                 /* fallthrough */
 222
 223         case SLJIT_F_EQUAL:
 224         case SLJIT_ORDERED_EQUAL:
 225                 return cc0;
 226
 227         case SLJIT_NOT_EQUAL:
 228                 if (SLJIT_ADD_SUB_NO_COMPARE(compiler->status_flags_state)) {
 229                         sljit_s32 type = GET_FLAG_TYPE(compiler->status_flags_state);
 230                         if (type >= SLJIT_SIG_LESS && type <= SLJIT_SIG_LESS_EQUAL)
 231                                 return (cc1 | cc2 | cc3);
 232                         if (type == SLJIT_OVERFLOW)
 233                                 return (cc1 | cc2);
 234                         return (cc1 | cc3);
 235                 }
 236                 /* fallthrough */
 237
 238         case SLJIT_UNORDERED_OR_NOT_EQUAL:
 239                 return (cc1 | cc2 | cc3);
 240
 241         case SLJIT_LESS:
 242                 return cc1;
 243
 244         case SLJIT_GREATER_EQUAL:
 245         case SLJIT_UNORDERED_OR_GREATER_EQUAL:
 246                 return (cc0 | cc2 | cc3);
 247
 248         case SLJIT_GREATER:
 249                 if (compiler->status_flags_state & SLJIT_CURRENT_FLAGS_COMPARE)
 250                         return cc2;
 251                 return cc3;
 252
 253         case SLJIT_LESS_EQUAL:
 254                 if (compiler->status_flags_state & SLJIT_CURRENT_FLAGS_COMPARE)
 255                         return (cc0 | cc1);
 256                 return (cc0 | cc1 | cc2);
 257
 258         case SLJIT_SIG_LESS:
 259         case SLJIT_F_LESS:
 260         case SLJIT_ORDERED_LESS:
 261                 return cc1;
 262
 263         case SLJIT_NOT_CARRY:
 264                 if (compiler->status_flags_state & SLJIT_CURRENT_FLAGS_SUB)
 265                         return (cc2 | cc3);
 266                 /* fallthrough */
 267
 268         case SLJIT_SIG_LESS_EQUAL:
 269         case SLJIT_F_LESS_EQUAL:
 270         case SLJIT_ORDERED_LESS_EQUAL:
 271                 return (cc0 | cc1);
 272
 273         case SLJIT_CARRY:
 274                 if (compiler->status_flags_state & SLJIT_CURRENT_FLAGS_SUB)
 275                         return (cc0 | cc1);
 276                 /* fallthrough */
 277
 278         case SLJIT_SIG_GREATER:
 279         case SLJIT_UNORDERED_OR_GREATER:
 280                 /* Overflow is considered greater, see SLJIT_SUB. */
 281                 return cc2 | cc3;
 282
 283         case SLJIT_SIG_GREATER_EQUAL:
 284                 return (cc0 | cc2 | cc3);
 285
 286         case SLJIT_OVERFLOW:
 287                 if (compiler->status_flags_state & SLJIT_SET_Z)
 288                         return (cc2 | cc3);
 289                 /* fallthrough */
 290
 291         case SLJIT_UNORDERED:
 292                 return cc3;
 293
 294         case SLJIT_NOT_OVERFLOW:
 295                 if (compiler->status_flags_state & SLJIT_SET_Z)
 296                         return (cc0 | cc1);
 297                 /* fallthrough */
 298
 299         case SLJIT_ORDERED:
 300                 return (cc0 | cc1 | cc2);
 301
 302         case SLJIT_F_NOT_EQUAL:
 303         case SLJIT_ORDERED_NOT_EQUAL:
 304                 return (cc1 | cc2);
 305
 306         case SLJIT_F_GREATER:
 307         case SLJIT_ORDERED_GREATER:
 308                 return cc2;
 309
 310         case SLJIT_F_GREATER_EQUAL:
 311         case SLJIT_ORDERED_GREATER_EQUAL:
 312                 return (cc0 | cc2);
 313
 314         case SLJIT_UNORDERED_OR_LESS_EQUAL:
 315                 return (cc0 | cc1 | cc3);
 316
 317         case SLJIT_UNORDERED_OR_EQUAL:
 318                 return (cc0 | cc3);
 319
 320         case SLJIT_UNORDERED_OR_LESS:
 321                 return (cc1 | cc3);
 322         }
 323
 324         SLJIT_UNREACHABLE();
 325         return (sljit_u8)-1;
 326 }
 327
 328 /* Facility to bit index mappings.
 329    Note: some facilities share the same bit index. */
 330 typedef sljit_uw facility_bit;
 331 #define STORE_FACILITY_LIST_EXTENDED_FACILITY 7
 332 #define FAST_LONG_DISPLACEMENT_FACILITY 19
 333 #define EXTENDED_IMMEDIATE_FACILITY 21
 334 #define GENERAL_INSTRUCTION_EXTENSION_FACILITY 34
 335 #define DISTINCT_OPERAND_FACILITY 45
 336 #define HIGH_WORD_FACILITY 45
 337 #define POPULATION_COUNT_FACILITY 45
 338 #define LOAD_STORE_ON_CONDITION_1_FACILITY 45
 339 #define MISCELLANEOUS_INSTRUCTION_EXTENSIONS_1_FACILITY 49
 340 #define LOAD_STORE_ON_CONDITION_2_FACILITY 53
 341 #define MISCELLANEOUS_INSTRUCTION_EXTENSIONS_2_FACILITY 58
 342 #define VECTOR_FACILITY 129
 343 #define VECTOR_ENHANCEMENTS_1_FACILITY 135
 344
 345 /* Report whether a facility is known to be present due to the compiler
 346    settings. This function should always be compiled to a constant
 347    value given a constant argument. */
 348 static SLJIT_INLINE int have_facility_static(facility_bit x)
 349 {
 350 #if ENABLE_STATIC_FACILITY_DETECTION
 351         switch (x) {
 352         case FAST_LONG_DISPLACEMENT_FACILITY:
 353                 return (__ARCH__ >=  6 /* z990 */);
 354         case EXTENDED_IMMEDIATE_FACILITY:
 355         case STORE_FACILITY_LIST_EXTENDED_FACILITY:
 356                 return (__ARCH__ >=  7 /* z9-109 */);
 357         case GENERAL_INSTRUCTION_EXTENSION_FACILITY:
 358                 return (__ARCH__ >=  8 /* z10 */);
 359         case DISTINCT_OPERAND_FACILITY:
 360                 return (__ARCH__ >=  9 /* z196 */);
 361         case MISCELLANEOUS_INSTRUCTION_EXTENSIONS_1_FACILITY:
 362                 return (__ARCH__ >= 10 /* zEC12 */);
 363         case LOAD_STORE_ON_CONDITION_2_FACILITY:
 364         case VECTOR_FACILITY:
 365                 return (__ARCH__ >= 11 /* z13 */);
 366         case MISCELLANEOUS_INSTRUCTION_EXTENSIONS_2_FACILITY:
 367         case VECTOR_ENHANCEMENTS_1_FACILITY:
 368                 return (__ARCH__ >= 12 /* z14 */);
 369         default:
 370                 SLJIT_UNREACHABLE();
 371         }
 372 #endif
 373         return 0;
 374 }
 375
 376 static SLJIT_INLINE unsigned long get_hwcap()
 377 {
 378         static unsigned long hwcap = 0;
 379         if (SLJIT_UNLIKELY(!hwcap)) {
 380                 hwcap = getauxval(AT_HWCAP);
 381                 SLJIT_ASSERT(hwcap != 0);
 382         }
 383         return hwcap;
 384 }
 385
 386 static SLJIT_INLINE int have_stfle()
 387 {
 388         if (have_facility_static(STORE_FACILITY_LIST_EXTENDED_FACILITY))
 389                 return 1;
 390
 391         return (get_hwcap() & HWCAP_S390_STFLE);
 392 }
 393
 394 /* Report whether the given facility is available. This function always
 395    performs a runtime check. */
 396 static int have_facility_dynamic(facility_bit x)
 397 {
 398 #if ENABLE_DYNAMIC_FACILITY_DETECTION
 399         static struct {
 400                 sljit_uw bits[4];
 401         } cpu_features;
 402         size_t size = sizeof(cpu_features);
 403         const sljit_uw word_index = x >> 6;
 404         const sljit_uw bit_index = ((1UL << 63) >> (x & 63));
 405
 406         SLJIT_ASSERT(x < size * 8);
 407         if (SLJIT_UNLIKELY(!have_stfle()))
 408                 return 0;
 409
 410         if (SLJIT_UNLIKELY(cpu_features.bits[0] == 0)) {
 411                 __asm__ __volatile__ (
 412                         "lgr   %%r0, %0;"
 413                         "stfle 0(%1);"
 414                         /* outputs  */:
 415                         /* inputs   */: "d" ((size / 8) - 1), "a" (&cpu_features)
 416                         /* clobbers */: "r0", "cc", "memory"
 417                 );
 418                 SLJIT_ASSERT(cpu_features.bits[0] != 0);
 419         }
 420         return (cpu_features.bits[word_index] & bit_index) != 0;
 421 #else
 422         return 0;
 423 #endif
 424 }
 425
 426 #define HAVE_FACILITY(name, bit) \
 427 static SLJIT_INLINE int name() \
 428 { \
 429         static int have = -1; \
 430         /* Static check first. May allow the function to be optimized away. */ \
 431         if (have_facility_static(bit)) \
 432                 have = 1; \
 433         else if (SLJIT_UNLIKELY(have < 0)) \
 434                 have = have_facility_dynamic(bit) ? 1 : 0; \
 435 \
 436         return have; \
 437 }
 438
 439 HAVE_FACILITY(have_eimm,    EXTENDED_IMMEDIATE_FACILITY)
 440 HAVE_FACILITY(have_ldisp,   FAST_LONG_DISPLACEMENT_FACILITY)
 441 HAVE_FACILITY(have_genext,  GENERAL_INSTRUCTION_EXTENSION_FACILITY)
 442 HAVE_FACILITY(have_lscond1, LOAD_STORE_ON_CONDITION_1_FACILITY)
 443 HAVE_FACILITY(have_lscond2, LOAD_STORE_ON_CONDITION_2_FACILITY)
 444 HAVE_FACILITY(have_misc2,   MISCELLANEOUS_INSTRUCTION_EXTENSIONS_2_FACILITY)
 445 #undef HAVE_FACILITY
 446
 447 #define is_u12(d)       (0 <= (d) && (d) <= 0x00000fffL)
 448 #define is_u32(d)       (0 <= (d) && (d) <= 0xffffffffL)
 449
 450 #define CHECK_SIGNED(v, bitlen) \
 451         ((v) >= -(1 << ((bitlen) - 1)) && (v) < (1 << ((bitlen) - 1)))
 452
 453 #define is_s8(d)        CHECK_SIGNED((d), 8)
 454 #define is_s16(d)       CHECK_SIGNED((d), 16)
 455 #define is_s20(d)       CHECK_SIGNED((d), 20)
 456 #define is_s32(d)       ((d) == (sljit_s32)(d))
 457
 458 static SLJIT_INLINE sljit_ins disp_s20(sljit_s32 d)
 459 {
 460         SLJIT_ASSERT(is_s20(d));
 461
 462         sljit_uw dh = (d >> 12) & 0xff;
 463         sljit_uw dl = (d << 8) & 0xfff00;
 464         return (dh | dl) << 8;
 465 }
 466
 467 /* TODO(carenas): variadic macro is not strictly needed */
 468 #define SLJIT_S390X_INSTRUCTION(op, ...) \
 469 static SLJIT_INLINE sljit_ins op(__VA_ARGS__)
 470
 471 /* RR form instructions. */
 472 #define SLJIT_S390X_RR(name, pattern) \
 473 SLJIT_S390X_INSTRUCTION(name, sljit_gpr dst, sljit_gpr src) \
 474 { \
 475         return (pattern) | ((dst & 0xf) << 4) | (src & 0xf); \
 476 }
 477
 478 /* AND */
 479 SLJIT_S390X_RR(nr,   0x1400)
 480
 481 /* BRANCH AND SAVE */
 482 SLJIT_S390X_RR(basr, 0x0d00)
 483
 484 /* BRANCH ON CONDITION */
 485 SLJIT_S390X_RR(bcr,  0x0700) /* TODO(mundaym): type for mask? */
 486
 487 /* DIVIDE */
 488 SLJIT_S390X_RR(dr,   0x1d00)
 489
 490 /* EXCLUSIVE OR */
 491 SLJIT_S390X_RR(xr,   0x1700)
 492
 493 /* LOAD */
 494 SLJIT_S390X_RR(lr,   0x1800)
 495
 496 /* LOAD COMPLEMENT */
 497 SLJIT_S390X_RR(lcr,  0x1300)
 498
 499 /* OR */
 500 SLJIT_S390X_RR(or,   0x1600)
 501
 502 #undef SLJIT_S390X_RR
 503
 504 /* RRE form instructions */
 505 #define SLJIT_S390X_RRE(name, pattern) \
 506 SLJIT_S390X_INSTRUCTION(name, sljit_gpr dst, sljit_gpr src) \
 507 { \
 508         return (pattern) | R4A(dst) | R0A(src); \
 509 }
 510
 511 /* AND */
 512 SLJIT_S390X_RRE(ngr,   0xb9800000)
 513
 514 /* DIVIDE LOGICAL */
 515 SLJIT_S390X_RRE(dlr,   0xb9970000)
 516 SLJIT_S390X_RRE(dlgr,  0xb9870000)
 517
 518 /* DIVIDE SINGLE */
 519 SLJIT_S390X_RRE(dsgr,  0xb90d0000)
 520
 521 /* EXCLUSIVE OR */
 522 SLJIT_S390X_RRE(xgr,   0xb9820000)
 523
 524 /* LOAD */
 525 SLJIT_S390X_RRE(lgr,   0xb9040000)
 526 SLJIT_S390X_RRE(lgfr,  0xb9140000)
 527
 528 /* LOAD BYTE */
 529 SLJIT_S390X_RRE(lbr,   0xb9260000)
 530 SLJIT_S390X_RRE(lgbr,  0xb9060000)
 531
 532 /* LOAD COMPLEMENT */
 533 SLJIT_S390X_RRE(lcgr,  0xb9030000)
 534
 535 /* LOAD HALFWORD */
 536 SLJIT_S390X_RRE(lhr,   0xb9270000)
 537 SLJIT_S390X_RRE(lghr,  0xb9070000)
 538
 539 /* LOAD LOGICAL */
 540 SLJIT_S390X_RRE(llgfr, 0xb9160000)
 541
 542 /* LOAD LOGICAL CHARACTER */
 543 SLJIT_S390X_RRE(llcr,  0xb9940000)
 544 SLJIT_S390X_RRE(llgcr, 0xb9840000)
 545
 546 /* LOAD LOGICAL HALFWORD */
 547 SLJIT_S390X_RRE(llhr,  0xb9950000)
 548 SLJIT_S390X_RRE(llghr, 0xb9850000)
 549
 550 /* MULTIPLY LOGICAL */
 551 SLJIT_S390X_RRE(mlgr,  0xb9860000)
 552
 553 /* MULTIPLY SINGLE */
 554 SLJIT_S390X_RRE(msgfr, 0xb91c0000)
 555
 556 /* OR */
 557 SLJIT_S390X_RRE(ogr,   0xb9810000)
 558
 559 /* SUBTRACT */
 560 SLJIT_S390X_RRE(sgr,   0xb9090000)
 561
 562 #undef SLJIT_S390X_RRE
 563
 564 /* RI-a form instructions */
 565 #define SLJIT_S390X_RIA(name, pattern, imm_type) \
 566 SLJIT_S390X_INSTRUCTION(name, sljit_gpr reg, imm_type imm) \
 567 { \
 568         return (pattern) | R20A(reg) | (imm & 0xffff); \
 569 }
 570
 571 /* ADD HALFWORD IMMEDIATE */
 572 SLJIT_S390X_RIA(aghi,  0xa70b0000, sljit_s16)
 573
 574 /* LOAD HALFWORD IMMEDIATE */
 575 SLJIT_S390X_RIA(lhi,   0xa7080000, sljit_s16)
 576 SLJIT_S390X_RIA(lghi,  0xa7090000, sljit_s16)
 577
 578 /* LOAD LOGICAL IMMEDIATE */
 579 SLJIT_S390X_RIA(llihh, 0xa50c0000, sljit_u16)
 580 SLJIT_S390X_RIA(llihl, 0xa50d0000, sljit_u16)
 581 SLJIT_S390X_RIA(llilh, 0xa50e0000, sljit_u16)
 582 SLJIT_S390X_RIA(llill, 0xa50f0000, sljit_u16)
 583
 584 /* MULTIPLY HALFWORD IMMEDIATE */
 585 SLJIT_S390X_RIA(mhi,   0xa70c0000, sljit_s16)
 586 SLJIT_S390X_RIA(mghi,  0xa70d0000, sljit_s16)
 587
 588 /* OR IMMEDIATE */
 589 SLJIT_S390X_RIA(oilh,  0xa50a0000, sljit_u16)
 590
 591 #undef SLJIT_S390X_RIA
 592
 593 /* RIL-a form instructions (requires extended immediate facility) */
 594 #define SLJIT_S390X_RILA(name, pattern, imm_type) \
 595 SLJIT_S390X_INSTRUCTION(name, sljit_gpr reg, imm_type imm) \
 596 { \
 597         SLJIT_ASSERT(have_eimm()); \
 598         return (pattern) | R36A(reg) | ((sljit_ins)imm & 0xffffffffu); \
 599 }
 600
 601 /* ADD IMMEDIATE */
 602 SLJIT_S390X_RILA(agfi,  0xc20800000000, sljit_s32)
 603
 604 /* ADD IMMEDIATE HIGH */
 605 SLJIT_S390X_RILA(aih,   0xcc0800000000, sljit_s32) /* TODO(mundaym): high-word facility? */
 606
 607 /* AND IMMEDIATE */
 608 SLJIT_S390X_RILA(nihf,  0xc00a00000000, sljit_u32)
 609
 610 /* EXCLUSIVE OR IMMEDIATE */
 611 SLJIT_S390X_RILA(xilf,  0xc00700000000, sljit_u32)
 612
 613 /* INSERT IMMEDIATE */
 614 SLJIT_S390X_RILA(iihf,  0xc00800000000, sljit_u32)
 615 SLJIT_S390X_RILA(iilf,  0xc00900000000, sljit_u32)
 616
 617 /* LOAD IMMEDIATE */
 618 SLJIT_S390X_RILA(lgfi,  0xc00100000000, sljit_s32)
 619
 620 /* LOAD LOGICAL IMMEDIATE */
 621 SLJIT_S390X_RILA(llihf, 0xc00e00000000, sljit_u32)
 622 SLJIT_S390X_RILA(llilf, 0xc00f00000000, sljit_u32)
 623
 624 /* SUBTRACT LOGICAL IMMEDIATE */
 625 SLJIT_S390X_RILA(slfi,  0xc20500000000, sljit_u32)
 626
 627 #undef SLJIT_S390X_RILA
 628
 629 /* RX-a form instructions */
 630 #define SLJIT_S390X_RXA(name, pattern) \
 631 SLJIT_S390X_INSTRUCTION(name, sljit_gpr r, sljit_s32 d, sljit_gpr x, sljit_gpr b) \
 632 { \
 633         SLJIT_ASSERT((d & 0xfff) == d); \
 634 \
 635         return (pattern) | R20A(r) | R16A(x) | R12A(b) | (sljit_ins)(d & 0xfff); \
 636 }
 637
 638 /* LOAD */
 639 SLJIT_S390X_RXA(l,   0x58000000)
 640
 641 /* LOAD ADDRESS */
 642 SLJIT_S390X_RXA(la,  0x41000000)
 643
 644 /* LOAD HALFWORD */
 645 SLJIT_S390X_RXA(lh,  0x48000000)
 646
 647 /* MULTIPLY SINGLE */
 648 SLJIT_S390X_RXA(ms,  0x71000000)
 649
 650 /* STORE */
 651 SLJIT_S390X_RXA(st,  0x50000000)
 652
 653 /* STORE CHARACTER */
 654 SLJIT_S390X_RXA(stc, 0x42000000)
 655
 656 /* STORE HALFWORD */
 657 SLJIT_S390X_RXA(sth, 0x40000000)
 658
 659 #undef SLJIT_S390X_RXA
 660
 661 /* RXY-a instructions */
 662 #define SLJIT_S390X_RXYA(name, pattern, cond) \
 663 SLJIT_S390X_INSTRUCTION(name, sljit_gpr r, sljit_s32 d, sljit_gpr x, sljit_gpr b) \
 664 { \
 665         SLJIT_ASSERT(cond); \
 666 \
 667         return (pattern) | R36A(r) | R32A(x) | R28A(b) | disp_s20(d); \
 668 }
 669
 670 /* LOAD */
 671 SLJIT_S390X_RXYA(ly,    0xe30000000058, have_ldisp())
 672 SLJIT_S390X_RXYA(lg,    0xe30000000004, 1)
 673 SLJIT_S390X_RXYA(lgf,   0xe30000000014, 1)
 674
 675 /* LOAD BYTE */
 676 SLJIT_S390X_RXYA(lb,    0xe30000000076, have_ldisp())
 677 SLJIT_S390X_RXYA(lgb,   0xe30000000077, have_ldisp())
 678
 679 /* LOAD HALFWORD */
 680 SLJIT_S390X_RXYA(lhy,   0xe30000000078, have_ldisp())
 681 SLJIT_S390X_RXYA(lgh,   0xe30000000015, 1)
 682
 683 /* LOAD LOGICAL */
 684 SLJIT_S390X_RXYA(llgf,  0xe30000000016, 1)
 685
 686 /* LOAD LOGICAL CHARACTER */
 687 SLJIT_S390X_RXYA(llc,   0xe30000000094, have_eimm())
 688 SLJIT_S390X_RXYA(llgc,  0xe30000000090, 1)
 689
 690 /* LOAD LOGICAL HALFWORD */
 691 SLJIT_S390X_RXYA(llh,   0xe30000000095, have_eimm())
 692 SLJIT_S390X_RXYA(llgh,  0xe30000000091, 1)
 693
 694 /* MULTIPLY SINGLE */
 695 SLJIT_S390X_RXYA(msy,   0xe30000000051, have_ldisp())
 696 SLJIT_S390X_RXYA(msg,   0xe3000000000c, 1)
 697
 698 /* STORE */
 699 SLJIT_S390X_RXYA(sty,   0xe30000000050, have_ldisp())
 700 SLJIT_S390X_RXYA(stg,   0xe30000000024, 1)
 701
 702 /* STORE CHARACTER */
 703 SLJIT_S390X_RXYA(stcy,  0xe30000000072, have_ldisp())
 704
 705 /* STORE HALFWORD */
 706 SLJIT_S390X_RXYA(sthy,  0xe30000000070, have_ldisp())
 707
 708 #undef SLJIT_S390X_RXYA
 709
 710 /* RSY-a instructions */
 711 #define SLJIT_S390X_RSYA(name, pattern, cond) \
 712 SLJIT_S390X_INSTRUCTION(name, sljit_gpr dst, sljit_gpr src, sljit_s32 d, sljit_gpr b) \
 713 { \
 714         SLJIT_ASSERT(cond); \
 715 \
 716         return (pattern) | R36A(dst) | R32A(src) | R28A(b) | disp_s20(d); \
 717 }
 718
 719 /* LOAD MULTIPLE */
 720 SLJIT_S390X_RSYA(lmg,   0xeb0000000004, 1)
 721
 722 /* SHIFT LEFT LOGICAL */
 723 SLJIT_S390X_RSYA(sllg,  0xeb000000000d, 1)
 724
 725 /* SHIFT RIGHT SINGLE */
 726 SLJIT_S390X_RSYA(srag,  0xeb000000000a, 1)
 727
 728 /* STORE MULTIPLE */
 729 SLJIT_S390X_RSYA(stmg,  0xeb0000000024, 1)
 730
 731 #undef SLJIT_S390X_RSYA
 732
 733 /* RIE-f instructions (require general-instructions-extension facility) */
 734 #define SLJIT_S390X_RIEF(name, pattern) \
 735 SLJIT_S390X_INSTRUCTION(name, sljit_gpr dst, sljit_gpr src, sljit_u8 start, sljit_u8 end, sljit_u8 rot) \
 736 { \
 737         sljit_ins i3, i4, i5; \
 738 \
 739         SLJIT_ASSERT(have_genext()); \
 740         i3 = (sljit_ins)start << 24; \
 741         i4 = (sljit_ins)end << 16; \
 742         i5 = (sljit_ins)rot << 8; \
 743 \
 744         return (pattern) | R36A(dst & 0xf) | R32A(src & 0xf) | i3 | i4 | i5; \
 745 }
 746
 747 /* ROTATE THEN AND SELECTED BITS */
 748 /* SLJIT_S390X_RIEF(rnsbg,  0xec0000000054) */
 749
 750 /* ROTATE THEN EXCLUSIVE OR SELECTED BITS */
 751 /* SLJIT_S390X_RIEF(rxsbg,  0xec0000000057) */
 752
 753 /* ROTATE THEN OR SELECTED BITS */
 754 SLJIT_S390X_RIEF(rosbg,  0xec0000000056)
 755
 756 /* ROTATE THEN INSERT SELECTED BITS */
 757 /* SLJIT_S390X_RIEF(risbg,  0xec0000000055) */
 758 /* SLJIT_S390X_RIEF(risbgn, 0xec0000000059) */
 759
 760 /* ROTATE THEN INSERT SELECTED BITS HIGH */
 761 SLJIT_S390X_RIEF(risbhg, 0xec000000005d)
 762
 763 /* ROTATE THEN INSERT SELECTED BITS LOW */
 764 /* SLJIT_S390X_RIEF(risblg, 0xec0000000051) */
 765
 766 #undef SLJIT_S390X_RIEF
 767
 768 /* RRF-c instructions (require load/store-on-condition 1 facility) */
 769 #define SLJIT_S390X_RRFC(name, pattern) \
 770 SLJIT_S390X_INSTRUCTION(name, sljit_gpr dst, sljit_gpr src, sljit_uw mask) \
 771 { \
 772         sljit_ins m3; \
 773 \
 774         SLJIT_ASSERT(have_lscond1()); \
 775         m3 = (sljit_ins)(mask & 0xf) << 12; \
 776 \
 777         return (pattern) | m3 | R4A(dst) | R0A(src); \
 778 }
 779
 780 /* LOAD HALFWORD IMMEDIATE ON CONDITION */
 781 SLJIT_S390X_RRFC(locr,  0xb9f20000)
 782 SLJIT_S390X_RRFC(locgr, 0xb9e20000)
 783
 784 #undef SLJIT_S390X_RRFC
 785
 786 /* RIE-g instructions (require load/store-on-condition 2 facility) */
 787 #define SLJIT_S390X_RIEG(name, pattern) \
 788 SLJIT_S390X_INSTRUCTION(name, sljit_gpr reg, sljit_sw imm, sljit_uw mask) \
 789 { \
 790         sljit_ins m3, i2; \
 791 \
 792         SLJIT_ASSERT(have_lscond2()); \
 793         m3 = (sljit_ins)(mask & 0xf) << 32; \
 794         i2 = (sljit_ins)(imm & 0xffffL) << 16; \
 795 \
 796         return (pattern) | R36A(reg) | m3 | i2; \
 797 }
 798
 799 /* LOAD HALFWORD IMMEDIATE ON CONDITION */
 800 SLJIT_S390X_RIEG(lochi,  0xec0000000042)
 801 SLJIT_S390X_RIEG(locghi, 0xec0000000046)
 802
 803 #undef SLJIT_S390X_RIEG
 804
 805 #define SLJIT_S390X_RILB(name, pattern, cond) \
 806 SLJIT_S390X_INSTRUCTION(name, sljit_gpr reg, sljit_sw ri) \
 807 { \
 808         SLJIT_ASSERT(cond); \
 809 \
 810         return (pattern) | R36A(reg) | (sljit_ins)(ri & 0xffffffff); \
 811 }
 812
 813 /* BRANCH RELATIVE AND SAVE LONG */
 814 SLJIT_S390X_RILB(brasl, 0xc00500000000, 1)
 815
 816 /* LOAD ADDRESS RELATIVE LONG */
 817 SLJIT_S390X_RILB(larl,  0xc00000000000, 1)
 818
 819 /* LOAD RELATIVE LONG */
 820 SLJIT_S390X_RILB(lgrl,  0xc40800000000, have_genext())
 821
 822 #undef SLJIT_S390X_RILB
 823
 824 SLJIT_S390X_INSTRUCTION(br, sljit_gpr target)
 825 {
 826         return 0x07f0 | target;
 827 }
 828
 829 SLJIT_S390X_INSTRUCTION(brc, sljit_uw mask, sljit_sw target)
 830 {
 831         sljit_ins m1 = (sljit_ins)(mask & 0xf) << 20;
 832         sljit_ins ri2 = (sljit_ins)target & 0xffff;
 833         return 0xa7040000L | m1 | ri2;
 834 }
 835
 836 SLJIT_S390X_INSTRUCTION(brcl, sljit_uw mask, sljit_sw target)
 837 {
 838         sljit_ins m1 = (sljit_ins)(mask & 0xf) << 36;
 839         sljit_ins ri2 = (sljit_ins)target & 0xffffffff;
 840         return 0xc00400000000L | m1 | ri2;
 841 }
 842
 843 SLJIT_S390X_INSTRUCTION(flogr, sljit_gpr dst, sljit_gpr src)
 844 {
 845         SLJIT_ASSERT(have_eimm());
 846         return 0xb9830000 | R8A(dst) | R0A(src);
 847 }
 848
 849 /* INSERT PROGRAM MASK */
 850 SLJIT_S390X_INSTRUCTION(ipm, sljit_gpr dst)
 851 {
 852         return 0xb2220000 | R4A(dst);
 853 }
 854
 855 /* SET PROGRAM MASK */
 856 SLJIT_S390X_INSTRUCTION(spm, sljit_gpr dst)
 857 {
 858         return 0x0400 | R4A(dst);
 859 }
 860
 861 /* ROTATE THEN INSERT SELECTED BITS HIGH (ZERO) */
 862 SLJIT_S390X_INSTRUCTION(risbhgz, sljit_gpr dst, sljit_gpr src, sljit_u8 start, sljit_u8 end, sljit_u8 rot)
 863 {
 864         return risbhg(dst, src, start, 0x8 | end, rot);
 865 }
 866
 867 #undef SLJIT_S390X_INSTRUCTION
 868
 869 static sljit_s32 update_zero_overflow(struct sljit_compiler *compiler, sljit_s32 op, sljit_gpr dst_r)
 870 {
 871         /* Condition codes: bits 18 and 19.
 872            Transformation:
 873              0 (zero and no overflow) : unchanged
 874              1 (non-zero and no overflow) : unchanged
 875              2 (zero and overflow) : decreased by 1
 876              3 (non-zero and overflow) : decreased by 1 if non-zero */
 877         FAIL_IF(push_inst(compiler, brc(0xc, 2 + 2 + ((op & SLJIT_32) ? 1 : 2) + 2 + 3 + 1)));
 878         FAIL_IF(push_inst(compiler, ipm(tmp1)));
 879         FAIL_IF(push_inst(compiler, (op & SLJIT_32) ? or(dst_r, dst_r) : ogr(dst_r, dst_r)));
 880         FAIL_IF(push_inst(compiler, brc(0x8, 2 + 3)));
 881         FAIL_IF(push_inst(compiler, slfi(tmp1, 0x10000000)));
 882         FAIL_IF(push_inst(compiler, spm(tmp1)));
 883         return SLJIT_SUCCESS;
 884 }
 885
 886 /* load 64-bit immediate into register without clobbering flags */
 887 static sljit_s32 push_load_imm_inst(struct sljit_compiler *compiler, sljit_gpr target, sljit_sw v)
 888 {
 889         /* 4 byte instructions */
 890         if (is_s16(v))
 891                 return push_inst(compiler, lghi(target, (sljit_s16)v));
 892
 893         if (((sljit_uw)v & ~(sljit_uw)0x000000000000ffff) == 0)
 894                 return push_inst(compiler, llill(target, (sljit_u16)v));
 895
 896         if (((sljit_uw)v & ~(sljit_uw)0x00000000ffff0000) == 0)
 897                 return push_inst(compiler, llilh(target, (sljit_u16)(v >> 16)));
 898
 899         if (((sljit_uw)v & ~(sljit_uw)0x0000ffff00000000) == 0)
 900                 return push_inst(compiler, llihl(target, (sljit_u16)(v >> 32)));
 901
 902         if (((sljit_uw)v & ~(sljit_uw)0xffff000000000000) == 0)
 903                 return push_inst(compiler, llihh(target, (sljit_u16)(v >> 48)));
 904
 905         /* 6 byte instructions (requires extended immediate facility) */
 906         if (have_eimm()) {
 907                 if (is_s32(v))
 908                         return push_inst(compiler, lgfi(target, (sljit_s32)v));
 909
 910                 if (((sljit_uw)v >> 32) == 0)
 911                         return push_inst(compiler, llilf(target, (sljit_u32)v));
 912
 913                 if (((sljit_uw)v << 32) == 0)
 914                         return push_inst(compiler, llihf(target, (sljit_u32)((sljit_uw)v >> 32)));
 915
 916                 FAIL_IF(push_inst(compiler, llilf(target, (sljit_u32)v)));
 917                 return push_inst(compiler, iihf(target, (sljit_u32)(v >> 32)));
 918         }
 919
 920         /* TODO(mundaym): instruction sequences that don't use extended immediates */
 921         abort();
 922 }
 923
 924 struct addr {
 925         sljit_gpr base;
 926         sljit_gpr index;
 927         sljit_s32 offset;
 928 };
 929
 930 /* transform memory operand into D(X,B) form with a signed 20-bit offset */
 931 static sljit_s32 make_addr_bxy(struct sljit_compiler *compiler,
 932         struct addr *addr, sljit_s32 mem, sljit_sw off,
 933         sljit_gpr tmp /* clobbered, must not be r0 */)
 934 {
 935         sljit_gpr base = r0;
 936         sljit_gpr index = r0;
 937
 938         SLJIT_ASSERT(tmp != r0);
 939         if (mem & REG_MASK)
 940                 base = gpr(mem & REG_MASK);
 941
 942         if (mem & OFFS_REG_MASK) {
 943                 index = gpr(OFFS_REG(mem));
 944                 if (off != 0) {
 945                         /* shift and put the result into tmp */
 946                         SLJIT_ASSERT(0 <= off && off < 64);
 947                         FAIL_IF(push_inst(compiler, sllg(tmp, index, (sljit_s32)off, 0)));
 948                         index = tmp;
 949                         off = 0; /* clear offset */
 950                 }
 951         }
 952         else if (!is_s20(off)) {
 953                 FAIL_IF(push_load_imm_inst(compiler, tmp, off));
 954                 index = tmp;
 955                 off = 0; /* clear offset */
 956         }
 957         addr->base = base;
 958         addr->index = index;
 959         addr->offset = (sljit_s32)off;
 960         return SLJIT_SUCCESS;
 961 }
 962
 963 /* transform memory operand into D(X,B) form with an unsigned 12-bit offset */
 964 static sljit_s32 make_addr_bx(struct sljit_compiler *compiler,
 965         struct addr *addr, sljit_s32 mem, sljit_sw off,
 966         sljit_gpr tmp /* clobbered, must not be r0 */)
 967 {
 968         sljit_gpr base = r0;
 969         sljit_gpr index = r0;
 970
 971         SLJIT_ASSERT(tmp != r0);
 972         if (mem & REG_MASK)
 973                 base = gpr(mem & REG_MASK);
 974
 975         if (mem & OFFS_REG_MASK) {
 976                 index = gpr(OFFS_REG(mem));
 977                 if (off != 0) {
 978                         /* shift and put the result into tmp */
 979                         SLJIT_ASSERT(0 <= off && off < 64);
 980                         FAIL_IF(push_inst(compiler, sllg(tmp, index, (sljit_s32)off, 0)));
 981                         index = tmp;
 982                         off = 0; /* clear offset */
 983                 }
 984         }
 985         else if (!is_u12(off)) {
 986                 FAIL_IF(push_load_imm_inst(compiler, tmp, off));
 987                 index = tmp;
 988                 off = 0; /* clear offset */
 989         }
 990         addr->base = base;
 991         addr->index = index;
 992         addr->offset = (sljit_s32)off;
 993         return SLJIT_SUCCESS;
 994 }
 995
 996 #define EVAL(op, r, addr) op(r, addr.offset, addr.index, addr.base)
 997 #define WHEN(cond, r, i1, i2, addr) \
 998         (cond) ? EVAL(i1, r, addr) : EVAL(i2, r, addr)
 999
1000 /* May clobber tmp1. */
1001 static sljit_s32 load_word(struct sljit_compiler *compiler, sljit_gpr dst,
1002                 sljit_s32 src, sljit_sw srcw,
1003                 sljit_s32 is_32bit)
1004 {
1005         struct addr addr;
1006         sljit_ins ins;
1007
1008         SLJIT_ASSERT(src & SLJIT_MEM);
1009         if (have_ldisp() || !is_32bit)
1010                 FAIL_IF(make_addr_bxy(compiler, &addr, src, srcw, tmp1));
1011         else
1012                 FAIL_IF(make_addr_bx(compiler, &addr, src, srcw, tmp1));
1013
1014         if (is_32bit)
1015                 ins = WHEN(is_u12(addr.offset), dst, l, ly, addr);
1016         else
1017                 ins = lg(dst, addr.offset, addr.index, addr.base);
1018
1019         return push_inst(compiler, ins);
1020 }
1021
1022 /* May clobber tmp1. */
1023 static sljit_s32 store_word(struct sljit_compiler *compiler, sljit_gpr src,
1024                 sljit_s32 dst, sljit_sw dstw,
1025                 sljit_s32 is_32bit)
1026 {
1027         struct addr addr;
1028         sljit_ins ins;
1029
1030         SLJIT_ASSERT(dst & SLJIT_MEM);
1031         if (have_ldisp() || !is_32bit)
1032                 FAIL_IF(make_addr_bxy(compiler, &addr, dst, dstw, tmp1));
1033         else
1034                 FAIL_IF(make_addr_bx(compiler, &addr, dst, dstw, tmp1));
1035
1036         if (is_32bit)
1037                 ins = WHEN(is_u12(addr.offset), src, st, sty, addr);
1038         else
1039                 ins = stg(src, addr.offset, addr.index, addr.base);
1040
1041         return push_inst(compiler, ins);
1042 }
1043
1044 #undef WHEN
1045
1046 static sljit_s32 emit_move(struct sljit_compiler *compiler,
1047         sljit_gpr dst_r,
1048         sljit_s32 src, sljit_sw srcw)
1049 {
1050         SLJIT_ASSERT(!IS_GPR_REG(src) || dst_r != gpr(src & REG_MASK));
1051
1052         if (src & SLJIT_IMM)
1053                 return push_load_imm_inst(compiler, dst_r, srcw);
1054
1055         if (src & SLJIT_MEM)
1056                 return load_word(compiler, dst_r, src, srcw, (compiler->mode & SLJIT_32) != 0);
1057
1058         sljit_gpr src_r = gpr(src & REG_MASK);
1059         return push_inst(compiler, (compiler->mode & SLJIT_32) ? lr(dst_r, src_r) : lgr(dst_r, src_r));
1060 }
1061
1062 static sljit_s32 emit_rr(struct sljit_compiler *compiler, sljit_ins ins,
1063         sljit_s32 dst,
1064         sljit_s32 src1, sljit_sw src1w,
1065         sljit_s32 src2, sljit_sw src2w)
1066 {
1067         sljit_gpr dst_r = tmp0;
1068         sljit_gpr src_r = tmp1;
1069         sljit_s32 needs_move = 1;
1070
1071         if (FAST_IS_REG(dst)) {
1072                 dst_r = gpr(dst);
1073
1074                 if (dst == src1)
1075                         needs_move = 0;
1076                 else if (dst == src2) {
1077                         dst_r = tmp0;
1078                         needs_move = 2;
1079                 }
1080         }
1081
1082         if (needs_move)
1083                 FAIL_IF(emit_move(compiler, dst_r, src1, src1w));
1084
1085         if (FAST_IS_REG(src2))
1086                 src_r = gpr(src2);
1087         else
1088                 FAIL_IF(emit_move(compiler, tmp1, src2, src2w));
1089
1090         FAIL_IF(push_inst(compiler, ins | R4A(dst_r) | R0A(src_r)));
1091
1092         if (needs_move != 2)
1093                 return SLJIT_SUCCESS;
1094
1095         dst_r = gpr(dst & REG_MASK);
1096         return push_inst(compiler, (compiler->mode & SLJIT_32) ? lr(dst_r, tmp0) : lgr(dst_r, tmp0));
1097 }
1098
1099 static sljit_s32 emit_rr1(struct sljit_compiler *compiler, sljit_ins ins,
1100         sljit_s32 dst,
1101         sljit_s32 src1, sljit_sw src1w)
1102 {
1103         sljit_gpr dst_r = FAST_IS_REG(dst) ? gpr(dst) : tmp0;
1104         sljit_gpr src_r = tmp1;
1105
1106         if (FAST_IS_REG(src1))
1107                 src_r = gpr(src1);
1108         else
1109                 FAIL_IF(emit_move(compiler, tmp1, src1, src1w));
1110
1111         return push_inst(compiler, ins | R4A(dst_r) | R0A(src_r));
1112 }
1113
1114 static sljit_s32 emit_rrf(struct sljit_compiler *compiler, sljit_ins ins,
1115         sljit_s32 dst,
1116         sljit_s32 src1, sljit_sw src1w,
1117         sljit_s32 src2, sljit_sw src2w)
1118 {
1119         sljit_gpr dst_r = FAST_IS_REG(dst) ? gpr(dst & REG_MASK) : tmp0;
1120         sljit_gpr src1_r = tmp0;
1121         sljit_gpr src2_r = tmp1;
1122
1123         if (FAST_IS_REG(src1))
1124                 src1_r = gpr(src1);
1125         else
1126                 FAIL_IF(emit_move(compiler, tmp0, src1, src1w));
1127
1128         if (FAST_IS_REG(src2))
1129                 src2_r = gpr(src2);
1130         else
1131                 FAIL_IF(emit_move(compiler, tmp1, src2, src2w));
1132
1133         return push_inst(compiler, ins | R4A(dst_r) | R0A(src1_r) | R12A(src2_r));
1134 }
1135
1136 typedef enum {
1137         RI_A,
1138         RIL_A,
1139 } emit_ril_type;
1140
1141 static sljit_s32 emit_ri(struct sljit_compiler *compiler, sljit_ins ins,
1142         sljit_s32 dst,
1143         sljit_s32 src1, sljit_sw src1w,
1144         sljit_sw src2w,
1145         emit_ril_type type)
1146 {
1147         sljit_gpr dst_r = tmp0;
1148         sljit_s32 needs_move = 1;
1149
1150         if (FAST_IS_REG(dst)) {
1151                 dst_r = gpr(dst);
1152
1153                 if (dst == src1)
1154                         needs_move = 0;
1155         }
1156
1157         if (needs_move)
1158                 FAIL_IF(emit_move(compiler, dst_r, src1, src1w));
1159
1160         if (type == RIL_A)
1161                 return push_inst(compiler, ins | R36A(dst_r) | (src2w & 0xffffffff));
1162         return push_inst(compiler, ins | R20A(dst_r) | (src2w & 0xffff));
1163 }
1164
1165 static sljit_s32 emit_rie_d(struct sljit_compiler *compiler, sljit_ins ins,
1166         sljit_s32 dst,
1167         sljit_s32 src1, sljit_sw src1w,
1168         sljit_sw src2w)
1169 {
1170         sljit_gpr dst_r = FAST_IS_REG(dst) ? gpr(dst) : tmp0;
1171         sljit_gpr src_r = tmp0;
1172
1173         if (!FAST_IS_REG(src1))
1174                 FAIL_IF(emit_move(compiler, tmp0, src1, src1w));
1175         else
1176                 src_r = gpr(src1 & REG_MASK);
1177
1178         return push_inst(compiler, ins | R36A(dst_r) | R32A(src_r) | (sljit_ins)(src2w & 0xffff) << 16);
1179 }
1180
1181 typedef enum {
1182         RX_A,
1183         RXY_A,
1184 } emit_rx_type;
1185
1186 static sljit_s32 emit_rx(struct sljit_compiler *compiler, sljit_ins ins,
1187         sljit_s32 dst,
1188         sljit_s32 src1, sljit_sw src1w,
1189         sljit_s32 src2, sljit_sw src2w,
1190         emit_rx_type type)
1191 {
1192         sljit_gpr dst_r = tmp0;
1193         sljit_s32 needs_move = 1;
1194         sljit_gpr base, index;
1195
1196         SLJIT_ASSERT(src2 & SLJIT_MEM);
1197
1198         if (FAST_IS_REG(dst)) {
1199                 dst_r = gpr(dst);
1200
1201                 if (dst == src1)
1202                         needs_move = 0;
1203                 else if (dst == (src2 & REG_MASK) || (dst == OFFS_REG(src2))) {
1204                         dst_r = tmp0;
1205                         needs_move = 2;
1206                 }
1207         }
1208
1209         if (needs_move)
1210                 FAIL_IF(emit_move(compiler, dst_r, src1, src1w));
1211
1212         base = gpr(src2 & REG_MASK);
1213         index = tmp0;
1214
1215         if (src2 & OFFS_REG_MASK) {
1216                 index = gpr(OFFS_REG(src2));
1217
1218                 if (src2w != 0) {
1219                         FAIL_IF(push_inst(compiler, sllg(tmp1, index, src2w & 0x3, 0)));
1220                         src2w = 0;
1221                         index = tmp1;
1222                 }
1223         } else if ((type == RX_A && !is_u12(src2w)) || (type == RXY_A && !is_s20(src2w))) {
1224                 FAIL_IF(push_load_imm_inst(compiler, tmp1, src2w));
1225
1226                 if (src2 & REG_MASK)
1227                         index = tmp1;
1228                 else
1229                         base = tmp1;
1230                 src2w = 0;
1231         }
1232
1233         if (type == RX_A)
1234                 ins |= R20A(dst_r) | R16A(index) | R12A(base) | (sljit_ins)src2w;
1235         else
1236                 ins |= R36A(dst_r) | R32A(index) | R28A(base) | disp_s20((sljit_s32)src2w);
1237
1238         FAIL_IF(push_inst(compiler, ins));
1239
1240         if (needs_move != 2)
1241                 return SLJIT_SUCCESS;
1242
1243         dst_r = gpr(dst);
1244         return push_inst(compiler, (compiler->mode & SLJIT_32) ? lr(dst_r, tmp0) : lgr(dst_r, tmp0));
1245 }
1246
1247 static sljit_s32 emit_siy(struct sljit_compiler *compiler, sljit_ins ins,
1248         sljit_s32 dst, sljit_sw dstw,
1249         sljit_sw srcw)
1250 {
1251         SLJIT_ASSERT(dst & SLJIT_MEM);
1252
1253         sljit_gpr dst_r = tmp1;
1254
1255         if (dst & OFFS_REG_MASK) {
1256                 sljit_gpr index = tmp1;
1257
1258                 if ((dstw & 0x3) == 0)
1259                         index = gpr(OFFS_REG(dst));
1260                 else
1261                         FAIL_IF(push_inst(compiler, sllg(tmp1, index, dstw & 0x3, 0)));
1262
1263                 FAIL_IF(push_inst(compiler, la(tmp1, 0, dst_r, index)));
1264                 dstw = 0;
1265         }
1266         else if (!is_s20(dstw)) {
1267                 FAIL_IF(push_load_imm_inst(compiler, tmp1, dstw));
1268
1269                 if (dst & REG_MASK)
1270                         FAIL_IF(push_inst(compiler, la(tmp1, 0, dst_r, tmp1)));
1271
1272                 dstw = 0;
1273         }
1274         else
1275                 dst_r = gpr(dst & REG_MASK);
1276
1277         return push_inst(compiler, ins | ((sljit_ins)(srcw & 0xff) << 32) | R28A(dst_r) | disp_s20((sljit_s32)dstw));
1278 }
1279
1280 struct ins_forms {
1281         sljit_ins op_r;
1282         sljit_ins op_gr;
1283         sljit_ins op_rk;
1284         sljit_ins op_grk;
1285         sljit_ins op;
1286         sljit_ins op_y;
1287         sljit_ins op_g;
1288 };
1289
1290 static sljit_s32 emit_commutative(struct sljit_compiler *compiler, const struct ins_forms *forms,
1291         sljit_s32 dst,
1292         sljit_s32 src1, sljit_sw src1w,
1293         sljit_s32 src2, sljit_sw src2w)
1294 {
1295         sljit_s32 mode = compiler->mode;
1296         sljit_ins ins, ins_k;
1297
1298         if ((src1 | src2) & SLJIT_MEM) {
1299                 sljit_ins ins12, ins20;
1300
1301                 if (mode & SLJIT_32) {
1302                         ins12 = forms->op;
1303                         ins20 = forms->op_y;
1304                 }
1305                 else {
1306                         ins12 = 0;
1307                         ins20 = forms->op_g;
1308                 }
1309
1310                 if (ins12 && ins20) {
1311                         /* Extra instructions needed for address computation can be executed independently. */
1312                         if ((src2 & SLJIT_MEM) && (!(src1 & SLJIT_MEM)
1313                                         || ((src1 & OFFS_REG_MASK) ? (src1w & 0x3) == 0 : is_s20(src1w)))) {
1314                                 if ((src2 & OFFS_REG_MASK) || is_u12(src2w) || !is_s20(src2w))
1315                                         return emit_rx(compiler, ins12, dst, src1, src1w, src2, src2w, RX_A);
1316
1317                                 return emit_rx(compiler, ins20, dst, src1, src1w, src2, src2w, RXY_A);
1318                         }
1319
1320                         if (src1 & SLJIT_MEM) {
1321                                 if ((src1 & OFFS_REG_MASK) || is_u12(src1w) || !is_s20(src1w))
1322                                         return emit_rx(compiler, ins12, dst, src2, src2w, src1, src1w, RX_A);
1323
1324                                 return emit_rx(compiler, ins20, dst, src2, src2w, src1, src1w, RXY_A);
1325                         }
1326                 }
1327                 else if (ins12 || ins20) {
1328                         emit_rx_type rx_type;
1329
1330                         if (ins12) {
1331                                 rx_type = RX_A;
1332                                 ins = ins12;
1333                         }
1334                         else {
1335                                 rx_type = RXY_A;
1336                                 ins = ins20;
1337                         }
1338
1339                         if ((src2 & SLJIT_MEM) && (!(src1 & SLJIT_MEM)
1340                                         || ((src1 & OFFS_REG_MASK) ? (src1w & 0x3) == 0 : (rx_type == RX_A ? is_u12(src1w) : is_s20(src1w)))))
1341                                 return emit_rx(compiler, ins, dst, src1, src1w, src2, src2w, rx_type);
1342
1343                         if (src1 & SLJIT_MEM)
1344                                 return emit_rx(compiler, ins, dst, src2, src2w, src1, src1w, rx_type);
1345                 }
1346         }
1347
1348         if (mode & SLJIT_32) {
1349                 ins = forms->op_r;
1350                 ins_k = forms->op_rk;
1351         }
1352         else {
1353                 ins = forms->op_gr;
1354                 ins_k = forms->op_grk;
1355         }
1356
1357         SLJIT_ASSERT(ins != 0 || ins_k != 0);
1358
1359         if (ins && FAST_IS_REG(dst)) {
1360                 if (dst == src1)
1361                         return emit_rr(compiler, ins, dst, src1, src1w, src2, src2w);
1362
1363                 if (dst == src2)
1364                         return emit_rr(compiler, ins, dst, src2, src2w, src1, src1w);
1365         }
1366
1367         if (ins_k == 0)
1368                 return emit_rr(compiler, ins, dst, src1, src1w, src2, src2w);
1369
1370         return emit_rrf(compiler, ins_k, dst, src1, src1w, src2, src2w);
1371 }
1372
1373 static sljit_s32 emit_non_commutative(struct sljit_compiler *compiler, const struct ins_forms *forms,
1374         sljit_s32 dst,
1375         sljit_s32 src1, sljit_sw src1w,
1376         sljit_s32 src2, sljit_sw src2w)
1377 {
1378         sljit_s32 mode = compiler->mode;
1379         sljit_ins ins;
1380
1381         if (src2 & SLJIT_MEM) {
1382                 sljit_ins ins12, ins20;
1383
1384                 if (mode & SLJIT_32) {
1385                         ins12 = forms->op;
1386                         ins20 = forms->op_y;
1387                 }
1388                 else {
1389                         ins12 = 0;
1390                         ins20 = forms->op_g;
1391                 }
1392
1393                 if (ins12 && ins20) {
1394                         if ((src2 & OFFS_REG_MASK) || is_u12(src2w) || !is_s20(src2w))
1395                                 return emit_rx(compiler, ins12, dst, src1, src1w, src2, src2w, RX_A);
1396
1397                         return emit_rx(compiler, ins20, dst, src1, src1w, src2, src2w, RXY_A);
1398                 }
1399                 else if (ins12)
1400                         return emit_rx(compiler, ins12, dst, src1, src1w, src2, src2w, RX_A);
1401                 else if (ins20)
1402                         return emit_rx(compiler, ins20, dst, src1, src1w, src2, src2w, RXY_A);
1403         }
1404
1405         ins = (mode & SLJIT_32) ? forms->op_rk : forms->op_grk;
1406
1407         if (ins == 0 || (FAST_IS_REG(dst) && dst == src1))
1408                 return emit_rr(compiler, (mode & SLJIT_32) ? forms->op_r : forms->op_gr, dst, src1, src1w, src2, src2w);
1409
1410         return emit_rrf(compiler, ins, dst, src1, src1w, src2, src2w);
1411 }
1412
1413 SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compiler)
1414 {
1415         struct sljit_label *label;
1416         struct sljit_jump *jump;
1417         struct sljit_s390x_const *const_;
1418         struct sljit_put_label *put_label;
1419         sljit_sw executable_offset;
1420         sljit_uw ins_size = 0; /* instructions */
1421         sljit_uw pool_size = 0; /* literal pool */
1422         sljit_uw pad_size;
1423         sljit_uw i, j = 0;
1424         struct sljit_memory_fragment *buf;
1425         void *code, *code_ptr;
1426         sljit_uw *pool, *pool_ptr;
1427         sljit_sw source, offset; /* TODO(carenas): only need 32 bit */
1428
1429         CHECK_ERROR_PTR();
1430         CHECK_PTR(check_sljit_generate_code(compiler));
1431         reverse_buf(compiler);
1432
1433         /* branch handling */
1434         label = compiler->labels;
1435         jump = compiler->jumps;
1436         put_label = compiler->put_labels;
1437
1438         /* TODO(carenas): compiler->executable_size could be calculated
1439          *                before to avoid the following loop (except for
1440          *                pool_size)
1441          */
1442         /* calculate the size of the code */
1443         for (buf = compiler->buf; buf != NULL; buf = buf->next) {
1444                 sljit_uw len = buf->used_size / sizeof(sljit_ins);
1445                 sljit_ins *ibuf = (sljit_ins *)buf->memory;
1446                 for (i = 0; i < len; ++i, ++j) {
1447                         sljit_ins ins = ibuf[i];
1448
1449                         /* TODO(carenas): instruction tag vs size/addr == j
1450                          * using instruction tags for const is creative
1451                          * but unlike all other architectures, and is not
1452                          * done consistently for all other objects.
1453                          * This might need reviewing later.
1454                          */
1455                         if (ins & sljit_ins_const) {
1456                                 pool_size += sizeof(*pool);
1457                                 ins &= ~sljit_ins_const;
1458                         }
1459                         if (label && label->size == j) {
1460                                 label->size = ins_size;
1461                                 label = label->next;
1462                         }
1463                         if (jump && jump->addr == j) {
1464                                 if ((jump->flags & SLJIT_REWRITABLE_JUMP) || (jump->flags & JUMP_ADDR)) {
1465                                         /* encoded: */
1466                                         /*   brasl %r14, <rel_addr> (or brcl <mask>, <rel_addr>) */
1467                                         /* replace with: */
1468                                         /*   lgrl %r1, <pool_addr> */
1469                                         /*   bras %r14, %r1 (or bcr <mask>, %r1) */
1470                                         pool_size += sizeof(*pool);
1471                                         ins_size += 2;
1472                                 }
1473                                 jump = jump->next;
1474                         }
1475                         if (put_label && put_label->addr == j) {
1476                                 pool_size += sizeof(*pool);
1477                                 put_label = put_label->next;
1478                         }
1479                         ins_size += sizeof_ins(ins);
1480                 }
1481         }
1482
1483         /* emit trailing label */
1484         if (label && label->size == j) {
1485                 label->size = ins_size;
1486                 label = label->next;
1487         }
1488
1489         SLJIT_ASSERT(!label);
1490         SLJIT_ASSERT(!jump);
1491         SLJIT_ASSERT(!put_label);
1492
1493         /* pad code size to 8 bytes so is accessible with half word offsets */
1494         /* the literal pool needs to be doubleword aligned */
1495         pad_size = ((ins_size + 7UL) & ~7UL) - ins_size;
1496         SLJIT_ASSERT(pad_size < 8UL);
1497
1498         /* allocate target buffer */
1499         code = SLJIT_MALLOC_EXEC(ins_size + pad_size + pool_size,
1500                                         compiler->exec_allocator_data);
1501         PTR_FAIL_WITH_EXEC_IF(code);
1502         code_ptr = code;
1503         executable_offset = SLJIT_EXEC_OFFSET(code);
1504
1505         /* TODO(carenas): pool is optional, and the ABI recommends it to
1506          *                be created before the function code, instead of
1507          *                globally; if generated code is too big could
1508          *                need offsets bigger than 32bit words and asser()
1509          */
1510         pool = (sljit_uw *)((sljit_uw)code + ins_size + pad_size);
1511         pool_ptr = pool;
1512         const_ = (struct sljit_s390x_const *)compiler->consts;
1513
1514         /* update label addresses */
1515         label = compiler->labels;
1516         while (label) {
1517                 label->addr = (sljit_uw)SLJIT_ADD_EXEC_OFFSET(
1518                         (sljit_uw)code_ptr + label->size, executable_offset);
1519                 label = label->next;
1520         }
1521
1522         /* reset jumps */
1523         jump = compiler->jumps;
1524         put_label = compiler->put_labels;
1525
1526         /* emit the code */
1527         j = 0;
1528         for (buf = compiler->buf; buf != NULL; buf = buf->next) {
1529                 sljit_uw len = buf->used_size / sizeof(sljit_ins);
1530                 sljit_ins *ibuf = (sljit_ins *)buf->memory;
1531                 for (i = 0; i < len; ++i, ++j) {
1532                         sljit_ins ins = ibuf[i];
1533                         if (ins & sljit_ins_const) {
1534                                 /* clear the const tag */
1535                                 ins &= ~sljit_ins_const;
1536
1537                                 /* update instruction with relative address of constant */
1538                                 source = (sljit_sw)code_ptr;
1539                                 offset = (sljit_sw)pool_ptr - source;
1540
1541                                 SLJIT_ASSERT(!(offset & 1));
1542                                 offset >>= 1; /* halfword (not byte) offset */
1543                                 SLJIT_ASSERT(is_s32(offset));
1544
1545                                 ins |= (sljit_ins)offset & 0xffffffff;
1546
1547                                 /* update address */
1548                                 const_->const_.addr = (sljit_uw)pool_ptr;
1549
1550                                 /* store initial value into pool and update pool address */
1551                                 *(pool_ptr++) = (sljit_uw)const_->init_value;
1552
1553                                 /* move to next constant */
1554                                 const_ = (struct sljit_s390x_const *)const_->const_.next;
1555                         }
1556                         if (jump && jump->addr == j) {
1557                                 sljit_sw target = (sljit_sw)((jump->flags & JUMP_LABEL) ? jump->u.label->addr : jump->u.target);
1558                                 if ((jump->flags & SLJIT_REWRITABLE_JUMP) || (jump->flags & JUMP_ADDR)) {
1559                                         jump->addr = (sljit_uw)pool_ptr;
1560
1561                                         /* load address into tmp1 */
1562                                         source = (sljit_sw)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset);
1563                                         offset = (sljit_sw)SLJIT_ADD_EXEC_OFFSET(pool_ptr, executable_offset) - source;
1564
1565                                         SLJIT_ASSERT(!(offset & 1));
1566                                         offset >>= 1;
1567                                         SLJIT_ASSERT(is_s32(offset));
1568
1569                                         encode_inst(&code_ptr, lgrl(tmp1, offset & 0xffffffff));
1570
1571                                         /* store jump target into pool and update pool address */
1572                                         *(pool_ptr++) = (sljit_uw)target;
1573
1574                                         /* branch to tmp1 */
1575                                         sljit_ins op = (ins >> 32) & 0xf;
1576                                         sljit_ins arg = (ins >> 36) & 0xf;
1577                                         switch (op) {
1578                                         case 4: /* brcl -> bcr */
1579                                                 ins = bcr(arg, tmp1);
1580                                                 break;
1581                                         case 5: /* brasl -> basr */
1582                                                 ins = basr(arg, tmp1);
1583                                                 break;
1584                                         default:
1585                                                 abort();
1586                                         }
1587                                 }
1588                                 else {
1589                                         jump->addr = (sljit_uw)code_ptr + 2;
1590                                         source = (sljit_sw)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset);
1591                                         offset = target - source;
1592
1593                                         /* offset must be halfword aligned */
1594                                         SLJIT_ASSERT(!(offset & 1));
1595                                         offset >>= 1;
1596                                         SLJIT_ASSERT(is_s32(offset)); /* TODO(mundaym): handle arbitrary offsets */
1597
1598                                         /* patch jump target */
1599                                         ins |= (sljit_ins)offset & 0xffffffff;
1600                                 }
1601                                 jump = jump->next;
1602                         }
1603                         if (put_label && put_label->addr == j) {
1604                                 source = (sljit_sw)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset);
1605
1606                                 SLJIT_ASSERT(put_label->label);
1607                                 put_label->addr = (sljit_uw)code_ptr;
1608
1609                                 /* store target into pool */
1610                                 *pool_ptr = put_label->label->addr;
1611                                 offset = (sljit_sw)SLJIT_ADD_EXEC_OFFSET(pool_ptr, executable_offset) - source;
1612                                 pool_ptr++;
1613
1614                                 SLJIT_ASSERT(!(offset & 1));
1615                                 offset >>= 1;
1616                                 SLJIT_ASSERT(is_s32(offset));
1617                                 ins |= (sljit_ins)offset & 0xffffffff;
1618
1619                                 put_label = put_label->next;
1620                         }
1621                         encode_inst(&code_ptr, ins);
1622                 }
1623         }
1624         SLJIT_ASSERT((sljit_u8 *)code + ins_size == code_ptr);
1625         SLJIT_ASSERT((sljit_u8 *)pool + pool_size == (sljit_u8 *)pool_ptr);
1626
1627         compiler->error = SLJIT_ERR_COMPILED;
1628         compiler->executable_offset = executable_offset;
1629         compiler->executable_size = ins_size;
1630         code = SLJIT_ADD_EXEC_OFFSET(code, executable_offset);
1631         code_ptr = SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset);
1632         SLJIT_CACHE_FLUSH(code, code_ptr);
1633         SLJIT_UPDATE_WX_FLAGS(code, code_ptr, 1);
1634         return code;
1635 }
1636
1637 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_has_cpu_feature(sljit_s32 feature_type)
1638 {
1639         /* TODO(mundaym): implement all */
1640         switch (feature_type) {
1641         case SLJIT_HAS_CLZ:
1642                 return have_eimm() ? 1 : 0; /* FLOGR instruction */
1643         case SLJIT_HAS_CMOV:
1644                 return have_lscond1() ? 1 : 0;
1645         case SLJIT_HAS_FPU:
1646                 return 1;
1647         }
1648         return 0;
1649 }
1650
1651 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_cmp_info(sljit_s32 type)
1652 {
1653         return (type >= SLJIT_UNORDERED && type <= SLJIT_ORDERED_LESS_EQUAL);
1654 }
1655
1656 /* --------------------------------------------------------------------- */
1657 /*  Entry, exit                                                          */
1658 /* --------------------------------------------------------------------- */
1659
1660 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_enter(struct sljit_compiler *compiler,
1661         sljit_s32 options, sljit_s32 arg_types, sljit_s32 scratches, sljit_s32 saveds,
1662         sljit_s32 fscratches, sljit_s32 fsaveds, sljit_s32 local_size)
1663 {
1664         sljit_s32 saved_arg_count = SLJIT_KEPT_SAVEDS_COUNT(options);
1665         sljit_s32 offset, i, tmp;
1666
1667         CHECK_ERROR();
1668         CHECK(check_sljit_emit_enter(compiler, options, arg_types, scratches, saveds, fscratches, fsaveds, local_size));
1669         set_emit_enter(compiler, options, arg_types, scratches, saveds, fscratches, fsaveds, local_size);
1670
1671         /* Saved registers are stored in callee allocated save area. */
1672         SLJIT_ASSERT(gpr(SLJIT_FIRST_SAVED_REG) == r6 && gpr(SLJIT_S0) == r13);
1673
1674         offset = 2 * SSIZE_OF(sw);
1675         if (saveds + scratches >= SLJIT_NUMBER_OF_REGISTERS) {
1676                 if (saved_arg_count == 0) {
1677                         FAIL_IF(push_inst(compiler, stmg(r6, r14, offset, r15)));
1678                         offset += 9 * SSIZE_OF(sw);
1679                 } else {
1680                         FAIL_IF(push_inst(compiler, stmg(r6, r13 - (sljit_gpr)saved_arg_count, offset, r15)));
1681                         offset += (8 - saved_arg_count) * SSIZE_OF(sw);
1682                 }
1683         } else {
1684                 if (scratches == SLJIT_FIRST_SAVED_REG) {
1685                         FAIL_IF(push_inst(compiler, stg(r6, offset, 0, r15)));
1686                         offset += SSIZE_OF(sw);
1687                 } else if (scratches > SLJIT_FIRST_SAVED_REG) {
1688                         FAIL_IF(push_inst(compiler, stmg(r6, r6 + (sljit_gpr)(scratches - SLJIT_FIRST_SAVED_REG), offset, r15)));
1689                         offset += (scratches - (SLJIT_FIRST_SAVED_REG - 1)) * SSIZE_OF(sw);
1690                 }
1691
1692                 if (saved_arg_count == 0) {
1693                         if (saveds == 0) {
1694                                 FAIL_IF(push_inst(compiler, stg(r14, offset, 0, r15)));
1695                                 offset += SSIZE_OF(sw);
1696                         } else {
1697                                 FAIL_IF(push_inst(compiler, stmg(r14 - (sljit_gpr)saveds, r14, offset, r15)));
1698                                 offset += (saveds + 1) * SSIZE_OF(sw);
1699                         }
1700                 } else if (saveds > saved_arg_count) {
1701                         if (saveds == saved_arg_count + 1) {
1702                                 FAIL_IF(push_inst(compiler, stg(r14 - (sljit_gpr)saveds, offset, 0, r15)));
1703                                 offset += SSIZE_OF(sw);
1704                         } else {
1705                                 FAIL_IF(push_inst(compiler, stmg(r14 - (sljit_gpr)saveds, r13 - (sljit_gpr)saved_arg_count, offset, r15)));
1706                                 offset += (saveds - saved_arg_count) * SSIZE_OF(sw);
1707                         }
1708                 }
1709         }
1710
1711         if (saved_arg_count > 0) {
1712                 FAIL_IF(push_inst(compiler, stg(r14, offset, 0, r15)));
1713                 offset += SSIZE_OF(sw);
1714         }
1715
1716         tmp = SLJIT_FS0 - fsaveds;
1717         for (i = SLJIT_FS0; i > tmp; i--) {
1718                 FAIL_IF(push_inst(compiler, 0x60000000 /* std */ | F20(i) | R12A(r15) | (sljit_ins)offset));
1719                 offset += SSIZE_OF(sw);
1720         }
1721
1722         for (i = fscratches; i >= SLJIT_FIRST_SAVED_FLOAT_REG; i--) {
1723                 FAIL_IF(push_inst(compiler, 0x60000000 /* std */ | F20(i) | R12A(r15) | (sljit_ins)offset));
1724                 offset += SSIZE_OF(sw);
1725         }
1726
1727         local_size = (local_size + SLJIT_S390X_DEFAULT_STACK_FRAME_SIZE + 0xf) & ~0xf;
1728         compiler->local_size = local_size;
1729
1730         FAIL_IF(push_inst(compiler, 0xe30000000071 /* lay */ | R36A(r15) | R28A(r15) | disp_s20(-local_size)));
1731
1732         if (options & SLJIT_ENTER_REG_ARG)
1733                 return SLJIT_SUCCESS;
1734
1735         arg_types >>= SLJIT_ARG_SHIFT;
1736         saved_arg_count = 0;
1737         tmp = 0;
1738         while (arg_types > 0) {
1739                 if ((arg_types & SLJIT_ARG_MASK) < SLJIT_ARG_TYPE_F64) {
1740                         if (!(arg_types & SLJIT_ARG_TYPE_SCRATCH_REG)) {
1741                                 FAIL_IF(push_inst(compiler, lgr(gpr(SLJIT_S0 - saved_arg_count), gpr(SLJIT_R0 + tmp))));
1742                                 saved_arg_count++;
1743                         }
1744                         tmp++;
1745                 }
1746
1747                 arg_types >>= SLJIT_ARG_SHIFT;
1748         }
1749
1750         return SLJIT_SUCCESS;
1751 }
1752
1753 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_set_context(struct sljit_compiler *compiler,
1754         sljit_s32 options, sljit_s32 arg_types, sljit_s32 scratches, sljit_s32 saveds,
1755         sljit_s32 fscratches, sljit_s32 fsaveds, sljit_s32 local_size)
1756 {
1757         CHECK_ERROR();
1758         CHECK(check_sljit_set_context(compiler, options, arg_types, scratches, saveds, fscratches, fsaveds, local_size));
1759         set_set_context(compiler, options, arg_types, scratches, saveds, fscratches, fsaveds, local_size);
1760
1761         compiler->local_size = (local_size + SLJIT_S390X_DEFAULT_STACK_FRAME_SIZE + 0xf) & ~0xf;
1762         return SLJIT_SUCCESS;
1763 }
1764
1765 static sljit_s32 emit_stack_frame_release(struct sljit_compiler *compiler)
1766 {
1767         sljit_s32 offset, i, tmp;
1768         sljit_s32 local_size = compiler->local_size;
1769         sljit_s32 saveds = compiler->saveds;
1770         sljit_s32 scratches = compiler->scratches;
1771         sljit_s32 kept_saveds_count = SLJIT_KEPT_SAVEDS_COUNT(compiler->options);
1772
1773         if (is_u12(local_size))
1774                 FAIL_IF(push_inst(compiler, 0x41000000 /* ly */ | R20A(r15) | R12A(r15) | (sljit_ins)local_size));
1775         else
1776                 FAIL_IF(push_inst(compiler, 0xe30000000071 /* lay */ | R36A(r15) | R28A(r15) | disp_s20(local_size)));
1777
1778         offset = 2 * SSIZE_OF(sw);
1779         if (saveds + scratches >= SLJIT_NUMBER_OF_REGISTERS) {
1780                 if (kept_saveds_count == 0) {
1781                         FAIL_IF(push_inst(compiler, lmg(r6, r14, offset, r15)));
1782                         offset += 9 * SSIZE_OF(sw);
1783                 } else {
1784                         FAIL_IF(push_inst(compiler, lmg(r6, r13 - (sljit_gpr)kept_saveds_count, offset, r15)));
1785                         offset += (8 - kept_saveds_count) * SSIZE_OF(sw);
1786                 }
1787         } else {
1788                 if (scratches == SLJIT_FIRST_SAVED_REG) {
1789                         FAIL_IF(push_inst(compiler, lg(r6, offset, 0, r15)));
1790                         offset += SSIZE_OF(sw);
1791                 } else if (scratches > SLJIT_FIRST_SAVED_REG) {
1792                         FAIL_IF(push_inst(compiler, lmg(r6, r6 + (sljit_gpr)(scratches - SLJIT_FIRST_SAVED_REG), offset, r15)));
1793                         offset += (scratches - (SLJIT_FIRST_SAVED_REG - 1)) * SSIZE_OF(sw);
1794                 }
1795
1796                 if (kept_saveds_count == 0) {
1797                         if (saveds == 0) {
1798                                 FAIL_IF(push_inst(compiler, lg(r14, offset, 0, r15)));
1799                                 offset += SSIZE_OF(sw);
1800                         } else {
1801                                 FAIL_IF(push_inst(compiler, lmg(r14 - (sljit_gpr)saveds, r14, offset, r15)));
1802                                 offset += (saveds + 1) * SSIZE_OF(sw);
1803                         }
1804                 } else if (saveds > kept_saveds_count) {
1805                         if (saveds == kept_saveds_count + 1) {
1806                                 FAIL_IF(push_inst(compiler, lg(r14 - (sljit_gpr)saveds, offset, 0, r15)));
1807                                 offset += SSIZE_OF(sw);
1808                         } else {
1809                                 FAIL_IF(push_inst(compiler, lmg(r14 - (sljit_gpr)saveds, r13 - (sljit_gpr)kept_saveds_count, offset, r15)));
1810                                 offset += (saveds - kept_saveds_count) * SSIZE_OF(sw);
1811                         }
1812                 }
1813         }
1814
1815         if (kept_saveds_count > 0) {
1816                 FAIL_IF(push_inst(compiler, lg(r14, offset, 0, r15)));
1817                 offset += SSIZE_OF(sw);
1818         }
1819
1820         tmp = SLJIT_FS0 - compiler->fsaveds;
1821         for (i = SLJIT_FS0; i > tmp; i--) {
1822                 FAIL_IF(push_inst(compiler, 0x68000000 /* ld */ | F20(i) | R12A(r15) | (sljit_ins)offset));
1823                 offset += SSIZE_OF(sw);
1824         }
1825
1826         for (i = compiler->fscratches; i >= SLJIT_FIRST_SAVED_FLOAT_REG; i--) {
1827                 FAIL_IF(push_inst(compiler, 0x68000000 /* ld */ | F20(i) | R12A(r15) | (sljit_ins)offset));
1828                 offset += SSIZE_OF(sw);
1829         }
1830
1831         return SLJIT_SUCCESS;
1832 }
1833
1834 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_return_void(struct sljit_compiler *compiler)
1835 {
1836         CHECK_ERROR();
1837         CHECK(check_sljit_emit_return_void(compiler));
1838
1839         FAIL_IF(emit_stack_frame_release(compiler));
1840         return push_inst(compiler, br(r14)); /* return */
1841 }
1842
1843 /* --------------------------------------------------------------------- */
1844 /*  Operators                                                            */
1845 /* --------------------------------------------------------------------- */
1846
1847 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op0(struct sljit_compiler *compiler, sljit_s32 op)
1848 {
1849         sljit_gpr arg0 = gpr(SLJIT_R0);
1850         sljit_gpr arg1 = gpr(SLJIT_R1);
1851
1852         CHECK_ERROR();
1853         CHECK(check_sljit_emit_op0(compiler, op));
1854
1855         op = GET_OPCODE(op) | (op & SLJIT_32);
1856         switch (op) {
1857         case SLJIT_BREAKPOINT:
1858                 /* The following invalid instruction is emitted by gdb. */
1859                 return push_inst(compiler, 0x0001 /* 2-byte trap */);
1860         case SLJIT_NOP:
1861                 return push_inst(compiler, 0x0700 /* 2-byte nop */);
1862         case SLJIT_LMUL_UW:
1863                 FAIL_IF(push_inst(compiler, mlgr(arg0, arg0)));
1864                 break;
1865         case SLJIT_LMUL_SW:
1866                 /* signed multiplication from: */
1867                 /* Hacker's Delight, Second Edition: Chapter 8-3. */
1868                 FAIL_IF(push_inst(compiler, srag(tmp0, arg0, 63, 0)));
1869                 FAIL_IF(push_inst(compiler, srag(tmp1, arg1, 63, 0)));
1870                 FAIL_IF(push_inst(compiler, ngr(tmp0, arg1)));
1871                 FAIL_IF(push_inst(compiler, ngr(tmp1, arg0)));
1872
1873                 /* unsigned multiplication */
1874                 FAIL_IF(push_inst(compiler, mlgr(arg0, arg0)));
1875
1876                 FAIL_IF(push_inst(compiler, sgr(arg0, tmp0)));
1877                 FAIL_IF(push_inst(compiler, sgr(arg0, tmp1)));
1878                 break;
1879         case SLJIT_DIV_U32:
1880         case SLJIT_DIVMOD_U32:
1881                 FAIL_IF(push_inst(compiler, lhi(tmp0, 0)));
1882                 FAIL_IF(push_inst(compiler, lr(tmp1, arg0)));
1883                 FAIL_IF(push_inst(compiler, dlr(tmp0, arg1)));
1884                 FAIL_IF(push_inst(compiler, lr(arg0, tmp1))); /* quotient */
1885                 if (op == SLJIT_DIVMOD_U32)
1886                         return push_inst(compiler, lr(arg1, tmp0)); /* remainder */
1887
1888                 return SLJIT_SUCCESS;
1889         case SLJIT_DIV_S32:
1890         case SLJIT_DIVMOD_S32:
1891                 FAIL_IF(push_inst(compiler, lhi(tmp0, 0)));
1892                 FAIL_IF(push_inst(compiler, lr(tmp1, arg0)));
1893                 FAIL_IF(push_inst(compiler, dr(tmp0, arg1)));
1894                 FAIL_IF(push_inst(compiler, lr(arg0, tmp1))); /* quotient */
1895                 if (op == SLJIT_DIVMOD_S32)
1896                         return push_inst(compiler, lr(arg1, tmp0)); /* remainder */
1897
1898                 return SLJIT_SUCCESS;
1899         case SLJIT_DIV_UW:
1900         case SLJIT_DIVMOD_UW:
1901                 FAIL_IF(push_inst(compiler, lghi(tmp0, 0)));
1902                 FAIL_IF(push_inst(compiler, lgr(tmp1, arg0)));
1903                 FAIL_IF(push_inst(compiler, dlgr(tmp0, arg1)));
1904                 FAIL_IF(push_inst(compiler, lgr(arg0, tmp1))); /* quotient */
1905                 if (op == SLJIT_DIVMOD_UW)
1906                         return push_inst(compiler, lgr(arg1, tmp0)); /* remainder */
1907
1908                 return SLJIT_SUCCESS;
1909         case SLJIT_DIV_SW:
1910         case SLJIT_DIVMOD_SW:
1911                 FAIL_IF(push_inst(compiler, lgr(tmp1, arg0)));
1912                 FAIL_IF(push_inst(compiler, dsgr(tmp0, arg1)));
1913                 FAIL_IF(push_inst(compiler, lgr(arg0, tmp1))); /* quotient */
1914                 if (op == SLJIT_DIVMOD_SW)
1915                         return push_inst(compiler, lgr(arg1, tmp0)); /* remainder */
1916
1917                 return SLJIT_SUCCESS;
1918         case SLJIT_ENDBR:
1919                 return SLJIT_SUCCESS;
1920         case SLJIT_SKIP_FRAMES_BEFORE_RETURN:
1921                 return SLJIT_SUCCESS;
1922         default:
1923                 SLJIT_UNREACHABLE();
1924         }
1925         /* swap result registers */
1926         FAIL_IF(push_inst(compiler, lgr(tmp0, arg0)));
1927         FAIL_IF(push_inst(compiler, lgr(arg0, arg1)));
1928         return push_inst(compiler, lgr(arg1, tmp0));
1929 }
1930
1931 /* LEVAL will be defined later with different parameters as needed */
1932 #define WHEN2(cond, i1, i2) (cond) ? LEVAL(i1) : LEVAL(i2)
1933
1934 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op1(struct sljit_compiler *compiler, sljit_s32 op,
1935         sljit_s32 dst, sljit_sw dstw,
1936         sljit_s32 src, sljit_sw srcw)
1937 {
1938         sljit_ins ins;
1939         struct addr mem;
1940         sljit_gpr dst_r;
1941         sljit_gpr src_r;
1942         sljit_s32 opcode = GET_OPCODE(op);
1943
1944         CHECK_ERROR();
1945         CHECK(check_sljit_emit_op1(compiler, op, dst, dstw, src, srcw));
1946         ADJUST_LOCAL_OFFSET(dst, dstw);
1947         ADJUST_LOCAL_OFFSET(src, srcw);
1948
1949         if (opcode >= SLJIT_MOV && opcode <= SLJIT_MOV_P) {
1950                 /* LOAD REGISTER */
1951                 if (FAST_IS_REG(dst) && FAST_IS_REG(src)) {
1952                         dst_r = gpr(dst);
1953                         src_r = gpr(src);
1954                         switch (opcode | (op & SLJIT_32)) {
1955                         /* 32-bit */
1956                         case SLJIT_MOV32_U8:
1957                                 ins = llcr(dst_r, src_r);
1958                                 break;
1959                         case SLJIT_MOV32_S8:
1960                                 ins = lbr(dst_r, src_r);
1961                                 break;
1962                         case SLJIT_MOV32_U16:
1963                                 ins = llhr(dst_r, src_r);
1964                                 break;
1965                         case SLJIT_MOV32_S16:
1966                                 ins = lhr(dst_r, src_r);
1967                                 break;
1968                         case SLJIT_MOV32:
1969                                 if (dst_r == src_r)
1970                                         return SLJIT_SUCCESS;
1971                                 ins = lr(dst_r, src_r);
1972                                 break;
1973                         /* 64-bit */
1974                         case SLJIT_MOV_U8:
1975                                 ins = llgcr(dst_r, src_r);
1976                                 break;
1977                         case SLJIT_MOV_S8:
1978                                 ins = lgbr(dst_r, src_r);
1979                                 break;
1980                         case SLJIT_MOV_U16:
1981                                 ins = llghr(dst_r, src_r);
1982                                 break;
1983                         case SLJIT_MOV_S16:
1984                                 ins = lghr(dst_r, src_r);
1985                                 break;
1986                         case SLJIT_MOV_U32:
1987                                 ins = llgfr(dst_r, src_r);
1988                                 break;
1989                         case SLJIT_MOV_S32:
1990                                 ins = lgfr(dst_r, src_r);
1991                                 break;
1992                         case SLJIT_MOV:
1993                         case SLJIT_MOV_P:
1994                                 if (dst_r == src_r)
1995                                         return SLJIT_SUCCESS;
1996                                 ins = lgr(dst_r, src_r);
1997                                 break;
1998                         default:
1999                                 ins = 0;
2000                                 SLJIT_UNREACHABLE();
2001                                 break;
2002                         }
2003                         FAIL_IF(push_inst(compiler, ins));
2004                         return SLJIT_SUCCESS;
2005                 }
2006                 /* LOAD IMMEDIATE */
2007                 if (FAST_IS_REG(dst) && (src & SLJIT_IMM)) {
2008                         switch (opcode) {
2009                         case SLJIT_MOV_U8:
2010                                 srcw = (sljit_sw)((sljit_u8)(srcw));
2011                                 break;
2012                         case SLJIT_MOV_S8:
2013                                 srcw = (sljit_sw)((sljit_s8)(srcw));
2014                                 break;
2015                         case SLJIT_MOV_U16:
2016                                 srcw = (sljit_sw)((sljit_u16)(srcw));
2017                                 break;
2018                         case SLJIT_MOV_S16:
2019                                 srcw = (sljit_sw)((sljit_s16)(srcw));
2020                                 break;
2021                         case SLJIT_MOV_U32:
2022                                 srcw = (sljit_sw)((sljit_u32)(srcw));
2023                                 break;
2024                         case SLJIT_MOV_S32:
2025                         case SLJIT_MOV32:
2026                                 srcw = (sljit_sw)((sljit_s32)(srcw));
2027                                 break;
2028                         }
2029                         return push_load_imm_inst(compiler, gpr(dst), srcw);
2030                 }
2031                 /* LOAD */
2032                 /* TODO(carenas): avoid reg being defined later */
2033                 #define LEVAL(i) EVAL(i, reg, mem)
2034                 if (FAST_IS_REG(dst) && (src & SLJIT_MEM)) {
2035                         sljit_gpr reg = gpr(dst);
2036
2037                         FAIL_IF(make_addr_bxy(compiler, &mem, src, srcw, tmp1));
2038                         /* TODO(carenas): convert all calls below to LEVAL */
2039                         switch (opcode | (op & SLJIT_32)) {
2040                         case SLJIT_MOV32_U8:
2041                                 ins = llc(reg, mem.offset, mem.index, mem.base);
2042                                 break;
2043                         case SLJIT_MOV32_S8:
2044                                 ins = lb(reg, mem.offset, mem.index, mem.base);
2045                                 break;
2046                         case SLJIT_MOV32_U16:
2047                                 ins = llh(reg, mem.offset, mem.index, mem.base);
2048                                 break;
2049                         case SLJIT_MOV32_S16:
2050                                 ins = WHEN2(is_u12(mem.offset), lh, lhy);
2051                                 break;
2052                         case SLJIT_MOV32:
2053                                 ins = WHEN2(is_u12(mem.offset), l, ly);
2054                                 break;
2055                         case SLJIT_MOV_U8:
2056                                 ins = LEVAL(llgc);
2057                                 break;
2058                         case SLJIT_MOV_S8:
2059                                 ins = lgb(reg, mem.offset, mem.index, mem.base);
2060                                 break;
2061                         case SLJIT_MOV_U16:
2062                                 ins = LEVAL(llgh);
2063                                 break;
2064                         case SLJIT_MOV_S16:
2065                                 ins = lgh(reg, mem.offset, mem.index, mem.base);
2066                                 break;
2067                         case SLJIT_MOV_U32:
2068                                 ins = LEVAL(llgf);
2069                                 break;
2070                         case SLJIT_MOV_S32:
2071                                 ins = lgf(reg, mem.offset, mem.index, mem.base);
2072                                 break;
2073                         case SLJIT_MOV_P:
2074                         case SLJIT_MOV:
2075                                 ins = lg(reg, mem.offset, mem.index, mem.base);
2076                                 break;
2077                         default:
2078                                 ins = 0;
2079                                 SLJIT_UNREACHABLE();
2080                                 break;
2081                         }
2082                         FAIL_IF(push_inst(compiler, ins));
2083                         return SLJIT_SUCCESS;
2084                 }
2085                 /* STORE and STORE IMMEDIATE */
2086                 if ((dst & SLJIT_MEM)
2087                         && (FAST_IS_REG(src) || (src & SLJIT_IMM))) {
2088                         sljit_gpr reg = FAST_IS_REG(src) ? gpr(src) : tmp0;
2089                         if (src & SLJIT_IMM) {
2090                                 /* TODO(mundaym): MOVE IMMEDIATE? */
2091                                 FAIL_IF(push_load_imm_inst(compiler, reg, srcw));
2092                         }
2093                         struct addr mem;
2094                         FAIL_IF(make_addr_bxy(compiler, &mem, dst, dstw, tmp1));
2095                         switch (opcode) {
2096                         case SLJIT_MOV_U8:
2097                         case SLJIT_MOV_S8:
2098                                 return push_inst(compiler,
2099                                         WHEN2(is_u12(mem.offset), stc, stcy));
2100                         case SLJIT_MOV_U16:
2101                         case SLJIT_MOV_S16:
2102                                 return push_inst(compiler,
2103                                         WHEN2(is_u12(mem.offset), sth, sthy));
2104                         case SLJIT_MOV_U32:
2105                         case SLJIT_MOV_S32:
2106                         case SLJIT_MOV32:
2107                                 return push_inst(compiler,
2108                                         WHEN2(is_u12(mem.offset), st, sty));
2109                         case SLJIT_MOV_P:
2110                         case SLJIT_MOV:
2111                                 FAIL_IF(push_inst(compiler, LEVAL(stg)));
2112                                 return SLJIT_SUCCESS;
2113                         default:
2114                                 SLJIT_UNREACHABLE();
2115                         }
2116                 }
2117                 #undef LEVAL
2118                 /* MOVE CHARACTERS */
2119                 if ((dst & SLJIT_MEM) && (src & SLJIT_MEM)) {
2120                         struct addr mem;
2121                         FAIL_IF(make_addr_bxy(compiler, &mem, src, srcw, tmp1));
2122                         switch (opcode) {
2123                         case SLJIT_MOV_U8:
2124                         case SLJIT_MOV_S8:
2125                                 FAIL_IF(push_inst(compiler,
2126                                         EVAL(llgc, tmp0, mem)));
2127                                 FAIL_IF(make_addr_bxy(compiler, &mem, dst, dstw, tmp1));
2128                                 return push_inst(compiler,
2129                                         EVAL(stcy, tmp0, mem));
2130                         case SLJIT_MOV_U16:
2131                         case SLJIT_MOV_S16:
2132                                 FAIL_IF(push_inst(compiler,
2133                                         EVAL(llgh, tmp0, mem)));
2134                                 FAIL_IF(make_addr_bxy(compiler, &mem, dst, dstw, tmp1));
2135                                 return push_inst(compiler,
2136                                         EVAL(sthy, tmp0, mem));
2137                         case SLJIT_MOV_U32:
2138                         case SLJIT_MOV_S32:
2139                         case SLJIT_MOV32:
2140                                 FAIL_IF(push_inst(compiler,
2141                                         EVAL(ly, tmp0, mem)));
2142                                 FAIL_IF(make_addr_bxy(compiler, &mem, dst, dstw, tmp1));
2143                                 return push_inst(compiler,
2144                                         EVAL(sty, tmp0, mem));
2145                         case SLJIT_MOV_P:
2146                         case SLJIT_MOV:
2147                                 FAIL_IF(push_inst(compiler,
2148                                         EVAL(lg, tmp0, mem)));
2149                                 FAIL_IF(make_addr_bxy(compiler, &mem, dst, dstw, tmp1));
2150                                 FAIL_IF(push_inst(compiler,
2151                                         EVAL(stg, tmp0, mem)));
2152                                 return SLJIT_SUCCESS;
2153                         default:
2154                                 SLJIT_UNREACHABLE();
2155                         }
2156                 }
2157                 SLJIT_UNREACHABLE();
2158         }
2159
2160         SLJIT_ASSERT((src & SLJIT_IMM) == 0); /* no immediates */
2161
2162         dst_r = FAST_IS_REG(dst) ? gpr(REG_MASK & dst) : tmp0;
2163         src_r = FAST_IS_REG(src) ? gpr(REG_MASK & src) : tmp0;
2164         if (src & SLJIT_MEM)
2165                 FAIL_IF(load_word(compiler, src_r, src, srcw, src & SLJIT_32));
2166
2167         compiler->status_flags_state = op & (VARIABLE_FLAG_MASK | SLJIT_SET_Z);
2168
2169         /* TODO(mundaym): optimize loads and stores */
2170         switch (opcode | (op & SLJIT_32)) {
2171         case SLJIT_NOT:
2172                 /* emulate ~x with x^-1 */
2173                 FAIL_IF(push_load_imm_inst(compiler, tmp1, -1));
2174                 if (src_r != dst_r)
2175                         FAIL_IF(push_inst(compiler, lgr(dst_r, src_r)));
2176
2177                 FAIL_IF(push_inst(compiler, xgr(dst_r, tmp1)));
2178                 break;
2179         case SLJIT_NOT32:
2180                 /* emulate ~x with x^-1 */
2181                 if (have_eimm())
2182                         FAIL_IF(push_inst(compiler, xilf(dst_r, 0xffffffff)));
2183                 else {
2184                         FAIL_IF(push_load_imm_inst(compiler, tmp1, -1));
2185                         if (src_r != dst_r)
2186                                 FAIL_IF(push_inst(compiler, lr(dst_r, src_r)));
2187
2188                         FAIL_IF(push_inst(compiler, xr(dst_r, tmp1)));
2189                 }
2190                 break;
2191         case SLJIT_CLZ:
2192                 if (have_eimm()) {
2193                         FAIL_IF(push_inst(compiler, flogr(tmp0, src_r))); /* clobbers tmp1 */
2194                         if (dst_r != tmp0)
2195                                 FAIL_IF(push_inst(compiler, lgr(dst_r, tmp0)));
2196                 } else {
2197                         abort(); /* TODO(mundaym): no eimm (?) */
2198                 }
2199                 break;
2200         case SLJIT_CLZ32:
2201                 if (have_eimm()) {
2202                         FAIL_IF(push_inst(compiler, sllg(tmp1, src_r, 32, 0)));
2203                         FAIL_IF(push_inst(compiler, iilf(tmp1, 0xffffffff)));
2204                         FAIL_IF(push_inst(compiler, flogr(tmp0, tmp1))); /* clobbers tmp1 */
2205                         if (dst_r != tmp0)
2206                                 FAIL_IF(push_inst(compiler, lr(dst_r, tmp0)));
2207                 } else {
2208                         abort(); /* TODO(mundaym): no eimm (?) */
2209                 }
2210                 break;
2211         default:
2212                 SLJIT_UNREACHABLE();
2213         }
2214
2215         if ((op & (SLJIT_SET_Z | VARIABLE_FLAG_MASK)) == (SLJIT_SET_Z | SLJIT_SET_OVERFLOW))
2216                 FAIL_IF(update_zero_overflow(compiler, op, dst_r));
2217
2218         /* TODO(carenas): doesn't need FAIL_IF */
2219         if (dst & SLJIT_MEM)
2220                 FAIL_IF(store_word(compiler, dst_r, dst, dstw, op & SLJIT_32));
2221
2222         return SLJIT_SUCCESS;
2223 }
2224
2225 static SLJIT_INLINE int is_commutative(sljit_s32 op)
2226 {
2227         switch (GET_OPCODE(op)) {
2228         case SLJIT_ADD:
2229         case SLJIT_ADDC:
2230         case SLJIT_MUL:
2231         case SLJIT_AND:
2232         case SLJIT_OR:
2233         case SLJIT_XOR:
2234                 return 1;
2235         }
2236         return 0;
2237 }
2238
2239 static SLJIT_INLINE int is_shift(sljit_s32 op) {
2240         sljit_s32 v = GET_OPCODE(op);
2241         return (v == SLJIT_SHL || v == SLJIT_ASHR || v == SLJIT_LSHR) ? 1 : 0;
2242 }
2243
2244 static const struct ins_forms add_forms = {
2245         0x1a00, /* ar */
2246         0xb9080000, /* agr */
2247         0xb9f80000, /* ark */
2248         0xb9e80000, /* agrk */
2249         0x5a000000, /* a */
2250         0xe3000000005a, /* ay */
2251         0xe30000000008, /* ag */
2252 };
2253
2254 static const struct ins_forms logical_add_forms = {
2255         0x1e00, /* alr */
2256         0xb90a0000, /* algr */
2257         0xb9fa0000, /* alrk */
2258         0xb9ea0000, /* algrk */
2259         0x5e000000, /* al */
2260         0xe3000000005e, /* aly */
2261         0xe3000000000a, /* alg */
2262 };
2263
2264 static sljit_s32 sljit_emit_add(struct sljit_compiler *compiler, sljit_s32 op,
2265         sljit_s32 dst, sljit_sw dstw,
2266         sljit_s32 src1, sljit_sw src1w,
2267         sljit_s32 src2, sljit_sw src2w)
2268 {
2269         int sets_overflow = (op & VARIABLE_FLAG_MASK) == SLJIT_SET_OVERFLOW;
2270         int sets_zero_overflow = (op & (SLJIT_SET_Z | VARIABLE_FLAG_MASK)) == (SLJIT_SET_Z | SLJIT_SET_OVERFLOW);
2271         const struct ins_forms *forms;
2272         sljit_ins ins;
2273
2274         if (src2 & SLJIT_IMM) {
2275                 if (!sets_zero_overflow && is_s8(src2w) && (src1 & SLJIT_MEM) && (dst == src1 && dstw == src1w)) {
2276                         if (sets_overflow)
2277                                 ins = (op & SLJIT_32) ? 0xeb000000006a /* asi */ : 0xeb000000007a /* agsi */;
2278                         else
2279                                 ins = (op & SLJIT_32) ? 0xeb000000006e /* alsi */ : 0xeb000000007e /* algsi */;
2280                         return emit_siy(compiler, ins, dst, dstw, src2w);
2281                 }
2282
2283                 if (is_s16(src2w)) {
2284                         if (sets_overflow)
2285                                 ins = (op & SLJIT_32) ? 0xec00000000d8 /* ahik */ : 0xec00000000d9 /* aghik */;
2286                         else
2287                                 ins = (op & SLJIT_32) ? 0xec00000000da /* alhsik */ : 0xec00000000db /* alghsik */;
2288                         FAIL_IF(emit_rie_d(compiler, ins, dst, src1, src1w, src2w));
2289                         goto done;
2290                 }
2291
2292                 if (!sets_overflow) {
2293                         if ((op & SLJIT_32) || is_u32(src2w)) {
2294                                 ins = (op & SLJIT_32) ? 0xc20b00000000 /* alfi */ : 0xc20a00000000 /* algfi */;
2295                                 FAIL_IF(emit_ri(compiler, ins, dst, src1, src1w, src2w, RIL_A));
2296                                 goto done;
2297                         }
2298                         if (is_u32(-src2w)) {
2299                                 FAIL_IF(emit_ri(compiler, 0xc20400000000 /* slgfi */, dst, src1, src1w, -src2w, RIL_A));
2300                                 goto done;
2301                         }
2302                 }
2303                 else if ((op & SLJIT_32) || is_s32(src2w)) {
2304                         ins = (op & SLJIT_32) ? 0xc20900000000 /* afi */ : 0xc20800000000 /* agfi */;
2305                         FAIL_IF(emit_ri(compiler, ins, dst, src1, src1w, src2w, RIL_A));
2306                         goto done;
2307                 }
2308         }
2309
2310         forms = sets_overflow ? &add_forms : &logical_add_forms;
2311         FAIL_IF(emit_commutative(compiler, forms, dst, src1, src1w, src2, src2w));
2312
2313 done:
2314         if (sets_zero_overflow)
2315                 FAIL_IF(update_zero_overflow(compiler, op, FAST_IS_REG(dst) ? gpr(dst & REG_MASK) : tmp0));
2316
2317         if (dst & SLJIT_MEM)
2318                 return store_word(compiler, tmp0, dst, dstw, op & SLJIT_32);
2319
2320         return SLJIT_SUCCESS;
2321 }
2322
2323 static const struct ins_forms sub_forms = {
2324         0x1b00, /* sr */
2325         0xb9090000, /* sgr */
2326         0xb9f90000, /* srk */
2327         0xb9e90000, /* sgrk */
2328         0x5b000000, /* s */
2329         0xe3000000005b, /* sy */
2330         0xe30000000009, /* sg */
2331 };
2332
2333 static const struct ins_forms logical_sub_forms = {
2334         0x1f00, /* slr */
2335         0xb90b0000, /* slgr */
2336         0xb9fb0000, /* slrk */
2337         0xb9eb0000, /* slgrk */
2338         0x5f000000, /* sl */
2339         0xe3000000005f, /* sly */
2340         0xe3000000000b, /* slg */
2341 };
2342
2343 static sljit_s32 sljit_emit_sub(struct sljit_compiler *compiler, sljit_s32 op,
2344         sljit_s32 dst, sljit_sw dstw,
2345         sljit_s32 src1, sljit_sw src1w,
2346         sljit_s32 src2, sljit_sw src2w)
2347 {
2348         sljit_s32 flag_type = GET_FLAG_TYPE(op);
2349         int sets_signed = (flag_type >= SLJIT_SIG_LESS && flag_type <= SLJIT_NOT_OVERFLOW);
2350         int sets_zero_overflow = (op & (SLJIT_SET_Z | VARIABLE_FLAG_MASK)) == (SLJIT_SET_Z | SLJIT_SET_OVERFLOW);
2351         const struct ins_forms *forms;
2352         sljit_ins ins;
2353
2354         if (dst == (sljit_s32)tmp0 && flag_type <= SLJIT_SIG_LESS_EQUAL) {
2355                 int compare_signed = flag_type >= SLJIT_SIG_LESS;
2356
2357                 compiler->status_flags_state |= SLJIT_CURRENT_FLAGS_COMPARE;
2358
2359                 if (src2 & SLJIT_IMM) {
2360                         if (compare_signed || ((op & VARIABLE_FLAG_MASK) == 0 && is_s32(src2w)))
2361                         {
2362                                 if ((op & SLJIT_32) || is_s32(src2w)) {
2363                                         ins = (op & SLJIT_32) ? 0xc20d00000000 /* cfi */ : 0xc20c00000000 /* cgfi */;
2364                                         return emit_ri(compiler, ins, src1, src1, src1w, src2w, RIL_A);
2365                                 }
2366                         }
2367                         else {
2368                                 if ((op & SLJIT_32) || is_u32(src2w)) {
2369                                         ins = (op & SLJIT_32) ? 0xc20f00000000 /* clfi */ : 0xc20e00000000 /* clgfi */;
2370                                         return emit_ri(compiler, ins, src1, src1, src1w, src2w, RIL_A);
2371                                 }
2372                                 if (is_s16(src2w))
2373                                         return emit_rie_d(compiler, 0xec00000000db /* alghsik */, (sljit_s32)tmp0, src1, src1w, src2w);
2374                         }
2375                 }
2376                 else if (src2 & SLJIT_MEM) {
2377                         if ((op & SLJIT_32) && ((src2 & OFFS_REG_MASK) || is_u12(src2w))) {
2378                                 ins = compare_signed ? 0x59000000 /* c */ : 0x55000000 /* cl */;
2379                                 return emit_rx(compiler, ins, src1, src1, src1w, src2, src2w, RX_A);
2380                         }
2381
2382                         if (compare_signed)
2383                                 ins = (op & SLJIT_32) ? 0xe30000000059 /* cy */ : 0xe30000000020 /* cg */;
2384                         else
2385                                 ins = (op & SLJIT_32) ? 0xe30000000055 /* cly */ : 0xe30000000021 /* clg */;
2386                         return emit_rx(compiler, ins, src1, src1, src1w, src2, src2w, RXY_A);
2387                 }
2388
2389                 if (compare_signed)
2390                         ins = (op & SLJIT_32) ? 0x1900 /* cr */ : 0xb9200000 /* cgr */;
2391                 else
2392                         ins = (op & SLJIT_32) ? 0x1500 /* clr */ : 0xb9210000 /* clgr */;
2393                 return emit_rr(compiler, ins, src1, src1, src1w, src2, src2w);
2394         }
2395
2396         if (src1 == SLJIT_IMM && src1w == 0 && (flag_type == 0 || sets_signed)) {
2397                 ins = (op & SLJIT_32) ? 0x1300 /* lcr */ : 0xb9030000 /* lcgr */;
2398                 FAIL_IF(emit_rr1(compiler, ins, dst, src2, src2w));
2399                 goto done;
2400         }
2401
2402         if (src2 & SLJIT_IMM) {
2403                 sljit_sw neg_src2w = -src2w;
2404
2405                 if (sets_signed || neg_src2w != 0 || (op & (SLJIT_SET_Z | VARIABLE_FLAG_MASK)) == 0) {
2406                         if (!sets_zero_overflow && is_s8(neg_src2w) && (src1 & SLJIT_MEM) && (dst == src1 && dstw == src1w)) {
2407                                 if (sets_signed)
2408                                         ins = (op & SLJIT_32) ? 0xeb000000006a /* asi */ : 0xeb000000007a /* agsi */;
2409                                 else
2410                                         ins = (op & SLJIT_32) ? 0xeb000000006e /* alsi */ : 0xeb000000007e /* algsi */;
2411                                 return emit_siy(compiler, ins, dst, dstw, neg_src2w);
2412                         }
2413
2414                         if (is_s16(neg_src2w)) {
2415                                 if (sets_signed)
2416                                         ins = (op & SLJIT_32) ? 0xec00000000d8 /* ahik */ : 0xec00000000d9 /* aghik */;
2417                                 else
2418                                         ins = (op & SLJIT_32) ? 0xec00000000da /* alhsik */ : 0xec00000000db /* alghsik */;
2419                                 FAIL_IF(emit_rie_d(compiler, ins, dst, src1, src1w, neg_src2w));
2420                                 goto done;
2421                         }
2422                 }
2423
2424                 if (!sets_signed) {
2425                         if ((op & SLJIT_32) || is_u32(src2w)) {
2426                                 ins = (op & SLJIT_32) ? 0xc20500000000 /* slfi */ : 0xc20400000000 /* slgfi */;
2427                                 FAIL_IF(emit_ri(compiler, ins, dst, src1, src1w, src2w, RIL_A));
2428                                 goto done;
2429                         }
2430                         if (is_u32(neg_src2w)) {
2431                                 FAIL_IF(emit_ri(compiler, 0xc20a00000000 /* algfi */, dst, src1, src1w, neg_src2w, RIL_A));
2432                                 goto done;
2433                         }
2434                 }
2435                 else if ((op & SLJIT_32) || is_s32(neg_src2w)) {
2436                         ins = (op & SLJIT_32) ? 0xc20900000000 /* afi */ : 0xc20800000000 /* agfi */;
2437                         FAIL_IF(emit_ri(compiler, ins, dst, src1, src1w, neg_src2w, RIL_A));
2438                         goto done;
2439                 }
2440         }
2441
2442         forms = sets_signed ? &sub_forms : &logical_sub_forms;
2443         FAIL_IF(emit_non_commutative(compiler, forms, dst, src1, src1w, src2, src2w));
2444
2445 done:
2446         if (sets_signed) {
2447                 sljit_gpr dst_r = FAST_IS_REG(dst) ? gpr(dst & REG_MASK) : tmp0;
2448
2449                 if ((op & VARIABLE_FLAG_MASK) != SLJIT_SET_OVERFLOW) {
2450                         /* In case of overflow, the sign bit of the two source operands must be different, and
2451                              - the first operand is greater if the sign bit of the result is set
2452                              - the first operand is less if the sign bit of the result is not set
2453                            The -result operation sets the corrent sign, because the result cannot be zero.
2454                            The overflow is considered greater, since the result must be equal to INT_MIN so its sign bit is set. */
2455                         FAIL_IF(push_inst(compiler, brc(0xe, 2 + 2)));
2456                         FAIL_IF(push_inst(compiler, (op & SLJIT_32) ? lcr(tmp1, dst_r) : lcgr(tmp1, dst_r)));
2457                 }
2458                 else if (op & SLJIT_SET_Z)
2459                         FAIL_IF(update_zero_overflow(compiler, op, dst_r));
2460         }
2461
2462         if (dst & SLJIT_MEM)
2463                 return store_word(compiler, tmp0, dst, dstw, op & SLJIT_32);
2464
2465         return SLJIT_SUCCESS;
2466 }
2467
2468 static const struct ins_forms multiply_forms = {
2469         0xb2520000, /* msr */
2470         0xb90c0000, /* msgr */
2471         0xb9fd0000, /* msrkc */
2472         0xb9ed0000, /* msgrkc */
2473         0x71000000, /* ms */
2474         0xe30000000051, /* msy */
2475         0xe3000000000c, /* msg */
2476 };
2477
2478 static const struct ins_forms multiply_overflow_forms = {
2479         0,
2480         0,
2481         0xb9fd0000, /* msrkc */
2482         0xb9ed0000, /* msgrkc */
2483         0,
2484         0xe30000000053, /* msc */
2485         0xe30000000083, /* msgc */
2486 };
2487
2488 static sljit_s32 sljit_emit_multiply(struct sljit_compiler *compiler, sljit_s32 op,
2489         sljit_s32 dst,
2490         sljit_s32 src1, sljit_sw src1w,
2491         sljit_s32 src2, sljit_sw src2w)
2492 {
2493         sljit_ins ins;
2494
2495         if (HAS_FLAGS(op)) {
2496                 /* if have_misc2 fails, this operation should be emulated. 32 bit emulation:
2497                 FAIL_IF(push_inst(compiler, lgfr(tmp0, src1_r)));
2498                 FAIL_IF(push_inst(compiler, msgfr(tmp0, src2_r)));
2499                 if (dst_r != tmp0) {
2500                         FAIL_IF(push_inst(compiler, lr(dst_r, tmp0)));
2501                 }
2502                 FAIL_IF(push_inst(compiler, aih(tmp0, 1)));
2503                 FAIL_IF(push_inst(compiler, nihf(tmp0, ~1U)));
2504                 FAIL_IF(push_inst(compiler, ipm(tmp1)));
2505                 FAIL_IF(push_inst(compiler, oilh(tmp1, 0x2000))); */
2506
2507                 return emit_commutative(compiler, &multiply_overflow_forms, dst, src1, src1w, src2, src2w);
2508         }
2509
2510         if (src2 & SLJIT_IMM) {
2511                 if (is_s16(src2w)) {
2512                         ins = (op & SLJIT_32) ? 0xa70c0000 /* mhi */ : 0xa70d0000 /* mghi */;
2513                         return emit_ri(compiler, ins, dst, src1, src1w, src2w, RI_A);
2514                 }
2515
2516                 if (is_s32(src2w)) {
2517                         ins = (op & SLJIT_32) ? 0xc20100000000 /* msfi */ : 0xc20000000000 /* msgfi */;
2518                         return emit_ri(compiler, ins, dst, src1, src1w, src2w, RIL_A);
2519                 }
2520         }
2521
2522         return emit_commutative(compiler, &multiply_forms, dst, src1, src1w, src2, src2w);
2523 }
2524
2525 static sljit_s32 sljit_emit_bitwise_imm(struct sljit_compiler *compiler, sljit_s32 type,
2526         sljit_s32 dst,
2527         sljit_s32 src1, sljit_sw src1w,
2528         sljit_uw imm, sljit_s32 count16)
2529 {
2530         sljit_s32 mode = compiler->mode;
2531         sljit_gpr dst_r = tmp0;
2532         sljit_s32 needs_move = 1;
2533
2534         if (IS_GPR_REG(dst)) {
2535                 dst_r = gpr(dst & REG_MASK);
2536                 if (dst == src1)
2537                         needs_move = 0;
2538         }
2539
2540         if (needs_move)
2541                 FAIL_IF(emit_move(compiler, dst_r, src1, src1w));
2542
2543         if (type == SLJIT_AND) {
2544                 if (!(mode & SLJIT_32))
2545                         FAIL_IF(push_inst(compiler, 0xc00a00000000 /* nihf */ | R36A(dst_r) | (imm >> 32)));
2546                 return push_inst(compiler, 0xc00b00000000 /* nilf */ | R36A(dst_r) | (imm & 0xffffffff));
2547         }
2548         else if (type == SLJIT_OR) {
2549                 if (count16 >= 3) {
2550                         FAIL_IF(push_inst(compiler, 0xc00c00000000 /* oihf */ | R36A(dst_r) | (imm >> 32)));
2551                         return push_inst(compiler, 0xc00d00000000 /* oilf */ | R36A(dst_r) | (imm & 0xffffffff));
2552                 }
2553
2554                 if (count16 >= 2) {
2555                         if ((imm & 0x00000000ffffffffull) == 0)
2556                                 return push_inst(compiler, 0xc00c00000000 /* oihf */ | R36A(dst_r) | (imm >> 32));
2557                         if ((imm & 0xffffffff00000000ull) == 0)
2558                                 return push_inst(compiler, 0xc00d00000000 /* oilf */ | R36A(dst_r) | (imm & 0xffffffff));
2559                 }
2560
2561                 if ((imm & 0xffff000000000000ull) != 0)
2562                         FAIL_IF(push_inst(compiler, 0xa5080000 /* oihh */ | R20A(dst_r) | (imm >> 48)));
2563                 if ((imm & 0x0000ffff00000000ull) != 0)
2564                         FAIL_IF(push_inst(compiler, 0xa5090000 /* oihl */ | R20A(dst_r) | ((imm >> 32) & 0xffff)));
2565                 if ((imm & 0x00000000ffff0000ull) != 0)
2566                         FAIL_IF(push_inst(compiler, 0xa50a0000 /* oilh */ | R20A(dst_r) | ((imm >> 16) & 0xffff)));
2567                 if ((imm & 0x000000000000ffffull) != 0 || imm == 0)
2568                         return push_inst(compiler, 0xa50b0000 /* oill */ | R20A(dst_r) | (imm & 0xffff));
2569                 return SLJIT_SUCCESS;
2570         }
2571
2572         if ((imm & 0xffffffff00000000ull) != 0)
2573                 FAIL_IF(push_inst(compiler, 0xc00600000000 /* xihf */ | R36A(dst_r) | (imm >> 32)));
2574         if ((imm & 0x00000000ffffffffull) != 0 || imm == 0)
2575                 return push_inst(compiler, 0xc00700000000 /* xilf */ | R36A(dst_r) | (imm & 0xffffffff));
2576         return SLJIT_SUCCESS;
2577 }
2578
2579 static const struct ins_forms bitwise_and_forms = {
2580         0x1400, /* nr */
2581         0xb9800000, /* ngr */
2582         0xb9f40000, /* nrk */
2583         0xb9e40000, /* ngrk */
2584         0x54000000, /* n */
2585         0xe30000000054, /* ny */
2586         0xe30000000080, /* ng */
2587 };
2588
2589 static const struct ins_forms bitwise_or_forms = {
2590         0x1600, /* or */
2591         0xb9810000, /* ogr */
2592         0xb9f60000, /* ork */
2593         0xb9e60000, /* ogrk */
2594         0x56000000, /* o */
2595         0xe30000000056, /* oy */
2596         0xe30000000081, /* og */
2597 };
2598
2599 static const struct ins_forms bitwise_xor_forms = {
2600         0x1700, /* xr */
2601         0xb9820000, /* xgr */
2602         0xb9f70000, /* xrk */
2603         0xb9e70000, /* xgrk */
2604         0x57000000, /* x */
2605         0xe30000000057, /* xy */
2606         0xe30000000082, /* xg */
2607 };
2608
2609 static sljit_s32 sljit_emit_bitwise(struct sljit_compiler *compiler, sljit_s32 op,
2610         sljit_s32 dst,
2611         sljit_s32 src1, sljit_sw src1w,
2612         sljit_s32 src2, sljit_sw src2w)
2613 {
2614         sljit_s32 type = GET_OPCODE(op);
2615         const struct ins_forms *forms;
2616
2617         if ((src2 & SLJIT_IMM) && (!(op & SLJIT_SET_Z) || (type == SLJIT_AND && dst == (sljit_s32)tmp0))) {
2618                 sljit_s32 count16 = 0;
2619                 sljit_uw imm = (sljit_uw)src2w;
2620
2621                 if (op & SLJIT_32)
2622                         imm &= 0xffffffffull;
2623
2624                 if ((imm & 0x000000000000ffffull) != 0 || imm == 0)
2625                         count16++;
2626                 if ((imm & 0x00000000ffff0000ull) != 0)
2627                         count16++;
2628                 if ((imm & 0x0000ffff00000000ull) != 0)
2629                         count16++;
2630                 if ((imm & 0xffff000000000000ull) != 0)
2631                         count16++;
2632
2633                 if (type == SLJIT_AND && dst == (sljit_s32)tmp0 && count16 == 1) {
2634                         sljit_gpr src_r = tmp0;
2635
2636                         if (FAST_IS_REG(src1))
2637                                 src_r = gpr(src1 & REG_MASK);
2638                         else
2639                                 FAIL_IF(emit_move(compiler, tmp0, src1, src1w));
2640
2641                         if ((imm & 0x000000000000ffffull) != 0 || imm == 0)
2642                                 return push_inst(compiler, 0xa7010000 | R20A(src_r) | imm);
2643                         if ((imm & 0x00000000ffff0000ull) != 0)
2644                                 return push_inst(compiler, 0xa7000000 | R20A(src_r) | (imm >> 16));
2645                         if ((imm & 0x0000ffff00000000ull) != 0)
2646                                 return push_inst(compiler, 0xa7030000 | R20A(src_r) | (imm >> 32));
2647                         return push_inst(compiler, 0xa7020000 | R20A(src_r) | (imm >> 48));
2648                 }
2649
2650                 if (!(op & SLJIT_SET_Z))
2651                         return sljit_emit_bitwise_imm(compiler, type, dst, src1, src1w, imm, count16);
2652         }
2653
2654         if (type == SLJIT_AND)
2655                 forms = &bitwise_and_forms;
2656         else if (type == SLJIT_OR)
2657                 forms = &bitwise_or_forms;
2658         else
2659                 forms = &bitwise_xor_forms;
2660
2661         return emit_commutative(compiler, forms, dst, src1, src1w, src2, src2w);
2662 }
2663
2664 static sljit_s32 sljit_emit_shift(struct sljit_compiler *compiler, sljit_s32 op,
2665         sljit_s32 dst,
2666         sljit_s32 src1, sljit_sw src1w,
2667         sljit_s32 src2, sljit_sw src2w)
2668 {
2669         sljit_s32 type = GET_OPCODE(op);
2670         sljit_gpr dst_r = FAST_IS_REG(dst) ? gpr(dst & REG_MASK) : tmp0;
2671         sljit_gpr src_r = tmp0;
2672         sljit_gpr base_r = tmp0;
2673         sljit_ins imm = 0;
2674         sljit_ins ins;
2675
2676         if (FAST_IS_REG(src1))
2677                 src_r = gpr(src1 & REG_MASK);
2678         else
2679                 FAIL_IF(emit_move(compiler, tmp0, src1, src1w));
2680
2681         if (src2 & SLJIT_IMM)
2682                 imm = (sljit_ins)(src2w & ((op & SLJIT_32) ? 0x1f : 0x3f));
2683         else if (FAST_IS_REG(src2))
2684                 base_r = gpr(src2 & REG_MASK);
2685         else {
2686                 FAIL_IF(emit_move(compiler, tmp1, src2, src2w));
2687                 base_r = tmp1;
2688         }
2689
2690         if ((op & SLJIT_32) && dst_r == src_r) {
2691                 if (type == SLJIT_SHL)
2692                         ins = 0x89000000 /* sll */;
2693                 else if (type == SLJIT_LSHR)
2694                         ins = 0x88000000 /* srl */;
2695                 else
2696                         ins = 0x8a000000 /* sra */;
2697
2698                 FAIL_IF(push_inst(compiler, ins | R20A(dst_r) | R12A(base_r) | imm));
2699         }
2700         else {
2701                 if (type == SLJIT_SHL)
2702                         ins = (op & SLJIT_32) ? 0xeb00000000df /* sllk */ : 0xeb000000000d /* sllg */;
2703                 else if (type == SLJIT_LSHR)
2704                         ins = (op & SLJIT_32) ? 0xeb00000000de /* srlk */ : 0xeb000000000c /* srlg */;
2705                 else
2706                         ins = (op & SLJIT_32) ? 0xeb00000000dc /* srak */ : 0xeb000000000a /* srag */;
2707
2708                 FAIL_IF(push_inst(compiler, ins | R36A(dst_r) | R32A(src_r) | R28A(base_r) | (imm << 16)));
2709         }
2710
2711         if ((op & SLJIT_SET_Z) && type != SLJIT_ASHR)
2712                 return push_inst(compiler, (op & SLJIT_32) ? or(dst_r, dst_r) : ogr(dst_r, dst_r));
2713
2714         return SLJIT_SUCCESS;
2715 }
2716
2717 static const struct ins_forms addc_forms = {
2718         0xb9980000, /* alcr */
2719         0xb9880000, /* alcgr */
2720         0,
2721         0,
2722         0,
2723         0xe30000000098, /* alc */
2724         0xe30000000088, /* alcg */
2725 };
2726
2727 static const struct ins_forms subc_forms = {
2728         0xb9990000, /* slbr */
2729         0xb9890000, /* slbgr */
2730         0,
2731         0,
2732         0,
2733         0xe30000000099, /* slb */
2734         0xe30000000089, /* slbg */
2735 };
2736
2737 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op2(struct sljit_compiler *compiler, sljit_s32 op,
2738         sljit_s32 dst, sljit_sw dstw,
2739         sljit_s32 src1, sljit_sw src1w,
2740         sljit_s32 src2, sljit_sw src2w)
2741 {
2742         CHECK_ERROR();
2743         CHECK(check_sljit_emit_op2(compiler, op, 0, dst, dstw, src1, src1w, src2, src2w));
2744         ADJUST_LOCAL_OFFSET(dst, dstw);
2745         ADJUST_LOCAL_OFFSET(src1, src1w);
2746         ADJUST_LOCAL_OFFSET(src2, src2w);
2747
2748         compiler->mode = op & SLJIT_32;
2749         compiler->status_flags_state = op & (VARIABLE_FLAG_MASK | SLJIT_SET_Z);
2750
2751         if (is_commutative(op) && (src1 & SLJIT_IMM) && !(src2 & SLJIT_IMM)) {
2752                 src1 ^= src2;
2753                 src2 ^= src1;
2754                 src1 ^= src2;
2755
2756                 src1w ^= src2w;
2757                 src2w ^= src1w;
2758                 src1w ^= src2w;
2759         }
2760
2761         switch (GET_OPCODE(op)) {
2762         case SLJIT_ADD:
2763                 compiler->status_flags_state |= SLJIT_CURRENT_FLAGS_ADD;
2764                 return sljit_emit_add(compiler, op, dst, dstw, src1, src1w, src2, src2w);
2765         case SLJIT_ADDC:
2766                 compiler->status_flags_state |= SLJIT_CURRENT_FLAGS_ADD;
2767                 FAIL_IF(emit_commutative(compiler, &addc_forms, dst, src1, src1w, src2, src2w));
2768                 if (dst & SLJIT_MEM)
2769                         return store_word(compiler, tmp0, dst, dstw, op & SLJIT_32);
2770                 return SLJIT_SUCCESS;
2771         case SLJIT_SUB:
2772                 compiler->status_flags_state |= SLJIT_CURRENT_FLAGS_SUB;
2773                 return sljit_emit_sub(compiler, op, dst, dstw, src1, src1w, src2, src2w);
2774         case SLJIT_SUBC:
2775                 compiler->status_flags_state |= SLJIT_CURRENT_FLAGS_SUB;
2776                 FAIL_IF(emit_non_commutative(compiler, &subc_forms, dst, src1, src1w, src2, src2w));
2777                 if (dst & SLJIT_MEM)
2778                         return store_word(compiler, tmp0, dst, dstw, op & SLJIT_32);
2779                 return SLJIT_SUCCESS;
2780         case SLJIT_MUL:
2781                 FAIL_IF(sljit_emit_multiply(compiler, op, dst, src1, src1w, src2, src2w));
2782                 break;
2783         case SLJIT_AND:
2784         case SLJIT_OR:
2785         case SLJIT_XOR:
2786                 FAIL_IF(sljit_emit_bitwise(compiler, op, dst, src1, src1w, src2, src2w));
2787                 break;
2788         case SLJIT_SHL:
2789         case SLJIT_LSHR:
2790         case SLJIT_ASHR:
2791                 FAIL_IF(sljit_emit_shift(compiler, op, dst, src1, src1w, src2, src2w));
2792                 break;
2793         }
2794
2795         if (dst & SLJIT_MEM)
2796                 return store_word(compiler, tmp0, dst, dstw, op & SLJIT_32);
2797         return SLJIT_SUCCESS;
2798 }
2799
2800 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op2u(struct sljit_compiler *compiler, sljit_s32 op,
2801         sljit_s32 src1, sljit_sw src1w,
2802         sljit_s32 src2, sljit_sw src2w)
2803 {
2804         CHECK_ERROR();
2805         CHECK(check_sljit_emit_op2(compiler, op, 1, 0, 0, src1, src1w, src2, src2w));
2806
2807         SLJIT_SKIP_CHECKS(compiler);
2808         return sljit_emit_op2(compiler, op, (sljit_s32)tmp0, 0, src1, src1w, src2, src2w);
2809 }
2810
2811 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_src(
2812         struct sljit_compiler *compiler,
2813         sljit_s32 op, sljit_s32 src, sljit_sw srcw)
2814 {
2815         sljit_gpr src_r;
2816
2817         CHECK_ERROR();
2818         CHECK(check_sljit_emit_op_src(compiler, op, src, srcw));
2819         ADJUST_LOCAL_OFFSET(src, srcw);
2820
2821         switch (op) {
2822         case SLJIT_FAST_RETURN:
2823                 src_r = FAST_IS_REG(src) ? gpr(src) : tmp1;
2824                 if (src & SLJIT_MEM)
2825                         FAIL_IF(load_word(compiler, tmp1, src, srcw, 0));
2826
2827                 return push_inst(compiler, br(src_r));
2828         case SLJIT_SKIP_FRAMES_BEFORE_FAST_RETURN:
2829                 /* TODO(carenas): implement? */
2830                 return SLJIT_SUCCESS;
2831         case SLJIT_PREFETCH_L1:
2832         case SLJIT_PREFETCH_L2:
2833         case SLJIT_PREFETCH_L3:
2834         case SLJIT_PREFETCH_ONCE:
2835                 /* TODO(carenas): implement */
2836                 return SLJIT_SUCCESS;
2837         default:
2838                 /* TODO(carenas): probably should not success by default */
2839                 return SLJIT_SUCCESS;
2840         }
2841
2842         return SLJIT_SUCCESS;
2843 }
2844
2845 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_get_register_index(sljit_s32 reg)
2846 {
2847         CHECK_REG_INDEX(check_sljit_get_register_index(reg));
2848         return (sljit_s32)gpr(reg);
2849 }
2850
2851 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_get_float_register_index(sljit_s32 reg)
2852 {
2853         CHECK_REG_INDEX(check_sljit_get_float_register_index(reg));
2854         return (sljit_s32)fgpr(reg);
2855 }
2856
2857 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_custom(struct sljit_compiler *compiler,
2858         void *instruction, sljit_u32 size)
2859 {
2860         sljit_ins ins = 0;
2861
2862         CHECK_ERROR();
2863         CHECK(check_sljit_emit_op_custom(compiler, instruction, size));
2864
2865         memcpy((sljit_u8 *)&ins + sizeof(ins) - size, instruction, size);
2866         return push_inst(compiler, ins);
2867 }
2868
2869 /* --------------------------------------------------------------------- */
2870 /*  Floating point operators                                             */
2871 /* --------------------------------------------------------------------- */
2872
2873 #define FLOAT_LOAD 0
2874 #define FLOAT_STORE 1
2875
2876 static sljit_s32 float_mem(struct sljit_compiler *compiler, sljit_s32 op,
2877         sljit_s32 reg,
2878         sljit_s32 mem, sljit_sw memw)
2879 {
2880         struct addr addr;
2881         sljit_ins ins;
2882
2883         SLJIT_ASSERT(mem & SLJIT_MEM);
2884
2885         if ((mem & OFFS_REG_MASK) || is_u12(memw) || !is_s20(memw)) {
2886                 FAIL_IF(make_addr_bx(compiler, &addr, mem, memw, tmp1));
2887
2888                 if (op & FLOAT_STORE)
2889                         ins = (op & SLJIT_32) ? 0x70000000 /* ste */ : 0x60000000 /* std */;
2890                 else
2891                         ins = (op & SLJIT_32) ? 0x78000000 /* le */ : 0x68000000 /* ld */;
2892
2893                 return push_inst(compiler, ins | F20(reg) | R16A(addr.index) | R12A(addr.base) | (sljit_ins)addr.offset);
2894         }
2895
2896         FAIL_IF(make_addr_bxy(compiler, &addr, mem, memw, tmp1));
2897
2898         if (op & FLOAT_STORE)
2899                 ins = (op & SLJIT_32) ? 0xed0000000066 /* stey */ : 0xed0000000067 /* stdy */;
2900         else
2901                 ins = (op & SLJIT_32) ? 0xed0000000064 /* ley */ : 0xed0000000065 /* ldy */;
2902
2903         return push_inst(compiler, ins | F36(reg) | R32A(addr.index) | R28A(addr.base) | disp_s20(addr.offset));
2904 }
2905
2906 static sljit_s32 emit_float(struct sljit_compiler *compiler, sljit_ins ins_r, sljit_ins ins,
2907         sljit_s32 reg,
2908         sljit_s32 src, sljit_sw srcw)
2909 {
2910         struct addr addr;
2911
2912         if (!(src & SLJIT_MEM))
2913                 return push_inst(compiler, ins_r | F4(reg) | F0(src));
2914
2915         FAIL_IF(make_addr_bx(compiler, &addr, src, srcw, tmp1));
2916         return push_inst(compiler, ins | F36(reg) | R32A(addr.index) | R28A(addr.base) | ((sljit_ins)addr.offset << 16));
2917 }
2918
2919 static SLJIT_INLINE sljit_s32 sljit_emit_fop1_conv_sw_from_f64(struct sljit_compiler *compiler, sljit_s32 op,
2920         sljit_s32 dst, sljit_sw dstw,
2921         sljit_s32 src, sljit_sw srcw)
2922 {
2923         sljit_ins dst_r = FAST_IS_REG(dst) ? gpr(dst) : tmp0;
2924         sljit_ins ins;
2925
2926         if (src & SLJIT_MEM) {
2927                 FAIL_IF(float_mem(compiler, FLOAT_LOAD | (op & SLJIT_32), TMP_FREG1, src, srcw));
2928                 src = TMP_FREG1;
2929         }
2930
2931         /* M3 is set to 5 */
2932         if (GET_OPCODE(op) == SLJIT_CONV_SW_FROM_F64)
2933                 ins = (op & SLJIT_32) ? 0xb3a85000 /* cgebr */ : 0xb3a95000 /* cgdbr */;
2934         else
2935                 ins = (op & SLJIT_32) ? 0xb3985000 /* cfebr */ : 0xb3995000 /* cfdbr */;
2936
2937         FAIL_IF(push_inst(compiler, ins | R4A(dst_r) | F0(src)));
2938
2939         if (dst & SLJIT_MEM)
2940                 return store_word(compiler, dst_r, dst, dstw, GET_OPCODE(op) >= SLJIT_CONV_S32_FROM_F64);
2941
2942         return SLJIT_SUCCESS;
2943 }
2944
2945 static SLJIT_INLINE sljit_s32 sljit_emit_fop1_conv_f64_from_sw(struct sljit_compiler *compiler, sljit_s32 op,
2946         sljit_s32 dst, sljit_sw dstw,
2947         sljit_s32 src, sljit_sw srcw)
2948 {
2949         sljit_s32 dst_r = FAST_IS_REG(dst) ? dst : TMP_FREG1;
2950         sljit_ins ins;
2951
2952         if (src & SLJIT_IMM) {
2953                 FAIL_IF(push_load_imm_inst(compiler, tmp0, srcw));
2954                 src = (sljit_s32)tmp0;
2955         }
2956         else if (src & SLJIT_MEM) {
2957                 FAIL_IF(load_word(compiler, tmp0, src, srcw, GET_OPCODE(op) >= SLJIT_CONV_F64_FROM_S32));
2958                 src = (sljit_s32)tmp0;
2959         }
2960
2961         if (GET_OPCODE(op) == SLJIT_CONV_F64_FROM_SW)
2962                 ins = (op & SLJIT_32) ? 0xb3a40000 /* cegbr */ : 0xb3a50000 /* cdgbr */;
2963         else
2964                 ins = (op & SLJIT_32) ? 0xb3940000 /* cefbr */ : 0xb3950000 /* cdfbr */;
2965
2966         FAIL_IF(push_inst(compiler, ins | F4(dst_r) | R0(src)));
2967
2968         if (dst & SLJIT_MEM)
2969                 return float_mem(compiler, FLOAT_STORE | (op & SLJIT_32), TMP_FREG1, dst, dstw);
2970
2971         return SLJIT_SUCCESS;
2972 }
2973
2974 static SLJIT_INLINE sljit_s32 sljit_emit_fop1_cmp(struct sljit_compiler *compiler, sljit_s32 op,
2975         sljit_s32 src1, sljit_sw src1w,
2976         sljit_s32 src2, sljit_sw src2w)
2977 {
2978         sljit_ins ins_r, ins;
2979
2980         if (src1 & SLJIT_MEM) {
2981                 FAIL_IF(float_mem(compiler, FLOAT_LOAD | (op & SLJIT_32), TMP_FREG1, src1, src1w));
2982                 src1 = TMP_FREG1;
2983         }
2984
2985         if (op & SLJIT_32) {
2986                 ins_r = 0xb3090000 /* cebr */;
2987                 ins = 0xed0000000009 /* ceb */;
2988         } else {
2989                 ins_r = 0xb3190000 /* cdbr */;
2990                 ins = 0xed0000000019 /* cdb */;
2991         }
2992
2993         return emit_float(compiler, ins_r, ins, src1, src2, src2w);
2994 }
2995
2996 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fop1(struct sljit_compiler *compiler, sljit_s32 op,
2997         sljit_s32 dst, sljit_sw dstw,
2998         sljit_s32 src, sljit_sw srcw)
2999 {
3000         sljit_s32 dst_r;
3001         sljit_ins ins;
3002
3003         CHECK_ERROR();
3004
3005         SELECT_FOP1_OPERATION_WITH_CHECKS(compiler, op, dst, dstw, src, srcw);
3006
3007         dst_r = FAST_IS_REG(dst) ? dst : TMP_FREG1;
3008
3009         if (op == SLJIT_CONV_F64_FROM_F32)
3010                 FAIL_IF(emit_float(compiler, 0xb3040000 /* ldebr */, 0xed0000000004 /* ldeb */, dst_r, src, srcw));
3011         else {
3012                 if (src & SLJIT_MEM) {
3013                         FAIL_IF(float_mem(compiler, FLOAT_LOAD | (op == SLJIT_CONV_F32_FROM_F64 ? 0 : (op & SLJIT_32)), dst_r, src, srcw));
3014                         src = dst_r;
3015                 }
3016
3017                 switch (GET_OPCODE(op)) {
3018                 case SLJIT_MOV_F64:
3019                         if (FAST_IS_REG(dst)) {
3020                                 if (dst == src)
3021                                         return SLJIT_SUCCESS;
3022
3023                                 ins = (op & SLJIT_32) ? 0x3800 /* ler */ : 0x2800 /* ldr */;
3024                                 break;
3025                         }
3026                         return float_mem(compiler, FLOAT_STORE | (op & SLJIT_32), src, dst, dstw);
3027                 case SLJIT_CONV_F64_FROM_F32:
3028                         /* Only SLJIT_CONV_F32_FROM_F64. */
3029                         ins = 0xb3440000 /* ledbr */;
3030                         break;
3031                 case SLJIT_NEG_F64:
3032                         ins = (op & SLJIT_32) ? 0xb3030000 /* lcebr */ : 0xb3130000 /* lcdbr */;
3033                         break;
3034                 default:
3035                         SLJIT_ASSERT(GET_OPCODE(op) == SLJIT_ABS_F64);
3036                         ins = (op & SLJIT_32) ? 0xb3000000 /* lpebr */ : 0xb3100000 /* lpdbr */;
3037                         break;
3038                 }
3039
3040                 FAIL_IF(push_inst(compiler, ins | F4(dst_r) | F0(src)));
3041         }
3042
3043         if (!(dst & SLJIT_MEM))
3044                 return SLJIT_SUCCESS;
3045
3046         SLJIT_ASSERT(dst_r == TMP_FREG1);
3047
3048         return float_mem(compiler, FLOAT_STORE | (op & SLJIT_32), TMP_FREG1, dst, dstw);
3049 }
3050
3051 #define FLOAT_MOV(op, dst_r, src_r) \
3052         (((op & SLJIT_32) ? 0x3800 /* ler */ : 0x2800 /* ldr */) | F4(dst_r) | F0(src_r))
3053
3054 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fop2(struct sljit_compiler *compiler, sljit_s32 op,
3055         sljit_s32 dst, sljit_sw dstw,
3056         sljit_s32 src1, sljit_sw src1w,
3057         sljit_s32 src2, sljit_sw src2w)
3058 {
3059         sljit_s32 dst_r = TMP_FREG1;
3060         sljit_ins ins_r, ins;
3061
3062         CHECK_ERROR();
3063         CHECK(check_sljit_emit_fop2(compiler, op, dst, dstw, src1, src1w, src2, src2w));
3064         ADJUST_LOCAL_OFFSET(dst, dstw);
3065         ADJUST_LOCAL_OFFSET(src1, src1w);
3066         ADJUST_LOCAL_OFFSET(src2, src2w);
3067
3068         do {
3069                 if (FAST_IS_REG(dst)) {
3070                         dst_r = dst;
3071
3072                         if (dst == src1)
3073                                 break;
3074
3075                         if (dst == src2) {
3076                                 if (GET_OPCODE(op) == SLJIT_ADD_F64 || GET_OPCODE(op) == SLJIT_MUL_F64) {
3077                                         src2 = src1;
3078                                         src2w = src1w;
3079                                         src1 = dst;
3080                                         break;
3081                                 }
3082
3083                                 FAIL_IF(push_inst(compiler, FLOAT_MOV(op, TMP_FREG1, src2)));
3084                                 src2 = TMP_FREG1;
3085                         }
3086                 }
3087
3088                 if (src1 & SLJIT_MEM)
3089                         FAIL_IF(float_mem(compiler, FLOAT_LOAD | (op & SLJIT_32), dst_r, src1, src1w));
3090                 else
3091                         FAIL_IF(push_inst(compiler, FLOAT_MOV(op, dst_r, src1)));
3092         } while (0);
3093
3094         switch (GET_OPCODE(op)) {
3095         case SLJIT_ADD_F64:
3096                 ins_r = (op & SLJIT_32) ? 0xb30a0000 /* aebr */ : 0xb31a0000 /* adbr */;
3097                 ins = (op & SLJIT_32) ? 0xed000000000a /* aeb */ : 0xed000000001a /* adb */;
3098                 break;
3099         case SLJIT_SUB_F64:
3100                 ins_r = (op & SLJIT_32) ? 0xb30b0000 /* sebr */ : 0xb31b0000 /* sdbr */;
3101                 ins = (op & SLJIT_32) ? 0xed000000000b /* seb */ : 0xed000000001b /* sdb */;
3102                 break;
3103         case SLJIT_MUL_F64:
3104                 ins_r = (op & SLJIT_32) ? 0xb3170000 /* meebr */ : 0xb31c0000 /* mdbr */;
3105                 ins = (op & SLJIT_32) ? 0xed0000000017 /* meeb */ : 0xed000000001c /* mdb */;
3106                 break;
3107         default:
3108                 SLJIT_ASSERT(GET_OPCODE(op) == SLJIT_DIV_F64);
3109                 ins_r = (op & SLJIT_32) ? 0xb30d0000 /* debr */ : 0xb31d0000 /* ddbr */;
3110                 ins = (op & SLJIT_32) ? 0xed000000000d /* deb */ : 0xed000000001d /* ddb */;
3111                 break;
3112         }
3113
3114         FAIL_IF(emit_float(compiler, ins_r, ins, dst_r, src2, src2w));
3115
3116         if (dst & SLJIT_MEM)
3117                 return float_mem(compiler, FLOAT_STORE | (op & SLJIT_32), TMP_FREG1, dst, dstw);
3118
3119         SLJIT_ASSERT(dst_r != TMP_FREG1);
3120         return SLJIT_SUCCESS;
3121 }
3122
3123 /* --------------------------------------------------------------------- */
3124 /*  Other instructions                                                   */
3125 /* --------------------------------------------------------------------- */
3126
3127 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fast_enter(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw)
3128 {
3129         CHECK_ERROR();
3130         CHECK(check_sljit_emit_fast_enter(compiler, dst, dstw));
3131         ADJUST_LOCAL_OFFSET(dst, dstw);
3132
3133         if (FAST_IS_REG(dst))
3134                 return push_inst(compiler, lgr(gpr(dst), fast_link_r));
3135
3136         /* memory */
3137         return store_word(compiler, fast_link_r, dst, dstw, 0);
3138 }
3139
3140 /* --------------------------------------------------------------------- */
3141 /*  Conditional instructions                                             */
3142 /* --------------------------------------------------------------------- */
3143
3144 SLJIT_API_FUNC_ATTRIBUTE struct sljit_label* sljit_emit_label(struct sljit_compiler *compiler)
3145 {
3146         struct sljit_label *label;
3147
3148         CHECK_ERROR_PTR();
3149         CHECK_PTR(check_sljit_emit_label(compiler));
3150
3151         if (compiler->last_label && compiler->last_label->size == compiler->size)
3152                 return compiler->last_label;
3153
3154         label = (struct sljit_label*)ensure_abuf(compiler, sizeof(struct sljit_label));
3155         PTR_FAIL_IF(!label);
3156         set_label(label, compiler);
3157         return label;
3158 }
3159
3160 SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_jump(struct sljit_compiler *compiler, sljit_s32 type)
3161 {
3162         sljit_u8 mask = ((type & 0xff) < SLJIT_JUMP) ? get_cc(compiler, type & 0xff) : 0xf;
3163
3164         CHECK_ERROR_PTR();
3165         CHECK_PTR(check_sljit_emit_jump(compiler, type));
3166
3167         /* record jump */
3168         struct sljit_jump *jump = (struct sljit_jump *)
3169                 ensure_abuf(compiler, sizeof(struct sljit_jump));
3170         PTR_FAIL_IF(!jump);
3171         set_jump(jump, compiler, type & SLJIT_REWRITABLE_JUMP);
3172         jump->addr = compiler->size;
3173
3174         /* emit jump instruction */
3175         type &= 0xff;
3176         if (type >= SLJIT_FAST_CALL)
3177                 PTR_FAIL_IF(push_inst(compiler, brasl(type == SLJIT_FAST_CALL ? fast_link_r : link_r, 0)));
3178         else
3179                 PTR_FAIL_IF(push_inst(compiler, brcl(mask, 0)));
3180
3181         return jump;
3182 }
3183
3184 SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_call(struct sljit_compiler *compiler, sljit_s32 type,
3185         sljit_s32 arg_types)
3186 {
3187         SLJIT_UNUSED_ARG(arg_types);
3188         CHECK_ERROR_PTR();
3189         CHECK_PTR(check_sljit_emit_call(compiler, type, arg_types));
3190
3191         if (type & SLJIT_CALL_RETURN) {
3192                 PTR_FAIL_IF(emit_stack_frame_release(compiler));
3193                 type = SLJIT_JUMP | (type & SLJIT_REWRITABLE_JUMP);
3194         }
3195
3196         SLJIT_SKIP_CHECKS(compiler);
3197         return sljit_emit_jump(compiler, type);
3198 }
3199
3200 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_ijump(struct sljit_compiler *compiler, sljit_s32 type, sljit_s32 src, sljit_sw srcw)
3201 {
3202         sljit_gpr src_r = FAST_IS_REG(src) ? gpr(src) : tmp1;
3203
3204         CHECK_ERROR();
3205         CHECK(check_sljit_emit_ijump(compiler, type, src, srcw));
3206
3207         if (src & SLJIT_IMM) {
3208                 SLJIT_ASSERT(!(srcw & 1)); /* target address must be even */
3209                 FAIL_IF(push_load_imm_inst(compiler, src_r, srcw));
3210         }
3211         else if (src & SLJIT_MEM) {
3212                 ADJUST_LOCAL_OFFSET(src, srcw);
3213                 FAIL_IF(load_word(compiler, src_r, src, srcw, 0 /* 64-bit */));
3214         }
3215
3216         /* emit jump instruction */
3217         if (type >= SLJIT_FAST_CALL)
3218                 return push_inst(compiler, basr(type == SLJIT_FAST_CALL ? fast_link_r : link_r, src_r));
3219
3220         return push_inst(compiler, br(src_r));
3221 }
3222
3223 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_icall(struct sljit_compiler *compiler, sljit_s32 type,
3224         sljit_s32 arg_types,
3225         sljit_s32 src, sljit_sw srcw)
3226 {
3227         CHECK_ERROR();
3228         CHECK(check_sljit_emit_icall(compiler, type, arg_types, src, srcw));
3229
3230         SLJIT_ASSERT(gpr(TMP_REG2) == tmp1);
3231
3232         if (src & SLJIT_MEM) {
3233                 ADJUST_LOCAL_OFFSET(src, srcw);
3234                 FAIL_IF(load_word(compiler, tmp1, src, srcw, 0 /* 64-bit */));
3235                 src = TMP_REG2;
3236         }
3237
3238         if (type & SLJIT_CALL_RETURN) {
3239                 if (src >= SLJIT_FIRST_SAVED_REG && src <= SLJIT_S0) {
3240                         FAIL_IF(push_inst(compiler, lgr(tmp1, gpr(src))));
3241                         src = TMP_REG2;
3242                 }
3243
3244                 FAIL_IF(emit_stack_frame_release(compiler));
3245                 type = SLJIT_JUMP;
3246         }
3247
3248         SLJIT_SKIP_CHECKS(compiler);
3249         return sljit_emit_ijump(compiler, type, src, srcw);
3250 }
3251
3252 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_flags(struct sljit_compiler *compiler, sljit_s32 op,
3253         sljit_s32 dst, sljit_sw dstw,
3254         sljit_s32 type)
3255 {
3256         sljit_u8 mask = get_cc(compiler, type);
3257
3258         CHECK_ERROR();
3259         CHECK(check_sljit_emit_op_flags(compiler, op, dst, dstw, type));
3260
3261         sljit_gpr dst_r = FAST_IS_REG(dst) ? gpr(dst & REG_MASK) : tmp0;
3262         sljit_gpr loc_r = tmp1;
3263         switch (GET_OPCODE(op)) {
3264         case SLJIT_AND:
3265         case SLJIT_OR:
3266         case SLJIT_XOR:
3267                 compiler->status_flags_state = op & SLJIT_SET_Z;
3268
3269                 /* dst is also source operand */
3270                 if (dst & SLJIT_MEM)
3271                         FAIL_IF(load_word(compiler, dst_r, dst, dstw, op & SLJIT_32));
3272
3273                 break;
3274         case SLJIT_MOV32:
3275                 op |= SLJIT_32;
3276                 /* fallthrough */
3277         case SLJIT_MOV:
3278                 /* can write straight into destination */
3279                 loc_r = dst_r;
3280                 break;
3281         default:
3282                 SLJIT_UNREACHABLE();
3283         }
3284
3285         /* TODO(mundaym): fold into cmov helper function? */
3286         #define LEVAL(i) i(loc_r, 1, mask)
3287         if (have_lscond2()) {
3288                 FAIL_IF(push_load_imm_inst(compiler, loc_r, 0));
3289                 FAIL_IF(push_inst(compiler,
3290                         WHEN2(op & SLJIT_32, lochi, locghi)));
3291         } else {
3292                 /* TODO(mundaym): no load/store-on-condition 2 facility (ipm? branch-and-set?) */
3293                 abort();
3294         }
3295         #undef LEVAL
3296
3297         /* apply bitwise op and set condition codes */
3298         switch (GET_OPCODE(op)) {
3299         #define LEVAL(i) i(dst_r, loc_r)
3300         case SLJIT_AND:
3301                 FAIL_IF(push_inst(compiler,
3302                         WHEN2(op & SLJIT_32, nr, ngr)));
3303                 break;
3304         case SLJIT_OR:
3305                 FAIL_IF(push_inst(compiler,
3306                         WHEN2(op & SLJIT_32, or, ogr)));
3307                 break;
3308         case SLJIT_XOR:
3309                 FAIL_IF(push_inst(compiler,
3310                         WHEN2(op & SLJIT_32, xr, xgr)));
3311                 break;
3312         #undef LEVAL
3313         }
3314
3315         /* store result to memory if required */
3316         if (dst & SLJIT_MEM)
3317                 return store_word(compiler, dst_r, dst, dstw, (op & SLJIT_32));
3318
3319         return SLJIT_SUCCESS;
3320 }
3321
3322 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_cmov(struct sljit_compiler *compiler, sljit_s32 type,
3323         sljit_s32 dst_reg,
3324         sljit_s32 src, sljit_sw srcw)
3325 {
3326         sljit_u8 mask = get_cc(compiler, type);
3327         sljit_gpr dst_r = gpr(dst_reg & ~SLJIT_32);
3328         sljit_gpr src_r = FAST_IS_REG(src) ? gpr(src) : tmp0;
3329
3330         CHECK_ERROR();
3331         CHECK(check_sljit_emit_cmov(compiler, type, dst_reg, src, srcw));
3332
3333         if (src & SLJIT_IMM) {
3334                 /* TODO(mundaym): fast path with lscond2 */
3335                 FAIL_IF(push_load_imm_inst(compiler, src_r, srcw));
3336         }
3337
3338         #define LEVAL(i) i(dst_r, src_r, mask)
3339         if (have_lscond1())
3340                 return push_inst(compiler,
3341                         WHEN2(dst_reg & SLJIT_32, locr, locgr));
3342
3343         #undef LEVAL
3344
3345         /* TODO(mundaym): implement */
3346         return SLJIT_ERR_UNSUPPORTED;
3347 }
3348
3349 /* --------------------------------------------------------------------- */
3350 /*  Other instructions                                                   */
3351 /* --------------------------------------------------------------------- */
3352
3353 /* On s390x we build a literal pool to hold constants. This has two main
3354    advantages:
3355
3356      1. we only need one instruction in the instruction stream (LGRL)
3357      2. we can store 64 bit addresses and use 32 bit offsets
3358
3359    To retrofit the extra information needed to build the literal pool we
3360    add a new sljit_s390x_const struct that contains the initial value but
3361    can still be cast to a sljit_const. */
3362
3363 SLJIT_API_FUNC_ATTRIBUTE struct sljit_const* sljit_emit_const(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw, sljit_sw init_value)
3364 {
3365         struct sljit_s390x_const *const_;
3366         sljit_gpr dst_r;
3367
3368         CHECK_ERROR_PTR();
3369         CHECK_PTR(check_sljit_emit_const(compiler, dst, dstw, init_value));
3370
3371         const_ = (struct sljit_s390x_const*)ensure_abuf(compiler,
3372                                         sizeof(struct sljit_s390x_const));
3373         PTR_FAIL_IF(!const_);
3374         set_const((struct sljit_const*)const_, compiler);
3375         const_->init_value = init_value;
3376
3377         dst_r = FAST_IS_REG(dst) ? gpr(dst & REG_MASK) : tmp0;
3378         if (have_genext())
3379                 PTR_FAIL_IF(push_inst(compiler, sljit_ins_const | lgrl(dst_r, 0)));
3380         else {
3381                 PTR_FAIL_IF(push_inst(compiler, sljit_ins_const | larl(tmp1, 0)));
3382                 PTR_FAIL_IF(push_inst(compiler, lg(dst_r, 0, r0, tmp1)));
3383         }
3384
3385         if (dst & SLJIT_MEM)
3386                 PTR_FAIL_IF(store_word(compiler, dst_r, dst, dstw, 0 /* always 64-bit */));
3387
3388         return (struct sljit_const*)const_;
3389 }
3390
3391 SLJIT_API_FUNC_ATTRIBUTE void sljit_set_jump_addr(sljit_uw addr, sljit_uw new_target, sljit_sw executable_offset)
3392 {
3393         /* Update the constant pool. */
3394         sljit_uw *ptr = (sljit_uw *)addr;
3395         SLJIT_UNUSED_ARG(executable_offset);
3396
3397         SLJIT_UPDATE_WX_FLAGS(ptr, ptr + 1, 0);
3398         *ptr = new_target;
3399         SLJIT_UPDATE_WX_FLAGS(ptr, ptr + 1, 1);
3400         SLJIT_CACHE_FLUSH(ptr, ptr + 1);
3401 }
3402
3403 SLJIT_API_FUNC_ATTRIBUTE void sljit_set_const(sljit_uw addr, sljit_sw new_constant, sljit_sw executable_offset)
3404 {
3405         sljit_set_jump_addr(addr, (sljit_uw)new_constant, executable_offset);
3406 }
3407
3408 SLJIT_API_FUNC_ATTRIBUTE struct sljit_put_label *sljit_emit_put_label(
3409         struct sljit_compiler *compiler,
3410         sljit_s32 dst, sljit_sw dstw)
3411 {
3412         struct sljit_put_label *put_label;
3413         sljit_gpr dst_r;
3414
3415         CHECK_ERROR_PTR();
3416         CHECK_PTR(check_sljit_emit_put_label(compiler, dst, dstw));
3417         ADJUST_LOCAL_OFFSET(dst, dstw);
3418
3419         put_label = (struct sljit_put_label*)ensure_abuf(compiler, sizeof(struct sljit_put_label));
3420         PTR_FAIL_IF(!put_label);
3421         set_put_label(put_label, compiler, 0);
3422
3423         dst_r = FAST_IS_REG(dst) ? gpr(dst & REG_MASK) : tmp0;
3424
3425         if (have_genext())
3426                 PTR_FAIL_IF(push_inst(compiler, lgrl(dst_r, 0)));
3427         else {
3428                 PTR_FAIL_IF(push_inst(compiler, larl(tmp1, 0)));
3429                 PTR_FAIL_IF(push_inst(compiler, lg(dst_r, 0, r0, tmp1)));
3430         }
3431
3432         if (dst & SLJIT_MEM)
3433                 PTR_FAIL_IF(store_word(compiler, dst_r, dst, dstw, 0));
3434
3435         return put_label;
3436 }
3437
3438 /* TODO(carenas): EVAL probably should move up or be refactored */
3439 #undef WHEN2
3440 #undef EVAL
3441
3442 #undef tmp1
3443 #undef tmp0
3444
3445 /* TODO(carenas): undef other macros that spill like is_u12? */