sljit_src/sljitNativeX86_common.c

   1 /*
   2  *    Stack-less Just-In-Time compiler
   3  *
   4  *    Copyright Zoltan Herczeg (hzmester@freemail.hu). All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without modification, are
   7  * permitted provided that the following conditions are met:
   8  *
   9  *   1. Redistributions of source code must retain the above copyright notice, this list of
  10  *      conditions and the following disclaimer.
  11  *
  12  *   2. Redistributions in binary form must reproduce the above copyright notice, this list
  13  *      of conditions and the following disclaimer in the documentation and/or other materials
  14  *      provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) AND CONTRIBUTORS ``AS IS'' AND ANY
  17  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  18  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
  19  * SHALL THE COPYRIGHT HOLDER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  20  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
  21  * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
  22  * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  23  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  24  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  */
  26
  27 SLJIT_API_FUNC_ATTRIBUTE const char* sljit_get_platform_name(void)
  28 {
  29 #if (defined SLJIT_X86_32_FASTCALL && SLJIT_X86_32_FASTCALL)
  30         return "x86" SLJIT_CPUINFO " ABI:fastcall";
  31 #else
  32         return "x86" SLJIT_CPUINFO;
  33 #endif
  34 }
  35
  36 /*
  37    32b register indexes:
  38      0 - EAX
  39      1 - ECX
  40      2 - EDX
  41      3 - EBX
  42      4 - ESP
  43      5 - EBP
  44      6 - ESI
  45      7 - EDI
  46 */
  47
  48 /*
  49    64b register indexes:
  50      0 - RAX
  51      1 - RCX
  52      2 - RDX
  53      3 - RBX
  54      4 - RSP
  55      5 - RBP
  56      6 - RSI
  57      7 - RDI
  58      8 - R8   - From now on REX prefix is required
  59      9 - R9
  60     10 - R10
  61     11 - R11
  62     12 - R12
  63     13 - R13
  64     14 - R14
  65     15 - R15
  66 */
  67
  68 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
  69
  70 /* Last register + 1. */
  71 #define TMP_REG1        (SLJIT_NUMBER_OF_REGISTERS + 2)
  72
  73 static const sljit_u8 reg_map[SLJIT_NUMBER_OF_REGISTERS + 3] = {
  74         0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 7, 6, 3, 4, 5
  75 };
  76
  77 #define CHECK_EXTRA_REGS(p, w, do) \
  78         if (p >= SLJIT_R3 && p <= SLJIT_S3) { \
  79                 if (p <= compiler->scratches) \
  80                         w = compiler->saveds_offset - ((p) - SLJIT_R2) * (sljit_sw)sizeof(sljit_sw); \
  81                 else \
  82                         w = compiler->locals_offset + ((p) - SLJIT_S2) * (sljit_sw)sizeof(sljit_sw); \
  83                 p = SLJIT_MEM1(SLJIT_SP); \
  84                 do; \
  85         }
  86
  87 #else /* SLJIT_CONFIG_X86_32 */
  88
  89 /* Last register + 1. */
  90 #define TMP_REG1        (SLJIT_NUMBER_OF_REGISTERS + 2)
  91 #define TMP_REG2        (SLJIT_NUMBER_OF_REGISTERS + 3)
  92
  93 /* Note: r12 & 0x7 == 0b100, which decoded as SIB byte present
  94    Note: avoid to use r12 and r13 for memory addessing
  95    therefore r12 is better to be a higher saved register. */
  96 #ifndef _WIN64
  97 /* Args: rdi(=7), rsi(=6), rdx(=2), rcx(=1), r8, r9. Scratches: rax(=0), r10, r11 */
  98 static const sljit_u8 reg_map[SLJIT_NUMBER_OF_REGISTERS + 4] = {
  99         0, 0, 6, 7, 1, 8, 11, 10, 12, 5, 13, 14, 15, 3, 4, 2, 9
 100 };
 101 /* low-map. reg_map & 0x7. */
 102 static const sljit_u8 reg_lmap[SLJIT_NUMBER_OF_REGISTERS + 4] = {
 103         0, 0, 6, 7, 1, 0, 3,  2,  4,  5,  5,  6,  7, 3, 4, 2, 1
 104 };
 105 #else
 106 /* Args: rcx(=1), rdx(=2), r8, r9. Scratches: rax(=0), r10, r11 */
 107 static const sljit_u8 reg_map[SLJIT_NUMBER_OF_REGISTERS + 4] = {
 108         0, 0, 2, 8, 1, 11, 12, 5, 13, 14, 15, 7, 6, 3, 4, 9, 10
 109 };
 110 /* low-map. reg_map & 0x7. */
 111 static const sljit_u8 reg_lmap[SLJIT_NUMBER_OF_REGISTERS + 4] = {
 112         0, 0, 2, 0, 1,  3,  4, 5,  5,  6,  7, 7, 6, 3, 4, 1,  2
 113 };
 114 #endif
 115
 116 /* Args: xmm0-xmm3 */
 117 static const sljit_u8 freg_map[SLJIT_NUMBER_OF_FLOAT_REGISTERS + 1] = {
 118         4, 0, 1, 2, 3, 5, 6
 119 };
 120 /* low-map. freg_map & 0x7. */
 121 static const sljit_u8 freg_lmap[SLJIT_NUMBER_OF_FLOAT_REGISTERS + 1] = {
 122         4, 0, 1, 2, 3, 5, 6
 123 };
 124
 125 #define REX_W           0x48
 126 #define REX_R           0x44
 127 #define REX_X           0x42
 128 #define REX_B           0x41
 129 #define REX             0x40
 130
 131 #ifndef _WIN64
 132 #define HALFWORD_MAX 0x7fffffffl
 133 #define HALFWORD_MIN -0x80000000l
 134 #else
 135 #define HALFWORD_MAX 0x7fffffffll
 136 #define HALFWORD_MIN -0x80000000ll
 137 #endif
 138
 139 #define IS_HALFWORD(x)          ((x) <= HALFWORD_MAX && (x) >= HALFWORD_MIN)
 140 #define NOT_HALFWORD(x)         ((x) > HALFWORD_MAX || (x) < HALFWORD_MIN)
 141
 142 #define CHECK_EXTRA_REGS(p, w, do)
 143
 144 #endif /* SLJIT_CONFIG_X86_32 */
 145
 146 #define TMP_FREG        (0)
 147
 148 /* Size flags for emit_x86_instruction: */
 149 #define EX86_BIN_INS            0x0010
 150 #define EX86_SHIFT_INS          0x0020
 151 #define EX86_REX                0x0040
 152 #define EX86_NO_REXW            0x0080
 153 #define EX86_BYTE_ARG           0x0100
 154 #define EX86_HALF_ARG           0x0200
 155 #define EX86_PREF_66            0x0400
 156 #define EX86_PREF_F2            0x0800
 157 #define EX86_PREF_F3            0x1000
 158 #define EX86_SSE2_OP1           0x2000
 159 #define EX86_SSE2_OP2           0x4000
 160 #define EX86_SSE2               (EX86_SSE2_OP1 | EX86_SSE2_OP2)
 161
 162 /* --------------------------------------------------------------------- */
 163 /*  Instrucion forms                                                     */
 164 /* --------------------------------------------------------------------- */
 165
 166 #define ADD             (/* BINARY */ 0 << 3)
 167 #define ADD_EAX_i32     0x05
 168 #define ADD_r_rm        0x03
 169 #define ADD_rm_r        0x01
 170 #define ADDSD_x_xm      0x58
 171 #define ADC             (/* BINARY */ 2 << 3)
 172 #define ADC_EAX_i32     0x15
 173 #define ADC_r_rm        0x13
 174 #define ADC_rm_r        0x11
 175 #define AND             (/* BINARY */ 4 << 3)
 176 #define AND_EAX_i32     0x25
 177 #define AND_r_rm        0x23
 178 #define AND_rm_r        0x21
 179 #define ANDPD_x_xm      0x54
 180 #define BSR_r_rm        (/* GROUP_0F */ 0xbd)
 181 #define CALL_i32        0xe8
 182 #define CALL_rm         (/* GROUP_FF */ 2 << 3)
 183 #define CDQ             0x99
 184 #define CMOVE_r_rm      (/* GROUP_0F */ 0x44)
 185 #define CMP             (/* BINARY */ 7 << 3)
 186 #define CMP_EAX_i32     0x3d
 187 #define CMP_r_rm        0x3b
 188 #define CMP_rm_r        0x39
 189 #define CVTPD2PS_x_xm   0x5a
 190 #define CVTSI2SD_x_rm   0x2a
 191 #define CVTTSD2SI_r_xm  0x2c
 192 #define DIV             (/* GROUP_F7 */ 6 << 3)
 193 #define DIVSD_x_xm      0x5e
 194 #define FSTPS           0xd9
 195 #define FSTPD           0xdd
 196 #define INT3            0xcc
 197 #define IDIV            (/* GROUP_F7 */ 7 << 3)
 198 #define IMUL            (/* GROUP_F7 */ 5 << 3)
 199 #define IMUL_r_rm       (/* GROUP_0F */ 0xaf)
 200 #define IMUL_r_rm_i8    0x6b
 201 #define IMUL_r_rm_i32   0x69
 202 #define JE_i8           0x74
 203 #define JNE_i8          0x75
 204 #define JMP_i8          0xeb
 205 #define JMP_i32         0xe9
 206 #define JMP_rm          (/* GROUP_FF */ 4 << 3)
 207 #define LEA_r_m         0x8d
 208 #define MOV_r_rm        0x8b
 209 #define MOV_r_i32       0xb8
 210 #define MOV_rm_r        0x89
 211 #define MOV_rm_i32      0xc7
 212 #define MOV_rm8_i8      0xc6
 213 #define MOV_rm8_r8      0x88
 214 #define MOVSD_x_xm      0x10
 215 #define MOVSD_xm_x      0x11
 216 #define MOVSXD_r_rm     0x63
 217 #define MOVSX_r_rm8     (/* GROUP_0F */ 0xbe)
 218 #define MOVSX_r_rm16    (/* GROUP_0F */ 0xbf)
 219 #define MOVZX_r_rm8     (/* GROUP_0F */ 0xb6)
 220 #define MOVZX_r_rm16    (/* GROUP_0F */ 0xb7)
 221 #define MUL             (/* GROUP_F7 */ 4 << 3)
 222 #define MULSD_x_xm      0x59
 223 #define NEG_rm          (/* GROUP_F7 */ 3 << 3)
 224 #define NOP             0x90
 225 #define NOT_rm          (/* GROUP_F7 */ 2 << 3)
 226 #define OR              (/* BINARY */ 1 << 3)
 227 #define OR_r_rm         0x0b
 228 #define OR_EAX_i32      0x0d
 229 #define OR_rm_r         0x09
 230 #define OR_rm8_r8       0x08
 231 #define POP_r           0x58
 232 #define POP_rm          0x8f
 233 #define POPF            0x9d
 234 #define PREFETCH        0x18
 235 #define PUSH_i32        0x68
 236 #define PUSH_r          0x50
 237 #define PUSH_rm         (/* GROUP_FF */ 6 << 3)
 238 #define PUSHF           0x9c
 239 #define RET_near        0xc3
 240 #define RET_i16         0xc2
 241 #define SBB             (/* BINARY */ 3 << 3)
 242 #define SBB_EAX_i32     0x1d
 243 #define SBB_r_rm        0x1b
 244 #define SBB_rm_r        0x19
 245 #define SAR             (/* SHIFT */ 7 << 3)
 246 #define SHL             (/* SHIFT */ 4 << 3)
 247 #define SHR             (/* SHIFT */ 5 << 3)
 248 #define SUB             (/* BINARY */ 5 << 3)
 249 #define SUB_EAX_i32     0x2d
 250 #define SUB_r_rm        0x2b
 251 #define SUB_rm_r        0x29
 252 #define SUBSD_x_xm      0x5c
 253 #define TEST_EAX_i32    0xa9
 254 #define TEST_rm_r       0x85
 255 #define UCOMISD_x_xm    0x2e
 256 #define UNPCKLPD_x_xm   0x14
 257 #define XCHG_EAX_r      0x90
 258 #define XCHG_r_rm       0x87
 259 #define XOR             (/* BINARY */ 6 << 3)
 260 #define XOR_EAX_i32     0x35
 261 #define XOR_r_rm        0x33
 262 #define XOR_rm_r        0x31
 263 #define XORPD_x_xm      0x57
 264
 265 #define GROUP_0F        0x0f
 266 #define GROUP_F7        0xf7
 267 #define GROUP_FF        0xff
 268 #define GROUP_BINARY_81 0x81
 269 #define GROUP_BINARY_83 0x83
 270 #define GROUP_SHIFT_1   0xd1
 271 #define GROUP_SHIFT_N   0xc1
 272 #define GROUP_SHIFT_CL  0xd3
 273
 274 #define MOD_REG         0xc0
 275 #define MOD_DISP8       0x40
 276
 277 #define INC_SIZE(s)                     (*inst++ = (s), compiler->size += (s))
 278
 279 #define PUSH_REG(r)                     (*inst++ = (PUSH_r + (r)))
 280 #define POP_REG(r)                      (*inst++ = (POP_r + (r)))
 281 #define RET()                           (*inst++ = (RET_near))
 282 #define RET_I16(n)                      (*inst++ = (RET_i16), *inst++ = n, *inst++ = 0)
 283 /* r32, r/m32 */
 284 #define MOV_RM(mod, reg, rm)            (*inst++ = (MOV_r_rm), *inst++ = (mod) << 6 | (reg) << 3 | (rm))
 285
 286 /* Multithreading does not affect these static variables, since they store
 287    built-in CPU features. Therefore they can be overwritten by different threads
 288    if they detect the CPU features in the same time. */
 289 #if (defined SLJIT_DETECT_SSE2 && SLJIT_DETECT_SSE2)
 290 static sljit_s32 cpu_has_sse2 = -1;
 291 #endif
 292 static sljit_s32 cpu_has_cmov = -1;
 293
 294 #ifdef _WIN32_WCE
 295 #include <cmnintrin.h>
 296 #elif defined(_MSC_VER) && _MSC_VER >= 1400
 297 #include <intrin.h>
 298 #endif
 299
 300 /******************************************************/
 301 /*    Unaligned-store functions                       */
 302 /******************************************************/
 303
 304 static SLJIT_INLINE void sljit_unaligned_store_s16(void *addr, sljit_s16 value)
 305 {
 306         SLJIT_MEMCPY(addr, &value, sizeof(value));
 307 }
 308
 309 static SLJIT_INLINE void sljit_unaligned_store_s32(void *addr, sljit_s32 value)
 310 {
 311         SLJIT_MEMCPY(addr, &value, sizeof(value));
 312 }
 313
 314 static SLJIT_INLINE void sljit_unaligned_store_sw(void *addr, sljit_sw value)
 315 {
 316         SLJIT_MEMCPY(addr, &value, sizeof(value));
 317 }
 318
 319 /******************************************************/
 320 /*    Utility functions                               */
 321 /******************************************************/
 322
 323 static void get_cpu_features(void)
 324 {
 325         sljit_u32 features;
 326
 327 #if defined(_MSC_VER) && _MSC_VER >= 1400
 328
 329         int CPUInfo[4];
 330         __cpuid(CPUInfo, 1);
 331         features = (sljit_u32)CPUInfo[3];
 332
 333 #elif defined(__GNUC__) || defined(__INTEL_COMPILER) || defined(__SUNPRO_C)
 334
 335         /* AT&T syntax. */
 336         __asm__ (
 337                 "movl $0x1, %%eax\n"
 338 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
 339                 /* On x86-32, there is no red zone, so this
 340                    should work (no need for a local variable). */
 341                 "push %%ebx\n"
 342 #endif
 343                 "cpuid\n"
 344 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
 345                 "pop %%ebx\n"
 346 #endif
 347                 "movl %%edx, %0\n"
 348                 : "=g" (features)
 349                 :
 350 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
 351                 : "%eax", "%ecx", "%edx"
 352 #else
 353                 : "%rax", "%rbx", "%rcx", "%rdx"
 354 #endif
 355         );
 356
 357 #else /* _MSC_VER && _MSC_VER >= 1400 */
 358
 359         /* Intel syntax. */
 360         __asm {
 361                 mov eax, 1
 362                 cpuid
 363                 mov features, edx
 364         }
 365
 366 #endif /* _MSC_VER && _MSC_VER >= 1400 */
 367
 368 #if (defined SLJIT_DETECT_SSE2 && SLJIT_DETECT_SSE2)
 369         cpu_has_sse2 = (features >> 26) & 0x1;
 370 #endif
 371         cpu_has_cmov = (features >> 15) & 0x1;
 372 }
 373
 374 static sljit_u8 get_jump_code(sljit_s32 type)
 375 {
 376         switch (type) {
 377         case SLJIT_EQUAL:
 378         case SLJIT_EQUAL_F64:
 379                 return 0x84 /* je */;
 380
 381         case SLJIT_NOT_EQUAL:
 382         case SLJIT_NOT_EQUAL_F64:
 383                 return 0x85 /* jne */;
 384
 385         case SLJIT_LESS:
 386         case SLJIT_LESS_F64:
 387                 return 0x82 /* jc */;
 388
 389         case SLJIT_GREATER_EQUAL:
 390         case SLJIT_GREATER_EQUAL_F64:
 391                 return 0x83 /* jae */;
 392
 393         case SLJIT_GREATER:
 394         case SLJIT_GREATER_F64:
 395                 return 0x87 /* jnbe */;
 396
 397         case SLJIT_LESS_EQUAL:
 398         case SLJIT_LESS_EQUAL_F64:
 399                 return 0x86 /* jbe */;
 400
 401         case SLJIT_SIG_LESS:
 402                 return 0x8c /* jl */;
 403
 404         case SLJIT_SIG_GREATER_EQUAL:
 405                 return 0x8d /* jnl */;
 406
 407         case SLJIT_SIG_GREATER:
 408                 return 0x8f /* jnle */;
 409
 410         case SLJIT_SIG_LESS_EQUAL:
 411                 return 0x8e /* jle */;
 412
 413         case SLJIT_OVERFLOW:
 414                 return 0x80 /* jo */;
 415
 416         case SLJIT_NOT_OVERFLOW:
 417                 return 0x81 /* jno */;
 418
 419         case SLJIT_UNORDERED_F64:
 420                 return 0x8a /* jp */;
 421
 422         case SLJIT_ORDERED_F64:
 423                 return 0x8b /* jpo */;
 424         }
 425         return 0;
 426 }
 427
 428 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
 429 static sljit_u8* generate_far_jump_code(struct sljit_jump *jump, sljit_u8 *code_ptr, sljit_sw executable_offset);
 430 #else
 431 static sljit_u8* generate_far_jump_code(struct sljit_jump *jump, sljit_u8 *code_ptr);
 432 static sljit_u8* generate_put_label_code(struct sljit_put_label *put_label, sljit_u8 *code_ptr, sljit_uw max_label);
 433 #endif
 434
 435 static sljit_u8* generate_near_jump_code(struct sljit_jump *jump, sljit_u8 *code_ptr, sljit_u8 *code, sljit_sw executable_offset)
 436 {
 437         sljit_s32 type = jump->flags >> TYPE_SHIFT;
 438         sljit_s32 short_jump;
 439         sljit_uw label_addr;
 440
 441         if (jump->flags & JUMP_LABEL)
 442                 label_addr = (sljit_uw)(code + jump->u.label->size);
 443         else
 444                 label_addr = jump->u.target - executable_offset;
 445
 446         short_jump = (sljit_sw)(label_addr - (jump->addr + 2)) >= -128 && (sljit_sw)(label_addr - (jump->addr + 2)) <= 127;
 447
 448 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
 449         if ((sljit_sw)(label_addr - (jump->addr + 1)) > HALFWORD_MAX || (sljit_sw)(label_addr - (jump->addr + 1)) < HALFWORD_MIN)
 450                 return generate_far_jump_code(jump, code_ptr);
 451 #endif
 452
 453         if (type == SLJIT_JUMP) {
 454                 if (short_jump)
 455                         *code_ptr++ = JMP_i8;
 456                 else
 457                         *code_ptr++ = JMP_i32;
 458                 jump->addr++;
 459         }
 460         else if (type >= SLJIT_FAST_CALL) {
 461                 short_jump = 0;
 462                 *code_ptr++ = CALL_i32;
 463                 jump->addr++;
 464         }
 465         else if (short_jump) {
 466                 *code_ptr++ = get_jump_code(type) - 0x10;
 467                 jump->addr++;
 468         }
 469         else {
 470                 *code_ptr++ = GROUP_0F;
 471                 *code_ptr++ = get_jump_code(type);
 472                 jump->addr += 2;
 473         }
 474
 475         if (short_jump) {
 476                 jump->flags |= PATCH_MB;
 477                 code_ptr += sizeof(sljit_s8);
 478         } else {
 479                 jump->flags |= PATCH_MW;
 480                 code_ptr += sizeof(sljit_s32);
 481         }
 482
 483         return code_ptr;
 484 }
 485
 486 SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compiler)
 487 {
 488         struct sljit_memory_fragment *buf;
 489         sljit_u8 *code;
 490         sljit_u8 *code_ptr;
 491         sljit_u8 *buf_ptr;
 492         sljit_u8 *buf_end;
 493         sljit_u8 len;
 494         sljit_sw executable_offset;
 495         sljit_sw jump_addr;
 496
 497         struct sljit_label *label;
 498         struct sljit_jump *jump;
 499         struct sljit_const *const_;
 500         struct sljit_put_label *put_label;
 501
 502         CHECK_ERROR_PTR();
 503         CHECK_PTR(check_sljit_generate_code(compiler));
 504         reverse_buf(compiler);
 505
 506         /* Second code generation pass. */
 507         code = (sljit_u8*)SLJIT_MALLOC_EXEC(compiler->size, compiler->exec_allocator_data);
 508         PTR_FAIL_WITH_EXEC_IF(code);
 509         buf = compiler->buf;
 510
 511         code_ptr = code;
 512         label = compiler->labels;
 513         jump = compiler->jumps;
 514         const_ = compiler->consts;
 515         put_label = compiler->put_labels;
 516         executable_offset = SLJIT_EXEC_OFFSET(code);
 517
 518         do {
 519                 buf_ptr = buf->memory;
 520                 buf_end = buf_ptr + buf->used_size;
 521                 do {
 522                         len = *buf_ptr++;
 523                         if (len > 0) {
 524                                 /* The code is already generated. */
 525                                 SLJIT_MEMCPY(code_ptr, buf_ptr, len);
 526                                 code_ptr += len;
 527                                 buf_ptr += len;
 528                         }
 529                         else {
 530                                 switch (*buf_ptr) {
 531                                 case 0:
 532                                         label->addr = (sljit_uw)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset);
 533                                         label->size = code_ptr - code;
 534                                         label = label->next;
 535                                         break;
 536                                 case 1:
 537                                         jump->addr = (sljit_uw)code_ptr;
 538                                         if (!(jump->flags & SLJIT_REWRITABLE_JUMP))
 539                                                 code_ptr = generate_near_jump_code(jump, code_ptr, code, executable_offset);
 540                                         else {
 541 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
 542                                                 code_ptr = generate_far_jump_code(jump, code_ptr, executable_offset);
 543 #else
 544                                                 code_ptr = generate_far_jump_code(jump, code_ptr);
 545 #endif
 546                                         }
 547                                         jump = jump->next;
 548                                         break;
 549                                 case 2:
 550                                         const_->addr = ((sljit_uw)code_ptr) - sizeof(sljit_sw);
 551                                         const_ = const_->next;
 552                                         break;
 553                                 default:
 554                                         SLJIT_ASSERT(*buf_ptr == 3);
 555                                         SLJIT_ASSERT(put_label->label);
 556                                         put_label->addr = (sljit_uw)code_ptr;
 557 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
 558                                         code_ptr = generate_put_label_code(put_label, code_ptr, (sljit_uw)SLJIT_ADD_EXEC_OFFSET(code, executable_offset) + put_label->label->size);
 559 #endif
 560                                         put_label = put_label->next;
 561                                         break;
 562                                 }
 563                                 buf_ptr++;
 564                         }
 565                 } while (buf_ptr < buf_end);
 566                 SLJIT_ASSERT(buf_ptr == buf_end);
 567                 buf = buf->next;
 568         } while (buf);
 569
 570         SLJIT_ASSERT(!label);
 571         SLJIT_ASSERT(!jump);
 572         SLJIT_ASSERT(!const_);
 573         SLJIT_ASSERT(!put_label);
 574         SLJIT_ASSERT(code_ptr <= code + compiler->size);
 575
 576         jump = compiler->jumps;
 577         while (jump) {
 578                 jump_addr = jump->addr + executable_offset;
 579
 580                 if (jump->flags & PATCH_MB) {
 581                         SLJIT_ASSERT((sljit_sw)(jump->u.label->addr - (jump_addr + sizeof(sljit_s8))) >= -128 && (sljit_sw)(jump->u.label->addr - (jump_addr + sizeof(sljit_s8))) <= 127);
 582                         *(sljit_u8*)jump->addr = (sljit_u8)(jump->u.label->addr - (jump_addr + sizeof(sljit_s8)));
 583                 } else if (jump->flags & PATCH_MW) {
 584                         if (jump->flags & JUMP_LABEL) {
 585 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
 586                                 sljit_unaligned_store_sw((void*)jump->addr, (sljit_sw)(jump->u.label->addr - (jump_addr + sizeof(sljit_sw))));
 587 #else
 588                                 SLJIT_ASSERT((sljit_sw)(jump->u.label->addr - (jump_addr + sizeof(sljit_s32))) >= HALFWORD_MIN && (sljit_sw)(jump->u.label->addr - (jump_addr + sizeof(sljit_s32))) <= HALFWORD_MAX);
 589                                 sljit_unaligned_store_s32((void*)jump->addr, (sljit_s32)(jump->u.label->addr - (jump_addr + sizeof(sljit_s32))));
 590 #endif
 591                         }
 592                         else {
 593 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
 594                                 sljit_unaligned_store_sw((void*)jump->addr, (sljit_sw)(jump->u.target - (jump_addr + sizeof(sljit_sw))));
 595 #else
 596                                 SLJIT_ASSERT((sljit_sw)(jump->u.target - (jump_addr + sizeof(sljit_s32))) >= HALFWORD_MIN && (sljit_sw)(jump->u.target - (jump_addr + sizeof(sljit_s32))) <= HALFWORD_MAX);
 597                                 sljit_unaligned_store_s32((void*)jump->addr, (sljit_s32)(jump->u.target - (jump_addr + sizeof(sljit_s32))));
 598 #endif
 599                         }
 600                 }
 601 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
 602                 else if (jump->flags & PATCH_MD)
 603                         sljit_unaligned_store_sw((void*)jump->addr, jump->u.label->addr);
 604 #endif
 605
 606                 jump = jump->next;
 607         }
 608
 609         put_label = compiler->put_labels;
 610         while (put_label) {
 611 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
 612                 sljit_unaligned_store_sw((void*)(put_label->addr - sizeof(sljit_sw)), (sljit_sw)put_label->label->addr);
 613 #else
 614                 if (put_label->flags & PATCH_MD) {
 615                         SLJIT_ASSERT(put_label->label->addr > HALFWORD_MAX);
 616                         sljit_unaligned_store_sw((void*)(put_label->addr - sizeof(sljit_sw)), (sljit_sw)put_label->label->addr);
 617                 }
 618                 else {
 619                         SLJIT_ASSERT(put_label->label->addr <= HALFWORD_MAX);
 620                         sljit_unaligned_store_s32((void*)(put_label->addr - sizeof(sljit_s32)), (sljit_s32)put_label->label->addr);
 621                 }
 622 #endif
 623
 624                 put_label = put_label->next;
 625         }
 626
 627         compiler->error = SLJIT_ERR_COMPILED;
 628         compiler->executable_offset = executable_offset;
 629         compiler->executable_size = code_ptr - code;
 630
 631         code = (sljit_u8*)SLJIT_ADD_EXEC_OFFSET(code, executable_offset);
 632
 633         SLJIT_UPDATE_WX_FLAGS(code, (sljit_u8*)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset), 1);
 634         return (void*)code;
 635 }
 636
 637 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_has_cpu_feature(sljit_s32 feature_type)
 638 {
 639         switch (feature_type) {
 640         case SLJIT_HAS_FPU:
 641 #ifdef SLJIT_IS_FPU_AVAILABLE
 642                 return SLJIT_IS_FPU_AVAILABLE;
 643 #elif (defined SLJIT_DETECT_SSE2 && SLJIT_DETECT_SSE2)
 644                 if (cpu_has_sse2 == -1)
 645                         get_cpu_features();
 646                 return cpu_has_sse2;
 647 #else /* SLJIT_DETECT_SSE2 */
 648                 return 1;
 649 #endif /* SLJIT_DETECT_SSE2 */
 650
 651 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
 652         case SLJIT_HAS_VIRTUAL_REGISTERS:
 653                 return 1;
 654 #endif
 655
 656         case SLJIT_HAS_CLZ:
 657         case SLJIT_HAS_CMOV:
 658                 if (cpu_has_cmov == -1)
 659                         get_cpu_features();
 660                 return cpu_has_cmov;
 661
 662         case SLJIT_HAS_PREFETCH:
 663                 return 1;
 664
 665         case SLJIT_HAS_SSE2:
 666 #if (defined SLJIT_DETECT_SSE2 && SLJIT_DETECT_SSE2)
 667                 if (cpu_has_sse2 == -1)
 668                         get_cpu_features();
 669                 return cpu_has_sse2;
 670 #else
 671                 return 1;
 672 #endif
 673
 674         default:
 675                 return 0;
 676         }
 677 }
 678
 679 /* --------------------------------------------------------------------- */
 680 /*  Operators                                                            */
 681 /* --------------------------------------------------------------------- */
 682
 683 #define BINARY_OPCODE(opcode) (((opcode ## _EAX_i32) << 24) | ((opcode ## _r_rm) << 16) | ((opcode ## _rm_r) << 8) | (opcode))
 684
 685 static sljit_s32 emit_cum_binary(struct sljit_compiler *compiler,
 686         sljit_u32 op_types,
 687         sljit_s32 dst, sljit_sw dstw,
 688         sljit_s32 src1, sljit_sw src1w,
 689         sljit_s32 src2, sljit_sw src2w);
 690
 691 static sljit_s32 emit_non_cum_binary(struct sljit_compiler *compiler,
 692         sljit_u32 op_types,
 693         sljit_s32 dst, sljit_sw dstw,
 694         sljit_s32 src1, sljit_sw src1w,
 695         sljit_s32 src2, sljit_sw src2w);
 696
 697 static sljit_s32 emit_mov(struct sljit_compiler *compiler,
 698         sljit_s32 dst, sljit_sw dstw,
 699         sljit_s32 src, sljit_sw srcw);
 700
 701 #define EMIT_MOV(compiler, dst, dstw, src, srcw) \
 702         FAIL_IF(emit_mov(compiler, dst, dstw, src, srcw));
 703
 704 static SLJIT_INLINE sljit_s32 emit_sse2_store(struct sljit_compiler *compiler,
 705         sljit_s32 single, sljit_s32 dst, sljit_sw dstw, sljit_s32 src);
 706
 707 static SLJIT_INLINE sljit_s32 emit_sse2_load(struct sljit_compiler *compiler,
 708         sljit_s32 single, sljit_s32 dst, sljit_s32 src, sljit_sw srcw);
 709
 710 static sljit_s32 emit_cmp_binary(struct sljit_compiler *compiler,
 711         sljit_s32 src1, sljit_sw src1w,
 712         sljit_s32 src2, sljit_sw src2w);
 713
 714 static SLJIT_INLINE sljit_s32 emit_endbranch(struct sljit_compiler *compiler)
 715 {
 716 #if (defined SLJIT_CONFIG_X86_CET && SLJIT_CONFIG_X86_CET)
 717         /* Emit endbr32/endbr64 when CET is enabled.  */
 718         sljit_u8 *inst;
 719         inst = (sljit_u8*)ensure_buf(compiler, 1 + 4);
 720         FAIL_IF(!inst);
 721         INC_SIZE(4);
 722         *inst++ = 0xf3;
 723         *inst++ = 0x0f;
 724         *inst++ = 0x1e;
 725 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
 726         *inst = 0xfb;
 727 #else
 728         *inst = 0xfa;
 729 #endif
 730 #else /* !SLJIT_CONFIG_X86_CET */
 731         SLJIT_UNUSED_ARG(compiler);
 732 #endif /* SLJIT_CONFIG_X86_CET */
 733         return SLJIT_SUCCESS;
 734 }
 735
 736 #if (defined SLJIT_CONFIG_X86_CET && SLJIT_CONFIG_X86_CET) && defined (__SHSTK__)
 737
 738 static SLJIT_INLINE sljit_s32 emit_rdssp(struct sljit_compiler *compiler, sljit_s32 reg)
 739 {
 740         sljit_u8 *inst;
 741         sljit_s32 size;
 742
 743 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
 744         size = 5;
 745 #else
 746         size = 4;
 747 #endif
 748
 749         inst = (sljit_u8*)ensure_buf(compiler, 1 + size);
 750         FAIL_IF(!inst);
 751         INC_SIZE(size);
 752         *inst++ = 0xf3;
 753 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
 754         *inst++ = REX_W | (reg_map[reg] <= 7 ? 0 : REX_B);
 755 #endif
 756         *inst++ = 0x0f;
 757         *inst++ = 0x1e;
 758         *inst = (0x3 << 6) | (0x1 << 3) | (reg_map[reg] & 0x7);
 759         return SLJIT_SUCCESS;
 760 }
 761
 762 static SLJIT_INLINE sljit_s32 emit_incssp(struct sljit_compiler *compiler, sljit_s32 reg)
 763 {
 764         sljit_u8 *inst;
 765         sljit_s32 size;
 766
 767 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
 768         size = 5;
 769 #else
 770         size = 4;
 771 #endif
 772
 773         inst = (sljit_u8*)ensure_buf(compiler, 1 + size);
 774         FAIL_IF(!inst);
 775         INC_SIZE(size);
 776         *inst++ = 0xf3;
 777 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
 778         *inst++ = REX_W | (reg_map[reg] <= 7 ? 0 : REX_B);
 779 #endif
 780         *inst++ = 0x0f;
 781         *inst++ = 0xae;
 782         *inst = (0x3 << 6) | (0x5 << 3) | (reg_map[reg] & 0x7);
 783         return SLJIT_SUCCESS;
 784 }
 785
 786 #endif /* SLJIT_CONFIG_X86_CET && __SHSTK__ */
 787
 788 static SLJIT_INLINE sljit_s32 cpu_has_shadow_stack(void)
 789 {
 790 #if (defined SLJIT_CONFIG_X86_CET && SLJIT_CONFIG_X86_CET) && defined (__SHSTK__)
 791         return _get_ssp() != 0;
 792 #else /* !SLJIT_CONFIG_X86_CET || !__SHSTK__ */
 793         return 0;
 794 #endif /* SLJIT_CONFIG_X86_CET && __SHSTK__ */
 795 }
 796
 797 static SLJIT_INLINE sljit_s32 adjust_shadow_stack(struct sljit_compiler *compiler,
 798         sljit_s32 src, sljit_sw srcw, sljit_s32 base, sljit_sw disp)
 799 {
 800 #if (defined SLJIT_CONFIG_X86_CET && SLJIT_CONFIG_X86_CET) && defined (__SHSTK__)
 801         sljit_u8 *inst, *jz_after_cmp_inst;
 802         sljit_uw size_jz_after_cmp_inst;
 803
 804         sljit_uw size_before_rdssp_inst = compiler->size;
 805
 806         /* Generate "RDSSP TMP_REG1". */
 807         FAIL_IF(emit_rdssp(compiler, TMP_REG1));
 808
 809         /* Load return address on shadow stack into TMP_REG1. */
 810 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
 811         SLJIT_ASSERT(reg_map[TMP_REG1] == 5);
 812
 813         /* Hand code unsupported "mov 0x0(%ebp),%ebp". */
 814         inst = (sljit_u8*)ensure_buf(compiler, 1 + 3);
 815         FAIL_IF(!inst);
 816         INC_SIZE(3);
 817         *inst++ = 0x8b;
 818         *inst++ = 0x6d;
 819         *inst = 0;
 820 #else /* !SLJIT_CONFIG_X86_32 */
 821         EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_MEM1(TMP_REG1), 0);
 822 #endif /* SLJIT_CONFIG_X86_32 */
 823
 824         if (src == SLJIT_UNUSED) {
 825                 /* Return address is on stack.  */
 826                 src = SLJIT_MEM1(base);
 827                 srcw = disp;
 828         }
 829
 830         /* Compare return address against TMP_REG1. */
 831         FAIL_IF(emit_cmp_binary (compiler, TMP_REG1, 0, src, srcw));
 832
 833         /* Generate JZ to skip shadow stack ajdustment when shadow
 834            stack matches normal stack. */
 835         inst = (sljit_u8*)ensure_buf(compiler, 1 + 2);
 836         FAIL_IF(!inst);
 837         INC_SIZE(2);
 838         *inst++ = get_jump_code(SLJIT_EQUAL) - 0x10;
 839         size_jz_after_cmp_inst = compiler->size;
 840         jz_after_cmp_inst = inst;
 841
 842 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
 843         /* REX_W is not necessary. */
 844         compiler->mode32 = 1;
 845 #endif
 846         /* Load 1 into TMP_REG1. */
 847         EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_IMM, 1);
 848
 849         /* Generate "INCSSP TMP_REG1". */
 850         FAIL_IF(emit_incssp(compiler, TMP_REG1));
 851
 852         /* Jump back to "RDSSP TMP_REG1" to check shadow stack again. */
 853         inst = (sljit_u8*)ensure_buf(compiler, 1 + 2);
 854         FAIL_IF(!inst);
 855         INC_SIZE(2);
 856         *inst++ = JMP_i8;
 857         *inst = size_before_rdssp_inst - compiler->size;
 858
 859         *jz_after_cmp_inst = compiler->size - size_jz_after_cmp_inst;
 860 #else /* !SLJIT_CONFIG_X86_CET || !__SHSTK__ */
 861         SLJIT_UNUSED_ARG(compiler);
 862         SLJIT_UNUSED_ARG(src);
 863         SLJIT_UNUSED_ARG(srcw);
 864         SLJIT_UNUSED_ARG(base);
 865         SLJIT_UNUSED_ARG(disp);
 866 #endif /* SLJIT_CONFIG_X86_CET && __SHSTK__ */
 867         return SLJIT_SUCCESS;
 868 }
 869
 870 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
 871 #include "sljitNativeX86_32.c"
 872 #else
 873 #include "sljitNativeX86_64.c"
 874 #endif
 875
 876 static sljit_s32 emit_mov(struct sljit_compiler *compiler,
 877         sljit_s32 dst, sljit_sw dstw,
 878         sljit_s32 src, sljit_sw srcw)
 879 {
 880         sljit_u8* inst;
 881
 882         SLJIT_ASSERT(dst != SLJIT_UNUSED);
 883
 884         if (FAST_IS_REG(src)) {
 885                 inst = emit_x86_instruction(compiler, 1, src, 0, dst, dstw);
 886                 FAIL_IF(!inst);
 887                 *inst = MOV_rm_r;
 888                 return SLJIT_SUCCESS;
 889         }
 890         if (src & SLJIT_IMM) {
 891                 if (FAST_IS_REG(dst)) {
 892 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
 893                         return emit_do_imm(compiler, MOV_r_i32 + reg_map[dst], srcw);
 894 #else
 895                         if (!compiler->mode32) {
 896                                 if (NOT_HALFWORD(srcw))
 897                                         return emit_load_imm64(compiler, dst, srcw);
 898                         }
 899                         else
 900                                 return emit_do_imm32(compiler, (reg_map[dst] >= 8) ? REX_B : 0, MOV_r_i32 + reg_lmap[dst], srcw);
 901 #endif
 902                 }
 903 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
 904                 if (!compiler->mode32 && NOT_HALFWORD(srcw)) {
 905                         /* Immediate to memory move. Only SLJIT_MOV operation copies
 906                            an immediate directly into memory so TMP_REG1 can be used. */
 907                         FAIL_IF(emit_load_imm64(compiler, TMP_REG1, srcw));
 908                         inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, dst, dstw);
 909                         FAIL_IF(!inst);
 910                         *inst = MOV_rm_r;
 911                         return SLJIT_SUCCESS;
 912                 }
 913 #endif
 914                 inst = emit_x86_instruction(compiler, 1, SLJIT_IMM, srcw, dst, dstw);
 915                 FAIL_IF(!inst);
 916                 *inst = MOV_rm_i32;
 917                 return SLJIT_SUCCESS;
 918         }
 919         if (FAST_IS_REG(dst)) {
 920                 inst = emit_x86_instruction(compiler, 1, dst, 0, src, srcw);
 921                 FAIL_IF(!inst);
 922                 *inst = MOV_r_rm;
 923                 return SLJIT_SUCCESS;
 924         }
 925
 926         /* Memory to memory move. Only SLJIT_MOV operation copies
 927            data from memory to memory so TMP_REG1 can be used. */
 928         inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, src, srcw);
 929         FAIL_IF(!inst);
 930         *inst = MOV_r_rm;
 931         inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, dst, dstw);
 932         FAIL_IF(!inst);
 933         *inst = MOV_rm_r;
 934         return SLJIT_SUCCESS;
 935 }
 936
 937 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op0(struct sljit_compiler *compiler, sljit_s32 op)
 938 {
 939         sljit_u8 *inst;
 940 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
 941         sljit_s32 size;
 942 #endif
 943
 944         CHECK_ERROR();
 945         CHECK(check_sljit_emit_op0(compiler, op));
 946
 947         switch (GET_OPCODE(op)) {
 948         case SLJIT_BREAKPOINT:
 949                 inst = (sljit_u8*)ensure_buf(compiler, 1 + 1);
 950                 FAIL_IF(!inst);
 951                 INC_SIZE(1);
 952                 *inst = INT3;
 953                 break;
 954         case SLJIT_NOP:
 955                 inst = (sljit_u8*)ensure_buf(compiler, 1 + 1);
 956                 FAIL_IF(!inst);
 957                 INC_SIZE(1);
 958                 *inst = NOP;
 959                 break;
 960         case SLJIT_LMUL_UW:
 961         case SLJIT_LMUL_SW:
 962         case SLJIT_DIVMOD_UW:
 963         case SLJIT_DIVMOD_SW:
 964         case SLJIT_DIV_UW:
 965         case SLJIT_DIV_SW:
 966 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
 967 #ifdef _WIN64
 968                 SLJIT_ASSERT(
 969                         reg_map[SLJIT_R0] == 0
 970                         && reg_map[SLJIT_R1] == 2
 971                         && reg_map[TMP_REG1] > 7);
 972 #else
 973                 SLJIT_ASSERT(
 974                         reg_map[SLJIT_R0] == 0
 975                         && reg_map[SLJIT_R1] < 7
 976                         && reg_map[TMP_REG1] == 2);
 977 #endif
 978                 compiler->mode32 = op & SLJIT_I32_OP;
 979 #endif
 980                 SLJIT_COMPILE_ASSERT((SLJIT_DIVMOD_UW & 0x2) == 0 && SLJIT_DIV_UW - 0x2 == SLJIT_DIVMOD_UW, bad_div_opcode_assignments);
 981
 982                 op = GET_OPCODE(op);
 983                 if ((op | 0x2) == SLJIT_DIV_UW) {
 984 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32) || defined(_WIN64)
 985                         EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_R1, 0);
 986                         inst = emit_x86_instruction(compiler, 1, SLJIT_R1, 0, SLJIT_R1, 0);
 987 #else
 988                         inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, TMP_REG1, 0);
 989 #endif
 990                         FAIL_IF(!inst);
 991                         *inst = XOR_r_rm;
 992                 }
 993
 994                 if ((op | 0x2) == SLJIT_DIV_SW) {
 995 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32) || defined(_WIN64)
 996                         EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_R1, 0);
 997 #endif
 998
 999 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1000                         inst = (sljit_u8*)ensure_buf(compiler, 1 + 1);
1001                         FAIL_IF(!inst);
1002                         INC_SIZE(1);
1003                         *inst = CDQ;
1004 #else
1005                         if (compiler->mode32) {
1006                                 inst = (sljit_u8*)ensure_buf(compiler, 1 + 1);
1007                                 FAIL_IF(!inst);
1008                                 INC_SIZE(1);
1009                                 *inst = CDQ;
1010                         } else {
1011                                 inst = (sljit_u8*)ensure_buf(compiler, 1 + 2);
1012                                 FAIL_IF(!inst);
1013                                 INC_SIZE(2);
1014                                 *inst++ = REX_W;
1015                                 *inst = CDQ;
1016                         }
1017 #endif
1018                 }
1019
1020 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1021                 inst = (sljit_u8*)ensure_buf(compiler, 1 + 2);
1022                 FAIL_IF(!inst);
1023                 INC_SIZE(2);
1024                 *inst++ = GROUP_F7;
1025                 *inst = MOD_REG | ((op >= SLJIT_DIVMOD_UW) ? reg_map[TMP_REG1] : reg_map[SLJIT_R1]);
1026 #else
1027 #ifdef _WIN64
1028                 size = (!compiler->mode32 || op >= SLJIT_DIVMOD_UW) ? 3 : 2;
1029 #else
1030                 size = (!compiler->mode32) ? 3 : 2;
1031 #endif
1032                 inst = (sljit_u8*)ensure_buf(compiler, 1 + size);
1033                 FAIL_IF(!inst);
1034                 INC_SIZE(size);
1035 #ifdef _WIN64
1036                 if (!compiler->mode32)
1037                         *inst++ = REX_W | ((op >= SLJIT_DIVMOD_UW) ? REX_B : 0);
1038                 else if (op >= SLJIT_DIVMOD_UW)
1039                         *inst++ = REX_B;
1040                 *inst++ = GROUP_F7;
1041                 *inst = MOD_REG | ((op >= SLJIT_DIVMOD_UW) ? reg_lmap[TMP_REG1] : reg_lmap[SLJIT_R1]);
1042 #else
1043                 if (!compiler->mode32)
1044                         *inst++ = REX_W;
1045                 *inst++ = GROUP_F7;
1046                 *inst = MOD_REG | reg_map[SLJIT_R1];
1047 #endif
1048 #endif
1049                 switch (op) {
1050                 case SLJIT_LMUL_UW:
1051                         *inst |= MUL;
1052                         break;
1053                 case SLJIT_LMUL_SW:
1054                         *inst |= IMUL;
1055                         break;
1056                 case SLJIT_DIVMOD_UW:
1057                 case SLJIT_DIV_UW:
1058                         *inst |= DIV;
1059                         break;
1060                 case SLJIT_DIVMOD_SW:
1061                 case SLJIT_DIV_SW:
1062                         *inst |= IDIV;
1063                         break;
1064                 }
1065 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64) && !defined(_WIN64)
1066                 if (op <= SLJIT_DIVMOD_SW)
1067                         EMIT_MOV(compiler, SLJIT_R1, 0, TMP_REG1, 0);
1068 #else
1069                 if (op >= SLJIT_DIV_UW)
1070                         EMIT_MOV(compiler, SLJIT_R1, 0, TMP_REG1, 0);
1071 #endif
1072                 break;
1073         case SLJIT_ENDBR:
1074                 return emit_endbranch(compiler);
1075         case SLJIT_SKIP_FRAMES_BEFORE_RETURN:
1076                 return skip_frames_before_return(compiler);
1077         }
1078
1079         return SLJIT_SUCCESS;
1080 }
1081
1082 #define ENCODE_PREFIX(prefix) \
1083         do { \
1084                 inst = (sljit_u8*)ensure_buf(compiler, 1 + 1); \
1085                 FAIL_IF(!inst); \
1086                 INC_SIZE(1); \
1087                 *inst = (prefix); \
1088         } while (0)
1089
1090 static sljit_s32 emit_mov_byte(struct sljit_compiler *compiler, sljit_s32 sign,
1091         sljit_s32 dst, sljit_sw dstw,
1092         sljit_s32 src, sljit_sw srcw)
1093 {
1094         sljit_u8* inst;
1095         sljit_s32 dst_r;
1096 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1097         sljit_s32 work_r;
1098 #endif
1099
1100 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1101         compiler->mode32 = 0;
1102 #endif
1103
1104         if (src & SLJIT_IMM) {
1105                 if (FAST_IS_REG(dst)) {
1106 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1107                         return emit_do_imm(compiler, MOV_r_i32 + reg_map[dst], srcw);
1108 #else
1109                         inst = emit_x86_instruction(compiler, 1, SLJIT_IMM, srcw, dst, 0);
1110                         FAIL_IF(!inst);
1111                         *inst = MOV_rm_i32;
1112                         return SLJIT_SUCCESS;
1113 #endif
1114                 }
1115                 inst = emit_x86_instruction(compiler, 1 | EX86_BYTE_ARG | EX86_NO_REXW, SLJIT_IMM, srcw, dst, dstw);
1116                 FAIL_IF(!inst);
1117                 *inst = MOV_rm8_i8;
1118                 return SLJIT_SUCCESS;
1119         }
1120
1121         dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
1122
1123         if ((dst & SLJIT_MEM) && FAST_IS_REG(src)) {
1124 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1125                 if (reg_map[src] >= 4) {
1126                         SLJIT_ASSERT(dst_r == TMP_REG1);
1127                         EMIT_MOV(compiler, TMP_REG1, 0, src, 0);
1128                 } else
1129                         dst_r = src;
1130 #else
1131                 dst_r = src;
1132 #endif
1133         }
1134 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1135         else if (FAST_IS_REG(src) && reg_map[src] >= 4) {
1136                 /* src, dst are registers. */
1137                 SLJIT_ASSERT(SLOW_IS_REG(dst));
1138                 if (reg_map[dst] < 4) {
1139                         if (dst != src)
1140                                 EMIT_MOV(compiler, dst, 0, src, 0);
1141                         inst = emit_x86_instruction(compiler, 2, dst, 0, dst, 0);
1142                         FAIL_IF(!inst);
1143                         *inst++ = GROUP_0F;
1144                         *inst = sign ? MOVSX_r_rm8 : MOVZX_r_rm8;
1145                 }
1146                 else {
1147                         if (dst != src)
1148                                 EMIT_MOV(compiler, dst, 0, src, 0);
1149                         if (sign) {
1150                                 /* shl reg, 24 */
1151                                 inst = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, SLJIT_IMM, 24, dst, 0);
1152                                 FAIL_IF(!inst);
1153                                 *inst |= SHL;
1154                                 /* sar reg, 24 */
1155                                 inst = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, SLJIT_IMM, 24, dst, 0);
1156                                 FAIL_IF(!inst);
1157                                 *inst |= SAR;
1158                         }
1159                         else {
1160                                 inst = emit_x86_instruction(compiler, 1 | EX86_BIN_INS, SLJIT_IMM, 0xff, dst, 0);
1161                                 FAIL_IF(!inst);
1162                                 *(inst + 1) |= AND;
1163                         }
1164                 }
1165                 return SLJIT_SUCCESS;
1166         }
1167 #endif
1168         else {
1169                 /* src can be memory addr or reg_map[src] < 4 on x86_32 architectures. */
1170                 inst = emit_x86_instruction(compiler, 2, dst_r, 0, src, srcw);
1171                 FAIL_IF(!inst);
1172                 *inst++ = GROUP_0F;
1173                 *inst = sign ? MOVSX_r_rm8 : MOVZX_r_rm8;
1174         }
1175
1176         if (dst & SLJIT_MEM) {
1177 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1178                 if (dst_r == TMP_REG1) {
1179                         /* Find a non-used register, whose reg_map[src] < 4. */
1180                         if ((dst & REG_MASK) == SLJIT_R0) {
1181                                 if ((dst & OFFS_REG_MASK) == TO_OFFS_REG(SLJIT_R1))
1182                                         work_r = SLJIT_R2;
1183                                 else
1184                                         work_r = SLJIT_R1;
1185                         }
1186                         else {
1187                                 if ((dst & OFFS_REG_MASK) != TO_OFFS_REG(SLJIT_R0))
1188                                         work_r = SLJIT_R0;
1189                                 else if ((dst & REG_MASK) == SLJIT_R1)
1190                                         work_r = SLJIT_R2;
1191                                 else
1192                                         work_r = SLJIT_R1;
1193                         }
1194
1195                         if (work_r == SLJIT_R0) {
1196                                 ENCODE_PREFIX(XCHG_EAX_r + reg_map[TMP_REG1]);
1197                         }
1198                         else {
1199                                 inst = emit_x86_instruction(compiler, 1, work_r, 0, dst_r, 0);
1200                                 FAIL_IF(!inst);
1201                                 *inst = XCHG_r_rm;
1202                         }
1203
1204                         inst = emit_x86_instruction(compiler, 1, work_r, 0, dst, dstw);
1205                         FAIL_IF(!inst);
1206                         *inst = MOV_rm8_r8;
1207
1208                         if (work_r == SLJIT_R0) {
1209                                 ENCODE_PREFIX(XCHG_EAX_r + reg_map[TMP_REG1]);
1210                         }
1211                         else {
1212                                 inst = emit_x86_instruction(compiler, 1, work_r, 0, dst_r, 0);
1213                                 FAIL_IF(!inst);
1214                                 *inst = XCHG_r_rm;
1215                         }
1216                 }
1217                 else {
1218                         inst = emit_x86_instruction(compiler, 1, dst_r, 0, dst, dstw);
1219                         FAIL_IF(!inst);
1220                         *inst = MOV_rm8_r8;
1221                 }
1222 #else
1223                 inst = emit_x86_instruction(compiler, 1 | EX86_REX | EX86_NO_REXW, dst_r, 0, dst, dstw);
1224                 FAIL_IF(!inst);
1225                 *inst = MOV_rm8_r8;
1226 #endif
1227         }
1228
1229         return SLJIT_SUCCESS;
1230 }
1231
1232 static sljit_s32 emit_prefetch(struct sljit_compiler *compiler, sljit_s32 op,
1233         sljit_s32 src, sljit_sw srcw)
1234 {
1235         sljit_u8* inst;
1236
1237 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1238         compiler->mode32 = 1;
1239 #endif
1240
1241         inst = emit_x86_instruction(compiler, 2, 0, 0, src, srcw);
1242         FAIL_IF(!inst);
1243         *inst++ = GROUP_0F;
1244         *inst++ = PREFETCH;
1245
1246         if (op == SLJIT_PREFETCH_L1)
1247                 *inst |= (1 << 3);
1248         else if (op == SLJIT_PREFETCH_L2)
1249                 *inst |= (2 << 3);
1250         else if (op == SLJIT_PREFETCH_L3)
1251                 *inst |= (3 << 3);
1252
1253         return SLJIT_SUCCESS;
1254 }
1255
1256 static sljit_s32 emit_mov_half(struct sljit_compiler *compiler, sljit_s32 sign,
1257         sljit_s32 dst, sljit_sw dstw,
1258         sljit_s32 src, sljit_sw srcw)
1259 {
1260         sljit_u8* inst;
1261         sljit_s32 dst_r;
1262
1263 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1264         compiler->mode32 = 0;
1265 #endif
1266
1267         if (src & SLJIT_IMM) {
1268                 if (FAST_IS_REG(dst)) {
1269 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1270                         return emit_do_imm(compiler, MOV_r_i32 + reg_map[dst], srcw);
1271 #else
1272                         inst = emit_x86_instruction(compiler, 1, SLJIT_IMM, srcw, dst, 0);
1273                         FAIL_IF(!inst);
1274                         *inst = MOV_rm_i32;
1275                         return SLJIT_SUCCESS;
1276 #endif
1277                 }
1278                 inst = emit_x86_instruction(compiler, 1 | EX86_HALF_ARG | EX86_NO_REXW | EX86_PREF_66, SLJIT_IMM, srcw, dst, dstw);
1279                 FAIL_IF(!inst);
1280                 *inst = MOV_rm_i32;
1281                 return SLJIT_SUCCESS;
1282         }
1283
1284         dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
1285
1286         if ((dst & SLJIT_MEM) && FAST_IS_REG(src))
1287                 dst_r = src;
1288         else {
1289                 inst = emit_x86_instruction(compiler, 2, dst_r, 0, src, srcw);
1290                 FAIL_IF(!inst);
1291                 *inst++ = GROUP_0F;
1292                 *inst = sign ? MOVSX_r_rm16 : MOVZX_r_rm16;
1293         }
1294
1295         if (dst & SLJIT_MEM) {
1296                 inst = emit_x86_instruction(compiler, 1 | EX86_NO_REXW | EX86_PREF_66, dst_r, 0, dst, dstw);
1297                 FAIL_IF(!inst);
1298                 *inst = MOV_rm_r;
1299         }
1300
1301         return SLJIT_SUCCESS;
1302 }
1303
1304 static sljit_s32 emit_unary(struct sljit_compiler *compiler, sljit_u8 opcode,
1305         sljit_s32 dst, sljit_sw dstw,
1306         sljit_s32 src, sljit_sw srcw)
1307 {
1308         sljit_u8* inst;
1309
1310         if (dst == src && dstw == srcw) {
1311                 /* Same input and output */
1312                 inst = emit_x86_instruction(compiler, 1, 0, 0, dst, dstw);
1313                 FAIL_IF(!inst);
1314                 *inst++ = GROUP_F7;
1315                 *inst |= opcode;
1316                 return SLJIT_SUCCESS;
1317         }
1318
1319         if (SLJIT_UNLIKELY(dst == SLJIT_UNUSED))
1320                 dst = TMP_REG1;
1321
1322         if (FAST_IS_REG(dst)) {
1323                 EMIT_MOV(compiler, dst, 0, src, srcw);
1324                 inst = emit_x86_instruction(compiler, 1, 0, 0, dst, 0);
1325                 FAIL_IF(!inst);
1326                 *inst++ = GROUP_F7;
1327                 *inst |= opcode;
1328                 return SLJIT_SUCCESS;
1329         }
1330
1331         EMIT_MOV(compiler, TMP_REG1, 0, src, srcw);
1332         inst = emit_x86_instruction(compiler, 1, 0, 0, TMP_REG1, 0);
1333         FAIL_IF(!inst);
1334         *inst++ = GROUP_F7;
1335         *inst |= opcode;
1336         EMIT_MOV(compiler, dst, dstw, TMP_REG1, 0);
1337         return SLJIT_SUCCESS;
1338 }
1339
1340 static sljit_s32 emit_not_with_flags(struct sljit_compiler *compiler,
1341         sljit_s32 dst, sljit_sw dstw,
1342         sljit_s32 src, sljit_sw srcw)
1343 {
1344         sljit_u8* inst;
1345
1346         if (dst == SLJIT_UNUSED)
1347                 dst = TMP_REG1;
1348
1349         if (FAST_IS_REG(dst)) {
1350                 EMIT_MOV(compiler, dst, 0, src, srcw);
1351                 inst = emit_x86_instruction(compiler, 1, 0, 0, dst, 0);
1352                 FAIL_IF(!inst);
1353                 *inst++ = GROUP_F7;
1354                 *inst |= NOT_rm;
1355                 inst = emit_x86_instruction(compiler, 1, dst, 0, dst, 0);
1356                 FAIL_IF(!inst);
1357                 *inst = OR_r_rm;
1358                 return SLJIT_SUCCESS;
1359         }
1360
1361         EMIT_MOV(compiler, TMP_REG1, 0, src, srcw);
1362         inst = emit_x86_instruction(compiler, 1, 0, 0, TMP_REG1, 0);
1363         FAIL_IF(!inst);
1364         *inst++ = GROUP_F7;
1365         *inst |= NOT_rm;
1366         inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, TMP_REG1, 0);
1367         FAIL_IF(!inst);
1368         *inst = OR_r_rm;
1369         EMIT_MOV(compiler, dst, dstw, TMP_REG1, 0);
1370         return SLJIT_SUCCESS;
1371 }
1372
1373 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1374 static const sljit_sw emit_clz_arg = 32 + 31;
1375 #endif
1376
1377 static sljit_s32 emit_clz(struct sljit_compiler *compiler, sljit_s32 op_flags,
1378         sljit_s32 dst, sljit_sw dstw,
1379         sljit_s32 src, sljit_sw srcw)
1380 {
1381         sljit_u8* inst;
1382         sljit_s32 dst_r;
1383
1384         SLJIT_UNUSED_ARG(op_flags);
1385
1386         if (cpu_has_cmov == -1)
1387                 get_cpu_features();
1388
1389         dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
1390
1391         inst = emit_x86_instruction(compiler, 2, dst_r, 0, src, srcw);
1392         FAIL_IF(!inst);
1393         *inst++ = GROUP_0F;
1394         *inst = BSR_r_rm;
1395
1396 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1397         if (cpu_has_cmov) {
1398                 if (dst_r != TMP_REG1) {
1399                         EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_IMM, 32 + 31);
1400                         inst = emit_x86_instruction(compiler, 2, dst_r, 0, TMP_REG1, 0);
1401                 }
1402                 else
1403                         inst = emit_x86_instruction(compiler, 2, dst_r, 0, SLJIT_MEM0(), (sljit_sw)&emit_clz_arg);
1404
1405                 FAIL_IF(!inst);
1406                 *inst++ = GROUP_0F;
1407                 *inst = CMOVE_r_rm;
1408         }
1409         else
1410                 FAIL_IF(sljit_emit_cmov_generic(compiler, SLJIT_EQUAL, dst_r, SLJIT_IMM, 32 + 31));
1411
1412         inst = emit_x86_instruction(compiler, 1 | EX86_BIN_INS, SLJIT_IMM, 31, dst_r, 0);
1413 #else
1414         if (cpu_has_cmov) {
1415                 EMIT_MOV(compiler, TMP_REG2, 0, SLJIT_IMM, !(op_flags & SLJIT_I32_OP) ? (64 + 63) : (32 + 31));
1416
1417                 inst = emit_x86_instruction(compiler, 2, dst_r, 0, TMP_REG2, 0);
1418                 FAIL_IF(!inst);
1419                 *inst++ = GROUP_0F;
1420                 *inst = CMOVE_r_rm;
1421         }
1422         else
1423                 FAIL_IF(sljit_emit_cmov_generic(compiler, SLJIT_EQUAL, dst_r, SLJIT_IMM, !(op_flags & SLJIT_I32_OP) ? (64 + 63) : (32 + 31)));
1424
1425         inst = emit_x86_instruction(compiler, 1 | EX86_BIN_INS, SLJIT_IMM, !(op_flags & SLJIT_I32_OP) ? 63 : 31, dst_r, 0);
1426 #endif
1427
1428         FAIL_IF(!inst);
1429         *(inst + 1) |= XOR;
1430
1431         if (dst & SLJIT_MEM)
1432                 EMIT_MOV(compiler, dst, dstw, TMP_REG1, 0);
1433         return SLJIT_SUCCESS;
1434 }
1435
1436 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op1(struct sljit_compiler *compiler, sljit_s32 op,
1437         sljit_s32 dst, sljit_sw dstw,
1438         sljit_s32 src, sljit_sw srcw)
1439 {
1440         sljit_s32 op_flags = GET_ALL_FLAGS(op);
1441 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1442         sljit_s32 dst_is_ereg = 0;
1443 #endif
1444
1445         CHECK_ERROR();
1446         CHECK(check_sljit_emit_op1(compiler, op, dst, dstw, src, srcw));
1447         ADJUST_LOCAL_OFFSET(dst, dstw);
1448         ADJUST_LOCAL_OFFSET(src, srcw);
1449
1450         CHECK_EXTRA_REGS(dst, dstw, dst_is_ereg = 1);
1451         CHECK_EXTRA_REGS(src, srcw, (void)0);
1452 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1453         compiler->mode32 = op_flags & SLJIT_I32_OP;
1454 #endif
1455
1456         op = GET_OPCODE(op);
1457
1458         if (op >= SLJIT_MOV && op <= SLJIT_MOV_P) {
1459 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1460                 compiler->mode32 = 0;
1461 #endif
1462
1463                 if (FAST_IS_REG(src) && src == dst) {
1464                         if (!TYPE_CAST_NEEDED(op))
1465                                 return SLJIT_SUCCESS;
1466                 }
1467
1468                 if (op_flags & SLJIT_I32_OP) {
1469 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1470                         if (src & SLJIT_MEM) {
1471                                 if (op == SLJIT_MOV_S32)
1472                                         op = SLJIT_MOV_U32;
1473                         }
1474                         else if (src & SLJIT_IMM) {
1475                                 if (op == SLJIT_MOV_U32)
1476                                         op = SLJIT_MOV_S32;
1477                         }
1478 #endif
1479                 }
1480
1481                 if (src & SLJIT_IMM) {
1482                         switch (op) {
1483                         case SLJIT_MOV_U8:
1484                                 srcw = (sljit_u8)srcw;
1485                                 break;
1486                         case SLJIT_MOV_S8:
1487                                 srcw = (sljit_s8)srcw;
1488                                 break;
1489                         case SLJIT_MOV_U16:
1490                                 srcw = (sljit_u16)srcw;
1491                                 break;
1492                         case SLJIT_MOV_S16:
1493                                 srcw = (sljit_s16)srcw;
1494                                 break;
1495 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1496                         case SLJIT_MOV_U32:
1497                                 srcw = (sljit_u32)srcw;
1498                                 break;
1499                         case SLJIT_MOV_S32:
1500                                 srcw = (sljit_s32)srcw;
1501                                 break;
1502 #endif
1503                         }
1504 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1505                         if (SLJIT_UNLIKELY(dst_is_ereg))
1506                                 return emit_mov(compiler, dst, dstw, src, srcw);
1507 #endif
1508                 }
1509
1510 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1511                 if (SLJIT_UNLIKELY(dst_is_ereg) && (!(op == SLJIT_MOV || op == SLJIT_MOV_U32 || op == SLJIT_MOV_S32 || op == SLJIT_MOV_P) || (src & SLJIT_MEM))) {
1512                         SLJIT_ASSERT(dst == SLJIT_MEM1(SLJIT_SP));
1513                         dst = TMP_REG1;
1514                 }
1515 #endif
1516
1517                 switch (op) {
1518                 case SLJIT_MOV:
1519                 case SLJIT_MOV_P:
1520 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1521                 case SLJIT_MOV_U32:
1522                 case SLJIT_MOV_S32:
1523 #endif
1524                         FAIL_IF(emit_mov(compiler, dst, dstw, src, srcw));
1525                         break;
1526                 case SLJIT_MOV_U8:
1527                         FAIL_IF(emit_mov_byte(compiler, 0, dst, dstw, src, srcw));
1528                         break;
1529                 case SLJIT_MOV_S8:
1530                         FAIL_IF(emit_mov_byte(compiler, 1, dst, dstw, src, srcw));
1531                         break;
1532                 case SLJIT_MOV_U16:
1533                         FAIL_IF(emit_mov_half(compiler, 0, dst, dstw, src, srcw));
1534                         break;
1535                 case SLJIT_MOV_S16:
1536                         FAIL_IF(emit_mov_half(compiler, 1, dst, dstw, src, srcw));
1537                         break;
1538 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1539                 case SLJIT_MOV_U32:
1540                         FAIL_IF(emit_mov_int(compiler, 0, dst, dstw, src, srcw));
1541                         break;
1542                 case SLJIT_MOV_S32:
1543                         FAIL_IF(emit_mov_int(compiler, 1, dst, dstw, src, srcw));
1544                         break;
1545 #endif
1546                 }
1547
1548 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1549                 if (SLJIT_UNLIKELY(dst_is_ereg) && dst == TMP_REG1)
1550                         return emit_mov(compiler, SLJIT_MEM1(SLJIT_SP), dstw, TMP_REG1, 0);
1551 #endif
1552                 return SLJIT_SUCCESS;
1553         }
1554
1555         switch (op) {
1556         case SLJIT_NOT:
1557                 if (SLJIT_UNLIKELY(op_flags & SLJIT_SET_Z))
1558                         return emit_not_with_flags(compiler, dst, dstw, src, srcw);
1559                 return emit_unary(compiler, NOT_rm, dst, dstw, src, srcw);
1560
1561         case SLJIT_NEG:
1562                 return emit_unary(compiler, NEG_rm, dst, dstw, src, srcw);
1563
1564         case SLJIT_CLZ:
1565                 return emit_clz(compiler, op_flags, dst, dstw, src, srcw);
1566         }
1567
1568         return SLJIT_SUCCESS;
1569 }
1570
1571 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1572
1573 #define BINARY_IMM(op_imm, op_mr, immw, arg, argw) \
1574         if (IS_HALFWORD(immw) || compiler->mode32) { \
1575                 inst = emit_x86_instruction(compiler, 1 | EX86_BIN_INS, SLJIT_IMM, immw, arg, argw); \
1576                 FAIL_IF(!inst); \
1577                 *(inst + 1) |= (op_imm); \
1578         } \
1579         else { \
1580                 FAIL_IF(emit_load_imm64(compiler, (arg == TMP_REG1) ? TMP_REG2 : TMP_REG1, immw)); \
1581                 inst = emit_x86_instruction(compiler, 1, (arg == TMP_REG1) ? TMP_REG2 : TMP_REG1, 0, arg, argw); \
1582                 FAIL_IF(!inst); \
1583                 *inst = (op_mr); \
1584         }
1585
1586 #define BINARY_EAX_IMM(op_eax_imm, immw) \
1587         FAIL_IF(emit_do_imm32(compiler, (!compiler->mode32) ? REX_W : 0, (op_eax_imm), immw))
1588
1589 #else
1590
1591 #define BINARY_IMM(op_imm, op_mr, immw, arg, argw) \
1592         inst = emit_x86_instruction(compiler, 1 | EX86_BIN_INS, SLJIT_IMM, immw, arg, argw); \
1593         FAIL_IF(!inst); \
1594         *(inst + 1) |= (op_imm);
1595
1596 #define BINARY_EAX_IMM(op_eax_imm, immw) \
1597         FAIL_IF(emit_do_imm(compiler, (op_eax_imm), immw))
1598
1599 #endif
1600
1601 static sljit_s32 emit_cum_binary(struct sljit_compiler *compiler,
1602         sljit_u32 op_types,
1603         sljit_s32 dst, sljit_sw dstw,
1604         sljit_s32 src1, sljit_sw src1w,
1605         sljit_s32 src2, sljit_sw src2w)
1606 {
1607         sljit_u8* inst;
1608         sljit_u8 op_eax_imm = (op_types >> 24);
1609         sljit_u8 op_rm = (op_types >> 16) & 0xff;
1610         sljit_u8 op_mr = (op_types >> 8) & 0xff;
1611         sljit_u8 op_imm = op_types & 0xff;
1612
1613         if (dst == SLJIT_UNUSED) {
1614                 EMIT_MOV(compiler, TMP_REG1, 0, src1, src1w);
1615                 if (src2 & SLJIT_IMM) {
1616                         BINARY_IMM(op_imm, op_mr, src2w, TMP_REG1, 0);
1617                 }
1618                 else {
1619                         inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, src2, src2w);
1620                         FAIL_IF(!inst);
1621                         *inst = op_rm;
1622                 }
1623                 return SLJIT_SUCCESS;
1624         }
1625
1626         if (dst == src1 && dstw == src1w) {
1627                 if (src2 & SLJIT_IMM) {
1628 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1629                         if ((dst == SLJIT_R0) && (src2w > 127 || src2w < -128) && (compiler->mode32 || IS_HALFWORD(src2w))) {
1630 #else
1631                         if ((dst == SLJIT_R0) && (src2w > 127 || src2w < -128)) {
1632 #endif
1633                                 BINARY_EAX_IMM(op_eax_imm, src2w);
1634                         }
1635                         else {
1636                                 BINARY_IMM(op_imm, op_mr, src2w, dst, dstw);
1637                         }
1638                 }
1639                 else if (FAST_IS_REG(dst)) {
1640                         inst = emit_x86_instruction(compiler, 1, dst, dstw, src2, src2w);
1641                         FAIL_IF(!inst);
1642                         *inst = op_rm;
1643                 }
1644                 else if (FAST_IS_REG(src2)) {
1645                         /* Special exception for sljit_emit_op_flags. */
1646                         inst = emit_x86_instruction(compiler, 1, src2, src2w, dst, dstw);
1647                         FAIL_IF(!inst);
1648                         *inst = op_mr;
1649                 }
1650                 else {
1651                         EMIT_MOV(compiler, TMP_REG1, 0, src2, src2w);
1652                         inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, dst, dstw);
1653                         FAIL_IF(!inst);
1654                         *inst = op_mr;
1655                 }
1656                 return SLJIT_SUCCESS;
1657         }
1658
1659         /* Only for cumulative operations. */
1660         if (dst == src2 && dstw == src2w) {
1661                 if (src1 & SLJIT_IMM) {
1662 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1663                         if ((dst == SLJIT_R0) && (src1w > 127 || src1w < -128) && (compiler->mode32 || IS_HALFWORD(src1w))) {
1664 #else
1665                         if ((dst == SLJIT_R0) && (src1w > 127 || src1w < -128)) {
1666 #endif
1667                                 BINARY_EAX_IMM(op_eax_imm, src1w);
1668                         }
1669                         else {
1670                                 BINARY_IMM(op_imm, op_mr, src1w, dst, dstw);
1671                         }
1672                 }
1673                 else if (FAST_IS_REG(dst)) {
1674                         inst = emit_x86_instruction(compiler, 1, dst, dstw, src1, src1w);
1675                         FAIL_IF(!inst);
1676                         *inst = op_rm;
1677                 }
1678                 else if (FAST_IS_REG(src1)) {
1679                         inst = emit_x86_instruction(compiler, 1, src1, src1w, dst, dstw);
1680                         FAIL_IF(!inst);
1681                         *inst = op_mr;
1682                 }
1683                 else {
1684                         EMIT_MOV(compiler, TMP_REG1, 0, src1, src1w);
1685                         inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, dst, dstw);
1686                         FAIL_IF(!inst);
1687                         *inst = op_mr;
1688                 }
1689                 return SLJIT_SUCCESS;
1690         }
1691
1692         /* General version. */
1693         if (FAST_IS_REG(dst)) {
1694                 EMIT_MOV(compiler, dst, 0, src1, src1w);
1695                 if (src2 & SLJIT_IMM) {
1696                         BINARY_IMM(op_imm, op_mr, src2w, dst, 0);
1697                 }
1698                 else {
1699                         inst = emit_x86_instruction(compiler, 1, dst, 0, src2, src2w);
1700                         FAIL_IF(!inst);
1701                         *inst = op_rm;
1702                 }
1703         }
1704         else {
1705                 /* This version requires less memory writing. */
1706                 EMIT_MOV(compiler, TMP_REG1, 0, src1, src1w);
1707                 if (src2 & SLJIT_IMM) {
1708                         BINARY_IMM(op_imm, op_mr, src2w, TMP_REG1, 0);
1709                 }
1710                 else {
1711                         inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, src2, src2w);
1712                         FAIL_IF(!inst);
1713                         *inst = op_rm;
1714                 }
1715                 EMIT_MOV(compiler, dst, dstw, TMP_REG1, 0);
1716         }
1717
1718         return SLJIT_SUCCESS;
1719 }
1720
1721 static sljit_s32 emit_non_cum_binary(struct sljit_compiler *compiler,
1722         sljit_u32 op_types,
1723         sljit_s32 dst, sljit_sw dstw,
1724         sljit_s32 src1, sljit_sw src1w,
1725         sljit_s32 src2, sljit_sw src2w)
1726 {
1727         sljit_u8* inst;
1728         sljit_u8 op_eax_imm = (op_types >> 24);
1729         sljit_u8 op_rm = (op_types >> 16) & 0xff;
1730         sljit_u8 op_mr = (op_types >> 8) & 0xff;
1731         sljit_u8 op_imm = op_types & 0xff;
1732
1733         if (dst == SLJIT_UNUSED) {
1734                 EMIT_MOV(compiler, TMP_REG1, 0, src1, src1w);
1735                 if (src2 & SLJIT_IMM) {
1736                         BINARY_IMM(op_imm, op_mr, src2w, TMP_REG1, 0);
1737                 }
1738                 else {
1739                         inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, src2, src2w);
1740                         FAIL_IF(!inst);
1741                         *inst = op_rm;
1742                 }
1743                 return SLJIT_SUCCESS;
1744         }
1745
1746         if (dst == src1 && dstw == src1w) {
1747                 if (src2 & SLJIT_IMM) {
1748 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1749                         if ((dst == SLJIT_R0) && (src2w > 127 || src2w < -128) && (compiler->mode32 || IS_HALFWORD(src2w))) {
1750 #else
1751                         if ((dst == SLJIT_R0) && (src2w > 127 || src2w < -128)) {
1752 #endif
1753                                 BINARY_EAX_IMM(op_eax_imm, src2w);
1754                         }
1755                         else {
1756                                 BINARY_IMM(op_imm, op_mr, src2w, dst, dstw);
1757                         }
1758                 }
1759                 else if (FAST_IS_REG(dst)) {
1760                         inst = emit_x86_instruction(compiler, 1, dst, dstw, src2, src2w);
1761                         FAIL_IF(!inst);
1762                         *inst = op_rm;
1763                 }
1764                 else if (FAST_IS_REG(src2)) {
1765                         inst = emit_x86_instruction(compiler, 1, src2, src2w, dst, dstw);
1766                         FAIL_IF(!inst);
1767                         *inst = op_mr;
1768                 }
1769                 else {
1770                         EMIT_MOV(compiler, TMP_REG1, 0, src2, src2w);
1771                         inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, dst, dstw);
1772                         FAIL_IF(!inst);
1773                         *inst = op_mr;
1774                 }
1775                 return SLJIT_SUCCESS;
1776         }
1777
1778         /* General version. */
1779         if (FAST_IS_REG(dst) && dst != src2) {
1780                 EMIT_MOV(compiler, dst, 0, src1, src1w);
1781                 if (src2 & SLJIT_IMM) {
1782                         BINARY_IMM(op_imm, op_mr, src2w, dst, 0);
1783                 }
1784                 else {
1785                         inst = emit_x86_instruction(compiler, 1, dst, 0, src2, src2w);
1786                         FAIL_IF(!inst);
1787                         *inst = op_rm;
1788                 }
1789         }
1790         else {
1791                 /* This version requires less memory writing. */
1792                 EMIT_MOV(compiler, TMP_REG1, 0, src1, src1w);
1793                 if (src2 & SLJIT_IMM) {
1794                         BINARY_IMM(op_imm, op_mr, src2w, TMP_REG1, 0);
1795                 }
1796                 else {
1797                         inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, src2, src2w);
1798                         FAIL_IF(!inst);
1799                         *inst = op_rm;
1800                 }
1801                 EMIT_MOV(compiler, dst, dstw, TMP_REG1, 0);
1802         }
1803
1804         return SLJIT_SUCCESS;
1805 }
1806
1807 static sljit_s32 emit_mul(struct sljit_compiler *compiler,
1808         sljit_s32 dst, sljit_sw dstw,
1809         sljit_s32 src1, sljit_sw src1w,
1810         sljit_s32 src2, sljit_sw src2w)
1811 {
1812         sljit_u8* inst;
1813         sljit_s32 dst_r;
1814
1815         dst_r = SLOW_IS_REG(dst) ? dst : TMP_REG1;
1816
1817         /* Register destination. */
1818         if (dst_r == src1 && !(src2 & SLJIT_IMM)) {
1819                 inst = emit_x86_instruction(compiler, 2, dst_r, 0, src2, src2w);
1820                 FAIL_IF(!inst);
1821                 *inst++ = GROUP_0F;
1822                 *inst = IMUL_r_rm;
1823         }
1824         else if (dst_r == src2 && !(src1 & SLJIT_IMM)) {
1825                 inst = emit_x86_instruction(compiler, 2, dst_r, 0, src1, src1w);
1826                 FAIL_IF(!inst);
1827                 *inst++ = GROUP_0F;
1828                 *inst = IMUL_r_rm;
1829         }
1830         else if (src1 & SLJIT_IMM) {
1831                 if (src2 & SLJIT_IMM) {
1832                         EMIT_MOV(compiler, dst_r, 0, SLJIT_IMM, src2w);
1833                         src2 = dst_r;
1834                         src2w = 0;
1835                 }
1836
1837                 if (src1w <= 127 && src1w >= -128) {
1838                         inst = emit_x86_instruction(compiler, 1, dst_r, 0, src2, src2w);
1839                         FAIL_IF(!inst);
1840                         *inst = IMUL_r_rm_i8;
1841                         inst = (sljit_u8*)ensure_buf(compiler, 1 + 1);
1842                         FAIL_IF(!inst);
1843                         INC_SIZE(1);
1844                         *inst = (sljit_s8)src1w;
1845                 }
1846 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1847                 else {
1848                         inst = emit_x86_instruction(compiler, 1, dst_r, 0, src2, src2w);
1849                         FAIL_IF(!inst);
1850                         *inst = IMUL_r_rm_i32;
1851                         inst = (sljit_u8*)ensure_buf(compiler, 1 + 4);
1852                         FAIL_IF(!inst);
1853                         INC_SIZE(4);
1854                         sljit_unaligned_store_sw(inst, src1w);
1855                 }
1856 #else
1857                 else if (IS_HALFWORD(src1w)) {
1858                         inst = emit_x86_instruction(compiler, 1, dst_r, 0, src2, src2w);
1859                         FAIL_IF(!inst);
1860                         *inst = IMUL_r_rm_i32;
1861                         inst = (sljit_u8*)ensure_buf(compiler, 1 + 4);
1862                         FAIL_IF(!inst);
1863                         INC_SIZE(4);
1864                         sljit_unaligned_store_s32(inst, (sljit_s32)src1w);
1865                 }
1866                 else {
1867                         if (dst_r != src2)
1868                                 EMIT_MOV(compiler, dst_r, 0, src2, src2w);
1869                         FAIL_IF(emit_load_imm64(compiler, TMP_REG2, src1w));
1870                         inst = emit_x86_instruction(compiler, 2, dst_r, 0, TMP_REG2, 0);
1871                         FAIL_IF(!inst);
1872                         *inst++ = GROUP_0F;
1873                         *inst = IMUL_r_rm;
1874                 }
1875 #endif
1876         }
1877         else if (src2 & SLJIT_IMM) {
1878                 /* Note: src1 is NOT immediate. */
1879
1880                 if (src2w <= 127 && src2w >= -128) {
1881                         inst = emit_x86_instruction(compiler, 1, dst_r, 0, src1, src1w);
1882                         FAIL_IF(!inst);
1883                         *inst = IMUL_r_rm_i8;
1884                         inst = (sljit_u8*)ensure_buf(compiler, 1 + 1);
1885                         FAIL_IF(!inst);
1886                         INC_SIZE(1);
1887                         *inst = (sljit_s8)src2w;
1888                 }
1889 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1890                 else {
1891                         inst = emit_x86_instruction(compiler, 1, dst_r, 0, src1, src1w);
1892                         FAIL_IF(!inst);
1893                         *inst = IMUL_r_rm_i32;
1894                         inst = (sljit_u8*)ensure_buf(compiler, 1 + 4);
1895                         FAIL_IF(!inst);
1896                         INC_SIZE(4);
1897                         sljit_unaligned_store_sw(inst, src2w);
1898                 }
1899 #else
1900                 else if (IS_HALFWORD(src2w)) {
1901                         inst = emit_x86_instruction(compiler, 1, dst_r, 0, src1, src1w);
1902                         FAIL_IF(!inst);
1903                         *inst = IMUL_r_rm_i32;
1904                         inst = (sljit_u8*)ensure_buf(compiler, 1 + 4);
1905                         FAIL_IF(!inst);
1906                         INC_SIZE(4);
1907                         sljit_unaligned_store_s32(inst, (sljit_s32)src2w);
1908                 }
1909                 else {
1910                         if (dst_r != src1)
1911                                 EMIT_MOV(compiler, dst_r, 0, src1, src1w);
1912                         FAIL_IF(emit_load_imm64(compiler, TMP_REG2, src2w));
1913                         inst = emit_x86_instruction(compiler, 2, dst_r, 0, TMP_REG2, 0);
1914                         FAIL_IF(!inst);
1915                         *inst++ = GROUP_0F;
1916                         *inst = IMUL_r_rm;
1917                 }
1918 #endif
1919         }
1920         else {
1921                 /* Neither argument is immediate. */
1922                 if (ADDRESSING_DEPENDS_ON(src2, dst_r))
1923                         dst_r = TMP_REG1;
1924                 EMIT_MOV(compiler, dst_r, 0, src1, src1w);
1925                 inst = emit_x86_instruction(compiler, 2, dst_r, 0, src2, src2w);
1926                 FAIL_IF(!inst);
1927                 *inst++ = GROUP_0F;
1928                 *inst = IMUL_r_rm;
1929         }
1930
1931         if (dst & SLJIT_MEM)
1932                 EMIT_MOV(compiler, dst, dstw, TMP_REG1, 0);
1933
1934         return SLJIT_SUCCESS;
1935 }
1936
1937 static sljit_s32 emit_lea_binary(struct sljit_compiler *compiler,
1938         sljit_s32 dst, sljit_sw dstw,
1939         sljit_s32 src1, sljit_sw src1w,
1940         sljit_s32 src2, sljit_sw src2w)
1941 {
1942         sljit_u8* inst;
1943         sljit_s32 dst_r, done = 0;
1944
1945         /* These cases better be left to handled by normal way. */
1946         if (dst == src1 && dstw == src1w)
1947                 return SLJIT_ERR_UNSUPPORTED;
1948         if (dst == src2 && dstw == src2w)
1949                 return SLJIT_ERR_UNSUPPORTED;
1950
1951         dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
1952
1953         if (FAST_IS_REG(src1)) {
1954                 if (FAST_IS_REG(src2)) {
1955                         inst = emit_x86_instruction(compiler, 1, dst_r, 0, SLJIT_MEM2(src1, src2), 0);
1956                         FAIL_IF(!inst);
1957                         *inst = LEA_r_m;
1958                         done = 1;
1959                 }
1960 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1961                 if ((src2 & SLJIT_IMM) && (compiler->mode32 || IS_HALFWORD(src2w))) {
1962                         inst = emit_x86_instruction(compiler, 1, dst_r, 0, SLJIT_MEM1(src1), (sljit_s32)src2w);
1963 #else
1964                 if (src2 & SLJIT_IMM) {
1965                         inst = emit_x86_instruction(compiler, 1, dst_r, 0, SLJIT_MEM1(src1), src2w);
1966 #endif
1967                         FAIL_IF(!inst);
1968                         *inst = LEA_r_m;
1969                         done = 1;
1970                 }
1971         }
1972         else if (FAST_IS_REG(src2)) {
1973 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1974                 if ((src1 & SLJIT_IMM) && (compiler->mode32 || IS_HALFWORD(src1w))) {
1975                         inst = emit_x86_instruction(compiler, 1, dst_r, 0, SLJIT_MEM1(src2), (sljit_s32)src1w);
1976 #else
1977                 if (src1 & SLJIT_IMM) {
1978                         inst = emit_x86_instruction(compiler, 1, dst_r, 0, SLJIT_MEM1(src2), src1w);
1979 #endif
1980                         FAIL_IF(!inst);
1981                         *inst = LEA_r_m;
1982                         done = 1;
1983                 }
1984         }
1985
1986         if (done) {
1987                 if (dst_r == TMP_REG1)
1988                         return emit_mov(compiler, dst, dstw, TMP_REG1, 0);
1989                 return SLJIT_SUCCESS;
1990         }
1991         return SLJIT_ERR_UNSUPPORTED;
1992 }
1993
1994 static sljit_s32 emit_cmp_binary(struct sljit_compiler *compiler,
1995         sljit_s32 src1, sljit_sw src1w,
1996         sljit_s32 src2, sljit_sw src2w)
1997 {
1998         sljit_u8* inst;
1999
2000 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2001         if (src1 == SLJIT_R0 && (src2 & SLJIT_IMM) && (src2w > 127 || src2w < -128) && (compiler->mode32 || IS_HALFWORD(src2w))) {
2002 #else
2003         if (src1 == SLJIT_R0 && (src2 & SLJIT_IMM) && (src2w > 127 || src2w < -128)) {
2004 #endif
2005                 BINARY_EAX_IMM(CMP_EAX_i32, src2w);
2006                 return SLJIT_SUCCESS;
2007         }
2008
2009         if (FAST_IS_REG(src1)) {
2010                 if (src2 & SLJIT_IMM) {
2011                         BINARY_IMM(CMP, CMP_rm_r, src2w, src1, 0);
2012                 }
2013                 else {
2014                         inst = emit_x86_instruction(compiler, 1, src1, 0, src2, src2w);
2015                         FAIL_IF(!inst);
2016                         *inst = CMP_r_rm;
2017                 }
2018                 return SLJIT_SUCCESS;
2019         }
2020
2021         if (FAST_IS_REG(src2) && !(src1 & SLJIT_IMM)) {
2022                 inst = emit_x86_instruction(compiler, 1, src2, 0, src1, src1w);
2023                 FAIL_IF(!inst);
2024                 *inst = CMP_rm_r;
2025                 return SLJIT_SUCCESS;
2026         }
2027
2028         if (src2 & SLJIT_IMM) {
2029                 if (src1 & SLJIT_IMM) {
2030                         EMIT_MOV(compiler, TMP_REG1, 0, src1, src1w);
2031                         src1 = TMP_REG1;
2032                         src1w = 0;
2033                 }
2034                 BINARY_IMM(CMP, CMP_rm_r, src2w, src1, src1w);
2035         }
2036         else {
2037                 EMIT_MOV(compiler, TMP_REG1, 0, src1, src1w);
2038                 inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, src2, src2w);
2039                 FAIL_IF(!inst);
2040                 *inst = CMP_r_rm;
2041         }
2042         return SLJIT_SUCCESS;
2043 }
2044
2045 static sljit_s32 emit_test_binary(struct sljit_compiler *compiler,
2046         sljit_s32 src1, sljit_sw src1w,
2047         sljit_s32 src2, sljit_sw src2w)
2048 {
2049         sljit_u8* inst;
2050
2051 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2052         if (src1 == SLJIT_R0 && (src2 & SLJIT_IMM) && (src2w > 127 || src2w < -128) && (compiler->mode32 || IS_HALFWORD(src2w))) {
2053 #else
2054         if (src1 == SLJIT_R0 && (src2 & SLJIT_IMM) && (src2w > 127 || src2w < -128)) {
2055 #endif
2056                 BINARY_EAX_IMM(TEST_EAX_i32, src2w);
2057                 return SLJIT_SUCCESS;
2058         }
2059
2060 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2061         if (src2 == SLJIT_R0 && (src1 & SLJIT_IMM) && (src1w > 127 || src1w < -128) && (compiler->mode32 || IS_HALFWORD(src1w))) {
2062 #else
2063         if (src2 == SLJIT_R0 && (src1 & SLJIT_IMM) && (src1w > 127 || src1w < -128)) {
2064 #endif
2065                 BINARY_EAX_IMM(TEST_EAX_i32, src1w);
2066                 return SLJIT_SUCCESS;
2067         }
2068
2069         if (!(src1 & SLJIT_IMM)) {
2070                 if (src2 & SLJIT_IMM) {
2071 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2072                         if (IS_HALFWORD(src2w) || compiler->mode32) {
2073                                 inst = emit_x86_instruction(compiler, 1, SLJIT_IMM, src2w, src1, src1w);
2074                                 FAIL_IF(!inst);
2075                                 *inst = GROUP_F7;
2076                         }
2077                         else {
2078                                 FAIL_IF(emit_load_imm64(compiler, TMP_REG1, src2w));
2079                                 inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, src1, src1w);
2080                                 FAIL_IF(!inst);
2081                                 *inst = TEST_rm_r;
2082                         }
2083 #else
2084                         inst = emit_x86_instruction(compiler, 1, SLJIT_IMM, src2w, src1, src1w);
2085                         FAIL_IF(!inst);
2086                         *inst = GROUP_F7;
2087 #endif
2088                         return SLJIT_SUCCESS;
2089                 }
2090                 else if (FAST_IS_REG(src1)) {
2091                         inst = emit_x86_instruction(compiler, 1, src1, 0, src2, src2w);
2092                         FAIL_IF(!inst);
2093                         *inst = TEST_rm_r;
2094                         return SLJIT_SUCCESS;
2095                 }
2096         }
2097
2098         if (!(src2 & SLJIT_IMM)) {
2099                 if (src1 & SLJIT_IMM) {
2100 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2101                         if (IS_HALFWORD(src1w) || compiler->mode32) {
2102                                 inst = emit_x86_instruction(compiler, 1, SLJIT_IMM, src1w, src2, src2w);
2103                                 FAIL_IF(!inst);
2104                                 *inst = GROUP_F7;
2105                         }
2106                         else {
2107                                 FAIL_IF(emit_load_imm64(compiler, TMP_REG1, src1w));
2108                                 inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, src2, src2w);
2109                                 FAIL_IF(!inst);
2110                                 *inst = TEST_rm_r;
2111                         }
2112 #else
2113                         inst = emit_x86_instruction(compiler, 1, src1, src1w, src2, src2w);
2114                         FAIL_IF(!inst);
2115                         *inst = GROUP_F7;
2116 #endif
2117                         return SLJIT_SUCCESS;
2118                 }
2119                 else if (FAST_IS_REG(src2)) {
2120                         inst = emit_x86_instruction(compiler, 1, src2, 0, src1, src1w);
2121                         FAIL_IF(!inst);
2122                         *inst = TEST_rm_r;
2123                         return SLJIT_SUCCESS;
2124                 }
2125         }
2126
2127         EMIT_MOV(compiler, TMP_REG1, 0, src1, src1w);
2128         if (src2 & SLJIT_IMM) {
2129 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2130                 if (IS_HALFWORD(src2w) || compiler->mode32) {
2131                         inst = emit_x86_instruction(compiler, 1, SLJIT_IMM, src2w, TMP_REG1, 0);
2132                         FAIL_IF(!inst);
2133                         *inst = GROUP_F7;
2134                 }
2135                 else {
2136                         FAIL_IF(emit_load_imm64(compiler, TMP_REG2, src2w));
2137                         inst = emit_x86_instruction(compiler, 1, TMP_REG2, 0, TMP_REG1, 0);
2138                         FAIL_IF(!inst);
2139                         *inst = TEST_rm_r;
2140                 }
2141 #else
2142                 inst = emit_x86_instruction(compiler, 1, SLJIT_IMM, src2w, TMP_REG1, 0);
2143                 FAIL_IF(!inst);
2144                 *inst = GROUP_F7;
2145 #endif
2146         }
2147         else {
2148                 inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, src2, src2w);
2149                 FAIL_IF(!inst);
2150                 *inst = TEST_rm_r;
2151         }
2152         return SLJIT_SUCCESS;
2153 }
2154
2155 static sljit_s32 emit_shift(struct sljit_compiler *compiler,
2156         sljit_u8 mode,
2157         sljit_s32 dst, sljit_sw dstw,
2158         sljit_s32 src1, sljit_sw src1w,
2159         sljit_s32 src2, sljit_sw src2w)
2160 {
2161         sljit_u8* inst;
2162
2163         if ((src2 & SLJIT_IMM) || (src2 == SLJIT_PREF_SHIFT_REG)) {
2164                 if (dst == src1 && dstw == src1w) {
2165                         inst = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, src2, src2w, dst, dstw);
2166                         FAIL_IF(!inst);
2167                         *inst |= mode;
2168                         return SLJIT_SUCCESS;
2169                 }
2170                 if (dst == SLJIT_UNUSED) {
2171                         EMIT_MOV(compiler, TMP_REG1, 0, src1, src1w);
2172                         inst = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, src2, src2w, TMP_REG1, 0);
2173                         FAIL_IF(!inst);
2174                         *inst |= mode;
2175                         return SLJIT_SUCCESS;
2176                 }
2177                 if (dst == SLJIT_PREF_SHIFT_REG && src2 == SLJIT_PREF_SHIFT_REG) {
2178                         EMIT_MOV(compiler, TMP_REG1, 0, src1, src1w);
2179                         inst = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, SLJIT_PREF_SHIFT_REG, 0, TMP_REG1, 0);
2180                         FAIL_IF(!inst);
2181                         *inst |= mode;
2182                         EMIT_MOV(compiler, SLJIT_PREF_SHIFT_REG, 0, TMP_REG1, 0);
2183                         return SLJIT_SUCCESS;
2184                 }
2185                 if (FAST_IS_REG(dst)) {
2186                         EMIT_MOV(compiler, dst, 0, src1, src1w);
2187                         inst = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, src2, src2w, dst, 0);
2188                         FAIL_IF(!inst);
2189                         *inst |= mode;
2190                         return SLJIT_SUCCESS;
2191                 }
2192
2193                 EMIT_MOV(compiler, TMP_REG1, 0, src1, src1w);
2194                 inst = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, src2, src2w, TMP_REG1, 0);
2195                 FAIL_IF(!inst);
2196                 *inst |= mode;
2197                 EMIT_MOV(compiler, dst, dstw, TMP_REG1, 0);
2198                 return SLJIT_SUCCESS;
2199         }
2200
2201         if (dst == SLJIT_PREF_SHIFT_REG) {
2202                 EMIT_MOV(compiler, TMP_REG1, 0, src1, src1w);
2203                 EMIT_MOV(compiler, SLJIT_PREF_SHIFT_REG, 0, src2, src2w);
2204                 inst = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, SLJIT_PREF_SHIFT_REG, 0, TMP_REG1, 0);
2205                 FAIL_IF(!inst);
2206                 *inst |= mode;
2207                 EMIT_MOV(compiler, SLJIT_PREF_SHIFT_REG, 0, TMP_REG1, 0);
2208         }
2209         else if (SLOW_IS_REG(dst) && dst != src2 && !ADDRESSING_DEPENDS_ON(src2, dst)) {
2210                 if (src1 != dst)
2211                         EMIT_MOV(compiler, dst, 0, src1, src1w);
2212                 EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_PREF_SHIFT_REG, 0);
2213                 EMIT_MOV(compiler, SLJIT_PREF_SHIFT_REG, 0, src2, src2w);
2214                 inst = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, SLJIT_PREF_SHIFT_REG, 0, dst, 0);
2215                 FAIL_IF(!inst);
2216                 *inst |= mode;
2217                 EMIT_MOV(compiler, SLJIT_PREF_SHIFT_REG, 0, TMP_REG1, 0);
2218         }
2219         else {
2220                 /* This case is complex since ecx itself may be used for
2221                    addressing, and this case must be supported as well. */
2222                 EMIT_MOV(compiler, TMP_REG1, 0, src1, src1w);
2223 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
2224                 EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), 0, SLJIT_PREF_SHIFT_REG, 0);
2225                 EMIT_MOV(compiler, SLJIT_PREF_SHIFT_REG, 0, src2, src2w);
2226                 inst = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, SLJIT_PREF_SHIFT_REG, 0, TMP_REG1, 0);
2227                 FAIL_IF(!inst);
2228                 *inst |= mode;
2229                 EMIT_MOV(compiler, SLJIT_PREF_SHIFT_REG, 0, SLJIT_MEM1(SLJIT_SP), 0);
2230 #else
2231                 EMIT_MOV(compiler, TMP_REG2, 0, SLJIT_PREF_SHIFT_REG, 0);
2232                 EMIT_MOV(compiler, SLJIT_PREF_SHIFT_REG, 0, src2, src2w);
2233                 inst = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, SLJIT_PREF_SHIFT_REG, 0, TMP_REG1, 0);
2234                 FAIL_IF(!inst);
2235                 *inst |= mode;
2236                 EMIT_MOV(compiler, SLJIT_PREF_SHIFT_REG, 0, TMP_REG2, 0);
2237 #endif
2238                 if (dst != SLJIT_UNUSED)
2239                         return emit_mov(compiler, dst, dstw, TMP_REG1, 0);
2240         }
2241
2242         return SLJIT_SUCCESS;
2243 }
2244
2245 static sljit_s32 emit_shift_with_flags(struct sljit_compiler *compiler,
2246         sljit_u8 mode, sljit_s32 set_flags,
2247         sljit_s32 dst, sljit_sw dstw,
2248         sljit_s32 src1, sljit_sw src1w,
2249         sljit_s32 src2, sljit_sw src2w)
2250 {
2251         /* The CPU does not set flags if the shift count is 0. */
2252         if (src2 & SLJIT_IMM) {
2253 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2254                 if ((src2w & 0x3f) != 0 || (compiler->mode32 && (src2w & 0x1f) != 0))
2255                         return emit_shift(compiler, mode, dst, dstw, src1, src1w, src2, src2w);
2256 #else
2257                 if ((src2w & 0x1f) != 0)
2258                         return emit_shift(compiler, mode, dst, dstw, src1, src1w, src2, src2w);
2259 #endif
2260                 if (!set_flags)
2261                         return emit_mov(compiler, dst, dstw, src1, src1w);
2262                 /* OR dst, src, 0 */
2263                 return emit_cum_binary(compiler, BINARY_OPCODE(OR),
2264                         dst, dstw, src1, src1w, SLJIT_IMM, 0);
2265         }
2266
2267         if (!set_flags)
2268                 return emit_shift(compiler, mode, dst, dstw, src1, src1w, src2, src2w);
2269
2270         if (!FAST_IS_REG(dst))
2271                 FAIL_IF(emit_cmp_binary(compiler, src1, src1w, SLJIT_IMM, 0));
2272
2273         FAIL_IF(emit_shift(compiler, mode, dst, dstw, src1, src1w, src2, src2w));
2274
2275         if (FAST_IS_REG(dst))
2276                 return emit_cmp_binary(compiler, (dst == SLJIT_UNUSED) ? TMP_REG1 : dst, dstw, SLJIT_IMM, 0);
2277         return SLJIT_SUCCESS;
2278 }
2279
2280 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op2(struct sljit_compiler *compiler, sljit_s32 op,
2281         sljit_s32 dst, sljit_sw dstw,
2282         sljit_s32 src1, sljit_sw src1w,
2283         sljit_s32 src2, sljit_sw src2w)
2284 {
2285         CHECK_ERROR();
2286         CHECK(check_sljit_emit_op2(compiler, op, dst, dstw, src1, src1w, src2, src2w));
2287         ADJUST_LOCAL_OFFSET(dst, dstw);
2288         ADJUST_LOCAL_OFFSET(src1, src1w);
2289         ADJUST_LOCAL_OFFSET(src2, src2w);
2290
2291         CHECK_EXTRA_REGS(dst, dstw, (void)0);
2292         CHECK_EXTRA_REGS(src1, src1w, (void)0);
2293         CHECK_EXTRA_REGS(src2, src2w, (void)0);
2294 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2295         compiler->mode32 = op & SLJIT_I32_OP;
2296 #endif
2297
2298         if (dst == SLJIT_UNUSED && !HAS_FLAGS(op))
2299                 return SLJIT_SUCCESS;
2300
2301         switch (GET_OPCODE(op)) {
2302         case SLJIT_ADD:
2303                 if (!HAS_FLAGS(op)) {
2304                         if (emit_lea_binary(compiler, dst, dstw, src1, src1w, src2, src2w) != SLJIT_ERR_UNSUPPORTED)
2305                                 return compiler->error;
2306                 }
2307                 return emit_cum_binary(compiler, BINARY_OPCODE(ADD),
2308                         dst, dstw, src1, src1w, src2, src2w);
2309         case SLJIT_ADDC:
2310                 return emit_cum_binary(compiler, BINARY_OPCODE(ADC),
2311                         dst, dstw, src1, src1w, src2, src2w);
2312         case SLJIT_SUB:
2313                 if (!HAS_FLAGS(op)) {
2314                         if ((src2 & SLJIT_IMM) && emit_lea_binary(compiler, dst, dstw, src1, src1w, SLJIT_IMM, -src2w) != SLJIT_ERR_UNSUPPORTED)
2315                                 return compiler->error;
2316                         if (SLOW_IS_REG(dst) && src2 == dst) {
2317                                 FAIL_IF(emit_non_cum_binary(compiler, BINARY_OPCODE(SUB), dst, 0, dst, 0, src1, src1w));
2318                                 return emit_unary(compiler, NEG_rm, dst, 0, dst, 0);
2319                         }
2320                 }
2321
2322                 if (dst == SLJIT_UNUSED)
2323                         return emit_cmp_binary(compiler, src1, src1w, src2, src2w);
2324                 return emit_non_cum_binary(compiler, BINARY_OPCODE(SUB),
2325                         dst, dstw, src1, src1w, src2, src2w);
2326         case SLJIT_SUBC:
2327                 return emit_non_cum_binary(compiler, BINARY_OPCODE(SBB),
2328                         dst, dstw, src1, src1w, src2, src2w);
2329         case SLJIT_MUL:
2330                 return emit_mul(compiler, dst, dstw, src1, src1w, src2, src2w);
2331         case SLJIT_AND:
2332                 if (dst == SLJIT_UNUSED)
2333                         return emit_test_binary(compiler, src1, src1w, src2, src2w);
2334                 return emit_cum_binary(compiler, BINARY_OPCODE(AND),
2335                         dst, dstw, src1, src1w, src2, src2w);
2336         case SLJIT_OR:
2337                 return emit_cum_binary(compiler, BINARY_OPCODE(OR),
2338                         dst, dstw, src1, src1w, src2, src2w);
2339         case SLJIT_XOR:
2340                 return emit_cum_binary(compiler, BINARY_OPCODE(XOR),
2341                         dst, dstw, src1, src1w, src2, src2w);
2342         case SLJIT_SHL:
2343                 return emit_shift_with_flags(compiler, SHL, HAS_FLAGS(op),
2344                         dst, dstw, src1, src1w, src2, src2w);
2345         case SLJIT_LSHR:
2346                 return emit_shift_with_flags(compiler, SHR, HAS_FLAGS(op),
2347                         dst, dstw, src1, src1w, src2, src2w);
2348         case SLJIT_ASHR:
2349                 return emit_shift_with_flags(compiler, SAR, HAS_FLAGS(op),
2350                         dst, dstw, src1, src1w, src2, src2w);
2351         }
2352
2353         return SLJIT_SUCCESS;
2354 }
2355
2356 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_src(struct sljit_compiler *compiler, sljit_s32 op,
2357         sljit_s32 src, sljit_sw srcw)
2358 {
2359         CHECK_ERROR();
2360         CHECK(check_sljit_emit_op_src(compiler, op, src, srcw));
2361         ADJUST_LOCAL_OFFSET(src, srcw);
2362
2363         CHECK_EXTRA_REGS(src, srcw, (void)0);
2364
2365         switch (op) {
2366         case SLJIT_FAST_RETURN:
2367                 return emit_fast_return(compiler, src, srcw);
2368         case SLJIT_SKIP_FRAMES_BEFORE_FAST_RETURN:
2369                 /* Don't adjust shadow stack if it isn't enabled.  */
2370                 if (!cpu_has_shadow_stack ())
2371                         return SLJIT_SUCCESS;
2372                 return adjust_shadow_stack(compiler, src, srcw, SLJIT_UNUSED, 0);
2373         case SLJIT_PREFETCH_L1:
2374         case SLJIT_PREFETCH_L2:
2375         case SLJIT_PREFETCH_L3:
2376         case SLJIT_PREFETCH_ONCE:
2377                 return emit_prefetch(compiler, op, src, srcw);
2378         }
2379
2380         return SLJIT_SUCCESS;
2381 }
2382
2383 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_get_register_index(sljit_s32 reg)
2384 {
2385         CHECK_REG_INDEX(check_sljit_get_register_index(reg));
2386 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
2387         if (reg >= SLJIT_R3 && reg <= SLJIT_R8)
2388                 return -1;
2389 #endif
2390         return reg_map[reg];
2391 }
2392
2393 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_get_float_register_index(sljit_s32 reg)
2394 {
2395         CHECK_REG_INDEX(check_sljit_get_float_register_index(reg));
2396 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
2397         return reg;
2398 #else
2399         return freg_map[reg];
2400 #endif
2401 }
2402
2403 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_custom(struct sljit_compiler *compiler,
2404         void *instruction, sljit_s32 size)
2405 {
2406         sljit_u8 *inst;
2407
2408         CHECK_ERROR();
2409         CHECK(check_sljit_emit_op_custom(compiler, instruction, size));
2410
2411         inst = (sljit_u8*)ensure_buf(compiler, 1 + size);
2412         FAIL_IF(!inst);
2413         INC_SIZE(size);
2414         SLJIT_MEMCPY(inst, instruction, size);
2415         return SLJIT_SUCCESS;
2416 }
2417
2418 /* --------------------------------------------------------------------- */
2419 /*  Floating point operators                                             */
2420 /* --------------------------------------------------------------------- */
2421
2422 /* Alignment(3) + 4 * 16 bytes. */
2423 static sljit_s32 sse2_data[3 + (4 * 4)];
2424 static sljit_s32 *sse2_buffer;
2425
2426 static void init_compiler(void)
2427 {
2428         /* Align to 16 bytes. */
2429         sse2_buffer = (sljit_s32*)(((sljit_uw)sse2_data + 15) & ~0xf);
2430
2431         /* Single precision constants (each constant is 16 byte long). */
2432         sse2_buffer[0] = 0x80000000;
2433         sse2_buffer[4] = 0x7fffffff;
2434         /* Double precision constants (each constant is 16 byte long). */
2435         sse2_buffer[8] = 0;
2436         sse2_buffer[9] = 0x80000000;
2437         sse2_buffer[12] = 0xffffffff;
2438         sse2_buffer[13] = 0x7fffffff;
2439 }
2440
2441 static sljit_s32 emit_sse2(struct sljit_compiler *compiler, sljit_u8 opcode,
2442         sljit_s32 single, sljit_s32 xmm1, sljit_s32 xmm2, sljit_sw xmm2w)
2443 {
2444         sljit_u8 *inst;
2445
2446         inst = emit_x86_instruction(compiler, 2 | (single ? EX86_PREF_F3 : EX86_PREF_F2) | EX86_SSE2, xmm1, 0, xmm2, xmm2w);
2447         FAIL_IF(!inst);
2448         *inst++ = GROUP_0F;
2449         *inst = opcode;
2450         return SLJIT_SUCCESS;
2451 }
2452
2453 static sljit_s32 emit_sse2_logic(struct sljit_compiler *compiler, sljit_u8 opcode,
2454         sljit_s32 pref66, sljit_s32 xmm1, sljit_s32 xmm2, sljit_sw xmm2w)
2455 {
2456         sljit_u8 *inst;
2457
2458         inst = emit_x86_instruction(compiler, 2 | (pref66 ? EX86_PREF_66 : 0) | EX86_SSE2, xmm1, 0, xmm2, xmm2w);
2459         FAIL_IF(!inst);
2460         *inst++ = GROUP_0F;
2461         *inst = opcode;
2462         return SLJIT_SUCCESS;
2463 }
2464
2465 static SLJIT_INLINE sljit_s32 emit_sse2_load(struct sljit_compiler *compiler,
2466         sljit_s32 single, sljit_s32 dst, sljit_s32 src, sljit_sw srcw)
2467 {
2468         return emit_sse2(compiler, MOVSD_x_xm, single, dst, src, srcw);
2469 }
2470
2471 static SLJIT_INLINE sljit_s32 emit_sse2_store(struct sljit_compiler *compiler,
2472         sljit_s32 single, sljit_s32 dst, sljit_sw dstw, sljit_s32 src)
2473 {
2474         return emit_sse2(compiler, MOVSD_xm_x, single, src, dst, dstw);
2475 }
2476
2477 static SLJIT_INLINE sljit_s32 sljit_emit_fop1_conv_sw_from_f64(struct sljit_compiler *compiler, sljit_s32 op,
2478         sljit_s32 dst, sljit_sw dstw,
2479         sljit_s32 src, sljit_sw srcw)
2480 {
2481         sljit_s32 dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
2482         sljit_u8 *inst;
2483
2484 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2485         if (GET_OPCODE(op) == SLJIT_CONV_SW_FROM_F64)
2486                 compiler->mode32 = 0;
2487 #endif
2488
2489         inst = emit_x86_instruction(compiler, 2 | ((op & SLJIT_F32_OP) ? EX86_PREF_F3 : EX86_PREF_F2) | EX86_SSE2_OP2, dst_r, 0, src, srcw);
2490         FAIL_IF(!inst);
2491         *inst++ = GROUP_0F;
2492         *inst = CVTTSD2SI_r_xm;
2493
2494         if (dst & SLJIT_MEM)
2495                 return emit_mov(compiler, dst, dstw, TMP_REG1, 0);
2496         return SLJIT_SUCCESS;
2497 }
2498
2499 static SLJIT_INLINE sljit_s32 sljit_emit_fop1_conv_f64_from_sw(struct sljit_compiler *compiler, sljit_s32 op,
2500         sljit_s32 dst, sljit_sw dstw,
2501         sljit_s32 src, sljit_sw srcw)
2502 {
2503         sljit_s32 dst_r = FAST_IS_REG(dst) ? dst : TMP_FREG;
2504         sljit_u8 *inst;
2505
2506 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2507         if (GET_OPCODE(op) == SLJIT_CONV_F64_FROM_SW)
2508                 compiler->mode32 = 0;
2509 #endif
2510
2511         if (src & SLJIT_IMM) {
2512 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2513                 if (GET_OPCODE(op) == SLJIT_CONV_F64_FROM_S32)
2514                         srcw = (sljit_s32)srcw;
2515 #endif
2516                 EMIT_MOV(compiler, TMP_REG1, 0, src, srcw);
2517                 src = TMP_REG1;
2518                 srcw = 0;
2519         }
2520
2521         inst = emit_x86_instruction(compiler, 2 | ((op & SLJIT_F32_OP) ? EX86_PREF_F3 : EX86_PREF_F2) | EX86_SSE2_OP1, dst_r, 0, src, srcw);
2522         FAIL_IF(!inst);
2523         *inst++ = GROUP_0F;
2524         *inst = CVTSI2SD_x_rm;
2525
2526 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2527         compiler->mode32 = 1;
2528 #endif
2529         if (dst_r == TMP_FREG)
2530                 return emit_sse2_store(compiler, op & SLJIT_F32_OP, dst, dstw, TMP_FREG);
2531         return SLJIT_SUCCESS;
2532 }
2533
2534 static SLJIT_INLINE sljit_s32 sljit_emit_fop1_cmp(struct sljit_compiler *compiler, sljit_s32 op,
2535         sljit_s32 src1, sljit_sw src1w,
2536         sljit_s32 src2, sljit_sw src2w)
2537 {
2538         if (!FAST_IS_REG(src1)) {
2539                 FAIL_IF(emit_sse2_load(compiler, op & SLJIT_F32_OP, TMP_FREG, src1, src1w));
2540                 src1 = TMP_FREG;
2541         }
2542
2543         return emit_sse2_logic(compiler, UCOMISD_x_xm, !(op & SLJIT_F32_OP), src1, src2, src2w);
2544 }
2545
2546 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fop1(struct sljit_compiler *compiler, sljit_s32 op,
2547         sljit_s32 dst, sljit_sw dstw,
2548         sljit_s32 src, sljit_sw srcw)
2549 {
2550         sljit_s32 dst_r;
2551
2552 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2553         compiler->mode32 = 1;
2554 #endif
2555
2556         CHECK_ERROR();
2557         SELECT_FOP1_OPERATION_WITH_CHECKS(compiler, op, dst, dstw, src, srcw);
2558
2559         if (GET_OPCODE(op) == SLJIT_MOV_F64) {
2560                 if (FAST_IS_REG(dst))
2561                         return emit_sse2_load(compiler, op & SLJIT_F32_OP, dst, src, srcw);
2562                 if (FAST_IS_REG(src))
2563                         return emit_sse2_store(compiler, op & SLJIT_F32_OP, dst, dstw, src);
2564                 FAIL_IF(emit_sse2_load(compiler, op & SLJIT_F32_OP, TMP_FREG, src, srcw));
2565                 return emit_sse2_store(compiler, op & SLJIT_F32_OP, dst, dstw, TMP_FREG);
2566         }
2567
2568         if (GET_OPCODE(op) == SLJIT_CONV_F64_FROM_F32) {
2569                 dst_r = FAST_IS_REG(dst) ? dst : TMP_FREG;
2570                 if (FAST_IS_REG(src)) {
2571                         /* We overwrite the high bits of source. From SLJIT point of view,
2572                            this is not an issue.
2573                            Note: In SSE3, we could also use MOVDDUP and MOVSLDUP. */
2574                         FAIL_IF(emit_sse2_logic(compiler, UNPCKLPD_x_xm, op & SLJIT_F32_OP, src, src, 0));
2575                 }
2576                 else {
2577                         FAIL_IF(emit_sse2_load(compiler, !(op & SLJIT_F32_OP), TMP_FREG, src, srcw));
2578                         src = TMP_FREG;
2579                 }
2580
2581                 FAIL_IF(emit_sse2_logic(compiler, CVTPD2PS_x_xm, op & SLJIT_F32_OP, dst_r, src, 0));
2582                 if (dst_r == TMP_FREG)
2583                         return emit_sse2_store(compiler, op & SLJIT_F32_OP, dst, dstw, TMP_FREG);
2584                 return SLJIT_SUCCESS;
2585         }
2586
2587         if (FAST_IS_REG(dst)) {
2588                 dst_r = dst;
2589                 if (dst != src)
2590                         FAIL_IF(emit_sse2_load(compiler, op & SLJIT_F32_OP, dst_r, src, srcw));
2591         }
2592         else {
2593                 dst_r = TMP_FREG;
2594                 FAIL_IF(emit_sse2_load(compiler, op & SLJIT_F32_OP, dst_r, src, srcw));
2595         }
2596
2597         switch (GET_OPCODE(op)) {
2598         case SLJIT_NEG_F64:
2599                 FAIL_IF(emit_sse2_logic(compiler, XORPD_x_xm, 1, dst_r, SLJIT_MEM0(), (sljit_sw)(op & SLJIT_F32_OP ? sse2_buffer : sse2_buffer + 8)));
2600                 break;
2601
2602         case SLJIT_ABS_F64:
2603                 FAIL_IF(emit_sse2_logic(compiler, ANDPD_x_xm, 1, dst_r, SLJIT_MEM0(), (sljit_sw)(op & SLJIT_F32_OP ? sse2_buffer + 4 : sse2_buffer + 12)));
2604                 break;
2605         }
2606
2607         if (dst_r == TMP_FREG)
2608                 return emit_sse2_store(compiler, op & SLJIT_F32_OP, dst, dstw, TMP_FREG);
2609         return SLJIT_SUCCESS;
2610 }
2611
2612 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fop2(struct sljit_compiler *compiler, sljit_s32 op,
2613         sljit_s32 dst, sljit_sw dstw,
2614         sljit_s32 src1, sljit_sw src1w,
2615         sljit_s32 src2, sljit_sw src2w)
2616 {
2617         sljit_s32 dst_r;
2618
2619         CHECK_ERROR();
2620         CHECK(check_sljit_emit_fop2(compiler, op, dst, dstw, src1, src1w, src2, src2w));
2621         ADJUST_LOCAL_OFFSET(dst, dstw);
2622         ADJUST_LOCAL_OFFSET(src1, src1w);
2623         ADJUST_LOCAL_OFFSET(src2, src2w);
2624
2625 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2626         compiler->mode32 = 1;
2627 #endif
2628
2629         if (FAST_IS_REG(dst)) {
2630                 dst_r = dst;
2631                 if (dst == src1)
2632                         ; /* Do nothing here. */
2633                 else if (dst == src2 && (op == SLJIT_ADD_F64 || op == SLJIT_MUL_F64)) {
2634                         /* Swap arguments. */
2635                         src2 = src1;
2636                         src2w = src1w;
2637                 }
2638                 else if (dst != src2)
2639                         FAIL_IF(emit_sse2_load(compiler, op & SLJIT_F32_OP, dst_r, src1, src1w));
2640                 else {
2641                         dst_r = TMP_FREG;
2642                         FAIL_IF(emit_sse2_load(compiler, op & SLJIT_F32_OP, TMP_FREG, src1, src1w));
2643                 }
2644         }
2645         else {
2646                 dst_r = TMP_FREG;
2647                 FAIL_IF(emit_sse2_load(compiler, op & SLJIT_F32_OP, TMP_FREG, src1, src1w));
2648         }
2649
2650         switch (GET_OPCODE(op)) {
2651         case SLJIT_ADD_F64:
2652                 FAIL_IF(emit_sse2(compiler, ADDSD_x_xm, op & SLJIT_F32_OP, dst_r, src2, src2w));
2653                 break;
2654
2655         case SLJIT_SUB_F64:
2656                 FAIL_IF(emit_sse2(compiler, SUBSD_x_xm, op & SLJIT_F32_OP, dst_r, src2, src2w));
2657                 break;
2658
2659         case SLJIT_MUL_F64:
2660                 FAIL_IF(emit_sse2(compiler, MULSD_x_xm, op & SLJIT_F32_OP, dst_r, src2, src2w));
2661                 break;
2662
2663         case SLJIT_DIV_F64:
2664                 FAIL_IF(emit_sse2(compiler, DIVSD_x_xm, op & SLJIT_F32_OP, dst_r, src2, src2w));
2665                 break;
2666         }
2667
2668         if (dst_r == TMP_FREG)
2669                 return emit_sse2_store(compiler, op & SLJIT_F32_OP, dst, dstw, TMP_FREG);
2670         return SLJIT_SUCCESS;
2671 }
2672
2673 /* --------------------------------------------------------------------- */
2674 /*  Conditional instructions                                             */
2675 /* --------------------------------------------------------------------- */
2676
2677 SLJIT_API_FUNC_ATTRIBUTE struct sljit_label* sljit_emit_label(struct sljit_compiler *compiler)
2678 {
2679         sljit_u8 *inst;
2680         struct sljit_label *label;
2681
2682         CHECK_ERROR_PTR();
2683         CHECK_PTR(check_sljit_emit_label(compiler));
2684
2685         if (compiler->last_label && compiler->last_label->size == compiler->size)
2686                 return compiler->last_label;
2687
2688         label = (struct sljit_label*)ensure_abuf(compiler, sizeof(struct sljit_label));
2689         PTR_FAIL_IF(!label);
2690         set_label(label, compiler);
2691
2692         inst = (sljit_u8*)ensure_buf(compiler, 2);
2693         PTR_FAIL_IF(!inst);
2694
2695         *inst++ = 0;
2696         *inst++ = 0;
2697
2698         return label;
2699 }
2700
2701 SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_jump(struct sljit_compiler *compiler, sljit_s32 type)
2702 {
2703         sljit_u8 *inst;
2704         struct sljit_jump *jump;
2705
2706         CHECK_ERROR_PTR();
2707         CHECK_PTR(check_sljit_emit_jump(compiler, type));
2708
2709         jump = (struct sljit_jump*)ensure_abuf(compiler, sizeof(struct sljit_jump));
2710         PTR_FAIL_IF_NULL(jump);
2711         set_jump(jump, compiler, (type & SLJIT_REWRITABLE_JUMP) | ((type & 0xff) << TYPE_SHIFT));
2712         type &= 0xff;
2713
2714         /* Worst case size. */
2715 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
2716         compiler->size += (type >= SLJIT_JUMP) ? 5 : 6;
2717 #else
2718         compiler->size += (type >= SLJIT_JUMP) ? (10 + 3) : (2 + 10 + 3);
2719 #endif
2720
2721         inst = (sljit_u8*)ensure_buf(compiler, 2);
2722         PTR_FAIL_IF_NULL(inst);
2723
2724         *inst++ = 0;
2725         *inst++ = 1;
2726         return jump;
2727 }
2728
2729 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_ijump(struct sljit_compiler *compiler, sljit_s32 type, sljit_s32 src, sljit_sw srcw)
2730 {
2731         sljit_u8 *inst;
2732         struct sljit_jump *jump;
2733
2734         CHECK_ERROR();
2735         CHECK(check_sljit_emit_ijump(compiler, type, src, srcw));
2736         ADJUST_LOCAL_OFFSET(src, srcw);
2737
2738         CHECK_EXTRA_REGS(src, srcw, (void)0);
2739
2740         if (src == SLJIT_IMM) {
2741                 jump = (struct sljit_jump*)ensure_abuf(compiler, sizeof(struct sljit_jump));
2742                 FAIL_IF_NULL(jump);
2743                 set_jump(jump, compiler, JUMP_ADDR | (type << TYPE_SHIFT));
2744                 jump->u.target = srcw;
2745
2746                 /* Worst case size. */
2747 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
2748                 compiler->size += 5;
2749 #else
2750                 compiler->size += 10 + 3;
2751 #endif
2752
2753                 inst = (sljit_u8*)ensure_buf(compiler, 2);
2754                 FAIL_IF_NULL(inst);
2755
2756                 *inst++ = 0;
2757                 *inst++ = 1;
2758         }
2759         else {
2760 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2761                 /* REX_W is not necessary (src is not immediate). */
2762                 compiler->mode32 = 1;
2763 #endif
2764                 inst = emit_x86_instruction(compiler, 1, 0, 0, src, srcw);
2765                 FAIL_IF(!inst);
2766                 *inst++ = GROUP_FF;
2767                 *inst |= (type >= SLJIT_FAST_CALL) ? CALL_rm : JMP_rm;
2768         }
2769         return SLJIT_SUCCESS;
2770 }
2771
2772 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_flags(struct sljit_compiler *compiler, sljit_s32 op,
2773         sljit_s32 dst, sljit_sw dstw,
2774         sljit_s32 type)
2775 {
2776         sljit_u8 *inst;
2777         sljit_u8 cond_set = 0;
2778 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2779         sljit_s32 reg;
2780 #endif
2781         /* ADJUST_LOCAL_OFFSET and CHECK_EXTRA_REGS might overwrite these values. */
2782         sljit_s32 dst_save = dst;
2783         sljit_sw dstw_save = dstw;
2784
2785         CHECK_ERROR();
2786         CHECK(check_sljit_emit_op_flags(compiler, op, dst, dstw, type));
2787
2788         ADJUST_LOCAL_OFFSET(dst, dstw);
2789         CHECK_EXTRA_REGS(dst, dstw, (void)0);
2790
2791         type &= 0xff;
2792         /* setcc = jcc + 0x10. */
2793         cond_set = get_jump_code(type) + 0x10;
2794
2795 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2796         if (GET_OPCODE(op) == SLJIT_OR && !GET_ALL_FLAGS(op) && FAST_IS_REG(dst)) {
2797                 inst = (sljit_u8*)ensure_buf(compiler, 1 + 4 + 3);
2798                 FAIL_IF(!inst);
2799                 INC_SIZE(4 + 3);
2800                 /* Set low register to conditional flag. */
2801                 *inst++ = (reg_map[TMP_REG1] <= 7) ? REX : REX_B;
2802                 *inst++ = GROUP_0F;
2803                 *inst++ = cond_set;
2804                 *inst++ = MOD_REG | reg_lmap[TMP_REG1];
2805                 *inst++ = REX | (reg_map[TMP_REG1] <= 7 ? 0 : REX_R) | (reg_map[dst] <= 7 ? 0 : REX_B);
2806                 *inst++ = OR_rm8_r8;
2807                 *inst++ = MOD_REG | (reg_lmap[TMP_REG1] << 3) | reg_lmap[dst];
2808                 return SLJIT_SUCCESS;
2809         }
2810
2811         reg = (GET_OPCODE(op) < SLJIT_ADD && FAST_IS_REG(dst)) ? dst : TMP_REG1;
2812
2813         inst = (sljit_u8*)ensure_buf(compiler, 1 + 4 + 4);
2814         FAIL_IF(!inst);
2815         INC_SIZE(4 + 4);
2816         /* Set low register to conditional flag. */
2817         *inst++ = (reg_map[reg] <= 7) ? REX : REX_B;
2818         *inst++ = GROUP_0F;
2819         *inst++ = cond_set;
2820         *inst++ = MOD_REG | reg_lmap[reg];
2821         *inst++ = REX_W | (reg_map[reg] <= 7 ? 0 : (REX_B | REX_R));
2822         /* The movzx instruction does not affect flags. */
2823         *inst++ = GROUP_0F;
2824         *inst++ = MOVZX_r_rm8;
2825         *inst = MOD_REG | (reg_lmap[reg] << 3) | reg_lmap[reg];
2826
2827         if (reg != TMP_REG1)
2828                 return SLJIT_SUCCESS;
2829
2830         if (GET_OPCODE(op) < SLJIT_ADD) {
2831                 compiler->mode32 = GET_OPCODE(op) != SLJIT_MOV;
2832                 return emit_mov(compiler, dst, dstw, TMP_REG1, 0);
2833         }
2834
2835 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE) \
2836                 || (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2837         compiler->skip_checks = 1;
2838 #endif
2839         return sljit_emit_op2(compiler, op, dst_save, dstw_save, dst_save, dstw_save, TMP_REG1, 0);
2840
2841 #else
2842         /* The SLJIT_CONFIG_X86_32 code path starts here. */
2843         if (GET_OPCODE(op) < SLJIT_ADD && FAST_IS_REG(dst)) {
2844                 if (reg_map[dst] <= 4) {
2845                         /* Low byte is accessible. */
2846                         inst = (sljit_u8*)ensure_buf(compiler, 1 + 3 + 3);
2847                         FAIL_IF(!inst);
2848                         INC_SIZE(3 + 3);
2849                         /* Set low byte to conditional flag. */
2850                         *inst++ = GROUP_0F;
2851                         *inst++ = cond_set;
2852                         *inst++ = MOD_REG | reg_map[dst];
2853
2854                         *inst++ = GROUP_0F;
2855                         *inst++ = MOVZX_r_rm8;
2856                         *inst = MOD_REG | (reg_map[dst] << 3) | reg_map[dst];
2857                         return SLJIT_SUCCESS;
2858                 }
2859
2860                 /* Low byte is not accessible. */
2861                 if (cpu_has_cmov == -1)
2862                         get_cpu_features();
2863
2864                 if (cpu_has_cmov) {
2865                         EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_IMM, 1);
2866                         /* a xor reg, reg operation would overwrite the flags. */
2867                         EMIT_MOV(compiler, dst, 0, SLJIT_IMM, 0);
2868
2869                         inst = (sljit_u8*)ensure_buf(compiler, 1 + 3);
2870                         FAIL_IF(!inst);
2871                         INC_SIZE(3);
2872
2873                         *inst++ = GROUP_0F;
2874                         /* cmovcc = setcc - 0x50. */
2875                         *inst++ = cond_set - 0x50;
2876                         *inst++ = MOD_REG | (reg_map[dst] << 3) | reg_map[TMP_REG1];
2877                         return SLJIT_SUCCESS;
2878                 }
2879
2880                 inst = (sljit_u8*)ensure_buf(compiler, 1 + 1 + 3 + 3 + 1);
2881                 FAIL_IF(!inst);
2882                 INC_SIZE(1 + 3 + 3 + 1);
2883                 *inst++ = XCHG_EAX_r + reg_map[TMP_REG1];
2884                 /* Set al to conditional flag. */
2885                 *inst++ = GROUP_0F;
2886                 *inst++ = cond_set;
2887                 *inst++ = MOD_REG | 0 /* eax */;
2888
2889                 *inst++ = GROUP_0F;
2890                 *inst++ = MOVZX_r_rm8;
2891                 *inst++ = MOD_REG | (reg_map[dst] << 3) | 0 /* eax */;
2892                 *inst++ = XCHG_EAX_r + reg_map[TMP_REG1];
2893                 return SLJIT_SUCCESS;
2894         }
2895
2896         if (GET_OPCODE(op) == SLJIT_OR && !GET_ALL_FLAGS(op) && FAST_IS_REG(dst) && reg_map[dst] <= 4) {
2897                 SLJIT_ASSERT(reg_map[SLJIT_R0] == 0);
2898
2899                 if (dst != SLJIT_R0) {
2900                         inst = (sljit_u8*)ensure_buf(compiler, 1 + 1 + 3 + 2 + 1);
2901                         FAIL_IF(!inst);
2902                         INC_SIZE(1 + 3 + 2 + 1);
2903                         /* Set low register to conditional flag. */
2904                         *inst++ = XCHG_EAX_r + reg_map[TMP_REG1];
2905                         *inst++ = GROUP_0F;
2906                         *inst++ = cond_set;
2907                         *inst++ = MOD_REG | 0 /* eax */;
2908                         *inst++ = OR_rm8_r8;
2909                         *inst++ = MOD_REG | (0 /* eax */ << 3) | reg_map[dst];
2910                         *inst++ = XCHG_EAX_r + reg_map[TMP_REG1];
2911                 }
2912                 else {
2913                         inst = (sljit_u8*)ensure_buf(compiler, 1 + 2 + 3 + 2 + 2);
2914                         FAIL_IF(!inst);
2915                         INC_SIZE(2 + 3 + 2 + 2);
2916                         /* Set low register to conditional flag. */
2917                         *inst++ = XCHG_r_rm;
2918                         *inst++ = MOD_REG | (1 /* ecx */ << 3) | reg_map[TMP_REG1];
2919                         *inst++ = GROUP_0F;
2920                         *inst++ = cond_set;
2921                         *inst++ = MOD_REG | 1 /* ecx */;
2922                         *inst++ = OR_rm8_r8;
2923                         *inst++ = MOD_REG | (1 /* ecx */ << 3) | 0 /* eax */;
2924                         *inst++ = XCHG_r_rm;
2925                         *inst++ = MOD_REG | (1 /* ecx */ << 3) | reg_map[TMP_REG1];
2926                 }
2927                 return SLJIT_SUCCESS;
2928         }
2929
2930         /* Set TMP_REG1 to the bit. */
2931         inst = (sljit_u8*)ensure_buf(compiler, 1 + 1 + 3 + 3 + 1);
2932         FAIL_IF(!inst);
2933         INC_SIZE(1 + 3 + 3 + 1);
2934         *inst++ = XCHG_EAX_r + reg_map[TMP_REG1];
2935         /* Set al to conditional flag. */
2936         *inst++ = GROUP_0F;
2937         *inst++ = cond_set;
2938         *inst++ = MOD_REG | 0 /* eax */;
2939
2940         *inst++ = GROUP_0F;
2941         *inst++ = MOVZX_r_rm8;
2942         *inst++ = MOD_REG | (0 << 3) /* eax */ | 0 /* eax */;
2943
2944         *inst++ = XCHG_EAX_r + reg_map[TMP_REG1];
2945
2946         if (GET_OPCODE(op) < SLJIT_ADD)
2947                 return emit_mov(compiler, dst, dstw, TMP_REG1, 0);
2948
2949 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE) \
2950                 || (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2951         compiler->skip_checks = 1;
2952 #endif
2953         return sljit_emit_op2(compiler, op, dst_save, dstw_save, dst_save, dstw_save, TMP_REG1, 0);
2954 #endif /* SLJIT_CONFIG_X86_64 */
2955 }
2956
2957 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_cmov(struct sljit_compiler *compiler, sljit_s32 type,
2958         sljit_s32 dst_reg,
2959         sljit_s32 src, sljit_sw srcw)
2960 {
2961         sljit_u8* inst;
2962
2963         CHECK_ERROR();
2964         CHECK(check_sljit_emit_cmov(compiler, type, dst_reg, src, srcw));
2965
2966 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
2967         dst_reg &= ~SLJIT_I32_OP;
2968
2969         if (!sljit_has_cpu_feature(SLJIT_HAS_CMOV) || (dst_reg >= SLJIT_R3 && dst_reg <= SLJIT_S3))
2970                 return sljit_emit_cmov_generic(compiler, type, dst_reg, src, srcw);
2971 #else
2972         if (!sljit_has_cpu_feature(SLJIT_HAS_CMOV))
2973                 return sljit_emit_cmov_generic(compiler, type, dst_reg, src, srcw);
2974 #endif
2975
2976         /* ADJUST_LOCAL_OFFSET is not needed. */
2977         CHECK_EXTRA_REGS(src, srcw, (void)0);
2978
2979 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2980         compiler->mode32 = dst_reg & SLJIT_I32_OP;
2981         dst_reg &= ~SLJIT_I32_OP;
2982 #endif
2983
2984         if (SLJIT_UNLIKELY(src & SLJIT_IMM)) {
2985                 EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_IMM, srcw);
2986                 src = TMP_REG1;
2987                 srcw = 0;
2988         }
2989
2990         inst = emit_x86_instruction(compiler, 2, dst_reg, 0, src, srcw);
2991         FAIL_IF(!inst);
2992         *inst++ = GROUP_0F;
2993         *inst = get_jump_code(type & 0xff) - 0x40;
2994         return SLJIT_SUCCESS;
2995 }
2996
2997 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_get_local_base(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw, sljit_sw offset)
2998 {
2999         CHECK_ERROR();
3000         CHECK(check_sljit_get_local_base(compiler, dst, dstw, offset));
3001         ADJUST_LOCAL_OFFSET(dst, dstw);
3002
3003         CHECK_EXTRA_REGS(dst, dstw, (void)0);
3004
3005 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
3006         compiler->mode32 = 0;
3007 #endif
3008
3009         ADJUST_LOCAL_OFFSET(SLJIT_MEM1(SLJIT_SP), offset);
3010
3011 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
3012         if (NOT_HALFWORD(offset)) {
3013                 FAIL_IF(emit_load_imm64(compiler, TMP_REG1, offset));
3014 #if (defined SLJIT_DEBUG && SLJIT_DEBUG)
3015                 SLJIT_ASSERT(emit_lea_binary(compiler, dst, dstw, SLJIT_SP, 0, TMP_REG1, 0) != SLJIT_ERR_UNSUPPORTED);
3016                 return compiler->error;
3017 #else
3018                 return emit_lea_binary(compiler, dst, dstw, SLJIT_SP, 0, TMP_REG1, 0);
3019 #endif
3020         }
3021 #endif
3022
3023         if (offset != 0)
3024                 return emit_lea_binary(compiler, dst, dstw, SLJIT_SP, 0, SLJIT_IMM, offset);
3025         return emit_mov(compiler, dst, dstw, SLJIT_SP, 0);
3026 }
3027
3028 SLJIT_API_FUNC_ATTRIBUTE struct sljit_const* sljit_emit_const(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw, sljit_sw init_value)
3029 {
3030         sljit_u8 *inst;
3031         struct sljit_const *const_;
3032 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
3033         sljit_s32 reg;
3034 #endif
3035
3036         CHECK_ERROR_PTR();
3037         CHECK_PTR(check_sljit_emit_const(compiler, dst, dstw, init_value));
3038         ADJUST_LOCAL_OFFSET(dst, dstw);
3039
3040         CHECK_EXTRA_REGS(dst, dstw, (void)0);
3041
3042         const_ = (struct sljit_const*)ensure_abuf(compiler, sizeof(struct sljit_const));
3043         PTR_FAIL_IF(!const_);
3044         set_const(const_, compiler);
3045
3046 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
3047         compiler->mode32 = 0;
3048         reg = FAST_IS_REG(dst) ? dst : TMP_REG1;
3049
3050         if (emit_load_imm64(compiler, reg, init_value))
3051                 return NULL;
3052 #else
3053         if (emit_mov(compiler, dst, dstw, SLJIT_IMM, init_value))
3054                 return NULL;
3055 #endif
3056
3057         inst = (sljit_u8*)ensure_buf(compiler, 2);
3058         PTR_FAIL_IF(!inst);
3059
3060         *inst++ = 0;
3061         *inst++ = 2;
3062
3063 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
3064         if (dst & SLJIT_MEM)
3065                 if (emit_mov(compiler, dst, dstw, TMP_REG1, 0))
3066                         return NULL;
3067 #endif
3068
3069         return const_;
3070 }
3071
3072 SLJIT_API_FUNC_ATTRIBUTE struct sljit_put_label* sljit_emit_put_label(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw)
3073 {
3074         struct sljit_put_label *put_label;
3075         sljit_u8 *inst;
3076 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
3077         sljit_s32 reg;
3078         sljit_uw start_size;
3079 #endif
3080
3081         CHECK_ERROR_PTR();
3082         CHECK_PTR(check_sljit_emit_put_label(compiler, dst, dstw));
3083         ADJUST_LOCAL_OFFSET(dst, dstw);
3084
3085         CHECK_EXTRA_REGS(dst, dstw, (void)0);
3086
3087         put_label = (struct sljit_put_label*)ensure_abuf(compiler, sizeof(struct sljit_put_label));
3088         PTR_FAIL_IF(!put_label);
3089         set_put_label(put_label, compiler, 0);
3090
3091 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
3092         compiler->mode32 = 0;
3093         reg = FAST_IS_REG(dst) ? dst : TMP_REG1;
3094
3095         if (emit_load_imm64(compiler, reg, 0))
3096                 return NULL;
3097 #else
3098         if (emit_mov(compiler, dst, dstw, SLJIT_IMM, 0))
3099                 return NULL;
3100 #endif
3101
3102 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
3103         if (dst & SLJIT_MEM) {
3104                 start_size = compiler->size;
3105                 if (emit_mov(compiler, dst, dstw, TMP_REG1, 0))
3106                         return NULL;
3107                 put_label->flags = compiler->size - start_size;
3108         }
3109 #endif
3110
3111         inst = (sljit_u8*)ensure_buf(compiler, 2);
3112         PTR_FAIL_IF(!inst);
3113
3114         *inst++ = 0;
3115         *inst++ = 3;
3116
3117         return put_label;
3118 }
3119
3120 SLJIT_API_FUNC_ATTRIBUTE void sljit_set_jump_addr(sljit_uw addr, sljit_uw new_target, sljit_sw executable_offset)
3121 {
3122         SLJIT_UNUSED_ARG(executable_offset);
3123
3124         SLJIT_UPDATE_WX_FLAGS((void*)addr, (void*)(addr + sizeof(sljit_uw)), 0);
3125 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
3126         sljit_unaligned_store_sw((void*)addr, new_target - (addr + 4) - (sljit_uw)executable_offset);
3127 #else
3128         sljit_unaligned_store_sw((void*)addr, (sljit_sw) new_target);
3129 #endif
3130         SLJIT_UPDATE_WX_FLAGS((void*)addr, (void*)(addr + sizeof(sljit_uw)), 1);
3131 }
3132
3133 SLJIT_API_FUNC_ATTRIBUTE void sljit_set_const(sljit_uw addr, sljit_sw new_constant, sljit_sw executable_offset)
3134 {
3135         SLJIT_UNUSED_ARG(executable_offset);
3136
3137         SLJIT_UPDATE_WX_FLAGS((void*)addr, (void*)(addr + sizeof(sljit_sw)), 0);
3138         sljit_unaligned_store_sw((void*)addr, new_constant);
3139         SLJIT_UPDATE_WX_FLAGS((void*)addr, (void*)(addr + sizeof(sljit_sw)), 1);
3140 }