?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Improve sljit abi calling convention
[sljit.git] / sljit_src / sljitNativeX86_common.c
blobe7d230d12f094f1ccf8c01e01c00a70a09c5a663
1 /*
2 * Stack-less Just-In-Time compiler
3 *
4 * Copyright Zoltan Herczeg (hzmester@freemail.hu). All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without modification, are
7 * permitted provided that the following conditions are met:
8 *
9 * 1. Redistributions of source code must retain the above copyright notice, this list of
10 * conditions and the following disclaimer.
11 *
12 * 2. Redistributions in binary form must reproduce the above copyright notice, this list
13 * of conditions and the following disclaimer in the documentation and/or other materials
14 * provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) AND CONTRIBUTORS ``AS IS'' AND ANY
17 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
19 * SHALL THE COPYRIGHT HOLDER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
21 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
22 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
23 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
24 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27 SLJIT_API_FUNC_ATTRIBUTE const char* sljit_get_platform_name(void)
28 {
29 return "x86" SLJIT_CPUINFO;
30 }
31
32 /*
33 32b register indexes:
34 0 - EAX
35 1 - ECX
36 2 - EDX
37 3 - EBX
38 4 - ESP
39 5 - EBP
40 6 - ESI
41 7 - EDI
42 */
43
44 /*
45 64b register indexes:
46 0 - RAX
47 1 - RCX
48 2 - RDX
49 3 - RBX
50 4 - RSP
51 5 - RBP
52 6 - RSI
53 7 - RDI
54 8 - R8 - From now on REX prefix is required
55 9 - R9
56 10 - R10
57 11 - R11
58 12 - R12
59 13 - R13
60 14 - R14
61 15 - R15
62 */
63
64 #define TMP_FREG (0)
65
66 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
67
68 /* Last register + 1. */
69 #define TMP_REG1 (SLJIT_NUMBER_OF_REGISTERS + 2)
70
71 static const sljit_u8 reg_map[SLJIT_NUMBER_OF_REGISTERS + 3] = {
72 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 7, 6, 3, 4, 5
73 };
74
75 #define CHECK_EXTRA_REGS(p, w, do) \
76 if (p >= SLJIT_R3 && p <= SLJIT_S3) { \
77 if (p <= compiler->scratches) \
78 w = compiler->scratches_offset + ((p) - SLJIT_R3) * SSIZE_OF(sw); \
79 else \
80 w = compiler->locals_offset + ((p) - SLJIT_S2) * SSIZE_OF(sw); \
81 p = SLJIT_MEM1(SLJIT_SP); \
82 do; \
83 }
84
85 #else /* SLJIT_CONFIG_X86_32 */
86
87 /* Last register + 1. */
88 #define TMP_REG1 (SLJIT_NUMBER_OF_REGISTERS + 2)
89 #define TMP_REG2 (SLJIT_NUMBER_OF_REGISTERS + 3)
90
91 /* Note: r12 & 0x7 == 0b100, which decoded as SIB byte present
92 Note: avoid to use r12 and r13 for memory addessing
93 therefore r12 is better to be a higher saved register. */
94 #ifndef _WIN64
95 /* Args: rdi(=7), rsi(=6), rdx(=2), rcx(=1), r8, r9. Scratches: rax(=0), r10, r11 */
96 static const sljit_u8 reg_map[SLJIT_NUMBER_OF_REGISTERS + 4] = {
97 0, 0, 6, 7, 1, 8, 11, 10, 12, 5, 13, 14, 15, 3, 4, 2, 9
98 };
99 /* low-map. reg_map & 0x7. */
100 static const sljit_u8 reg_lmap[SLJIT_NUMBER_OF_REGISTERS + 4] = {
101 0, 0, 6, 7, 1, 0, 3, 2, 4, 5, 5, 6, 7, 3, 4, 2, 1
102 };
103 #else
104 /* Args: rcx(=1), rdx(=2), r8, r9. Scratches: rax(=0), r10, r11 */
105 static const sljit_u8 reg_map[SLJIT_NUMBER_OF_REGISTERS + 4] = {
106 0, 0, 2, 8, 1, 11, 12, 5, 13, 14, 15, 7, 6, 3, 4, 9, 10
107 };
108 /* low-map. reg_map & 0x7. */
109 static const sljit_u8 reg_lmap[SLJIT_NUMBER_OF_REGISTERS + 4] = {
110 0, 0, 2, 0, 1, 3, 4, 5, 5, 6, 7, 7, 6, 3, 4, 1, 2
111 };
112 #endif
113
114 /* Args: xmm0-xmm3 */
115 static const sljit_u8 freg_map[SLJIT_NUMBER_OF_FLOAT_REGISTERS + 1] = {
116 4, 0, 1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
117 };
118 /* low-map. freg_map & 0x7. */
119 static const sljit_u8 freg_lmap[SLJIT_NUMBER_OF_FLOAT_REGISTERS + 1] = {
120 4, 0, 1, 2, 3, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7
121 };
122
123 #define REX_W 0x48
124 #define REX_R 0x44
125 #define REX_X 0x42
126 #define REX_B 0x41
127 #define REX 0x40
128
129 #ifndef _WIN64
130 #define HALFWORD_MAX 0x7fffffffl
131 #define HALFWORD_MIN -0x80000000l
132 #else
133 #define HALFWORD_MAX 0x7fffffffll
134 #define HALFWORD_MIN -0x80000000ll
135 #endif
136
137 #define IS_HALFWORD(x) ((x) <= HALFWORD_MAX && (x) >= HALFWORD_MIN)
138 #define NOT_HALFWORD(x) ((x) > HALFWORD_MAX || (x) < HALFWORD_MIN)
139
140 #define CHECK_EXTRA_REGS(p, w, do)
141
142 #endif /* SLJIT_CONFIG_X86_32 */
143
144 #define U8(v) ((sljit_u8)(v))
145
146
147 /* Size flags for emit_x86_instruction: */
148 #define EX86_BIN_INS 0x0010
149 #define EX86_SHIFT_INS 0x0020
150 #define EX86_REX 0x0040
151 #define EX86_NO_REXW 0x0080
152 #define EX86_BYTE_ARG 0x0100
153 #define EX86_HALF_ARG 0x0200
154 #define EX86_PREF_66 0x0400
155 #define EX86_PREF_F2 0x0800
156 #define EX86_PREF_F3 0x1000
157 #define EX86_SSE2_OP1 0x2000
158 #define EX86_SSE2_OP2 0x4000
159 #define EX86_SSE2 (EX86_SSE2_OP1 | EX86_SSE2_OP2)
160
161 /* --------------------------------------------------------------------- */
162 /* Instrucion forms */
163 /* --------------------------------------------------------------------- */
164
165 #define ADD (/* BINARY */ 0 << 3)
166 #define ADD_EAX_i32 0x05
167 #define ADD_r_rm 0x03
168 #define ADD_rm_r 0x01
169 #define ADDSD_x_xm 0x58
170 #define ADC (/* BINARY */ 2 << 3)
171 #define ADC_EAX_i32 0x15
172 #define ADC_r_rm 0x13
173 #define ADC_rm_r 0x11
174 #define AND (/* BINARY */ 4 << 3)
175 #define AND_EAX_i32 0x25
176 #define AND_r_rm 0x23
177 #define AND_rm_r 0x21
178 #define ANDPD_x_xm 0x54
179 #define BSR_r_rm (/* GROUP_0F */ 0xbd)
180 #define CALL_i32 0xe8
181 #define CALL_rm (/* GROUP_FF */ 2 << 3)
182 #define CDQ 0x99
183 #define CMOVE_r_rm (/* GROUP_0F */ 0x44)
184 #define CMP (/* BINARY */ 7 << 3)
185 #define CMP_EAX_i32 0x3d
186 #define CMP_r_rm 0x3b
187 #define CMP_rm_r 0x39
188 #define CVTPD2PS_x_xm 0x5a
189 #define CVTSI2SD_x_rm 0x2a
190 #define CVTTSD2SI_r_xm 0x2c
191 #define DIV (/* GROUP_F7 */ 6 << 3)
192 #define DIVSD_x_xm 0x5e
193 #define FSTPS 0xd9
194 #define FSTPD 0xdd
195 #define INT3 0xcc
196 #define IDIV (/* GROUP_F7 */ 7 << 3)
197 #define IMUL (/* GROUP_F7 */ 5 << 3)
198 #define IMUL_r_rm (/* GROUP_0F */ 0xaf)
199 #define IMUL_r_rm_i8 0x6b
200 #define IMUL_r_rm_i32 0x69
201 #define JE_i8 0x74
202 #define JNE_i8 0x75
203 #define JMP_i8 0xeb
204 #define JMP_i32 0xe9
205 #define JMP_rm (/* GROUP_FF */ 4 << 3)
206 #define LEA_r_m 0x8d
207 #define LOOP_i8 0xe2
208 #define MOV_r_rm 0x8b
209 #define MOV_r_i32 0xb8
210 #define MOV_rm_r 0x89
211 #define MOV_rm_i32 0xc7
212 #define MOV_rm8_i8 0xc6
213 #define MOV_rm8_r8 0x88
214 #define MOVAPS_x_xm 0x28
215 #define MOVAPS_xm_x 0x29
216 #define MOVSD_x_xm 0x10
217 #define MOVSD_xm_x 0x11
218 #define MOVSXD_r_rm 0x63
219 #define MOVSX_r_rm8 (/* GROUP_0F */ 0xbe)
220 #define MOVSX_r_rm16 (/* GROUP_0F */ 0xbf)
221 #define MOVZX_r_rm8 (/* GROUP_0F */ 0xb6)
222 #define MOVZX_r_rm16 (/* GROUP_0F */ 0xb7)
223 #define MUL (/* GROUP_F7 */ 4 << 3)
224 #define MULSD_x_xm 0x59
225 #define NEG_rm (/* GROUP_F7 */ 3 << 3)
226 #define NOP 0x90
227 #define NOT_rm (/* GROUP_F7 */ 2 << 3)
228 #define OR (/* BINARY */ 1 << 3)
229 #define OR_r_rm 0x0b
230 #define OR_EAX_i32 0x0d
231 #define OR_rm_r 0x09
232 #define OR_rm8_r8 0x08
233 #define POP_r 0x58
234 #define POP_rm 0x8f
235 #define POPF 0x9d
236 #define PREFETCH 0x18
237 #define PUSH_i32 0x68
238 #define PUSH_r 0x50
239 #define PUSH_rm (/* GROUP_FF */ 6 << 3)
240 #define PUSHF 0x9c
241 #define RET_near 0xc3
242 #define RET_i16 0xc2
243 #define SBB (/* BINARY */ 3 << 3)
244 #define SBB_EAX_i32 0x1d
245 #define SBB_r_rm 0x1b
246 #define SBB_rm_r 0x19
247 #define SAR (/* SHIFT */ 7 << 3)
248 #define SHL (/* SHIFT */ 4 << 3)
249 #define SHR (/* SHIFT */ 5 << 3)
250 #define SUB (/* BINARY */ 5 << 3)
251 #define SUB_EAX_i32 0x2d
252 #define SUB_r_rm 0x2b
253 #define SUB_rm_r 0x29
254 #define SUBSD_x_xm 0x5c
255 #define TEST_EAX_i32 0xa9
256 #define TEST_rm_r 0x85
257 #define UCOMISD_x_xm 0x2e
258 #define UNPCKLPD_x_xm 0x14
259 #define XCHG_EAX_r 0x90
260 #define XCHG_r_rm 0x87
261 #define XOR (/* BINARY */ 6 << 3)
262 #define XOR_EAX_i32 0x35
263 #define XOR_r_rm 0x33
264 #define XOR_rm_r 0x31
265 #define XORPD_x_xm 0x57
266
267 #define GROUP_0F 0x0f
268 #define GROUP_F7 0xf7
269 #define GROUP_FF 0xff
270 #define GROUP_BINARY_81 0x81
271 #define GROUP_BINARY_83 0x83
272 #define GROUP_SHIFT_1 0xd1
273 #define GROUP_SHIFT_N 0xc1
274 #define GROUP_SHIFT_CL 0xd3
275
276 #define MOD_REG 0xc0
277 #define MOD_DISP8 0x40
278
279 #define INC_SIZE(s) (*inst++ = U8(s), compiler->size += (s))
280
281 #define PUSH_REG(r) (*inst++ = U8(PUSH_r + (r)))
282 #define POP_REG(r) (*inst++ = U8(POP_r + (r)))
283 #define RET() (*inst++ = RET_near)
284 #define RET_I16(n) (*inst++ = RET_i16, *inst++ = U8(n), *inst++ = 0)
285
286 /* Multithreading does not affect these static variables, since they store
287 built-in CPU features. Therefore they can be overwritten by different threads
288 if they detect the CPU features in the same time. */
289 #if (defined SLJIT_DETECT_SSE2 && SLJIT_DETECT_SSE2)
290 static sljit_s32 cpu_has_sse2 = -1;
291 #endif
292 static sljit_s32 cpu_has_cmov = -1;
293
294 #ifdef _WIN32_WCE
295 #include <cmnintrin.h>
296 #elif defined(_MSC_VER) && _MSC_VER >= 1400
297 #include <intrin.h>
298 #endif
299
300 /******************************************************/
301 /* Unaligned-store functions */
302 /******************************************************/
303
304 static SLJIT_INLINE void sljit_unaligned_store_s16(void *addr, sljit_s16 value)
305 {
306 SLJIT_MEMCPY(addr, &value, sizeof(value));
307 }
308
309 static SLJIT_INLINE void sljit_unaligned_store_s32(void *addr, sljit_s32 value)
310 {
311 SLJIT_MEMCPY(addr, &value, sizeof(value));
312 }
313
314 static SLJIT_INLINE void sljit_unaligned_store_sw(void *addr, sljit_sw value)
315 {
316 SLJIT_MEMCPY(addr, &value, sizeof(value));
317 }
318
319 /******************************************************/
320 /* Utility functions */
321 /******************************************************/
322
323 static void get_cpu_features(void)
324 {
325 sljit_u32 features;
326
327 #if defined(_MSC_VER) && _MSC_VER >= 1400
328
329 int CPUInfo[4];
330 __cpuid(CPUInfo, 1);
331 features = (sljit_u32)CPUInfo[3];
332
333 #elif defined(__GNUC__) || defined(__INTEL_COMPILER) || defined(__SUNPRO_C)
334
335 /* AT&T syntax. */
336 __asm__ (
337 "movl $0x1, %%eax\n"
338 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
339 /* On x86-32, there is no red zone, so this
340 should work (no need for a local variable). */
341 "push %%ebx\n"
342 #endif
343 "cpuid\n"
344 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
345 "pop %%ebx\n"
346 #endif
347 "movl %%edx, %0\n"
348 : "=g" (features)
349 :
350 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
351 : "%eax", "%ecx", "%edx"
352 #else
353 : "%rax", "%rbx", "%rcx", "%rdx"
354 #endif
355 );
356
357 #else /* _MSC_VER && _MSC_VER >= 1400 */
358
359 /* Intel syntax. */
360 __asm {
361 mov eax, 1
362 cpuid
363 mov features, edx
364 }
365
366 #endif /* _MSC_VER && _MSC_VER >= 1400 */
367
368 #if (defined SLJIT_DETECT_SSE2 && SLJIT_DETECT_SSE2)
369 cpu_has_sse2 = (features >> 26) & 0x1;
370 #endif
371 cpu_has_cmov = (features >> 15) & 0x1;
372 }
373
374 static sljit_u8 get_jump_code(sljit_uw type)
375 {
376 switch (type) {
377 case SLJIT_EQUAL:
378 case SLJIT_F_EQUAL:
379 case SLJIT_UNORDERED_OR_EQUAL:
380 case SLJIT_ORDERED_EQUAL: /* Not supported. */
381 return 0x84 /* je */;
382
383 case SLJIT_NOT_EQUAL:
384 case SLJIT_F_NOT_EQUAL:
385 case SLJIT_ORDERED_NOT_EQUAL:
386 case SLJIT_UNORDERED_OR_NOT_EQUAL: /* Not supported. */
387 return 0x85 /* jne */;
388
389 case SLJIT_LESS:
390 case SLJIT_CARRY:
391 case SLJIT_F_LESS:
392 case SLJIT_UNORDERED_OR_LESS:
393 case SLJIT_UNORDERED_OR_GREATER:
394 return 0x82 /* jc */;
395
396 case SLJIT_GREATER_EQUAL:
397 case SLJIT_NOT_CARRY:
398 case SLJIT_F_GREATER_EQUAL:
399 case SLJIT_ORDERED_GREATER_EQUAL:
400 case SLJIT_ORDERED_LESS_EQUAL:
401 return 0x83 /* jae */;
402
403 case SLJIT_GREATER:
404 case SLJIT_F_GREATER:
405 case SLJIT_ORDERED_LESS:
406 case SLJIT_ORDERED_GREATER:
407 return 0x87 /* jnbe */;
408
409 case SLJIT_LESS_EQUAL:
410 case SLJIT_F_LESS_EQUAL:
411 case SLJIT_UNORDERED_OR_GREATER_EQUAL:
412 case SLJIT_UNORDERED_OR_LESS_EQUAL:
413 return 0x86 /* jbe */;
414
415 case SLJIT_SIG_LESS:
416 return 0x8c /* jl */;
417
418 case SLJIT_SIG_GREATER_EQUAL:
419 return 0x8d /* jnl */;
420
421 case SLJIT_SIG_GREATER:
422 return 0x8f /* jnle */;
423
424 case SLJIT_SIG_LESS_EQUAL:
425 return 0x8e /* jle */;
426
427 case SLJIT_OVERFLOW:
428 return 0x80 /* jo */;
429
430 case SLJIT_NOT_OVERFLOW:
431 return 0x81 /* jno */;
432
433 case SLJIT_UNORDERED:
434 return 0x8a /* jp */;
435
436 case SLJIT_ORDERED:
437 return 0x8b /* jpo */;
438 }
439 return 0;
440 }
441
442 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
443 static sljit_u8* generate_far_jump_code(struct sljit_jump *jump, sljit_u8 *code_ptr, sljit_sw executable_offset);
444 #else
445 static sljit_u8* generate_far_jump_code(struct sljit_jump *jump, sljit_u8 *code_ptr);
446 static sljit_u8* generate_put_label_code(struct sljit_put_label *put_label, sljit_u8 *code_ptr, sljit_uw max_label);
447 #endif
448
449 static sljit_u8* generate_near_jump_code(struct sljit_jump *jump, sljit_u8 *code_ptr, sljit_u8 *code, sljit_sw executable_offset)
450 {
451 sljit_uw type = jump->flags >> TYPE_SHIFT;
452 sljit_s32 short_jump;
453 sljit_uw label_addr;
454
455 if (jump->flags & JUMP_LABEL)
456 label_addr = (sljit_uw)(code + jump->u.label->size);
457 else
458 label_addr = jump->u.target - (sljit_uw)executable_offset;
459
460 short_jump = (sljit_sw)(label_addr - (jump->addr + 2)) >= -128 && (sljit_sw)(label_addr - (jump->addr + 2)) <= 127;
461
462 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
463 if ((sljit_sw)(label_addr - (jump->addr + 1)) > HALFWORD_MAX || (sljit_sw)(label_addr - (jump->addr + 1)) < HALFWORD_MIN)
464 return generate_far_jump_code(jump, code_ptr);
465 #endif
466
467 if (type == SLJIT_JUMP) {
468 if (short_jump)
469 *code_ptr++ = JMP_i8;
470 else
471 *code_ptr++ = JMP_i32;
472 jump->addr++;
473 }
474 else if (type >= SLJIT_FAST_CALL) {
475 short_jump = 0;
476 *code_ptr++ = CALL_i32;
477 jump->addr++;
478 }
479 else if (short_jump) {
480 *code_ptr++ = U8(get_jump_code(type) - 0x10);
481 jump->addr++;
482 }
483 else {
484 *code_ptr++ = GROUP_0F;
485 *code_ptr++ = get_jump_code(type);
486 jump->addr += 2;
487 }
488
489 if (short_jump) {
490 jump->flags |= PATCH_MB;
491 code_ptr += sizeof(sljit_s8);
492 } else {
493 jump->flags |= PATCH_MW;
494 code_ptr += sizeof(sljit_s32);
495 }
496
497 return code_ptr;
498 }
499
500 SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compiler)
501 {
502 struct sljit_memory_fragment *buf;
503 sljit_u8 *code;
504 sljit_u8 *code_ptr;
505 sljit_u8 *buf_ptr;
506 sljit_u8 *buf_end;
507 sljit_u8 len;
508 sljit_sw executable_offset;
509 sljit_uw jump_addr;
510
511 struct sljit_label *label;
512 struct sljit_jump *jump;
513 struct sljit_const *const_;
514 struct sljit_put_label *put_label;
515
516 CHECK_ERROR_PTR();
517 CHECK_PTR(check_sljit_generate_code(compiler));
518 reverse_buf(compiler);
519
520 /* Second code generation pass. */
521 code = (sljit_u8*)SLJIT_MALLOC_EXEC(compiler->size, compiler->exec_allocator_data);
522 PTR_FAIL_WITH_EXEC_IF(code);
523 buf = compiler->buf;
524
525 code_ptr = code;
526 label = compiler->labels;
527 jump = compiler->jumps;
528 const_ = compiler->consts;
529 put_label = compiler->put_labels;
530 executable_offset = SLJIT_EXEC_OFFSET(code);
531
532 do {
533 buf_ptr = buf->memory;
534 buf_end = buf_ptr + buf->used_size;
535 do {
536 len = *buf_ptr++;
537 if (len > 0) {
538 /* The code is already generated. */
539 SLJIT_MEMCPY(code_ptr, buf_ptr, len);
540 code_ptr += len;
541 buf_ptr += len;
542 }
543 else {
544 switch (*buf_ptr) {
545 case 0:
546 label->addr = (sljit_uw)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset);
547 label->size = (sljit_uw)(code_ptr - code);
548 label = label->next;
549 break;
550 case 1:
551 jump->addr = (sljit_uw)code_ptr;
552 if (!(jump->flags & SLJIT_REWRITABLE_JUMP))
553 code_ptr = generate_near_jump_code(jump, code_ptr, code, executable_offset);
554 else {
555 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
556 code_ptr = generate_far_jump_code(jump, code_ptr, executable_offset);
557 #else
558 code_ptr = generate_far_jump_code(jump, code_ptr);
559 #endif
560 }
561 jump = jump->next;
562 break;
563 case 2:
564 const_->addr = ((sljit_uw)code_ptr) - sizeof(sljit_sw);
565 const_ = const_->next;
566 break;
567 default:
568 SLJIT_ASSERT(*buf_ptr == 3);
569 SLJIT_ASSERT(put_label->label);
570 put_label->addr = (sljit_uw)code_ptr;
571 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
572 code_ptr = generate_put_label_code(put_label, code_ptr, (sljit_uw)SLJIT_ADD_EXEC_OFFSET(code, executable_offset) + put_label->label->size);
573 #endif
574 put_label = put_label->next;
575 break;
576 }
577 buf_ptr++;
578 }
579 } while (buf_ptr < buf_end);
580 SLJIT_ASSERT(buf_ptr == buf_end);
581 buf = buf->next;
582 } while (buf);
583
584 SLJIT_ASSERT(!label);
585 SLJIT_ASSERT(!jump);
586 SLJIT_ASSERT(!const_);
587 SLJIT_ASSERT(!put_label);
588 SLJIT_ASSERT(code_ptr <= code + compiler->size);
589
590 jump = compiler->jumps;
591 while (jump) {
592 jump_addr = jump->addr + (sljit_uw)executable_offset;
593
594 if (jump->flags & PATCH_MB) {
595 SLJIT_ASSERT((sljit_sw)(jump->u.label->addr - (jump_addr + sizeof(sljit_s8))) >= -128 && (sljit_sw)(jump->u.label->addr - (jump_addr + sizeof(sljit_s8))) <= 127);
596 *(sljit_u8*)jump->addr = U8(jump->u.label->addr - (jump_addr + sizeof(sljit_s8)));
597 } else if (jump->flags & PATCH_MW) {
598 if (jump->flags & JUMP_LABEL) {
599 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
600 sljit_unaligned_store_sw((void*)jump->addr, (sljit_sw)(jump->u.label->addr - (jump_addr + sizeof(sljit_sw))));
601 #else
602 SLJIT_ASSERT((sljit_sw)(jump->u.label->addr - (jump_addr + sizeof(sljit_s32))) >= HALFWORD_MIN && (sljit_sw)(jump->u.label->addr - (jump_addr + sizeof(sljit_s32))) <= HALFWORD_MAX);
603 sljit_unaligned_store_s32((void*)jump->addr, (sljit_s32)(jump->u.label->addr - (jump_addr + sizeof(sljit_s32))));
604 #endif
605 }
606 else {
607 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
608 sljit_unaligned_store_sw((void*)jump->addr, (sljit_sw)(jump->u.target - (jump_addr + sizeof(sljit_sw))));
609 #else
610 SLJIT_ASSERT((sljit_sw)(jump->u.target - (jump_addr + sizeof(sljit_s32))) >= HALFWORD_MIN && (sljit_sw)(jump->u.target - (jump_addr + sizeof(sljit_s32))) <= HALFWORD_MAX);
611 sljit_unaligned_store_s32((void*)jump->addr, (sljit_s32)(jump->u.target - (jump_addr + sizeof(sljit_s32))));
612 #endif
613 }
614 }
615 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
616 else if (jump->flags & PATCH_MD)
617 sljit_unaligned_store_sw((void*)jump->addr, (sljit_sw)jump->u.label->addr);
618 #endif
619
620 jump = jump->next;
621 }
622
623 put_label = compiler->put_labels;
624 while (put_label) {
625 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
626 sljit_unaligned_store_sw((void*)(put_label->addr - sizeof(sljit_sw)), (sljit_sw)put_label->label->addr);
627 #else
628 if (put_label->flags & PATCH_MD) {
629 SLJIT_ASSERT(put_label->label->addr > HALFWORD_MAX);
630 sljit_unaligned_store_sw((void*)(put_label->addr - sizeof(sljit_sw)), (sljit_sw)put_label->label->addr);
631 }
632 else {
633 SLJIT_ASSERT(put_label->label->addr <= HALFWORD_MAX);
634 sljit_unaligned_store_s32((void*)(put_label->addr - sizeof(sljit_s32)), (sljit_s32)put_label->label->addr);
635 }
636 #endif
637
638 put_label = put_label->next;
639 }
640
641 compiler->error = SLJIT_ERR_COMPILED;
642 compiler->executable_offset = executable_offset;
643 compiler->executable_size = (sljit_uw)(code_ptr - code);
644
645 code = (sljit_u8*)SLJIT_ADD_EXEC_OFFSET(code, executable_offset);
646
647 SLJIT_UPDATE_WX_FLAGS(code, (sljit_u8*)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset), 1);
648 return (void*)code;
649 }
650
651 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_has_cpu_feature(sljit_s32 feature_type)
652 {
653 switch (feature_type) {
654 case SLJIT_HAS_FPU:
655 #ifdef SLJIT_IS_FPU_AVAILABLE
656 return SLJIT_IS_FPU_AVAILABLE;
657 #elif (defined SLJIT_DETECT_SSE2 && SLJIT_DETECT_SSE2)
658 if (cpu_has_sse2 == -1)
659 get_cpu_features();
660 return cpu_has_sse2;
661 #else /* SLJIT_DETECT_SSE2 */
662 return 1;
663 #endif /* SLJIT_DETECT_SSE2 */
664
665 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
666 case SLJIT_HAS_VIRTUAL_REGISTERS:
667 return 1;
668 #endif
669
670 case SLJIT_HAS_CLZ:
671 case SLJIT_HAS_CMOV:
672 if (cpu_has_cmov == -1)
673 get_cpu_features();
674 return cpu_has_cmov;
675
676 case SLJIT_HAS_PREFETCH:
677 return 1;
678
679 case SLJIT_HAS_SSE2:
680 #if (defined SLJIT_DETECT_SSE2 && SLJIT_DETECT_SSE2)
681 if (cpu_has_sse2 == -1)
682 get_cpu_features();
683 return cpu_has_sse2;
684 #else
685 return 1;
686 #endif
687
688 default:
689 return 0;
690 }
691 }
692
693 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_cmp_info(sljit_s32 type)
694 {
695 if (type < SLJIT_UNORDERED || type > SLJIT_ORDERED_LESS_EQUAL)
696 return 0;
697
698 switch (type) {
699 case SLJIT_ORDERED_EQUAL:
700 case SLJIT_UNORDERED_OR_NOT_EQUAL:
701 return 0;
702 }
703
704 return 1;
705 }
706
707 /* --------------------------------------------------------------------- */
708 /* Operators */
709 /* --------------------------------------------------------------------- */
710
711 #define BINARY_OPCODE(opcode) (((opcode ## _EAX_i32) << 24) | ((opcode ## _r_rm) << 16) | ((opcode ## _rm_r) << 8) | (opcode))
712
713 #define BINARY_IMM32(op_imm, immw, arg, argw) \
714 do { \
715 inst = emit_x86_instruction(compiler, 1 | EX86_BIN_INS, SLJIT_IMM, immw, arg, argw); \
716 FAIL_IF(!inst); \
717 *(inst + 1) |= (op_imm); \
718 } while (0)
719
720 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
721
722 #define BINARY_IMM(op_imm, op_mr, immw, arg, argw) \
723 do { \
724 if (IS_HALFWORD(immw) || compiler->mode32) { \
725 BINARY_IMM32(op_imm, immw, arg, argw); \
726 } \
727 else { \
728 FAIL_IF(emit_load_imm64(compiler, (arg == TMP_REG1) ? TMP_REG2 : TMP_REG1, immw)); \
729 inst = emit_x86_instruction(compiler, 1, (arg == TMP_REG1) ? TMP_REG2 : TMP_REG1, 0, arg, argw); \
730 FAIL_IF(!inst); \
731 *inst = (op_mr); \
732 } \
733 } while (0)
734
735 #define BINARY_EAX_IMM(op_eax_imm, immw) \
736 FAIL_IF(emit_do_imm32(compiler, (!compiler->mode32) ? REX_W : 0, (op_eax_imm), immw))
737
738 #else /* !SLJIT_CONFIG_X86_64 */
739
740 #define BINARY_IMM(op_imm, op_mr, immw, arg, argw) \
741 BINARY_IMM32(op_imm, immw, arg, argw)
742
743 #define BINARY_EAX_IMM(op_eax_imm, immw) \
744 FAIL_IF(emit_do_imm(compiler, (op_eax_imm), immw))
745
746 #endif /* SLJIT_CONFIG_X86_64 */
747
748 static sljit_s32 emit_mov(struct sljit_compiler *compiler,
749 sljit_s32 dst, sljit_sw dstw,
750 sljit_s32 src, sljit_sw srcw);
751
752 #define EMIT_MOV(compiler, dst, dstw, src, srcw) \
753 FAIL_IF(emit_mov(compiler, dst, dstw, src, srcw));
754
755 static SLJIT_INLINE sljit_s32 emit_sse2_store(struct sljit_compiler *compiler,
756 sljit_s32 single, sljit_s32 dst, sljit_sw dstw, sljit_s32 src);
757
758 static SLJIT_INLINE sljit_s32 emit_sse2_load(struct sljit_compiler *compiler,
759 sljit_s32 single, sljit_s32 dst, sljit_s32 src, sljit_sw srcw);
760
761 static sljit_s32 emit_cmp_binary(struct sljit_compiler *compiler,
762 sljit_s32 src1, sljit_sw src1w,
763 sljit_s32 src2, sljit_sw src2w);
764
765 static SLJIT_INLINE sljit_s32 emit_endbranch(struct sljit_compiler *compiler)
766 {
767 #if (defined SLJIT_CONFIG_X86_CET && SLJIT_CONFIG_X86_CET)
768 /* Emit endbr32/endbr64 when CET is enabled. */
769 sljit_u8 *inst;
770 inst = (sljit_u8*)ensure_buf(compiler, 1 + 4);
771 FAIL_IF(!inst);
772 INC_SIZE(4);
773 *inst++ = 0xf3;
774 *inst++ = 0x0f;
775 *inst++ = 0x1e;
776 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
777 *inst = 0xfb;
778 #else
779 *inst = 0xfa;
780 #endif
781 #else /* !SLJIT_CONFIG_X86_CET */
782 SLJIT_UNUSED_ARG(compiler);
783 #endif /* SLJIT_CONFIG_X86_CET */
784 return SLJIT_SUCCESS;
785 }
786
787 #if (defined SLJIT_CONFIG_X86_CET && SLJIT_CONFIG_X86_CET) && defined (__SHSTK__)
788
789 static SLJIT_INLINE sljit_s32 emit_rdssp(struct sljit_compiler *compiler, sljit_s32 reg)
790 {
791 sljit_u8 *inst;
792 sljit_s32 size;
793
794 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
795 size = 5;
796 #else
797 size = 4;
798 #endif
799
800 inst = (sljit_u8*)ensure_buf(compiler, 1 + size);
801 FAIL_IF(!inst);
802 INC_SIZE(size);
803 *inst++ = 0xf3;
804 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
805 *inst++ = REX_W | (reg_map[reg] <= 7 ? 0 : REX_B);
806 #endif
807 *inst++ = 0x0f;
808 *inst++ = 0x1e;
809 *inst = (0x3 << 6) | (0x1 << 3) | (reg_map[reg] & 0x7);
810 return SLJIT_SUCCESS;
811 }
812
813 static SLJIT_INLINE sljit_s32 emit_incssp(struct sljit_compiler *compiler, sljit_s32 reg)
814 {
815 sljit_u8 *inst;
816 sljit_s32 size;
817
818 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
819 size = 5;
820 #else
821 size = 4;
822 #endif
823
824 inst = (sljit_u8*)ensure_buf(compiler, 1 + size);
825 FAIL_IF(!inst);
826 INC_SIZE(size);
827 *inst++ = 0xf3;
828 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
829 *inst++ = REX_W | (reg_map[reg] <= 7 ? 0 : REX_B);
830 #endif
831 *inst++ = 0x0f;
832 *inst++ = 0xae;
833 *inst = (0x3 << 6) | (0x5 << 3) | (reg_map[reg] & 0x7);
834 return SLJIT_SUCCESS;
835 }
836
837 #endif /* SLJIT_CONFIG_X86_CET && __SHSTK__ */
838
839 static SLJIT_INLINE sljit_s32 cpu_has_shadow_stack(void)
840 {
841 #if (defined SLJIT_CONFIG_X86_CET && SLJIT_CONFIG_X86_CET) && defined (__SHSTK__)
842 return _get_ssp() != 0;
843 #else /* !SLJIT_CONFIG_X86_CET || !__SHSTK__ */
844 return 0;
845 #endif /* SLJIT_CONFIG_X86_CET && __SHSTK__ */
846 }
847
848 static SLJIT_INLINE sljit_s32 adjust_shadow_stack(struct sljit_compiler *compiler,
849 sljit_s32 src, sljit_sw srcw)
850 {
851 #if (defined SLJIT_CONFIG_X86_CET && SLJIT_CONFIG_X86_CET) && defined (__SHSTK__)
852 sljit_u8 *inst, *jz_after_cmp_inst;
853 sljit_uw size_jz_after_cmp_inst;
854
855 sljit_uw size_before_rdssp_inst = compiler->size;
856
857 /* Generate "RDSSP TMP_REG1". */
858 FAIL_IF(emit_rdssp(compiler, TMP_REG1));
859
860 /* Load return address on shadow stack into TMP_REG1. */
861 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
862 SLJIT_ASSERT(reg_map[TMP_REG1] == 5);
863
864 /* Hand code unsupported "mov 0x0(%ebp),%ebp". */
865 inst = (sljit_u8*)ensure_buf(compiler, 1 + 3);
866 FAIL_IF(!inst);
867 INC_SIZE(3);
868 *inst++ = 0x8b;
869 *inst++ = 0x6d;
870 *inst = 0;
871 #else /* !SLJIT_CONFIG_X86_32 */
872 EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_MEM1(TMP_REG1), 0);
873 #endif /* SLJIT_CONFIG_X86_32 */
874
875 /* Compare return address against TMP_REG1. */
876 FAIL_IF(emit_cmp_binary (compiler, TMP_REG1, 0, src, srcw));
877
878 /* Generate JZ to skip shadow stack ajdustment when shadow
879 stack matches normal stack. */
880 inst = (sljit_u8*)ensure_buf(compiler, 1 + 2);
881 FAIL_IF(!inst);
882 INC_SIZE(2);
883 *inst++ = get_jump_code(SLJIT_EQUAL) - 0x10;
884 size_jz_after_cmp_inst = compiler->size;
885 jz_after_cmp_inst = inst;
886
887 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
888 /* REX_W is not necessary. */
889 compiler->mode32 = 1;
890 #endif
891 /* Load 1 into TMP_REG1. */
892 EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_IMM, 1);
893
894 /* Generate "INCSSP TMP_REG1". */
895 FAIL_IF(emit_incssp(compiler, TMP_REG1));
896
897 /* Jump back to "RDSSP TMP_REG1" to check shadow stack again. */
898 inst = (sljit_u8*)ensure_buf(compiler, 1 + 2);
899 FAIL_IF(!inst);
900 INC_SIZE(2);
901 *inst++ = JMP_i8;
902 *inst = size_before_rdssp_inst - compiler->size;
903
904 *jz_after_cmp_inst = compiler->size - size_jz_after_cmp_inst;
905 #else /* !SLJIT_CONFIG_X86_CET || !__SHSTK__ */
906 SLJIT_UNUSED_ARG(compiler);
907 SLJIT_UNUSED_ARG(src);
908 SLJIT_UNUSED_ARG(srcw);
909 #endif /* SLJIT_CONFIG_X86_CET && __SHSTK__ */
910 return SLJIT_SUCCESS;
911 }
912
913 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
914 #include "sljitNativeX86_32.c"
915 #else
916 #include "sljitNativeX86_64.c"
917 #endif
918
919 static sljit_s32 emit_mov(struct sljit_compiler *compiler,
920 sljit_s32 dst, sljit_sw dstw,
921 sljit_s32 src, sljit_sw srcw)
922 {
923 sljit_u8* inst;
924
925 if (FAST_IS_REG(src)) {
926 inst = emit_x86_instruction(compiler, 1, src, 0, dst, dstw);
927 FAIL_IF(!inst);
928 *inst = MOV_rm_r;
929 return SLJIT_SUCCESS;
930 }
931 if (src & SLJIT_IMM) {
932 if (FAST_IS_REG(dst)) {
933 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
934 return emit_do_imm(compiler, MOV_r_i32 | reg_map[dst], srcw);
935 #else
936 if (!compiler->mode32) {
937 if (NOT_HALFWORD(srcw))
938 return emit_load_imm64(compiler, dst, srcw);
939 }
940 else
941 return emit_do_imm32(compiler, (reg_map[dst] >= 8) ? REX_B : 0, U8(MOV_r_i32 | reg_lmap[dst]), srcw);
942 #endif
943 }
944 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
945 if (!compiler->mode32 && NOT_HALFWORD(srcw)) {
946 /* Immediate to memory move. Only SLJIT_MOV operation copies
947 an immediate directly into memory so TMP_REG1 can be used. */
948 FAIL_IF(emit_load_imm64(compiler, TMP_REG1, srcw));
949 inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, dst, dstw);
950 FAIL_IF(!inst);
951 *inst = MOV_rm_r;
952 return SLJIT_SUCCESS;
953 }
954 #endif
955 inst = emit_x86_instruction(compiler, 1, SLJIT_IMM, srcw, dst, dstw);
956 FAIL_IF(!inst);
957 *inst = MOV_rm_i32;
958 return SLJIT_SUCCESS;
959 }
960 if (FAST_IS_REG(dst)) {
961 inst = emit_x86_instruction(compiler, 1, dst, 0, src, srcw);
962 FAIL_IF(!inst);
963 *inst = MOV_r_rm;
964 return SLJIT_SUCCESS;
965 }
966
967 /* Memory to memory move. Only SLJIT_MOV operation copies
968 data from memory to memory so TMP_REG1 can be used. */
969 inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, src, srcw);
970 FAIL_IF(!inst);
971 *inst = MOV_r_rm;
972 inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, dst, dstw);
973 FAIL_IF(!inst);
974 *inst = MOV_rm_r;
975 return SLJIT_SUCCESS;
976 }
977
978 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op0(struct sljit_compiler *compiler, sljit_s32 op)
979 {
980 sljit_u8 *inst;
981 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
982 sljit_uw size;
983 #endif
984
985 CHECK_ERROR();
986 CHECK(check_sljit_emit_op0(compiler, op));
987
988 switch (GET_OPCODE(op)) {
989 case SLJIT_BREAKPOINT:
990 inst = (sljit_u8*)ensure_buf(compiler, 1 + 1);
991 FAIL_IF(!inst);
992 INC_SIZE(1);
993 *inst = INT3;
994 break;
995 case SLJIT_NOP:
996 inst = (sljit_u8*)ensure_buf(compiler, 1 + 1);
997 FAIL_IF(!inst);
998 INC_SIZE(1);
999 *inst = NOP;
1000 break;
1001 case SLJIT_LMUL_UW:
1002 case SLJIT_LMUL_SW:
1003 case SLJIT_DIVMOD_UW:
1004 case SLJIT_DIVMOD_SW:
1005 case SLJIT_DIV_UW:
1006 case SLJIT_DIV_SW:
1007 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1008 #ifdef _WIN64
1009 SLJIT_ASSERT(
1010 reg_map[SLJIT_R0] == 0
1011 && reg_map[SLJIT_R1] == 2
1012 && reg_map[TMP_REG1] > 7);
1013 #else
1014 SLJIT_ASSERT(
1015 reg_map[SLJIT_R0] == 0
1016 && reg_map[SLJIT_R1] < 7
1017 && reg_map[TMP_REG1] == 2);
1018 #endif
1019 compiler->mode32 = op & SLJIT_32;
1020 #endif
1021 SLJIT_COMPILE_ASSERT((SLJIT_DIVMOD_UW & 0x2) == 0 && SLJIT_DIV_UW - 0x2 == SLJIT_DIVMOD_UW, bad_div_opcode_assignments);
1022
1023 op = GET_OPCODE(op);
1024 if ((op | 0x2) == SLJIT_DIV_UW) {
1025 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32) || defined(_WIN64)
1026 EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_R1, 0);
1027 inst = emit_x86_instruction(compiler, 1, SLJIT_R1, 0, SLJIT_R1, 0);
1028 #else
1029 inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, TMP_REG1, 0);
1030 #endif
1031 FAIL_IF(!inst);
1032 *inst = XOR_r_rm;
1033 }
1034
1035 if ((op | 0x2) == SLJIT_DIV_SW) {
1036 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32) || defined(_WIN64)
1037 EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_R1, 0);
1038 #endif
1039
1040 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1041 inst = (sljit_u8*)ensure_buf(compiler, 1 + 1);
1042 FAIL_IF(!inst);
1043 INC_SIZE(1);
1044 *inst = CDQ;
1045 #else
1046 if (compiler->mode32) {
1047 inst = (sljit_u8*)ensure_buf(compiler, 1 + 1);
1048 FAIL_IF(!inst);
1049 INC_SIZE(1);
1050 *inst = CDQ;
1051 } else {
1052 inst = (sljit_u8*)ensure_buf(compiler, 1 + 2);
1053 FAIL_IF(!inst);
1054 INC_SIZE(2);
1055 *inst++ = REX_W;
1056 *inst = CDQ;
1057 }
1058 #endif
1059 }
1060
1061 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1062 inst = (sljit_u8*)ensure_buf(compiler, 1 + 2);
1063 FAIL_IF(!inst);
1064 INC_SIZE(2);
1065 *inst++ = GROUP_F7;
1066 *inst = MOD_REG | ((op >= SLJIT_DIVMOD_UW) ? reg_map[TMP_REG1] : reg_map[SLJIT_R1]);
1067 #else
1068 #ifdef _WIN64
1069 size = (!compiler->mode32 || op >= SLJIT_DIVMOD_UW) ? 3 : 2;
1070 #else
1071 size = (!compiler->mode32) ? 3 : 2;
1072 #endif
1073 inst = (sljit_u8*)ensure_buf(compiler, 1 + size);
1074 FAIL_IF(!inst);
1075 INC_SIZE(size);
1076 #ifdef _WIN64
1077 if (!compiler->mode32)
1078 *inst++ = REX_W | ((op >= SLJIT_DIVMOD_UW) ? REX_B : 0);
1079 else if (op >= SLJIT_DIVMOD_UW)
1080 *inst++ = REX_B;
1081 *inst++ = GROUP_F7;
1082 *inst = MOD_REG | ((op >= SLJIT_DIVMOD_UW) ? reg_lmap[TMP_REG1] : reg_lmap[SLJIT_R1]);
1083 #else
1084 if (!compiler->mode32)
1085 *inst++ = REX_W;
1086 *inst++ = GROUP_F7;
1087 *inst = MOD_REG | reg_map[SLJIT_R1];
1088 #endif
1089 #endif
1090 switch (op) {
1091 case SLJIT_LMUL_UW:
1092 *inst |= MUL;
1093 break;
1094 case SLJIT_LMUL_SW:
1095 *inst |= IMUL;
1096 break;
1097 case SLJIT_DIVMOD_UW:
1098 case SLJIT_DIV_UW:
1099 *inst |= DIV;
1100 break;
1101 case SLJIT_DIVMOD_SW:
1102 case SLJIT_DIV_SW:
1103 *inst |= IDIV;
1104 break;
1105 }
1106 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64) && !defined(_WIN64)
1107 if (op <= SLJIT_DIVMOD_SW)
1108 EMIT_MOV(compiler, SLJIT_R1, 0, TMP_REG1, 0);
1109 #else
1110 if (op >= SLJIT_DIV_UW)
1111 EMIT_MOV(compiler, SLJIT_R1, 0, TMP_REG1, 0);
1112 #endif
1113 break;
1114 case SLJIT_ENDBR:
1115 return emit_endbranch(compiler);
1116 case SLJIT_SKIP_FRAMES_BEFORE_RETURN:
1117 return skip_frames_before_return(compiler);
1118 }
1119
1120 return SLJIT_SUCCESS;
1121 }
1122
1123 #define ENCODE_PREFIX(prefix) \
1124 do { \
1125 inst = (sljit_u8*)ensure_buf(compiler, 1 + 1); \
1126 FAIL_IF(!inst); \
1127 INC_SIZE(1); \
1128 *inst = U8(prefix); \
1129 } while (0)
1130
1131 static sljit_s32 emit_mov_byte(struct sljit_compiler *compiler, sljit_s32 sign,
1132 sljit_s32 dst, sljit_sw dstw,
1133 sljit_s32 src, sljit_sw srcw)
1134 {
1135 sljit_u8* inst;
1136 sljit_s32 dst_r;
1137 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1138 sljit_s32 work_r;
1139 #endif
1140
1141 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1142 compiler->mode32 = 0;
1143 #endif
1144
1145 if (src & SLJIT_IMM) {
1146 if (FAST_IS_REG(dst)) {
1147 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1148 return emit_do_imm(compiler, MOV_r_i32 | reg_map[dst], srcw);
1149 #else
1150 inst = emit_x86_instruction(compiler, 1, SLJIT_IMM, srcw, dst, 0);
1151 FAIL_IF(!inst);
1152 *inst = MOV_rm_i32;
1153 return SLJIT_SUCCESS;
1154 #endif
1155 }
1156 inst = emit_x86_instruction(compiler, 1 | EX86_BYTE_ARG | EX86_NO_REXW, SLJIT_IMM, srcw, dst, dstw);
1157 FAIL_IF(!inst);
1158 *inst = MOV_rm8_i8;
1159 return SLJIT_SUCCESS;
1160 }
1161
1162 dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
1163
1164 if ((dst & SLJIT_MEM) && FAST_IS_REG(src)) {
1165 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1166 if (reg_map[src] >= 4) {
1167 SLJIT_ASSERT(dst_r == TMP_REG1);
1168 EMIT_MOV(compiler, TMP_REG1, 0, src, 0);
1169 } else
1170 dst_r = src;
1171 #else
1172 dst_r = src;
1173 #endif
1174 }
1175 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1176 else if (FAST_IS_REG(src) && reg_map[src] >= 4) {
1177 /* src, dst are registers. */
1178 SLJIT_ASSERT(FAST_IS_REG(dst));
1179 if (reg_map[dst] < 4) {
1180 if (dst != src)
1181 EMIT_MOV(compiler, dst, 0, src, 0);
1182 inst = emit_x86_instruction(compiler, 2, dst, 0, dst, 0);
1183 FAIL_IF(!inst);
1184 *inst++ = GROUP_0F;
1185 *inst = sign ? MOVSX_r_rm8 : MOVZX_r_rm8;
1186 }
1187 else {
1188 if (dst != src)
1189 EMIT_MOV(compiler, dst, 0, src, 0);
1190 if (sign) {
1191 /* shl reg, 24 */
1192 inst = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, SLJIT_IMM, 24, dst, 0);
1193 FAIL_IF(!inst);
1194 *inst |= SHL;
1195 /* sar reg, 24 */
1196 inst = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, SLJIT_IMM, 24, dst, 0);
1197 FAIL_IF(!inst);
1198 *inst |= SAR;
1199 }
1200 else {
1201 inst = emit_x86_instruction(compiler, 1 | EX86_BIN_INS, SLJIT_IMM, 0xff, dst, 0);
1202 FAIL_IF(!inst);
1203 *(inst + 1) |= AND;
1204 }
1205 }
1206 return SLJIT_SUCCESS;
1207 }
1208 #endif
1209 else {
1210 /* src can be memory addr or reg_map[src] < 4 on x86_32 architectures. */
1211 inst = emit_x86_instruction(compiler, 2, dst_r, 0, src, srcw);
1212 FAIL_IF(!inst);
1213 *inst++ = GROUP_0F;
1214 *inst = sign ? MOVSX_r_rm8 : MOVZX_r_rm8;
1215 }
1216
1217 if (dst & SLJIT_MEM) {
1218 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1219 if (dst_r == TMP_REG1) {
1220 /* Find a non-used register, whose reg_map[src] < 4. */
1221 if ((dst & REG_MASK) == SLJIT_R0) {
1222 if ((dst & OFFS_REG_MASK) == TO_OFFS_REG(SLJIT_R1))
1223 work_r = SLJIT_R2;
1224 else
1225 work_r = SLJIT_R1;
1226 }
1227 else {
1228 if ((dst & OFFS_REG_MASK) != TO_OFFS_REG(SLJIT_R0))
1229 work_r = SLJIT_R0;
1230 else if ((dst & REG_MASK) == SLJIT_R1)
1231 work_r = SLJIT_R2;
1232 else
1233 work_r = SLJIT_R1;
1234 }
1235
1236 if (work_r == SLJIT_R0) {
1237 ENCODE_PREFIX(XCHG_EAX_r | reg_map[TMP_REG1]);
1238 }
1239 else {
1240 inst = emit_x86_instruction(compiler, 1, work_r, 0, dst_r, 0);
1241 FAIL_IF(!inst);
1242 *inst = XCHG_r_rm;
1243 }
1244
1245 inst = emit_x86_instruction(compiler, 1, work_r, 0, dst, dstw);
1246 FAIL_IF(!inst);
1247 *inst = MOV_rm8_r8;
1248
1249 if (work_r == SLJIT_R0) {
1250 ENCODE_PREFIX(XCHG_EAX_r | reg_map[TMP_REG1]);
1251 }
1252 else {
1253 inst = emit_x86_instruction(compiler, 1, work_r, 0, dst_r, 0);
1254 FAIL_IF(!inst);
1255 *inst = XCHG_r_rm;
1256 }
1257 }
1258 else {
1259 inst = emit_x86_instruction(compiler, 1, dst_r, 0, dst, dstw);
1260 FAIL_IF(!inst);
1261 *inst = MOV_rm8_r8;
1262 }
1263 #else
1264 inst = emit_x86_instruction(compiler, 1 | EX86_REX | EX86_NO_REXW, dst_r, 0, dst, dstw);
1265 FAIL_IF(!inst);
1266 *inst = MOV_rm8_r8;
1267 #endif
1268 }
1269
1270 return SLJIT_SUCCESS;
1271 }
1272
1273 static sljit_s32 emit_prefetch(struct sljit_compiler *compiler, sljit_s32 op,
1274 sljit_s32 src, sljit_sw srcw)
1275 {
1276 sljit_u8* inst;
1277
1278 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1279 compiler->mode32 = 1;
1280 #endif
1281
1282 inst = emit_x86_instruction(compiler, 2, 0, 0, src, srcw);
1283 FAIL_IF(!inst);
1284 *inst++ = GROUP_0F;
1285 *inst++ = PREFETCH;
1286
1287 if (op == SLJIT_PREFETCH_L1)
1288 *inst |= (1 << 3);
1289 else if (op == SLJIT_PREFETCH_L2)
1290 *inst |= (2 << 3);
1291 else if (op == SLJIT_PREFETCH_L3)
1292 *inst |= (3 << 3);
1293
1294 return SLJIT_SUCCESS;
1295 }
1296
1297 static sljit_s32 emit_mov_half(struct sljit_compiler *compiler, sljit_s32 sign,
1298 sljit_s32 dst, sljit_sw dstw,
1299 sljit_s32 src, sljit_sw srcw)
1300 {
1301 sljit_u8* inst;
1302 sljit_s32 dst_r;
1303
1304 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1305 compiler->mode32 = 0;
1306 #endif
1307
1308 if (src & SLJIT_IMM) {
1309 if (FAST_IS_REG(dst)) {
1310 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1311 return emit_do_imm(compiler, MOV_r_i32 | reg_map[dst], srcw);
1312 #else
1313 inst = emit_x86_instruction(compiler, 1, SLJIT_IMM, srcw, dst, 0);
1314 FAIL_IF(!inst);
1315 *inst = MOV_rm_i32;
1316 return SLJIT_SUCCESS;
1317 #endif
1318 }
1319 inst = emit_x86_instruction(compiler, 1 | EX86_HALF_ARG | EX86_NO_REXW | EX86_PREF_66, SLJIT_IMM, srcw, dst, dstw);
1320 FAIL_IF(!inst);
1321 *inst = MOV_rm_i32;
1322 return SLJIT_SUCCESS;
1323 }
1324
1325 dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
1326
1327 if ((dst & SLJIT_MEM) && FAST_IS_REG(src))
1328 dst_r = src;
1329 else {
1330 inst = emit_x86_instruction(compiler, 2, dst_r, 0, src, srcw);
1331 FAIL_IF(!inst);
1332 *inst++ = GROUP_0F;
1333 *inst = sign ? MOVSX_r_rm16 : MOVZX_r_rm16;
1334 }
1335
1336 if (dst & SLJIT_MEM) {
1337 inst = emit_x86_instruction(compiler, 1 | EX86_NO_REXW | EX86_PREF_66, dst_r, 0, dst, dstw);
1338 FAIL_IF(!inst);
1339 *inst = MOV_rm_r;
1340 }
1341
1342 return SLJIT_SUCCESS;
1343 }
1344
1345 static sljit_s32 emit_unary(struct sljit_compiler *compiler, sljit_u8 opcode,
1346 sljit_s32 dst, sljit_sw dstw,
1347 sljit_s32 src, sljit_sw srcw)
1348 {
1349 sljit_u8* inst;
1350
1351 if (dst == src && dstw == srcw) {
1352 /* Same input and output */
1353 inst = emit_x86_instruction(compiler, 1, 0, 0, dst, dstw);
1354 FAIL_IF(!inst);
1355 *inst++ = GROUP_F7;
1356 *inst |= opcode;
1357 return SLJIT_SUCCESS;
1358 }
1359
1360 if (FAST_IS_REG(dst)) {
1361 EMIT_MOV(compiler, dst, 0, src, srcw);
1362 inst = emit_x86_instruction(compiler, 1, 0, 0, dst, 0);
1363 FAIL_IF(!inst);
1364 *inst++ = GROUP_F7;
1365 *inst |= opcode;
1366 return SLJIT_SUCCESS;
1367 }
1368
1369 EMIT_MOV(compiler, TMP_REG1, 0, src, srcw);
1370 inst = emit_x86_instruction(compiler, 1, 0, 0, TMP_REG1, 0);
1371 FAIL_IF(!inst);
1372 *inst++ = GROUP_F7;
1373 *inst |= opcode;
1374 EMIT_MOV(compiler, dst, dstw, TMP_REG1, 0);
1375 return SLJIT_SUCCESS;
1376 }
1377
1378 static sljit_s32 emit_not_with_flags(struct sljit_compiler *compiler,
1379 sljit_s32 dst, sljit_sw dstw,
1380 sljit_s32 src, sljit_sw srcw)
1381 {
1382 sljit_u8* inst;
1383
1384 if (FAST_IS_REG(dst)) {
1385 EMIT_MOV(compiler, dst, 0, src, srcw);
1386 inst = emit_x86_instruction(compiler, 1, 0, 0, dst, 0);
1387 FAIL_IF(!inst);
1388 *inst++ = GROUP_F7;
1389 *inst |= NOT_rm;
1390 inst = emit_x86_instruction(compiler, 1, dst, 0, dst, 0);
1391 FAIL_IF(!inst);
1392 *inst = OR_r_rm;
1393 return SLJIT_SUCCESS;
1394 }
1395
1396 EMIT_MOV(compiler, TMP_REG1, 0, src, srcw);
1397 inst = emit_x86_instruction(compiler, 1, 0, 0, TMP_REG1, 0);
1398 FAIL_IF(!inst);
1399 *inst++ = GROUP_F7;
1400 *inst |= NOT_rm;
1401 inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, TMP_REG1, 0);
1402 FAIL_IF(!inst);
1403 *inst = OR_r_rm;
1404 EMIT_MOV(compiler, dst, dstw, TMP_REG1, 0);
1405 return SLJIT_SUCCESS;
1406 }
1407
1408 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1409 static const sljit_sw emit_clz_arg = 32 + 31;
1410 #endif
1411
1412 static sljit_s32 emit_clz(struct sljit_compiler *compiler, sljit_s32 op_flags,
1413 sljit_s32 dst, sljit_sw dstw,
1414 sljit_s32 src, sljit_sw srcw)
1415 {
1416 sljit_u8* inst;
1417 sljit_s32 dst_r;
1418
1419 SLJIT_UNUSED_ARG(op_flags);
1420
1421 if (cpu_has_cmov == -1)
1422 get_cpu_features();
1423
1424 dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
1425
1426 inst = emit_x86_instruction(compiler, 2, dst_r, 0, src, srcw);
1427 FAIL_IF(!inst);
1428 *inst++ = GROUP_0F;
1429 *inst = BSR_r_rm;
1430
1431 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1432 if (cpu_has_cmov) {
1433 if (dst_r != TMP_REG1) {
1434 EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_IMM, 32 + 31);
1435 inst = emit_x86_instruction(compiler, 2, dst_r, 0, TMP_REG1, 0);
1436 }
1437 else
1438 inst = emit_x86_instruction(compiler, 2, dst_r, 0, SLJIT_MEM0(), (sljit_sw)&emit_clz_arg);
1439
1440 FAIL_IF(!inst);
1441 *inst++ = GROUP_0F;
1442 *inst = CMOVE_r_rm;
1443 }
1444 else
1445 FAIL_IF(sljit_emit_cmov_generic(compiler, SLJIT_EQUAL, dst_r, SLJIT_IMM, 32 + 31));
1446
1447 inst = emit_x86_instruction(compiler, 1 | EX86_BIN_INS, SLJIT_IMM, 31, dst_r, 0);
1448 #else
1449 if (cpu_has_cmov) {
1450 EMIT_MOV(compiler, TMP_REG2, 0, SLJIT_IMM, !(op_flags & SLJIT_32) ? (64 + 63) : (32 + 31));
1451
1452 inst = emit_x86_instruction(compiler, 2, dst_r, 0, TMP_REG2, 0);
1453 FAIL_IF(!inst);
1454 *inst++ = GROUP_0F;
1455 *inst = CMOVE_r_rm;
1456 }
1457 else
1458 FAIL_IF(sljit_emit_cmov_generic(compiler, SLJIT_EQUAL, dst_r, SLJIT_IMM, !(op_flags & SLJIT_32) ? (64 + 63) : (32 + 31)));
1459
1460 inst = emit_x86_instruction(compiler, 1 | EX86_BIN_INS, SLJIT_IMM, !(op_flags & SLJIT_32) ? 63 : 31, dst_r, 0);
1461 #endif
1462
1463 FAIL_IF(!inst);
1464 *(inst + 1) |= XOR;
1465
1466 if (dst & SLJIT_MEM)
1467 EMIT_MOV(compiler, dst, dstw, TMP_REG1, 0);
1468 return SLJIT_SUCCESS;
1469 }
1470
1471 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op1(struct sljit_compiler *compiler, sljit_s32 op,
1472 sljit_s32 dst, sljit_sw dstw,
1473 sljit_s32 src, sljit_sw srcw)
1474 {
1475 sljit_s32 op_flags = GET_ALL_FLAGS(op);
1476 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1477 sljit_s32 dst_is_ereg = 0;
1478 #endif
1479
1480 CHECK_ERROR();
1481 CHECK(check_sljit_emit_op1(compiler, op, dst, dstw, src, srcw));
1482 ADJUST_LOCAL_OFFSET(dst, dstw);
1483 ADJUST_LOCAL_OFFSET(src, srcw);
1484
1485 CHECK_EXTRA_REGS(dst, dstw, dst_is_ereg = 1);
1486 CHECK_EXTRA_REGS(src, srcw, (void)0);
1487 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1488 compiler->mode32 = op_flags & SLJIT_32;
1489 #endif
1490
1491 op = GET_OPCODE(op);
1492
1493 if (op >= SLJIT_MOV && op <= SLJIT_MOV_P) {
1494 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1495 compiler->mode32 = 0;
1496 #endif
1497
1498 if (FAST_IS_REG(src) && src == dst) {
1499 if (!TYPE_CAST_NEEDED(op))
1500 return SLJIT_SUCCESS;
1501 }
1502
1503 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1504 if (op_flags & SLJIT_32) {
1505 if (src & SLJIT_MEM) {
1506 if (op == SLJIT_MOV_S32)
1507 op = SLJIT_MOV_U32;
1508 }
1509 else if (src & SLJIT_IMM) {
1510 if (op == SLJIT_MOV_U32)
1511 op = SLJIT_MOV_S32;
1512 }
1513 }
1514 #endif
1515
1516 if (src & SLJIT_IMM) {
1517 switch (op) {
1518 case SLJIT_MOV_U8:
1519 srcw = (sljit_u8)srcw;
1520 break;
1521 case SLJIT_MOV_S8:
1522 srcw = (sljit_s8)srcw;
1523 break;
1524 case SLJIT_MOV_U16:
1525 srcw = (sljit_u16)srcw;
1526 break;
1527 case SLJIT_MOV_S16:
1528 srcw = (sljit_s16)srcw;
1529 break;
1530 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1531 case SLJIT_MOV_U32:
1532 srcw = (sljit_u32)srcw;
1533 break;
1534 case SLJIT_MOV_S32:
1535 srcw = (sljit_s32)srcw;
1536 break;
1537 #endif
1538 }
1539 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1540 if (SLJIT_UNLIKELY(dst_is_ereg))
1541 return emit_mov(compiler, dst, dstw, src, srcw);
1542 #endif
1543 }
1544
1545 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1546 if (SLJIT_UNLIKELY(dst_is_ereg) && (!(op == SLJIT_MOV || op == SLJIT_MOV_U32 || op == SLJIT_MOV_S32 || op == SLJIT_MOV_P) || (src & SLJIT_MEM))) {
1547 SLJIT_ASSERT(dst == SLJIT_MEM1(SLJIT_SP));
1548 dst = TMP_REG1;
1549 }
1550 #endif
1551
1552 switch (op) {
1553 case SLJIT_MOV:
1554 case SLJIT_MOV_P:
1555 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1556 case SLJIT_MOV_U32:
1557 case SLJIT_MOV_S32:
1558 case SLJIT_MOV32:
1559 #endif
1560 EMIT_MOV(compiler, dst, dstw, src, srcw);
1561 break;
1562 case SLJIT_MOV_U8:
1563 FAIL_IF(emit_mov_byte(compiler, 0, dst, dstw, src, srcw));
1564 break;
1565 case SLJIT_MOV_S8:
1566 FAIL_IF(emit_mov_byte(compiler, 1, dst, dstw, src, srcw));
1567 break;
1568 case SLJIT_MOV_U16:
1569 FAIL_IF(emit_mov_half(compiler, 0, dst, dstw, src, srcw));
1570 break;
1571 case SLJIT_MOV_S16:
1572 FAIL_IF(emit_mov_half(compiler, 1, dst, dstw, src, srcw));
1573 break;
1574 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1575 case SLJIT_MOV_U32:
1576 FAIL_IF(emit_mov_int(compiler, 0, dst, dstw, src, srcw));
1577 break;
1578 case SLJIT_MOV_S32:
1579 FAIL_IF(emit_mov_int(compiler, 1, dst, dstw, src, srcw));
1580 break;
1581 case SLJIT_MOV32:
1582 compiler->mode32 = 1;
1583 EMIT_MOV(compiler, dst, dstw, src, srcw);
1584 compiler->mode32 = 0;
1585 break;
1586 #endif
1587 }
1588
1589 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1590 if (SLJIT_UNLIKELY(dst_is_ereg) && dst == TMP_REG1)
1591 return emit_mov(compiler, SLJIT_MEM1(SLJIT_SP), dstw, TMP_REG1, 0);
1592 #endif
1593 return SLJIT_SUCCESS;
1594 }
1595
1596 switch (op) {
1597 case SLJIT_NOT:
1598 if (SLJIT_UNLIKELY(op_flags & SLJIT_SET_Z))
1599 return emit_not_with_flags(compiler, dst, dstw, src, srcw);
1600 return emit_unary(compiler, NOT_rm, dst, dstw, src, srcw);
1601
1602 case SLJIT_CLZ:
1603 return emit_clz(compiler, op_flags, dst, dstw, src, srcw);
1604 }
1605
1606 return SLJIT_SUCCESS;
1607 }
1608
1609 static sljit_s32 emit_cum_binary(struct sljit_compiler *compiler,
1610 sljit_u32 op_types,
1611 sljit_s32 dst, sljit_sw dstw,
1612 sljit_s32 src1, sljit_sw src1w,
1613 sljit_s32 src2, sljit_sw src2w)
1614 {
1615 sljit_u8* inst;
1616 sljit_u8 op_eax_imm = U8(op_types >> 24);
1617 sljit_u8 op_rm = U8((op_types >> 16) & 0xff);
1618 sljit_u8 op_mr = U8((op_types >> 8) & 0xff);
1619 sljit_u8 op_imm = U8(op_types & 0xff);
1620
1621 if (dst == src1 && dstw == src1w) {
1622 if (src2 & SLJIT_IMM) {
1623 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1624 if ((dst == SLJIT_R0) && (src2w > 127 || src2w < -128) && (compiler->mode32 || IS_HALFWORD(src2w))) {
1625 #else
1626 if ((dst == SLJIT_R0) && (src2w > 127 || src2w < -128)) {
1627 #endif
1628 BINARY_EAX_IMM(op_eax_imm, src2w);
1629 }
1630 else {
1631 BINARY_IMM(op_imm, op_mr, src2w, dst, dstw);
1632 }
1633 }
1634 else if (FAST_IS_REG(dst)) {
1635 inst = emit_x86_instruction(compiler, 1, dst, dstw, src2, src2w);
1636 FAIL_IF(!inst);
1637 *inst = op_rm;
1638 }
1639 else if (FAST_IS_REG(src2)) {
1640 /* Special exception for sljit_emit_op_flags. */
1641 inst = emit_x86_instruction(compiler, 1, src2, src2w, dst, dstw);
1642 FAIL_IF(!inst);
1643 *inst = op_mr;
1644 }
1645 else {
1646 EMIT_MOV(compiler, TMP_REG1, 0, src2, src2w);
1647 inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, dst, dstw);
1648 FAIL_IF(!inst);
1649 *inst = op_mr;
1650 }
1651 return SLJIT_SUCCESS;
1652 }
1653
1654 /* Only for cumulative operations. */
1655 if (dst == src2 && dstw == src2w) {
1656 if (src1 & SLJIT_IMM) {
1657 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1658 if ((dst == SLJIT_R0) && (src1w > 127 || src1w < -128) && (compiler->mode32 || IS_HALFWORD(src1w))) {
1659 #else
1660 if ((dst == SLJIT_R0) && (src1w > 127 || src1w < -128)) {
1661 #endif
1662 BINARY_EAX_IMM(op_eax_imm, src1w);
1663 }
1664 else {
1665 BINARY_IMM(op_imm, op_mr, src1w, dst, dstw);
1666 }
1667 }
1668 else if (FAST_IS_REG(dst)) {
1669 inst = emit_x86_instruction(compiler, 1, dst, dstw, src1, src1w);
1670 FAIL_IF(!inst);
1671 *inst = op_rm;
1672 }
1673 else if (FAST_IS_REG(src1)) {
1674 inst = emit_x86_instruction(compiler, 1, src1, src1w, dst, dstw);
1675 FAIL_IF(!inst);
1676 *inst = op_mr;
1677 }
1678 else {
1679 EMIT_MOV(compiler, TMP_REG1, 0, src1, src1w);
1680 inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, dst, dstw);
1681 FAIL_IF(!inst);
1682 *inst = op_mr;
1683 }
1684 return SLJIT_SUCCESS;
1685 }
1686
1687 /* General version. */
1688 if (FAST_IS_REG(dst)) {
1689 EMIT_MOV(compiler, dst, 0, src1, src1w);
1690 if (src2 & SLJIT_IMM) {
1691 BINARY_IMM(op_imm, op_mr, src2w, dst, 0);
1692 }
1693 else {
1694 inst = emit_x86_instruction(compiler, 1, dst, 0, src2, src2w);
1695 FAIL_IF(!inst);
1696 *inst = op_rm;
1697 }
1698 }
1699 else {
1700 /* This version requires less memory writing. */
1701 EMIT_MOV(compiler, TMP_REG1, 0, src1, src1w);
1702 if (src2 & SLJIT_IMM) {
1703 BINARY_IMM(op_imm, op_mr, src2w, TMP_REG1, 0);
1704 }
1705 else {
1706 inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, src2, src2w);
1707 FAIL_IF(!inst);
1708 *inst = op_rm;
1709 }
1710 EMIT_MOV(compiler, dst, dstw, TMP_REG1, 0);
1711 }
1712
1713 return SLJIT_SUCCESS;
1714 }
1715
1716 static sljit_s32 emit_non_cum_binary(struct sljit_compiler *compiler,
1717 sljit_u32 op_types,
1718 sljit_s32 dst, sljit_sw dstw,
1719 sljit_s32 src1, sljit_sw src1w,
1720 sljit_s32 src2, sljit_sw src2w)
1721 {
1722 sljit_u8* inst;
1723 sljit_u8 op_eax_imm = U8(op_types >> 24);
1724 sljit_u8 op_rm = U8((op_types >> 16) & 0xff);
1725 sljit_u8 op_mr = U8((op_types >> 8) & 0xff);
1726 sljit_u8 op_imm = U8(op_types & 0xff);
1727
1728 if (dst == src1 && dstw == src1w) {
1729 if (src2 & SLJIT_IMM) {
1730 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1731 if ((dst == SLJIT_R0) && (src2w > 127 || src2w < -128) && (compiler->mode32 || IS_HALFWORD(src2w))) {
1732 #else
1733 if ((dst == SLJIT_R0) && (src2w > 127 || src2w < -128)) {
1734 #endif
1735 BINARY_EAX_IMM(op_eax_imm, src2w);
1736 }
1737 else {
1738 BINARY_IMM(op_imm, op_mr, src2w, dst, dstw);
1739 }
1740 }
1741 else if (FAST_IS_REG(dst)) {
1742 inst = emit_x86_instruction(compiler, 1, dst, dstw, src2, src2w);
1743 FAIL_IF(!inst);
1744 *inst = op_rm;
1745 }
1746 else if (FAST_IS_REG(src2)) {
1747 inst = emit_x86_instruction(compiler, 1, src2, src2w, dst, dstw);
1748 FAIL_IF(!inst);
1749 *inst = op_mr;
1750 }
1751 else {
1752 EMIT_MOV(compiler, TMP_REG1, 0, src2, src2w);
1753 inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, dst, dstw);
1754 FAIL_IF(!inst);
1755 *inst = op_mr;
1756 }
1757 return SLJIT_SUCCESS;
1758 }
1759
1760 /* General version. */
1761 if (FAST_IS_REG(dst) && dst != src2) {
1762 EMIT_MOV(compiler, dst, 0, src1, src1w);
1763 if (src2 & SLJIT_IMM) {
1764 BINARY_IMM(op_imm, op_mr, src2w, dst, 0);
1765 }
1766 else {
1767 inst = emit_x86_instruction(compiler, 1, dst, 0, src2, src2w);
1768 FAIL_IF(!inst);
1769 *inst = op_rm;
1770 }
1771 }
1772 else {
1773 /* This version requires less memory writing. */
1774 EMIT_MOV(compiler, TMP_REG1, 0, src1, src1w);
1775 if (src2 & SLJIT_IMM) {
1776 BINARY_IMM(op_imm, op_mr, src2w, TMP_REG1, 0);
1777 }
1778 else {
1779 inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, src2, src2w);
1780 FAIL_IF(!inst);
1781 *inst = op_rm;
1782 }
1783 EMIT_MOV(compiler, dst, dstw, TMP_REG1, 0);
1784 }
1785
1786 return SLJIT_SUCCESS;
1787 }
1788
1789 static sljit_s32 emit_mul(struct sljit_compiler *compiler,
1790 sljit_s32 dst, sljit_sw dstw,
1791 sljit_s32 src1, sljit_sw src1w,
1792 sljit_s32 src2, sljit_sw src2w)
1793 {
1794 sljit_u8* inst;
1795 sljit_s32 dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
1796
1797 /* Register destination. */
1798 if (dst_r == src1 && !(src2 & SLJIT_IMM)) {
1799 inst = emit_x86_instruction(compiler, 2, dst_r, 0, src2, src2w);
1800 FAIL_IF(!inst);
1801 *inst++ = GROUP_0F;
1802 *inst = IMUL_r_rm;
1803 }
1804 else if (dst_r == src2 && !(src1 & SLJIT_IMM)) {
1805 inst = emit_x86_instruction(compiler, 2, dst_r, 0, src1, src1w);
1806 FAIL_IF(!inst);
1807 *inst++ = GROUP_0F;
1808 *inst = IMUL_r_rm;
1809 }
1810 else if (src1 & SLJIT_IMM) {
1811 if (src2 & SLJIT_IMM) {
1812 EMIT_MOV(compiler, dst_r, 0, SLJIT_IMM, src2w);
1813 src2 = dst_r;
1814 src2w = 0;
1815 }
1816
1817 if (src1w <= 127 && src1w >= -128) {
1818 inst = emit_x86_instruction(compiler, 1, dst_r, 0, src2, src2w);
1819 FAIL_IF(!inst);
1820 *inst = IMUL_r_rm_i8;
1821 inst = (sljit_u8*)ensure_buf(compiler, 1 + 1);
1822 FAIL_IF(!inst);
1823 INC_SIZE(1);
1824 *inst = U8(src1w);
1825 }
1826 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1827 else {
1828 inst = emit_x86_instruction(compiler, 1, dst_r, 0, src2, src2w);
1829 FAIL_IF(!inst);
1830 *inst = IMUL_r_rm_i32;
1831 inst = (sljit_u8*)ensure_buf(compiler, 1 + 4);
1832 FAIL_IF(!inst);
1833 INC_SIZE(4);
1834 sljit_unaligned_store_sw(inst, src1w);
1835 }
1836 #else
1837 else if (IS_HALFWORD(src1w)) {
1838 inst = emit_x86_instruction(compiler, 1, dst_r, 0, src2, src2w);
1839 FAIL_IF(!inst);
1840 *inst = IMUL_r_rm_i32;
1841 inst = (sljit_u8*)ensure_buf(compiler, 1 + 4);
1842 FAIL_IF(!inst);
1843 INC_SIZE(4);
1844 sljit_unaligned_store_s32(inst, (sljit_s32)src1w);
1845 }
1846 else {
1847 if (dst_r != src2)
1848 EMIT_MOV(compiler, dst_r, 0, src2, src2w);
1849 FAIL_IF(emit_load_imm64(compiler, TMP_REG2, src1w));
1850 inst = emit_x86_instruction(compiler, 2, dst_r, 0, TMP_REG2, 0);
1851 FAIL_IF(!inst);
1852 *inst++ = GROUP_0F;
1853 *inst = IMUL_r_rm;
1854 }
1855 #endif
1856 }
1857 else if (src2 & SLJIT_IMM) {
1858 /* Note: src1 is NOT immediate. */
1859
1860 if (src2w <= 127 && src2w >= -128) {
1861 inst = emit_x86_instruction(compiler, 1, dst_r, 0, src1, src1w);
1862 FAIL_IF(!inst);
1863 *inst = IMUL_r_rm_i8;
1864 inst = (sljit_u8*)ensure_buf(compiler, 1 + 1);
1865 FAIL_IF(!inst);
1866 INC_SIZE(1);
1867 *inst = U8(src2w);
1868 }
1869 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1870 else {
1871 inst = emit_x86_instruction(compiler, 1, dst_r, 0, src1, src1w);
1872 FAIL_IF(!inst);
1873 *inst = IMUL_r_rm_i32;
1874 inst = (sljit_u8*)ensure_buf(compiler, 1 + 4);
1875 FAIL_IF(!inst);
1876 INC_SIZE(4);
1877 sljit_unaligned_store_sw(inst, src2w);
1878 }
1879 #else
1880 else if (IS_HALFWORD(src2w)) {
1881 inst = emit_x86_instruction(compiler, 1, dst_r, 0, src1, src1w);
1882 FAIL_IF(!inst);
1883 *inst = IMUL_r_rm_i32;
1884 inst = (sljit_u8*)ensure_buf(compiler, 1 + 4);
1885 FAIL_IF(!inst);
1886 INC_SIZE(4);
1887 sljit_unaligned_store_s32(inst, (sljit_s32)src2w);
1888 }
1889 else {
1890 if (dst_r != src1)
1891 EMIT_MOV(compiler, dst_r, 0, src1, src1w);
1892 FAIL_IF(emit_load_imm64(compiler, TMP_REG2, src2w));
1893 inst = emit_x86_instruction(compiler, 2, dst_r, 0, TMP_REG2, 0);
1894 FAIL_IF(!inst);
1895 *inst++ = GROUP_0F;
1896 *inst = IMUL_r_rm;
1897 }
1898 #endif
1899 }
1900 else {
1901 /* Neither argument is immediate. */
1902 if (ADDRESSING_DEPENDS_ON(src2, dst_r))
1903 dst_r = TMP_REG1;
1904 EMIT_MOV(compiler, dst_r, 0, src1, src1w);
1905 inst = emit_x86_instruction(compiler, 2, dst_r, 0, src2, src2w);
1906 FAIL_IF(!inst);
1907 *inst++ = GROUP_0F;
1908 *inst = IMUL_r_rm;
1909 }
1910
1911 if (dst & SLJIT_MEM)
1912 EMIT_MOV(compiler, dst, dstw, TMP_REG1, 0);
1913
1914 return SLJIT_SUCCESS;
1915 }
1916
1917 static sljit_s32 emit_lea_binary(struct sljit_compiler *compiler,
1918 sljit_s32 dst, sljit_sw dstw,
1919 sljit_s32 src1, sljit_sw src1w,
1920 sljit_s32 src2, sljit_sw src2w)
1921 {
1922 sljit_u8* inst;
1923 sljit_s32 dst_r, done = 0;
1924
1925 /* These cases better be left to handled by normal way. */
1926 if (dst == src1 && dstw == src1w)
1927 return SLJIT_ERR_UNSUPPORTED;
1928 if (dst == src2 && dstw == src2w)
1929 return SLJIT_ERR_UNSUPPORTED;
1930
1931 dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
1932
1933 if (FAST_IS_REG(src1)) {
1934 if (FAST_IS_REG(src2)) {
1935 inst = emit_x86_instruction(compiler, 1, dst_r, 0, SLJIT_MEM2(src1, src2), 0);
1936 FAIL_IF(!inst);
1937 *inst = LEA_r_m;
1938 done = 1;
1939 }
1940 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1941 if ((src2 & SLJIT_IMM) && (compiler->mode32 || IS_HALFWORD(src2w))) {
1942 inst = emit_x86_instruction(compiler, 1, dst_r, 0, SLJIT_MEM1(src1), (sljit_s32)src2w);
1943 #else
1944 if (src2 & SLJIT_IMM) {
1945 inst = emit_x86_instruction(compiler, 1, dst_r, 0, SLJIT_MEM1(src1), src2w);
1946 #endif
1947 FAIL_IF(!inst);
1948 *inst = LEA_r_m;
1949 done = 1;
1950 }
1951 }
1952 else if (FAST_IS_REG(src2)) {
1953 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1954 if ((src1 & SLJIT_IMM) && (compiler->mode32 || IS_HALFWORD(src1w))) {
1955 inst = emit_x86_instruction(compiler, 1, dst_r, 0, SLJIT_MEM1(src2), (sljit_s32)src1w);
1956 #else
1957 if (src1 & SLJIT_IMM) {
1958 inst = emit_x86_instruction(compiler, 1, dst_r, 0, SLJIT_MEM1(src2), src1w);
1959 #endif
1960 FAIL_IF(!inst);
1961 *inst = LEA_r_m;
1962 done = 1;
1963 }
1964 }
1965
1966 if (done) {
1967 if (dst_r == TMP_REG1)
1968 return emit_mov(compiler, dst, dstw, TMP_REG1, 0);
1969 return SLJIT_SUCCESS;
1970 }
1971 return SLJIT_ERR_UNSUPPORTED;
1972 }
1973
1974 static sljit_s32 emit_cmp_binary(struct sljit_compiler *compiler,
1975 sljit_s32 src1, sljit_sw src1w,
1976 sljit_s32 src2, sljit_sw src2w)
1977 {
1978 sljit_u8* inst;
1979
1980 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1981 if (src1 == SLJIT_R0 && (src2 & SLJIT_IMM) && (src2w > 127 || src2w < -128) && (compiler->mode32 || IS_HALFWORD(src2w))) {
1982 #else
1983 if (src1 == SLJIT_R0 && (src2 & SLJIT_IMM) && (src2w > 127 || src2w < -128)) {
1984 #endif
1985 BINARY_EAX_IMM(CMP_EAX_i32, src2w);
1986 return SLJIT_SUCCESS;
1987 }
1988
1989 if (FAST_IS_REG(src1)) {
1990 if (src2 & SLJIT_IMM) {
1991 BINARY_IMM(CMP, CMP_rm_r, src2w, src1, 0);
1992 }
1993 else {
1994 inst = emit_x86_instruction(compiler, 1, src1, 0, src2, src2w);
1995 FAIL_IF(!inst);
1996 *inst = CMP_r_rm;
1997 }
1998 return SLJIT_SUCCESS;
1999 }
2000
2001 if (FAST_IS_REG(src2) && !(src1 & SLJIT_IMM)) {
2002 inst = emit_x86_instruction(compiler, 1, src2, 0, src1, src1w);
2003 FAIL_IF(!inst);
2004 *inst = CMP_rm_r;
2005 return SLJIT_SUCCESS;
2006 }
2007
2008 if (src2 & SLJIT_IMM) {
2009 if (src1 & SLJIT_IMM) {
2010 EMIT_MOV(compiler, TMP_REG1, 0, src1, src1w);
2011 src1 = TMP_REG1;
2012 src1w = 0;
2013 }
2014 BINARY_IMM(CMP, CMP_rm_r, src2w, src1, src1w);
2015 }
2016 else {
2017 EMIT_MOV(compiler, TMP_REG1, 0, src1, src1w);
2018 inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, src2, src2w);
2019 FAIL_IF(!inst);
2020 *inst = CMP_r_rm;
2021 }
2022 return SLJIT_SUCCESS;
2023 }
2024
2025 static sljit_s32 emit_test_binary(struct sljit_compiler *compiler,
2026 sljit_s32 src1, sljit_sw src1w,
2027 sljit_s32 src2, sljit_sw src2w)
2028 {
2029 sljit_u8* inst;
2030
2031 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2032 if (src1 == SLJIT_R0 && (src2 & SLJIT_IMM) && (src2w > 127 || src2w < -128) && (compiler->mode32 || IS_HALFWORD(src2w))) {
2033 #else
2034 if (src1 == SLJIT_R0 && (src2 & SLJIT_IMM) && (src2w > 127 || src2w < -128)) {
2035 #endif
2036 BINARY_EAX_IMM(TEST_EAX_i32, src2w);
2037 return SLJIT_SUCCESS;
2038 }
2039
2040 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2041 if (src2 == SLJIT_R0 && (src1 & SLJIT_IMM) && (src1w > 127 || src1w < -128) && (compiler->mode32 || IS_HALFWORD(src1w))) {
2042 #else
2043 if (src2 == SLJIT_R0 && (src1 & SLJIT_IMM) && (src1w > 127 || src1w < -128)) {
2044 #endif
2045 BINARY_EAX_IMM(TEST_EAX_i32, src1w);
2046 return SLJIT_SUCCESS;
2047 }
2048
2049 if (!(src1 & SLJIT_IMM)) {
2050 if (src2 & SLJIT_IMM) {
2051 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2052 if (IS_HALFWORD(src2w) || compiler->mode32) {
2053 inst = emit_x86_instruction(compiler, 1, SLJIT_IMM, src2w, src1, src1w);
2054 FAIL_IF(!inst);
2055 *inst = GROUP_F7;
2056 }
2057 else {
2058 FAIL_IF(emit_load_imm64(compiler, TMP_REG1, src2w));
2059 inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, src1, src1w);
2060 FAIL_IF(!inst);
2061 *inst = TEST_rm_r;
2062 }
2063 #else
2064 inst = emit_x86_instruction(compiler, 1, SLJIT_IMM, src2w, src1, src1w);
2065 FAIL_IF(!inst);
2066 *inst = GROUP_F7;
2067 #endif
2068 return SLJIT_SUCCESS;
2069 }
2070 else if (FAST_IS_REG(src1)) {
2071 inst = emit_x86_instruction(compiler, 1, src1, 0, src2, src2w);
2072 FAIL_IF(!inst);
2073 *inst = TEST_rm_r;
2074 return SLJIT_SUCCESS;
2075 }
2076 }
2077
2078 if (!(src2 & SLJIT_IMM)) {
2079 if (src1 & SLJIT_IMM) {
2080 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2081 if (IS_HALFWORD(src1w) || compiler->mode32) {
2082 inst = emit_x86_instruction(compiler, 1, SLJIT_IMM, src1w, src2, src2w);
2083 FAIL_IF(!inst);
2084 *inst = GROUP_F7;
2085 }
2086 else {
2087 FAIL_IF(emit_load_imm64(compiler, TMP_REG1, src1w));
2088 inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, src2, src2w);
2089 FAIL_IF(!inst);
2090 *inst = TEST_rm_r;
2091 }
2092 #else
2093 inst = emit_x86_instruction(compiler, 1, src1, src1w, src2, src2w);
2094 FAIL_IF(!inst);
2095 *inst = GROUP_F7;
2096 #endif
2097 return SLJIT_SUCCESS;
2098 }
2099 else if (FAST_IS_REG(src2)) {
2100 inst = emit_x86_instruction(compiler, 1, src2, 0, src1, src1w);
2101 FAIL_IF(!inst);
2102 *inst = TEST_rm_r;
2103 return SLJIT_SUCCESS;
2104 }
2105 }
2106
2107 EMIT_MOV(compiler, TMP_REG1, 0, src1, src1w);
2108 if (src2 & SLJIT_IMM) {
2109 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2110 if (IS_HALFWORD(src2w) || compiler->mode32) {
2111 inst = emit_x86_instruction(compiler, 1, SLJIT_IMM, src2w, TMP_REG1, 0);
2112 FAIL_IF(!inst);
2113 *inst = GROUP_F7;
2114 }
2115 else {
2116 FAIL_IF(emit_load_imm64(compiler, TMP_REG2, src2w));
2117 inst = emit_x86_instruction(compiler, 1, TMP_REG2, 0, TMP_REG1, 0);
2118 FAIL_IF(!inst);
2119 *inst = TEST_rm_r;
2120 }
2121 #else
2122 inst = emit_x86_instruction(compiler, 1, SLJIT_IMM, src2w, TMP_REG1, 0);
2123 FAIL_IF(!inst);
2124 *inst = GROUP_F7;
2125 #endif
2126 }
2127 else {
2128 inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, src2, src2w);
2129 FAIL_IF(!inst);
2130 *inst = TEST_rm_r;
2131 }
2132 return SLJIT_SUCCESS;
2133 }
2134
2135 static sljit_s32 emit_shift(struct sljit_compiler *compiler,
2136 sljit_u8 mode,
2137 sljit_s32 dst, sljit_sw dstw,
2138 sljit_s32 src1, sljit_sw src1w,
2139 sljit_s32 src2, sljit_sw src2w)
2140 {
2141 sljit_u8* inst;
2142
2143 if ((src2 & SLJIT_IMM) || (src2 == SLJIT_PREF_SHIFT_REG)) {
2144 if (dst == src1 && dstw == src1w) {
2145 inst = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, src2, src2w, dst, dstw);
2146 FAIL_IF(!inst);
2147 *inst |= mode;
2148 return SLJIT_SUCCESS;
2149 }
2150 if (dst == SLJIT_PREF_SHIFT_REG && src2 == SLJIT_PREF_SHIFT_REG) {
2151 EMIT_MOV(compiler, TMP_REG1, 0, src1, src1w);
2152 inst = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, SLJIT_PREF_SHIFT_REG, 0, TMP_REG1, 0);
2153 FAIL_IF(!inst);
2154 *inst |= mode;
2155 EMIT_MOV(compiler, SLJIT_PREF_SHIFT_REG, 0, TMP_REG1, 0);
2156 return SLJIT_SUCCESS;
2157 }
2158 if (FAST_IS_REG(dst)) {
2159 EMIT_MOV(compiler, dst, 0, src1, src1w);
2160 inst = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, src2, src2w, dst, 0);
2161 FAIL_IF(!inst);
2162 *inst |= mode;
2163 return SLJIT_SUCCESS;
2164 }
2165
2166 EMIT_MOV(compiler, TMP_REG1, 0, src1, src1w);
2167 inst = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, src2, src2w, TMP_REG1, 0);
2168 FAIL_IF(!inst);
2169 *inst |= mode;
2170 EMIT_MOV(compiler, dst, dstw, TMP_REG1, 0);
2171 return SLJIT_SUCCESS;
2172 }
2173
2174 if (dst == SLJIT_PREF_SHIFT_REG) {
2175 EMIT_MOV(compiler, TMP_REG1, 0, src1, src1w);
2176 EMIT_MOV(compiler, SLJIT_PREF_SHIFT_REG, 0, src2, src2w);
2177 inst = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, SLJIT_PREF_SHIFT_REG, 0, TMP_REG1, 0);
2178 FAIL_IF(!inst);
2179 *inst |= mode;
2180 EMIT_MOV(compiler, SLJIT_PREF_SHIFT_REG, 0, TMP_REG1, 0);
2181 }
2182 else if (FAST_IS_REG(dst) && dst != src2 && dst != TMP_REG1 && !ADDRESSING_DEPENDS_ON(src2, dst)) {
2183 if (src1 != dst)
2184 EMIT_MOV(compiler, dst, 0, src1, src1w);
2185 EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_PREF_SHIFT_REG, 0);
2186 EMIT_MOV(compiler, SLJIT_PREF_SHIFT_REG, 0, src2, src2w);
2187 inst = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, SLJIT_PREF_SHIFT_REG, 0, dst, 0);
2188 FAIL_IF(!inst);
2189 *inst |= mode;
2190 EMIT_MOV(compiler, SLJIT_PREF_SHIFT_REG, 0, TMP_REG1, 0);
2191 }
2192 else {
2193 /* This case is complex since ecx itself may be used for
2194 addressing, and this case must be supported as well. */
2195 EMIT_MOV(compiler, TMP_REG1, 0, src1, src1w);
2196 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
2197 EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), 0, SLJIT_PREF_SHIFT_REG, 0);
2198 EMIT_MOV(compiler, SLJIT_PREF_SHIFT_REG, 0, src2, src2w);
2199 inst = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, SLJIT_PREF_SHIFT_REG, 0, TMP_REG1, 0);
2200 FAIL_IF(!inst);
2201 *inst |= mode;
2202 EMIT_MOV(compiler, SLJIT_PREF_SHIFT_REG, 0, SLJIT_MEM1(SLJIT_SP), 0);
2203 #else
2204 EMIT_MOV(compiler, TMP_REG2, 0, SLJIT_PREF_SHIFT_REG, 0);
2205 EMIT_MOV(compiler, SLJIT_PREF_SHIFT_REG, 0, src2, src2w);
2206 inst = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, SLJIT_PREF_SHIFT_REG, 0, TMP_REG1, 0);
2207 FAIL_IF(!inst);
2208 *inst |= mode;
2209 EMIT_MOV(compiler, SLJIT_PREF_SHIFT_REG, 0, TMP_REG2, 0);
2210 #endif
2211 if (dst != TMP_REG1)
2212 return emit_mov(compiler, dst, dstw, TMP_REG1, 0);
2213 }
2214
2215 return SLJIT_SUCCESS;
2216 }
2217
2218 static sljit_s32 emit_shift_with_flags(struct sljit_compiler *compiler,
2219 sljit_u8 mode, sljit_s32 set_flags,
2220 sljit_s32 dst, sljit_sw dstw,
2221 sljit_s32 src1, sljit_sw src1w,
2222 sljit_s32 src2, sljit_sw src2w)
2223 {
2224 /* The CPU does not set flags if the shift count is 0. */
2225 if (src2 & SLJIT_IMM) {
2226 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2227 if ((src2w & 0x3f) != 0 || (compiler->mode32 && (src2w & 0x1f) != 0))
2228 return emit_shift(compiler, mode, dst, dstw, src1, src1w, src2, src2w);
2229 #else
2230 if ((src2w & 0x1f) != 0)
2231 return emit_shift(compiler, mode, dst, dstw, src1, src1w, src2, src2w);
2232 #endif
2233 if (!set_flags)
2234 return emit_mov(compiler, dst, dstw, src1, src1w);
2235 /* OR dst, src, 0 */
2236 return emit_cum_binary(compiler, BINARY_OPCODE(OR),
2237 dst, dstw, src1, src1w, SLJIT_IMM, 0);
2238 }
2239
2240 if (!set_flags)
2241 return emit_shift(compiler, mode, dst, dstw, src1, src1w, src2, src2w);
2242
2243 if (!FAST_IS_REG(dst))
2244 FAIL_IF(emit_cmp_binary(compiler, src1, src1w, SLJIT_IMM, 0));
2245
2246 FAIL_IF(emit_shift(compiler, mode, dst, dstw, src1, src1w, src2, src2w));
2247
2248 if (FAST_IS_REG(dst))
2249 return emit_cmp_binary(compiler, dst, dstw, SLJIT_IMM, 0);
2250 return SLJIT_SUCCESS;
2251 }
2252
2253 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op2(struct sljit_compiler *compiler, sljit_s32 op,
2254 sljit_s32 dst, sljit_sw dstw,
2255 sljit_s32 src1, sljit_sw src1w,
2256 sljit_s32 src2, sljit_sw src2w)
2257 {
2258 CHECK_ERROR();
2259 CHECK(check_sljit_emit_op2(compiler, op, 0, dst, dstw, src1, src1w, src2, src2w));
2260 ADJUST_LOCAL_OFFSET(dst, dstw);
2261 ADJUST_LOCAL_OFFSET(src1, src1w);
2262 ADJUST_LOCAL_OFFSET(src2, src2w);
2263
2264 CHECK_EXTRA_REGS(dst, dstw, (void)0);
2265 CHECK_EXTRA_REGS(src1, src1w, (void)0);
2266 CHECK_EXTRA_REGS(src2, src2w, (void)0);
2267 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2268 compiler->mode32 = op & SLJIT_32;
2269 #endif
2270
2271 SLJIT_ASSERT(dst != TMP_REG1 || HAS_FLAGS(op));
2272
2273 switch (GET_OPCODE(op)) {
2274 case SLJIT_ADD:
2275 if (!HAS_FLAGS(op)) {
2276 if (emit_lea_binary(compiler, dst, dstw, src1, src1w, src2, src2w) != SLJIT_ERR_UNSUPPORTED)
2277 return compiler->error;
2278 }
2279 return emit_cum_binary(compiler, BINARY_OPCODE(ADD),
2280 dst, dstw, src1, src1w, src2, src2w);
2281 case SLJIT_ADDC:
2282 return emit_cum_binary(compiler, BINARY_OPCODE(ADC),
2283 dst, dstw, src1, src1w, src2, src2w);
2284 case SLJIT_SUB:
2285 if (src1 == SLJIT_IMM && src1w == 0)
2286 return emit_unary(compiler, NEG_rm, dst, dstw, src2, src2w);
2287
2288 if (!HAS_FLAGS(op)) {
2289 if ((src2 & SLJIT_IMM) && emit_lea_binary(compiler, dst, dstw, src1, src1w, SLJIT_IMM, -src2w) != SLJIT_ERR_UNSUPPORTED)
2290 return compiler->error;
2291 if (FAST_IS_REG(dst) && src2 == dst) {
2292 FAIL_IF(emit_non_cum_binary(compiler, BINARY_OPCODE(SUB), dst, 0, dst, 0, src1, src1w));
2293 return emit_unary(compiler, NEG_rm, dst, 0, dst, 0);
2294 }
2295 }
2296
2297 return emit_non_cum_binary(compiler, BINARY_OPCODE(SUB),
2298 dst, dstw, src1, src1w, src2, src2w);
2299 case SLJIT_SUBC:
2300 return emit_non_cum_binary(compiler, BINARY_OPCODE(SBB),
2301 dst, dstw, src1, src1w, src2, src2w);
2302 case SLJIT_MUL:
2303 return emit_mul(compiler, dst, dstw, src1, src1w, src2, src2w);
2304 case SLJIT_AND:
2305 return emit_cum_binary(compiler, BINARY_OPCODE(AND),
2306 dst, dstw, src1, src1w, src2, src2w);
2307 case SLJIT_OR:
2308 return emit_cum_binary(compiler, BINARY_OPCODE(OR),
2309 dst, dstw, src1, src1w, src2, src2w);
2310 case SLJIT_XOR:
2311 return emit_cum_binary(compiler, BINARY_OPCODE(XOR),
2312 dst, dstw, src1, src1w, src2, src2w);
2313 case SLJIT_SHL:
2314 return emit_shift_with_flags(compiler, SHL, HAS_FLAGS(op),
2315 dst, dstw, src1, src1w, src2, src2w);
2316 case SLJIT_LSHR:
2317 return emit_shift_with_flags(compiler, SHR, HAS_FLAGS(op),
2318 dst, dstw, src1, src1w, src2, src2w);
2319 case SLJIT_ASHR:
2320 return emit_shift_with_flags(compiler, SAR, HAS_FLAGS(op),
2321 dst, dstw, src1, src1w, src2, src2w);
2322 }
2323
2324 return SLJIT_SUCCESS;
2325 }
2326
2327 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op2u(struct sljit_compiler *compiler, sljit_s32 op,
2328 sljit_s32 src1, sljit_sw src1w,
2329 sljit_s32 src2, sljit_sw src2w)
2330 {
2331 sljit_s32 opcode = GET_OPCODE(op);
2332
2333 CHECK_ERROR();
2334 CHECK(check_sljit_emit_op2(compiler, op, 1, 0, 0, src1, src1w, src2, src2w));
2335
2336 if (opcode != SLJIT_SUB && opcode != SLJIT_AND) {
2337 SLJIT_SKIP_CHECKS(compiler);
2338 return sljit_emit_op2(compiler, op, TMP_REG1, 0, src1, src1w, src2, src2w);
2339 }
2340
2341 ADJUST_LOCAL_OFFSET(src1, src1w);
2342 ADJUST_LOCAL_OFFSET(src2, src2w);
2343
2344 CHECK_EXTRA_REGS(src1, src1w, (void)0);
2345 CHECK_EXTRA_REGS(src2, src2w, (void)0);
2346 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2347 compiler->mode32 = op & SLJIT_32;
2348 #endif
2349
2350 if (opcode == SLJIT_SUB) {
2351 return emit_cmp_binary(compiler, src1, src1w, src2, src2w);
2352 }
2353 return emit_test_binary(compiler, src1, src1w, src2, src2w);
2354 }
2355
2356 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_src(struct sljit_compiler *compiler, sljit_s32 op,
2357 sljit_s32 src, sljit_sw srcw)
2358 {
2359 CHECK_ERROR();
2360 CHECK(check_sljit_emit_op_src(compiler, op, src, srcw));
2361 ADJUST_LOCAL_OFFSET(src, srcw);
2362
2363 CHECK_EXTRA_REGS(src, srcw, (void)0);
2364
2365 switch (op) {
2366 case SLJIT_FAST_RETURN:
2367 return emit_fast_return(compiler, src, srcw);
2368 case SLJIT_SKIP_FRAMES_BEFORE_FAST_RETURN:
2369 /* Don't adjust shadow stack if it isn't enabled. */
2370 if (!cpu_has_shadow_stack ())
2371 return SLJIT_SUCCESS;
2372 return adjust_shadow_stack(compiler, src, srcw);
2373 case SLJIT_PREFETCH_L1:
2374 case SLJIT_PREFETCH_L2:
2375 case SLJIT_PREFETCH_L3:
2376 case SLJIT_PREFETCH_ONCE:
2377 return emit_prefetch(compiler, op, src, srcw);
2378 }
2379
2380 return SLJIT_SUCCESS;
2381 }
2382
2383 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_get_register_index(sljit_s32 reg)
2384 {
2385 CHECK_REG_INDEX(check_sljit_get_register_index(reg));
2386 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
2387 if (reg >= SLJIT_R3 && reg <= SLJIT_R8)
2388 return -1;
2389 #endif
2390 return reg_map[reg];
2391 }
2392
2393 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_get_float_register_index(sljit_s32 reg)
2394 {
2395 CHECK_REG_INDEX(check_sljit_get_float_register_index(reg));
2396 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
2397 return reg;
2398 #else
2399 return freg_map[reg];
2400 #endif
2401 }
2402
2403 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_custom(struct sljit_compiler *compiler,
2404 void *instruction, sljit_u32 size)
2405 {
2406 sljit_u8 *inst;
2407
2408 CHECK_ERROR();
2409 CHECK(check_sljit_emit_op_custom(compiler, instruction, size));
2410
2411 inst = (sljit_u8*)ensure_buf(compiler, 1 + size);
2412 FAIL_IF(!inst);
2413 INC_SIZE(size);
2414 SLJIT_MEMCPY(inst, instruction, size);
2415 return SLJIT_SUCCESS;
2416 }
2417
2418 /* --------------------------------------------------------------------- */
2419 /* Floating point operators */
2420 /* --------------------------------------------------------------------- */
2421
2422 /* Alignment(3) + 4 * 16 bytes. */
2423 static sljit_u32 sse2_data[3 + (4 * 4)];
2424 static sljit_u32 *sse2_buffer;
2425
2426 static void init_compiler(void)
2427 {
2428 /* Align to 16 bytes. */
2429 sse2_buffer = (sljit_u32*)(((sljit_uw)sse2_data + 15) & ~(sljit_uw)0xf);
2430
2431 /* Single precision constants (each constant is 16 byte long). */
2432 sse2_buffer[0] = 0x80000000;
2433 sse2_buffer[4] = 0x7fffffff;
2434 /* Double precision constants (each constant is 16 byte long). */
2435 sse2_buffer[8] = 0;
2436 sse2_buffer[9] = 0x80000000;
2437 sse2_buffer[12] = 0xffffffff;
2438 sse2_buffer[13] = 0x7fffffff;
2439 }
2440
2441 static sljit_s32 emit_sse2(struct sljit_compiler *compiler, sljit_u8 opcode,
2442 sljit_s32 single, sljit_s32 xmm1, sljit_s32 xmm2, sljit_sw xmm2w)
2443 {
2444 sljit_u8 *inst;
2445
2446 inst = emit_x86_instruction(compiler, 2 | (single ? EX86_PREF_F3 : EX86_PREF_F2) | EX86_SSE2, xmm1, 0, xmm2, xmm2w);
2447 FAIL_IF(!inst);
2448 *inst++ = GROUP_0F;
2449 *inst = opcode;
2450 return SLJIT_SUCCESS;
2451 }
2452
2453 static sljit_s32 emit_sse2_logic(struct sljit_compiler *compiler, sljit_u8 opcode,
2454 sljit_s32 pref66, sljit_s32 xmm1, sljit_s32 xmm2, sljit_sw xmm2w)
2455 {
2456 sljit_u8 *inst;
2457
2458 inst = emit_x86_instruction(compiler, 2 | (pref66 ? EX86_PREF_66 : 0) | EX86_SSE2, xmm1, 0, xmm2, xmm2w);
2459 FAIL_IF(!inst);
2460 *inst++ = GROUP_0F;
2461 *inst = opcode;
2462 return SLJIT_SUCCESS;
2463 }
2464
2465 static SLJIT_INLINE sljit_s32 emit_sse2_load(struct sljit_compiler *compiler,
2466 sljit_s32 single, sljit_s32 dst, sljit_s32 src, sljit_sw srcw)
2467 {
2468 return emit_sse2(compiler, MOVSD_x_xm, single, dst, src, srcw);
2469 }
2470
2471 static SLJIT_INLINE sljit_s32 emit_sse2_store(struct sljit_compiler *compiler,
2472 sljit_s32 single, sljit_s32 dst, sljit_sw dstw, sljit_s32 src)
2473 {
2474 return emit_sse2(compiler, MOVSD_xm_x, single, src, dst, dstw);
2475 }
2476
2477 static SLJIT_INLINE sljit_s32 sljit_emit_fop1_conv_sw_from_f64(struct sljit_compiler *compiler, sljit_s32 op,
2478 sljit_s32 dst, sljit_sw dstw,
2479 sljit_s32 src, sljit_sw srcw)
2480 {
2481 sljit_s32 dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
2482 sljit_u8 *inst;
2483
2484 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2485 if (GET_OPCODE(op) == SLJIT_CONV_SW_FROM_F64)
2486 compiler->mode32 = 0;
2487 #endif
2488
2489 inst = emit_x86_instruction(compiler, 2 | ((op & SLJIT_32) ? EX86_PREF_F3 : EX86_PREF_F2) | EX86_SSE2_OP2, dst_r, 0, src, srcw);
2490 FAIL_IF(!inst);
2491 *inst++ = GROUP_0F;
2492 *inst = CVTTSD2SI_r_xm;
2493
2494 if (dst & SLJIT_MEM)
2495 return emit_mov(compiler, dst, dstw, TMP_REG1, 0);
2496 return SLJIT_SUCCESS;
2497 }
2498
2499 static SLJIT_INLINE sljit_s32 sljit_emit_fop1_conv_f64_from_sw(struct sljit_compiler *compiler, sljit_s32 op,
2500 sljit_s32 dst, sljit_sw dstw,
2501 sljit_s32 src, sljit_sw srcw)
2502 {
2503 sljit_s32 dst_r = FAST_IS_REG(dst) ? dst : TMP_FREG;
2504 sljit_u8 *inst;
2505
2506 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2507 if (GET_OPCODE(op) == SLJIT_CONV_F64_FROM_SW)
2508 compiler->mode32 = 0;
2509 #endif
2510
2511 if (src & SLJIT_IMM) {
2512 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2513 if (GET_OPCODE(op) == SLJIT_CONV_F64_FROM_S32)
2514 srcw = (sljit_s32)srcw;
2515 #endif
2516 EMIT_MOV(compiler, TMP_REG1, 0, src, srcw);
2517 src = TMP_REG1;
2518 srcw = 0;
2519 }
2520
2521 inst = emit_x86_instruction(compiler, 2 | ((op & SLJIT_32) ? EX86_PREF_F3 : EX86_PREF_F2) | EX86_SSE2_OP1, dst_r, 0, src, srcw);
2522 FAIL_IF(!inst);
2523 *inst++ = GROUP_0F;
2524 *inst = CVTSI2SD_x_rm;
2525
2526 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2527 compiler->mode32 = 1;
2528 #endif
2529 if (dst_r == TMP_FREG)
2530 return emit_sse2_store(compiler, op & SLJIT_32, dst, dstw, TMP_FREG);
2531 return SLJIT_SUCCESS;
2532 }
2533
2534 static SLJIT_INLINE sljit_s32 sljit_emit_fop1_cmp(struct sljit_compiler *compiler, sljit_s32 op,
2535 sljit_s32 src1, sljit_sw src1w,
2536 sljit_s32 src2, sljit_sw src2w)
2537 {
2538 switch (GET_FLAG_TYPE(op)) {
2539 case SLJIT_ORDERED_LESS:
2540 case SLJIT_UNORDERED_OR_GREATER_EQUAL:
2541 case SLJIT_UNORDERED_OR_GREATER:
2542 case SLJIT_ORDERED_LESS_EQUAL:
2543 if (!FAST_IS_REG(src2)) {
2544 FAIL_IF(emit_sse2_load(compiler, op & SLJIT_32, TMP_FREG, src2, src2w));
2545 src2 = TMP_FREG;
2546 }
2547
2548 return emit_sse2_logic(compiler, UCOMISD_x_xm, !(op & SLJIT_32), src2, src1, src1w);
2549 }
2550
2551 if (!FAST_IS_REG(src1)) {
2552 FAIL_IF(emit_sse2_load(compiler, op & SLJIT_32, TMP_FREG, src1, src1w));
2553 src1 = TMP_FREG;
2554 }
2555
2556 return emit_sse2_logic(compiler, UCOMISD_x_xm, !(op & SLJIT_32), src1, src2, src2w);
2557 }
2558
2559 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fop1(struct sljit_compiler *compiler, sljit_s32 op,
2560 sljit_s32 dst, sljit_sw dstw,
2561 sljit_s32 src, sljit_sw srcw)
2562 {
2563 sljit_s32 dst_r;
2564
2565 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2566 compiler->mode32 = 1;
2567 #endif
2568
2569 CHECK_ERROR();
2570 SELECT_FOP1_OPERATION_WITH_CHECKS(compiler, op, dst, dstw, src, srcw);
2571
2572 if (GET_OPCODE(op) == SLJIT_MOV_F64) {
2573 if (FAST_IS_REG(dst))
2574 return emit_sse2_load(compiler, op & SLJIT_32, dst, src, srcw);
2575 if (FAST_IS_REG(src))
2576 return emit_sse2_store(compiler, op & SLJIT_32, dst, dstw, src);
2577 FAIL_IF(emit_sse2_load(compiler, op & SLJIT_32, TMP_FREG, src, srcw));
2578 return emit_sse2_store(compiler, op & SLJIT_32, dst, dstw, TMP_FREG);
2579 }
2580
2581 if (GET_OPCODE(op) == SLJIT_CONV_F64_FROM_F32) {
2582 dst_r = FAST_IS_REG(dst) ? dst : TMP_FREG;
2583 if (FAST_IS_REG(src)) {
2584 /* We overwrite the high bits of source. From SLJIT point of view,
2585 this is not an issue.
2586 Note: In SSE3, we could also use MOVDDUP and MOVSLDUP. */
2587 FAIL_IF(emit_sse2_logic(compiler, UNPCKLPD_x_xm, op & SLJIT_32, src, src, 0));
2588 }
2589 else {
2590 FAIL_IF(emit_sse2_load(compiler, !(op & SLJIT_32), TMP_FREG, src, srcw));
2591 src = TMP_FREG;
2592 }
2593
2594 FAIL_IF(emit_sse2_logic(compiler, CVTPD2PS_x_xm, op & SLJIT_32, dst_r, src, 0));
2595 if (dst_r == TMP_FREG)
2596 return emit_sse2_store(compiler, op & SLJIT_32, dst, dstw, TMP_FREG);
2597 return SLJIT_SUCCESS;
2598 }
2599
2600 if (FAST_IS_REG(dst)) {
2601 dst_r = dst;
2602 if (dst != src)
2603 FAIL_IF(emit_sse2_load(compiler, op & SLJIT_32, dst_r, src, srcw));
2604 }
2605 else {
2606 dst_r = TMP_FREG;
2607 FAIL_IF(emit_sse2_load(compiler, op & SLJIT_32, dst_r, src, srcw));
2608 }
2609
2610 switch (GET_OPCODE(op)) {
2611 case SLJIT_NEG_F64:
2612 FAIL_IF(emit_sse2_logic(compiler, XORPD_x_xm, 1, dst_r, SLJIT_MEM0(), (sljit_sw)(op & SLJIT_32 ? sse2_buffer : sse2_buffer + 8)));
2613 break;
2614
2615 case SLJIT_ABS_F64:
2616 FAIL_IF(emit_sse2_logic(compiler, ANDPD_x_xm, 1, dst_r, SLJIT_MEM0(), (sljit_sw)(op & SLJIT_32 ? sse2_buffer + 4 : sse2_buffer + 12)));
2617 break;
2618 }
2619
2620 if (dst_r == TMP_FREG)
2621 return emit_sse2_store(compiler, op & SLJIT_32, dst, dstw, TMP_FREG);
2622 return SLJIT_SUCCESS;
2623 }
2624
2625 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fop2(struct sljit_compiler *compiler, sljit_s32 op,
2626 sljit_s32 dst, sljit_sw dstw,
2627 sljit_s32 src1, sljit_sw src1w,
2628 sljit_s32 src2, sljit_sw src2w)
2629 {
2630 sljit_s32 dst_r;
2631
2632 CHECK_ERROR();
2633 CHECK(check_sljit_emit_fop2(compiler, op, dst, dstw, src1, src1w, src2, src2w));
2634 ADJUST_LOCAL_OFFSET(dst, dstw);
2635 ADJUST_LOCAL_OFFSET(src1, src1w);
2636 ADJUST_LOCAL_OFFSET(src2, src2w);
2637
2638 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2639 compiler->mode32 = 1;
2640 #endif
2641
2642 if (FAST_IS_REG(dst)) {
2643 dst_r = dst;
2644 if (dst == src1)
2645 ; /* Do nothing here. */
2646 else if (dst == src2 && (op == SLJIT_ADD_F64 || op == SLJIT_MUL_F64)) {
2647 /* Swap arguments. */
2648 src2 = src1;
2649 src2w = src1w;
2650 }
2651 else if (dst != src2)
2652 FAIL_IF(emit_sse2_load(compiler, op & SLJIT_32, dst_r, src1, src1w));
2653 else {
2654 dst_r = TMP_FREG;
2655 FAIL_IF(emit_sse2_load(compiler, op & SLJIT_32, TMP_FREG, src1, src1w));
2656 }
2657 }
2658 else {
2659 dst_r = TMP_FREG;
2660 FAIL_IF(emit_sse2_load(compiler, op & SLJIT_32, TMP_FREG, src1, src1w));
2661 }
2662
2663 switch (GET_OPCODE(op)) {
2664 case SLJIT_ADD_F64:
2665 FAIL_IF(emit_sse2(compiler, ADDSD_x_xm, op & SLJIT_32, dst_r, src2, src2w));
2666 break;
2667
2668 case SLJIT_SUB_F64:
2669 FAIL_IF(emit_sse2(compiler, SUBSD_x_xm, op & SLJIT_32, dst_r, src2, src2w));
2670 break;
2671
2672 case SLJIT_MUL_F64:
2673 FAIL_IF(emit_sse2(compiler, MULSD_x_xm, op & SLJIT_32, dst_r, src2, src2w));
2674 break;
2675
2676 case SLJIT_DIV_F64:
2677 FAIL_IF(emit_sse2(compiler, DIVSD_x_xm, op & SLJIT_32, dst_r, src2, src2w));
2678 break;
2679 }
2680
2681 if (dst_r == TMP_FREG)
2682 return emit_sse2_store(compiler, op & SLJIT_32, dst, dstw, TMP_FREG);
2683 return SLJIT_SUCCESS;
2684 }
2685
2686 /* --------------------------------------------------------------------- */
2687 /* Conditional instructions */
2688 /* --------------------------------------------------------------------- */
2689
2690 SLJIT_API_FUNC_ATTRIBUTE struct sljit_label* sljit_emit_label(struct sljit_compiler *compiler)
2691 {
2692 sljit_u8 *inst;
2693 struct sljit_label *label;
2694
2695 CHECK_ERROR_PTR();
2696 CHECK_PTR(check_sljit_emit_label(compiler));
2697
2698 if (compiler->last_label && compiler->last_label->size == compiler->size)
2699 return compiler->last_label;
2700
2701 label = (struct sljit_label*)ensure_abuf(compiler, sizeof(struct sljit_label));
2702 PTR_FAIL_IF(!label);
2703 set_label(label, compiler);
2704
2705 inst = (sljit_u8*)ensure_buf(compiler, 2);
2706 PTR_FAIL_IF(!inst);
2707
2708 *inst++ = 0;
2709 *inst++ = 0;
2710
2711 return label;
2712 }
2713
2714 SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_jump(struct sljit_compiler *compiler, sljit_s32 type)
2715 {
2716 sljit_u8 *inst;
2717 struct sljit_jump *jump;
2718
2719 CHECK_ERROR_PTR();
2720 CHECK_PTR(check_sljit_emit_jump(compiler, type));
2721
2722 jump = (struct sljit_jump*)ensure_abuf(compiler, sizeof(struct sljit_jump));
2723 PTR_FAIL_IF_NULL(jump);
2724 set_jump(jump, compiler, (sljit_u32)((type & SLJIT_REWRITABLE_JUMP) | ((type & 0xff) << TYPE_SHIFT)));
2725 type &= 0xff;
2726
2727 /* Worst case size. */
2728 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
2729 compiler->size += (type >= SLJIT_JUMP) ? 5 : 6;
2730 #else
2731 compiler->size += (type >= SLJIT_JUMP) ? (10 + 3) : (2 + 10 + 3);
2732 #endif
2733
2734 inst = (sljit_u8*)ensure_buf(compiler, 2);
2735 PTR_FAIL_IF_NULL(inst);
2736
2737 *inst++ = 0;
2738 *inst++ = 1;
2739 return jump;
2740 }
2741
2742 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_ijump(struct sljit_compiler *compiler, sljit_s32 type, sljit_s32 src, sljit_sw srcw)
2743 {
2744 sljit_u8 *inst;
2745 struct sljit_jump *jump;
2746
2747 CHECK_ERROR();
2748 CHECK(check_sljit_emit_ijump(compiler, type, src, srcw));
2749 ADJUST_LOCAL_OFFSET(src, srcw);
2750
2751 CHECK_EXTRA_REGS(src, srcw, (void)0);
2752
2753 if (src == SLJIT_IMM) {
2754 jump = (struct sljit_jump*)ensure_abuf(compiler, sizeof(struct sljit_jump));
2755 FAIL_IF_NULL(jump);
2756 set_jump(jump, compiler, (sljit_u32)(JUMP_ADDR | (type << TYPE_SHIFT)));
2757 jump->u.target = (sljit_uw)srcw;
2758
2759 /* Worst case size. */
2760 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
2761 compiler->size += 5;
2762 #else
2763 compiler->size += 10 + 3;
2764 #endif
2765
2766 inst = (sljit_u8*)ensure_buf(compiler, 2);
2767 FAIL_IF_NULL(inst);
2768
2769 *inst++ = 0;
2770 *inst++ = 1;
2771 }
2772 else {
2773 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2774 /* REX_W is not necessary (src is not immediate). */
2775 compiler->mode32 = 1;
2776 #endif
2777 inst = emit_x86_instruction(compiler, 1, 0, 0, src, srcw);
2778 FAIL_IF(!inst);
2779 *inst++ = GROUP_FF;
2780 *inst = U8(*inst | ((type >= SLJIT_FAST_CALL) ? CALL_rm : JMP_rm));
2781 }
2782 return SLJIT_SUCCESS;
2783 }
2784
2785 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_flags(struct sljit_compiler *compiler, sljit_s32 op,
2786 sljit_s32 dst, sljit_sw dstw,
2787 sljit_s32 type)
2788 {
2789 sljit_u8 *inst;
2790 sljit_u8 cond_set = 0;
2791 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2792 sljit_s32 reg;
2793 #endif
2794 /* ADJUST_LOCAL_OFFSET and CHECK_EXTRA_REGS might overwrite these values. */
2795 sljit_s32 dst_save = dst;
2796 sljit_sw dstw_save = dstw;
2797
2798 CHECK_ERROR();
2799 CHECK(check_sljit_emit_op_flags(compiler, op, dst, dstw, type));
2800
2801 ADJUST_LOCAL_OFFSET(dst, dstw);
2802 CHECK_EXTRA_REGS(dst, dstw, (void)0);
2803
2804 /* setcc = jcc + 0x10. */
2805 cond_set = U8(get_jump_code((sljit_uw)type) + 0x10);
2806
2807 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2808 if (GET_OPCODE(op) == SLJIT_OR && !GET_ALL_FLAGS(op) && FAST_IS_REG(dst)) {
2809 inst = (sljit_u8*)ensure_buf(compiler, 1 + 4 + 3);
2810 FAIL_IF(!inst);
2811 INC_SIZE(4 + 3);
2812 /* Set low register to conditional flag. */
2813 *inst++ = (reg_map[TMP_REG1] <= 7) ? REX : REX_B;
2814 *inst++ = GROUP_0F;
2815 *inst++ = cond_set;
2816 *inst++ = MOD_REG | reg_lmap[TMP_REG1];
2817 *inst++ = U8(REX | (reg_map[TMP_REG1] <= 7 ? 0 : REX_R) | (reg_map[dst] <= 7 ? 0 : REX_B));
2818 *inst++ = OR_rm8_r8;
2819 *inst++ = U8(MOD_REG | (reg_lmap[TMP_REG1] << 3) | reg_lmap[dst]);
2820 return SLJIT_SUCCESS;
2821 }
2822
2823 reg = (GET_OPCODE(op) < SLJIT_ADD && FAST_IS_REG(dst)) ? dst : TMP_REG1;
2824
2825 inst = (sljit_u8*)ensure_buf(compiler, 1 + 4 + 4);
2826 FAIL_IF(!inst);
2827 INC_SIZE(4 + 4);
2828 /* Set low register to conditional flag. */
2829 *inst++ = (reg_map[reg] <= 7) ? REX : REX_B;
2830 *inst++ = GROUP_0F;
2831 *inst++ = cond_set;
2832 *inst++ = MOD_REG | reg_lmap[reg];
2833 *inst++ = REX_W | (reg_map[reg] <= 7 ? 0 : (REX_B | REX_R));
2834 /* The movzx instruction does not affect flags. */
2835 *inst++ = GROUP_0F;
2836 *inst++ = MOVZX_r_rm8;
2837 *inst = U8(MOD_REG | (reg_lmap[reg] << 3) | reg_lmap[reg]);
2838
2839 if (reg != TMP_REG1)
2840 return SLJIT_SUCCESS;
2841
2842 if (GET_OPCODE(op) < SLJIT_ADD) {
2843 compiler->mode32 = GET_OPCODE(op) != SLJIT_MOV;
2844 return emit_mov(compiler, dst, dstw, TMP_REG1, 0);
2845 }
2846
2847 SLJIT_SKIP_CHECKS(compiler);
2848 return sljit_emit_op2(compiler, op, dst_save, dstw_save, dst_save, dstw_save, TMP_REG1, 0);
2849
2850 #else
2851 /* The SLJIT_CONFIG_X86_32 code path starts here. */
2852 if (GET_OPCODE(op) < SLJIT_ADD && FAST_IS_REG(dst)) {
2853 if (reg_map[dst] <= 4) {
2854 /* Low byte is accessible. */
2855 inst = (sljit_u8*)ensure_buf(compiler, 1 + 3 + 3);
2856 FAIL_IF(!inst);
2857 INC_SIZE(3 + 3);
2858 /* Set low byte to conditional flag. */
2859 *inst++ = GROUP_0F;
2860 *inst++ = cond_set;
2861 *inst++ = U8(MOD_REG | reg_map[dst]);
2862
2863 *inst++ = GROUP_0F;
2864 *inst++ = MOVZX_r_rm8;
2865 *inst = U8(MOD_REG | (reg_map[dst] << 3) | reg_map[dst]);
2866 return SLJIT_SUCCESS;
2867 }
2868
2869 /* Low byte is not accessible. */
2870 if (cpu_has_cmov == -1)
2871 get_cpu_features();
2872
2873 if (cpu_has_cmov) {
2874 EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_IMM, 1);
2875 /* a xor reg, reg operation would overwrite the flags. */
2876 EMIT_MOV(compiler, dst, 0, SLJIT_IMM, 0);
2877
2878 inst = (sljit_u8*)ensure_buf(compiler, 1 + 3);
2879 FAIL_IF(!inst);
2880 INC_SIZE(3);
2881
2882 *inst++ = GROUP_0F;
2883 /* cmovcc = setcc - 0x50. */
2884 *inst++ = U8(cond_set - 0x50);
2885 *inst++ = U8(MOD_REG | (reg_map[dst] << 3) | reg_map[TMP_REG1]);
2886 return SLJIT_SUCCESS;
2887 }
2888
2889 inst = (sljit_u8*)ensure_buf(compiler, 1 + 1 + 3 + 3 + 1);
2890 FAIL_IF(!inst);
2891 INC_SIZE(1 + 3 + 3 + 1);
2892 *inst++ = U8(XCHG_EAX_r | reg_map[TMP_REG1]);
2893 /* Set al to conditional flag. */
2894 *inst++ = GROUP_0F;
2895 *inst++ = cond_set;
2896 *inst++ = MOD_REG | 0 /* eax */;
2897
2898 *inst++ = GROUP_0F;
2899 *inst++ = MOVZX_r_rm8;
2900 *inst++ = U8(MOD_REG | (reg_map[dst] << 3) | 0 /* eax */);
2901 *inst++ = U8(XCHG_EAX_r | reg_map[TMP_REG1]);
2902 return SLJIT_SUCCESS;
2903 }
2904
2905 if (GET_OPCODE(op) == SLJIT_OR && !GET_ALL_FLAGS(op) && FAST_IS_REG(dst) && reg_map[dst] <= 4) {
2906 SLJIT_ASSERT(reg_map[SLJIT_R0] == 0);
2907
2908 if (dst != SLJIT_R0) {
2909 inst = (sljit_u8*)ensure_buf(compiler, 1 + 1 + 3 + 2 + 1);
2910 FAIL_IF(!inst);
2911 INC_SIZE(1 + 3 + 2 + 1);
2912 /* Set low register to conditional flag. */
2913 *inst++ = U8(XCHG_EAX_r | reg_map[TMP_REG1]);
2914 *inst++ = GROUP_0F;
2915 *inst++ = cond_set;
2916 *inst++ = MOD_REG | 0 /* eax */;
2917 *inst++ = OR_rm8_r8;
2918 *inst++ = MOD_REG | (0 /* eax */ << 3) | reg_map[dst];
2919 *inst++ = U8(XCHG_EAX_r | reg_map[TMP_REG1]);
2920 }
2921 else {
2922 inst = (sljit_u8*)ensure_buf(compiler, 1 + 2 + 3 + 2 + 2);
2923 FAIL_IF(!inst);
2924 INC_SIZE(2 + 3 + 2 + 2);
2925 /* Set low register to conditional flag. */
2926 *inst++ = XCHG_r_rm;
2927 *inst++ = U8(MOD_REG | (1 /* ecx */ << 3) | reg_map[TMP_REG1]);
2928 *inst++ = GROUP_0F;
2929 *inst++ = cond_set;
2930 *inst++ = MOD_REG | 1 /* ecx */;
2931 *inst++ = OR_rm8_r8;
2932 *inst++ = MOD_REG | (1 /* ecx */ << 3) | 0 /* eax */;
2933 *inst++ = XCHG_r_rm;
2934 *inst++ = U8(MOD_REG | (1 /* ecx */ << 3) | reg_map[TMP_REG1]);
2935 }
2936 return SLJIT_SUCCESS;
2937 }
2938
2939 /* Set TMP_REG1 to the bit. */
2940 inst = (sljit_u8*)ensure_buf(compiler, 1 + 1 + 3 + 3 + 1);
2941 FAIL_IF(!inst);
2942 INC_SIZE(1 + 3 + 3 + 1);
2943 *inst++ = U8(XCHG_EAX_r | reg_map[TMP_REG1]);
2944 /* Set al to conditional flag. */
2945 *inst++ = GROUP_0F;
2946 *inst++ = cond_set;
2947 *inst++ = MOD_REG | 0 /* eax */;
2948
2949 *inst++ = GROUP_0F;
2950 *inst++ = MOVZX_r_rm8;
2951 *inst++ = MOD_REG | (0 << 3) /* eax */ | 0 /* eax */;
2952
2953 *inst++ = U8(XCHG_EAX_r | reg_map[TMP_REG1]);
2954
2955 if (GET_OPCODE(op) < SLJIT_ADD)
2956 return emit_mov(compiler, dst, dstw, TMP_REG1, 0);
2957
2958 SLJIT_SKIP_CHECKS(compiler);
2959 return sljit_emit_op2(compiler, op, dst_save, dstw_save, dst_save, dstw_save, TMP_REG1, 0);
2960 #endif /* SLJIT_CONFIG_X86_64 */
2961 }
2962
2963 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_cmov(struct sljit_compiler *compiler, sljit_s32 type,
2964 sljit_s32 dst_reg,
2965 sljit_s32 src, sljit_sw srcw)
2966 {
2967 sljit_u8* inst;
2968
2969 CHECK_ERROR();
2970 CHECK(check_sljit_emit_cmov(compiler, type, dst_reg, src, srcw));
2971
2972 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
2973 dst_reg &= ~SLJIT_32;
2974
2975 if (!sljit_has_cpu_feature(SLJIT_HAS_CMOV) || (dst_reg >= SLJIT_R3 && dst_reg <= SLJIT_S3))
2976 return sljit_emit_cmov_generic(compiler, type, dst_reg, src, srcw);
2977 #else
2978 if (!sljit_has_cpu_feature(SLJIT_HAS_CMOV))
2979 return sljit_emit_cmov_generic(compiler, type, dst_reg, src, srcw);
2980 #endif
2981
2982 /* ADJUST_LOCAL_OFFSET is not needed. */
2983 CHECK_EXTRA_REGS(src, srcw, (void)0);
2984
2985 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2986 compiler->mode32 = dst_reg & SLJIT_32;
2987 dst_reg &= ~SLJIT_32;
2988 #endif
2989
2990 if (SLJIT_UNLIKELY(src & SLJIT_IMM)) {
2991 EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_IMM, srcw);
2992 src = TMP_REG1;
2993 srcw = 0;
2994 }
2995
2996 inst = emit_x86_instruction(compiler, 2, dst_reg, 0, src, srcw);
2997 FAIL_IF(!inst);
2998 *inst++ = GROUP_0F;
2999 *inst = U8(get_jump_code((sljit_uw)type) - 0x40);
3000 return SLJIT_SUCCESS;
3001 }
3002
3003 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_get_local_base(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw, sljit_sw offset)
3004 {
3005 CHECK_ERROR();
3006 CHECK(check_sljit_get_local_base(compiler, dst, dstw, offset));
3007 ADJUST_LOCAL_OFFSET(dst, dstw);
3008
3009 CHECK_EXTRA_REGS(dst, dstw, (void)0);
3010
3011 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
3012 compiler->mode32 = 0;
3013 #endif
3014
3015 ADJUST_LOCAL_OFFSET(SLJIT_MEM1(SLJIT_SP), offset);
3016
3017 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
3018 if (NOT_HALFWORD(offset)) {
3019 FAIL_IF(emit_load_imm64(compiler, TMP_REG1, offset));
3020 #if (defined SLJIT_DEBUG && SLJIT_DEBUG)
3021 SLJIT_ASSERT(emit_lea_binary(compiler, dst, dstw, SLJIT_SP, 0, TMP_REG1, 0) != SLJIT_ERR_UNSUPPORTED);
3022 return compiler->error;
3023 #else
3024 return emit_lea_binary(compiler, dst, dstw, SLJIT_SP, 0, TMP_REG1, 0);
3025 #endif
3026 }
3027 #endif
3028
3029 if (offset != 0)
3030 return emit_lea_binary(compiler, dst, dstw, SLJIT_SP, 0, SLJIT_IMM, offset);
3031 return emit_mov(compiler, dst, dstw, SLJIT_SP, 0);
3032 }
3033
3034 SLJIT_API_FUNC_ATTRIBUTE struct sljit_const* sljit_emit_const(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw, sljit_sw init_value)
3035 {
3036 sljit_u8 *inst;
3037 struct sljit_const *const_;
3038 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
3039 sljit_s32 reg;
3040 #endif
3041
3042 CHECK_ERROR_PTR();
3043 CHECK_PTR(check_sljit_emit_const(compiler, dst, dstw, init_value));
3044 ADJUST_LOCAL_OFFSET(dst, dstw);
3045
3046 CHECK_EXTRA_REGS(dst, dstw, (void)0);
3047
3048 const_ = (struct sljit_const*)ensure_abuf(compiler, sizeof(struct sljit_const));
3049 PTR_FAIL_IF(!const_);
3050 set_const(const_, compiler);
3051
3052 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
3053 compiler->mode32 = 0;
3054 reg = FAST_IS_REG(dst) ? dst : TMP_REG1;
3055
3056 if (emit_load_imm64(compiler, reg, init_value))
3057 return NULL;
3058 #else
3059 if (emit_mov(compiler, dst, dstw, SLJIT_IMM, init_value))
3060 return NULL;
3061 #endif
3062
3063 inst = (sljit_u8*)ensure_buf(compiler, 2);
3064 PTR_FAIL_IF(!inst);
3065
3066 *inst++ = 0;
3067 *inst++ = 2;
3068
3069 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
3070 if (dst & SLJIT_MEM)
3071 if (emit_mov(compiler, dst, dstw, TMP_REG1, 0))
3072 return NULL;
3073 #endif
3074
3075 return const_;
3076 }
3077
3078 SLJIT_API_FUNC_ATTRIBUTE struct sljit_put_label* sljit_emit_put_label(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw)
3079 {
3080 struct sljit_put_label *put_label;
3081 sljit_u8 *inst;
3082 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
3083 sljit_s32 reg;
3084 sljit_uw start_size;
3085 #endif
3086
3087 CHECK_ERROR_PTR();
3088 CHECK_PTR(check_sljit_emit_put_label(compiler, dst, dstw));
3089 ADJUST_LOCAL_OFFSET(dst, dstw);
3090
3091 CHECK_EXTRA_REGS(dst, dstw, (void)0);
3092
3093 put_label = (struct sljit_put_label*)ensure_abuf(compiler, sizeof(struct sljit_put_label));
3094 PTR_FAIL_IF(!put_label);
3095 set_put_label(put_label, compiler, 0);
3096
3097 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
3098 compiler->mode32 = 0;
3099 reg = FAST_IS_REG(dst) ? dst : TMP_REG1;
3100
3101 if (emit_load_imm64(compiler, reg, 0))
3102 return NULL;
3103 #else
3104 if (emit_mov(compiler, dst, dstw, SLJIT_IMM, 0))
3105 return NULL;
3106 #endif
3107
3108 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
3109 if (dst & SLJIT_MEM) {
3110 start_size = compiler->size;
3111 if (emit_mov(compiler, dst, dstw, TMP_REG1, 0))
3112 return NULL;
3113 put_label->flags = compiler->size - start_size;
3114 }
3115 #endif
3116
3117 inst = (sljit_u8*)ensure_buf(compiler, 2);
3118 PTR_FAIL_IF(!inst);
3119
3120 *inst++ = 0;
3121 *inst++ = 3;
3122
3123 return put_label;
3124 }
3125
3126 SLJIT_API_FUNC_ATTRIBUTE void sljit_set_jump_addr(sljit_uw addr, sljit_uw new_target, sljit_sw executable_offset)
3127 {
3128 SLJIT_UNUSED_ARG(executable_offset);
3129
3130 SLJIT_UPDATE_WX_FLAGS((void*)addr, (void*)(addr + sizeof(sljit_uw)), 0);
3131 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
3132 sljit_unaligned_store_sw((void*)addr, (sljit_sw)(new_target - (addr + 4) - (sljit_uw)executable_offset));
3133 #else
3134 sljit_unaligned_store_sw((void*)addr, (sljit_sw)new_target);
3135 #endif
3136 SLJIT_UPDATE_WX_FLAGS((void*)addr, (void*)(addr + sizeof(sljit_uw)), 1);
3137 }
3138
3139 SLJIT_API_FUNC_ATTRIBUTE void sljit_set_const(sljit_uw addr, sljit_sw new_constant, sljit_sw executable_offset)
3140 {
3141 SLJIT_UNUSED_ARG(executable_offset);
3142
3143 SLJIT_UPDATE_WX_FLAGS((void*)addr, (void*)(addr + sizeof(sljit_sw)), 0);
3144 sljit_unaligned_store_sw((void*)addr, new_constant);
3145 SLJIT_UPDATE_WX_FLAGS((void*)addr, (void*)(addr + sizeof(sljit_sw)), 1);
3146 }
Platform independent low-level JIT compiler (github mirror, with mob branch)