?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Replacing constants with instruction names in x86. Greatly improves maintainability.
[sljit.git] / sljit_src / sljitNativeX86_common.c
blobaa2b12b3e9012f59612d73390989921302daf0d3
1 /*
2 * Stack-less Just-In-Time compiler
3 *
4 * Copyright 2009-2012 Zoltan Herczeg (hzmester@freemail.hu). All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without modification, are
7 * permitted provided that the following conditions are met:
8 *
9 * 1. Redistributions of source code must retain the above copyright notice, this list of
10 * conditions and the following disclaimer.
11 *
12 * 2. Redistributions in binary form must reproduce the above copyright notice, this list
13 * of conditions and the following disclaimer in the documentation and/or other materials
14 * provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) AND CONTRIBUTORS ``AS IS'' AND ANY
17 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
19 * SHALL THE COPYRIGHT HOLDER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
21 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
22 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
23 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
24 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27 SLJIT_API_FUNC_ATTRIBUTE SLJIT_CONST char* sljit_get_platform_name(void)
28 {
29 return "x86" SLJIT_CPUINFO;
30 }
31
32 /*
33 32b register indexes:
34 0 - EAX
35 1 - ECX
36 2 - EDX
37 3 - EBX
38 4 - none
39 5 - EBP
40 6 - ESI
41 7 - EDI
42 */
43
44 /*
45 64b register indexes:
46 0 - RAX
47 1 - RCX
48 2 - RDX
49 3 - RBX
50 4 - none
51 5 - RBP
52 6 - RSI
53 7 - RDI
54 8 - R8 - From now on REX prefix is required
55 9 - R9
56 10 - R10
57 11 - R11
58 12 - R12
59 13 - R13
60 14 - R14
61 15 - R15
62 */
63
64 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
65
66 /* Last register + 1. */
67 #define TMP_REGISTER (SLJIT_NO_REGISTERS + 1)
68
69 static SLJIT_CONST sljit_ub reg_map[SLJIT_NO_REGISTERS + 2] = {
70 0, 0, 2, 1, 0, 0, 3, 6, 7, 0, 0, 4, 5
71 };
72
73 #define CHECK_EXTRA_REGS(p, w, do) \
74 if (p >= SLJIT_TEMPORARY_EREG1 && p <= SLJIT_TEMPORARY_EREG2) { \
75 w = compiler->temporaries_start + (p - SLJIT_TEMPORARY_EREG1) * sizeof(sljit_sw); \
76 p = SLJIT_MEM1(SLJIT_LOCALS_REG); \
77 do; \
78 } \
79 else if (p >= SLJIT_SAVED_EREG1 && p <= SLJIT_SAVED_EREG2) { \
80 w = compiler->saveds_start + (p - SLJIT_SAVED_EREG1) * sizeof(sljit_sw); \
81 p = SLJIT_MEM1(SLJIT_LOCALS_REG); \
82 do; \
83 }
84
85 #else /* SLJIT_CONFIG_X86_32 */
86
87 /* Last register + 1. */
88 #define TMP_REGISTER (SLJIT_NO_REGISTERS + 1)
89 #define TMP_REG2 (SLJIT_NO_REGISTERS + 2)
90 #define TMP_REG3 (SLJIT_NO_REGISTERS + 3)
91
92 /* Note: r12 & 0x7 == 0b100, which decoded as SIB byte present
93 Note: avoid to use r12 and r13 for memory addessing
94 therefore r12 is better for SAVED_EREG than SAVED_REG. */
95 #ifndef _WIN64
96 /* 1st passed in rdi, 2nd argument passed in rsi, 3rd in rdx. */
97 static SLJIT_CONST sljit_ub reg_map[SLJIT_NO_REGISTERS + 4] = {
98 0, 0, 6, 1, 8, 11, 3, 15, 14, 13, 12, 4, 2, 7, 9
99 };
100 /* low-map. reg_map & 0x7. */
101 static SLJIT_CONST sljit_ub reg_lmap[SLJIT_NO_REGISTERS + 4] = {
102 0, 0, 6, 1, 0, 3, 3, 7, 6, 5, 4, 4, 2, 7, 1
103 };
104 #else
105 /* 1st passed in rcx, 2nd argument passed in rdx, 3rd in r8. */
106 static SLJIT_CONST sljit_ub reg_map[SLJIT_NO_REGISTERS + 4] = {
107 0, 0, 2, 1, 11, 13, 3, 6, 7, 14, 15, 4, 10, 8, 9
108 };
109 /* low-map. reg_map & 0x7. */
110 static SLJIT_CONST sljit_ub reg_lmap[SLJIT_NO_REGISTERS + 4] = {
111 0, 0, 2, 1, 3, 5, 3, 6, 7, 6, 7, 4, 2, 0, 1
112 };
113 #endif
114
115 #define REX_W 0x48
116 #define REX_R 0x44
117 #define REX_X 0x42
118 #define REX_B 0x41
119 #define REX 0x40
120
121 #define IS_HALFWORD(x) ((x) <= 0x7fffffffll && (x) >= -0x80000000ll)
122 #define NOT_HALFWORD(x) ((x) > 0x7fffffffll || (x) < -0x80000000ll)
123
124 #define CHECK_EXTRA_REGS(p, w, do)
125
126 #endif /* SLJIT_CONFIG_X86_32 */
127
128 #if (defined SLJIT_SSE2 && SLJIT_SSE2)
129 #define TMP_FREG (0)
130 #endif
131
132 /* Size flags for emit_x86_instruction: */
133 #define EX86_BIN_INS 0x0010
134 #define EX86_SHIFT_INS 0x0020
135 #define EX86_REX 0x0040
136 #define EX86_NO_REXW 0x0080
137 #define EX86_BYTE_ARG 0x0100
138 #define EX86_HALF_ARG 0x0200
139 #define EX86_PREF_66 0x0400
140
141 #if (defined SLJIT_SSE2 && SLJIT_SSE2)
142 #define EX86_SSE2 0x0800
143 #define EX86_PREF_F2 0x1000
144 #define EX86_PREF_F3 0x2000
145 #endif
146
147 /* --------------------------------------------------------------------- */
148 /* Instrucion forms */
149 /* --------------------------------------------------------------------- */
150
151 #define ADD (/* BINARY */ 0 << 3)
152 #define ADD_EAX_i32 0x05
153 #define ADD_r_rm 0x03
154 #define ADD_rm_r 0x01
155 #define ADDSD_x_xm 0x58
156 #define ADC (/* BINARY */ 2 << 3)
157 #define ADC_EAX_i32 0x15
158 #define ADC_r_rm 0x13
159 #define ADC_rm_r 0x11
160 #define AND (/* BINARY */ 4 << 3)
161 #define AND_EAX_i32 0x25
162 #define AND_r_rm 0x23
163 #define AND_rm_r 0x21
164 #define ANDPD_x_xm 0x54
165 #define BSR_r_rm (/* GROUP_0F */ 0xbd)
166 #define CALL_i32 0xe8
167 #define CALL_rm (/* GROUP_FF */ 2 << 3)
168 #define CDQ 0x99
169 #define CMOVNE_r_rm (/* GROUP_0F */ 0x45)
170 #define CMP (/* BINARY */ 7 << 3)
171 #define CMP_EAX_i32 0x3d
172 #define CMP_r_rm 0x3b
173 #define CMP_rm_r 0x39
174 #define DIV (/* GROUP_F7 */ 6 << 3)
175 #define DIVSD_x_xm 0x5e
176 #define INT3 0xcc
177 #define IDIV (/* GROUP_F7 */ 7 << 3)
178 #define IMUL (/* GROUP_F7 */ 5 << 3)
179 #define IMUL_r_rm (/* GROUP_0F */ 0xaf)
180 #define IMUL_r_rm_i8 0x6b
181 #define IMUL_r_rm_i32 0x69
182 #define JMP_i8 0xeb
183 #define JMP_i32 0xe9
184 #define JMP_rm (/* GROUP_FF */ 4 << 3)
185 #define LEA_r_m 0x8d
186 #define MOV_r_rm 0x8b
187 #define MOV_r_i32 0xb8
188 #define MOV_rm_r 0x89
189 #define MOV_rm_i32 0xc7
190 #define MOV_rm8_i8 0xc6
191 #define MOV_rm8_r8 0x88
192 #define MOVSD_x_xm 0x10
193 #define MOVSD_xm_x 0x11
194 #define MOVSXD_r_rm 0x63
195 #define MOVSX_r_rm8 (/* GROUP_0F */ 0xbe)
196 #define MOVSX_r_rm16 (/* GROUP_0F */ 0xbf)
197 #define MOVZX_r_rm8 (/* GROUP_0F */ 0xb6)
198 #define MOVZX_r_rm16 (/* GROUP_0F */ 0xb7)
199 #define MUL (/* GROUP_F7 */ 4 << 3)
200 #define MULSD_x_xm 0x59
201 #define NEG_rm (/* GROUP_F7 */ 3 << 3)
202 #define NOP 0x90
203 #define NOT_rm (/* GROUP_F7 */ 2 << 3)
204 #define OR (/* BINARY */ 1 << 3)
205 #define OR_r_rm 0x0b
206 #define OR_EAX_i32 0x0d
207 #define OR_rm_r 0x09
208 #define POP_r 0x58
209 #define POP_rm 0x8f
210 #define POPF 0x9d
211 #define PUSH_i32 0x68
212 #define PUSH_r 0x50
213 #define PUSH_rm (/* GROUP_FF */ 6 << 3)
214 #define PUSHF 0x9c
215 #define RET_near 0xc3
216 #define RET_i16 0xc2
217 #define SBB (/* BINARY */ 3 << 3)
218 #define SBB_EAX_i32 0x1d
219 #define SBB_r_rm 0x1b
220 #define SBB_rm_r 0x19
221 #define SAR (/* SHIFT */ 7 << 3)
222 #define SHL (/* SHIFT */ 4 << 3)
223 #define SHR (/* SHIFT */ 5 << 3)
224 #define SUB (/* BINARY */ 5 << 3)
225 #define SUB_EAX_i32 0x2d
226 #define SUB_r_rm 0x2b
227 #define SUB_rm_r 0x29
228 #define SUBSD_x_xm 0x5c
229 #define TEST_EAX_i32 0xa9
230 #define TEST_rm_r 0x85
231 #define UCOMISD_x_xm 0x2e
232 #define XCHG_EAX_r 0x90
233 #define XCHG_r_rm 0x87
234 #define XOR (/* BINARY */ 6 << 3)
235 #define XOR_EAX_i32 0x35
236 #define XOR_r_rm 0x33
237 #define XOR_rm_r 0x31
238 #define XORPD_x_xm 0x57
239
240 #define GROUP_0F 0x0f
241 #define GROUP_F7 0xf7
242 #define GROUP_FF 0xff
243 #define GROUP_BIN_81 0x81
244 #define GROUP_BIN_83 0x83
245
246 #define MOD_REG 0xc0
247
248 #define INC_SIZE(s) (*buf++ = (s), compiler->size += (s))
249 #define INC_CSIZE(s) (*code++ = (s), compiler->size += (s))
250
251 #define PUSH_REG(r) (*buf++ = (PUSH_r + (r)))
252 #define POP_REG(r) (*buf++ = (POP_r + (r)))
253 #define RET() (*buf++ = (RET_near))
254 #define RET_I16(n) (*buf++ = (RET_i16), *buf++ = n, *buf++ = 0)
255 /* r32, r/m32 */
256 #define MOV_RM(mod, reg, rm) (*buf++ = (MOV_r_rm), *buf++ = (mod) << 6 | (reg) << 3 | (rm))
257
258 static sljit_ub get_jump_code(sljit_si type)
259 {
260 switch (type) {
261 case SLJIT_C_EQUAL:
262 case SLJIT_C_FLOAT_EQUAL:
263 return 0x84 /* je */;
264
265 case SLJIT_C_NOT_EQUAL:
266 case SLJIT_C_FLOAT_NOT_EQUAL:
267 return 0x85 /* jne */;
268
269 case SLJIT_C_LESS:
270 case SLJIT_C_FLOAT_LESS:
271 return 0x82 /* jc */;
272
273 case SLJIT_C_GREATER_EQUAL:
274 case SLJIT_C_FLOAT_GREATER_EQUAL:
275 return 0x83 /* jae */;
276
277 case SLJIT_C_GREATER:
278 case SLJIT_C_FLOAT_GREATER:
279 return 0x87 /* jnbe */;
280
281 case SLJIT_C_LESS_EQUAL:
282 case SLJIT_C_FLOAT_LESS_EQUAL:
283 return 0x86 /* jbe */;
284
285 case SLJIT_C_SIG_LESS:
286 return 0x8c /* jl */;
287
288 case SLJIT_C_SIG_GREATER_EQUAL:
289 return 0x8d /* jnl */;
290
291 case SLJIT_C_SIG_GREATER:
292 return 0x8f /* jnle */;
293
294 case SLJIT_C_SIG_LESS_EQUAL:
295 return 0x8e /* jle */;
296
297 case SLJIT_C_OVERFLOW:
298 case SLJIT_C_MUL_OVERFLOW:
299 return 0x80 /* jo */;
300
301 case SLJIT_C_NOT_OVERFLOW:
302 case SLJIT_C_MUL_NOT_OVERFLOW:
303 return 0x81 /* jno */;
304
305 case SLJIT_C_FLOAT_UNORDERED:
306 return 0x8a /* jp */;
307
308 case SLJIT_C_FLOAT_ORDERED:
309 return 0x8b /* jpo */;
310 }
311 return 0;
312 }
313
314 static sljit_ub* generate_far_jump_code(struct sljit_jump *jump, sljit_ub *code_ptr, sljit_si type);
315
316 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
317 static sljit_ub* generate_fixed_jump(sljit_ub *code_ptr, sljit_sw addr, sljit_si type);
318 #endif
319
320 static sljit_ub* generate_near_jump_code(struct sljit_jump *jump, sljit_ub *code_ptr, sljit_ub *code, sljit_si type)
321 {
322 sljit_si short_jump;
323 sljit_uw label_addr;
324
325 if (jump->flags & JUMP_LABEL)
326 label_addr = (sljit_uw)(code + jump->u.label->size);
327 else
328 label_addr = jump->u.target;
329 short_jump = (sljit_sw)(label_addr - (jump->addr + 2)) >= -128 && (sljit_sw)(label_addr - (jump->addr + 2)) <= 127;
330
331 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
332 if ((sljit_sw)(label_addr - (jump->addr + 1)) > 0x7fffffffll || (sljit_sw)(label_addr - (jump->addr + 1)) < -0x80000000ll)
333 return generate_far_jump_code(jump, code_ptr, type);
334 #endif
335
336 if (type == SLJIT_JUMP) {
337 if (short_jump)
338 *code_ptr++ = JMP_i8;
339 else
340 *code_ptr++ = JMP_i32;
341 jump->addr++;
342 }
343 else if (type >= SLJIT_FAST_CALL) {
344 short_jump = 0;
345 *code_ptr++ = CALL_i32;
346 jump->addr++;
347 }
348 else if (short_jump) {
349 *code_ptr++ = get_jump_code(type) - 0x10;
350 jump->addr++;
351 }
352 else {
353 *code_ptr++ = GROUP_0F;
354 *code_ptr++ = get_jump_code(type);
355 jump->addr += 2;
356 }
357
358 if (short_jump) {
359 jump->flags |= PATCH_MB;
360 code_ptr += sizeof(sljit_sb);
361 } else {
362 jump->flags |= PATCH_MW;
363 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
364 code_ptr += sizeof(sljit_sw);
365 #else
366 code_ptr += sizeof(sljit_si);
367 #endif
368 }
369
370 return code_ptr;
371 }
372
373 SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compiler)
374 {
375 struct sljit_memory_fragment *buf;
376 sljit_ub *code;
377 sljit_ub *code_ptr;
378 sljit_ub *buf_ptr;
379 sljit_ub *buf_end;
380 sljit_ub len;
381
382 struct sljit_label *label;
383 struct sljit_jump *jump;
384 struct sljit_const *const_;
385
386 CHECK_ERROR_PTR();
387 check_sljit_generate_code(compiler);
388 reverse_buf(compiler);
389
390 /* Second code generation pass. */
391 code = (sljit_ub*)SLJIT_MALLOC_EXEC(compiler->size);
392 PTR_FAIL_WITH_EXEC_IF(code);
393 buf = compiler->buf;
394
395 code_ptr = code;
396 label = compiler->labels;
397 jump = compiler->jumps;
398 const_ = compiler->consts;
399 do {
400 buf_ptr = buf->memory;
401 buf_end = buf_ptr + buf->used_size;
402 do {
403 len = *buf_ptr++;
404 if (len > 0) {
405 /* The code is already generated. */
406 SLJIT_MEMMOVE(code_ptr, buf_ptr, len);
407 code_ptr += len;
408 buf_ptr += len;
409 }
410 else {
411 if (*buf_ptr >= 4) {
412 jump->addr = (sljit_uw)code_ptr;
413 if (!(jump->flags & SLJIT_REWRITABLE_JUMP))
414 code_ptr = generate_near_jump_code(jump, code_ptr, code, *buf_ptr - 4);
415 else
416 code_ptr = generate_far_jump_code(jump, code_ptr, *buf_ptr - 4);
417 jump = jump->next;
418 }
419 else if (*buf_ptr == 0) {
420 label->addr = (sljit_uw)code_ptr;
421 label->size = code_ptr - code;
422 label = label->next;
423 }
424 else if (*buf_ptr == 1) {
425 const_->addr = ((sljit_uw)code_ptr) - sizeof(sljit_sw);
426 const_ = const_->next;
427 }
428 else {
429 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
430 *code_ptr++ = (*buf_ptr == 2) ? CALL_i32 : JMP_i32;
431 buf_ptr++;
432 *(sljit_sw*)code_ptr = *(sljit_sw*)buf_ptr - ((sljit_sw)code_ptr + sizeof(sljit_sw));
433 code_ptr += sizeof(sljit_sw);
434 buf_ptr += sizeof(sljit_sw) - 1;
435 #else
436 code_ptr = generate_fixed_jump(code_ptr, *(sljit_sw*)(buf_ptr + 1), *buf_ptr);
437 buf_ptr += sizeof(sljit_sw);
438 #endif
439 }
440 buf_ptr++;
441 }
442 } while (buf_ptr < buf_end);
443 SLJIT_ASSERT(buf_ptr == buf_end);
444 buf = buf->next;
445 } while (buf);
446
447 SLJIT_ASSERT(!label);
448 SLJIT_ASSERT(!jump);
449 SLJIT_ASSERT(!const_);
450
451 jump = compiler->jumps;
452 while (jump) {
453 if (jump->flags & PATCH_MB) {
454 SLJIT_ASSERT((sljit_sw)(jump->u.label->addr - (jump->addr + sizeof(sljit_sb))) >= -128 && (sljit_sw)(jump->u.label->addr - (jump->addr + sizeof(sljit_sb))) <= 127);
455 *(sljit_ub*)jump->addr = (sljit_ub)(jump->u.label->addr - (jump->addr + sizeof(sljit_sb)));
456 } else if (jump->flags & PATCH_MW) {
457 if (jump->flags & JUMP_LABEL) {
458 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
459 *(sljit_sw*)jump->addr = (sljit_sw)(jump->u.label->addr - (jump->addr + sizeof(sljit_sw)));
460 #else
461 SLJIT_ASSERT((sljit_sw)(jump->u.label->addr - (jump->addr + sizeof(sljit_si))) >= -0x80000000ll && (sljit_sw)(jump->u.label->addr - (jump->addr + sizeof(sljit_si))) <= 0x7fffffffll);
462 *(sljit_si*)jump->addr = (sljit_si)(jump->u.label->addr - (jump->addr + sizeof(sljit_si)));
463 #endif
464 }
465 else {
466 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
467 *(sljit_sw*)jump->addr = (sljit_sw)(jump->u.target - (jump->addr + sizeof(sljit_sw)));
468 #else
469 SLJIT_ASSERT((sljit_sw)(jump->u.target - (jump->addr + sizeof(sljit_si))) >= -0x80000000ll && (sljit_sw)(jump->u.target - (jump->addr + sizeof(sljit_si))) <= 0x7fffffffll);
470 *(sljit_si*)jump->addr = (sljit_si)(jump->u.target - (jump->addr + sizeof(sljit_si)));
471 #endif
472 }
473 }
474 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
475 else if (jump->flags & PATCH_MD)
476 *(sljit_sw*)jump->addr = jump->u.label->addr;
477 #endif
478
479 jump = jump->next;
480 }
481
482 /* Maybe we waste some space because of short jumps. */
483 SLJIT_ASSERT(code_ptr <= code + compiler->size);
484 compiler->error = SLJIT_ERR_COMPILED;
485 compiler->executable_size = compiler->size;
486 return (void*)code;
487 }
488
489 /* --------------------------------------------------------------------- */
490 /* Operators */
491 /* --------------------------------------------------------------------- */
492
493 static sljit_si emit_cum_binary(struct sljit_compiler *compiler,
494 sljit_ub op_rm, sljit_ub op_mr, sljit_ub op_imm, sljit_ub op_eax_imm,
495 sljit_si dst, sljit_sw dstw,
496 sljit_si src1, sljit_sw src1w,
497 sljit_si src2, sljit_sw src2w);
498
499 static sljit_si emit_non_cum_binary(struct sljit_compiler *compiler,
500 sljit_ub op_rm, sljit_ub op_mr, sljit_ub op_imm, sljit_ub op_eax_imm,
501 sljit_si dst, sljit_sw dstw,
502 sljit_si src1, sljit_sw src1w,
503 sljit_si src2, sljit_sw src2w);
504
505 static sljit_si emit_mov(struct sljit_compiler *compiler,
506 sljit_si dst, sljit_sw dstw,
507 sljit_si src, sljit_sw srcw);
508
509 static SLJIT_INLINE sljit_si emit_save_flags(struct sljit_compiler *compiler)
510 {
511 sljit_ub *buf;
512
513 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
514 buf = (sljit_ub*)ensure_buf(compiler, 1 + 5);
515 FAIL_IF(!buf);
516 INC_SIZE(5);
517 #else
518 buf = (sljit_ub*)ensure_buf(compiler, 1 + 6);
519 FAIL_IF(!buf);
520 INC_SIZE(6);
521 *buf++ = REX_W;
522 #endif
523 *buf++ = LEA_r_m; /* lea esp/rsp, [esp/rsp + sizeof(sljit_sw)] */
524 *buf++ = 0x64;
525 *buf++ = 0x24;
526 *buf++ = (sljit_ub)sizeof(sljit_sw);
527 *buf++ = PUSHF;
528 compiler->flags_saved = 1;
529 return SLJIT_SUCCESS;
530 }
531
532 static SLJIT_INLINE sljit_si emit_restore_flags(struct sljit_compiler *compiler, sljit_si keep_flags)
533 {
534 sljit_ub *buf;
535
536 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
537 buf = (sljit_ub*)ensure_buf(compiler, 1 + 5);
538 FAIL_IF(!buf);
539 INC_SIZE(5);
540 *buf++ = POPF;
541 #else
542 buf = (sljit_ub*)ensure_buf(compiler, 1 + 6);
543 FAIL_IF(!buf);
544 INC_SIZE(6);
545 *buf++ = POPF;
546 *buf++ = REX_W;
547 #endif
548 *buf++ = LEA_r_m; /* lea esp/rsp, [esp/rsp - sizeof(sljit_sw)] */
549 *buf++ = 0x64;
550 *buf++ = 0x24;
551 *buf++ = (sljit_ub)-(sljit_sb)sizeof(sljit_sw);
552 compiler->flags_saved = keep_flags;
553 return SLJIT_SUCCESS;
554 }
555
556 #ifdef _WIN32
557 #include <malloc.h>
558
559 static void SLJIT_CALL sljit_grow_stack(sljit_sw local_size)
560 {
561 /* Workaround for calling the internal _chkstk() function on Windows.
562 This function touches all 4k pages belongs to the requested stack space,
563 which size is passed in local_size. This is necessary on Windows where
564 the stack can only grow in 4k steps. However, this function just burn
565 CPU cycles if the stack is large enough, but you don't know it in advance.
566 I think this is a bad design even if it has some reasons. */
567 alloca(local_size);
568 }
569
570 #endif
571
572 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
573 #include "sljitNativeX86_32.c"
574 #else
575 #include "sljitNativeX86_64.c"
576 #endif
577
578 static sljit_si emit_mov(struct sljit_compiler *compiler,
579 sljit_si dst, sljit_sw dstw,
580 sljit_si src, sljit_sw srcw)
581 {
582 sljit_ub* code;
583
584 if (dst == SLJIT_UNUSED) {
585 /* No destination, doesn't need to setup flags. */
586 if (src & SLJIT_MEM) {
587 code = emit_x86_instruction(compiler, 1, TMP_REGISTER, 0, src, srcw);
588 FAIL_IF(!code);
589 *code = MOV_r_rm;
590 }
591 return SLJIT_SUCCESS;
592 }
593 if (src >= SLJIT_TEMPORARY_REG1 && src <= TMP_REGISTER) {
594 code = emit_x86_instruction(compiler, 1, src, 0, dst, dstw);
595 FAIL_IF(!code);
596 *code = MOV_rm_r;
597 return SLJIT_SUCCESS;
598 }
599 if (src & SLJIT_IMM) {
600 if (dst >= SLJIT_TEMPORARY_REG1 && dst <= TMP_REGISTER) {
601 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
602 return emit_do_imm(compiler, MOV_r_i32 + reg_map[dst], srcw);
603 #else
604 if (!compiler->mode32) {
605 if (NOT_HALFWORD(srcw))
606 return emit_load_imm64(compiler, dst, srcw);
607 }
608 else
609 return emit_do_imm32(compiler, (reg_map[dst] >= 8) ? REX_B : 0, MOV_r_i32 + reg_lmap[dst], srcw);
610 #endif
611 }
612 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
613 if (!compiler->mode32 && NOT_HALFWORD(srcw)) {
614 FAIL_IF(emit_load_imm64(compiler, TMP_REG2, srcw));
615 code = emit_x86_instruction(compiler, 1, TMP_REG2, 0, dst, dstw);
616 FAIL_IF(!code);
617 *code = MOV_rm_r;
618 return SLJIT_SUCCESS;
619 }
620 #endif
621 code = emit_x86_instruction(compiler, 1, SLJIT_IMM, srcw, dst, dstw);
622 FAIL_IF(!code);
623 *code = MOV_rm_i32;
624 return SLJIT_SUCCESS;
625 }
626 if (dst >= SLJIT_TEMPORARY_REG1 && dst <= TMP_REGISTER) {
627 code = emit_x86_instruction(compiler, 1, dst, 0, src, srcw);
628 FAIL_IF(!code);
629 *code = MOV_r_rm;
630 return SLJIT_SUCCESS;
631 }
632
633 /* Memory to memory move. Requires two instruction. */
634 code = emit_x86_instruction(compiler, 1, TMP_REGISTER, 0, src, srcw);
635 FAIL_IF(!code);
636 *code = MOV_r_rm;
637 code = emit_x86_instruction(compiler, 1, TMP_REGISTER, 0, dst, dstw);
638 FAIL_IF(!code);
639 *code = MOV_rm_r;
640 return SLJIT_SUCCESS;
641 }
642
643 #define EMIT_MOV(compiler, dst, dstw, src, srcw) \
644 FAIL_IF(emit_mov(compiler, dst, dstw, src, srcw));
645
646 SLJIT_API_FUNC_ATTRIBUTE sljit_si sljit_emit_op0(struct sljit_compiler *compiler, sljit_si op)
647 {
648 sljit_ub *buf;
649 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
650 sljit_si size;
651 #endif
652
653 CHECK_ERROR();
654 check_sljit_emit_op0(compiler, op);
655
656 switch (GET_OPCODE(op)) {
657 case SLJIT_BREAKPOINT:
658 buf = (sljit_ub*)ensure_buf(compiler, 1 + 1);
659 FAIL_IF(!buf);
660 INC_SIZE(1);
661 *buf = INT3;
662 break;
663 case SLJIT_NOP:
664 buf = (sljit_ub*)ensure_buf(compiler, 1 + 1);
665 FAIL_IF(!buf);
666 INC_SIZE(1);
667 *buf = NOP;
668 break;
669 case SLJIT_UMUL:
670 case SLJIT_SMUL:
671 case SLJIT_UDIV:
672 case SLJIT_SDIV:
673 compiler->flags_saved = 0;
674 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
675 #ifdef _WIN64
676 SLJIT_COMPILE_ASSERT(
677 reg_map[SLJIT_TEMPORARY_REG1] == 0
678 && reg_map[SLJIT_TEMPORARY_REG2] == 2
679 && reg_map[TMP_REGISTER] > 7,
680 invalid_register_assignment_for_div_mul);
681 #else
682 SLJIT_COMPILE_ASSERT(
683 reg_map[SLJIT_TEMPORARY_REG1] == 0
684 && reg_map[SLJIT_TEMPORARY_REG2] < 7
685 && reg_map[TMP_REGISTER] == 2,
686 invalid_register_assignment_for_div_mul);
687 #endif
688 compiler->mode32 = op & SLJIT_INT_OP;
689 #endif
690
691 op = GET_OPCODE(op);
692 if (op == SLJIT_UDIV) {
693 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32) || defined(_WIN64)
694 EMIT_MOV(compiler, TMP_REGISTER, 0, SLJIT_TEMPORARY_REG2, 0);
695 buf = emit_x86_instruction(compiler, 1, SLJIT_TEMPORARY_REG2, 0, SLJIT_TEMPORARY_REG2, 0);
696 #else
697 buf = emit_x86_instruction(compiler, 1, TMP_REGISTER, 0, TMP_REGISTER, 0);
698 #endif
699 FAIL_IF(!buf);
700 *buf = XOR_r_rm;
701 }
702
703 if (op == SLJIT_SDIV) {
704 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32) || defined(_WIN64)
705 EMIT_MOV(compiler, TMP_REGISTER, 0, SLJIT_TEMPORARY_REG2, 0);
706 #endif
707
708 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
709 buf = (sljit_ub*)ensure_buf(compiler, 1 + 1);
710 FAIL_IF(!buf);
711 INC_SIZE(1);
712 *buf = CDQ;
713 #else
714 if (compiler->mode32) {
715 buf = (sljit_ub*)ensure_buf(compiler, 1 + 1);
716 FAIL_IF(!buf);
717 INC_SIZE(1);
718 *buf = CDQ;
719 } else {
720 buf = (sljit_ub*)ensure_buf(compiler, 1 + 2);
721 FAIL_IF(!buf);
722 INC_SIZE(2);
723 *buf++ = REX_W;
724 *buf = CDQ;
725 }
726 #endif
727 }
728
729 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
730 buf = (sljit_ub*)ensure_buf(compiler, 1 + 2);
731 FAIL_IF(!buf);
732 INC_SIZE(2);
733 *buf++ = GROUP_F7;
734 *buf = MOD_REG | ((op >= SLJIT_UDIV) ? reg_map[TMP_REGISTER] : reg_map[SLJIT_TEMPORARY_REG2]);
735 #else
736 #ifdef _WIN64
737 size = (!compiler->mode32 || op >= SLJIT_UDIV) ? 3 : 2;
738 #else
739 size = (!compiler->mode32) ? 3 : 2;
740 #endif
741 buf = (sljit_ub*)ensure_buf(compiler, 1 + size);
742 FAIL_IF(!buf);
743 INC_SIZE(size);
744 #ifdef _WIN64
745 if (!compiler->mode32)
746 *buf++ = REX_W | ((op >= SLJIT_UDIV) ? REX_B : 0);
747 else if (op >= SLJIT_UDIV)
748 *buf++ = REX_B;
749 *buf++ = GROUP_F7;
750 *buf = MOD_REG | ((op >= SLJIT_UDIV) ? reg_lmap[TMP_REGISTER] : reg_lmap[SLJIT_TEMPORARY_REG2]);
751 #else
752 if (!compiler->mode32)
753 *buf++ = REX_W;
754 *buf++ = GROUP_F7;
755 *buf = MOD_REG | reg_map[SLJIT_TEMPORARY_REG2];
756 #endif
757 #endif
758 switch (op) {
759 case SLJIT_UMUL:
760 *buf |= MUL;
761 break;
762 case SLJIT_SMUL:
763 *buf |= IMUL;
764 break;
765 case SLJIT_UDIV:
766 *buf |= DIV;
767 break;
768 case SLJIT_SDIV:
769 *buf |= IDIV;
770 break;
771 }
772 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64) && !defined(_WIN64)
773 EMIT_MOV(compiler, SLJIT_TEMPORARY_REG2, 0, TMP_REGISTER, 0);
774 #endif
775 break;
776 }
777
778 return SLJIT_SUCCESS;
779 }
780
781 #define ENCODE_PREFIX(prefix) \
782 do { \
783 code = (sljit_ub*)ensure_buf(compiler, 1 + 1); \
784 FAIL_IF(!code); \
785 INC_CSIZE(1); \
786 *code = (prefix); \
787 } while (0)
788
789 static sljit_si emit_mov_byte(struct sljit_compiler *compiler, sljit_si sign,
790 sljit_si dst, sljit_sw dstw,
791 sljit_si src, sljit_sw srcw)
792 {
793 sljit_ub* code;
794 sljit_si dst_r;
795 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
796 sljit_si work_r;
797 #endif
798
799 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
800 compiler->mode32 = 0;
801 #endif
802
803 if (dst == SLJIT_UNUSED && !(src & SLJIT_MEM))
804 return SLJIT_SUCCESS; /* Empty instruction. */
805
806 if (src & SLJIT_IMM) {
807 if (dst >= SLJIT_TEMPORARY_REG1 && dst <= TMP_REGISTER) {
808 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
809 return emit_do_imm(compiler, MOV_r_i32 + reg_map[dst], srcw);
810 #else
811 return emit_load_imm64(compiler, dst, srcw);
812 #endif
813 }
814 code = emit_x86_instruction(compiler, 1 | EX86_BYTE_ARG | EX86_NO_REXW, SLJIT_IMM, srcw, dst, dstw);
815 FAIL_IF(!code);
816 *code = MOV_rm8_i8;
817 return SLJIT_SUCCESS;
818 }
819
820 dst_r = (dst >= SLJIT_TEMPORARY_REG1 && dst <= TMP_REGISTER) ? dst : TMP_REGISTER;
821
822 if ((dst & SLJIT_MEM) && src >= SLJIT_TEMPORARY_REG1 && src <= SLJIT_NO_REGISTERS) {
823 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
824 if (reg_map[src] >= 4) {
825 SLJIT_ASSERT(dst_r == TMP_REGISTER);
826 EMIT_MOV(compiler, TMP_REGISTER, 0, src, 0);
827 } else
828 dst_r = src;
829 #else
830 dst_r = src;
831 #endif
832 }
833 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
834 else if (src >= SLJIT_TEMPORARY_REG1 && src <= SLJIT_NO_REGISTERS && reg_map[src] >= 4) {
835 /* src, dst are registers. */
836 SLJIT_ASSERT(dst >= SLJIT_TEMPORARY_REG1 && dst <= TMP_REGISTER);
837 if (reg_map[dst] < 4) {
838 if (dst != src)
839 EMIT_MOV(compiler, dst, 0, src, 0);
840 code = emit_x86_instruction(compiler, 2, dst, 0, dst, 0);
841 FAIL_IF(!code);
842 *code++ = GROUP_0F;
843 *code = sign ? MOVSX_r_rm8 : MOVZX_r_rm8;
844 }
845 else {
846 if (dst != src)
847 EMIT_MOV(compiler, dst, 0, src, 0);
848 if (sign) {
849 /* shl reg, 24 */
850 code = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, SLJIT_IMM, 24, dst, 0);
851 FAIL_IF(!code);
852 *code |= SHL;
853 /* sar reg, 24 */
854 code = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, SLJIT_IMM, 24, dst, 0);
855 FAIL_IF(!code);
856 *code |= SAR;
857 }
858 else {
859 code = emit_x86_instruction(compiler, 1 | EX86_BIN_INS, SLJIT_IMM, 0xff, dst, 0);
860 FAIL_IF(!code);
861 *(code + 1) |= AND;
862 }
863 }
864 return SLJIT_SUCCESS;
865 }
866 #endif
867 else {
868 /* src can be memory addr or reg_map[src] < 4 on x86_32 architectures. */
869 code = emit_x86_instruction(compiler, 2, dst_r, 0, src, srcw);
870 FAIL_IF(!code);
871 *code++ = GROUP_0F;
872 *code = sign ? MOVSX_r_rm8 : MOVZX_r_rm8;
873 }
874
875 if (dst & SLJIT_MEM) {
876 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
877 if (dst_r == TMP_REGISTER) {
878 /* Find a non-used register, whose reg_map[src] < 4. */
879 if ((dst & 0xf) == SLJIT_TEMPORARY_REG1) {
880 if ((dst & 0xf0) == (SLJIT_TEMPORARY_REG2 << 4))
881 work_r = SLJIT_TEMPORARY_REG3;
882 else
883 work_r = SLJIT_TEMPORARY_REG2;
884 }
885 else {
886 if ((dst & 0xf0) != (SLJIT_TEMPORARY_REG1 << 4))
887 work_r = SLJIT_TEMPORARY_REG1;
888 else if ((dst & 0xf) == SLJIT_TEMPORARY_REG2)
889 work_r = SLJIT_TEMPORARY_REG3;
890 else
891 work_r = SLJIT_TEMPORARY_REG2;
892 }
893
894 if (work_r == SLJIT_TEMPORARY_REG1) {
895 ENCODE_PREFIX(XCHG_EAX_r + reg_map[TMP_REGISTER]);
896 }
897 else {
898 code = emit_x86_instruction(compiler, 1, work_r, 0, dst_r, 0);
899 FAIL_IF(!code);
900 *code = XCHG_r_rm;
901 }
902
903 code = emit_x86_instruction(compiler, 1, work_r, 0, dst, dstw);
904 FAIL_IF(!code);
905 *code = MOV_rm8_r8;
906
907 if (work_r == SLJIT_TEMPORARY_REG1) {
908 ENCODE_PREFIX(XCHG_EAX_r + reg_map[TMP_REGISTER]);
909 }
910 else {
911 code = emit_x86_instruction(compiler, 1, work_r, 0, dst_r, 0);
912 FAIL_IF(!code);
913 *code = XCHG_r_rm;
914 }
915 }
916 else {
917 code = emit_x86_instruction(compiler, 1, dst_r, 0, dst, dstw);
918 FAIL_IF(!code);
919 *code = MOV_rm8_r8;
920 }
921 #else
922 code = emit_x86_instruction(compiler, 1 | EX86_REX | EX86_NO_REXW, dst_r, 0, dst, dstw);
923 FAIL_IF(!code);
924 *code = MOV_rm8_r8;
925 #endif
926 }
927
928 return SLJIT_SUCCESS;
929 }
930
931 static sljit_si emit_mov_half(struct sljit_compiler *compiler, sljit_si sign,
932 sljit_si dst, sljit_sw dstw,
933 sljit_si src, sljit_sw srcw)
934 {
935 sljit_ub* code;
936 sljit_si dst_r;
937
938 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
939 compiler->mode32 = 0;
940 #endif
941
942 if (dst == SLJIT_UNUSED && !(src & SLJIT_MEM))
943 return SLJIT_SUCCESS; /* Empty instruction. */
944
945 if (src & SLJIT_IMM) {
946 if (dst >= SLJIT_TEMPORARY_REG1 && dst <= TMP_REGISTER) {
947 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
948 return emit_do_imm(compiler, MOV_r_i32 + reg_map[dst], srcw);
949 #else
950 return emit_load_imm64(compiler, dst, srcw);
951 #endif
952 }
953 code = emit_x86_instruction(compiler, 1 | EX86_HALF_ARG | EX86_NO_REXW | EX86_PREF_66, SLJIT_IMM, srcw, dst, dstw);
954 FAIL_IF(!code);
955 *code = MOV_rm_i32;
956 return SLJIT_SUCCESS;
957 }
958
959 dst_r = (dst >= SLJIT_TEMPORARY_REG1 && dst <= TMP_REGISTER) ? dst : TMP_REGISTER;
960
961 if ((dst & SLJIT_MEM) && (src >= SLJIT_TEMPORARY_REG1 && src <= SLJIT_NO_REGISTERS))
962 dst_r = src;
963 else {
964 code = emit_x86_instruction(compiler, 2, dst_r, 0, src, srcw);
965 FAIL_IF(!code);
966 *code++ = GROUP_0F;
967 *code = sign ? MOVSX_r_rm16 : MOVZX_r_rm16;
968 }
969
970 if (dst & SLJIT_MEM) {
971 code = emit_x86_instruction(compiler, 1 | EX86_NO_REXW | EX86_PREF_66, dst_r, 0, dst, dstw);
972 FAIL_IF(!code);
973 *code = MOV_rm_r;
974 }
975
976 return SLJIT_SUCCESS;
977 }
978
979 static sljit_si emit_unary(struct sljit_compiler *compiler, sljit_ub opcode,
980 sljit_si dst, sljit_sw dstw,
981 sljit_si src, sljit_sw srcw)
982 {
983 sljit_ub* code;
984
985 if (dst == SLJIT_UNUSED) {
986 EMIT_MOV(compiler, TMP_REGISTER, 0, src, srcw);
987 code = emit_x86_instruction(compiler, 1, 0, 0, TMP_REGISTER, 0);
988 FAIL_IF(!code);
989 *code++ = GROUP_F7;
990 *code |= opcode;
991 return SLJIT_SUCCESS;
992 }
993 if (dst == src && dstw == srcw) {
994 /* Same input and output */
995 code = emit_x86_instruction(compiler, 1, 0, 0, dst, dstw);
996 FAIL_IF(!code);
997 *code++ = GROUP_F7;
998 *code |= opcode;
999 return SLJIT_SUCCESS;
1000 }
1001 if (dst >= SLJIT_TEMPORARY_REG1 && dst <= SLJIT_NO_REGISTERS) {
1002 EMIT_MOV(compiler, dst, 0, src, srcw);
1003 code = emit_x86_instruction(compiler, 1, 0, 0, dst, dstw);
1004 FAIL_IF(!code);
1005 *code++ = GROUP_F7;
1006 *code |= opcode;
1007 return SLJIT_SUCCESS;
1008 }
1009 EMIT_MOV(compiler, TMP_REGISTER, 0, src, srcw);
1010 code = emit_x86_instruction(compiler, 1, 0, 0, TMP_REGISTER, 0);
1011 FAIL_IF(!code);
1012 *code++ = GROUP_F7;
1013 *code |= opcode;
1014 EMIT_MOV(compiler, dst, dstw, TMP_REGISTER, 0);
1015 return SLJIT_SUCCESS;
1016 }
1017
1018 static sljit_si emit_not_with_flags(struct sljit_compiler *compiler,
1019 sljit_si dst, sljit_sw dstw,
1020 sljit_si src, sljit_sw srcw)
1021 {
1022 sljit_ub* code;
1023
1024 if (dst == SLJIT_UNUSED) {
1025 EMIT_MOV(compiler, TMP_REGISTER, 0, src, srcw);
1026 code = emit_x86_instruction(compiler, 1, 0, 0, TMP_REGISTER, 0);
1027 FAIL_IF(!code);
1028 *code++ = GROUP_F7;
1029 *code |= NOT_rm;
1030 code = emit_x86_instruction(compiler, 1, TMP_REGISTER, 0, TMP_REGISTER, 0);
1031 FAIL_IF(!code);
1032 *code = OR_r_rm;
1033 return SLJIT_SUCCESS;
1034 }
1035 if (dst >= SLJIT_TEMPORARY_REG1 && dst <= SLJIT_NO_REGISTERS) {
1036 EMIT_MOV(compiler, dst, 0, src, srcw);
1037 code = emit_x86_instruction(compiler, 1, 0, 0, dst, dstw);
1038 FAIL_IF(!code);
1039 *code++ = GROUP_F7;
1040 *code |= NOT_rm;
1041 code = emit_x86_instruction(compiler, 1, dst, 0, dst, 0);
1042 FAIL_IF(!code);
1043 *code = OR_r_rm;
1044 return SLJIT_SUCCESS;
1045 }
1046 EMIT_MOV(compiler, TMP_REGISTER, 0, src, srcw);
1047 code = emit_x86_instruction(compiler, 1, 0, 0, TMP_REGISTER, 0);
1048 FAIL_IF(!code);
1049 *code++ = GROUP_F7;
1050 *code |= NOT_rm;
1051 code = emit_x86_instruction(compiler, 1, TMP_REGISTER, 0, TMP_REGISTER, 0);
1052 FAIL_IF(!code);
1053 *code = OR_r_rm;
1054 EMIT_MOV(compiler, dst, dstw, TMP_REGISTER, 0);
1055 return SLJIT_SUCCESS;
1056 }
1057
1058 static sljit_si emit_clz(struct sljit_compiler *compiler, sljit_si op_flags,
1059 sljit_si dst, sljit_sw dstw,
1060 sljit_si src, sljit_sw srcw)
1061 {
1062 sljit_ub* code;
1063 sljit_si dst_r;
1064
1065 SLJIT_UNUSED_ARG(op_flags);
1066 if (SLJIT_UNLIKELY(dst == SLJIT_UNUSED)) {
1067 /* Just set the zero flag. */
1068 EMIT_MOV(compiler, TMP_REGISTER, 0, src, srcw);
1069 code = emit_x86_instruction(compiler, 1, 0, 0, TMP_REGISTER, 0);
1070 FAIL_IF(!code);
1071 *code++ = GROUP_F7;
1072 *code |= NOT_rm;
1073 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1074 code = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, SLJIT_IMM, 31, TMP_REGISTER, 0);
1075 #else
1076 code = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, SLJIT_IMM, !(op_flags & SLJIT_INT_OP) ? 63 : 31, TMP_REGISTER, 0);
1077 #endif
1078 FAIL_IF(!code);
1079 *code |= SHR;
1080 return SLJIT_SUCCESS;
1081 }
1082
1083 if (SLJIT_UNLIKELY(src & SLJIT_IMM)) {
1084 EMIT_MOV(compiler, TMP_REGISTER, 0, src, srcw);
1085 src = TMP_REGISTER;
1086 srcw = 0;
1087 }
1088
1089 code = emit_x86_instruction(compiler, 2, TMP_REGISTER, 0, src, srcw);
1090 FAIL_IF(!code);
1091 *code++ = GROUP_0F;
1092 *code = BSR_r_rm;
1093
1094 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1095 if (dst >= SLJIT_TEMPORARY_REG1 && dst <= TMP_REGISTER)
1096 dst_r = dst;
1097 else {
1098 /* Find an unused temporary register. */
1099 if ((dst & 0xf) != SLJIT_TEMPORARY_REG1 && (dst & 0xf0) != (SLJIT_TEMPORARY_REG1 << 4))
1100 dst_r = SLJIT_TEMPORARY_REG1;
1101 else if ((dst & 0xf) != SLJIT_TEMPORARY_REG2 && (dst & 0xf0) != (SLJIT_TEMPORARY_REG2 << 4))
1102 dst_r = SLJIT_TEMPORARY_REG2;
1103 else
1104 dst_r = SLJIT_TEMPORARY_REG3;
1105 EMIT_MOV(compiler, dst, dstw, dst_r, 0);
1106 }
1107 EMIT_MOV(compiler, dst_r, 0, SLJIT_IMM, 32 + 31);
1108 #else
1109 dst_r = (dst >= SLJIT_TEMPORARY_REG1 && dst <= TMP_REGISTER) ? dst : TMP_REG2;
1110 compiler->mode32 = 0;
1111 EMIT_MOV(compiler, dst_r, 0, SLJIT_IMM, !(op_flags & SLJIT_INT_OP) ? 64 + 63 : 32 + 31);
1112 compiler->mode32 = op_flags & SLJIT_INT_OP;
1113 #endif
1114
1115 code = emit_x86_instruction(compiler, 2, dst_r, 0, TMP_REGISTER, 0);
1116 FAIL_IF(!code);
1117 *code++ = GROUP_0F;
1118 *code = CMOVNE_r_rm;
1119
1120 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1121 code = emit_x86_instruction(compiler, 1 | EX86_BIN_INS, SLJIT_IMM, 31, dst_r, 0);
1122 #else
1123 code = emit_x86_instruction(compiler, 1 | EX86_BIN_INS, SLJIT_IMM, !(op_flags & SLJIT_INT_OP) ? 63 : 31, dst_r, 0);
1124 #endif
1125 FAIL_IF(!code);
1126 *(code + 1) |= XOR;
1127
1128 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1129 if (dst & SLJIT_MEM) {
1130 code = emit_x86_instruction(compiler, 1, dst_r, 0, dst, dstw);
1131 FAIL_IF(!code);
1132 *code = XCHG_r_rm;
1133 }
1134 #else
1135 if (dst & SLJIT_MEM)
1136 EMIT_MOV(compiler, dst, dstw, TMP_REG2, 0);
1137 #endif
1138 return SLJIT_SUCCESS;
1139 }
1140
1141 SLJIT_API_FUNC_ATTRIBUTE sljit_si sljit_emit_op1(struct sljit_compiler *compiler, sljit_si op,
1142 sljit_si dst, sljit_sw dstw,
1143 sljit_si src, sljit_sw srcw)
1144 {
1145 sljit_ub* code;
1146 sljit_si update = 0;
1147 sljit_si op_flags = GET_ALL_FLAGS(op);
1148 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1149 sljit_si dst_is_ereg = 0;
1150 sljit_si src_is_ereg = 0;
1151 #else
1152 #define src_is_ereg 0
1153 #endif
1154
1155 CHECK_ERROR();
1156 check_sljit_emit_op1(compiler, op, dst, dstw, src, srcw);
1157 ADJUST_LOCAL_OFFSET(dst, dstw);
1158 ADJUST_LOCAL_OFFSET(src, srcw);
1159
1160 CHECK_EXTRA_REGS(dst, dstw, dst_is_ereg = 1);
1161 CHECK_EXTRA_REGS(src, srcw, src_is_ereg = 1);
1162 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1163 compiler->mode32 = op_flags & SLJIT_INT_OP;
1164 #endif
1165
1166 op = GET_OPCODE(op);
1167 if (op >= SLJIT_MOV && op <= SLJIT_MOVU_P) {
1168 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1169 compiler->mode32 = 0;
1170 #endif
1171
1172 if (op_flags & SLJIT_INT_OP) {
1173 if (src <= SLJIT_NO_REGISTERS && src == dst) {
1174 if (!TYPE_CAST_NEEDED(op))
1175 return SLJIT_SUCCESS;
1176 }
1177 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1178 if (op == SLJIT_MOV_SI && (src & SLJIT_MEM))
1179 op = SLJIT_MOV_UI;
1180 if (op == SLJIT_MOVU_SI && (src & SLJIT_MEM))
1181 op = SLJIT_MOVU_UI;
1182 if (op == SLJIT_MOV_UI && (src & SLJIT_IMM))
1183 op = SLJIT_MOV_SI;
1184 if (op == SLJIT_MOVU_UI && (src & SLJIT_IMM))
1185 op = SLJIT_MOVU_SI;
1186 #endif
1187 }
1188
1189 SLJIT_COMPILE_ASSERT(SLJIT_MOV + 8 == SLJIT_MOVU, movu_offset);
1190 if (op >= SLJIT_MOVU) {
1191 update = 1;
1192 op -= 8;
1193 }
1194
1195 if (src & SLJIT_IMM) {
1196 switch (op) {
1197 case SLJIT_MOV_UB:
1198 srcw = (sljit_ub)srcw;
1199 break;
1200 case SLJIT_MOV_SB:
1201 srcw = (sljit_sb)srcw;
1202 break;
1203 case SLJIT_MOV_UH:
1204 srcw = (sljit_uh)srcw;
1205 break;
1206 case SLJIT_MOV_SH:
1207 srcw = (sljit_sh)srcw;
1208 break;
1209 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1210 case SLJIT_MOV_UI:
1211 srcw = (sljit_ui)srcw;
1212 break;
1213 case SLJIT_MOV_SI:
1214 srcw = (sljit_si)srcw;
1215 break;
1216 #endif
1217 }
1218 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1219 if (SLJIT_UNLIKELY(dst_is_ereg))
1220 return emit_mov(compiler, dst, dstw, src, srcw);
1221 #endif
1222 }
1223
1224 if (SLJIT_UNLIKELY(update) && (src & SLJIT_MEM) && !src_is_ereg && (src & 0xf) && (srcw != 0 || (src & 0xf0) != 0)) {
1225 code = emit_x86_instruction(compiler, 1, src & 0xf, 0, src, srcw);
1226 FAIL_IF(!code);
1227 *code = LEA_r_m;
1228 src &= SLJIT_MEM | 0xf;
1229 srcw = 0;
1230 }
1231
1232 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1233 if (SLJIT_UNLIKELY(dst_is_ereg) && (!(op == SLJIT_MOV || op == SLJIT_MOV_UI || op == SLJIT_MOV_SI || op == SLJIT_MOV_P) || (src & SLJIT_MEM))) {
1234 SLJIT_ASSERT(dst == SLJIT_MEM1(SLJIT_LOCALS_REG));
1235 dst = TMP_REGISTER;
1236 }
1237 #endif
1238
1239 switch (op) {
1240 case SLJIT_MOV:
1241 case SLJIT_MOV_P:
1242 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1243 case SLJIT_MOV_UI:
1244 case SLJIT_MOV_SI:
1245 #endif
1246 FAIL_IF(emit_mov(compiler, dst, dstw, src, srcw));
1247 break;
1248 case SLJIT_MOV_UB:
1249 FAIL_IF(emit_mov_byte(compiler, 0, dst, dstw, src, srcw));
1250 break;
1251 case SLJIT_MOV_SB:
1252 FAIL_IF(emit_mov_byte(compiler, 1, dst, dstw, src, srcw));
1253 break;
1254 case SLJIT_MOV_UH:
1255 FAIL_IF(emit_mov_half(compiler, 0, dst, dstw, src, srcw));
1256 break;
1257 case SLJIT_MOV_SH:
1258 FAIL_IF(emit_mov_half(compiler, 1, dst, dstw, src, srcw));
1259 break;
1260 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1261 case SLJIT_MOV_UI:
1262 FAIL_IF(emit_mov_int(compiler, 0, dst, dstw, src, srcw));
1263 break;
1264 case SLJIT_MOV_SI:
1265 FAIL_IF(emit_mov_int(compiler, 1, dst, dstw, src, srcw));
1266 break;
1267 #endif
1268 }
1269
1270 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1271 if (SLJIT_UNLIKELY(dst_is_ereg) && dst == TMP_REGISTER)
1272 return emit_mov(compiler, SLJIT_MEM1(SLJIT_LOCALS_REG), dstw, TMP_REGISTER, 0);
1273 #endif
1274
1275 if (SLJIT_UNLIKELY(update) && (dst & SLJIT_MEM) && (dst & 0xf) && (dstw != 0 || (dst & 0xf0) != 0)) {
1276 code = emit_x86_instruction(compiler, 1, dst & 0xf, 0, dst, dstw);
1277 FAIL_IF(!code);
1278 *code = LEA_r_m;
1279 }
1280 return SLJIT_SUCCESS;
1281 }
1282
1283 if (SLJIT_UNLIKELY(GET_FLAGS(op_flags)))
1284 compiler->flags_saved = 0;
1285
1286 switch (op) {
1287 case SLJIT_NOT:
1288 if (SLJIT_UNLIKELY(op_flags & SLJIT_SET_E))
1289 return emit_not_with_flags(compiler, dst, dstw, src, srcw);
1290 return emit_unary(compiler, NOT_rm, dst, dstw, src, srcw);
1291
1292 case SLJIT_NEG:
1293 if (SLJIT_UNLIKELY(op_flags & SLJIT_KEEP_FLAGS) && !compiler->flags_saved)
1294 FAIL_IF(emit_save_flags(compiler));
1295 return emit_unary(compiler, NEG_rm, dst, dstw, src, srcw);
1296
1297 case SLJIT_CLZ:
1298 if (SLJIT_UNLIKELY(op_flags & SLJIT_KEEP_FLAGS) && !compiler->flags_saved)
1299 FAIL_IF(emit_save_flags(compiler));
1300 return emit_clz(compiler, op_flags, dst, dstw, src, srcw);
1301 }
1302
1303 return SLJIT_SUCCESS;
1304
1305 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1306 #undef src_is_ereg
1307 #endif
1308 }
1309
1310 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1311
1312 #define BINARY_IMM(_op_imm_, _op_mr_, immw, arg, argw) \
1313 if (IS_HALFWORD(immw) || compiler->mode32) { \
1314 code = emit_x86_instruction(compiler, 1 | EX86_BIN_INS, SLJIT_IMM, immw, arg, argw); \
1315 FAIL_IF(!code); \
1316 *(code + 1) |= (_op_imm_); \
1317 } \
1318 else { \
1319 FAIL_IF(emit_load_imm64(compiler, TMP_REG2, immw)); \
1320 code = emit_x86_instruction(compiler, 1, TMP_REG2, 0, arg, argw); \
1321 FAIL_IF(!code); \
1322 *code = (_op_mr_); \
1323 }
1324
1325 #define BINARY_EAX_IMM(_op_eax_imm_, immw) \
1326 FAIL_IF(emit_do_imm32(compiler, (!compiler->mode32) ? REX_W : 0, (_op_eax_imm_), immw))
1327
1328 #else
1329
1330 #define BINARY_IMM(_op_imm_, _op_mr_, immw, arg, argw) \
1331 code = emit_x86_instruction(compiler, 1 | EX86_BIN_INS, SLJIT_IMM, immw, arg, argw); \
1332 FAIL_IF(!code); \
1333 *(code + 1) |= (_op_imm_);
1334
1335 #define BINARY_EAX_IMM(_op_eax_imm_, immw) \
1336 FAIL_IF(emit_do_imm(compiler, (_op_eax_imm_), immw))
1337
1338 #endif
1339
1340 static sljit_si emit_cum_binary(struct sljit_compiler *compiler,
1341 sljit_ub op_rm, sljit_ub op_mr, sljit_ub op_imm, sljit_ub op_eax_imm,
1342 sljit_si dst, sljit_sw dstw,
1343 sljit_si src1, sljit_sw src1w,
1344 sljit_si src2, sljit_sw src2w)
1345 {
1346 sljit_ub* code;
1347
1348 if (dst == SLJIT_UNUSED) {
1349 EMIT_MOV(compiler, TMP_REGISTER, 0, src1, src1w);
1350 if (src2 & SLJIT_IMM) {
1351 BINARY_IMM(op_imm, op_mr, src2w, TMP_REGISTER, 0);
1352 }
1353 else {
1354 code = emit_x86_instruction(compiler, 1, TMP_REGISTER, 0, src2, src2w);
1355 FAIL_IF(!code);
1356 *code = op_rm;
1357 }
1358 return SLJIT_SUCCESS;
1359 }
1360
1361 if (dst == src1 && dstw == src1w) {
1362 if (src2 & SLJIT_IMM) {
1363 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1364 if ((dst == SLJIT_TEMPORARY_REG1) && (src2w > 127 || src2w < -128) && (compiler->mode32 || IS_HALFWORD(src2w))) {
1365 #else
1366 if ((dst == SLJIT_TEMPORARY_REG1) && (src2w > 127 || src2w < -128)) {
1367 #endif
1368 BINARY_EAX_IMM(op_eax_imm, src2w);
1369 }
1370 else {
1371 BINARY_IMM(op_imm, op_mr, src2w, dst, dstw);
1372 }
1373 }
1374 else if (dst >= SLJIT_TEMPORARY_REG1 && dst <= SLJIT_NO_REGISTERS) {
1375 code = emit_x86_instruction(compiler, 1, dst, dstw, src2, src2w);
1376 FAIL_IF(!code);
1377 *code = op_rm;
1378 }
1379 else if (src2 >= SLJIT_TEMPORARY_REG1 && src2 <= TMP_REGISTER) {
1380 /* Special exception for sljit_emit_cond_value. */
1381 code = emit_x86_instruction(compiler, 1, src2, src2w, dst, dstw);
1382 FAIL_IF(!code);
1383 *code = op_mr;
1384 }
1385 else {
1386 EMIT_MOV(compiler, TMP_REGISTER, 0, src2, src2w);
1387 code = emit_x86_instruction(compiler, 1, TMP_REGISTER, 0, dst, dstw);
1388 FAIL_IF(!code);
1389 *code = op_mr;
1390 }
1391 return SLJIT_SUCCESS;
1392 }
1393
1394 /* Only for cumulative operations. */
1395 if (dst == src2 && dstw == src2w) {
1396 if (src1 & SLJIT_IMM) {
1397 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1398 if ((dst == SLJIT_TEMPORARY_REG1) && (src1w > 127 || src1w < -128) && (compiler->mode32 || IS_HALFWORD(src1w))) {
1399 #else
1400 if ((dst == SLJIT_TEMPORARY_REG1) && (src1w > 127 || src1w < -128)) {
1401 #endif
1402 BINARY_EAX_IMM(op_eax_imm, src1w);
1403 }
1404 else {
1405 BINARY_IMM(op_imm, op_mr, src1w, dst, dstw);
1406 }
1407 }
1408 else if (dst >= SLJIT_TEMPORARY_REG1 && dst <= SLJIT_NO_REGISTERS) {
1409 code = emit_x86_instruction(compiler, 1, dst, dstw, src1, src1w);
1410 FAIL_IF(!code);
1411 *code = op_rm;
1412 }
1413 else if (src1 >= SLJIT_TEMPORARY_REG1 && src1 <= SLJIT_NO_REGISTERS) {
1414 code = emit_x86_instruction(compiler, 1, src1, src1w, dst, dstw);
1415 FAIL_IF(!code);
1416 *code = op_mr;
1417 }
1418 else {
1419 EMIT_MOV(compiler, TMP_REGISTER, 0, src1, src1w);
1420 code = emit_x86_instruction(compiler, 1, TMP_REGISTER, 0, dst, dstw);
1421 FAIL_IF(!code);
1422 *code = op_mr;
1423 }
1424 return SLJIT_SUCCESS;
1425 }
1426
1427 /* General version. */
1428 if (dst >= SLJIT_TEMPORARY_REG1 && dst <= SLJIT_NO_REGISTERS) {
1429 EMIT_MOV(compiler, dst, 0, src1, src1w);
1430 if (src2 & SLJIT_IMM) {
1431 BINARY_IMM(op_imm, op_mr, src2w, dst, 0);
1432 }
1433 else {
1434 code = emit_x86_instruction(compiler, 1, dst, 0, src2, src2w);
1435 FAIL_IF(!code);
1436 *code = op_rm;
1437 }
1438 }
1439 else {
1440 /* This version requires less memory writing. */
1441 EMIT_MOV(compiler, TMP_REGISTER, 0, src1, src1w);
1442 if (src2 & SLJIT_IMM) {
1443 BINARY_IMM(op_imm, op_mr, src2w, TMP_REGISTER, 0);
1444 }
1445 else {
1446 code = emit_x86_instruction(compiler, 1, TMP_REGISTER, 0, src2, src2w);
1447 FAIL_IF(!code);
1448 *code = op_rm;
1449 }
1450 EMIT_MOV(compiler, dst, dstw, TMP_REGISTER, 0);
1451 }
1452
1453 return SLJIT_SUCCESS;
1454 }
1455
1456 static sljit_si emit_non_cum_binary(struct sljit_compiler *compiler,
1457 sljit_ub op_rm, sljit_ub op_mr, sljit_ub op_imm, sljit_ub op_eax_imm,
1458 sljit_si dst, sljit_sw dstw,
1459 sljit_si src1, sljit_sw src1w,
1460 sljit_si src2, sljit_sw src2w)
1461 {
1462 sljit_ub* code;
1463
1464 if (dst == SLJIT_UNUSED) {
1465 EMIT_MOV(compiler, TMP_REGISTER, 0, src1, src1w);
1466 if (src2 & SLJIT_IMM) {
1467 BINARY_IMM(op_imm, op_mr, src2w, TMP_REGISTER, 0);
1468 }
1469 else {
1470 code = emit_x86_instruction(compiler, 1, TMP_REGISTER, 0, src2, src2w);
1471 FAIL_IF(!code);
1472 *code = op_rm;
1473 }
1474 return SLJIT_SUCCESS;
1475 }
1476
1477 if (dst == src1 && dstw == src1w) {
1478 if (src2 & SLJIT_IMM) {
1479 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1480 if ((dst == SLJIT_TEMPORARY_REG1) && (src2w > 127 || src2w < -128) && (compiler->mode32 || IS_HALFWORD(src2w))) {
1481 #else
1482 if ((dst == SLJIT_TEMPORARY_REG1) && (src2w > 127 || src2w < -128)) {
1483 #endif
1484 BINARY_EAX_IMM(op_eax_imm, src2w);
1485 }
1486 else {
1487 BINARY_IMM(op_imm, op_mr, src2w, dst, dstw);
1488 }
1489 }
1490 else if (dst >= SLJIT_TEMPORARY_REG1 && dst <= SLJIT_NO_REGISTERS) {
1491 code = emit_x86_instruction(compiler, 1, dst, dstw, src2, src2w);
1492 FAIL_IF(!code);
1493 *code = op_rm;
1494 }
1495 else if (src2 >= SLJIT_TEMPORARY_REG1 && src2 <= SLJIT_NO_REGISTERS) {
1496 code = emit_x86_instruction(compiler, 1, src2, src2w, dst, dstw);
1497 FAIL_IF(!code);
1498 *code = op_mr;
1499 }
1500 else {
1501 EMIT_MOV(compiler, TMP_REGISTER, 0, src2, src2w);
1502 code = emit_x86_instruction(compiler, 1, TMP_REGISTER, 0, dst, dstw);
1503 FAIL_IF(!code);
1504 *code = op_mr;
1505 }
1506 return SLJIT_SUCCESS;
1507 }
1508
1509 /* General version. */
1510 if ((dst >= SLJIT_TEMPORARY_REG1 && dst <= SLJIT_NO_REGISTERS) && dst != src2) {
1511 EMIT_MOV(compiler, dst, 0, src1, src1w);
1512 if (src2 & SLJIT_IMM) {
1513 BINARY_IMM(op_imm, op_mr, src2w, dst, 0);
1514 }
1515 else {
1516 code = emit_x86_instruction(compiler, 1, dst, 0, src2, src2w);
1517 FAIL_IF(!code);
1518 *code = op_rm;
1519 }
1520 }
1521 else {
1522 /* This version requires less memory writing. */
1523 EMIT_MOV(compiler, TMP_REGISTER, 0, src1, src1w);
1524 if (src2 & SLJIT_IMM) {
1525 BINARY_IMM(op_imm, op_mr, src2w, TMP_REGISTER, 0);
1526 }
1527 else {
1528 code = emit_x86_instruction(compiler, 1, TMP_REGISTER, 0, src2, src2w);
1529 FAIL_IF(!code);
1530 *code = op_rm;
1531 }
1532 EMIT_MOV(compiler, dst, dstw, TMP_REGISTER, 0);
1533 }
1534
1535 return SLJIT_SUCCESS;
1536 }
1537
1538 static sljit_si emit_mul(struct sljit_compiler *compiler,
1539 sljit_si dst, sljit_sw dstw,
1540 sljit_si src1, sljit_sw src1w,
1541 sljit_si src2, sljit_sw src2w)
1542 {
1543 sljit_ub* code;
1544 sljit_si dst_r;
1545
1546 dst_r = (dst >= SLJIT_TEMPORARY_REG1 && dst <= SLJIT_NO_REGISTERS) ? dst : TMP_REGISTER;
1547
1548 /* Register destination. */
1549 if (dst_r == src1 && !(src2 & SLJIT_IMM)) {
1550 code = emit_x86_instruction(compiler, 2, dst_r, 0, src2, src2w);
1551 FAIL_IF(!code);
1552 *code++ = GROUP_0F;
1553 *code = IMUL_r_rm;
1554 }
1555 else if (dst_r == src2 && !(src1 & SLJIT_IMM)) {
1556 code = emit_x86_instruction(compiler, 2, dst_r, 0, src1, src1w);
1557 FAIL_IF(!code);
1558 *code++ = GROUP_0F;
1559 *code = IMUL_r_rm;
1560 }
1561 else if (src1 & SLJIT_IMM) {
1562 if (src2 & SLJIT_IMM) {
1563 EMIT_MOV(compiler, dst_r, 0, SLJIT_IMM, src2w);
1564 src2 = dst_r;
1565 src2w = 0;
1566 }
1567
1568 if (src1w <= 127 && src1w >= -128) {
1569 code = emit_x86_instruction(compiler, 1, dst_r, 0, src2, src2w);
1570 FAIL_IF(!code);
1571 *code = IMUL_r_rm_i8;
1572 code = (sljit_ub*)ensure_buf(compiler, 1 + 1);
1573 FAIL_IF(!code);
1574 INC_CSIZE(1);
1575 *code = (sljit_sb)src1w;
1576 }
1577 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1578 else {
1579 code = emit_x86_instruction(compiler, 1, dst_r, 0, src2, src2w);
1580 FAIL_IF(!code);
1581 *code = IMUL_r_rm_i32;
1582 code = (sljit_ub*)ensure_buf(compiler, 1 + 4);
1583 FAIL_IF(!code);
1584 INC_CSIZE(4);
1585 *(sljit_sw*)code = src1w;
1586 }
1587 #else
1588 else if (IS_HALFWORD(src1w)) {
1589 code = emit_x86_instruction(compiler, 1, dst_r, 0, src2, src2w);
1590 FAIL_IF(!code);
1591 *code = IMUL_r_rm_i32;
1592 code = (sljit_ub*)ensure_buf(compiler, 1 + 4);
1593 FAIL_IF(!code);
1594 INC_CSIZE(4);
1595 *(sljit_si*)code = (sljit_si)src1w;
1596 }
1597 else {
1598 EMIT_MOV(compiler, TMP_REG2, 0, SLJIT_IMM, src1w);
1599 if (dst_r != src2)
1600 EMIT_MOV(compiler, dst_r, 0, src2, src2w);
1601 code = emit_x86_instruction(compiler, 2, dst_r, 0, TMP_REG2, 0);
1602 FAIL_IF(!code);
1603 *code++ = GROUP_0F;
1604 *code = IMUL_r_rm;
1605 }
1606 #endif
1607 }
1608 else if (src2 & SLJIT_IMM) {
1609 /* Note: src1 is NOT immediate. */
1610
1611 if (src2w <= 127 && src2w >= -128) {
1612 code = emit_x86_instruction(compiler, 1, dst_r, 0, src1, src1w);
1613 FAIL_IF(!code);
1614 *code = IMUL_r_rm_i8;
1615 code = (sljit_ub*)ensure_buf(compiler, 1 + 1);
1616 FAIL_IF(!code);
1617 INC_CSIZE(1);
1618 *code = (sljit_sb)src2w;
1619 }
1620 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1621 else {
1622 code = emit_x86_instruction(compiler, 1, dst_r, 0, src1, src1w);
1623 FAIL_IF(!code);
1624 *code = IMUL_r_rm_i32;
1625 code = (sljit_ub*)ensure_buf(compiler, 1 + 4);
1626 FAIL_IF(!code);
1627 INC_CSIZE(4);
1628 *(sljit_sw*)code = src2w;
1629 }
1630 #else
1631 else if (IS_HALFWORD(src2w)) {
1632 code = emit_x86_instruction(compiler, 1, dst_r, 0, src1, src1w);
1633 FAIL_IF(!code);
1634 *code = IMUL_r_rm_i32;
1635 code = (sljit_ub*)ensure_buf(compiler, 1 + 4);
1636 FAIL_IF(!code);
1637 INC_CSIZE(4);
1638 *(sljit_si*)code = (sljit_si)src2w;
1639 }
1640 else {
1641 EMIT_MOV(compiler, TMP_REG2, 0, SLJIT_IMM, src1w);
1642 if (dst_r != src1)
1643 EMIT_MOV(compiler, dst_r, 0, src1, src1w);
1644 code = emit_x86_instruction(compiler, 2, dst_r, 0, TMP_REG2, 0);
1645 FAIL_IF(!code);
1646 *code++ = GROUP_0F;
1647 *code = IMUL_r_rm;
1648 }
1649 #endif
1650 }
1651 else {
1652 /* Neither argument is immediate. */
1653 if (ADDRESSING_DEPENDS_ON(src2, dst_r))
1654 dst_r = TMP_REGISTER;
1655 EMIT_MOV(compiler, dst_r, 0, src1, src1w);
1656 code = emit_x86_instruction(compiler, 2, dst_r, 0, src2, src2w);
1657 FAIL_IF(!code);
1658 *code++ = GROUP_0F;
1659 *code = IMUL_r_rm;
1660 }
1661
1662 if (dst_r == TMP_REGISTER)
1663 EMIT_MOV(compiler, dst, dstw, TMP_REGISTER, 0);
1664
1665 return SLJIT_SUCCESS;
1666 }
1667
1668 static sljit_si emit_lea_binary(struct sljit_compiler *compiler,
1669 sljit_si dst, sljit_sw dstw,
1670 sljit_si src1, sljit_sw src1w,
1671 sljit_si src2, sljit_sw src2w)
1672 {
1673 sljit_ub* code;
1674 sljit_si dst_r, done = 0;
1675
1676 /* These cases better be left to handled by normal way. */
1677 if (dst == src1 && dstw == src1w)
1678 return SLJIT_ERR_UNSUPPORTED;
1679 if (dst == src2 && dstw == src2w)
1680 return SLJIT_ERR_UNSUPPORTED;
1681
1682 dst_r = (dst >= SLJIT_TEMPORARY_REG1 && dst <= SLJIT_NO_REGISTERS) ? dst : TMP_REGISTER;
1683
1684 if (src1 >= SLJIT_TEMPORARY_REG1 && src1 <= SLJIT_NO_REGISTERS) {
1685 if ((src2 >= SLJIT_TEMPORARY_REG1 && src2 <= SLJIT_NO_REGISTERS) || src2 == TMP_REGISTER) {
1686 code = emit_x86_instruction(compiler, 1, dst_r, 0, SLJIT_MEM2(src1, src2), 0);
1687 FAIL_IF(!code);
1688 *code = LEA_r_m;
1689 done = 1;
1690 }
1691 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1692 if ((src2 & SLJIT_IMM) && (compiler->mode32 || IS_HALFWORD(src2w))) {
1693 code = emit_x86_instruction(compiler, 1, dst_r, 0, SLJIT_MEM1(src1), (sljit_si)src2w);
1694 #else
1695 if (src2 & SLJIT_IMM) {
1696 code = emit_x86_instruction(compiler, 1, dst_r, 0, SLJIT_MEM1(src1), src2w);
1697 #endif
1698 FAIL_IF(!code);
1699 *code = LEA_r_m;
1700 done = 1;
1701 }
1702 }
1703 else if (src2 >= SLJIT_TEMPORARY_REG1 && src2 <= SLJIT_NO_REGISTERS) {
1704 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1705 if ((src1 & SLJIT_IMM) && (compiler->mode32 || IS_HALFWORD(src1w))) {
1706 code = emit_x86_instruction(compiler, 1, dst_r, 0, SLJIT_MEM1(src2), (sljit_si)src1w);
1707 #else
1708 if (src1 & SLJIT_IMM) {
1709 code = emit_x86_instruction(compiler, 1, dst_r, 0, SLJIT_MEM1(src2), src1w);
1710 #endif
1711 FAIL_IF(!code);
1712 *code = LEA_r_m;
1713 done = 1;
1714 }
1715 }
1716
1717 if (done) {
1718 if (dst_r == TMP_REGISTER)
1719 return emit_mov(compiler, dst, dstw, TMP_REGISTER, 0);
1720 return SLJIT_SUCCESS;
1721 }
1722 return SLJIT_ERR_UNSUPPORTED;
1723 }
1724
1725 static sljit_si emit_cmp_binary(struct sljit_compiler *compiler,
1726 sljit_si src1, sljit_sw src1w,
1727 sljit_si src2, sljit_sw src2w)
1728 {
1729 sljit_ub* code;
1730
1731 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1732 if (src1 == SLJIT_TEMPORARY_REG1 && (src2 & SLJIT_IMM) && (src2w > 127 || src2w < -128) && (compiler->mode32 || IS_HALFWORD(src2w))) {
1733 #else
1734 if (src1 == SLJIT_TEMPORARY_REG1 && (src2 & SLJIT_IMM) && (src2w > 127 || src2w < -128)) {
1735 #endif
1736 BINARY_EAX_IMM(CMP_EAX_i32, src2w);
1737 return SLJIT_SUCCESS;
1738 }
1739
1740 if (src1 >= SLJIT_TEMPORARY_REG1 && src1 <= SLJIT_NO_REGISTERS) {
1741 if (src2 & SLJIT_IMM) {
1742 BINARY_IMM(CMP, CMP_rm_r, src2w, src1, 0);
1743 }
1744 else {
1745 code = emit_x86_instruction(compiler, 1, src1, 0, src2, src2w);
1746 FAIL_IF(!code);
1747 *code = CMP_r_rm;
1748 }
1749 return SLJIT_SUCCESS;
1750 }
1751
1752 if (src2 >= SLJIT_TEMPORARY_REG1 && src2 <= SLJIT_NO_REGISTERS && !(src1 & SLJIT_IMM)) {
1753 code = emit_x86_instruction(compiler, 1, src2, 0, src1, src1w);
1754 FAIL_IF(!code);
1755 *code = CMP_rm_r;
1756 return SLJIT_SUCCESS;
1757 }
1758
1759 if (src2 & SLJIT_IMM) {
1760 if (src1 & SLJIT_IMM) {
1761 EMIT_MOV(compiler, TMP_REGISTER, 0, src1, src1w);
1762 src1 = TMP_REGISTER;
1763 src1w = 0;
1764 }
1765 BINARY_IMM(CMP, CMP_rm_r, src2w, src1, src1w);
1766 }
1767 else {
1768 EMIT_MOV(compiler, TMP_REGISTER, 0, src1, src1w);
1769 code = emit_x86_instruction(compiler, 1, TMP_REGISTER, 0, src2, src2w);
1770 FAIL_IF(!code);
1771 *code = CMP_r_rm;
1772 }
1773 return SLJIT_SUCCESS;
1774 }
1775
1776 static sljit_si emit_test_binary(struct sljit_compiler *compiler,
1777 sljit_si src1, sljit_sw src1w,
1778 sljit_si src2, sljit_sw src2w)
1779 {
1780 sljit_ub* code;
1781
1782 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1783 if (src1 == SLJIT_TEMPORARY_REG1 && (src2 & SLJIT_IMM) && (src2w > 127 || src2w < -128) && (compiler->mode32 || IS_HALFWORD(src2w))) {
1784 #else
1785 if (src1 == SLJIT_TEMPORARY_REG1 && (src2 & SLJIT_IMM) && (src2w > 127 || src2w < -128)) {
1786 #endif
1787 BINARY_EAX_IMM(TEST_EAX_i32, src2w);
1788 return SLJIT_SUCCESS;
1789 }
1790
1791 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1792 if (src2 == SLJIT_TEMPORARY_REG1 && (src2 & SLJIT_IMM) && (src1w > 127 || src1w < -128) && (compiler->mode32 || IS_HALFWORD(src1w))) {
1793 #else
1794 if (src2 == SLJIT_TEMPORARY_REG1 && (src1 & SLJIT_IMM) && (src1w > 127 || src1w < -128)) {
1795 #endif
1796 BINARY_EAX_IMM(TEST_EAX_i32, src1w);
1797 return SLJIT_SUCCESS;
1798 }
1799
1800 if (src1 >= SLJIT_TEMPORARY_REG1 && src1 <= SLJIT_NO_REGISTERS) {
1801 if (src2 & SLJIT_IMM) {
1802 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1803 if (IS_HALFWORD(src2w) || compiler->mode32) {
1804 code = emit_x86_instruction(compiler, 1, SLJIT_IMM, src2w, src1, 0);
1805 FAIL_IF(!code);
1806 *code = GROUP_F7;
1807 }
1808 else {
1809 FAIL_IF(emit_load_imm64(compiler, TMP_REG2, src2w));
1810 code = emit_x86_instruction(compiler, 1, TMP_REG2, 0, src1, 0);
1811 FAIL_IF(!code);
1812 *code = TEST_rm_r;
1813 }
1814 #else
1815 code = emit_x86_instruction(compiler, 1, SLJIT_IMM, src2w, src1, 0);
1816 FAIL_IF(!code);
1817 *code = GROUP_F7;
1818 #endif
1819 }
1820 else {
1821 code = emit_x86_instruction(compiler, 1, src1, 0, src2, src2w);
1822 FAIL_IF(!code);
1823 *code = TEST_rm_r;
1824 }
1825 return SLJIT_SUCCESS;
1826 }
1827
1828 if (src2 >= SLJIT_TEMPORARY_REG1 && src2 <= SLJIT_NO_REGISTERS) {
1829 if (src1 & SLJIT_IMM) {
1830 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1831 if (IS_HALFWORD(src1w) || compiler->mode32) {
1832 code = emit_x86_instruction(compiler, 1, SLJIT_IMM, src1w, src2, 0);
1833 FAIL_IF(!code);
1834 *code = GROUP_F7;
1835 }
1836 else {
1837 FAIL_IF(emit_load_imm64(compiler, TMP_REG2, src1w));
1838 code = emit_x86_instruction(compiler, 1, TMP_REG2, 0, src2, 0);
1839 FAIL_IF(!code);
1840 *code = TEST_rm_r;
1841 }
1842 #else
1843 code = emit_x86_instruction(compiler, 1, src1, src1w, src2, 0);
1844 FAIL_IF(!code);
1845 *code = GROUP_F7;
1846 #endif
1847 }
1848 else {
1849 code = emit_x86_instruction(compiler, 1, src2, 0, src1, src1w);
1850 FAIL_IF(!code);
1851 *code = TEST_rm_r;
1852 }
1853 return SLJIT_SUCCESS;
1854 }
1855
1856 EMIT_MOV(compiler, TMP_REGISTER, 0, src1, src1w);
1857 if (src2 & SLJIT_IMM) {
1858 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1859 if (IS_HALFWORD(src2w) || compiler->mode32) {
1860 code = emit_x86_instruction(compiler, 1, SLJIT_IMM, src2w, TMP_REGISTER, 0);
1861 FAIL_IF(!code);
1862 *code = GROUP_F7;
1863 }
1864 else {
1865 FAIL_IF(emit_load_imm64(compiler, TMP_REG2, src2w));
1866 code = emit_x86_instruction(compiler, 1, TMP_REG2, 0, TMP_REGISTER, 0);
1867 FAIL_IF(!code);
1868 *code = TEST_rm_r;
1869 }
1870 #else
1871 code = emit_x86_instruction(compiler, 1, SLJIT_IMM, src2w, TMP_REGISTER, 0);
1872 FAIL_IF(!code);
1873 *code = GROUP_F7;
1874 #endif
1875 }
1876 else {
1877 code = emit_x86_instruction(compiler, 1, TMP_REGISTER, 0, src2, src2w);
1878 FAIL_IF(!code);
1879 *code = TEST_rm_r;
1880 }
1881 return SLJIT_SUCCESS;
1882 }
1883
1884 static sljit_si emit_shift(struct sljit_compiler *compiler,
1885 sljit_ub mode,
1886 sljit_si dst, sljit_sw dstw,
1887 sljit_si src1, sljit_sw src1w,
1888 sljit_si src2, sljit_sw src2w)
1889 {
1890 sljit_ub* code;
1891
1892 if ((src2 & SLJIT_IMM) || (src2 == SLJIT_PREF_SHIFT_REG)) {
1893 if (dst == src1 && dstw == src1w) {
1894 code = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, src2, src2w, dst, dstw);
1895 FAIL_IF(!code);
1896 *code |= mode;
1897 return SLJIT_SUCCESS;
1898 }
1899 if (dst == SLJIT_UNUSED) {
1900 EMIT_MOV(compiler, TMP_REGISTER, 0, src1, src1w);
1901 code = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, src2, src2w, TMP_REGISTER, 0);
1902 FAIL_IF(!code);
1903 *code |= mode;
1904 return SLJIT_SUCCESS;
1905 }
1906 if (dst == SLJIT_PREF_SHIFT_REG && src2 == SLJIT_PREF_SHIFT_REG) {
1907 EMIT_MOV(compiler, TMP_REGISTER, 0, src1, src1w);
1908 code = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, SLJIT_PREF_SHIFT_REG, 0, TMP_REGISTER, 0);
1909 FAIL_IF(!code);
1910 *code |= mode;
1911 EMIT_MOV(compiler, SLJIT_PREF_SHIFT_REG, 0, TMP_REGISTER, 0);
1912 return SLJIT_SUCCESS;
1913 }
1914 if (dst >= SLJIT_TEMPORARY_REG1 && dst <= SLJIT_NO_REGISTERS) {
1915 EMIT_MOV(compiler, dst, 0, src1, src1w);
1916 code = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, src2, src2w, dst, 0);
1917 FAIL_IF(!code);
1918 *code |= mode;
1919 return SLJIT_SUCCESS;
1920 }
1921
1922 EMIT_MOV(compiler, TMP_REGISTER, 0, src1, src1w);
1923 code = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, src2, src2w, TMP_REGISTER, 0);
1924 FAIL_IF(!code);
1925 *code |= mode;
1926 EMIT_MOV(compiler, dst, dstw, TMP_REGISTER, 0);
1927 return SLJIT_SUCCESS;
1928 }
1929
1930 if (dst == SLJIT_PREF_SHIFT_REG) {
1931 EMIT_MOV(compiler, TMP_REGISTER, 0, src1, src1w);
1932 EMIT_MOV(compiler, SLJIT_PREF_SHIFT_REG, 0, src2, src2w);
1933 code = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, SLJIT_PREF_SHIFT_REG, 0, TMP_REGISTER, 0);
1934 FAIL_IF(!code);
1935 *code |= mode;
1936 EMIT_MOV(compiler, SLJIT_PREF_SHIFT_REG, 0, TMP_REGISTER, 0);
1937 }
1938 else if (dst >= SLJIT_TEMPORARY_REG1 && dst <= SLJIT_NO_REGISTERS && dst != src2 && !ADDRESSING_DEPENDS_ON(src2, dst)) {
1939 if (src1 != dst)
1940 EMIT_MOV(compiler, dst, 0, src1, src1w);
1941 EMIT_MOV(compiler, TMP_REGISTER, 0, SLJIT_PREF_SHIFT_REG, 0);
1942 EMIT_MOV(compiler, SLJIT_PREF_SHIFT_REG, 0, src2, src2w);
1943 code = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, SLJIT_PREF_SHIFT_REG, 0, dst, 0);
1944 FAIL_IF(!code);
1945 *code |= mode;
1946 EMIT_MOV(compiler, SLJIT_PREF_SHIFT_REG, 0, TMP_REGISTER, 0);
1947 }
1948 else {
1949 /* This case is really difficult, since ecx itself may used for
1950 addressing, and we must ensure to work even in that case. */
1951 EMIT_MOV(compiler, TMP_REGISTER, 0, src1, src1w);
1952 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1953 EMIT_MOV(compiler, TMP_REG2, 0, SLJIT_PREF_SHIFT_REG, 0);
1954 #else
1955 /* [esp+0] contains the flags. */
1956 EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_LOCALS_REG), sizeof(sljit_sw), SLJIT_PREF_SHIFT_REG, 0);
1957 #endif
1958 EMIT_MOV(compiler, SLJIT_PREF_SHIFT_REG, 0, src2, src2w);
1959 code = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, SLJIT_PREF_SHIFT_REG, 0, TMP_REGISTER, 0);
1960 FAIL_IF(!code);
1961 *code |= mode;
1962 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1963 EMIT_MOV(compiler, SLJIT_PREF_SHIFT_REG, 0, TMP_REG2, 0);
1964 #else
1965 EMIT_MOV(compiler, SLJIT_PREF_SHIFT_REG, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), sizeof(sljit_sw));
1966 #endif
1967 EMIT_MOV(compiler, dst, dstw, TMP_REGISTER, 0);
1968 }
1969
1970 return SLJIT_SUCCESS;
1971 }
1972
1973 static sljit_si emit_shift_with_flags(struct sljit_compiler *compiler,
1974 sljit_ub mode, sljit_si set_flags,
1975 sljit_si dst, sljit_sw dstw,
1976 sljit_si src1, sljit_sw src1w,
1977 sljit_si src2, sljit_sw src2w)
1978 {
1979 /* The CPU does not set flags if the shift count is 0. */
1980 if (src2 & SLJIT_IMM) {
1981 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1982 if ((src2w & 0x3f) != 0 || (compiler->mode32 && (src2w & 0x1f) != 0))
1983 return emit_shift(compiler, mode, dst, dstw, src1, src1w, src2, src2w);
1984 #else
1985 if ((src2w & 0x1f) != 0)
1986 return emit_shift(compiler, mode, dst, dstw, src1, src1w, src2, src2w);
1987 #endif
1988 if (!set_flags)
1989 return emit_mov(compiler, dst, dstw, src1, src1w);
1990 /* OR dst, src, 0 */
1991 return emit_cum_binary(compiler, OR_r_rm, OR_rm_r, OR, OR_EAX_i32,
1992 dst, dstw, src1, src1w, SLJIT_IMM, 0);
1993 }
1994
1995 if (!set_flags)
1996 return emit_shift(compiler, mode, dst, dstw, src1, src1w, src2, src2w);
1997
1998 if (!(dst >= SLJIT_TEMPORARY_REG1 && dst <= SLJIT_NO_REGISTERS))
1999 FAIL_IF(emit_cmp_binary(compiler, src1, src1w, SLJIT_IMM, 0));
2000
2001 FAIL_IF(emit_shift(compiler,mode, dst, dstw, src1, src1w, src2, src2w));
2002
2003 if (dst >= SLJIT_TEMPORARY_REG1 && dst <= SLJIT_NO_REGISTERS)
2004 return emit_cmp_binary(compiler, dst, dstw, SLJIT_IMM, 0);
2005 return SLJIT_SUCCESS;
2006 }
2007
2008 SLJIT_API_FUNC_ATTRIBUTE sljit_si sljit_emit_op2(struct sljit_compiler *compiler, sljit_si op,
2009 sljit_si dst, sljit_sw dstw,
2010 sljit_si src1, sljit_sw src1w,
2011 sljit_si src2, sljit_sw src2w)
2012 {
2013 CHECK_ERROR();
2014 check_sljit_emit_op2(compiler, op, dst, dstw, src1, src1w, src2, src2w);
2015 ADJUST_LOCAL_OFFSET(dst, dstw);
2016 ADJUST_LOCAL_OFFSET(src1, src1w);
2017 ADJUST_LOCAL_OFFSET(src2, src2w);
2018
2019 CHECK_EXTRA_REGS(dst, dstw, (void)0);
2020 CHECK_EXTRA_REGS(src1, src1w, (void)0);
2021 CHECK_EXTRA_REGS(src2, src2w, (void)0);
2022 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2023 compiler->mode32 = op & SLJIT_INT_OP;
2024 #endif
2025
2026 if (GET_OPCODE(op) >= SLJIT_MUL) {
2027 if (SLJIT_UNLIKELY(GET_FLAGS(op)))
2028 compiler->flags_saved = 0;
2029 else if (SLJIT_UNLIKELY(op & SLJIT_KEEP_FLAGS) && !compiler->flags_saved)
2030 FAIL_IF(emit_save_flags(compiler));
2031 }
2032
2033 switch (GET_OPCODE(op)) {
2034 case SLJIT_ADD:
2035 if (!GET_FLAGS(op)) {
2036 if (emit_lea_binary(compiler, dst, dstw, src1, src1w, src2, src2w) != SLJIT_ERR_UNSUPPORTED)
2037 return compiler->error;
2038 }
2039 else
2040 compiler->flags_saved = 0;
2041 if (SLJIT_UNLIKELY(op & SLJIT_KEEP_FLAGS) && !compiler->flags_saved)
2042 FAIL_IF(emit_save_flags(compiler));
2043 return emit_cum_binary(compiler, ADD_r_rm, ADD_rm_r, ADD, ADD_EAX_i32,
2044 dst, dstw, src1, src1w, src2, src2w);
2045 case SLJIT_ADDC:
2046 if (SLJIT_UNLIKELY(compiler->flags_saved)) /* C flag must be restored. */
2047 FAIL_IF(emit_restore_flags(compiler, 1));
2048 else if (SLJIT_UNLIKELY(op & SLJIT_KEEP_FLAGS))
2049 FAIL_IF(emit_save_flags(compiler));
2050 if (SLJIT_UNLIKELY(GET_FLAGS(op)))
2051 compiler->flags_saved = 0;
2052 return emit_cum_binary(compiler, ADC_r_rm, ADC_rm_r, ADC, ADC_EAX_i32,
2053 dst, dstw, src1, src1w, src2, src2w);
2054 case SLJIT_SUB:
2055 if (!GET_FLAGS(op)) {
2056 if ((src2 & SLJIT_IMM) && emit_lea_binary(compiler, dst, dstw, src1, src1w, SLJIT_IMM, -src2w) != SLJIT_ERR_UNSUPPORTED)
2057 return compiler->error;
2058 }
2059 else
2060 compiler->flags_saved = 0;
2061 if (SLJIT_UNLIKELY(op & SLJIT_KEEP_FLAGS) && !compiler->flags_saved)
2062 FAIL_IF(emit_save_flags(compiler));
2063 if (dst == SLJIT_UNUSED)
2064 return emit_cmp_binary(compiler, src1, src1w, src2, src2w);
2065 return emit_non_cum_binary(compiler, SUB_r_rm, SUB_rm_r, SUB, SUB_EAX_i32,
2066 dst, dstw, src1, src1w, src2, src2w);
2067 case SLJIT_SUBC:
2068 if (SLJIT_UNLIKELY(compiler->flags_saved)) /* C flag must be restored. */
2069 FAIL_IF(emit_restore_flags(compiler, 1));
2070 else if (SLJIT_UNLIKELY(op & SLJIT_KEEP_FLAGS))
2071 FAIL_IF(emit_save_flags(compiler));
2072 if (SLJIT_UNLIKELY(GET_FLAGS(op)))
2073 compiler->flags_saved = 0;
2074 return emit_non_cum_binary(compiler, SBB_r_rm, SBB_rm_r, SBB, SBB_EAX_i32,
2075 dst, dstw, src1, src1w, src2, src2w);
2076 case SLJIT_MUL:
2077 return emit_mul(compiler, dst, dstw, src1, src1w, src2, src2w);
2078 case SLJIT_AND:
2079 if (dst == SLJIT_UNUSED)
2080 return emit_test_binary(compiler, src1, src1w, src2, src2w);
2081 return emit_cum_binary(compiler, AND_r_rm, AND_rm_r, AND, AND_EAX_i32,
2082 dst, dstw, src1, src1w, src2, src2w);
2083 case SLJIT_OR:
2084 return emit_cum_binary(compiler, OR_r_rm, OR_rm_r, OR, OR_EAX_i32,
2085 dst, dstw, src1, src1w, src2, src2w);
2086 case SLJIT_XOR:
2087 return emit_cum_binary(compiler, XOR_r_rm, XOR_rm_r, XOR, XOR_EAX_i32,
2088 dst, dstw, src1, src1w, src2, src2w);
2089 case SLJIT_SHL:
2090 return emit_shift_with_flags(compiler, SHL, GET_FLAGS(op),
2091 dst, dstw, src1, src1w, src2, src2w);
2092 case SLJIT_LSHR:
2093 return emit_shift_with_flags(compiler, SHR, GET_FLAGS(op),
2094 dst, dstw, src1, src1w, src2, src2w);
2095 case SLJIT_ASHR:
2096 return emit_shift_with_flags(compiler, SAR, GET_FLAGS(op),
2097 dst, dstw, src1, src1w, src2, src2w);
2098 }
2099
2100 return SLJIT_SUCCESS;
2101 }
2102
2103 SLJIT_API_FUNC_ATTRIBUTE sljit_si sljit_get_register_index(sljit_si reg)
2104 {
2105 check_sljit_get_register_index(reg);
2106 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
2107 if (reg == SLJIT_TEMPORARY_EREG1 || reg == SLJIT_TEMPORARY_EREG2
2108 || reg == SLJIT_SAVED_EREG1 || reg == SLJIT_SAVED_EREG2)
2109 return -1;
2110 #endif
2111 return reg_map[reg];
2112 }
2113
2114 SLJIT_API_FUNC_ATTRIBUTE sljit_si sljit_emit_op_custom(struct sljit_compiler *compiler,
2115 void *instruction, sljit_si size)
2116 {
2117 sljit_ub *buf;
2118
2119 CHECK_ERROR();
2120 check_sljit_emit_op_custom(compiler, instruction, size);
2121 SLJIT_ASSERT(size > 0 && size < 16);
2122
2123 buf = (sljit_ub*)ensure_buf(compiler, 1 + size);
2124 FAIL_IF(!buf);
2125 INC_SIZE(size);
2126 SLJIT_MEMMOVE(buf, instruction, size);
2127 return SLJIT_SUCCESS;
2128 }
2129
2130 /* --------------------------------------------------------------------- */
2131 /* Floating point operators */
2132 /* --------------------------------------------------------------------- */
2133
2134 #if (defined SLJIT_SSE2 && SLJIT_SSE2)
2135
2136 /* Alignment + 2 * 16 bytes. */
2137 static sljit_si sse2_data[3 + (4 + 4) * 2];
2138 static sljit_si *sse2_buffer;
2139
2140 static void init_compiler(void)
2141 {
2142 sse2_buffer = (sljit_si*)(((sljit_uw)sse2_data + 15) & ~0xf);
2143 /* Single precision constants. */
2144 sse2_buffer[0] = 0x80000000;
2145 sse2_buffer[4] = 0x7fffffff;
2146 /* Double precision constants. */
2147 sse2_buffer[8] = 0;
2148 sse2_buffer[9] = 0x80000000;
2149 sse2_buffer[12] = 0xffffffff;
2150 sse2_buffer[13] = 0x7fffffff;
2151 }
2152
2153 #endif
2154
2155 SLJIT_API_FUNC_ATTRIBUTE sljit_si sljit_is_fpu_available(void)
2156 {
2157 #if (defined SLJIT_SSE2 && SLJIT_SSE2)
2158 #if (defined SLJIT_DETECT_SSE2 && SLJIT_DETECT_SSE2)
2159 static sljit_si sse2_available = -1;
2160 sljit_si features;
2161
2162 if (sse2_available != -1)
2163 return sse2_available;
2164
2165 #ifdef __GNUC__
2166 /* AT&T syntax. */
2167 asm (
2168 "pushl %%ebx\n"
2169 "movl $0x1, %%eax\n"
2170 "cpuid\n"
2171 "popl %%ebx\n"
2172 "movl %%edx, %0\n"
2173 : "=g" (features)
2174 :
2175 : "%eax", "%ecx", "%edx"
2176 );
2177 #elif defined(_MSC_VER) || defined(__BORLANDC__)
2178 /* Intel syntax. */
2179 __asm {
2180 mov eax, 1
2181 push ebx
2182 cpuid
2183 pop ebx
2184 mov features, edx
2185 }
2186 #else
2187 #error "SLJIT_DETECT_SSE2 is not implemented for this C compiler"
2188 #endif
2189 sse2_available = (features >> 26) & 0x1;
2190 return sse2_available;
2191 #else
2192 return 1;
2193 #endif
2194 #else
2195 return 0;
2196 #endif
2197 }
2198
2199 #if (defined SLJIT_SSE2 && SLJIT_SSE2)
2200
2201 static sljit_si emit_sse2(struct sljit_compiler *compiler, sljit_ub opcode,
2202 sljit_si single, sljit_si xmm1, sljit_si xmm2, sljit_sw xmm2w)
2203 {
2204 sljit_ub *buf;
2205
2206 buf = emit_x86_instruction(compiler, 2 | (single ? EX86_PREF_F3 : EX86_PREF_F2) | EX86_SSE2, xmm1, 0, xmm2, xmm2w);
2207 FAIL_IF(!buf);
2208 *buf++ = GROUP_0F;
2209 *buf = opcode;
2210 return SLJIT_SUCCESS;
2211 }
2212
2213 static sljit_si emit_sse2_logic(struct sljit_compiler *compiler, sljit_ub opcode,
2214 sljit_si pref66, sljit_si xmm1, sljit_si xmm2, sljit_sw xmm2w)
2215 {
2216 sljit_ub *buf;
2217
2218 buf = emit_x86_instruction(compiler, 2 | (pref66 ? EX86_PREF_66 : 0) | EX86_SSE2, xmm1, 0, xmm2, xmm2w);
2219 FAIL_IF(!buf);
2220 *buf++ = GROUP_0F;
2221 *buf = opcode;
2222 return SLJIT_SUCCESS;
2223 }
2224
2225 static SLJIT_INLINE sljit_si emit_sse2_load(struct sljit_compiler *compiler,
2226 sljit_si single, sljit_si dst, sljit_si src, sljit_sw srcw)
2227 {
2228 return emit_sse2(compiler, MOVSD_x_xm, single, dst, src, srcw);
2229 }
2230
2231 static SLJIT_INLINE sljit_si emit_sse2_store(struct sljit_compiler *compiler,
2232 sljit_si single, sljit_si dst, sljit_sw dstw, sljit_si src)
2233 {
2234 return emit_sse2(compiler, MOVSD_xm_x, single, src, dst, dstw);
2235 }
2236
2237 SLJIT_API_FUNC_ATTRIBUTE sljit_si sljit_emit_fop1(struct sljit_compiler *compiler, sljit_si op,
2238 sljit_si dst, sljit_sw dstw,
2239 sljit_si src, sljit_sw srcw)
2240 {
2241 sljit_si dst_r;
2242
2243 CHECK_ERROR();
2244 check_sljit_emit_fop1(compiler, op, dst, dstw, src, srcw);
2245
2246 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2247 compiler->mode32 = 1;
2248 #endif
2249
2250 if (GET_OPCODE(op) == SLJIT_CMPD) {
2251 compiler->flags_saved = 0;
2252 if (dst <= SLJIT_FLOAT_REG6)
2253 dst_r = dst;
2254 else {
2255 dst_r = TMP_FREG;
2256 FAIL_IF(emit_sse2_load(compiler, op & SLJIT_SINGLE_OP, dst_r, dst, dstw));
2257 }
2258 return emit_sse2_logic(compiler, UCOMISD_x_xm, !(op & SLJIT_SINGLE_OP), dst_r, src, srcw);
2259 }
2260
2261 if (op == SLJIT_MOVD) {
2262 if (dst <= SLJIT_FLOAT_REG6)
2263 return emit_sse2_load(compiler, op & SLJIT_SINGLE_OP, dst, src, srcw);
2264 if (src <= SLJIT_FLOAT_REG6)
2265 return emit_sse2_store(compiler, op & SLJIT_SINGLE_OP, dst, dstw, src);
2266 FAIL_IF(emit_sse2_load(compiler, op & SLJIT_SINGLE_OP, TMP_FREG, src, srcw));
2267 return emit_sse2_store(compiler, op & SLJIT_SINGLE_OP, dst, dstw, TMP_FREG);
2268 }
2269
2270 if (dst >= SLJIT_FLOAT_REG1 && dst <= SLJIT_FLOAT_REG6) {
2271 dst_r = dst;
2272 if (dst != src)
2273 FAIL_IF(emit_sse2_load(compiler, op & SLJIT_SINGLE_OP, dst_r, src, srcw));
2274 }
2275 else {
2276 dst_r = TMP_FREG;
2277 FAIL_IF(emit_sse2_load(compiler, op & SLJIT_SINGLE_OP, dst_r, src, srcw));
2278 }
2279
2280 switch (GET_OPCODE(op)) {
2281 case SLJIT_NEGD:
2282 FAIL_IF(emit_sse2_logic(compiler, XORPD_x_xm, 1, dst_r, SLJIT_MEM0(), (sljit_sw)(op & SLJIT_SINGLE_OP ? sse2_buffer : sse2_buffer + 8)));
2283 break;
2284
2285 case SLJIT_ABSD:
2286 FAIL_IF(emit_sse2_logic(compiler, ANDPD_x_xm, 1, dst_r, SLJIT_MEM0(), (sljit_sw)(op & SLJIT_SINGLE_OP ? sse2_buffer + 4 : sse2_buffer + 12)));
2287 break;
2288 }
2289
2290 if (dst_r == TMP_FREG)
2291 return emit_sse2_store(compiler, op & SLJIT_SINGLE_OP, dst, dstw, TMP_FREG);
2292 return SLJIT_SUCCESS;
2293 }
2294
2295 SLJIT_API_FUNC_ATTRIBUTE sljit_si sljit_emit_fop2(struct sljit_compiler *compiler, sljit_si op,
2296 sljit_si dst, sljit_sw dstw,
2297 sljit_si src1, sljit_sw src1w,
2298 sljit_si src2, sljit_sw src2w)
2299 {
2300 sljit_si dst_r;
2301
2302 CHECK_ERROR();
2303 check_sljit_emit_fop2(compiler, op, dst, dstw, src1, src1w, src2, src2w);
2304
2305 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2306 compiler->mode32 = 1;
2307 #endif
2308
2309 if (dst <= SLJIT_FLOAT_REG6) {
2310 dst_r = dst;
2311 if (dst == src1)
2312 ; /* Do nothing here. */
2313 else if (dst == src2 && (op == SLJIT_ADDD || op == SLJIT_MULD)) {
2314 /* Swap arguments. */
2315 src2 = src1;
2316 src2w = src1w;
2317 }
2318 else if (dst != src2)
2319 FAIL_IF(emit_sse2_load(compiler, op & SLJIT_SINGLE_OP, dst_r, src1, src1w));
2320 else {
2321 dst_r = TMP_FREG;
2322 FAIL_IF(emit_sse2_load(compiler, op & SLJIT_SINGLE_OP, TMP_FREG, src1, src1w));
2323 }
2324 }
2325 else {
2326 dst_r = TMP_FREG;
2327 FAIL_IF(emit_sse2_load(compiler, op & SLJIT_SINGLE_OP, TMP_FREG, src1, src1w));
2328 }
2329
2330 switch (GET_OPCODE(op)) {
2331 case SLJIT_ADDD:
2332 FAIL_IF(emit_sse2(compiler, ADDSD_x_xm, op & SLJIT_SINGLE_OP, dst_r, src2, src2w));
2333 break;
2334
2335 case SLJIT_SUBD:
2336 FAIL_IF(emit_sse2(compiler, SUBSD_x_xm, op & SLJIT_SINGLE_OP, dst_r, src2, src2w));
2337 break;
2338
2339 case SLJIT_MULD:
2340 FAIL_IF(emit_sse2(compiler, MULSD_x_xm, op & SLJIT_SINGLE_OP, dst_r, src2, src2w));
2341 break;
2342
2343 case SLJIT_DIVD:
2344 FAIL_IF(emit_sse2(compiler, DIVSD_x_xm, op & SLJIT_SINGLE_OP, dst_r, src2, src2w));
2345 break;
2346 }
2347
2348 if (dst_r == TMP_FREG)
2349 return emit_sse2_store(compiler, op & SLJIT_SINGLE_OP, dst, dstw, TMP_FREG);
2350 return SLJIT_SUCCESS;
2351 }
2352
2353 #else
2354
2355 SLJIT_API_FUNC_ATTRIBUTE sljit_si sljit_emit_fop1(struct sljit_compiler *compiler, sljit_si op,
2356 sljit_si dst, sljit_sw dstw,
2357 sljit_si src, sljit_sw srcw)
2358 {
2359 CHECK_ERROR();
2360 /* Should cause an assertion fail. */
2361 check_sljit_emit_fop1(compiler, op, dst, dstw, src, srcw);
2362 compiler->error = SLJIT_ERR_UNSUPPORTED;
2363 return SLJIT_ERR_UNSUPPORTED;
2364 }
2365
2366 SLJIT_API_FUNC_ATTRIBUTE sljit_si sljit_emit_fop2(struct sljit_compiler *compiler, sljit_si op,
2367 sljit_si dst, sljit_sw dstw,
2368 sljit_si src1, sljit_sw src1w,
2369 sljit_si src2, sljit_sw src2w)
2370 {
2371 CHECK_ERROR();
2372 /* Should cause an assertion fail. */
2373 check_sljit_emit_fop2(compiler, op, dst, dstw, src1, src1w, src2, src2w);
2374 compiler->error = SLJIT_ERR_UNSUPPORTED;
2375 return SLJIT_ERR_UNSUPPORTED;
2376 }
2377
2378 #endif
2379
2380 /* --------------------------------------------------------------------- */
2381 /* Conditional instructions */
2382 /* --------------------------------------------------------------------- */
2383
2384 SLJIT_API_FUNC_ATTRIBUTE struct sljit_label* sljit_emit_label(struct sljit_compiler *compiler)
2385 {
2386 sljit_ub *buf;
2387 struct sljit_label *label;
2388
2389 CHECK_ERROR_PTR();
2390 check_sljit_emit_label(compiler);
2391
2392 /* We should restore the flags before the label,
2393 since other taken jumps has their own flags as well. */
2394 if (SLJIT_UNLIKELY(compiler->flags_saved))
2395 PTR_FAIL_IF(emit_restore_flags(compiler, 0));
2396
2397 if (compiler->last_label && compiler->last_label->size == compiler->size)
2398 return compiler->last_label;
2399
2400 label = (struct sljit_label*)ensure_abuf(compiler, sizeof(struct sljit_label));
2401 PTR_FAIL_IF(!label);
2402 set_label(label, compiler);
2403
2404 buf = (sljit_ub*)ensure_buf(compiler, 2);
2405 PTR_FAIL_IF(!buf);
2406
2407 *buf++ = 0;
2408 *buf++ = 0;
2409
2410 return label;
2411 }
2412
2413 SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_jump(struct sljit_compiler *compiler, sljit_si type)
2414 {
2415 sljit_ub *buf;
2416 struct sljit_jump *jump;
2417
2418 CHECK_ERROR_PTR();
2419 check_sljit_emit_jump(compiler, type);
2420
2421 if (SLJIT_UNLIKELY(compiler->flags_saved)) {
2422 if ((type & 0xff) <= SLJIT_JUMP)
2423 PTR_FAIL_IF(emit_restore_flags(compiler, 0));
2424 compiler->flags_saved = 0;
2425 }
2426
2427 jump = (struct sljit_jump*)ensure_abuf(compiler, sizeof(struct sljit_jump));
2428 PTR_FAIL_IF_NULL(jump);
2429 set_jump(jump, compiler, type & SLJIT_REWRITABLE_JUMP);
2430 type &= 0xff;
2431
2432 if (type >= SLJIT_CALL1)
2433 PTR_FAIL_IF(call_with_args(compiler, type));
2434
2435 /* Worst case size. */
2436 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
2437 compiler->size += (type >= SLJIT_JUMP) ? 5 : 6;
2438 #else
2439 compiler->size += (type >= SLJIT_JUMP) ? (10 + 3) : (2 + 10 + 3);
2440 #endif
2441
2442 buf = (sljit_ub*)ensure_buf(compiler, 2);
2443 PTR_FAIL_IF_NULL(buf);
2444
2445 *buf++ = 0;
2446 *buf++ = type + 4;
2447 return jump;
2448 }
2449
2450 SLJIT_API_FUNC_ATTRIBUTE sljit_si sljit_emit_ijump(struct sljit_compiler *compiler, sljit_si type, sljit_si src, sljit_sw srcw)
2451 {
2452 sljit_ub *code;
2453 struct sljit_jump *jump;
2454
2455 CHECK_ERROR();
2456 check_sljit_emit_ijump(compiler, type, src, srcw);
2457 ADJUST_LOCAL_OFFSET(src, srcw);
2458
2459 CHECK_EXTRA_REGS(src, srcw, (void)0);
2460
2461 if (SLJIT_UNLIKELY(compiler->flags_saved)) {
2462 if (type <= SLJIT_JUMP)
2463 FAIL_IF(emit_restore_flags(compiler, 0));
2464 compiler->flags_saved = 0;
2465 }
2466
2467 if (type >= SLJIT_CALL1) {
2468 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
2469 #if (defined SLJIT_X86_32_FASTCALL && SLJIT_X86_32_FASTCALL)
2470 if (src == SLJIT_TEMPORARY_REG3) {
2471 EMIT_MOV(compiler, TMP_REGISTER, 0, src, 0);
2472 src = TMP_REGISTER;
2473 }
2474 if (src == SLJIT_MEM1(SLJIT_LOCALS_REG) && type >= SLJIT_CALL3)
2475 srcw += sizeof(sljit_sw);
2476 #else
2477 if (src == SLJIT_MEM1(SLJIT_LOCALS_REG))
2478 srcw += sizeof(sljit_sw) * (type - SLJIT_CALL0);
2479 #endif
2480 #endif
2481 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64) && defined(_WIN64)
2482 if (src == SLJIT_TEMPORARY_REG3) {
2483 EMIT_MOV(compiler, TMP_REGISTER, 0, src, 0);
2484 src = TMP_REGISTER;
2485 }
2486 #endif
2487 FAIL_IF(call_with_args(compiler, type));
2488 }
2489
2490 if (src == SLJIT_IMM) {
2491 jump = (struct sljit_jump*)ensure_abuf(compiler, sizeof(struct sljit_jump));
2492 FAIL_IF_NULL(jump);
2493 set_jump(jump, compiler, JUMP_ADDR);
2494 jump->u.target = srcw;
2495
2496 /* Worst case size. */
2497 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
2498 compiler->size += 5;
2499 #else
2500 compiler->size += 10 + 3;
2501 #endif
2502
2503 code = (sljit_ub*)ensure_buf(compiler, 2);
2504 FAIL_IF_NULL(code);
2505
2506 *code++ = 0;
2507 *code++ = type + 4;
2508 }
2509 else {
2510 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2511 /* REX_W is not necessary (src is not immediate). */
2512 compiler->mode32 = 1;
2513 #endif
2514 code = emit_x86_instruction(compiler, 1, 0, 0, src, srcw);
2515 FAIL_IF(!code);
2516 *code++ = GROUP_FF;
2517 *code |= (type >= SLJIT_FAST_CALL) ? CALL_rm : JMP_rm;
2518 }
2519 return SLJIT_SUCCESS;
2520 }
2521
2522 SLJIT_API_FUNC_ATTRIBUTE sljit_si sljit_emit_cond_value(struct sljit_compiler *compiler, sljit_si op, sljit_si dst, sljit_sw dstw, sljit_si type)
2523 {
2524 sljit_ub *buf;
2525 sljit_ub cond_set = 0;
2526 sljit_si dst_save = dst;
2527 sljit_sw dstw_save = dstw;
2528 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2529 sljit_si reg;
2530 #endif
2531
2532 CHECK_ERROR();
2533 check_sljit_emit_cond_value(compiler, op, dst, dstw, type);
2534
2535 if (dst == SLJIT_UNUSED)
2536 return SLJIT_SUCCESS;
2537
2538 ADJUST_LOCAL_OFFSET(dst, dstw);
2539 CHECK_EXTRA_REGS(dst, dstw, (void)0);
2540 if (SLJIT_UNLIKELY(compiler->flags_saved))
2541 FAIL_IF(emit_restore_flags(compiler, op & SLJIT_KEEP_FLAGS));
2542
2543 switch (type) {
2544 case SLJIT_C_EQUAL:
2545 case SLJIT_C_FLOAT_EQUAL:
2546 cond_set = 0x94 /* sete */;
2547 break;
2548
2549 case SLJIT_C_NOT_EQUAL:
2550 case SLJIT_C_FLOAT_NOT_EQUAL:
2551 cond_set = 0x95 /* setne */;
2552 break;
2553
2554 case SLJIT_C_LESS:
2555 case SLJIT_C_FLOAT_LESS:
2556 cond_set = 0x92 /* setnae */;
2557 break;
2558
2559 case SLJIT_C_GREATER_EQUAL:
2560 case SLJIT_C_FLOAT_GREATER_EQUAL:
2561 cond_set = 0x93 /* setnb */;
2562 break;
2563
2564 case SLJIT_C_GREATER:
2565 case SLJIT_C_FLOAT_GREATER:
2566 cond_set = 0x97 /* seta */;
2567 break;
2568
2569 case SLJIT_C_LESS_EQUAL:
2570 case SLJIT_C_FLOAT_LESS_EQUAL:
2571 cond_set = 0x96 /* setbe */;
2572 break;
2573
2574 case SLJIT_C_SIG_LESS:
2575 cond_set = 0x9c /* setnge */;
2576 break;
2577
2578 case SLJIT_C_SIG_GREATER_EQUAL:
2579 cond_set = 0x9d /* setnl */;
2580 break;
2581
2582 case SLJIT_C_SIG_GREATER:
2583 cond_set = 0x9f /* setg */;
2584 break;
2585
2586 case SLJIT_C_SIG_LESS_EQUAL:
2587 cond_set = 0x9e /* setle */;
2588 break;
2589
2590 case SLJIT_C_OVERFLOW:
2591 case SLJIT_C_MUL_OVERFLOW:
2592 cond_set = 0x90 /* seto */;
2593 break;
2594
2595 case SLJIT_C_NOT_OVERFLOW:
2596 case SLJIT_C_MUL_NOT_OVERFLOW:
2597 cond_set = 0x91 /* setno */;
2598 break;
2599
2600 case SLJIT_C_FLOAT_UNORDERED:
2601 cond_set = 0x9a /* setp */;
2602 break;
2603
2604 case SLJIT_C_FLOAT_ORDERED:
2605 cond_set = 0x9b /* setpo */;
2606 break;
2607 }
2608
2609 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2610 reg = (op == SLJIT_MOV && dst >= SLJIT_TEMPORARY_REG1 && dst <= SLJIT_NO_REGISTERS) ? dst : TMP_REGISTER;
2611
2612 buf = (sljit_ub*)ensure_buf(compiler, 1 + 4 + 4);
2613 FAIL_IF(!buf);
2614 INC_SIZE(4 + 4);
2615 /* Set low register to conditional flag. */
2616 *buf++ = (reg_map[reg] <= 7) ? REX : REX_B;
2617 *buf++ = GROUP_0F;
2618 *buf++ = cond_set;
2619 *buf++ = MOD_REG | reg_lmap[reg];
2620 *buf++ = REX_W | (reg_map[reg] <= 7 ? 0 : (REX_B | REX_R));
2621 *buf++ = GROUP_0F;
2622 *buf++ = MOVZX_r_rm8;
2623 *buf = MOD_REG | (reg_lmap[reg] << 3) | reg_lmap[reg];
2624
2625 if (reg == TMP_REGISTER) {
2626 if (op == SLJIT_MOV) {
2627 compiler->mode32 = 0;
2628 EMIT_MOV(compiler, dst, dstw, TMP_REGISTER, 0);
2629 }
2630 else {
2631 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE) || (defined SLJIT_DEBUG && SLJIT_DEBUG)
2632 compiler->skip_checks = 1;
2633 #endif
2634 return sljit_emit_op2(compiler, op, dst_save, dstw_save, dst_save, dstw_save, TMP_REGISTER, 0);
2635 }
2636 }
2637 #else
2638 if (op == SLJIT_MOV) {
2639 if (dst >= SLJIT_TEMPORARY_REG1 && dst <= SLJIT_TEMPORARY_REG3) {
2640 buf = (sljit_ub*)ensure_buf(compiler, 1 + 3 + 3);
2641 FAIL_IF(!buf);
2642 INC_SIZE(3 + 3);
2643 /* Set low byte to conditional flag. */
2644 *buf++ = GROUP_0F;
2645 *buf++ = cond_set;
2646 *buf++ = MOD_REG | reg_map[dst];
2647
2648 *buf++ = GROUP_0F;
2649 *buf++ = MOVZX_r_rm8;
2650 *buf = MOD_REG | (reg_map[dst] << 3) | reg_map[dst];
2651 }
2652 else {
2653 EMIT_MOV(compiler, TMP_REGISTER, 0, SLJIT_TEMPORARY_REG1, 0);
2654
2655 buf = (sljit_ub*)ensure_buf(compiler, 1 + 3 + 3);
2656 FAIL_IF(!buf);
2657 INC_SIZE(3 + 3);
2658 /* Set al to conditional flag. */
2659 *buf++ = GROUP_0F;
2660 *buf++ = cond_set;
2661 *buf++ = MOD_REG;
2662
2663 *buf++ = GROUP_0F;
2664 *buf++ = MOVZX_r_rm8;
2665 if (dst >= SLJIT_SAVED_REG1 && dst <= SLJIT_NO_REGISTERS)
2666 *buf = MOD_REG | (reg_map[dst] << 3);
2667 else {
2668 *buf = MOD_REG;
2669 EMIT_MOV(compiler, dst, dstw, SLJIT_TEMPORARY_REG1, 0);
2670 }
2671
2672 EMIT_MOV(compiler, SLJIT_TEMPORARY_REG1, 0, TMP_REGISTER, 0);
2673 }
2674 }
2675 else {
2676 if (dst >= SLJIT_TEMPORARY_REG1 && dst <= SLJIT_TEMPORARY_REG3) {
2677 EMIT_MOV(compiler, TMP_REGISTER, 0, dst, 0);
2678 buf = (sljit_ub*)ensure_buf(compiler, 1 + 3);
2679 FAIL_IF(!buf);
2680 INC_SIZE(3);
2681
2682 *buf++ = GROUP_0F;
2683 *buf++ = cond_set;
2684 *buf++ = MOD_REG | reg_map[dst];
2685 }
2686 else {
2687 EMIT_MOV(compiler, TMP_REGISTER, 0, SLJIT_TEMPORARY_REG1, 0);
2688
2689 buf = (sljit_ub*)ensure_buf(compiler, 1 + 3 + 3 + 1);
2690 FAIL_IF(!buf);
2691 INC_SIZE(3 + 3 + 1);
2692 /* Set al to conditional flag. */
2693 *buf++ = GROUP_0F;
2694 *buf++ = cond_set;
2695 *buf++ = MOD_REG;
2696
2697 *buf++ = GROUP_0F;
2698 *buf++ = MOVZX_r_rm8;
2699 *buf++ = MOD_REG;
2700
2701 *buf++ = XCHG_EAX_r + reg_map[TMP_REGISTER];
2702 }
2703 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE) || (defined SLJIT_DEBUG && SLJIT_DEBUG)
2704 compiler->skip_checks = 1;
2705 #endif
2706 return sljit_emit_op2(compiler, op, dst_save, dstw_save, dst_save, dstw_save, TMP_REGISTER, 0);
2707 }
2708 #endif
2709
2710 return SLJIT_SUCCESS;
2711 }
2712
2713 SLJIT_API_FUNC_ATTRIBUTE sljit_si sljit_get_local_base(struct sljit_compiler *compiler, sljit_si dst, sljit_sw dstw, sljit_sw offset)
2714 {
2715 CHECK_ERROR();
2716 check_sljit_get_local_base(compiler, dst, dstw, offset);
2717 ADJUST_LOCAL_OFFSET(dst, dstw);
2718
2719 CHECK_EXTRA_REGS(dst, dstw, (void)0);
2720
2721 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2722 compiler->mode32 = 0;
2723 #endif
2724
2725 ADJUST_LOCAL_OFFSET(SLJIT_MEM1(SLJIT_LOCALS_REG), offset);
2726
2727 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2728 if (NOT_HALFWORD(offset)) {
2729 FAIL_IF(emit_load_imm64(compiler, TMP_REGISTER, offset));
2730 #if (defined SLJIT_DEBUG && SLJIT_DEBUG)
2731 SLJIT_ASSERT(emit_lea_binary(compiler, dst, dstw, SLJIT_LOCALS_REG, 0, TMP_REGISTER, 0) != SLJIT_ERR_UNSUPPORTED);
2732 return compiler->error;
2733 #else
2734 return emit_lea_binary(compiler, dst, dstw, SLJIT_LOCALS_REG, 0, TMP_REGISTER, 0);
2735 #endif
2736 }
2737 #endif
2738
2739 if (offset != 0)
2740 return emit_lea_binary(compiler, dst, dstw, SLJIT_LOCALS_REG, 0, SLJIT_IMM, offset);
2741 return emit_mov(compiler, dst, dstw, SLJIT_LOCALS_REG, 0);
2742 }
2743
2744 SLJIT_API_FUNC_ATTRIBUTE struct sljit_const* sljit_emit_const(struct sljit_compiler *compiler, sljit_si dst, sljit_sw dstw, sljit_sw init_value)
2745 {
2746 sljit_ub *buf;
2747 struct sljit_const *const_;
2748 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2749 sljit_si reg;
2750 #endif
2751
2752 CHECK_ERROR_PTR();
2753 check_sljit_emit_const(compiler, dst, dstw, init_value);
2754 ADJUST_LOCAL_OFFSET(dst, dstw);
2755
2756 CHECK_EXTRA_REGS(dst, dstw, (void)0);
2757
2758 const_ = (struct sljit_const*)ensure_abuf(compiler, sizeof(struct sljit_const));
2759 PTR_FAIL_IF(!const_);
2760 set_const(const_, compiler);
2761
2762 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2763 compiler->mode32 = 0;
2764 reg = (dst >= SLJIT_TEMPORARY_REG1 && dst <= SLJIT_NO_REGISTERS) ? dst : TMP_REGISTER;
2765
2766 if (emit_load_imm64(compiler, reg, init_value))
2767 return NULL;
2768 #else
2769 if (dst == SLJIT_UNUSED)
2770 dst = TMP_REGISTER;
2771
2772 if (emit_mov(compiler, dst, dstw, SLJIT_IMM, init_value))
2773 return NULL;
2774 #endif
2775
2776 buf = (sljit_ub*)ensure_buf(compiler, 2);
2777 PTR_FAIL_IF(!buf);
2778
2779 *buf++ = 0;
2780 *buf++ = 1;
2781
2782 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2783 if (reg == TMP_REGISTER && dst != SLJIT_UNUSED)
2784 if (emit_mov(compiler, dst, dstw, TMP_REGISTER, 0))
2785 return NULL;
2786 #endif
2787
2788 return const_;
2789 }
2790
2791 SLJIT_API_FUNC_ATTRIBUTE void sljit_set_jump_addr(sljit_uw addr, sljit_uw new_addr)
2792 {
2793 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
2794 *(sljit_sw*)addr = new_addr - (addr + 4);
2795 #else
2796 *(sljit_uw*)addr = new_addr;
2797 #endif
2798 }
2799
2800 SLJIT_API_FUNC_ATTRIBUTE void sljit_set_const(sljit_uw addr, sljit_sw new_constant)
2801 {
2802 *(sljit_sw*)addr = new_constant;
2803 }
Platform independent low-level JIT compiler (github mirror, with mob branch)