2 * Stack-less Just-In-Time compiler
4 * Copyright Zoltan Herczeg (hzmester@freemail.hu). All rights reserved.
6 * Redistribution and use in source and binary forms, with or without modification, are
7 * permitted provided that the following conditions are met:
9 * 1. Redistributions of source code must retain the above copyright notice, this list of
10 * conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright notice, this list
13 * of conditions and the following disclaimer in the documentation and/or other materials
14 * provided with the distribution.
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) AND CONTRIBUTORS ``AS IS'' AND ANY
17 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
19 * SHALL THE COPYRIGHT HOLDER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
21 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
22 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
23 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
24 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 /* x86 32-bit arch dependent functions. */
29 /* --------------------------------------------------------------------- */
31 /* --------------------------------------------------------------------- */
33 static sljit_s32
emit_do_imm(struct sljit_compiler
*compiler
, sljit_u8 opcode
, sljit_sw imm
)
37 inst
= (sljit_u8
*)ensure_buf(compiler
, 1 + 1 + sizeof(sljit_sw
));
39 INC_SIZE(1 + sizeof(sljit_sw
));
41 sljit_unaligned_store_sw(inst
, imm
);
45 /* Size contains the flags as well. */
46 static sljit_u8
* emit_x86_instruction(struct sljit_compiler
*compiler
, sljit_uw size
,
47 /* The register or immediate operand. */
48 sljit_s32 a
, sljit_sw imma
,
49 /* The general operand (not immediate). */
50 sljit_s32 b
, sljit_sw immb
)
55 sljit_uw flags
= size
;
58 /* Both cannot be switched on. */
59 SLJIT_ASSERT((flags
& (EX86_BIN_INS
| EX86_SHIFT_INS
)) != (EX86_BIN_INS
| EX86_SHIFT_INS
));
60 /* Size flags not allowed for typed instructions. */
61 SLJIT_ASSERT(!(flags
& (EX86_BIN_INS
| EX86_SHIFT_INS
)) || (flags
& (EX86_BYTE_ARG
| EX86_HALF_ARG
)) == 0);
62 /* Both size flags cannot be switched on. */
63 SLJIT_ASSERT((flags
& (EX86_BYTE_ARG
| EX86_HALF_ARG
)) != (EX86_BYTE_ARG
| EX86_HALF_ARG
));
64 /* SSE2 and immediate is not possible. */
65 SLJIT_ASSERT(!(a
& SLJIT_IMM
) || !(flags
& EX86_SSE2
));
66 SLJIT_ASSERT((flags
& (EX86_PREF_F2
| EX86_PREF_F3
)) != (EX86_PREF_F2
| EX86_PREF_F3
)
67 && (flags
& (EX86_PREF_F2
| EX86_PREF_66
)) != (EX86_PREF_F2
| EX86_PREF_66
)
68 && (flags
& (EX86_PREF_F3
| EX86_PREF_66
)) != (EX86_PREF_F3
| EX86_PREF_66
));
73 if (flags
& (EX86_PREF_F2
| EX86_PREF_F3
))
75 if (flags
& EX86_PREF_66
)
78 /* Calculate size of b. */
79 inst_size
+= 1; /* mod r/m byte. */
82 inst_size
+= sizeof(sljit_sw
);
84 if (immb
!= 0 && !(b
& OFFS_REG_MASK
)) {
85 /* Immediate operand. */
86 if (immb
<= 127 && immb
>= -128)
87 inst_size
+= sizeof(sljit_s8
);
89 inst_size
+= sizeof(sljit_sw
);
91 else if (reg_map
[b
& REG_MASK
] == 5) {
92 /* Swap registers if possible. */
93 if ((b
& OFFS_REG_MASK
) && (immb
& 0x3) == 0 && reg_map
[OFFS_REG(b
)] != 5)
94 b
= SLJIT_MEM
| OFFS_REG(b
) | TO_OFFS_REG(b
& REG_MASK
);
96 inst_size
+= sizeof(sljit_s8
);
99 if (reg_map
[b
& REG_MASK
] == 4 && !(b
& OFFS_REG_MASK
))
100 b
|= TO_OFFS_REG(SLJIT_SP
);
102 if (b
& OFFS_REG_MASK
)
103 inst_size
+= 1; /* SIB byte. */
107 /* Calculate size of a. */
109 if (flags
& EX86_BIN_INS
) {
110 if (imma
<= 127 && imma
>= -128) {
112 flags
|= EX86_BYTE_ARG
;
116 else if (flags
& EX86_SHIFT_INS
) {
120 flags
|= EX86_BYTE_ARG
;
122 } else if (flags
& EX86_BYTE_ARG
)
124 else if (flags
& EX86_HALF_ARG
)
125 inst_size
+= sizeof(short);
127 inst_size
+= sizeof(sljit_sw
);
130 SLJIT_ASSERT(!(flags
& EX86_SHIFT_INS
) || a
== SLJIT_PREF_SHIFT_REG
);
132 inst
= (sljit_u8
*)ensure_buf(compiler
, 1 + inst_size
);
135 /* Encoding the byte. */
137 if (flags
& EX86_PREF_F2
)
139 if (flags
& EX86_PREF_F3
)
141 if (flags
& EX86_PREF_66
)
144 buf_ptr
= inst
+ size
;
146 /* Encode mod/rm byte. */
147 if (!(flags
& EX86_SHIFT_INS
)) {
148 if ((flags
& EX86_BIN_INS
) && (a
& SLJIT_IMM
))
149 *inst
= (flags
& EX86_BYTE_ARG
) ? GROUP_BINARY_83
: GROUP_BINARY_81
;
153 else if (!(flags
& EX86_SSE2_OP1
))
154 *buf_ptr
= U8(reg_map
[a
] << 3);
156 *buf_ptr
= U8(a
<< 3);
161 *inst
= GROUP_SHIFT_1
;
163 *inst
= GROUP_SHIFT_N
;
165 *inst
= GROUP_SHIFT_CL
;
169 if (!(b
& SLJIT_MEM
)) {
170 *buf_ptr
= U8(*buf_ptr
| MOD_REG
| (!(flags
& EX86_SSE2_OP2
) ? reg_map
[b
] : b
));
172 } else if (b
& REG_MASK
) {
173 reg_map_b
= reg_map
[b
& REG_MASK
];
175 if (!(b
& OFFS_REG_MASK
) || (b
& OFFS_REG_MASK
) == TO_OFFS_REG(SLJIT_SP
)) {
176 if (immb
!= 0 || reg_map_b
== 5) {
177 if (immb
<= 127 && immb
>= -128)
183 if (!(b
& OFFS_REG_MASK
))
184 *buf_ptr
++ |= reg_map_b
;
187 *buf_ptr
++ = U8(reg_map_b
| (reg_map
[OFFS_REG(b
)] << 3));
190 if (immb
!= 0 || reg_map_b
== 5) {
191 if (immb
<= 127 && immb
>= -128)
192 *buf_ptr
++ = U8(immb
); /* 8 bit displacement. */
194 sljit_unaligned_store_sw(buf_ptr
, immb
); /* 32 bit displacement. */
195 buf_ptr
+= sizeof(sljit_sw
);
204 *buf_ptr
++ = U8(reg_map_b
| (reg_map
[OFFS_REG(b
)] << 3) | (immb
<< 6));
212 sljit_unaligned_store_sw(buf_ptr
, immb
); /* 32 bit displacement. */
213 buf_ptr
+= sizeof(sljit_sw
);
217 if (flags
& EX86_BYTE_ARG
)
219 else if (flags
& EX86_HALF_ARG
)
220 sljit_unaligned_store_s16(buf_ptr
, (sljit_s16
)imma
);
221 else if (!(flags
& EX86_SHIFT_INS
))
222 sljit_unaligned_store_sw(buf_ptr
, imma
);
225 return !(flags
& EX86_SHIFT_INS
) ? inst
: (inst
+ 1);
228 /* --------------------------------------------------------------------- */
230 /* --------------------------------------------------------------------- */
232 static sljit_u8
* generate_far_jump_code(struct sljit_jump
*jump
, sljit_u8
*code_ptr
, sljit_sw executable_offset
)
234 sljit_uw type
= jump
->flags
>> TYPE_SHIFT
;
236 if (type
== SLJIT_JUMP
) {
237 *code_ptr
++ = JMP_i32
;
240 else if (type
>= SLJIT_FAST_CALL
) {
241 *code_ptr
++ = CALL_i32
;
245 *code_ptr
++ = GROUP_0F
;
246 *code_ptr
++ = get_jump_code(type
);
250 if (jump
->flags
& JUMP_LABEL
)
251 jump
->flags
|= PATCH_MW
;
253 sljit_unaligned_store_sw(code_ptr
, (sljit_sw
)(jump
->u
.target
- (jump
->addr
+ 4) - (sljit_uw
)executable_offset
));
259 #define ENTER_TMP_TO_R4 0x00001
260 #define ENTER_TMP_TO_S 0x00002
262 SLJIT_API_FUNC_ATTRIBUTE sljit_s32
sljit_emit_enter(struct sljit_compiler
*compiler
,
263 sljit_s32 options
, sljit_s32 arg_types
, sljit_s32 scratches
, sljit_s32 saveds
,
264 sljit_s32 fscratches
, sljit_s32 fsaveds
, sljit_s32 local_size
)
266 sljit_s32 word_arg_count
, saved_arg_count
, float_arg_count
;
267 sljit_s32 size
, args_size
, types
, status
;
268 sljit_s32 kept_saveds_count
= SLJIT_KEPT_SAVEDS_COUNT(options
);
271 sljit_s32 r2_offset
= -1;
275 CHECK(check_sljit_emit_enter(compiler
, options
, arg_types
, scratches
, saveds
, fscratches
, fsaveds
, local_size
));
276 set_emit_enter(compiler
, options
, arg_types
, scratches
, saveds
, fscratches
, fsaveds
, local_size
);
278 /* Emit ENDBR32 at function entry if needed. */
279 FAIL_IF(emit_endbranch(compiler
));
281 SLJIT_COMPILE_ASSERT(SLJIT_FR0
== 1, float_register_index_start
);
283 arg_types
>>= SLJIT_ARG_SHIFT
;
287 if (options
& SLJIT_ENTER_REG_ARG
) {
288 args_size
= 3 * SSIZE_OF(sw
);
291 if ((arg_types
& SLJIT_ARG_MASK
) < SLJIT_ARG_TYPE_F64
) {
293 if (word_arg_count
>= 4)
294 status
|= ENTER_TMP_TO_R4
;
297 arg_types
>>= SLJIT_ARG_SHIFT
;
300 compiler
->args_size
= 0;
305 args_size
= SSIZE_OF(sw
);
307 switch (types
& SLJIT_ARG_MASK
) {
308 case SLJIT_ARG_TYPE_F64
:
310 FAIL_IF(emit_sse2_load(compiler
, 0, float_arg_count
, SLJIT_MEM1(SLJIT_SP
), args_size
));
311 args_size
+= SSIZE_OF(f64
);
313 case SLJIT_ARG_TYPE_F32
:
315 FAIL_IF(emit_sse2_load(compiler
, 1, float_arg_count
, SLJIT_MEM1(SLJIT_SP
), args_size
));
316 args_size
+= SSIZE_OF(f32
);
321 if (!(types
& SLJIT_ARG_TYPE_SCRATCH_REG
))
324 if (word_arg_count
== 4) {
325 if (types
& SLJIT_ARG_TYPE_SCRATCH_REG
) {
326 status
|= ENTER_TMP_TO_R4
;
327 arg_types
&= ~(SLJIT_ARG_FULL_MASK
<< 3 * SLJIT_ARG_SHIFT
);
328 } else if (saved_arg_count
== 4) {
329 status
|= ENTER_TMP_TO_S
;
330 arg_types
&= ~(SLJIT_ARG_FULL_MASK
<< 3 * SLJIT_ARG_SHIFT
);
334 args_size
+= SSIZE_OF(sw
);
337 types
>>= SLJIT_ARG_SHIFT
;
340 args_size
-= SSIZE_OF(sw
);
341 compiler
->args_size
= args_size
;
344 size
= (scratches
> 9 ? (scratches
- 9) : 0) + (saveds
<= 3 ? saveds
: 3) - kept_saveds_count
;
345 if (!(options
& SLJIT_ENTER_REG_ARG
))
349 inst
= (sljit_u8
*)ensure_buf(compiler
, (sljit_uw
)(size
+ 1));
352 INC_SIZE((sljit_uw
)size
);
354 if (!(options
& SLJIT_ENTER_REG_ARG
))
355 PUSH_REG(reg_map
[TMP_REG1
]);
357 if ((saveds
> 2 && kept_saveds_count
<= 2) || scratches
> 9)
358 PUSH_REG(reg_map
[SLJIT_S2
]);
359 if ((saveds
> 1 && kept_saveds_count
<= 1) || scratches
> 10)
360 PUSH_REG(reg_map
[SLJIT_S1
]);
361 if ((saveds
> 0 && kept_saveds_count
== 0) || scratches
> 11)
362 PUSH_REG(reg_map
[SLJIT_S0
]);
364 size
*= SSIZE_OF(sw
);
367 if (status
& (ENTER_TMP_TO_R4
| ENTER_TMP_TO_S
))
368 EMIT_MOV(compiler
, TMP_REG1
, 0, SLJIT_MEM1(SLJIT_SP
), args_size
+ size
);
370 size
+= SSIZE_OF(sw
);
372 local_size
= ((SLJIT_LOCALS_OFFSET_BASE
+ local_size
+ size
+ 0xf) & ~0xf) - size
;
373 compiler
->local_size
= local_size
;
379 switch (arg_types
& SLJIT_ARG_MASK
) {
380 case SLJIT_ARG_TYPE_F64
:
381 args_size
+= SSIZE_OF(f64
);
383 case SLJIT_ARG_TYPE_F32
:
384 args_size
+= SSIZE_OF(f32
);
388 SLJIT_ASSERT(word_arg_count
<= 3 || (word_arg_count
== 4 && !(status
& (ENTER_TMP_TO_R4
| ENTER_TMP_TO_S
))));
390 if (arg_types
& SLJIT_ARG_TYPE_SCRATCH_REG
) {
392 if (word_arg_count
== 3 && local_size
> 4 * 4096)
393 r2_offset
= local_size
+ args_size
;
396 EMIT_MOV(compiler
, word_arg_count
, 0, SLJIT_MEM1(SLJIT_SP
), args_size
);
399 EMIT_MOV(compiler
, SLJIT_S0
- saved_arg_count
, 0, SLJIT_MEM1(SLJIT_SP
), args_size
);
403 args_size
+= SSIZE_OF(sw
);
406 arg_types
>>= SLJIT_ARG_SHIFT
;
409 SLJIT_ASSERT(SLJIT_LOCALS_OFFSET
> 0);
412 SLJIT_ASSERT(r2_offset
== -1 || local_size
> 4 * 4096);
414 if (local_size
> 4096) {
415 if (local_size
<= 4 * 4096) {
416 BINARY_IMM32(OR
, 0, SLJIT_MEM1(SLJIT_SP
), -4096);
418 if (local_size
> 2 * 4096)
419 BINARY_IMM32(OR
, 0, SLJIT_MEM1(SLJIT_SP
), -4096 * 2);
420 if (local_size
> 3 * 4096)
421 BINARY_IMM32(OR
, 0, SLJIT_MEM1(SLJIT_SP
), -4096 * 3);
424 if (options
& SLJIT_ENTER_REG_ARG
) {
425 SLJIT_ASSERT(r2_offset
== -1);
427 inst
= (sljit_u8
*)ensure_buf(compiler
, (sljit_uw
)(1 + 1));
430 PUSH_REG(reg_map
[SLJIT_R2
]);
432 local_size
-= SSIZE_OF(sw
);
433 r2_offset
= local_size
;
436 EMIT_MOV(compiler
, SLJIT_R2
, 0, SLJIT_IMM
, local_size
>> 12);
438 BINARY_IMM32(OR
, 0, SLJIT_MEM1(SLJIT_SP
), -4096);
439 BINARY_IMM32(SUB
, 4096, SLJIT_SP
, 0);
441 inst
= (sljit_u8
*)ensure_buf(compiler
, 1 + 2);
446 inst
[1] = (sljit_u8
)-16;
451 if (local_size
> 0) {
452 BINARY_IMM32(OR
, 0, SLJIT_MEM1(SLJIT_SP
), -local_size
);
453 BINARY_IMM32(SUB
, local_size
, SLJIT_SP
, 0);
457 EMIT_MOV(compiler
, SLJIT_R2
, 0, SLJIT_MEM1(SLJIT_SP
), r2_offset
);
461 SLJIT_ASSERT(local_size
> 0);
463 BINARY_IMM32(SUB
, local_size
, SLJIT_SP
, 0);
467 size
= SLJIT_LOCALS_OFFSET_BASE
- SSIZE_OF(sw
);
468 kept_saveds_count
= SLJIT_R3
- kept_saveds_count
;
470 while (saved_arg_count
> 3) {
471 EMIT_MOV(compiler
, SLJIT_MEM1(SLJIT_SP
), size
, kept_saveds_count
, 0);
473 size
-= SSIZE_OF(sw
);
477 if (status
& (ENTER_TMP_TO_R4
| ENTER_TMP_TO_S
)) {
478 if (status
& ENTER_TMP_TO_R4
)
479 size
= 2 * SSIZE_OF(sw
);
481 EMIT_MOV(compiler
, SLJIT_MEM1(SLJIT_SP
), size
, TMP_REG1
, 0);
484 return SLJIT_SUCCESS
;
487 SLJIT_API_FUNC_ATTRIBUTE sljit_s32
sljit_set_context(struct sljit_compiler
*compiler
,
488 sljit_s32 options
, sljit_s32 arg_types
, sljit_s32 scratches
, sljit_s32 saveds
,
489 sljit_s32 fscratches
, sljit_s32 fsaveds
, sljit_s32 local_size
)
494 CHECK(check_sljit_set_context(compiler
, options
, arg_types
, scratches
, saveds
, fscratches
, fsaveds
, local_size
));
495 set_set_context(compiler
, options
, arg_types
, scratches
, saveds
, fscratches
, fsaveds
, local_size
);
497 arg_types
>>= SLJIT_ARG_SHIFT
;
500 if (!(options
& SLJIT_ENTER_REG_ARG
)) {
502 switch (arg_types
& SLJIT_ARG_MASK
) {
503 case SLJIT_ARG_TYPE_F64
:
504 args_size
+= SSIZE_OF(f64
);
506 case SLJIT_ARG_TYPE_F32
:
507 args_size
+= SSIZE_OF(f32
);
510 args_size
+= SSIZE_OF(sw
);
513 arg_types
>>= SLJIT_ARG_SHIFT
;
517 compiler
->args_size
= args_size
;
519 /* [esp+0] for saving temporaries and for function calls. */
521 saveds
= (1 + (scratches
> 9 ? (scratches
- 9) : 0) + (saveds
<= 3 ? saveds
: 3) - SLJIT_KEPT_SAVEDS_COUNT(options
)) * SSIZE_OF(sw
);
524 if (!(options
& SLJIT_ENTER_REG_ARG
))
525 saveds
+= SSIZE_OF(sw
);
527 compiler
->local_size
= ((SLJIT_LOCALS_OFFSET_BASE
+ local_size
+ saveds
+ 0xf) & ~0xf) - saveds
;
528 return SLJIT_SUCCESS
;
531 static sljit_s32
emit_stack_frame_release(struct sljit_compiler
*compiler
)
533 sljit_s32 kept_saveds_count
= SLJIT_KEPT_SAVEDS_COUNT(compiler
->options
);
538 BINARY_IMM32(ADD
, compiler
->local_size
, SLJIT_SP
, 0);
540 size
= (sljit_uw
)((compiler
->scratches
> 9 ? (compiler
->scratches
- 9) : 0) +
541 (compiler
->saveds
<= 3 ? compiler
->saveds
: 3) - kept_saveds_count
);
543 if (!(compiler
->options
& SLJIT_ENTER_REG_ARG
))
547 return SLJIT_SUCCESS
;
549 inst
= (sljit_u8
*)ensure_buf(compiler
, 1 + size
);
554 saveds
= compiler
->saveds
;
556 if ((saveds
> 0 && kept_saveds_count
== 0) || compiler
->scratches
> 11)
557 POP_REG(reg_map
[SLJIT_S0
]);
558 if ((saveds
> 1 && kept_saveds_count
<= 1) || compiler
->scratches
> 10)
559 POP_REG(reg_map
[SLJIT_S1
]);
560 if ((saveds
> 2 && kept_saveds_count
<= 2) || compiler
->scratches
> 9)
561 POP_REG(reg_map
[SLJIT_S2
]);
563 if (!(compiler
->options
& SLJIT_ENTER_REG_ARG
))
564 POP_REG(reg_map
[TMP_REG1
]);
566 return SLJIT_SUCCESS
;
569 SLJIT_API_FUNC_ATTRIBUTE sljit_s32
sljit_emit_return_void(struct sljit_compiler
*compiler
)
574 CHECK(check_sljit_emit_return_void(compiler
));
576 SLJIT_ASSERT(compiler
->args_size
>= 0);
577 SLJIT_ASSERT(compiler
->local_size
> 0);
579 FAIL_IF(emit_stack_frame_release(compiler
));
581 inst
= (sljit_u8
*)ensure_buf(compiler
, 1 + 1);
585 return SLJIT_SUCCESS
;
588 /* --------------------------------------------------------------------- */
589 /* Call / return instructions */
590 /* --------------------------------------------------------------------- */
592 static sljit_s32
call_get_stack_size(sljit_s32 arg_types
, sljit_s32
*word_arg_count_ptr
)
594 sljit_sw stack_size
= 0;
595 sljit_s32 word_arg_count
= 0;
597 arg_types
>>= SLJIT_ARG_SHIFT
;
600 switch (arg_types
& SLJIT_ARG_MASK
) {
601 case SLJIT_ARG_TYPE_F64
:
602 stack_size
+= SSIZE_OF(f64
);
604 case SLJIT_ARG_TYPE_F32
:
605 stack_size
+= SSIZE_OF(f32
);
609 stack_size
+= SSIZE_OF(sw
);
613 arg_types
>>= SLJIT_ARG_SHIFT
;
616 if (word_arg_count_ptr
)
617 *word_arg_count_ptr
= word_arg_count
;
619 if (stack_size
<= 4 * SSIZE_OF(sw
))
622 return ((stack_size
- (4 * SSIZE_OF(sw
)) + 0xf) & ~0xf);
625 static sljit_s32
call_with_args(struct sljit_compiler
*compiler
,
626 sljit_s32 arg_types
, sljit_sw stack_size
, sljit_s32 word_arg_count
, sljit_s32 keep_tmp1
)
628 sljit_s32 float_arg_count
= 0, arg4_reg
= 0, arg_offset
;
631 if (word_arg_count
>= 4) {
635 EMIT_MOV(compiler
, TMP_REG1
, 0, SLJIT_MEM1(SLJIT_SP
), 2 * SSIZE_OF(sw
));
641 BINARY_IMM32(SUB
, stack_size
, SLJIT_SP
, 0);
645 arg_types
>>= SLJIT_ARG_SHIFT
;
648 switch (arg_types
& SLJIT_ARG_MASK
) {
649 case SLJIT_ARG_TYPE_F64
:
651 FAIL_IF(emit_sse2_store(compiler
, 0, SLJIT_MEM1(SLJIT_SP
), arg_offset
, float_arg_count
));
652 arg_offset
+= SSIZE_OF(f64
);
654 case SLJIT_ARG_TYPE_F32
:
656 FAIL_IF(emit_sse2_store(compiler
, 1, SLJIT_MEM1(SLJIT_SP
), arg_offset
, float_arg_count
));
657 arg_offset
+= SSIZE_OF(f32
);
661 EMIT_MOV(compiler
, SLJIT_MEM1(SLJIT_SP
), arg_offset
, (word_arg_count
>= 4) ? arg4_reg
: word_arg_count
, 0);
663 if (word_arg_count
== 1 && arg4_reg
== SLJIT_R0
)
664 EMIT_MOV(compiler
, SLJIT_R0
, 0, SLJIT_MEM1(SLJIT_SP
), 2 * SSIZE_OF(sw
) + stack_size
);
666 arg_offset
+= SSIZE_OF(sw
);
670 arg_types
>>= SLJIT_ARG_SHIFT
;
673 return SLJIT_SUCCESS
;
676 static sljit_s32
post_call_with_args(struct sljit_compiler
*compiler
,
677 sljit_s32 arg_types
, sljit_s32 stack_size
)
683 BINARY_IMM32(ADD
, stack_size
, SLJIT_SP
, 0);
685 if ((arg_types
& SLJIT_ARG_MASK
) < SLJIT_ARG_TYPE_F64
)
686 return SLJIT_SUCCESS
;
688 single
= ((arg_types
& SLJIT_ARG_MASK
) == SLJIT_ARG_TYPE_F32
);
690 inst
= (sljit_u8
*)ensure_buf(compiler
, 1 + 3);
693 inst
[0] = single
? FSTPS
: FSTPD
;
694 inst
[1] = (0x03 << 3) | 0x04;
695 inst
[2] = (0x04 << 3) | reg_map
[SLJIT_SP
];
697 return emit_sse2_load(compiler
, single
, SLJIT_FR0
, SLJIT_MEM1(SLJIT_SP
), 0);
700 static sljit_s32
tail_call_with_args(struct sljit_compiler
*compiler
,
701 sljit_s32
*extra_space
, sljit_s32 arg_types
,
702 sljit_s32 src
, sljit_sw srcw
)
704 sljit_sw args_size
, saved_regs_size
;
705 sljit_sw types
, word_arg_count
, float_arg_count
;
706 sljit_sw stack_size
, prev_stack_size
, min_size
, offset
;
707 sljit_sw word_arg4_offset
;
708 sljit_u8 r2_offset
= 0;
709 sljit_s32 kept_saveds_count
= SLJIT_KEPT_SAVEDS_COUNT(compiler
->options
);
712 ADJUST_LOCAL_OFFSET(src
, srcw
);
713 CHECK_EXTRA_REGS(src
, srcw
, (void)0);
715 saved_regs_size
= (1 + (compiler
->scratches
> 9 ? (compiler
->scratches
- 9) : 0)
716 + (compiler
->saveds
<= 3 ? compiler
->saveds
: 3) - kept_saveds_count
) * SSIZE_OF(sw
);
720 arg_types
>>= SLJIT_ARG_SHIFT
;
724 while (arg_types
!= 0) {
725 types
= (types
<< SLJIT_ARG_SHIFT
) | (arg_types
& SLJIT_ARG_MASK
);
727 switch (arg_types
& SLJIT_ARG_MASK
) {
728 case SLJIT_ARG_TYPE_F64
:
729 args_size
+= SSIZE_OF(f64
);
732 case SLJIT_ARG_TYPE_F32
:
733 args_size
+= SSIZE_OF(f32
);
738 args_size
+= SSIZE_OF(sw
);
741 arg_types
>>= SLJIT_ARG_SHIFT
;
744 if (args_size
<= compiler
->args_size
) {
746 stack_size
= args_size
+ SSIZE_OF(sw
) + saved_regs_size
;
748 offset
= stack_size
+ compiler
->local_size
;
750 if (!(src
& SLJIT_IMM
) && src
!= SLJIT_R0
) {
751 if (word_arg_count
>= 1) {
752 EMIT_MOV(compiler
, SLJIT_MEM1(SLJIT_SP
), 0, SLJIT_R0
, 0);
753 r2_offset
= sizeof(sljit_sw
);
755 EMIT_MOV(compiler
, SLJIT_R0
, 0, src
, srcw
);
759 switch (types
& SLJIT_ARG_MASK
) {
760 case SLJIT_ARG_TYPE_F64
:
761 offset
-= SSIZE_OF(f64
);
762 FAIL_IF(emit_sse2_store(compiler
, 0, SLJIT_MEM1(SLJIT_SP
), offset
, float_arg_count
));
765 case SLJIT_ARG_TYPE_F32
:
766 offset
-= SSIZE_OF(f32
);
767 FAIL_IF(emit_sse2_store(compiler
, 0, SLJIT_MEM1(SLJIT_SP
), offset
, float_arg_count
));
771 switch (word_arg_count
) {
773 offset
-= SSIZE_OF(sw
);
774 if (r2_offset
!= 0) {
775 EMIT_MOV(compiler
, SLJIT_R2
, 0, SLJIT_MEM1(SLJIT_SP
), 0);
776 EMIT_MOV(compiler
, SLJIT_MEM1(SLJIT_SP
), offset
, SLJIT_R2
, 0);
778 EMIT_MOV(compiler
, SLJIT_MEM1(SLJIT_SP
), offset
, SLJIT_R0
, 0);
781 offset
-= SSIZE_OF(sw
);
782 EMIT_MOV(compiler
, SLJIT_MEM1(SLJIT_SP
), offset
, SLJIT_R1
, 0);
785 offset
-= SSIZE_OF(sw
);
788 offset
-= SSIZE_OF(sw
);
789 EMIT_MOV(compiler
, SLJIT_R2
, 0, SLJIT_MEM1(SLJIT_SP
), 2 * SSIZE_OF(sw
));
790 EMIT_MOV(compiler
, SLJIT_MEM1(SLJIT_SP
), offset
, SLJIT_R2
, 0);
796 types
>>= SLJIT_ARG_SHIFT
;
799 return emit_stack_frame_release(compiler
);
802 stack_size
= args_size
+ SSIZE_OF(sw
);
804 if (word_arg_count
>= 1 && !(src
& SLJIT_IMM
) && src
!= SLJIT_R0
) {
805 r2_offset
= SSIZE_OF(sw
);
806 stack_size
+= SSIZE_OF(sw
);
809 if (word_arg_count
>= 3)
810 stack_size
+= SSIZE_OF(sw
);
812 prev_stack_size
= SSIZE_OF(sw
) + saved_regs_size
;
813 min_size
= prev_stack_size
+ compiler
->local_size
;
815 word_arg4_offset
= 2 * SSIZE_OF(sw
);
817 if (stack_size
> min_size
) {
818 BINARY_IMM32(SUB
, stack_size
- min_size
, SLJIT_SP
, 0);
819 if (src
== SLJIT_MEM1(SLJIT_SP
))
820 srcw
+= stack_size
- min_size
;
821 word_arg4_offset
+= stack_size
- min_size
;
824 stack_size
= min_size
;
826 if (word_arg_count
>= 3) {
827 EMIT_MOV(compiler
, SLJIT_MEM1(SLJIT_SP
), r2_offset
, SLJIT_R2
, 0);
829 if (word_arg_count
>= 4)
830 EMIT_MOV(compiler
, SLJIT_R2
, 0, SLJIT_MEM1(SLJIT_SP
), word_arg4_offset
);
833 if (!(src
& SLJIT_IMM
) && src
!= SLJIT_R0
) {
834 if (word_arg_count
>= 1) {
835 SLJIT_ASSERT(r2_offset
== sizeof(sljit_sw
));
836 EMIT_MOV(compiler
, SLJIT_MEM1(SLJIT_SP
), 0, SLJIT_R0
, 0);
838 EMIT_MOV(compiler
, SLJIT_R0
, 0, src
, srcw
);
841 /* Restore saved registers. */
842 offset
= stack_size
- 2 * SSIZE_OF(sw
);
843 EMIT_MOV(compiler
, TMP_REG1
, 0, SLJIT_MEM1(SLJIT_SP
), offset
);
845 if (compiler
->saveds
> 2 || compiler
->scratches
> 9) {
846 offset
-= SSIZE_OF(sw
);
847 EMIT_MOV(compiler
, SLJIT_S2
, 0, SLJIT_MEM1(SLJIT_SP
), offset
);
849 if ((compiler
->saveds
> 1 && kept_saveds_count
<= 1) || compiler
->scratches
> 10) {
850 offset
-= SSIZE_OF(sw
);
851 EMIT_MOV(compiler
, SLJIT_S1
, 0, SLJIT_MEM1(SLJIT_SP
), offset
);
853 if ((compiler
->saveds
> 0 && kept_saveds_count
== 0) || compiler
->scratches
> 11) {
854 offset
-= SSIZE_OF(sw
);
855 EMIT_MOV(compiler
, SLJIT_S0
, 0, SLJIT_MEM1(SLJIT_SP
), offset
);
858 /* Copy fourth argument and return address. */
859 offset
= stack_size
- SSIZE_OF(sw
);
860 *extra_space
= args_size
;
862 if (word_arg_count
>= 4) {
863 offset
-= SSIZE_OF(sw
);
864 EMIT_MOV(compiler
, SLJIT_MEM1(SLJIT_SP
), offset
, SLJIT_R2
, 0);
868 switch (types
& SLJIT_ARG_MASK
) {
869 case SLJIT_ARG_TYPE_F64
:
870 offset
-= SSIZE_OF(f64
);
871 FAIL_IF(emit_sse2_store(compiler
, 0, SLJIT_MEM1(SLJIT_SP
), offset
, float_arg_count
));
874 case SLJIT_ARG_TYPE_F32
:
875 offset
-= SSIZE_OF(f32
);
876 FAIL_IF(emit_sse2_store(compiler
, 0, SLJIT_MEM1(SLJIT_SP
), offset
, float_arg_count
));
880 switch (word_arg_count
) {
882 offset
-= SSIZE_OF(sw
);
883 if (r2_offset
!= 0) {
884 EMIT_MOV(compiler
, SLJIT_R2
, 0, SLJIT_MEM1(SLJIT_SP
), 0);
885 EMIT_MOV(compiler
, SLJIT_MEM1(SLJIT_SP
), offset
, SLJIT_R2
, 0);
887 EMIT_MOV(compiler
, SLJIT_MEM1(SLJIT_SP
), offset
, SLJIT_R0
, 0);
890 offset
-= SSIZE_OF(sw
);
891 EMIT_MOV(compiler
, SLJIT_MEM1(SLJIT_SP
), offset
, SLJIT_R1
, 0);
894 offset
-= SSIZE_OF(sw
);
895 EMIT_MOV(compiler
, SLJIT_R2
, 0, SLJIT_MEM1(SLJIT_SP
), r2_offset
);
896 EMIT_MOV(compiler
, SLJIT_MEM1(SLJIT_SP
), offset
, SLJIT_R2
, 0);
902 types
>>= SLJIT_ARG_SHIFT
;
905 SLJIT_ASSERT(offset
>= 0);
908 return SLJIT_SUCCESS
;
910 BINARY_IMM32(ADD
, offset
, SLJIT_SP
, 0);
911 return SLJIT_SUCCESS
;
914 static sljit_s32
emit_tail_call_end(struct sljit_compiler
*compiler
, sljit_s32 extra_space
)
916 /* Called when stack consumption cannot be reduced to 0. */
919 BINARY_IMM32(ADD
, extra_space
, SLJIT_SP
, 0);
921 inst
= (sljit_u8
*)ensure_buf(compiler
, 1 + 1);
926 return SLJIT_SUCCESS
;
929 static sljit_s32
tail_call_reg_arg_with_args(struct sljit_compiler
*compiler
, sljit_s32 arg_types
)
931 sljit_s32 word_arg_count
= 0;
932 sljit_s32 kept_saveds_count
, offset
;
934 arg_types
>>= SLJIT_ARG_SHIFT
;
937 if ((arg_types
& SLJIT_ARG_MASK
) < SLJIT_ARG_TYPE_F64
)
940 arg_types
>>= SLJIT_ARG_SHIFT
;
943 if (word_arg_count
< 4)
944 return SLJIT_SUCCESS
;
946 EMIT_MOV(compiler
, TMP_REG1
, 0, SLJIT_MEM1(SLJIT_SP
), 2 * SSIZE_OF(sw
));
948 kept_saveds_count
= SLJIT_KEPT_SAVEDS_COUNT(compiler
->options
);
949 offset
= compiler
->local_size
+ 3 * SSIZE_OF(sw
);
951 if ((compiler
->saveds
> 0 && kept_saveds_count
== 0) || compiler
->scratches
> 11)
952 offset
+= SSIZE_OF(sw
);
953 if ((compiler
->saveds
> 1 && kept_saveds_count
<= 1) || compiler
->scratches
> 10)
954 offset
+= SSIZE_OF(sw
);
955 if ((compiler
->saveds
> 2 && kept_saveds_count
<= 2) || compiler
->scratches
> 9)
956 offset
+= SSIZE_OF(sw
);
958 return emit_mov(compiler
, SLJIT_MEM1(SLJIT_SP
), offset
, TMP_REG1
, 0);
961 SLJIT_API_FUNC_ATTRIBUTE
struct sljit_jump
* sljit_emit_call(struct sljit_compiler
*compiler
, sljit_s32 type
,
964 struct sljit_jump
*jump
;
965 sljit_sw stack_size
= 0;
966 sljit_s32 word_arg_count
;
969 CHECK_PTR(check_sljit_emit_call(compiler
, type
, arg_types
));
971 if (type
& SLJIT_CALL_RETURN
) {
972 if ((type
& 0xff) == SLJIT_CALL_REG_ARG
) {
973 PTR_FAIL_IF(tail_call_reg_arg_with_args(compiler
, arg_types
));
974 PTR_FAIL_IF(emit_stack_frame_release(compiler
));
976 SLJIT_SKIP_CHECKS(compiler
);
977 return sljit_emit_jump(compiler
, SLJIT_JUMP
| (type
& SLJIT_REWRITABLE_JUMP
));
981 PTR_FAIL_IF(tail_call_with_args(compiler
, &stack_size
, arg_types
, SLJIT_IMM
, 0));
983 SLJIT_SKIP_CHECKS(compiler
);
986 return sljit_emit_jump(compiler
, SLJIT_JUMP
| (type
& SLJIT_REWRITABLE_JUMP
));
988 jump
= sljit_emit_jump(compiler
, type
);
989 PTR_FAIL_IF(jump
== NULL
);
991 PTR_FAIL_IF(emit_tail_call_end(compiler
, stack_size
));
995 if ((type
& 0xff) == SLJIT_CALL_REG_ARG
) {
996 SLJIT_SKIP_CHECKS(compiler
);
997 return sljit_emit_jump(compiler
, type
);
1000 stack_size
= call_get_stack_size(arg_types
, &word_arg_count
);
1001 PTR_FAIL_IF(call_with_args(compiler
, arg_types
, stack_size
, word_arg_count
, 0));
1003 SLJIT_SKIP_CHECKS(compiler
);
1004 jump
= sljit_emit_jump(compiler
, type
);
1005 PTR_FAIL_IF(jump
== NULL
);
1007 PTR_FAIL_IF(post_call_with_args(compiler
, arg_types
, stack_size
));
1011 SLJIT_API_FUNC_ATTRIBUTE sljit_s32
sljit_emit_icall(struct sljit_compiler
*compiler
, sljit_s32 type
,
1012 sljit_s32 arg_types
,
1013 sljit_s32 src
, sljit_sw srcw
)
1015 sljit_sw stack_size
= 0;
1016 sljit_s32 word_arg_count
;
1019 CHECK(check_sljit_emit_icall(compiler
, type
, arg_types
, src
, srcw
));
1021 if (type
& SLJIT_CALL_RETURN
) {
1022 if ((type
& 0xff) == SLJIT_CALL_REG_ARG
) {
1023 FAIL_IF(tail_call_reg_arg_with_args(compiler
, arg_types
));
1025 if ((src
& SLJIT_MEM
) || (src
>= SLJIT_FIRST_SAVED_REG
&& src
<= SLJIT_S0
)) {
1026 ADJUST_LOCAL_OFFSET(src
, srcw
);
1027 CHECK_EXTRA_REGS(src
, srcw
, (void)0);
1029 EMIT_MOV(compiler
, TMP_REG1
, 0, src
, srcw
);
1034 FAIL_IF(emit_stack_frame_release(compiler
));
1036 SLJIT_SKIP_CHECKS(compiler
);
1037 return sljit_emit_ijump(compiler
, SLJIT_JUMP
, src
, srcw
);
1041 FAIL_IF(tail_call_with_args(compiler
, &stack_size
, arg_types
, src
, srcw
));
1043 if (!(src
& SLJIT_IMM
)) {
1048 SLJIT_SKIP_CHECKS(compiler
);
1050 if (stack_size
== 0)
1051 return sljit_emit_ijump(compiler
, SLJIT_JUMP
, src
, srcw
);
1053 FAIL_IF(sljit_emit_ijump(compiler
, type
, src
, srcw
));
1054 return emit_tail_call_end(compiler
, stack_size
);
1057 if ((type
& 0xff) == SLJIT_CALL_REG_ARG
) {
1058 SLJIT_SKIP_CHECKS(compiler
);
1059 return sljit_emit_ijump(compiler
, type
, src
, srcw
);
1062 ADJUST_LOCAL_OFFSET(src
, srcw
);
1063 CHECK_EXTRA_REGS(src
, srcw
, (void)0);
1065 if (src
& SLJIT_MEM
) {
1066 EMIT_MOV(compiler
, TMP_REG1
, 0, src
, srcw
);
1071 stack_size
= call_get_stack_size(arg_types
, &word_arg_count
);
1072 FAIL_IF(call_with_args(compiler
, arg_types
, stack_size
, word_arg_count
, src
== TMP_REG1
));
1074 if (stack_size
> 0 && src
== SLJIT_MEM1(SLJIT_SP
))
1077 SLJIT_SKIP_CHECKS(compiler
);
1078 FAIL_IF(sljit_emit_ijump(compiler
, type
, src
, srcw
));
1080 return post_call_with_args(compiler
, arg_types
, stack_size
);
1083 SLJIT_API_FUNC_ATTRIBUTE sljit_s32
sljit_emit_fast_enter(struct sljit_compiler
*compiler
, sljit_s32 dst
, sljit_sw dstw
)
1088 CHECK(check_sljit_emit_fast_enter(compiler
, dst
, dstw
));
1089 ADJUST_LOCAL_OFFSET(dst
, dstw
);
1091 CHECK_EXTRA_REGS(dst
, dstw
, (void)0);
1093 if (FAST_IS_REG(dst
)) {
1094 /* Unused dest is possible here. */
1095 inst
= (sljit_u8
*)ensure_buf(compiler
, 1 + 1);
1099 POP_REG(reg_map
[dst
]);
1100 return SLJIT_SUCCESS
;
1104 inst
= emit_x86_instruction(compiler
, 1, 0, 0, dst
, dstw
);
1107 return SLJIT_SUCCESS
;
1110 static sljit_s32
emit_fast_return(struct sljit_compiler
*compiler
, sljit_s32 src
, sljit_sw srcw
)
1114 CHECK_EXTRA_REGS(src
, srcw
, (void)0);
1116 if (FAST_IS_REG(src
)) {
1117 inst
= (sljit_u8
*)ensure_buf(compiler
, 1 + 1 + 1);
1121 PUSH_REG(reg_map
[src
]);
1124 inst
= emit_x86_instruction(compiler
, 1, 0, 0, src
, srcw
);
1129 inst
= (sljit_u8
*)ensure_buf(compiler
, 1 + 1);
1135 return SLJIT_SUCCESS
;
1138 static sljit_s32
skip_frames_before_return(struct sljit_compiler
*compiler
)
1142 /* Don't adjust shadow stack if it isn't enabled. */
1143 if (!cpu_has_shadow_stack())
1144 return SLJIT_SUCCESS
;
1146 SLJIT_ASSERT(compiler
->args_size
>= 0);
1147 SLJIT_ASSERT(compiler
->local_size
> 0);
1149 size
= compiler
->local_size
;
1150 size
+= (1 + (compiler
->scratches
> 9 ? (compiler
->scratches
- 9) : 0)
1151 + (compiler
->saveds
<= 3 ? compiler
->saveds
: 3)) * SSIZE_OF(sw
);
1153 return adjust_shadow_stack(compiler
, SLJIT_MEM1(SLJIT_SP
), size
);