Rework x86-32 stack layout
[sljit.git] / sljit_src / sljitNativeX86_32.c
blob68e53653bb3f84b6ff3849629fe9b52a7905359f
1 /*
2 * Stack-less Just-In-Time compiler
4 * Copyright Zoltan Herczeg (hzmester@freemail.hu). All rights reserved.
6 * Redistribution and use in source and binary forms, with or without modification, are
7 * permitted provided that the following conditions are met:
9 * 1. Redistributions of source code must retain the above copyright notice, this list of
10 * conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright notice, this list
13 * of conditions and the following disclaimer in the documentation and/or other materials
14 * provided with the distribution.
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) AND CONTRIBUTORS ``AS IS'' AND ANY
17 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
19 * SHALL THE COPYRIGHT HOLDER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
21 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
22 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
23 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
24 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 /* x86 32-bit arch dependent functions. */
29 /* --------------------------------------------------------------------- */
30 /* Operators */
31 /* --------------------------------------------------------------------- */
33 static sljit_s32 emit_do_imm(struct sljit_compiler *compiler, sljit_u8 opcode, sljit_sw imm)
35 sljit_u8 *inst;
37 inst = (sljit_u8*)ensure_buf(compiler, 1 + 1 + sizeof(sljit_sw));
38 FAIL_IF(!inst);
39 INC_SIZE(1 + sizeof(sljit_sw));
40 *inst++ = opcode;
41 sljit_unaligned_store_sw(inst, imm);
42 return SLJIT_SUCCESS;
45 /* Size contains the flags as well. */
46 static sljit_u8* emit_x86_instruction(struct sljit_compiler *compiler, sljit_uw size,
47 /* The register or immediate operand. */
48 sljit_s32 a, sljit_sw imma,
49 /* The general operand (not immediate). */
50 sljit_s32 b, sljit_sw immb)
52 sljit_u8 *inst;
53 sljit_u8 *buf_ptr;
54 sljit_u8 reg_map_b;
55 sljit_uw flags = size;
56 sljit_uw inst_size;
58 /* Both cannot be switched on. */
59 SLJIT_ASSERT((flags & (EX86_BIN_INS | EX86_SHIFT_INS)) != (EX86_BIN_INS | EX86_SHIFT_INS));
60 /* Size flags not allowed for typed instructions. */
61 SLJIT_ASSERT(!(flags & (EX86_BIN_INS | EX86_SHIFT_INS)) || (flags & (EX86_BYTE_ARG | EX86_HALF_ARG)) == 0);
62 /* Both size flags cannot be switched on. */
63 SLJIT_ASSERT((flags & (EX86_BYTE_ARG | EX86_HALF_ARG)) != (EX86_BYTE_ARG | EX86_HALF_ARG));
64 /* SSE2 and immediate is not possible. */
65 SLJIT_ASSERT(!(a & SLJIT_IMM) || !(flags & EX86_SSE2));
66 SLJIT_ASSERT((flags & (EX86_PREF_F2 | EX86_PREF_F3)) != (EX86_PREF_F2 | EX86_PREF_F3)
67 && (flags & (EX86_PREF_F2 | EX86_PREF_66)) != (EX86_PREF_F2 | EX86_PREF_66)
68 && (flags & (EX86_PREF_F3 | EX86_PREF_66)) != (EX86_PREF_F3 | EX86_PREF_66));
70 size &= 0xf;
71 inst_size = size;
73 if (flags & (EX86_PREF_F2 | EX86_PREF_F3))
74 inst_size++;
75 if (flags & EX86_PREF_66)
76 inst_size++;
78 /* Calculate size of b. */
79 inst_size += 1; /* mod r/m byte. */
80 if (b & SLJIT_MEM) {
81 if (!(b & REG_MASK))
82 inst_size += sizeof(sljit_sw);
83 else {
84 if (immb != 0 && !(b & OFFS_REG_MASK)) {
85 /* Immediate operand. */
86 if (immb <= 127 && immb >= -128)
87 inst_size += sizeof(sljit_s8);
88 else
89 inst_size += sizeof(sljit_sw);
91 else if (reg_map[b & REG_MASK] == 5) {
92 /* Swap registers if possible. */
93 if ((b & OFFS_REG_MASK) && (immb & 0x3) == 0 && reg_map[OFFS_REG(b)] != 5)
94 b = SLJIT_MEM | OFFS_REG(b) | TO_OFFS_REG(b & REG_MASK);
95 else
96 inst_size += sizeof(sljit_s8);
99 if (reg_map[b & REG_MASK] == 4 && !(b & OFFS_REG_MASK))
100 b |= TO_OFFS_REG(SLJIT_SP);
102 if (b & OFFS_REG_MASK)
103 inst_size += 1; /* SIB byte. */
107 /* Calculate size of a. */
108 if (a & SLJIT_IMM) {
109 if (flags & EX86_BIN_INS) {
110 if (imma <= 127 && imma >= -128) {
111 inst_size += 1;
112 flags |= EX86_BYTE_ARG;
113 } else
114 inst_size += 4;
116 else if (flags & EX86_SHIFT_INS) {
117 imma &= 0x1f;
118 if (imma != 1) {
119 inst_size++;
120 flags |= EX86_BYTE_ARG;
122 } else if (flags & EX86_BYTE_ARG)
123 inst_size++;
124 else if (flags & EX86_HALF_ARG)
125 inst_size += sizeof(short);
126 else
127 inst_size += sizeof(sljit_sw);
129 else
130 SLJIT_ASSERT(!(flags & EX86_SHIFT_INS) || a == SLJIT_PREF_SHIFT_REG);
132 inst = (sljit_u8*)ensure_buf(compiler, 1 + inst_size);
133 PTR_FAIL_IF(!inst);
135 /* Encoding the byte. */
136 INC_SIZE(inst_size);
137 if (flags & EX86_PREF_F2)
138 *inst++ = 0xf2;
139 if (flags & EX86_PREF_F3)
140 *inst++ = 0xf3;
141 if (flags & EX86_PREF_66)
142 *inst++ = 0x66;
144 buf_ptr = inst + size;
146 /* Encode mod/rm byte. */
147 if (!(flags & EX86_SHIFT_INS)) {
148 if ((flags & EX86_BIN_INS) && (a & SLJIT_IMM))
149 *inst = (flags & EX86_BYTE_ARG) ? GROUP_BINARY_83 : GROUP_BINARY_81;
151 if (a & SLJIT_IMM)
152 *buf_ptr = 0;
153 else if (!(flags & EX86_SSE2_OP1))
154 *buf_ptr = U8(reg_map[a] << 3);
155 else
156 *buf_ptr = U8(a << 3);
158 else {
159 if (a & SLJIT_IMM) {
160 if (imma == 1)
161 *inst = GROUP_SHIFT_1;
162 else
163 *inst = GROUP_SHIFT_N;
164 } else
165 *inst = GROUP_SHIFT_CL;
166 *buf_ptr = 0;
169 if (!(b & SLJIT_MEM)) {
170 *buf_ptr = U8(*buf_ptr | MOD_REG | (!(flags & EX86_SSE2_OP2) ? reg_map[b] : b));
171 buf_ptr++;
172 } else if (b & REG_MASK) {
173 reg_map_b = reg_map[b & REG_MASK];
175 if (!(b & OFFS_REG_MASK) || (b & OFFS_REG_MASK) == TO_OFFS_REG(SLJIT_SP)) {
176 if (immb != 0 || reg_map_b == 5) {
177 if (immb <= 127 && immb >= -128)
178 *buf_ptr |= 0x40;
179 else
180 *buf_ptr |= 0x80;
183 if (!(b & OFFS_REG_MASK))
184 *buf_ptr++ |= reg_map_b;
185 else {
186 *buf_ptr++ |= 0x04;
187 *buf_ptr++ = U8(reg_map_b | (reg_map[OFFS_REG(b)] << 3));
190 if (immb != 0 || reg_map_b == 5) {
191 if (immb <= 127 && immb >= -128)
192 *buf_ptr++ = U8(immb); /* 8 bit displacement. */
193 else {
194 sljit_unaligned_store_sw(buf_ptr, immb); /* 32 bit displacement. */
195 buf_ptr += sizeof(sljit_sw);
199 else {
200 if (reg_map_b == 5)
201 *buf_ptr |= 0x40;
203 *buf_ptr++ |= 0x04;
204 *buf_ptr++ = U8(reg_map_b | (reg_map[OFFS_REG(b)] << 3) | (immb << 6));
206 if (reg_map_b == 5)
207 *buf_ptr++ = 0;
210 else {
211 *buf_ptr++ |= 0x05;
212 sljit_unaligned_store_sw(buf_ptr, immb); /* 32 bit displacement. */
213 buf_ptr += sizeof(sljit_sw);
216 if (a & SLJIT_IMM) {
217 if (flags & EX86_BYTE_ARG)
218 *buf_ptr = U8(imma);
219 else if (flags & EX86_HALF_ARG)
220 sljit_unaligned_store_s16(buf_ptr, (sljit_s16)imma);
221 else if (!(flags & EX86_SHIFT_INS))
222 sljit_unaligned_store_sw(buf_ptr, imma);
225 return !(flags & EX86_SHIFT_INS) ? inst : (inst + 1);
228 /* --------------------------------------------------------------------- */
229 /* Enter / return */
230 /* --------------------------------------------------------------------- */
232 static sljit_u8* generate_far_jump_code(struct sljit_jump *jump, sljit_u8 *code_ptr, sljit_sw executable_offset)
234 sljit_uw type = jump->flags >> TYPE_SHIFT;
236 if (type == SLJIT_JUMP) {
237 *code_ptr++ = JMP_i32;
238 jump->addr++;
240 else if (type >= SLJIT_FAST_CALL) {
241 *code_ptr++ = CALL_i32;
242 jump->addr++;
244 else {
245 *code_ptr++ = GROUP_0F;
246 *code_ptr++ = get_jump_code(type);
247 jump->addr += 2;
250 if (jump->flags & JUMP_LABEL)
251 jump->flags |= PATCH_MW;
252 else
253 sljit_unaligned_store_sw(code_ptr, (sljit_sw)(jump->u.target - (jump->addr + 4) - (sljit_uw)executable_offset));
254 code_ptr += 4;
256 return code_ptr;
259 #define ENTER_TMP_TO_R4 0x00001
260 #define ENTER_TMP_TO_S 0x00002
262 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_enter(struct sljit_compiler *compiler,
263 sljit_s32 options, sljit_s32 arg_types, sljit_s32 scratches, sljit_s32 saveds,
264 sljit_s32 fscratches, sljit_s32 fsaveds, sljit_s32 local_size)
266 sljit_s32 word_arg_count, saved_arg_count, float_arg_count;
267 sljit_s32 size, args_size, types, status;
268 sljit_s32 kept_saveds_count = SLJIT_KEPT_SAVEDS_COUNT(options);
269 sljit_u8 *inst;
270 #ifdef _WIN32
271 sljit_s32 r2_offset = -1;
272 #endif
274 CHECK_ERROR();
275 CHECK(check_sljit_emit_enter(compiler, options, arg_types, scratches, saveds, fscratches, fsaveds, local_size));
276 set_emit_enter(compiler, options, arg_types, scratches, saveds, fscratches, fsaveds, local_size);
278 /* Emit ENDBR32 at function entry if needed. */
279 FAIL_IF(emit_endbranch(compiler));
281 SLJIT_COMPILE_ASSERT(SLJIT_FR0 == 1, float_register_index_start);
283 arg_types >>= SLJIT_ARG_SHIFT;
284 word_arg_count = 0;
285 status = 0;
287 if (options & SLJIT_ENTER_REG_ARG) {
288 args_size = 3 * SSIZE_OF(sw);
290 while (arg_types) {
291 if ((arg_types & SLJIT_ARG_MASK) < SLJIT_ARG_TYPE_F64) {
292 word_arg_count++;
293 if (word_arg_count >= 4)
294 status |= ENTER_TMP_TO_R4;
297 arg_types >>= SLJIT_ARG_SHIFT;
300 compiler->args_size = 0;
301 } else {
302 types = arg_types;
303 saved_arg_count = 0;
304 float_arg_count = 0;
305 args_size = SSIZE_OF(sw);
306 while (types) {
307 switch (types & SLJIT_ARG_MASK) {
308 case SLJIT_ARG_TYPE_F64:
309 float_arg_count++;
310 FAIL_IF(emit_sse2_load(compiler, 0, float_arg_count, SLJIT_MEM1(SLJIT_SP), args_size));
311 args_size += SSIZE_OF(f64);
312 break;
313 case SLJIT_ARG_TYPE_F32:
314 float_arg_count++;
315 FAIL_IF(emit_sse2_load(compiler, 1, float_arg_count, SLJIT_MEM1(SLJIT_SP), args_size));
316 args_size += SSIZE_OF(f32);
317 break;
318 default:
319 word_arg_count++;
321 if (!(types & SLJIT_ARG_TYPE_SCRATCH_REG))
322 saved_arg_count++;
324 if (word_arg_count == 4) {
325 if (types & SLJIT_ARG_TYPE_SCRATCH_REG) {
326 status |= ENTER_TMP_TO_R4;
327 arg_types &= ~(SLJIT_ARG_FULL_MASK << 3 * SLJIT_ARG_SHIFT);
328 } else if (saved_arg_count == 4) {
329 status |= ENTER_TMP_TO_S;
330 arg_types &= ~(SLJIT_ARG_FULL_MASK << 3 * SLJIT_ARG_SHIFT);
334 args_size += SSIZE_OF(sw);
335 break;
337 types >>= SLJIT_ARG_SHIFT;
340 args_size -= SSIZE_OF(sw);
341 compiler->args_size = args_size;
344 size = (scratches > 9 ? (scratches - 9) : 0) + (saveds <= 3 ? saveds : 3) - kept_saveds_count;
345 if (!(options & SLJIT_ENTER_REG_ARG))
346 size++;
348 if (size != 0) {
349 inst = (sljit_u8*)ensure_buf(compiler, (sljit_uw)(size + 1));
350 FAIL_IF(!inst);
352 INC_SIZE((sljit_uw)size);
354 if (!(options & SLJIT_ENTER_REG_ARG))
355 PUSH_REG(reg_map[TMP_REG1]);
357 if ((saveds > 2 && kept_saveds_count <= 2) || scratches > 9)
358 PUSH_REG(reg_map[SLJIT_S2]);
359 if ((saveds > 1 && kept_saveds_count <= 1) || scratches > 10)
360 PUSH_REG(reg_map[SLJIT_S1]);
361 if ((saveds > 0 && kept_saveds_count == 0) || scratches > 11)
362 PUSH_REG(reg_map[SLJIT_S0]);
364 size *= SSIZE_OF(sw);
367 if (status & (ENTER_TMP_TO_R4 | ENTER_TMP_TO_S))
368 EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_MEM1(SLJIT_SP), args_size + size);
370 size += SSIZE_OF(sw);
372 local_size = ((SLJIT_LOCALS_OFFSET_BASE + local_size + size + 0xf) & ~0xf) - size;
373 compiler->local_size = local_size;
375 word_arg_count = 0;
376 saved_arg_count = 0;
377 args_size = size;
378 while (arg_types) {
379 switch (arg_types & SLJIT_ARG_MASK) {
380 case SLJIT_ARG_TYPE_F64:
381 args_size += SSIZE_OF(f64);
382 break;
383 case SLJIT_ARG_TYPE_F32:
384 args_size += SSIZE_OF(f32);
385 break;
386 default:
387 word_arg_count++;
388 SLJIT_ASSERT(word_arg_count <= 3 || (word_arg_count == 4 && !(status & (ENTER_TMP_TO_R4 | ENTER_TMP_TO_S))));
390 if (arg_types & SLJIT_ARG_TYPE_SCRATCH_REG) {
391 #ifdef _WIN32
392 if (word_arg_count == 3 && local_size > 4 * 4096)
393 r2_offset = local_size + args_size;
394 else
395 #endif
396 EMIT_MOV(compiler, word_arg_count, 0, SLJIT_MEM1(SLJIT_SP), args_size);
398 } else {
399 EMIT_MOV(compiler, SLJIT_S0 - saved_arg_count, 0, SLJIT_MEM1(SLJIT_SP), args_size);
400 saved_arg_count++;
403 args_size += SSIZE_OF(sw);
404 break;
406 arg_types >>= SLJIT_ARG_SHIFT;
409 SLJIT_ASSERT(SLJIT_LOCALS_OFFSET > 0);
411 #ifdef _WIN32
412 SLJIT_ASSERT(r2_offset == -1 || local_size > 4 * 4096);
414 if (local_size > 4096) {
415 if (local_size <= 4 * 4096) {
416 BINARY_IMM32(OR, 0, SLJIT_MEM1(SLJIT_SP), -4096);
418 if (local_size > 2 * 4096)
419 BINARY_IMM32(OR, 0, SLJIT_MEM1(SLJIT_SP), -4096 * 2);
420 if (local_size > 3 * 4096)
421 BINARY_IMM32(OR, 0, SLJIT_MEM1(SLJIT_SP), -4096 * 3);
423 else {
424 if (options & SLJIT_ENTER_REG_ARG) {
425 SLJIT_ASSERT(r2_offset == -1);
427 inst = (sljit_u8*)ensure_buf(compiler, (sljit_uw)(1 + 1));
428 FAIL_IF(!inst);
429 INC_SIZE(1);
430 PUSH_REG(reg_map[SLJIT_R2]);
432 local_size -= SSIZE_OF(sw);
433 r2_offset = local_size;
436 EMIT_MOV(compiler, SLJIT_R2, 0, SLJIT_IMM, local_size >> 12);
438 BINARY_IMM32(OR, 0, SLJIT_MEM1(SLJIT_SP), -4096);
439 BINARY_IMM32(SUB, 4096, SLJIT_SP, 0);
441 inst = (sljit_u8*)ensure_buf(compiler, 1 + 2);
442 FAIL_IF(!inst);
444 INC_SIZE(2);
445 inst[0] = LOOP_i8;
446 inst[1] = (sljit_u8)-16;
447 local_size &= 0xfff;
451 if (local_size > 0) {
452 BINARY_IMM32(OR, 0, SLJIT_MEM1(SLJIT_SP), -local_size);
453 BINARY_IMM32(SUB, local_size, SLJIT_SP, 0);
456 if (r2_offset != -1)
457 EMIT_MOV(compiler, SLJIT_R2, 0, SLJIT_MEM1(SLJIT_SP), r2_offset);
459 #else /* !_WIN32 */
461 SLJIT_ASSERT(local_size > 0);
463 BINARY_IMM32(SUB, local_size, SLJIT_SP, 0);
465 #endif /* _WIN32 */
467 size = SLJIT_LOCALS_OFFSET_BASE - SSIZE_OF(sw);
468 kept_saveds_count = SLJIT_R3 - kept_saveds_count;
470 while (saved_arg_count > 3) {
471 EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), size, kept_saveds_count, 0);
472 kept_saveds_count++;
473 size -= SSIZE_OF(sw);
474 saved_arg_count--;
477 if (status & (ENTER_TMP_TO_R4 | ENTER_TMP_TO_S)) {
478 if (status & ENTER_TMP_TO_R4)
479 size = 2 * SSIZE_OF(sw);
481 EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), size, TMP_REG1, 0);
484 return SLJIT_SUCCESS;
487 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_set_context(struct sljit_compiler *compiler,
488 sljit_s32 options, sljit_s32 arg_types, sljit_s32 scratches, sljit_s32 saveds,
489 sljit_s32 fscratches, sljit_s32 fsaveds, sljit_s32 local_size)
491 sljit_s32 args_size;
493 CHECK_ERROR();
494 CHECK(check_sljit_set_context(compiler, options, arg_types, scratches, saveds, fscratches, fsaveds, local_size));
495 set_set_context(compiler, options, arg_types, scratches, saveds, fscratches, fsaveds, local_size);
497 arg_types >>= SLJIT_ARG_SHIFT;
498 args_size = 0;
500 if (!(options & SLJIT_ENTER_REG_ARG)) {
501 while (arg_types) {
502 switch (arg_types & SLJIT_ARG_MASK) {
503 case SLJIT_ARG_TYPE_F64:
504 args_size += SSIZE_OF(f64);
505 break;
506 case SLJIT_ARG_TYPE_F32:
507 args_size += SSIZE_OF(f32);
508 break;
509 default:
510 args_size += SSIZE_OF(sw);
511 break;
513 arg_types >>= SLJIT_ARG_SHIFT;
517 compiler->args_size = args_size;
519 /* [esp+0] for saving temporaries and for function calls. */
521 saveds = (1 + (scratches > 9 ? (scratches - 9) : 0) + (saveds <= 3 ? saveds : 3) - SLJIT_KEPT_SAVEDS_COUNT(options)) * SSIZE_OF(sw);
523 /* Saving ebp. */
524 if (!(options & SLJIT_ENTER_REG_ARG))
525 saveds += SSIZE_OF(sw);
527 compiler->local_size = ((SLJIT_LOCALS_OFFSET_BASE + local_size + saveds + 0xf) & ~0xf) - saveds;
528 return SLJIT_SUCCESS;
531 static sljit_s32 emit_stack_frame_release(struct sljit_compiler *compiler)
533 sljit_s32 kept_saveds_count = SLJIT_KEPT_SAVEDS_COUNT(compiler->options);
534 sljit_s32 saveds;
535 sljit_uw size;
536 sljit_u8 *inst;
538 BINARY_IMM32(ADD, compiler->local_size, SLJIT_SP, 0);
540 size = (sljit_uw)((compiler->scratches > 9 ? (compiler->scratches - 9) : 0) +
541 (compiler->saveds <= 3 ? compiler->saveds : 3) - kept_saveds_count);
543 if (!(compiler->options & SLJIT_ENTER_REG_ARG))
544 size++;
546 if (size == 0)
547 return SLJIT_SUCCESS;
549 inst = (sljit_u8*)ensure_buf(compiler, 1 + size);
550 FAIL_IF(!inst);
552 INC_SIZE(size);
554 saveds = compiler->saveds;
556 if ((saveds > 0 && kept_saveds_count == 0) || compiler->scratches > 11)
557 POP_REG(reg_map[SLJIT_S0]);
558 if ((saveds > 1 && kept_saveds_count <= 1) || compiler->scratches > 10)
559 POP_REG(reg_map[SLJIT_S1]);
560 if ((saveds > 2 && kept_saveds_count <= 2) || compiler->scratches > 9)
561 POP_REG(reg_map[SLJIT_S2]);
563 if (!(compiler->options & SLJIT_ENTER_REG_ARG))
564 POP_REG(reg_map[TMP_REG1]);
566 return SLJIT_SUCCESS;
569 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_return_void(struct sljit_compiler *compiler)
571 sljit_u8 *inst;
573 CHECK_ERROR();
574 CHECK(check_sljit_emit_return_void(compiler));
576 SLJIT_ASSERT(compiler->args_size >= 0);
577 SLJIT_ASSERT(compiler->local_size > 0);
579 FAIL_IF(emit_stack_frame_release(compiler));
581 inst = (sljit_u8*)ensure_buf(compiler, 1 + 1);
582 FAIL_IF(!inst);
583 INC_SIZE(1);
584 RET();
585 return SLJIT_SUCCESS;
588 /* --------------------------------------------------------------------- */
589 /* Call / return instructions */
590 /* --------------------------------------------------------------------- */
592 static sljit_s32 call_get_stack_size(sljit_s32 arg_types, sljit_s32 *word_arg_count_ptr)
594 sljit_sw stack_size = 0;
595 sljit_s32 word_arg_count = 0;
597 arg_types >>= SLJIT_ARG_SHIFT;
599 while (arg_types) {
600 switch (arg_types & SLJIT_ARG_MASK) {
601 case SLJIT_ARG_TYPE_F64:
602 stack_size += SSIZE_OF(f64);
603 break;
604 case SLJIT_ARG_TYPE_F32:
605 stack_size += SSIZE_OF(f32);
606 break;
607 default:
608 word_arg_count++;
609 stack_size += SSIZE_OF(sw);
610 break;
613 arg_types >>= SLJIT_ARG_SHIFT;
616 if (word_arg_count_ptr)
617 *word_arg_count_ptr = word_arg_count;
619 if (stack_size <= 4 * SSIZE_OF(sw))
620 return 0;
622 return ((stack_size - (4 * SSIZE_OF(sw)) + 0xf) & ~0xf);
625 static sljit_s32 call_with_args(struct sljit_compiler *compiler,
626 sljit_s32 arg_types, sljit_sw stack_size, sljit_s32 word_arg_count, sljit_s32 keep_tmp1)
628 sljit_s32 float_arg_count = 0, arg4_reg = 0, arg_offset;
629 sljit_u8 *inst;
631 if (word_arg_count >= 4) {
632 arg4_reg = SLJIT_R0;
634 if (!keep_tmp1) {
635 EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_MEM1(SLJIT_SP), 2 * SSIZE_OF(sw));
636 arg4_reg = TMP_REG1;
640 if (stack_size > 0)
641 BINARY_IMM32(SUB, stack_size, SLJIT_SP, 0);
643 arg_offset = 0;
644 word_arg_count = 0;
645 arg_types >>= SLJIT_ARG_SHIFT;
647 while (arg_types) {
648 switch (arg_types & SLJIT_ARG_MASK) {
649 case SLJIT_ARG_TYPE_F64:
650 float_arg_count++;
651 FAIL_IF(emit_sse2_store(compiler, 0, SLJIT_MEM1(SLJIT_SP), arg_offset, float_arg_count));
652 arg_offset += SSIZE_OF(f64);
653 break;
654 case SLJIT_ARG_TYPE_F32:
655 float_arg_count++;
656 FAIL_IF(emit_sse2_store(compiler, 1, SLJIT_MEM1(SLJIT_SP), arg_offset, float_arg_count));
657 arg_offset += SSIZE_OF(f32);
658 break;
659 default:
660 word_arg_count++;
661 EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), arg_offset, (word_arg_count >= 4) ? arg4_reg : word_arg_count, 0);
663 if (word_arg_count == 1 && arg4_reg == SLJIT_R0)
664 EMIT_MOV(compiler, SLJIT_R0, 0, SLJIT_MEM1(SLJIT_SP), 2 * SSIZE_OF(sw) + stack_size);
666 arg_offset += SSIZE_OF(sw);
667 break;
670 arg_types >>= SLJIT_ARG_SHIFT;
673 return SLJIT_SUCCESS;
676 static sljit_s32 post_call_with_args(struct sljit_compiler *compiler,
677 sljit_s32 arg_types, sljit_s32 stack_size)
679 sljit_u8 *inst;
680 sljit_s32 single;
682 if (stack_size > 0)
683 BINARY_IMM32(ADD, stack_size, SLJIT_SP, 0);
685 if ((arg_types & SLJIT_ARG_MASK) < SLJIT_ARG_TYPE_F64)
686 return SLJIT_SUCCESS;
688 single = ((arg_types & SLJIT_ARG_MASK) == SLJIT_ARG_TYPE_F32);
690 inst = (sljit_u8*)ensure_buf(compiler, 1 + 3);
691 FAIL_IF(!inst);
692 INC_SIZE(3);
693 inst[0] = single ? FSTPS : FSTPD;
694 inst[1] = (0x03 << 3) | 0x04;
695 inst[2] = (0x04 << 3) | reg_map[SLJIT_SP];
697 return emit_sse2_load(compiler, single, SLJIT_FR0, SLJIT_MEM1(SLJIT_SP), 0);
700 static sljit_s32 tail_call_with_args(struct sljit_compiler *compiler,
701 sljit_s32 *extra_space, sljit_s32 arg_types,
702 sljit_s32 src, sljit_sw srcw)
704 sljit_sw args_size, saved_regs_size;
705 sljit_sw types, word_arg_count, float_arg_count;
706 sljit_sw stack_size, prev_stack_size, min_size, offset;
707 sljit_sw word_arg4_offset;
708 sljit_u8 r2_offset = 0;
709 sljit_s32 kept_saveds_count = SLJIT_KEPT_SAVEDS_COUNT(compiler->options);
710 sljit_u8* inst;
712 ADJUST_LOCAL_OFFSET(src, srcw);
713 CHECK_EXTRA_REGS(src, srcw, (void)0);
715 saved_regs_size = (1 + (compiler->scratches > 9 ? (compiler->scratches - 9) : 0)
716 + (compiler->saveds <= 3 ? compiler->saveds : 3) - kept_saveds_count) * SSIZE_OF(sw);
718 word_arg_count = 0;
719 float_arg_count = 0;
720 arg_types >>= SLJIT_ARG_SHIFT;
721 types = 0;
722 args_size = 0;
724 while (arg_types != 0) {
725 types = (types << SLJIT_ARG_SHIFT) | (arg_types & SLJIT_ARG_MASK);
727 switch (arg_types & SLJIT_ARG_MASK) {
728 case SLJIT_ARG_TYPE_F64:
729 args_size += SSIZE_OF(f64);
730 float_arg_count++;
731 break;
732 case SLJIT_ARG_TYPE_F32:
733 args_size += SSIZE_OF(f32);
734 float_arg_count++;
735 break;
736 default:
737 word_arg_count++;
738 args_size += SSIZE_OF(sw);
739 break;
741 arg_types >>= SLJIT_ARG_SHIFT;
744 if (args_size <= compiler->args_size) {
745 *extra_space = 0;
746 stack_size = args_size + SSIZE_OF(sw) + saved_regs_size;
748 offset = stack_size + compiler->local_size;
750 if (!(src & SLJIT_IMM) && src != SLJIT_R0) {
751 if (word_arg_count >= 1) {
752 EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), 0, SLJIT_R0, 0);
753 r2_offset = sizeof(sljit_sw);
755 EMIT_MOV(compiler, SLJIT_R0, 0, src, srcw);
758 while (types != 0) {
759 switch (types & SLJIT_ARG_MASK) {
760 case SLJIT_ARG_TYPE_F64:
761 offset -= SSIZE_OF(f64);
762 FAIL_IF(emit_sse2_store(compiler, 0, SLJIT_MEM1(SLJIT_SP), offset, float_arg_count));
763 float_arg_count--;
764 break;
765 case SLJIT_ARG_TYPE_F32:
766 offset -= SSIZE_OF(f32);
767 FAIL_IF(emit_sse2_store(compiler, 0, SLJIT_MEM1(SLJIT_SP), offset, float_arg_count));
768 float_arg_count--;
769 break;
770 default:
771 switch (word_arg_count) {
772 case 1:
773 offset -= SSIZE_OF(sw);
774 if (r2_offset != 0) {
775 EMIT_MOV(compiler, SLJIT_R2, 0, SLJIT_MEM1(SLJIT_SP), 0);
776 EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), offset, SLJIT_R2, 0);
777 } else
778 EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), offset, SLJIT_R0, 0);
779 break;
780 case 2:
781 offset -= SSIZE_OF(sw);
782 EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), offset, SLJIT_R1, 0);
783 break;
784 case 3:
785 offset -= SSIZE_OF(sw);
786 break;
787 case 4:
788 offset -= SSIZE_OF(sw);
789 EMIT_MOV(compiler, SLJIT_R2, 0, SLJIT_MEM1(SLJIT_SP), 2 * SSIZE_OF(sw));
790 EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), offset, SLJIT_R2, 0);
791 break;
793 word_arg_count--;
794 break;
796 types >>= SLJIT_ARG_SHIFT;
799 return emit_stack_frame_release(compiler);
802 stack_size = args_size + SSIZE_OF(sw);
804 if (word_arg_count >= 1 && !(src & SLJIT_IMM) && src != SLJIT_R0) {
805 r2_offset = SSIZE_OF(sw);
806 stack_size += SSIZE_OF(sw);
809 if (word_arg_count >= 3)
810 stack_size += SSIZE_OF(sw);
812 prev_stack_size = SSIZE_OF(sw) + saved_regs_size;
813 min_size = prev_stack_size + compiler->local_size;
815 word_arg4_offset = 2 * SSIZE_OF(sw);
817 if (stack_size > min_size) {
818 BINARY_IMM32(SUB, stack_size - min_size, SLJIT_SP, 0);
819 if (src == SLJIT_MEM1(SLJIT_SP))
820 srcw += stack_size - min_size;
821 word_arg4_offset += stack_size - min_size;
823 else
824 stack_size = min_size;
826 if (word_arg_count >= 3) {
827 EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), r2_offset, SLJIT_R2, 0);
829 if (word_arg_count >= 4)
830 EMIT_MOV(compiler, SLJIT_R2, 0, SLJIT_MEM1(SLJIT_SP), word_arg4_offset);
833 if (!(src & SLJIT_IMM) && src != SLJIT_R0) {
834 if (word_arg_count >= 1) {
835 SLJIT_ASSERT(r2_offset == sizeof(sljit_sw));
836 EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), 0, SLJIT_R0, 0);
838 EMIT_MOV(compiler, SLJIT_R0, 0, src, srcw);
841 /* Restore saved registers. */
842 offset = stack_size - 2 * SSIZE_OF(sw);
843 EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_MEM1(SLJIT_SP), offset);
845 if (compiler->saveds > 2 || compiler->scratches > 9) {
846 offset -= SSIZE_OF(sw);
847 EMIT_MOV(compiler, SLJIT_S2, 0, SLJIT_MEM1(SLJIT_SP), offset);
849 if ((compiler->saveds > 1 && kept_saveds_count <= 1) || compiler->scratches > 10) {
850 offset -= SSIZE_OF(sw);
851 EMIT_MOV(compiler, SLJIT_S1, 0, SLJIT_MEM1(SLJIT_SP), offset);
853 if ((compiler->saveds > 0 && kept_saveds_count == 0) || compiler->scratches > 11) {
854 offset -= SSIZE_OF(sw);
855 EMIT_MOV(compiler, SLJIT_S0, 0, SLJIT_MEM1(SLJIT_SP), offset);
858 /* Copy fourth argument and return address. */
859 offset = stack_size - SSIZE_OF(sw);
860 *extra_space = args_size;
862 if (word_arg_count >= 4) {
863 offset -= SSIZE_OF(sw);
864 EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), offset, SLJIT_R2, 0);
867 while (types != 0) {
868 switch (types & SLJIT_ARG_MASK) {
869 case SLJIT_ARG_TYPE_F64:
870 offset -= SSIZE_OF(f64);
871 FAIL_IF(emit_sse2_store(compiler, 0, SLJIT_MEM1(SLJIT_SP), offset, float_arg_count));
872 float_arg_count--;
873 break;
874 case SLJIT_ARG_TYPE_F32:
875 offset -= SSIZE_OF(f32);
876 FAIL_IF(emit_sse2_store(compiler, 0, SLJIT_MEM1(SLJIT_SP), offset, float_arg_count));
877 float_arg_count--;
878 break;
879 default:
880 switch (word_arg_count) {
881 case 1:
882 offset -= SSIZE_OF(sw);
883 if (r2_offset != 0) {
884 EMIT_MOV(compiler, SLJIT_R2, 0, SLJIT_MEM1(SLJIT_SP), 0);
885 EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), offset, SLJIT_R2, 0);
886 } else
887 EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), offset, SLJIT_R0, 0);
888 break;
889 case 2:
890 offset -= SSIZE_OF(sw);
891 EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), offset, SLJIT_R1, 0);
892 break;
893 case 3:
894 offset -= SSIZE_OF(sw);
895 EMIT_MOV(compiler, SLJIT_R2, 0, SLJIT_MEM1(SLJIT_SP), r2_offset);
896 EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), offset, SLJIT_R2, 0);
897 break;
899 word_arg_count--;
900 break;
902 types >>= SLJIT_ARG_SHIFT;
905 SLJIT_ASSERT(offset >= 0);
907 if (offset == 0)
908 return SLJIT_SUCCESS;
910 BINARY_IMM32(ADD, offset, SLJIT_SP, 0);
911 return SLJIT_SUCCESS;
914 static sljit_s32 emit_tail_call_end(struct sljit_compiler *compiler, sljit_s32 extra_space)
916 /* Called when stack consumption cannot be reduced to 0. */
917 sljit_u8 *inst;
919 BINARY_IMM32(ADD, extra_space, SLJIT_SP, 0);
921 inst = (sljit_u8*)ensure_buf(compiler, 1 + 1);
922 FAIL_IF(!inst);
923 INC_SIZE(1);
924 RET();
926 return SLJIT_SUCCESS;
929 static sljit_s32 tail_call_reg_arg_with_args(struct sljit_compiler *compiler, sljit_s32 arg_types)
931 sljit_s32 word_arg_count = 0;
932 sljit_s32 kept_saveds_count, offset;
934 arg_types >>= SLJIT_ARG_SHIFT;
936 while (arg_types) {
937 if ((arg_types & SLJIT_ARG_MASK) < SLJIT_ARG_TYPE_F64)
938 word_arg_count++;
940 arg_types >>= SLJIT_ARG_SHIFT;
943 if (word_arg_count < 4)
944 return SLJIT_SUCCESS;
946 EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_MEM1(SLJIT_SP), 2 * SSIZE_OF(sw));
948 kept_saveds_count = SLJIT_KEPT_SAVEDS_COUNT(compiler->options);
949 offset = compiler->local_size + 3 * SSIZE_OF(sw);
951 if ((compiler->saveds > 0 && kept_saveds_count == 0) || compiler->scratches > 11)
952 offset += SSIZE_OF(sw);
953 if ((compiler->saveds > 1 && kept_saveds_count <= 1) || compiler->scratches > 10)
954 offset += SSIZE_OF(sw);
955 if ((compiler->saveds > 2 && kept_saveds_count <= 2) || compiler->scratches > 9)
956 offset += SSIZE_OF(sw);
958 return emit_mov(compiler, SLJIT_MEM1(SLJIT_SP), offset, TMP_REG1, 0);
961 SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_call(struct sljit_compiler *compiler, sljit_s32 type,
962 sljit_s32 arg_types)
964 struct sljit_jump *jump;
965 sljit_sw stack_size = 0;
966 sljit_s32 word_arg_count;
968 CHECK_ERROR_PTR();
969 CHECK_PTR(check_sljit_emit_call(compiler, type, arg_types));
971 if (type & SLJIT_CALL_RETURN) {
972 if ((type & 0xff) == SLJIT_CALL_REG_ARG) {
973 PTR_FAIL_IF(tail_call_reg_arg_with_args(compiler, arg_types));
974 PTR_FAIL_IF(emit_stack_frame_release(compiler));
976 SLJIT_SKIP_CHECKS(compiler);
977 return sljit_emit_jump(compiler, SLJIT_JUMP | (type & SLJIT_REWRITABLE_JUMP));
980 stack_size = type;
981 PTR_FAIL_IF(tail_call_with_args(compiler, &stack_size, arg_types, SLJIT_IMM, 0));
983 SLJIT_SKIP_CHECKS(compiler);
985 if (stack_size == 0)
986 return sljit_emit_jump(compiler, SLJIT_JUMP | (type & SLJIT_REWRITABLE_JUMP));
988 jump = sljit_emit_jump(compiler, type);
989 PTR_FAIL_IF(jump == NULL);
991 PTR_FAIL_IF(emit_tail_call_end(compiler, stack_size));
992 return jump;
995 if ((type & 0xff) == SLJIT_CALL_REG_ARG) {
996 SLJIT_SKIP_CHECKS(compiler);
997 return sljit_emit_jump(compiler, type);
1000 stack_size = call_get_stack_size(arg_types, &word_arg_count);
1001 PTR_FAIL_IF(call_with_args(compiler, arg_types, stack_size, word_arg_count, 0));
1003 SLJIT_SKIP_CHECKS(compiler);
1004 jump = sljit_emit_jump(compiler, type);
1005 PTR_FAIL_IF(jump == NULL);
1007 PTR_FAIL_IF(post_call_with_args(compiler, arg_types, stack_size));
1008 return jump;
1011 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_icall(struct sljit_compiler *compiler, sljit_s32 type,
1012 sljit_s32 arg_types,
1013 sljit_s32 src, sljit_sw srcw)
1015 sljit_sw stack_size = 0;
1016 sljit_s32 word_arg_count;
1018 CHECK_ERROR();
1019 CHECK(check_sljit_emit_icall(compiler, type, arg_types, src, srcw));
1021 if (type & SLJIT_CALL_RETURN) {
1022 if ((type & 0xff) == SLJIT_CALL_REG_ARG) {
1023 FAIL_IF(tail_call_reg_arg_with_args(compiler, arg_types));
1025 if ((src & SLJIT_MEM) || (src >= SLJIT_FIRST_SAVED_REG && src <= SLJIT_S0)) {
1026 ADJUST_LOCAL_OFFSET(src, srcw);
1027 CHECK_EXTRA_REGS(src, srcw, (void)0);
1029 EMIT_MOV(compiler, TMP_REG1, 0, src, srcw);
1030 src = TMP_REG1;
1031 srcw = 0;
1034 FAIL_IF(emit_stack_frame_release(compiler));
1036 SLJIT_SKIP_CHECKS(compiler);
1037 return sljit_emit_ijump(compiler, SLJIT_JUMP, src, srcw);
1040 stack_size = type;
1041 FAIL_IF(tail_call_with_args(compiler, &stack_size, arg_types, src, srcw));
1043 if (!(src & SLJIT_IMM)) {
1044 src = SLJIT_R0;
1045 srcw = 0;
1048 SLJIT_SKIP_CHECKS(compiler);
1050 if (stack_size == 0)
1051 return sljit_emit_ijump(compiler, SLJIT_JUMP, src, srcw);
1053 FAIL_IF(sljit_emit_ijump(compiler, type, src, srcw));
1054 return emit_tail_call_end(compiler, stack_size);
1057 if ((type & 0xff) == SLJIT_CALL_REG_ARG) {
1058 SLJIT_SKIP_CHECKS(compiler);
1059 return sljit_emit_ijump(compiler, type, src, srcw);
1062 ADJUST_LOCAL_OFFSET(src, srcw);
1063 CHECK_EXTRA_REGS(src, srcw, (void)0);
1065 if (src & SLJIT_MEM) {
1066 EMIT_MOV(compiler, TMP_REG1, 0, src, srcw);
1067 src = TMP_REG1;
1068 srcw = 0;
1071 stack_size = call_get_stack_size(arg_types, &word_arg_count);
1072 FAIL_IF(call_with_args(compiler, arg_types, stack_size, word_arg_count, src == TMP_REG1));
1074 if (stack_size > 0 && src == SLJIT_MEM1(SLJIT_SP))
1075 srcw += stack_size;
1077 SLJIT_SKIP_CHECKS(compiler);
1078 FAIL_IF(sljit_emit_ijump(compiler, type, src, srcw));
1080 return post_call_with_args(compiler, arg_types, stack_size);
1083 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fast_enter(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw)
1085 sljit_u8 *inst;
1087 CHECK_ERROR();
1088 CHECK(check_sljit_emit_fast_enter(compiler, dst, dstw));
1089 ADJUST_LOCAL_OFFSET(dst, dstw);
1091 CHECK_EXTRA_REGS(dst, dstw, (void)0);
1093 if (FAST_IS_REG(dst)) {
1094 /* Unused dest is possible here. */
1095 inst = (sljit_u8*)ensure_buf(compiler, 1 + 1);
1096 FAIL_IF(!inst);
1098 INC_SIZE(1);
1099 POP_REG(reg_map[dst]);
1100 return SLJIT_SUCCESS;
1103 /* Memory. */
1104 inst = emit_x86_instruction(compiler, 1, 0, 0, dst, dstw);
1105 FAIL_IF(!inst);
1106 *inst++ = POP_rm;
1107 return SLJIT_SUCCESS;
1110 static sljit_s32 emit_fast_return(struct sljit_compiler *compiler, sljit_s32 src, sljit_sw srcw)
1112 sljit_u8 *inst;
1114 CHECK_EXTRA_REGS(src, srcw, (void)0);
1116 if (FAST_IS_REG(src)) {
1117 inst = (sljit_u8*)ensure_buf(compiler, 1 + 1 + 1);
1118 FAIL_IF(!inst);
1120 INC_SIZE(1 + 1);
1121 PUSH_REG(reg_map[src]);
1123 else {
1124 inst = emit_x86_instruction(compiler, 1, 0, 0, src, srcw);
1125 FAIL_IF(!inst);
1126 *inst++ = GROUP_FF;
1127 *inst |= PUSH_rm;
1129 inst = (sljit_u8*)ensure_buf(compiler, 1 + 1);
1130 FAIL_IF(!inst);
1131 INC_SIZE(1);
1134 RET();
1135 return SLJIT_SUCCESS;
1138 static sljit_s32 skip_frames_before_return(struct sljit_compiler *compiler)
1140 sljit_sw size;
1142 /* Don't adjust shadow stack if it isn't enabled. */
1143 if (!cpu_has_shadow_stack())
1144 return SLJIT_SUCCESS;
1146 SLJIT_ASSERT(compiler->args_size >= 0);
1147 SLJIT_ASSERT(compiler->local_size > 0);
1149 size = compiler->local_size;
1150 size += (1 + (compiler->scratches > 9 ? (compiler->scratches - 9) : 0)
1151 + (compiler->saveds <= 3 ? compiler->saveds : 3)) * SSIZE_OF(sw);
1153 return adjust_shadow_stack(compiler, SLJIT_MEM1(SLJIT_SP), size);