sljit_src/sljitNativeX86_32.c

   1 /*
   2  *    Stack-less Just-In-Time compiler
   3  *
   4  *    Copyright Zoltan Herczeg (hzmester@freemail.hu). All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without modification, are
   7  * permitted provided that the following conditions are met:
   8  *
   9  *   1. Redistributions of source code must retain the above copyright notice, this list of
  10  *      conditions and the following disclaimer.
  11  *
  12  *   2. Redistributions in binary form must reproduce the above copyright notice, this list
  13  *      of conditions and the following disclaimer in the documentation and/or other materials
  14  *      provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) AND CONTRIBUTORS ``AS IS'' AND ANY
  17  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  18  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
  19  * SHALL THE COPYRIGHT HOLDER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  20  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
  21  * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
  22  * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  23  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  24  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  */
  26
  27 /* x86 32-bit arch dependent functions. */
  28
  29 static sljit_s32 emit_do_imm(struct sljit_compiler *compiler, sljit_u8 opcode, sljit_sw imm)
  30 {
  31         sljit_u8 *inst;
  32
  33         inst = (sljit_u8*)ensure_buf(compiler, 1 + 1 + sizeof(sljit_sw));
  34         FAIL_IF(!inst);
  35         INC_SIZE(1 + sizeof(sljit_sw));
  36         *inst++ = opcode;
  37         sljit_unaligned_store_sw(inst, imm);
  38         return SLJIT_SUCCESS;
  39 }
  40
  41 static sljit_u8* generate_far_jump_code(struct sljit_jump *jump, sljit_u8 *code_ptr, sljit_sw executable_offset)
  42 {
  43         sljit_s32 type = jump->flags >> TYPE_SHIFT;
  44
  45         if (type == SLJIT_JUMP) {
  46                 *code_ptr++ = JMP_i32;
  47                 jump->addr++;
  48         }
  49         else if (type >= SLJIT_FAST_CALL) {
  50                 *code_ptr++ = CALL_i32;
  51                 jump->addr++;
  52         }
  53         else {
  54                 *code_ptr++ = GROUP_0F;
  55                 *code_ptr++ = get_jump_code(type);
  56                 jump->addr += 2;
  57         }
  58
  59         if (jump->flags & JUMP_LABEL)
  60                 jump->flags |= PATCH_MW;
  61         else
  62                 sljit_unaligned_store_sw(code_ptr, jump->u.target - (jump->addr + 4) - (sljit_uw)executable_offset);
  63         code_ptr += 4;
  64
  65         return code_ptr;
  66 }
  67
  68 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_enter(struct sljit_compiler *compiler,
  69         sljit_s32 options, sljit_s32 arg_types, sljit_s32 scratches, sljit_s32 saveds,
  70         sljit_s32 fscratches, sljit_s32 fsaveds, sljit_s32 local_size)
  71 {
  72         sljit_s32 args, size;
  73         sljit_u8 *inst;
  74
  75         CHECK_ERROR();
  76         CHECK(check_sljit_emit_enter(compiler, options, arg_types, scratches, saveds, fscratches, fsaveds, local_size));
  77         set_emit_enter(compiler, options, arg_types, scratches, saveds, fscratches, fsaveds, local_size);
  78
  79         /* Emit ENDBR32 at function entry if needed.  */
  80         FAIL_IF(emit_endbranch(compiler));
  81
  82         args = get_arg_count(arg_types);
  83         compiler->args = args;
  84
  85         /* [esp+0] for saving temporaries and function calls. */
  86         compiler->stack_tmp_size = 2 * sizeof(sljit_sw);
  87
  88 #if !(defined SLJIT_X86_32_FASTCALL && SLJIT_X86_32_FASTCALL)
  89         if (scratches > 3)
  90                 compiler->stack_tmp_size = 3 * sizeof(sljit_sw);
  91 #endif
  92
  93         compiler->saveds_offset = compiler->stack_tmp_size;
  94         if (scratches > 3)
  95                 compiler->saveds_offset += ((scratches > (3 + 6)) ? 6 : (scratches - 3)) * sizeof(sljit_sw);
  96
  97         compiler->locals_offset = compiler->saveds_offset;
  98
  99         if (saveds > 3)
 100                 compiler->locals_offset += (saveds - 3) * sizeof(sljit_sw);
 101
 102         if (options & SLJIT_F64_ALIGNMENT)
 103                 compiler->locals_offset = (compiler->locals_offset + sizeof(sljit_f64) - 1) & ~(sizeof(sljit_f64) - 1);
 104
 105         size = 1 + (scratches > 9 ? (scratches - 9) : 0) + (saveds <= 3 ? saveds : 3);
 106 #if (defined SLJIT_X86_32_FASTCALL && SLJIT_X86_32_FASTCALL)
 107         size += (args > 0 ? (args * 2) : 0) + (args > 2 ? 2 : 0);
 108 #else
 109         size += (args > 0 ? (2 + args * 3) : 0);
 110 #endif
 111         inst = (sljit_u8*)ensure_buf(compiler, 1 + size);
 112         FAIL_IF(!inst);
 113
 114         INC_SIZE(size);
 115         PUSH_REG(reg_map[TMP_REG1]);
 116 #if !(defined SLJIT_X86_32_FASTCALL && SLJIT_X86_32_FASTCALL)
 117         if (args > 0) {
 118                 *inst++ = MOV_r_rm;
 119                 *inst++ = MOD_REG | (reg_map[TMP_REG1] << 3) | 0x4 /* esp */;
 120         }
 121 #endif
 122         if (saveds > 2 || scratches > 9)
 123                 PUSH_REG(reg_map[SLJIT_S2]);
 124         if (saveds > 1 || scratches > 10)
 125                 PUSH_REG(reg_map[SLJIT_S1]);
 126         if (saveds > 0 || scratches > 11)
 127                 PUSH_REG(reg_map[SLJIT_S0]);
 128
 129 #if (defined SLJIT_X86_32_FASTCALL && SLJIT_X86_32_FASTCALL)
 130         if (args > 0) {
 131                 inst[0] = MOV_r_rm;
 132                 inst[1] = MOD_REG | (reg_map[SLJIT_S0] << 3) | reg_map[SLJIT_R2];
 133                 inst += 2;
 134         }
 135         if (args > 1) {
 136                 inst[0] = MOV_r_rm;
 137                 inst[1] = MOD_REG | (reg_map[SLJIT_S1] << 3) | reg_map[SLJIT_R1];
 138                 inst += 2;
 139         }
 140         if (args > 2) {
 141                 inst[0] = MOV_r_rm;
 142                 inst[1] = MOD_DISP8 | (reg_map[SLJIT_S2] << 3) | 0x4 /* esp */;
 143                 inst[2] = 0x24;
 144                 inst[3] = sizeof(sljit_sw) * (3 + 2); /* saveds >= 3 as well. */
 145         }
 146 #else
 147         if (args > 0) {
 148                 inst[0] = MOV_r_rm;
 149                 inst[1] = MOD_DISP8 | (reg_map[SLJIT_S0] << 3) | reg_map[TMP_REG1];
 150                 inst[2] = sizeof(sljit_sw) * 2;
 151                 inst += 3;
 152         }
 153         if (args > 1) {
 154                 inst[0] = MOV_r_rm;
 155                 inst[1] = MOD_DISP8 | (reg_map[SLJIT_S1] << 3) | reg_map[TMP_REG1];
 156                 inst[2] = sizeof(sljit_sw) * 3;
 157                 inst += 3;
 158         }
 159         if (args > 2) {
 160                 inst[0] = MOV_r_rm;
 161                 inst[1] = MOD_DISP8 | (reg_map[SLJIT_S2] << 3) | reg_map[TMP_REG1];
 162                 inst[2] = sizeof(sljit_sw) * 4;
 163         }
 164 #endif
 165
 166         SLJIT_ASSERT(SLJIT_LOCALS_OFFSET > 0);
 167
 168 #if defined(__APPLE__)
 169         /* Ignore pushed registers and SLJIT_LOCALS_OFFSET when computing the aligned local size. */
 170         saveds = (2 + (scratches > 9 ? (scratches - 9) : 0) + (saveds <= 3 ? saveds : 3)) * sizeof(sljit_uw);
 171         local_size = ((SLJIT_LOCALS_OFFSET + saveds + local_size + 15) & ~15) - saveds;
 172 #else
 173         if (options & SLJIT_F64_ALIGNMENT)
 174                 local_size = SLJIT_LOCALS_OFFSET + ((local_size + sizeof(sljit_f64) - 1) & ~(sizeof(sljit_f64) - 1));
 175         else
 176                 local_size = SLJIT_LOCALS_OFFSET + ((local_size + sizeof(sljit_sw) - 1) & ~(sizeof(sljit_sw) - 1));
 177 #endif
 178
 179         compiler->local_size = local_size;
 180
 181 #ifdef _WIN32
 182         if (local_size > 0) {
 183                 if (local_size <= 4 * 4096) {
 184                         if (local_size > 4096)
 185                                 EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_MEM1(SLJIT_SP), -4096);
 186                         if (local_size > 2 * 4096)
 187                                 EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_MEM1(SLJIT_SP), -4096 * 2);
 188                         if (local_size > 3 * 4096)
 189                                 EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_MEM1(SLJIT_SP), -4096 * 3);
 190                 }
 191                 else {
 192                         EMIT_MOV(compiler, SLJIT_R0, 0, SLJIT_SP, 0);
 193                         EMIT_MOV(compiler, SLJIT_R1, 0, SLJIT_IMM, (local_size - 1) >> 12);
 194
 195                         SLJIT_ASSERT (reg_map[SLJIT_R0] == 0);
 196
 197                         EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_MEM1(SLJIT_R0), -4096);
 198                         FAIL_IF(emit_non_cum_binary(compiler, BINARY_OPCODE(SUB),
 199                                 SLJIT_R0, 0, SLJIT_R0, 0, SLJIT_IMM, 4096));
 200                         FAIL_IF(emit_non_cum_binary(compiler, BINARY_OPCODE(SUB),
 201                                 SLJIT_R1, 0, SLJIT_R1, 0, SLJIT_IMM, 1));
 202
 203                         inst = (sljit_u8*)ensure_buf(compiler, 1 + 2);
 204                         FAIL_IF(!inst);
 205
 206                         INC_SIZE(2);
 207                         inst[0] = JNE_i8;
 208                         inst[1] = (sljit_s8) -16;
 209                 }
 210
 211                 EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_MEM1(SLJIT_SP), -local_size);
 212         }
 213 #endif
 214
 215         SLJIT_ASSERT(local_size > 0);
 216
 217 #if !defined(__APPLE__)
 218         if (options & SLJIT_F64_ALIGNMENT) {
 219                 EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_SP, 0);
 220
 221                 /* Some space might allocated during sljit_grow_stack() above on WIN32. */
 222                 FAIL_IF(emit_non_cum_binary(compiler, BINARY_OPCODE(SUB),
 223                         SLJIT_SP, 0, SLJIT_SP, 0, SLJIT_IMM, local_size + sizeof(sljit_sw)));
 224
 225 #if defined _WIN32 && !(defined SLJIT_X86_32_FASTCALL && SLJIT_X86_32_FASTCALL)
 226                 if (compiler->local_size > 1024)
 227                         FAIL_IF(emit_cum_binary(compiler, BINARY_OPCODE(ADD),
 228                                 TMP_REG1, 0, TMP_REG1, 0, SLJIT_IMM, sizeof(sljit_sw)));
 229 #endif
 230
 231                 inst = (sljit_u8*)ensure_buf(compiler, 1 + 6);
 232                 FAIL_IF(!inst);
 233
 234                 INC_SIZE(6);
 235                 inst[0] = GROUP_BINARY_81;
 236                 inst[1] = MOD_REG | AND | reg_map[SLJIT_SP];
 237                 sljit_unaligned_store_sw(inst + 2, ~(sizeof(sljit_f64) - 1));
 238
 239                 /* The real local size must be used. */
 240                 return emit_mov(compiler, SLJIT_MEM1(SLJIT_SP), compiler->local_size, TMP_REG1, 0);
 241         }
 242 #endif
 243         return emit_non_cum_binary(compiler, BINARY_OPCODE(SUB),
 244                 SLJIT_SP, 0, SLJIT_SP, 0, SLJIT_IMM, local_size);
 245 }
 246
 247 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_set_context(struct sljit_compiler *compiler,
 248         sljit_s32 options, sljit_s32 arg_types, sljit_s32 scratches, sljit_s32 saveds,
 249         sljit_s32 fscratches, sljit_s32 fsaveds, sljit_s32 local_size)
 250 {
 251         CHECK_ERROR();
 252         CHECK(check_sljit_set_context(compiler, options, arg_types, scratches, saveds, fscratches, fsaveds, local_size));
 253         set_set_context(compiler, options, arg_types, scratches, saveds, fscratches, fsaveds, local_size);
 254
 255         compiler->args = get_arg_count(arg_types);
 256
 257         /* [esp+0] for saving temporaries and function calls. */
 258         compiler->stack_tmp_size = 2 * sizeof(sljit_sw);
 259
 260 #if !(defined SLJIT_X86_32_FASTCALL && SLJIT_X86_32_FASTCALL)
 261         if (scratches > 3)
 262                 compiler->stack_tmp_size = 3 * sizeof(sljit_sw);
 263 #endif
 264
 265         compiler->saveds_offset = compiler->stack_tmp_size;
 266         if (scratches > 3)
 267                 compiler->saveds_offset += ((scratches > (3 + 6)) ? 6 : (scratches - 3)) * sizeof(sljit_sw);
 268
 269         compiler->locals_offset = compiler->saveds_offset;
 270
 271         if (saveds > 3)
 272                 compiler->locals_offset += (saveds - 3) * sizeof(sljit_sw);
 273
 274         if (options & SLJIT_F64_ALIGNMENT)
 275                 compiler->locals_offset = (compiler->locals_offset + sizeof(sljit_f64) - 1) & ~(sizeof(sljit_f64) - 1);
 276
 277 #if defined(__APPLE__)
 278         saveds = (2 + (scratches > 9 ? (scratches - 9) : 0) + (saveds <= 3 ? saveds : 3)) * sizeof(sljit_uw);
 279         compiler->local_size = ((SLJIT_LOCALS_OFFSET + saveds + local_size + 15) & ~15) - saveds;
 280 #else
 281         if (options & SLJIT_F64_ALIGNMENT)
 282                 compiler->local_size = SLJIT_LOCALS_OFFSET + ((local_size + sizeof(sljit_f64) - 1) & ~(sizeof(sljit_f64) - 1));
 283         else
 284                 compiler->local_size = SLJIT_LOCALS_OFFSET + ((local_size + sizeof(sljit_sw) - 1) & ~(sizeof(sljit_sw) - 1));
 285 #endif
 286         return SLJIT_SUCCESS;
 287 }
 288
 289 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_return(struct sljit_compiler *compiler, sljit_s32 op, sljit_s32 src, sljit_sw srcw)
 290 {
 291         sljit_s32 size;
 292         sljit_u8 *inst;
 293
 294         CHECK_ERROR();
 295         CHECK(check_sljit_emit_return(compiler, op, src, srcw));
 296         SLJIT_ASSERT(compiler->args >= 0);
 297
 298         FAIL_IF(emit_mov_before_return(compiler, op, src, srcw));
 299
 300         SLJIT_ASSERT(compiler->local_size > 0);
 301
 302 #if !defined(__APPLE__)
 303         if (compiler->options & SLJIT_F64_ALIGNMENT)
 304                 EMIT_MOV(compiler, SLJIT_SP, 0, SLJIT_MEM1(SLJIT_SP), compiler->local_size)
 305         else
 306                 FAIL_IF(emit_cum_binary(compiler, BINARY_OPCODE(ADD),
 307                         SLJIT_SP, 0, SLJIT_SP, 0, SLJIT_IMM, compiler->local_size));
 308 #else
 309         FAIL_IF(emit_cum_binary(compiler, BINARY_OPCODE(ADD),
 310                 SLJIT_SP, 0, SLJIT_SP, 0, SLJIT_IMM, compiler->local_size));
 311 #endif
 312
 313         size = 2 + (compiler->scratches > 7 ? (compiler->scratches - 7) : 0) +
 314                 (compiler->saveds <= 3 ? compiler->saveds : 3);
 315 #if (defined SLJIT_X86_32_FASTCALL && SLJIT_X86_32_FASTCALL)
 316         if (compiler->args > 2)
 317                 size += 2;
 318 #else
 319         if (compiler->args > 0)
 320                 size += 2;
 321 #endif
 322         inst = (sljit_u8*)ensure_buf(compiler, 1 + size);
 323         FAIL_IF(!inst);
 324
 325         INC_SIZE(size);
 326
 327         if (compiler->saveds > 0 || compiler->scratches > 11)
 328                 POP_REG(reg_map[SLJIT_S0]);
 329         if (compiler->saveds > 1 || compiler->scratches > 10)
 330                 POP_REG(reg_map[SLJIT_S1]);
 331         if (compiler->saveds > 2 || compiler->scratches > 9)
 332                 POP_REG(reg_map[SLJIT_S2]);
 333         POP_REG(reg_map[TMP_REG1]);
 334 #if (defined SLJIT_X86_32_FASTCALL && SLJIT_X86_32_FASTCALL)
 335         if (compiler->args > 2)
 336                 RET_I16(sizeof(sljit_sw));
 337         else
 338                 RET();
 339 #else
 340         RET();
 341 #endif
 342
 343         return SLJIT_SUCCESS;
 344 }
 345
 346 /* --------------------------------------------------------------------- */
 347 /*  Operators                                                            */
 348 /* --------------------------------------------------------------------- */
 349
 350 /* Size contains the flags as well. */
 351 static sljit_u8* emit_x86_instruction(struct sljit_compiler *compiler, sljit_s32 size,
 352         /* The register or immediate operand. */
 353         sljit_s32 a, sljit_sw imma,
 354         /* The general operand (not immediate). */
 355         sljit_s32 b, sljit_sw immb)
 356 {
 357         sljit_u8 *inst;
 358         sljit_u8 *buf_ptr;
 359         sljit_s32 flags = size & ~0xf;
 360         sljit_s32 inst_size;
 361
 362         /* Both cannot be switched on. */
 363         SLJIT_ASSERT((flags & (EX86_BIN_INS | EX86_SHIFT_INS)) != (EX86_BIN_INS | EX86_SHIFT_INS));
 364         /* Size flags not allowed for typed instructions. */
 365         SLJIT_ASSERT(!(flags & (EX86_BIN_INS | EX86_SHIFT_INS)) || (flags & (EX86_BYTE_ARG | EX86_HALF_ARG)) == 0);
 366         /* Both size flags cannot be switched on. */
 367         SLJIT_ASSERT((flags & (EX86_BYTE_ARG | EX86_HALF_ARG)) != (EX86_BYTE_ARG | EX86_HALF_ARG));
 368         /* SSE2 and immediate is not possible. */
 369         SLJIT_ASSERT(!(a & SLJIT_IMM) || !(flags & EX86_SSE2));
 370         SLJIT_ASSERT((flags & (EX86_PREF_F2 | EX86_PREF_F3)) != (EX86_PREF_F2 | EX86_PREF_F3)
 371                 && (flags & (EX86_PREF_F2 | EX86_PREF_66)) != (EX86_PREF_F2 | EX86_PREF_66)
 372                 && (flags & (EX86_PREF_F3 | EX86_PREF_66)) != (EX86_PREF_F3 | EX86_PREF_66));
 373
 374         size &= 0xf;
 375         inst_size = size;
 376
 377         if (flags & (EX86_PREF_F2 | EX86_PREF_F3))
 378                 inst_size++;
 379         if (flags & EX86_PREF_66)
 380                 inst_size++;
 381
 382         /* Calculate size of b. */
 383         inst_size += 1; /* mod r/m byte. */
 384         if (b & SLJIT_MEM) {
 385                 if ((b & REG_MASK) == SLJIT_UNUSED)
 386                         inst_size += sizeof(sljit_sw);
 387                 else if (immb != 0 && !(b & OFFS_REG_MASK)) {
 388                         /* Immediate operand. */
 389                         if (immb <= 127 && immb >= -128)
 390                                 inst_size += sizeof(sljit_s8);
 391                         else
 392                                 inst_size += sizeof(sljit_sw);
 393                 }
 394
 395                 if ((b & REG_MASK) == SLJIT_SP && !(b & OFFS_REG_MASK))
 396                         b |= TO_OFFS_REG(SLJIT_SP);
 397
 398                 if ((b & OFFS_REG_MASK) != SLJIT_UNUSED)
 399                         inst_size += 1; /* SIB byte. */
 400         }
 401
 402         /* Calculate size of a. */
 403         if (a & SLJIT_IMM) {
 404                 if (flags & EX86_BIN_INS) {
 405                         if (imma <= 127 && imma >= -128) {
 406                                 inst_size += 1;
 407                                 flags |= EX86_BYTE_ARG;
 408                         } else
 409                                 inst_size += 4;
 410                 }
 411                 else if (flags & EX86_SHIFT_INS) {
 412                         imma &= 0x1f;
 413                         if (imma != 1) {
 414                                 inst_size ++;
 415                                 flags |= EX86_BYTE_ARG;
 416                         }
 417                 } else if (flags & EX86_BYTE_ARG)
 418                         inst_size++;
 419                 else if (flags & EX86_HALF_ARG)
 420                         inst_size += sizeof(short);
 421                 else
 422                         inst_size += sizeof(sljit_sw);
 423         }
 424         else
 425                 SLJIT_ASSERT(!(flags & EX86_SHIFT_INS) || a == SLJIT_PREF_SHIFT_REG);
 426
 427         inst = (sljit_u8*)ensure_buf(compiler, 1 + inst_size);
 428         PTR_FAIL_IF(!inst);
 429
 430         /* Encoding the byte. */
 431         INC_SIZE(inst_size);
 432         if (flags & EX86_PREF_F2)
 433                 *inst++ = 0xf2;
 434         if (flags & EX86_PREF_F3)
 435                 *inst++ = 0xf3;
 436         if (flags & EX86_PREF_66)
 437                 *inst++ = 0x66;
 438
 439         buf_ptr = inst + size;
 440
 441         /* Encode mod/rm byte. */
 442         if (!(flags & EX86_SHIFT_INS)) {
 443                 if ((flags & EX86_BIN_INS) && (a & SLJIT_IMM))
 444                         *inst = (flags & EX86_BYTE_ARG) ? GROUP_BINARY_83 : GROUP_BINARY_81;
 445
 446                 if (a & SLJIT_IMM)
 447                         *buf_ptr = 0;
 448                 else if (!(flags & EX86_SSE2_OP1))
 449                         *buf_ptr = reg_map[a] << 3;
 450                 else
 451                         *buf_ptr = a << 3;
 452         }
 453         else {
 454                 if (a & SLJIT_IMM) {
 455                         if (imma == 1)
 456                                 *inst = GROUP_SHIFT_1;
 457                         else
 458                                 *inst = GROUP_SHIFT_N;
 459                 } else
 460                         *inst = GROUP_SHIFT_CL;
 461                 *buf_ptr = 0;
 462         }
 463
 464         if (!(b & SLJIT_MEM))
 465                 *buf_ptr++ |= MOD_REG + ((!(flags & EX86_SSE2_OP2)) ? reg_map[b] : b);
 466         else if ((b & REG_MASK) != SLJIT_UNUSED) {
 467                 if ((b & OFFS_REG_MASK) == SLJIT_UNUSED || (b & OFFS_REG_MASK) == TO_OFFS_REG(SLJIT_SP)) {
 468                         if (immb != 0) {
 469                                 if (immb <= 127 && immb >= -128)
 470                                         *buf_ptr |= 0x40;
 471                                 else
 472                                         *buf_ptr |= 0x80;
 473                         }
 474
 475                         if ((b & OFFS_REG_MASK) == SLJIT_UNUSED)
 476                                 *buf_ptr++ |= reg_map[b & REG_MASK];
 477                         else {
 478                                 *buf_ptr++ |= 0x04;
 479                                 *buf_ptr++ = reg_map[b & REG_MASK] | (reg_map[OFFS_REG(b)] << 3);
 480                         }
 481
 482                         if (immb != 0) {
 483                                 if (immb <= 127 && immb >= -128)
 484                                         *buf_ptr++ = immb; /* 8 bit displacement. */
 485                                 else {
 486                                         sljit_unaligned_store_sw(buf_ptr, immb); /* 32 bit displacement. */
 487                                         buf_ptr += sizeof(sljit_sw);
 488                                 }
 489                         }
 490                 }
 491                 else {
 492                         *buf_ptr++ |= 0x04;
 493                         *buf_ptr++ = reg_map[b & REG_MASK] | (reg_map[OFFS_REG(b)] << 3) | (immb << 6);
 494                 }
 495         }
 496         else {
 497                 *buf_ptr++ |= 0x05;
 498                 sljit_unaligned_store_sw(buf_ptr, immb); /* 32 bit displacement. */
 499                 buf_ptr += sizeof(sljit_sw);
 500         }
 501
 502         if (a & SLJIT_IMM) {
 503                 if (flags & EX86_BYTE_ARG)
 504                         *buf_ptr = imma;
 505                 else if (flags & EX86_HALF_ARG)
 506                         sljit_unaligned_store_s16(buf_ptr, imma);
 507                 else if (!(flags & EX86_SHIFT_INS))
 508                         sljit_unaligned_store_sw(buf_ptr, imma);
 509         }
 510
 511         return !(flags & EX86_SHIFT_INS) ? inst : (inst + 1);
 512 }
 513
 514 /* --------------------------------------------------------------------- */
 515 /*  Call / return instructions                                           */
 516 /* --------------------------------------------------------------------- */
 517
 518 #if (defined SLJIT_X86_32_FASTCALL && SLJIT_X86_32_FASTCALL)
 519
 520 static sljit_s32 c_fast_call_get_stack_size(sljit_s32 arg_types, sljit_s32 *word_arg_count_ptr)
 521 {
 522         sljit_s32 stack_size = 0;
 523         sljit_s32 word_arg_count = 0;
 524
 525         arg_types >>= SLJIT_DEF_SHIFT;
 526
 527         while (arg_types) {
 528                 switch (arg_types & SLJIT_DEF_MASK) {
 529                 case SLJIT_ARG_TYPE_F32:
 530                         stack_size += sizeof(sljit_f32);
 531                         break;
 532                 case SLJIT_ARG_TYPE_F64:
 533                         stack_size += sizeof(sljit_f64);
 534                         break;
 535                 default:
 536                         word_arg_count++;
 537                         if (word_arg_count > 2)
 538                                 stack_size += sizeof(sljit_sw);
 539                         break;
 540                 }
 541
 542                 arg_types >>= SLJIT_DEF_SHIFT;
 543         }
 544
 545         if (word_arg_count_ptr)
 546                 *word_arg_count_ptr = word_arg_count;
 547
 548         return stack_size;
 549 }
 550
 551 static sljit_s32 c_fast_call_with_args(struct sljit_compiler *compiler,
 552         sljit_s32 arg_types, sljit_s32 stack_size, sljit_s32 word_arg_count, sljit_s32 swap_args)
 553 {
 554         sljit_u8 *inst;
 555         sljit_s32 float_arg_count;
 556
 557         if (stack_size == sizeof(sljit_sw) && word_arg_count == 3) {
 558                 inst = (sljit_u8*)ensure_buf(compiler, 1 + 1);
 559                 FAIL_IF(!inst);
 560                 INC_SIZE(1);
 561                 PUSH_REG(reg_map[SLJIT_R2]);
 562         }
 563         else if (stack_size > 0) {
 564                 if (word_arg_count >= 4)
 565                         EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_MEM1(SLJIT_SP), compiler->saveds_offset - sizeof(sljit_sw));
 566
 567                 FAIL_IF(emit_non_cum_binary(compiler, BINARY_OPCODE(SUB),
 568                         SLJIT_SP, 0, SLJIT_SP, 0, SLJIT_IMM, stack_size));
 569
 570                 stack_size = 0;
 571                 arg_types >>= SLJIT_DEF_SHIFT;
 572                 word_arg_count = 0;
 573                 float_arg_count = 0;
 574                 while (arg_types) {
 575                         switch (arg_types & SLJIT_DEF_MASK) {
 576                         case SLJIT_ARG_TYPE_F32:
 577                                 float_arg_count++;
 578                                 FAIL_IF(emit_sse2_store(compiler, 1, SLJIT_MEM1(SLJIT_SP), stack_size, float_arg_count));
 579                                 stack_size += sizeof(sljit_f32);
 580                                 break;
 581                         case SLJIT_ARG_TYPE_F64:
 582                                 float_arg_count++;
 583                                 FAIL_IF(emit_sse2_store(compiler, 0, SLJIT_MEM1(SLJIT_SP), stack_size, float_arg_count));
 584                                 stack_size += sizeof(sljit_f64);
 585                                 break;
 586                         default:
 587                                 word_arg_count++;
 588                                 if (word_arg_count == 3) {
 589                                         EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), stack_size, SLJIT_R2, 0);
 590                                         stack_size += sizeof(sljit_sw);
 591                                 }
 592                                 else if (word_arg_count == 4) {
 593                                         EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), stack_size, TMP_REG1, 0);
 594                                         stack_size += sizeof(sljit_sw);
 595                                 }
 596                                 break;
 597                         }
 598
 599                         arg_types >>= SLJIT_DEF_SHIFT;
 600                 }
 601         }
 602
 603         if (word_arg_count > 0) {
 604                 if (swap_args) {
 605                         inst = (sljit_u8*)ensure_buf(compiler, 1 + 1);
 606                         FAIL_IF(!inst);
 607                         INC_SIZE(1);
 608
 609                         *inst++ = XCHG_EAX_r | reg_map[SLJIT_R2];
 610                 }
 611                 else {
 612                         inst = (sljit_u8*)ensure_buf(compiler, 1 + 2);
 613                         FAIL_IF(!inst);
 614                         INC_SIZE(2);
 615
 616                         *inst++ = MOV_r_rm;
 617                         *inst++ = MOD_REG | (reg_map[SLJIT_R2] << 3) | reg_map[SLJIT_R0];
 618                 }
 619         }
 620
 621         return SLJIT_SUCCESS;
 622 }
 623
 624 #endif
 625
 626 static sljit_s32 cdecl_call_get_stack_size(struct sljit_compiler *compiler, sljit_s32 arg_types, sljit_s32 *word_arg_count_ptr)
 627 {
 628         sljit_s32 stack_size = 0;
 629         sljit_s32 word_arg_count = 0;
 630
 631         arg_types >>= SLJIT_DEF_SHIFT;
 632
 633         while (arg_types) {
 634                 switch (arg_types & SLJIT_DEF_MASK) {
 635                 case SLJIT_ARG_TYPE_F32:
 636                         stack_size += sizeof(sljit_f32);
 637                         break;
 638                 case SLJIT_ARG_TYPE_F64:
 639                         stack_size += sizeof(sljit_f64);
 640                         break;
 641                 default:
 642                         word_arg_count++;
 643                         stack_size += sizeof(sljit_sw);
 644                         break;
 645                 }
 646
 647                 arg_types >>= SLJIT_DEF_SHIFT;
 648         }
 649
 650         if (word_arg_count_ptr)
 651                 *word_arg_count_ptr = word_arg_count;
 652
 653         if (stack_size <= compiler->stack_tmp_size)
 654                 return 0;
 655
 656 #if defined(__APPLE__)
 657         return ((stack_size - compiler->stack_tmp_size + 15) & ~15);
 658 #else
 659         return stack_size - compiler->stack_tmp_size;
 660 #endif
 661 }
 662
 663 static sljit_s32 cdecl_call_with_args(struct sljit_compiler *compiler,
 664         sljit_s32 arg_types, sljit_s32 stack_size, sljit_s32 word_arg_count)
 665 {
 666         sljit_s32 float_arg_count = 0;
 667
 668         if (word_arg_count >= 4)
 669                 EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_MEM1(SLJIT_SP), compiler->saveds_offset - sizeof(sljit_sw));
 670
 671         if (stack_size > 0)
 672                 FAIL_IF(emit_non_cum_binary(compiler, BINARY_OPCODE(SUB),
 673                         SLJIT_SP, 0, SLJIT_SP, 0, SLJIT_IMM, stack_size));
 674
 675         stack_size = 0;
 676         word_arg_count = 0;
 677         arg_types >>= SLJIT_DEF_SHIFT;
 678
 679         while (arg_types) {
 680                 switch (arg_types & SLJIT_DEF_MASK) {
 681                 case SLJIT_ARG_TYPE_F32:
 682                         float_arg_count++;
 683                         FAIL_IF(emit_sse2_store(compiler, 1, SLJIT_MEM1(SLJIT_SP), stack_size, float_arg_count));
 684                         stack_size += sizeof(sljit_f32);
 685                         break;
 686                 case SLJIT_ARG_TYPE_F64:
 687                         float_arg_count++;
 688                         FAIL_IF(emit_sse2_store(compiler, 0, SLJIT_MEM1(SLJIT_SP), stack_size, float_arg_count));
 689                         stack_size += sizeof(sljit_f64);
 690                         break;
 691                 default:
 692                         word_arg_count++;
 693                         EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), stack_size, (word_arg_count >= 4) ? TMP_REG1 : word_arg_count, 0);
 694                         stack_size += sizeof(sljit_sw);
 695                         break;
 696                 }
 697
 698                 arg_types >>= SLJIT_DEF_SHIFT;
 699         }
 700
 701         return SLJIT_SUCCESS;
 702 }
 703
 704 static sljit_s32 post_call_with_args(struct sljit_compiler *compiler,
 705         sljit_s32 arg_types, sljit_s32 stack_size)
 706 {
 707         sljit_u8 *inst;
 708         sljit_s32 single;
 709
 710         if (stack_size > 0)
 711                 FAIL_IF(emit_cum_binary(compiler, BINARY_OPCODE(ADD),
 712                         SLJIT_SP, 0, SLJIT_SP, 0, SLJIT_IMM, stack_size));
 713
 714         if ((arg_types & SLJIT_DEF_MASK) < SLJIT_ARG_TYPE_F32)
 715                 return SLJIT_SUCCESS;
 716
 717         single = ((arg_types & SLJIT_DEF_MASK) == SLJIT_ARG_TYPE_F32);
 718
 719         inst = (sljit_u8*)ensure_buf(compiler, 1 + 3);
 720         FAIL_IF(!inst);
 721         INC_SIZE(3);
 722         inst[0] = single ? FSTPS : FSTPD;
 723         inst[1] = (0x03 << 3) | 0x04;
 724         inst[2] = (0x04 << 3) | reg_map[SLJIT_SP];
 725
 726         return emit_sse2_load(compiler, single, SLJIT_FR0, SLJIT_MEM1(SLJIT_SP), 0);
 727 }
 728
 729 SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_call(struct sljit_compiler *compiler, sljit_s32 type,
 730         sljit_s32 arg_types)
 731 {
 732         struct sljit_jump *jump;
 733         sljit_s32 stack_size = 0;
 734         sljit_s32 word_arg_count;
 735
 736         CHECK_ERROR_PTR();
 737         CHECK_PTR(check_sljit_emit_call(compiler, type, arg_types));
 738
 739 #if (defined SLJIT_X86_32_FASTCALL && SLJIT_X86_32_FASTCALL)
 740         if ((type & 0xff) == SLJIT_CALL) {
 741                 stack_size = c_fast_call_get_stack_size(arg_types, &word_arg_count);
 742                 PTR_FAIL_IF(c_fast_call_with_args(compiler, arg_types, stack_size, word_arg_count, 0));
 743
 744 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE) \
 745                 || (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
 746                 compiler->skip_checks = 1;
 747 #endif
 748
 749                 jump = sljit_emit_jump(compiler, type);
 750                 PTR_FAIL_IF(jump == NULL);
 751
 752                 PTR_FAIL_IF(post_call_with_args(compiler, arg_types, 0));
 753                 return jump;
 754         }
 755 #endif
 756
 757         stack_size = cdecl_call_get_stack_size(compiler, arg_types, &word_arg_count);
 758         PTR_FAIL_IF(cdecl_call_with_args(compiler, arg_types, stack_size, word_arg_count));
 759
 760 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE) \
 761                 || (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
 762         compiler->skip_checks = 1;
 763 #endif
 764
 765         jump = sljit_emit_jump(compiler, type);
 766         PTR_FAIL_IF(jump == NULL);
 767
 768         PTR_FAIL_IF(post_call_with_args(compiler, arg_types, stack_size));
 769         return jump;
 770 }
 771
 772 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_icall(struct sljit_compiler *compiler, sljit_s32 type,
 773         sljit_s32 arg_types,
 774         sljit_s32 src, sljit_sw srcw)
 775 {
 776         sljit_s32 stack_size = 0;
 777         sljit_s32 word_arg_count;
 778 #if (defined SLJIT_X86_32_FASTCALL && SLJIT_X86_32_FASTCALL)
 779         sljit_s32 swap_args;
 780 #endif
 781
 782         CHECK_ERROR();
 783         CHECK(check_sljit_emit_icall(compiler, type, arg_types, src, srcw));
 784
 785 #if (defined SLJIT_X86_32_FASTCALL && SLJIT_X86_32_FASTCALL)
 786         SLJIT_ASSERT(reg_map[SLJIT_R0] == 0 && reg_map[SLJIT_R2] == 1 && SLJIT_R0 == 1 && SLJIT_R2 == 3);
 787
 788         if ((type & 0xff) == SLJIT_CALL) {
 789                 stack_size = c_fast_call_get_stack_size(arg_types, &word_arg_count);
 790                 swap_args = 0;
 791
 792                 if (word_arg_count > 0) {
 793                         if ((src & REG_MASK) == SLJIT_R2 || OFFS_REG(src) == SLJIT_R2) {
 794                                 swap_args = 1;
 795                                 if (((src & REG_MASK) | 0x2) == SLJIT_R2)
 796                                         src ^= 0x2;
 797                                 if ((OFFS_REG(src) | 0x2) == SLJIT_R2)
 798                                         src ^= TO_OFFS_REG(0x2);
 799                         }
 800                 }
 801
 802                 FAIL_IF(c_fast_call_with_args(compiler, arg_types, stack_size, word_arg_count, swap_args));
 803
 804                 compiler->saveds_offset += stack_size;
 805                 compiler->locals_offset += stack_size;
 806
 807 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE) \
 808                 || (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
 809                 compiler->skip_checks = 1;
 810 #endif
 811                 FAIL_IF(sljit_emit_ijump(compiler, type, src, srcw));
 812
 813                 compiler->saveds_offset -= stack_size;
 814                 compiler->locals_offset -= stack_size;
 815
 816                 return post_call_with_args(compiler, arg_types, 0);
 817         }
 818 #endif
 819
 820         stack_size = cdecl_call_get_stack_size(compiler, arg_types, &word_arg_count);
 821         FAIL_IF(cdecl_call_with_args(compiler, arg_types, stack_size, word_arg_count));
 822
 823         compiler->saveds_offset += stack_size;
 824         compiler->locals_offset += stack_size;
 825
 826 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE) \
 827                 || (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
 828         compiler->skip_checks = 1;
 829 #endif
 830         FAIL_IF(sljit_emit_ijump(compiler, type, src, srcw));
 831
 832         compiler->saveds_offset -= stack_size;
 833         compiler->locals_offset -= stack_size;
 834
 835         return post_call_with_args(compiler, arg_types, stack_size);
 836 }
 837
 838 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fast_enter(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw)
 839 {
 840         sljit_u8 *inst;
 841
 842         CHECK_ERROR();
 843         CHECK(check_sljit_emit_fast_enter(compiler, dst, dstw));
 844         ADJUST_LOCAL_OFFSET(dst, dstw);
 845
 846         CHECK_EXTRA_REGS(dst, dstw, (void)0);
 847
 848         /* For UNUSED dst. Uncommon, but possible. */
 849         if (dst == SLJIT_UNUSED)
 850                 dst = TMP_REG1;
 851
 852         if (FAST_IS_REG(dst)) {
 853                 /* Unused dest is possible here. */
 854                 inst = (sljit_u8*)ensure_buf(compiler, 1 + 1);
 855                 FAIL_IF(!inst);
 856
 857                 INC_SIZE(1);
 858                 POP_REG(reg_map[dst]);
 859                 return SLJIT_SUCCESS;
 860         }
 861
 862         /* Memory. */
 863         inst = emit_x86_instruction(compiler, 1, 0, 0, dst, dstw);
 864         FAIL_IF(!inst);
 865         *inst++ = POP_rm;
 866         return SLJIT_SUCCESS;
 867 }
 868
 869 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fast_return(struct sljit_compiler *compiler, sljit_s32 src, sljit_sw srcw)
 870 {
 871         sljit_u8 *inst;
 872
 873         CHECK_ERROR();
 874         CHECK(check_sljit_emit_fast_return(compiler, src, srcw));
 875         ADJUST_LOCAL_OFFSET(src, srcw);
 876
 877         CHECK_EXTRA_REGS(src, srcw, (void)0);
 878
 879         if (FAST_IS_REG(src)) {
 880                 inst = (sljit_u8*)ensure_buf(compiler, 1 + 1 + 1);
 881                 FAIL_IF(!inst);
 882
 883                 INC_SIZE(1 + 1);
 884                 PUSH_REG(reg_map[src]);
 885         }
 886         else {
 887                 inst = emit_x86_instruction(compiler, 1, 0, 0, src, srcw);
 888                 FAIL_IF(!inst);
 889                 *inst++ = GROUP_FF;
 890                 *inst |= PUSH_rm;
 891
 892                 inst = (sljit_u8*)ensure_buf(compiler, 1 + 1);
 893                 FAIL_IF(!inst);
 894                 INC_SIZE(1);
 895         }
 896
 897         RET();
 898         return SLJIT_SUCCESS;
 899 }