?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Cmov fix for ARM-32 and ARM-64.
[sljit.git] / sljit_src / sljitNativeARM_32.c
blob745da99f614efff42c287ba3cb8b9c2256fb3c3b
1 /*
2 * Stack-less Just-In-Time compiler
3 *
4 * Copyright Zoltan Herczeg (hzmester@freemail.hu). All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without modification, are
7 * permitted provided that the following conditions are met:
8 *
9 * 1. Redistributions of source code must retain the above copyright notice, this list of
10 * conditions and the following disclaimer.
11 *
12 * 2. Redistributions in binary form must reproduce the above copyright notice, this list
13 * of conditions and the following disclaimer in the documentation and/or other materials
14 * provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) AND CONTRIBUTORS ``AS IS'' AND ANY
17 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
19 * SHALL THE COPYRIGHT HOLDER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
21 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
22 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
23 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
24 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27 SLJIT_API_FUNC_ATTRIBUTE const char* sljit_get_platform_name(void)
28 {
29 #if (defined SLJIT_CONFIG_ARM_V7 && SLJIT_CONFIG_ARM_V7)
30 return "ARMv7" SLJIT_CPUINFO;
31 #elif (defined SLJIT_CONFIG_ARM_V5 && SLJIT_CONFIG_ARM_V5)
32 return "ARMv5" SLJIT_CPUINFO;
33 #else
34 #error "Internal error: Unknown ARM architecture"
35 #endif
36 }
37
38 /* Last register + 1. */
39 #define TMP_REG1 (SLJIT_NUMBER_OF_REGISTERS + 2)
40 #define TMP_REG2 (SLJIT_NUMBER_OF_REGISTERS + 3)
41 #define TMP_PC (SLJIT_NUMBER_OF_REGISTERS + 4)
42
43 #define TMP_FREG1 (0)
44 #define TMP_FREG2 (SLJIT_NUMBER_OF_FLOAT_REGISTERS + 1)
45
46 /* In ARM instruction words.
47 Cache lines are usually 32 byte aligned. */
48 #define CONST_POOL_ALIGNMENT 8
49 #define CONST_POOL_EMPTY 0xffffffff
50
51 #define ALIGN_INSTRUCTION(ptr) \
52 (sljit_uw*)(((sljit_uw)(ptr) + (CONST_POOL_ALIGNMENT * sizeof(sljit_uw)) - 1) & ~((CONST_POOL_ALIGNMENT * sizeof(sljit_uw)) - 1))
53 #define MAX_DIFFERENCE(max_diff) \
54 (((max_diff) / (sljit_s32)sizeof(sljit_uw)) - (CONST_POOL_ALIGNMENT - 1))
55
56 /* See sljit_emit_enter and sljit_emit_op0 if you want to change them. */
57 static const sljit_u8 reg_map[SLJIT_NUMBER_OF_REGISTERS + 5] = {
58 0, 0, 1, 2, 3, 11, 10, 9, 8, 7, 6, 5, 4, 13, 14, 12, 15
59 };
60
61 #define RM(rm) (reg_map[rm])
62 #define RD(rd) (reg_map[rd] << 12)
63 #define RN(rn) (reg_map[rn] << 16)
64
65 /* --------------------------------------------------------------------- */
66 /* Instrucion forms */
67 /* --------------------------------------------------------------------- */
68
69 /* The instruction includes the AL condition.
70 INST_NAME - CONDITIONAL remove this flag. */
71 #define COND_MASK 0xf0000000
72 #define CONDITIONAL 0xe0000000
73 #define PUSH_POOL 0xff000000
74
75 /* DP - Data Processing instruction (use with EMIT_DATA_PROCESS_INS). */
76 #define ADC_DP 0x5
77 #define ADD_DP 0x4
78 #define AND_DP 0x0
79 #define B 0xea000000
80 #define BIC_DP 0xe
81 #define BL 0xeb000000
82 #define BLX 0xe12fff30
83 #define BX 0xe12fff10
84 #define CLZ 0xe16f0f10
85 #define CMN_DP 0xb
86 #define CMP_DP 0xa
87 #define BKPT 0xe1200070
88 #define EOR_DP 0x1
89 #define MOV_DP 0xd
90 #define MUL 0xe0000090
91 #define MVN_DP 0xf
92 #define NOP 0xe1a00000
93 #define ORR_DP 0xc
94 #define PUSH 0xe92d0000
95 #define POP 0xe8bd0000
96 #define RSB_DP 0x3
97 #define RSC_DP 0x7
98 #define SBC_DP 0x6
99 #define SMULL 0xe0c00090
100 #define SUB_DP 0x2
101 #define UMULL 0xe0800090
102 #define VABS_F32 0xeeb00ac0
103 #define VADD_F32 0xee300a00
104 #define VCMP_F32 0xeeb40a40
105 #define VCVT_F32_S32 0xeeb80ac0
106 #define VCVT_F64_F32 0xeeb70ac0
107 #define VCVT_S32_F32 0xeebd0ac0
108 #define VDIV_F32 0xee800a00
109 #define VMOV_F32 0xeeb00a40
110 #define VMOV 0xee000a10
111 #define VMRS 0xeef1fa10
112 #define VMUL_F32 0xee200a00
113 #define VNEG_F32 0xeeb10a40
114 #define VSTR_F32 0xed000a00
115 #define VSUB_F32 0xee300a40
116
117 #if (defined SLJIT_CONFIG_ARM_V7 && SLJIT_CONFIG_ARM_V7)
118 /* Arm v7 specific instructions. */
119 #define MOVW 0xe3000000
120 #define MOVT 0xe3400000
121 #define SXTB 0xe6af0070
122 #define SXTH 0xe6bf0070
123 #define UXTB 0xe6ef0070
124 #define UXTH 0xe6ff0070
125 #endif
126
127 #if (defined SLJIT_CONFIG_ARM_V5 && SLJIT_CONFIG_ARM_V5)
128
129 static sljit_s32 push_cpool(struct sljit_compiler *compiler)
130 {
131 /* Pushing the constant pool into the instruction stream. */
132 sljit_uw* inst;
133 sljit_uw* cpool_ptr;
134 sljit_uw* cpool_end;
135 sljit_s32 i;
136
137 /* The label could point the address after the constant pool. */
138 if (compiler->last_label && compiler->last_label->size == compiler->size)
139 compiler->last_label->size += compiler->cpool_fill + (CONST_POOL_ALIGNMENT - 1) + 1;
140
141 SLJIT_ASSERT(compiler->cpool_fill > 0 && compiler->cpool_fill <= CPOOL_SIZE);
142 inst = (sljit_uw*)ensure_buf(compiler, sizeof(sljit_uw));
143 FAIL_IF(!inst);
144 compiler->size++;
145 *inst = 0xff000000 | compiler->cpool_fill;
146
147 for (i = 0; i < CONST_POOL_ALIGNMENT - 1; i++) {
148 inst = (sljit_uw*)ensure_buf(compiler, sizeof(sljit_uw));
149 FAIL_IF(!inst);
150 compiler->size++;
151 *inst = 0;
152 }
153
154 cpool_ptr = compiler->cpool;
155 cpool_end = cpool_ptr + compiler->cpool_fill;
156 while (cpool_ptr < cpool_end) {
157 inst = (sljit_uw*)ensure_buf(compiler, sizeof(sljit_uw));
158 FAIL_IF(!inst);
159 compiler->size++;
160 *inst = *cpool_ptr++;
161 }
162 compiler->cpool_diff = CONST_POOL_EMPTY;
163 compiler->cpool_fill = 0;
164 return SLJIT_SUCCESS;
165 }
166
167 static sljit_s32 push_inst(struct sljit_compiler *compiler, sljit_uw inst)
168 {
169 sljit_uw* ptr;
170
171 if (SLJIT_UNLIKELY(compiler->cpool_diff != CONST_POOL_EMPTY && compiler->size - compiler->cpool_diff >= MAX_DIFFERENCE(4092)))
172 FAIL_IF(push_cpool(compiler));
173
174 ptr = (sljit_uw*)ensure_buf(compiler, sizeof(sljit_uw));
175 FAIL_IF(!ptr);
176 compiler->size++;
177 *ptr = inst;
178 return SLJIT_SUCCESS;
179 }
180
181 static sljit_s32 push_inst_with_literal(struct sljit_compiler *compiler, sljit_uw inst, sljit_uw literal)
182 {
183 sljit_uw* ptr;
184 sljit_uw cpool_index = CPOOL_SIZE;
185 sljit_uw* cpool_ptr;
186 sljit_uw* cpool_end;
187 sljit_u8* cpool_unique_ptr;
188
189 if (SLJIT_UNLIKELY(compiler->cpool_diff != CONST_POOL_EMPTY && compiler->size - compiler->cpool_diff >= MAX_DIFFERENCE(4092)))
190 FAIL_IF(push_cpool(compiler));
191 else if (compiler->cpool_fill > 0) {
192 cpool_ptr = compiler->cpool;
193 cpool_end = cpool_ptr + compiler->cpool_fill;
194 cpool_unique_ptr = compiler->cpool_unique;
195 do {
196 if ((*cpool_ptr == literal) && !(*cpool_unique_ptr)) {
197 cpool_index = cpool_ptr - compiler->cpool;
198 break;
199 }
200 cpool_ptr++;
201 cpool_unique_ptr++;
202 } while (cpool_ptr < cpool_end);
203 }
204
205 if (cpool_index == CPOOL_SIZE) {
206 /* Must allocate a new entry in the literal pool. */
207 if (compiler->cpool_fill < CPOOL_SIZE) {
208 cpool_index = compiler->cpool_fill;
209 compiler->cpool_fill++;
210 }
211 else {
212 FAIL_IF(push_cpool(compiler));
213 cpool_index = 0;
214 compiler->cpool_fill = 1;
215 }
216 }
217
218 SLJIT_ASSERT((inst & 0xfff) == 0);
219 ptr = (sljit_uw*)ensure_buf(compiler, sizeof(sljit_uw));
220 FAIL_IF(!ptr);
221 compiler->size++;
222 *ptr = inst | cpool_index;
223
224 compiler->cpool[cpool_index] = literal;
225 compiler->cpool_unique[cpool_index] = 0;
226 if (compiler->cpool_diff == CONST_POOL_EMPTY)
227 compiler->cpool_diff = compiler->size;
228 return SLJIT_SUCCESS;
229 }
230
231 static sljit_s32 push_inst_with_unique_literal(struct sljit_compiler *compiler, sljit_uw inst, sljit_uw literal)
232 {
233 sljit_uw* ptr;
234 if (SLJIT_UNLIKELY((compiler->cpool_diff != CONST_POOL_EMPTY && compiler->size - compiler->cpool_diff >= MAX_DIFFERENCE(4092)) || compiler->cpool_fill >= CPOOL_SIZE))
235 FAIL_IF(push_cpool(compiler));
236
237 SLJIT_ASSERT(compiler->cpool_fill < CPOOL_SIZE && (inst & 0xfff) == 0);
238 ptr = (sljit_uw*)ensure_buf(compiler, sizeof(sljit_uw));
239 FAIL_IF(!ptr);
240 compiler->size++;
241 *ptr = inst | compiler->cpool_fill;
242
243 compiler->cpool[compiler->cpool_fill] = literal;
244 compiler->cpool_unique[compiler->cpool_fill] = 1;
245 compiler->cpool_fill++;
246 if (compiler->cpool_diff == CONST_POOL_EMPTY)
247 compiler->cpool_diff = compiler->size;
248 return SLJIT_SUCCESS;
249 }
250
251 static SLJIT_INLINE sljit_s32 prepare_blx(struct sljit_compiler *compiler)
252 {
253 /* Place for at least two instruction (doesn't matter whether the first has a literal). */
254 if (SLJIT_UNLIKELY(compiler->cpool_diff != CONST_POOL_EMPTY && compiler->size - compiler->cpool_diff >= MAX_DIFFERENCE(4088)))
255 return push_cpool(compiler);
256 return SLJIT_SUCCESS;
257 }
258
259 static SLJIT_INLINE sljit_s32 emit_blx(struct sljit_compiler *compiler)
260 {
261 /* Must follow tightly the previous instruction (to be able to convert it to bl instruction). */
262 SLJIT_ASSERT(compiler->cpool_diff == CONST_POOL_EMPTY || compiler->size - compiler->cpool_diff < MAX_DIFFERENCE(4092));
263 return push_inst(compiler, BLX | RM(TMP_REG2));
264 }
265
266 static sljit_uw patch_pc_relative_loads(sljit_uw *last_pc_patch, sljit_uw *code_ptr, sljit_uw* const_pool, sljit_uw cpool_size)
267 {
268 sljit_uw diff;
269 sljit_uw ind;
270 sljit_uw counter = 0;
271 sljit_uw* clear_const_pool = const_pool;
272 sljit_uw* clear_const_pool_end = const_pool + cpool_size;
273
274 SLJIT_ASSERT(const_pool - code_ptr <= CONST_POOL_ALIGNMENT);
275 /* Set unused flag for all literals in the constant pool.
276 I.e.: unused literals can belong to branches, which can be encoded as B or BL.
277 We can "compress" the constant pool by discarding these literals. */
278 while (clear_const_pool < clear_const_pool_end)
279 *clear_const_pool++ = (sljit_uw)(-1);
280
281 while (last_pc_patch < code_ptr) {
282 /* Data transfer instruction with Rn == r15. */
283 if ((*last_pc_patch & 0x0c0f0000) == 0x040f0000) {
284 diff = const_pool - last_pc_patch;
285 ind = (*last_pc_patch) & 0xfff;
286
287 /* Must be a load instruction with immediate offset. */
288 SLJIT_ASSERT(ind < cpool_size && !(*last_pc_patch & (1 << 25)) && (*last_pc_patch & (1 << 20)));
289 if ((sljit_s32)const_pool[ind] < 0) {
290 const_pool[ind] = counter;
291 ind = counter;
292 counter++;
293 }
294 else
295 ind = const_pool[ind];
296
297 SLJIT_ASSERT(diff >= 1);
298 if (diff >= 2 || ind > 0) {
299 diff = (diff + ind - 2) << 2;
300 SLJIT_ASSERT(diff <= 0xfff);
301 *last_pc_patch = (*last_pc_patch & ~0xfff) | diff;
302 }
303 else
304 *last_pc_patch = (*last_pc_patch & ~(0xfff | (1 << 23))) | 0x004;
305 }
306 last_pc_patch++;
307 }
308 return counter;
309 }
310
311 /* In some rare ocasions we may need future patches. The probability is close to 0 in practice. */
312 struct future_patch {
313 struct future_patch* next;
314 sljit_s32 index;
315 sljit_s32 value;
316 };
317
318 static sljit_s32 resolve_const_pool_index(struct sljit_compiler *compiler, struct future_patch **first_patch, sljit_uw cpool_current_index, sljit_uw *cpool_start_address, sljit_uw *buf_ptr)
319 {
320 sljit_s32 value;
321 struct future_patch *curr_patch, *prev_patch;
322
323 SLJIT_UNUSED_ARG(compiler);
324
325 /* Using the values generated by patch_pc_relative_loads. */
326 if (!*first_patch)
327 value = (sljit_s32)cpool_start_address[cpool_current_index];
328 else {
329 curr_patch = *first_patch;
330 prev_patch = NULL;
331 while (1) {
332 if (!curr_patch) {
333 value = (sljit_s32)cpool_start_address[cpool_current_index];
334 break;
335 }
336 if ((sljit_uw)curr_patch->index == cpool_current_index) {
337 value = curr_patch->value;
338 if (prev_patch)
339 prev_patch->next = curr_patch->next;
340 else
341 *first_patch = curr_patch->next;
342 SLJIT_FREE(curr_patch, compiler->allocator_data);
343 break;
344 }
345 prev_patch = curr_patch;
346 curr_patch = curr_patch->next;
347 }
348 }
349
350 if (value >= 0) {
351 if ((sljit_uw)value > cpool_current_index) {
352 curr_patch = (struct future_patch*)SLJIT_MALLOC(sizeof(struct future_patch), compiler->allocator_data);
353 if (!curr_patch) {
354 while (*first_patch) {
355 curr_patch = *first_patch;
356 *first_patch = (*first_patch)->next;
357 SLJIT_FREE(curr_patch, compiler->allocator_data);
358 }
359 return SLJIT_ERR_ALLOC_FAILED;
360 }
361 curr_patch->next = *first_patch;
362 curr_patch->index = value;
363 curr_patch->value = cpool_start_address[value];
364 *first_patch = curr_patch;
365 }
366 cpool_start_address[value] = *buf_ptr;
367 }
368 return SLJIT_SUCCESS;
369 }
370
371 #else
372
373 static sljit_s32 push_inst(struct sljit_compiler *compiler, sljit_uw inst)
374 {
375 sljit_uw* ptr;
376
377 ptr = (sljit_uw*)ensure_buf(compiler, sizeof(sljit_uw));
378 FAIL_IF(!ptr);
379 compiler->size++;
380 *ptr = inst;
381 return SLJIT_SUCCESS;
382 }
383
384 static SLJIT_INLINE sljit_s32 emit_imm(struct sljit_compiler *compiler, sljit_s32 reg, sljit_sw imm)
385 {
386 FAIL_IF(push_inst(compiler, MOVW | RD(reg) | ((imm << 4) & 0xf0000) | (imm & 0xfff)));
387 return push_inst(compiler, MOVT | RD(reg) | ((imm >> 12) & 0xf0000) | ((imm >> 16) & 0xfff));
388 }
389
390 #endif
391
392 static SLJIT_INLINE sljit_s32 detect_jump_type(struct sljit_jump *jump, sljit_uw *code_ptr, sljit_uw *code, sljit_sw executable_offset)
393 {
394 sljit_sw diff;
395
396 if (jump->flags & SLJIT_REWRITABLE_JUMP)
397 return 0;
398
399 #if (defined SLJIT_CONFIG_ARM_V5 && SLJIT_CONFIG_ARM_V5)
400 if (jump->flags & IS_BL)
401 code_ptr--;
402
403 if (jump->flags & JUMP_ADDR)
404 diff = ((sljit_sw)jump->u.target - (sljit_sw)(code_ptr + 2) - executable_offset);
405 else {
406 SLJIT_ASSERT(jump->flags & JUMP_LABEL);
407 diff = ((sljit_sw)(code + jump->u.label->size) - (sljit_sw)(code_ptr + 2));
408 }
409
410 /* Branch to Thumb code has not been optimized yet. */
411 if (diff & 0x3)
412 return 0;
413
414 if (jump->flags & IS_BL) {
415 if (diff <= 0x01ffffff && diff >= -0x02000000) {
416 *code_ptr = (BL - CONDITIONAL) | (*(code_ptr + 1) & COND_MASK);
417 jump->flags |= PATCH_B;
418 return 1;
419 }
420 }
421 else {
422 if (diff <= 0x01ffffff && diff >= -0x02000000) {
423 *code_ptr = (B - CONDITIONAL) | (*code_ptr & COND_MASK);
424 jump->flags |= PATCH_B;
425 }
426 }
427 #else
428 if (jump->flags & JUMP_ADDR)
429 diff = ((sljit_sw)jump->u.target - (sljit_sw)code_ptr - executable_offset);
430 else {
431 SLJIT_ASSERT(jump->flags & JUMP_LABEL);
432 diff = ((sljit_sw)(code + jump->u.label->size) - (sljit_sw)code_ptr);
433 }
434
435 /* Branch to Thumb code has not been optimized yet. */
436 if (diff & 0x3)
437 return 0;
438
439 if (diff <= 0x01ffffff && diff >= -0x02000000) {
440 code_ptr -= 2;
441 *code_ptr = ((jump->flags & IS_BL) ? (BL - CONDITIONAL) : (B - CONDITIONAL)) | (code_ptr[2] & COND_MASK);
442 jump->flags |= PATCH_B;
443 return 1;
444 }
445 #endif
446 return 0;
447 }
448
449 static SLJIT_INLINE void inline_set_jump_addr(sljit_uw jump_ptr, sljit_sw executable_offset, sljit_uw new_addr, sljit_s32 flush_cache)
450 {
451 #if (defined SLJIT_CONFIG_ARM_V5 && SLJIT_CONFIG_ARM_V5)
452 sljit_uw *ptr = (sljit_uw *)jump_ptr;
453 sljit_uw *inst = (sljit_uw *)ptr[0];
454 sljit_uw mov_pc = ptr[1];
455 sljit_s32 bl = (mov_pc & 0x0000f000) != RD(TMP_PC);
456 sljit_sw diff = (sljit_sw)(((sljit_sw)new_addr - (sljit_sw)(inst + 2) - executable_offset) >> 2);
457
458 if (diff <= 0x7fffff && diff >= -0x800000) {
459 /* Turn to branch. */
460 if (!bl) {
461 inst[0] = (mov_pc & COND_MASK) | (B - CONDITIONAL) | (diff & 0xffffff);
462 if (flush_cache) {
463 inst = (sljit_uw *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset);
464 SLJIT_CACHE_FLUSH(inst, inst + 1);
465 }
466 } else {
467 inst[0] = (mov_pc & COND_MASK) | (BL - CONDITIONAL) | (diff & 0xffffff);
468 inst[1] = NOP;
469 if (flush_cache) {
470 inst = (sljit_uw *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset);
471 SLJIT_CACHE_FLUSH(inst, inst + 2);
472 }
473 }
474 } else {
475 /* Get the position of the constant. */
476 if (mov_pc & (1 << 23))
477 ptr = inst + ((mov_pc & 0xfff) >> 2) + 2;
478 else
479 ptr = inst + 1;
480
481 if (*inst != mov_pc) {
482 inst[0] = mov_pc;
483 if (!bl) {
484 if (flush_cache) {
485 inst = (sljit_uw *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset);
486 SLJIT_CACHE_FLUSH(inst, inst + 1);
487 }
488 } else {
489 inst[1] = BLX | RM(TMP_REG1);
490 if (flush_cache) {
491 inst = (sljit_uw *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset);
492 SLJIT_CACHE_FLUSH(inst, inst + 2);
493 }
494 }
495 }
496 *ptr = new_addr;
497 }
498 #else
499 sljit_uw *inst = (sljit_uw*)jump_ptr;
500 SLJIT_ASSERT((inst[0] & 0xfff00000) == MOVW && (inst[1] & 0xfff00000) == MOVT);
501 inst[0] = MOVW | (inst[0] & 0xf000) | ((new_addr << 4) & 0xf0000) | (new_addr & 0xfff);
502 inst[1] = MOVT | (inst[1] & 0xf000) | ((new_addr >> 12) & 0xf0000) | ((new_addr >> 16) & 0xfff);
503 if (flush_cache) {
504 inst = (sljit_uw *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset);
505 SLJIT_CACHE_FLUSH(inst, inst + 2);
506 }
507 #endif
508 }
509
510 static sljit_uw get_imm(sljit_uw imm);
511
512 static SLJIT_INLINE void inline_set_const(sljit_uw addr, sljit_sw executable_offset, sljit_sw new_constant, sljit_s32 flush_cache)
513 {
514 #if (defined SLJIT_CONFIG_ARM_V5 && SLJIT_CONFIG_ARM_V5)
515 sljit_uw *ptr = (sljit_uw*)addr;
516 sljit_uw *inst = (sljit_uw*)ptr[0];
517 sljit_uw ldr_literal = ptr[1];
518 sljit_uw src2;
519
520 src2 = get_imm(new_constant);
521 if (src2) {
522 *inst = 0xe3a00000 | (ldr_literal & 0xf000) | src2;
523 if (flush_cache) {
524 inst = (sljit_uw *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset);
525 SLJIT_CACHE_FLUSH(inst, inst + 1);
526 }
527 return;
528 }
529
530 src2 = get_imm(~new_constant);
531 if (src2) {
532 *inst = 0xe3e00000 | (ldr_literal & 0xf000) | src2;
533 if (flush_cache) {
534 inst = (sljit_uw *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset);
535 SLJIT_CACHE_FLUSH(inst, inst + 1);
536 }
537 return;
538 }
539
540 if (ldr_literal & (1 << 23))
541 ptr = inst + ((ldr_literal & 0xfff) >> 2) + 2;
542 else
543 ptr = inst + 1;
544
545 if (*inst != ldr_literal) {
546 *inst = ldr_literal;
547 if (flush_cache) {
548 inst = (sljit_uw *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset);
549 SLJIT_CACHE_FLUSH(inst, inst + 1);
550 }
551 }
552 *ptr = new_constant;
553 #else
554 sljit_uw *inst = (sljit_uw*)addr;
555 SLJIT_ASSERT((inst[0] & 0xfff00000) == MOVW && (inst[1] & 0xfff00000) == MOVT);
556 inst[0] = MOVW | (inst[0] & 0xf000) | ((new_constant << 4) & 0xf0000) | (new_constant & 0xfff);
557 inst[1] = MOVT | (inst[1] & 0xf000) | ((new_constant >> 12) & 0xf0000) | ((new_constant >> 16) & 0xfff);
558 if (flush_cache) {
559 inst = (sljit_uw *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset);
560 SLJIT_CACHE_FLUSH(inst, inst + 2);
561 }
562 #endif
563 }
564
565 SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compiler)
566 {
567 struct sljit_memory_fragment *buf;
568 sljit_uw *code;
569 sljit_uw *code_ptr;
570 sljit_uw *buf_ptr;
571 sljit_uw *buf_end;
572 sljit_uw size;
573 sljit_uw word_count;
574 sljit_sw executable_offset;
575 sljit_sw jump_addr;
576 #if (defined SLJIT_CONFIG_ARM_V5 && SLJIT_CONFIG_ARM_V5)
577 sljit_uw cpool_size;
578 sljit_uw cpool_skip_alignment;
579 sljit_uw cpool_current_index;
580 sljit_uw *cpool_start_address;
581 sljit_uw *last_pc_patch;
582 struct future_patch *first_patch;
583 #endif
584
585 struct sljit_label *label;
586 struct sljit_jump *jump;
587 struct sljit_const *const_;
588
589 CHECK_ERROR_PTR();
590 CHECK_PTR(check_sljit_generate_code(compiler));
591 reverse_buf(compiler);
592
593 /* Second code generation pass. */
594 #if (defined SLJIT_CONFIG_ARM_V5 && SLJIT_CONFIG_ARM_V5)
595 size = compiler->size + (compiler->patches << 1);
596 if (compiler->cpool_fill > 0)
597 size += compiler->cpool_fill + CONST_POOL_ALIGNMENT - 1;
598 #else
599 size = compiler->size;
600 #endif
601 code = (sljit_uw*)SLJIT_MALLOC_EXEC(size * sizeof(sljit_uw));
602 PTR_FAIL_WITH_EXEC_IF(code);
603 buf = compiler->buf;
604
605 #if (defined SLJIT_CONFIG_ARM_V5 && SLJIT_CONFIG_ARM_V5)
606 cpool_size = 0;
607 cpool_skip_alignment = 0;
608 cpool_current_index = 0;
609 cpool_start_address = NULL;
610 first_patch = NULL;
611 last_pc_patch = code;
612 #endif
613
614 code_ptr = code;
615 word_count = 0;
616 executable_offset = SLJIT_EXEC_OFFSET(code);
617
618 label = compiler->labels;
619 jump = compiler->jumps;
620 const_ = compiler->consts;
621
622 if (label && label->size == 0) {
623 label->addr = (sljit_uw)SLJIT_ADD_EXEC_OFFSET(code, executable_offset);
624 label = label->next;
625 }
626
627 do {
628 buf_ptr = (sljit_uw*)buf->memory;
629 buf_end = buf_ptr + (buf->used_size >> 2);
630 do {
631 word_count++;
632 #if (defined SLJIT_CONFIG_ARM_V5 && SLJIT_CONFIG_ARM_V5)
633 if (cpool_size > 0) {
634 if (cpool_skip_alignment > 0) {
635 buf_ptr++;
636 cpool_skip_alignment--;
637 }
638 else {
639 if (SLJIT_UNLIKELY(resolve_const_pool_index(compiler, &first_patch, cpool_current_index, cpool_start_address, buf_ptr))) {
640 SLJIT_FREE_EXEC(code);
641 compiler->error = SLJIT_ERR_ALLOC_FAILED;
642 return NULL;
643 }
644 buf_ptr++;
645 if (++cpool_current_index >= cpool_size) {
646 SLJIT_ASSERT(!first_patch);
647 cpool_size = 0;
648 if (label && label->size == word_count) {
649 /* Points after the current instruction. */
650 label->addr = (sljit_uw)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset);
651 label->size = code_ptr - code;
652 label = label->next;
653 }
654 }
655 }
656 }
657 else if ((*buf_ptr & 0xff000000) != PUSH_POOL) {
658 #endif
659 *code_ptr = *buf_ptr++;
660 /* These structures are ordered by their address. */
661 SLJIT_ASSERT(!label || label->size >= word_count);
662 SLJIT_ASSERT(!jump || jump->addr >= word_count);
663 SLJIT_ASSERT(!const_ || const_->addr >= word_count);
664 if (jump && jump->addr == word_count) {
665 #if (defined SLJIT_CONFIG_ARM_V5 && SLJIT_CONFIG_ARM_V5)
666 if (detect_jump_type(jump, code_ptr, code, executable_offset))
667 code_ptr--;
668 jump->addr = (sljit_uw)code_ptr;
669 #else
670 jump->addr = (sljit_uw)(code_ptr - 2);
671 if (detect_jump_type(jump, code_ptr, code, executable_offset))
672 code_ptr -= 2;
673 #endif
674 jump = jump->next;
675 }
676 if (label && label->size == word_count) {
677 /* code_ptr can be affected above. */
678 label->addr = (sljit_uw)SLJIT_ADD_EXEC_OFFSET(code_ptr + 1, executable_offset);
679 label->size = (code_ptr + 1) - code;
680 label = label->next;
681 }
682 if (const_ && const_->addr == word_count) {
683 #if (defined SLJIT_CONFIG_ARM_V5 && SLJIT_CONFIG_ARM_V5)
684 const_->addr = (sljit_uw)code_ptr;
685 #else
686 const_->addr = (sljit_uw)(code_ptr - 1);
687 #endif
688 const_ = const_->next;
689 }
690 code_ptr++;
691 #if (defined SLJIT_CONFIG_ARM_V5 && SLJIT_CONFIG_ARM_V5)
692 }
693 else {
694 /* Fortunately, no need to shift. */
695 cpool_size = *buf_ptr++ & ~PUSH_POOL;
696 SLJIT_ASSERT(cpool_size > 0);
697 cpool_start_address = ALIGN_INSTRUCTION(code_ptr + 1);
698 cpool_current_index = patch_pc_relative_loads(last_pc_patch, code_ptr, cpool_start_address, cpool_size);
699 if (cpool_current_index > 0) {
700 /* Unconditional branch. */
701 *code_ptr = B | (((cpool_start_address - code_ptr) + cpool_current_index - 2) & ~PUSH_POOL);
702 code_ptr = cpool_start_address + cpool_current_index;
703 }
704 cpool_skip_alignment = CONST_POOL_ALIGNMENT - 1;
705 cpool_current_index = 0;
706 last_pc_patch = code_ptr;
707 }
708 #endif
709 } while (buf_ptr < buf_end);
710 buf = buf->next;
711 } while (buf);
712
713 SLJIT_ASSERT(!label);
714 SLJIT_ASSERT(!jump);
715 SLJIT_ASSERT(!const_);
716
717 #if (defined SLJIT_CONFIG_ARM_V5 && SLJIT_CONFIG_ARM_V5)
718 SLJIT_ASSERT(cpool_size == 0);
719 if (compiler->cpool_fill > 0) {
720 cpool_start_address = ALIGN_INSTRUCTION(code_ptr);
721 cpool_current_index = patch_pc_relative_loads(last_pc_patch, code_ptr, cpool_start_address, compiler->cpool_fill);
722 if (cpool_current_index > 0)
723 code_ptr = cpool_start_address + cpool_current_index;
724
725 buf_ptr = compiler->cpool;
726 buf_end = buf_ptr + compiler->cpool_fill;
727 cpool_current_index = 0;
728 while (buf_ptr < buf_end) {
729 if (SLJIT_UNLIKELY(resolve_const_pool_index(compiler, &first_patch, cpool_current_index, cpool_start_address, buf_ptr))) {
730 SLJIT_FREE_EXEC(code);
731 compiler->error = SLJIT_ERR_ALLOC_FAILED;
732 return NULL;
733 }
734 buf_ptr++;
735 cpool_current_index++;
736 }
737 SLJIT_ASSERT(!first_patch);
738 }
739 #endif
740
741 jump = compiler->jumps;
742 while (jump) {
743 buf_ptr = (sljit_uw *)jump->addr;
744
745 if (jump->flags & PATCH_B) {
746 jump_addr = (sljit_sw)SLJIT_ADD_EXEC_OFFSET(buf_ptr + 2, executable_offset);
747 if (!(jump->flags & JUMP_ADDR)) {
748 SLJIT_ASSERT(jump->flags & JUMP_LABEL);
749 SLJIT_ASSERT(((sljit_sw)jump->u.label->addr - jump_addr) <= 0x01ffffff && ((sljit_sw)jump->u.label->addr - jump_addr) >= -0x02000000);
750 *buf_ptr |= (((sljit_sw)jump->u.label->addr - jump_addr) >> 2) & 0x00ffffff;
751 }
752 else {
753 SLJIT_ASSERT(((sljit_sw)jump->u.target - jump_addr) <= 0x01ffffff && ((sljit_sw)jump->u.target - jump_addr) >= -0x02000000);
754 *buf_ptr |= (((sljit_sw)jump->u.target - jump_addr) >> 2) & 0x00ffffff;
755 }
756 }
757 else if (jump->flags & SLJIT_REWRITABLE_JUMP) {
758 #if (defined SLJIT_CONFIG_ARM_V5 && SLJIT_CONFIG_ARM_V5)
759 jump->addr = (sljit_uw)code_ptr;
760 code_ptr[0] = (sljit_uw)buf_ptr;
761 code_ptr[1] = *buf_ptr;
762 inline_set_jump_addr((sljit_uw)code_ptr, executable_offset, (jump->flags & JUMP_LABEL) ? jump->u.label->addr : jump->u.target, 0);
763 code_ptr += 2;
764 #else
765 inline_set_jump_addr((sljit_uw)buf_ptr, executable_offset, (jump->flags & JUMP_LABEL) ? jump->u.label->addr : jump->u.target, 0);
766 #endif
767 }
768 else {
769 #if (defined SLJIT_CONFIG_ARM_V5 && SLJIT_CONFIG_ARM_V5)
770 if (jump->flags & IS_BL)
771 buf_ptr--;
772 if (*buf_ptr & (1 << 23))
773 buf_ptr += ((*buf_ptr & 0xfff) >> 2) + 2;
774 else
775 buf_ptr += 1;
776 *buf_ptr = (jump->flags & JUMP_LABEL) ? jump->u.label->addr : jump->u.target;
777 #else
778 inline_set_jump_addr((sljit_uw)buf_ptr, executable_offset, (jump->flags & JUMP_LABEL) ? jump->u.label->addr : jump->u.target, 0);
779 #endif
780 }
781 jump = jump->next;
782 }
783
784 #if (defined SLJIT_CONFIG_ARM_V5 && SLJIT_CONFIG_ARM_V5)
785 const_ = compiler->consts;
786 while (const_) {
787 buf_ptr = (sljit_uw*)const_->addr;
788 const_->addr = (sljit_uw)code_ptr;
789
790 code_ptr[0] = (sljit_uw)buf_ptr;
791 code_ptr[1] = *buf_ptr;
792 if (*buf_ptr & (1 << 23))
793 buf_ptr += ((*buf_ptr & 0xfff) >> 2) + 2;
794 else
795 buf_ptr += 1;
796 /* Set the value again (can be a simple constant). */
797 inline_set_const((sljit_uw)code_ptr, executable_offset, *buf_ptr, 0);
798 code_ptr += 2;
799
800 const_ = const_->next;
801 }
802 #endif
803
804 SLJIT_ASSERT(code_ptr - code <= (sljit_s32)size);
805
806 compiler->error = SLJIT_ERR_COMPILED;
807 compiler->executable_offset = executable_offset;
808 compiler->executable_size = (code_ptr - code) * sizeof(sljit_uw);
809
810 code = (sljit_uw *)SLJIT_ADD_EXEC_OFFSET(code, executable_offset);
811 code_ptr = (sljit_uw *)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset);
812
813 SLJIT_CACHE_FLUSH(code, code_ptr);
814 return code;
815 }
816
817 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_has_cpu_feature(sljit_s32 feature_type)
818 {
819 switch (feature_type) {
820 case SLJIT_HAS_FPU:
821 #ifdef SLJIT_IS_FPU_AVAILABLE
822 return SLJIT_IS_FPU_AVAILABLE;
823 #else
824 /* Available by default. */
825 return 1;
826 #endif
827
828 case SLJIT_HAS_PRE_UPDATE:
829 case SLJIT_HAS_CLZ:
830 case SLJIT_HAS_CMOV:
831 return 1;
832
833 default:
834 return 0;
835 }
836 }
837
838 /* --------------------------------------------------------------------- */
839 /* Entry, exit */
840 /* --------------------------------------------------------------------- */
841
842 /* Creates an index in data_transfer_insts array. */
843 #define WORD_DATA 0x00
844 #define BYTE_DATA 0x01
845 #define HALF_DATA 0x02
846 #define PRELOAD_DATA 0x03
847 #define SIGNED_DATA 0x04
848 #define LOAD_DATA 0x08
849
850 /* emit_op inp_flags.
851 WRITE_BACK must be the first, since it is a flag. */
852 #define WRITE_BACK 0x10
853 #define ALLOW_IMM 0x20
854 #define ALLOW_INV_IMM 0x40
855 #define ALLOW_ANY_IMM (ALLOW_IMM | ALLOW_INV_IMM)
856
857 /* s/l - store/load (1 bit)
858 u/s - signed/unsigned (1 bit)
859 w/b/h/N - word/byte/half/NOT allowed (2 bit)
860 Storing signed and unsigned values are the same operations. */
861
862 static const sljit_uw data_transfer_insts[16] = {
863 /* s u w */ 0xe5000000 /* str */,
864 /* s u b */ 0xe5400000 /* strb */,
865 /* s u h */ 0xe10000b0 /* strh */,
866 /* s u N */ 0x00000000 /* not allowed */,
867 /* s s w */ 0xe5000000 /* str */,
868 /* s s b */ 0xe5400000 /* strb */,
869 /* s s h */ 0xe10000b0 /* strh */,
870 /* s s N */ 0x00000000 /* not allowed */,
871
872 /* l u w */ 0xe5100000 /* ldr */,
873 /* l u b */ 0xe5500000 /* ldrb */,
874 /* l u h */ 0xe11000b0 /* ldrh */,
875 /* l u p */ 0xf5500000 /* preload data */,
876 /* l s w */ 0xe5100000 /* ldr */,
877 /* l s b */ 0xe11000d0 /* ldrsb */,
878 /* l s h */ 0xe11000f0 /* ldrsh */,
879 /* l s N */ 0x00000000 /* not allowed */,
880 };
881
882 #define EMIT_DATA_TRANSFER(type, add, wb, target_reg, base_reg, arg) \
883 (data_transfer_insts[(type) & 0xf] | ((add) << 23) | ((wb) << (21 - 4)) | RD(target_reg) | RN(base_reg) | (arg))
884
885 /* Normal ldr/str instruction.
886 Type2: ldrsb, ldrh, ldrsh */
887 #define IS_TYPE1_TRANSFER(type) \
888 (data_transfer_insts[(type) & 0xf] & 0x04000000)
889 #define TYPE2_TRANSFER_IMM(imm) \
890 (((imm) & 0xf) | (((imm) & 0xf0) << 4) | (1 << 22))
891
892 /* Condition: AL. */
893 #define EMIT_DATA_PROCESS_INS(opcode, set_flags, dst, src1, src2) \
894 (0xe0000000 | ((opcode) << 21) | (set_flags) | RD(dst) | RN(src1) | (src2))
895
896 static sljit_s32 emit_op(struct sljit_compiler *compiler, sljit_s32 op, sljit_s32 inp_flags,
897 sljit_s32 dst, sljit_sw dstw,
898 sljit_s32 src1, sljit_sw src1w,
899 sljit_s32 src2, sljit_sw src2w);
900
901 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_enter(struct sljit_compiler *compiler,
902 sljit_s32 options, sljit_s32 args, sljit_s32 scratches, sljit_s32 saveds,
903 sljit_s32 fscratches, sljit_s32 fsaveds, sljit_s32 local_size)
904 {
905 sljit_s32 size, i, tmp;
906 sljit_uw push;
907
908 CHECK_ERROR();
909 CHECK(check_sljit_emit_enter(compiler, options, args, scratches, saveds, fscratches, fsaveds, local_size));
910 set_emit_enter(compiler, options, args, scratches, saveds, fscratches, fsaveds, local_size);
911
912 /* Push saved registers, temporary registers
913 stmdb sp!, {..., lr} */
914 push = PUSH | (1 << 14);
915
916 tmp = saveds < SLJIT_NUMBER_OF_SAVED_REGISTERS ? (SLJIT_S0 + 1 - saveds) : SLJIT_FIRST_SAVED_REG;
917 for (i = SLJIT_S0; i >= tmp; i--)
918 push |= 1 << reg_map[i];
919
920 for (i = scratches; i >= SLJIT_FIRST_SAVED_REG; i--)
921 push |= 1 << reg_map[i];
922
923 FAIL_IF(push_inst(compiler, push));
924
925 /* Stack must be aligned to 8 bytes: */
926 size = GET_SAVED_REGISTERS_SIZE(scratches, saveds, 1);
927 local_size = ((size + local_size + 7) & ~7) - size;
928 compiler->local_size = local_size;
929 if (local_size > 0)
930 FAIL_IF(emit_op(compiler, SLJIT_SUB, ALLOW_IMM, SLJIT_SP, 0, SLJIT_SP, 0, SLJIT_IMM, local_size));
931
932 if (args >= 1)
933 FAIL_IF(push_inst(compiler, EMIT_DATA_PROCESS_INS(MOV_DP, 0, SLJIT_S0, SLJIT_UNUSED, RM(SLJIT_R0))));
934 if (args >= 2)
935 FAIL_IF(push_inst(compiler, EMIT_DATA_PROCESS_INS(MOV_DP, 0, SLJIT_S1, SLJIT_UNUSED, RM(SLJIT_R1))));
936 if (args >= 3)
937 FAIL_IF(push_inst(compiler, EMIT_DATA_PROCESS_INS(MOV_DP, 0, SLJIT_S2, SLJIT_UNUSED, RM(SLJIT_R2))));
938
939 return SLJIT_SUCCESS;
940 }
941
942 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_set_context(struct sljit_compiler *compiler,
943 sljit_s32 options, sljit_s32 args, sljit_s32 scratches, sljit_s32 saveds,
944 sljit_s32 fscratches, sljit_s32 fsaveds, sljit_s32 local_size)
945 {
946 sljit_s32 size;
947
948 CHECK_ERROR();
949 CHECK(check_sljit_set_context(compiler, options, args, scratches, saveds, fscratches, fsaveds, local_size));
950 set_set_context(compiler, options, args, scratches, saveds, fscratches, fsaveds, local_size);
951
952 size = GET_SAVED_REGISTERS_SIZE(scratches, saveds, 1);
953 compiler->local_size = ((size + local_size + 7) & ~7) - size;
954 return SLJIT_SUCCESS;
955 }
956
957 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_return(struct sljit_compiler *compiler, sljit_s32 op, sljit_s32 src, sljit_sw srcw)
958 {
959 sljit_s32 i, tmp;
960 sljit_uw pop;
961
962 CHECK_ERROR();
963 CHECK(check_sljit_emit_return(compiler, op, src, srcw));
964
965 FAIL_IF(emit_mov_before_return(compiler, op, src, srcw));
966
967 if (compiler->local_size > 0)
968 FAIL_IF(emit_op(compiler, SLJIT_ADD, ALLOW_IMM, SLJIT_SP, 0, SLJIT_SP, 0, SLJIT_IMM, compiler->local_size));
969
970 /* Push saved registers, temporary registers
971 ldmia sp!, {..., pc} */
972 pop = POP | (1 << 15);
973
974 tmp = compiler->saveds < SLJIT_NUMBER_OF_SAVED_REGISTERS ? (SLJIT_S0 + 1 - compiler->saveds) : SLJIT_FIRST_SAVED_REG;
975 for (i = SLJIT_S0; i >= tmp; i--)
976 pop |= 1 << reg_map[i];
977
978 for (i = compiler->scratches; i >= SLJIT_FIRST_SAVED_REG; i--)
979 pop |= 1 << reg_map[i];
980
981 return push_inst(compiler, pop);
982 }
983
984 /* --------------------------------------------------------------------- */
985 /* Operators */
986 /* --------------------------------------------------------------------- */
987
988 /* flags: */
989 /* Arguments are swapped. */
990 #define ARGS_SWAPPED 0x01
991 /* Inverted immediate. */
992 #define INV_IMM 0x02
993 /* Source and destination is register. */
994 #define MOVE_REG_CONV 0x04
995 /* Unused return value. */
996 #define UNUSED_RETURN 0x08
997 /* SET_FLAGS must be (1 << 20) as it is also the value of S bit (can be used for optimization). */
998 #define SET_FLAGS (1 << 20)
999 /* dst: reg
1000 src1: reg
1001 src2: reg or imm (if allowed)
1002 SRC2_IMM must be (1 << 25) as it is also the value of I bit (can be used for optimization). */
1003 #define SRC2_IMM (1 << 25)
1004
1005 #define EMIT_SHIFT_INS_AND_RETURN(opcode) \
1006 SLJIT_ASSERT(!(flags & INV_IMM) && !(src2 & SRC2_IMM)); \
1007 if (compiler->shift_imm != 0x20) { \
1008 SLJIT_ASSERT(src1 == TMP_REG1); \
1009 SLJIT_ASSERT(!(flags & ARGS_SWAPPED)); \
1010 \
1011 if (compiler->shift_imm != 0) \
1012 return push_inst(compiler, EMIT_DATA_PROCESS_INS(MOV_DP, flags & SET_FLAGS, \
1013 dst, SLJIT_UNUSED, (compiler->shift_imm << 7) | (opcode << 5) | RM(src2))); \
1014 return push_inst(compiler, EMIT_DATA_PROCESS_INS(MOV_DP, flags & SET_FLAGS, dst, SLJIT_UNUSED, RM(src2))); \
1015 } \
1016 return push_inst(compiler, EMIT_DATA_PROCESS_INS(MOV_DP, flags & SET_FLAGS, \
1017 dst, SLJIT_UNUSED, (reg_map[(flags & ARGS_SWAPPED) ? src1 : src2] << 8) | (opcode << 5) | 0x10 | RM((flags & ARGS_SWAPPED) ? src2 : src1)));
1018
1019 static SLJIT_INLINE sljit_s32 emit_single_op(struct sljit_compiler *compiler, sljit_s32 op, sljit_s32 flags,
1020 sljit_s32 dst, sljit_s32 src1, sljit_s32 src2)
1021 {
1022 switch (GET_OPCODE(op)) {
1023 case SLJIT_MOV:
1024 SLJIT_ASSERT(src1 == TMP_REG1 && !(flags & ARGS_SWAPPED));
1025 if (dst != src2) {
1026 if (src2 & SRC2_IMM) {
1027 return push_inst(compiler, EMIT_DATA_PROCESS_INS((flags & INV_IMM) ? MVN_DP : MOV_DP, 0,
1028 dst, SLJIT_UNUSED, src2));
1029 }
1030 return push_inst(compiler, EMIT_DATA_PROCESS_INS(MOV_DP, 0, dst, SLJIT_UNUSED, RM(src2)));
1031 }
1032 return SLJIT_SUCCESS;
1033
1034 case SLJIT_MOV_U8:
1035 case SLJIT_MOV_S8:
1036 SLJIT_ASSERT(src1 == TMP_REG1 && !(flags & ARGS_SWAPPED));
1037 if (flags & MOVE_REG_CONV) {
1038 #if (defined SLJIT_CONFIG_ARM_V5 && SLJIT_CONFIG_ARM_V5)
1039 if (op == SLJIT_MOV_U8)
1040 return push_inst(compiler, EMIT_DATA_PROCESS_INS(AND_DP, 0, dst, src2, SRC2_IMM | 0xff));
1041 FAIL_IF(push_inst(compiler, EMIT_DATA_PROCESS_INS(MOV_DP, 0, dst, SLJIT_UNUSED, (24 << 7) | RM(src2))));
1042 return push_inst(compiler, EMIT_DATA_PROCESS_INS(MOV_DP, 0, dst, SLJIT_UNUSED, (24 << 7) | (op == SLJIT_MOV_U8 ? 0x20 : 0x40) | RM(dst)));
1043 #else
1044 return push_inst(compiler, (op == SLJIT_MOV_U8 ? UXTB : SXTB) | RD(dst) | RM(src2));
1045 #endif
1046 }
1047 else if (dst != src2) {
1048 SLJIT_ASSERT(src2 & SRC2_IMM);
1049 return push_inst(compiler, EMIT_DATA_PROCESS_INS((flags & INV_IMM) ? MVN_DP : MOV_DP, 0,
1050 dst, SLJIT_UNUSED, src2));
1051 }
1052 return SLJIT_SUCCESS;
1053
1054 case SLJIT_MOV_U16:
1055 case SLJIT_MOV_S16:
1056 SLJIT_ASSERT(src1 == TMP_REG1 && !(flags & ARGS_SWAPPED));
1057 if (flags & MOVE_REG_CONV) {
1058 #if (defined SLJIT_CONFIG_ARM_V5 && SLJIT_CONFIG_ARM_V5)
1059 FAIL_IF(push_inst(compiler, EMIT_DATA_PROCESS_INS(MOV_DP, 0, dst, SLJIT_UNUSED, (16 << 7) | RM(src2))));
1060 return push_inst(compiler, EMIT_DATA_PROCESS_INS(MOV_DP, 0, dst, SLJIT_UNUSED, (16 << 7) | (op == SLJIT_MOV_U16 ? 0x20 : 0x40) | RM(dst)));
1061 #else
1062 return push_inst(compiler, (op == SLJIT_MOV_U16 ? UXTH : SXTH) | RD(dst) | RM(src2));
1063 #endif
1064 }
1065 else if (dst != src2) {
1066 SLJIT_ASSERT(src2 & SRC2_IMM);
1067 return push_inst(compiler, EMIT_DATA_PROCESS_INS((flags & INV_IMM) ? MVN_DP : MOV_DP, 0,
1068 dst, SLJIT_UNUSED, src2));
1069 }
1070 return SLJIT_SUCCESS;
1071
1072 case SLJIT_NOT:
1073 if (src2 & SRC2_IMM) {
1074 return push_inst(compiler, EMIT_DATA_PROCESS_INS((flags & INV_IMM) ? MOV_DP : MVN_DP, flags & SET_FLAGS,
1075 dst, SLJIT_UNUSED, src2));
1076 }
1077 return push_inst(compiler, EMIT_DATA_PROCESS_INS(MVN_DP, flags & SET_FLAGS, dst, SLJIT_UNUSED, RM(src2)));
1078
1079 case SLJIT_CLZ:
1080 SLJIT_ASSERT(!(flags & INV_IMM));
1081 SLJIT_ASSERT(!(src2 & SRC2_IMM));
1082 FAIL_IF(push_inst(compiler, CLZ | RD(dst) | RM(src2)));
1083 return SLJIT_SUCCESS;
1084
1085 case SLJIT_ADD:
1086 SLJIT_ASSERT(!(flags & INV_IMM));
1087 if ((flags & (UNUSED_RETURN | SET_FLAGS)) == (UNUSED_RETURN | SET_FLAGS) && !(flags & ARGS_SWAPPED))
1088 return push_inst(compiler, EMIT_DATA_PROCESS_INS(CMN_DP, SET_FLAGS,
1089 SLJIT_UNUSED, src1, (src2 & SRC2_IMM) ? src2 : RM(src2)));
1090 return push_inst(compiler, EMIT_DATA_PROCESS_INS(ADD_DP, flags & SET_FLAGS,
1091 dst, src1, (src2 & SRC2_IMM) ? src2 : RM(src2)));
1092
1093 case SLJIT_ADDC:
1094 SLJIT_ASSERT(!(flags & INV_IMM));
1095 return push_inst(compiler, EMIT_DATA_PROCESS_INS(ADC_DP, flags & SET_FLAGS,
1096 dst, src1, (src2 & SRC2_IMM) ? src2 : RM(src2)));
1097
1098 case SLJIT_SUB:
1099 SLJIT_ASSERT(!(flags & INV_IMM));
1100 if ((flags & (UNUSED_RETURN | SET_FLAGS)) == (UNUSED_RETURN | SET_FLAGS) && !(flags & ARGS_SWAPPED))
1101 return push_inst(compiler, EMIT_DATA_PROCESS_INS(CMP_DP, SET_FLAGS,
1102 SLJIT_UNUSED, src1, (src2 & SRC2_IMM) ? src2 : RM(src2)));
1103 return push_inst(compiler, EMIT_DATA_PROCESS_INS(!(flags & ARGS_SWAPPED) ? SUB_DP : RSB_DP, flags & SET_FLAGS,
1104 dst, src1, (src2 & SRC2_IMM) ? src2 : RM(src2)));
1105
1106 case SLJIT_SUBC:
1107 SLJIT_ASSERT(!(flags & INV_IMM));
1108 return push_inst(compiler, EMIT_DATA_PROCESS_INS(!(flags & ARGS_SWAPPED) ? SBC_DP : RSC_DP, flags & SET_FLAGS,
1109 dst, src1, (src2 & SRC2_IMM) ? src2 : RM(src2)));
1110
1111 case SLJIT_MUL:
1112 SLJIT_ASSERT(!(flags & INV_IMM));
1113 SLJIT_ASSERT(!(src2 & SRC2_IMM));
1114
1115 if (!HAS_FLAGS(op))
1116 return push_inst(compiler, MUL | (reg_map[dst] << 16) | (reg_map[src2] << 8) | reg_map[src1]);
1117
1118 FAIL_IF(push_inst(compiler, SMULL | (reg_map[TMP_REG1] << 16) | (reg_map[dst] << 12) | (reg_map[src2] << 8) | reg_map[src1]));
1119
1120 /* cmp TMP_REG1, dst asr #31. */
1121 return push_inst(compiler, EMIT_DATA_PROCESS_INS(CMP_DP, SET_FLAGS, SLJIT_UNUSED, TMP_REG1, RM(dst) | 0xfc0));
1122
1123 case SLJIT_AND:
1124 return push_inst(compiler, EMIT_DATA_PROCESS_INS(!(flags & INV_IMM) ? AND_DP : BIC_DP, flags & SET_FLAGS,
1125 dst, src1, (src2 & SRC2_IMM) ? src2 : RM(src2)));
1126
1127 case SLJIT_OR:
1128 SLJIT_ASSERT(!(flags & INV_IMM));
1129 return push_inst(compiler, EMIT_DATA_PROCESS_INS(ORR_DP, flags & SET_FLAGS, dst, src1, (src2 & SRC2_IMM) ? src2 : RM(src2)));
1130
1131 case SLJIT_XOR:
1132 SLJIT_ASSERT(!(flags & INV_IMM));
1133 return push_inst(compiler, EMIT_DATA_PROCESS_INS(EOR_DP, flags & SET_FLAGS, dst, src1, (src2 & SRC2_IMM) ? src2 : RM(src2)));
1134
1135 case SLJIT_SHL:
1136 EMIT_SHIFT_INS_AND_RETURN(0);
1137
1138 case SLJIT_LSHR:
1139 EMIT_SHIFT_INS_AND_RETURN(1);
1140
1141 case SLJIT_ASHR:
1142 EMIT_SHIFT_INS_AND_RETURN(2);
1143 }
1144
1145 SLJIT_UNREACHABLE();
1146 return SLJIT_SUCCESS;
1147 }
1148
1149 #undef EMIT_SHIFT_INS_AND_RETURN
1150
1151 /* Tests whether the immediate can be stored in the 12 bit imm field.
1152 Returns with 0 if not possible. */
1153 static sljit_uw get_imm(sljit_uw imm)
1154 {
1155 sljit_s32 rol;
1156
1157 if (imm <= 0xff)
1158 return SRC2_IMM | imm;
1159
1160 if (!(imm & 0xff000000)) {
1161 imm <<= 8;
1162 rol = 8;
1163 }
1164 else {
1165 imm = (imm << 24) | (imm >> 8);
1166 rol = 0;
1167 }
1168
1169 if (!(imm & 0xff000000)) {
1170 imm <<= 8;
1171 rol += 4;
1172 }
1173
1174 if (!(imm & 0xf0000000)) {
1175 imm <<= 4;
1176 rol += 2;
1177 }
1178
1179 if (!(imm & 0xc0000000)) {
1180 imm <<= 2;
1181 rol += 1;
1182 }
1183
1184 if (!(imm & 0x00ffffff))
1185 return SRC2_IMM | (imm >> 24) | (rol << 8);
1186 else
1187 return 0;
1188 }
1189
1190 #if (defined SLJIT_CONFIG_ARM_V5 && SLJIT_CONFIG_ARM_V5)
1191 static sljit_s32 generate_int(struct sljit_compiler *compiler, sljit_s32 reg, sljit_uw imm, sljit_s32 positive)
1192 {
1193 sljit_uw mask;
1194 sljit_uw imm1;
1195 sljit_uw imm2;
1196 sljit_s32 rol;
1197
1198 /* Step1: Search a zero byte (8 continous zero bit). */
1199 mask = 0xff000000;
1200 rol = 8;
1201 while(1) {
1202 if (!(imm & mask)) {
1203 /* Rol imm by rol. */
1204 imm = (imm << rol) | (imm >> (32 - rol));
1205 /* Calculate arm rol. */
1206 rol = 4 + (rol >> 1);
1207 break;
1208 }
1209 rol += 2;
1210 mask >>= 2;
1211 if (mask & 0x3) {
1212 /* rol by 8. */
1213 imm = (imm << 8) | (imm >> 24);
1214 mask = 0xff00;
1215 rol = 24;
1216 while (1) {
1217 if (!(imm & mask)) {
1218 /* Rol imm by rol. */
1219 imm = (imm << rol) | (imm >> (32 - rol));
1220 /* Calculate arm rol. */
1221 rol = (rol >> 1) - 8;
1222 break;
1223 }
1224 rol += 2;
1225 mask >>= 2;
1226 if (mask & 0x3)
1227 return 0;
1228 }
1229 break;
1230 }
1231 }
1232
1233 /* The low 8 bit must be zero. */
1234 SLJIT_ASSERT(!(imm & 0xff));
1235
1236 if (!(imm & 0xff000000)) {
1237 imm1 = SRC2_IMM | ((imm >> 16) & 0xff) | (((rol + 4) & 0xf) << 8);
1238 imm2 = SRC2_IMM | ((imm >> 8) & 0xff) | (((rol + 8) & 0xf) << 8);
1239 }
1240 else if (imm & 0xc0000000) {
1241 imm1 = SRC2_IMM | ((imm >> 24) & 0xff) | ((rol & 0xf) << 8);
1242 imm <<= 8;
1243 rol += 4;
1244
1245 if (!(imm & 0xff000000)) {
1246 imm <<= 8;
1247 rol += 4;
1248 }
1249
1250 if (!(imm & 0xf0000000)) {
1251 imm <<= 4;
1252 rol += 2;
1253 }
1254
1255 if (!(imm & 0xc0000000)) {
1256 imm <<= 2;
1257 rol += 1;
1258 }
1259
1260 if (!(imm & 0x00ffffff))
1261 imm2 = SRC2_IMM | (imm >> 24) | ((rol & 0xf) << 8);
1262 else
1263 return 0;
1264 }
1265 else {
1266 if (!(imm & 0xf0000000)) {
1267 imm <<= 4;
1268 rol += 2;
1269 }
1270
1271 if (!(imm & 0xc0000000)) {
1272 imm <<= 2;
1273 rol += 1;
1274 }
1275
1276 imm1 = SRC2_IMM | ((imm >> 24) & 0xff) | ((rol & 0xf) << 8);
1277 imm <<= 8;
1278 rol += 4;
1279
1280 if (!(imm & 0xf0000000)) {
1281 imm <<= 4;
1282 rol += 2;
1283 }
1284
1285 if (!(imm & 0xc0000000)) {
1286 imm <<= 2;
1287 rol += 1;
1288 }
1289
1290 if (!(imm & 0x00ffffff))
1291 imm2 = SRC2_IMM | (imm >> 24) | ((rol & 0xf) << 8);
1292 else
1293 return 0;
1294 }
1295
1296 FAIL_IF(push_inst(compiler, EMIT_DATA_PROCESS_INS(positive ? MOV_DP : MVN_DP, 0, reg, SLJIT_UNUSED, imm1)));
1297 FAIL_IF(push_inst(compiler, EMIT_DATA_PROCESS_INS(positive ? ORR_DP : BIC_DP, 0, reg, reg, imm2)));
1298 return 1;
1299 }
1300 #endif
1301
1302 static sljit_s32 load_immediate(struct sljit_compiler *compiler, sljit_s32 reg, sljit_uw imm)
1303 {
1304 sljit_uw tmp;
1305
1306 #if (defined SLJIT_CONFIG_ARM_V7 && SLJIT_CONFIG_ARM_V7)
1307 if (!(imm & ~0xffff))
1308 return push_inst(compiler, MOVW | RD(reg) | ((imm << 4) & 0xf0000) | (imm & 0xfff));
1309 #endif
1310
1311 /* Create imm by 1 inst. */
1312 tmp = get_imm(imm);
1313 if (tmp)
1314 return push_inst(compiler, EMIT_DATA_PROCESS_INS(MOV_DP, 0, reg, SLJIT_UNUSED, tmp));
1315
1316 tmp = get_imm(~imm);
1317 if (tmp)
1318 return push_inst(compiler, EMIT_DATA_PROCESS_INS(MVN_DP, 0, reg, SLJIT_UNUSED, tmp));
1319
1320 #if (defined SLJIT_CONFIG_ARM_V5 && SLJIT_CONFIG_ARM_V5)
1321 /* Create imm by 2 inst. */
1322 FAIL_IF(generate_int(compiler, reg, imm, 1));
1323 FAIL_IF(generate_int(compiler, reg, ~imm, 0));
1324
1325 /* Load integer. */
1326 return push_inst_with_literal(compiler, EMIT_DATA_TRANSFER(WORD_DATA | LOAD_DATA, 1, 0, reg, TMP_PC, 0), imm);
1327 #else
1328 FAIL_IF(push_inst(compiler, MOVW | RD(reg) | ((imm << 4) & 0xf0000) | (imm & 0xfff)));
1329 if (imm <= 0xffff)
1330 return SLJIT_SUCCESS;
1331 return push_inst(compiler, MOVT | RD(reg) | ((imm >> 12) & 0xf0000) | ((imm >> 16) & 0xfff));
1332 #endif
1333 }
1334
1335 static SLJIT_INLINE sljit_s32 emit_op_mem(struct sljit_compiler *compiler, sljit_s32 flags, sljit_s32 reg,
1336 sljit_s32 arg, sljit_sw argw, sljit_s32 tmp_reg)
1337 {
1338 sljit_uw offset_reg, imm;
1339 sljit_uw is_type1_transfer = IS_TYPE1_TRANSFER(flags);
1340
1341 SLJIT_ASSERT (arg & SLJIT_MEM);
1342 SLJIT_ASSERT((arg & REG_MASK) != tmp_reg);
1343
1344 SLJIT_COMPILE_ASSERT(WRITE_BACK == 0x10, optimized_for_emit_data_transfer);
1345
1346 if ((arg & REG_MASK) == SLJIT_UNUSED) {
1347 /* Write back is not used. */
1348 if (is_type1_transfer) {
1349 FAIL_IF(load_immediate(compiler, tmp_reg, argw & ~0xfff));
1350 argw &= 0xfff;
1351 }
1352 else {
1353 FAIL_IF(load_immediate(compiler, tmp_reg, argw & ~0xff));
1354 argw &= 0xff;
1355 }
1356
1357 return push_inst(compiler, EMIT_DATA_TRANSFER(flags, 1, 0, reg, tmp_reg, is_type1_transfer ? argw : TYPE2_TRANSFER_IMM(argw)));
1358 }
1359
1360 if (arg & OFFS_REG_MASK) {
1361 offset_reg = OFFS_REG(arg);
1362 arg &= REG_MASK;
1363 argw &= 0x3;
1364
1365 if (argw != 0 && !is_type1_transfer) {
1366 SLJIT_ASSERT(!(flags & WRITE_BACK));
1367
1368 FAIL_IF(push_inst(compiler, EMIT_DATA_PROCESS_INS(ADD_DP, 0, tmp_reg, arg, RM(offset_reg) | (argw << 7))));
1369 return push_inst(compiler, EMIT_DATA_TRANSFER(flags, 1, 0, reg, tmp_reg, TYPE2_TRANSFER_IMM(0)));
1370 }
1371
1372 /* Bit 25: RM is offset. */
1373 return push_inst(compiler, EMIT_DATA_TRANSFER(flags, 1, flags & WRITE_BACK, reg, arg,
1374 RM(offset_reg) | (is_type1_transfer ? (1 << 25) : 0) | (argw << 7)));
1375 }
1376
1377 arg &= REG_MASK;
1378
1379 if (is_type1_transfer) {
1380 if (argw > 0xfff) {
1381 imm = get_imm(argw & ~0xfff);
1382 if (imm) {
1383 offset_reg = (flags & WRITE_BACK) ? arg : tmp_reg;
1384 FAIL_IF(push_inst(compiler, EMIT_DATA_PROCESS_INS(ADD_DP, 0, offset_reg, arg, imm)));
1385 argw = argw & 0xfff;
1386 arg = offset_reg;
1387 }
1388 }
1389 else if (argw < -0xfff) {
1390 imm = get_imm(-argw & ~0xfff);
1391 if (imm) {
1392 offset_reg = (flags & WRITE_BACK) ? arg : tmp_reg;
1393 FAIL_IF(push_inst(compiler, EMIT_DATA_PROCESS_INS(SUB_DP, 0, offset_reg, arg, imm)));
1394 argw = -(-argw & 0xfff);
1395 arg = offset_reg;
1396 }
1397 }
1398
1399 if (argw >= 0 && argw <= 0xfff) {
1400 return push_inst(compiler, EMIT_DATA_TRANSFER(flags, 1, flags & WRITE_BACK, reg, arg & REG_MASK, argw));
1401 }
1402 if (argw < 0 && argw >= -0xfff) {
1403 return push_inst(compiler, EMIT_DATA_TRANSFER(flags, 0, flags & WRITE_BACK, reg, arg & REG_MASK, -argw));
1404 }
1405 }
1406 else {
1407 if (argw > 0xff) {
1408 imm = get_imm(argw & ~0xff);
1409 if (imm) {
1410 offset_reg = (flags & WRITE_BACK) ? arg : tmp_reg;
1411 FAIL_IF(push_inst(compiler, EMIT_DATA_PROCESS_INS(ADD_DP, 0, offset_reg, arg, imm)));
1412 argw = argw & 0xff;
1413 arg = offset_reg;
1414 }
1415 }
1416 else if (argw < -0xff) {
1417 imm = get_imm(-argw & ~0xff);
1418 if (imm) {
1419 offset_reg = (flags & WRITE_BACK) ? arg : tmp_reg;
1420 FAIL_IF(push_inst(compiler, EMIT_DATA_PROCESS_INS(SUB_DP, 0, offset_reg, arg, imm)));
1421 argw = -(-argw & 0xff);
1422 arg = offset_reg;
1423 }
1424 }
1425
1426 if (argw >= 0 && argw <= 0xff) {
1427 return push_inst(compiler, EMIT_DATA_TRANSFER(flags, 1, flags & WRITE_BACK, reg, arg, TYPE2_TRANSFER_IMM(argw)));
1428 }
1429 if (argw < 0 && argw >= -0xff) {
1430 argw = -argw;
1431 return push_inst(compiler, EMIT_DATA_TRANSFER(flags, 0, flags & WRITE_BACK, reg, arg, TYPE2_TRANSFER_IMM(argw)));
1432 }
1433 }
1434
1435 FAIL_IF(load_immediate(compiler, tmp_reg, argw));
1436 return push_inst(compiler, EMIT_DATA_TRANSFER(flags, 1, flags & WRITE_BACK, reg, arg,
1437 RM(tmp_reg) | (is_type1_transfer ? (1 << 25) : 0)));
1438 }
1439
1440 static sljit_s32 emit_op(struct sljit_compiler *compiler, sljit_s32 op, sljit_s32 inp_flags,
1441 sljit_s32 dst, sljit_sw dstw,
1442 sljit_s32 src1, sljit_sw src1w,
1443 sljit_s32 src2, sljit_sw src2w)
1444 {
1445 /* src1 is reg or TMP_REG1
1446 src2 is reg, TMP_REG2, or imm
1447 result goes to TMP_REG2, so put result can use TMP_REG1. */
1448
1449 /* We prefers register and simple consts. */
1450 sljit_s32 dst_reg;
1451 sljit_s32 src1_reg;
1452 sljit_s32 src2_reg;
1453 sljit_s32 flags = HAS_FLAGS(op) ? SET_FLAGS : 0;
1454
1455 /* Destination check. */
1456 if (SLJIT_UNLIKELY(dst == SLJIT_UNUSED))
1457 flags |= UNUSED_RETURN;
1458
1459 SLJIT_ASSERT(!(inp_flags & ALLOW_INV_IMM) || (inp_flags & ALLOW_IMM));
1460
1461 src2_reg = 0;
1462
1463 do {
1464 if (!(inp_flags & ALLOW_IMM))
1465 break;
1466
1467 if (src2 & SLJIT_IMM) {
1468 src2_reg = get_imm(src2w);
1469 if (src2_reg)
1470 break;
1471 if (inp_flags & ALLOW_INV_IMM) {
1472 src2_reg = get_imm(~src2w);
1473 if (src2_reg) {
1474 flags |= INV_IMM;
1475 break;
1476 }
1477 }
1478 if (GET_OPCODE(op) == SLJIT_ADD) {
1479 src2_reg = get_imm(-src2w);
1480 if (src2_reg) {
1481 op = SLJIT_SUB | GET_ALL_FLAGS(op);
1482 break;
1483 }
1484 }
1485 if (GET_OPCODE(op) == SLJIT_SUB) {
1486 src2_reg = get_imm(-src2w);
1487 if (src2_reg) {
1488 op = SLJIT_ADD | GET_ALL_FLAGS(op);
1489 break;
1490 }
1491 }
1492 }
1493
1494 if (src1 & SLJIT_IMM) {
1495 src2_reg = get_imm(src1w);
1496 if (src2_reg) {
1497 flags |= ARGS_SWAPPED;
1498 src1 = src2;
1499 src1w = src2w;
1500 break;
1501 }
1502 if (inp_flags & ALLOW_INV_IMM) {
1503 src2_reg = get_imm(~src1w);
1504 if (src2_reg) {
1505 flags |= ARGS_SWAPPED | INV_IMM;
1506 src1 = src2;
1507 src1w = src2w;
1508 break;
1509 }
1510 }
1511 if (GET_OPCODE(op) == SLJIT_ADD) {
1512 src2_reg = get_imm(-src1w);
1513 if (src2_reg) {
1514 /* Note: add is commutative operation. */
1515 src1 = src2;
1516 src1w = src2w;
1517 op = SLJIT_SUB | GET_ALL_FLAGS(op);
1518 break;
1519 }
1520 }
1521 }
1522 } while(0);
1523
1524 /* Source 1. */
1525 if (FAST_IS_REG(src1))
1526 src1_reg = src1;
1527 else if (src1 & SLJIT_MEM) {
1528 FAIL_IF(emit_op_mem(compiler, inp_flags | LOAD_DATA, TMP_REG1, src1, src1w, TMP_REG1));
1529 src1_reg = TMP_REG1;
1530 }
1531 else {
1532 FAIL_IF(load_immediate(compiler, TMP_REG1, src1w));
1533 src1_reg = TMP_REG1;
1534 }
1535
1536 /* Destination. */
1537 dst_reg = SLOW_IS_REG(dst) ? dst : TMP_REG2;
1538
1539 if (op <= SLJIT_MOVU_P) {
1540 if (dst & SLJIT_MEM) {
1541 if (inp_flags & BYTE_DATA)
1542 inp_flags &= ~SIGNED_DATA;
1543
1544 if (FAST_IS_REG(src2))
1545 return emit_op_mem(compiler, inp_flags, src2, dst, dstw, TMP_REG2);
1546 }
1547
1548 if (FAST_IS_REG(src2) && dst_reg != TMP_REG2)
1549 flags |= MOVE_REG_CONV;
1550 }
1551
1552 /* Source 2. */
1553 if (src2_reg == 0) {
1554 src2_reg = (op <= SLJIT_MOVU_P) ? dst_reg : TMP_REG2;
1555
1556 if (FAST_IS_REG(src2))
1557 src2_reg = src2;
1558 else if (src2 & SLJIT_MEM)
1559 FAIL_IF(emit_op_mem(compiler, inp_flags | LOAD_DATA, src2_reg, src2, src2w, TMP_REG2));
1560 else
1561 FAIL_IF(load_immediate(compiler, src2_reg, src2w));
1562 }
1563
1564 FAIL_IF(emit_single_op(compiler, op, flags, dst_reg, src1_reg, src2_reg));
1565
1566 if (!(dst & SLJIT_MEM))
1567 return SLJIT_SUCCESS;
1568
1569 return emit_op_mem(compiler, inp_flags, dst_reg, dst, dstw, TMP_REG1);
1570 }
1571
1572 #ifdef __cplusplus
1573 extern "C" {
1574 #endif
1575
1576 #if defined(__GNUC__)
1577 extern unsigned int __aeabi_uidivmod(unsigned int numerator, unsigned int denominator);
1578 extern int __aeabi_idivmod(int numerator, int denominator);
1579 #else
1580 #error "Software divmod functions are needed"
1581 #endif
1582
1583 #ifdef __cplusplus
1584 }
1585 #endif
1586
1587 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op0(struct sljit_compiler *compiler, sljit_s32 op)
1588 {
1589 sljit_sw saved_reg_list[3];
1590 sljit_sw saved_reg_count;
1591
1592 CHECK_ERROR();
1593 CHECK(check_sljit_emit_op0(compiler, op));
1594
1595 op = GET_OPCODE(op);
1596 switch (op) {
1597 case SLJIT_BREAKPOINT:
1598 FAIL_IF(push_inst(compiler, BKPT));
1599 break;
1600 case SLJIT_NOP:
1601 FAIL_IF(push_inst(compiler, NOP));
1602 break;
1603 case SLJIT_LMUL_UW:
1604 case SLJIT_LMUL_SW:
1605 return push_inst(compiler, (op == SLJIT_LMUL_UW ? UMULL : SMULL)
1606 | (reg_map[SLJIT_R1] << 16)
1607 | (reg_map[SLJIT_R0] << 12)
1608 | (reg_map[SLJIT_R0] << 8)
1609 | reg_map[SLJIT_R1]);
1610 case SLJIT_DIVMOD_UW:
1611 case SLJIT_DIVMOD_SW:
1612 case SLJIT_DIV_UW:
1613 case SLJIT_DIV_SW:
1614 SLJIT_COMPILE_ASSERT((SLJIT_DIVMOD_UW & 0x2) == 0 && SLJIT_DIV_UW - 0x2 == SLJIT_DIVMOD_UW, bad_div_opcode_assignments);
1615 SLJIT_ASSERT(reg_map[2] == 1 && reg_map[3] == 2 && reg_map[4] == 3);
1616
1617 saved_reg_count = 0;
1618 if (compiler->scratches >= 4)
1619 saved_reg_list[saved_reg_count++] = 3;
1620 if (compiler->scratches >= 3)
1621 saved_reg_list[saved_reg_count++] = 2;
1622 if (op >= SLJIT_DIV_UW)
1623 saved_reg_list[saved_reg_count++] = 1;
1624
1625 if (saved_reg_count > 0) {
1626 FAIL_IF(push_inst(compiler, 0xe52d0000 | (saved_reg_count >= 3 ? 16 : 8)
1627 | (saved_reg_list[0] << 12) /* str rX, [sp, #-8/-16]! */));
1628 if (saved_reg_count >= 2) {
1629 SLJIT_ASSERT(saved_reg_list[1] < 8);
1630 FAIL_IF(push_inst(compiler, 0xe58d0004 | (saved_reg_list[1] << 12) /* str rX, [sp, #4] */));
1631 }
1632 if (saved_reg_count >= 3) {
1633 SLJIT_ASSERT(saved_reg_list[2] < 8);
1634 FAIL_IF(push_inst(compiler, 0xe58d0008 | (saved_reg_list[2] << 12) /* str rX, [sp, #8] */));
1635 }
1636 }
1637
1638 #if defined(__GNUC__)
1639 FAIL_IF(sljit_emit_ijump(compiler, SLJIT_FAST_CALL, SLJIT_IMM,
1640 ((op | 0x2) == SLJIT_DIV_UW ? SLJIT_FUNC_OFFSET(__aeabi_uidivmod) : SLJIT_FUNC_OFFSET(__aeabi_idivmod))));
1641 #else
1642 #error "Software divmod functions are needed"
1643 #endif
1644
1645 if (saved_reg_count > 0) {
1646 if (saved_reg_count >= 3) {
1647 SLJIT_ASSERT(saved_reg_list[2] < 8);
1648 FAIL_IF(push_inst(compiler, 0xe59d0008 | (saved_reg_list[2] << 12) /* ldr rX, [sp, #8] */));
1649 }
1650 if (saved_reg_count >= 2) {
1651 SLJIT_ASSERT(saved_reg_list[1] < 8);
1652 FAIL_IF(push_inst(compiler, 0xe59d0004 | (saved_reg_list[1] << 12) /* ldr rX, [sp, #4] */));
1653 }
1654 return push_inst(compiler, 0xe49d0000 | (saved_reg_count >= 3 ? 16 : 8)
1655 | (saved_reg_list[0] << 12) /* ldr rX, [sp], #8/16 */);
1656 }
1657 return SLJIT_SUCCESS;
1658 }
1659
1660 return SLJIT_SUCCESS;
1661 }
1662
1663 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op1(struct sljit_compiler *compiler, sljit_s32 op,
1664 sljit_s32 dst, sljit_sw dstw,
1665 sljit_s32 src, sljit_sw srcw)
1666 {
1667 CHECK_ERROR();
1668 CHECK(check_sljit_emit_op1(compiler, op, dst, dstw, src, srcw));
1669 ADJUST_LOCAL_OFFSET(dst, dstw);
1670 ADJUST_LOCAL_OFFSET(src, srcw);
1671
1672 if (dst == SLJIT_UNUSED && !HAS_FLAGS(op)) {
1673 #if (defined SLJIT_CONFIG_ARM_V7 && SLJIT_CONFIG_ARM_V7)
1674 if (op <= SLJIT_MOV_P && (src & SLJIT_MEM))
1675 return emit_op_mem(compiler, PRELOAD_DATA | LOAD_DATA, TMP_PC, src, srcw, TMP_REG1);
1676 #endif
1677 return SLJIT_SUCCESS;
1678 }
1679
1680 switch (GET_OPCODE(op)) {
1681 case SLJIT_MOV:
1682 case SLJIT_MOV_U32:
1683 case SLJIT_MOV_S32:
1684 case SLJIT_MOV_P:
1685 return emit_op(compiler, SLJIT_MOV, ALLOW_ANY_IMM, dst, dstw, TMP_REG1, 0, src, srcw);
1686
1687 case SLJIT_MOV_U8:
1688 return emit_op(compiler, SLJIT_MOV_U8, ALLOW_ANY_IMM | BYTE_DATA, dst, dstw, TMP_REG1, 0, src, (src & SLJIT_IMM) ? (sljit_u8)srcw : srcw);
1689
1690 case SLJIT_MOV_S8:
1691 return emit_op(compiler, SLJIT_MOV_S8, ALLOW_ANY_IMM | SIGNED_DATA | BYTE_DATA, dst, dstw, TMP_REG1, 0, src, (src & SLJIT_IMM) ? (sljit_s8)srcw : srcw);
1692
1693 case SLJIT_MOV_U16:
1694 return emit_op(compiler, SLJIT_MOV_U16, ALLOW_ANY_IMM | HALF_DATA, dst, dstw, TMP_REG1, 0, src, (src & SLJIT_IMM) ? (sljit_u16)srcw : srcw);
1695
1696 case SLJIT_MOV_S16:
1697 return emit_op(compiler, SLJIT_MOV_S16, ALLOW_ANY_IMM | SIGNED_DATA | HALF_DATA, dst, dstw, TMP_REG1, 0, src, (src & SLJIT_IMM) ? (sljit_s16)srcw : srcw);
1698
1699 case SLJIT_MOVU:
1700 case SLJIT_MOVU_U32:
1701 case SLJIT_MOVU_S32:
1702 case SLJIT_MOVU_P:
1703 return emit_op(compiler, SLJIT_MOV, ALLOW_ANY_IMM | WRITE_BACK, dst, dstw, TMP_REG1, 0, src, srcw);
1704
1705 case SLJIT_MOVU_U8:
1706 return emit_op(compiler, SLJIT_MOV_U8, ALLOW_ANY_IMM | BYTE_DATA | WRITE_BACK, dst, dstw, TMP_REG1, 0, src, (src & SLJIT_IMM) ? (sljit_u8)srcw : srcw);
1707
1708 case SLJIT_MOVU_S8:
1709 return emit_op(compiler, SLJIT_MOV_S8, ALLOW_ANY_IMM | SIGNED_DATA | BYTE_DATA | WRITE_BACK, dst, dstw, TMP_REG1, 0, src, (src & SLJIT_IMM) ? (sljit_s8)srcw : srcw);
1710
1711 case SLJIT_MOVU_U16:
1712 return emit_op(compiler, SLJIT_MOV_U16, ALLOW_ANY_IMM | HALF_DATA | WRITE_BACK, dst, dstw, TMP_REG1, 0, src, (src & SLJIT_IMM) ? (sljit_u16)srcw : srcw);
1713
1714 case SLJIT_MOVU_S16:
1715 return emit_op(compiler, SLJIT_MOV_S16, ALLOW_ANY_IMM | SIGNED_DATA | HALF_DATA | WRITE_BACK, dst, dstw, TMP_REG1, 0, src, (src & SLJIT_IMM) ? (sljit_s16)srcw : srcw);
1716
1717 case SLJIT_NOT:
1718 return emit_op(compiler, op, ALLOW_ANY_IMM, dst, dstw, TMP_REG1, 0, src, srcw);
1719
1720 case SLJIT_NEG:
1721 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE) \
1722 || (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1723 compiler->skip_checks = 1;
1724 #endif
1725 return sljit_emit_op2(compiler, SLJIT_SUB | GET_ALL_FLAGS(op), dst, dstw, SLJIT_IMM, 0, src, srcw);
1726
1727 case SLJIT_CLZ:
1728 return emit_op(compiler, op, 0, dst, dstw, TMP_REG1, 0, src, srcw);
1729 }
1730
1731 return SLJIT_SUCCESS;
1732 }
1733
1734 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op2(struct sljit_compiler *compiler, sljit_s32 op,
1735 sljit_s32 dst, sljit_sw dstw,
1736 sljit_s32 src1, sljit_sw src1w,
1737 sljit_s32 src2, sljit_sw src2w)
1738 {
1739 CHECK_ERROR();
1740 CHECK(check_sljit_emit_op2(compiler, op, dst, dstw, src1, src1w, src2, src2w));
1741 ADJUST_LOCAL_OFFSET(dst, dstw);
1742 ADJUST_LOCAL_OFFSET(src1, src1w);
1743 ADJUST_LOCAL_OFFSET(src2, src2w);
1744
1745 if (dst == SLJIT_UNUSED && !HAS_FLAGS(op))
1746 return SLJIT_SUCCESS;
1747
1748 switch (GET_OPCODE(op)) {
1749 case SLJIT_ADD:
1750 case SLJIT_ADDC:
1751 case SLJIT_SUB:
1752 case SLJIT_SUBC:
1753 case SLJIT_OR:
1754 case SLJIT_XOR:
1755 return emit_op(compiler, op, ALLOW_IMM, dst, dstw, src1, src1w, src2, src2w);
1756
1757 case SLJIT_MUL:
1758 return emit_op(compiler, op, 0, dst, dstw, src1, src1w, src2, src2w);
1759
1760 case SLJIT_AND:
1761 return emit_op(compiler, op, ALLOW_ANY_IMM, dst, dstw, src1, src1w, src2, src2w);
1762
1763 case SLJIT_SHL:
1764 case SLJIT_LSHR:
1765 case SLJIT_ASHR:
1766 if (src2 & SLJIT_IMM) {
1767 compiler->shift_imm = src2w & 0x1f;
1768 return emit_op(compiler, op, 0, dst, dstw, TMP_REG1, 0, src1, src1w);
1769 }
1770 else {
1771 compiler->shift_imm = 0x20;
1772 return emit_op(compiler, op, 0, dst, dstw, src1, src1w, src2, src2w);
1773 }
1774 }
1775
1776 return SLJIT_SUCCESS;
1777 }
1778
1779 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_get_register_index(sljit_s32 reg)
1780 {
1781 CHECK_REG_INDEX(check_sljit_get_register_index(reg));
1782 return reg_map[reg];
1783 }
1784
1785 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_get_float_register_index(sljit_s32 reg)
1786 {
1787 CHECK_REG_INDEX(check_sljit_get_float_register_index(reg));
1788 return reg << 1;
1789 }
1790
1791 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_custom(struct sljit_compiler *compiler,
1792 void *instruction, sljit_s32 size)
1793 {
1794 CHECK_ERROR();
1795 CHECK(check_sljit_emit_op_custom(compiler, instruction, size));
1796
1797 return push_inst(compiler, *(sljit_uw*)instruction);
1798 }
1799
1800 /* --------------------------------------------------------------------- */
1801 /* Floating point operators */
1802 /* --------------------------------------------------------------------- */
1803
1804
1805 #define FPU_LOAD (1 << 20)
1806 #define EMIT_FPU_DATA_TRANSFER(inst, add, base, freg, offs) \
1807 ((inst) | ((add) << 23) | (reg_map[base] << 16) | (freg << 12) | (offs))
1808 #define EMIT_FPU_OPERATION(opcode, mode, dst, src1, src2) \
1809 ((opcode) | (mode) | ((dst) << 12) | (src1) | ((src2) << 16))
1810
1811 static sljit_s32 emit_fop_mem(struct sljit_compiler *compiler, sljit_s32 flags, sljit_s32 reg, sljit_s32 arg, sljit_sw argw)
1812 {
1813 sljit_uw imm;
1814 sljit_sw inst = VSTR_F32 | (flags & (SLJIT_F32_OP | FPU_LOAD));
1815
1816 SLJIT_ASSERT(arg & SLJIT_MEM);
1817 arg &= ~SLJIT_MEM;
1818
1819 if (SLJIT_UNLIKELY(arg & OFFS_REG_MASK)) {
1820 FAIL_IF(push_inst(compiler, EMIT_DATA_PROCESS_INS(ADD_DP, 0, TMP_REG2, arg & REG_MASK, RM(OFFS_REG(arg)) | ((argw & 0x3) << 7))));
1821 arg = TMP_REG2;
1822 argw = 0;
1823 }
1824
1825 /* Fast loads and stores. */
1826 if (arg) {
1827 if (!(argw & ~0x3fc))
1828 return push_inst(compiler, EMIT_FPU_DATA_TRANSFER(inst, 1, arg & REG_MASK, reg, argw >> 2));
1829 if (!(-argw & ~0x3fc))
1830 return push_inst(compiler, EMIT_FPU_DATA_TRANSFER(inst, 0, arg & REG_MASK, reg, (-argw) >> 2));
1831
1832 imm = get_imm(argw & ~0x3fc);
1833 if (imm) {
1834 FAIL_IF(push_inst(compiler, EMIT_DATA_PROCESS_INS(ADD_DP, 0, TMP_REG2, arg & REG_MASK, imm)));
1835 return push_inst(compiler, EMIT_FPU_DATA_TRANSFER(inst, 1, TMP_REG2, reg, (argw & 0x3fc) >> 2));
1836 }
1837 imm = get_imm(-argw & ~0x3fc);
1838 if (imm) {
1839 argw = -argw;
1840 FAIL_IF(push_inst(compiler, EMIT_DATA_PROCESS_INS(SUB_DP, 0, TMP_REG2, arg & REG_MASK, imm)));
1841 return push_inst(compiler, EMIT_FPU_DATA_TRANSFER(inst, 0, TMP_REG2, reg, (argw & 0x3fc) >> 2));
1842 }
1843 }
1844
1845 if (arg) {
1846 FAIL_IF(load_immediate(compiler, TMP_REG2, argw));
1847 FAIL_IF(push_inst(compiler, EMIT_DATA_PROCESS_INS(ADD_DP, 0, TMP_REG2, arg & REG_MASK, RM(TMP_REG2))));
1848 }
1849 else
1850 FAIL_IF(load_immediate(compiler, TMP_REG2, argw));
1851
1852 return push_inst(compiler, EMIT_FPU_DATA_TRANSFER(inst, 1, TMP_REG2, reg, 0));
1853 }
1854
1855 static SLJIT_INLINE sljit_s32 sljit_emit_fop1_conv_sw_from_f64(struct sljit_compiler *compiler, sljit_s32 op,
1856 sljit_s32 dst, sljit_sw dstw,
1857 sljit_s32 src, sljit_sw srcw)
1858 {
1859 op ^= SLJIT_F32_OP;
1860
1861 if (src & SLJIT_MEM) {
1862 FAIL_IF(emit_fop_mem(compiler, (op & SLJIT_F32_OP) | FPU_LOAD, TMP_FREG1, src, srcw));
1863 src = TMP_FREG1;
1864 }
1865
1866 FAIL_IF(push_inst(compiler, EMIT_FPU_OPERATION(VCVT_S32_F32, op & SLJIT_F32_OP, TMP_FREG1, src, 0)));
1867
1868 if (FAST_IS_REG(dst))
1869 return push_inst(compiler, VMOV | (1 << 20) | RD(dst) | (TMP_FREG1 << 16));
1870
1871 /* Store the integer value from a VFP register. */
1872 return emit_fop_mem(compiler, 0, TMP_FREG1, dst, dstw);
1873 }
1874
1875 static SLJIT_INLINE sljit_s32 sljit_emit_fop1_conv_f64_from_sw(struct sljit_compiler *compiler, sljit_s32 op,
1876 sljit_s32 dst, sljit_sw dstw,
1877 sljit_s32 src, sljit_sw srcw)
1878 {
1879 sljit_s32 dst_r = FAST_IS_REG(dst) ? dst : TMP_FREG1;
1880
1881 op ^= SLJIT_F32_OP;
1882
1883 if (FAST_IS_REG(src))
1884 FAIL_IF(push_inst(compiler, VMOV | RD(src) | (TMP_FREG1 << 16)));
1885 else if (src & SLJIT_MEM) {
1886 /* Load the integer value into a VFP register. */
1887 FAIL_IF(emit_fop_mem(compiler, FPU_LOAD, TMP_FREG1, src, srcw));
1888 }
1889 else {
1890 FAIL_IF(load_immediate(compiler, TMP_REG1, srcw));
1891 FAIL_IF(push_inst(compiler, VMOV | RD(TMP_REG1) | (TMP_FREG1 << 16)));
1892 }
1893
1894 FAIL_IF(push_inst(compiler, EMIT_FPU_OPERATION(VCVT_F32_S32, op & SLJIT_F32_OP, dst_r, TMP_FREG1, 0)));
1895
1896 if (dst & SLJIT_MEM)
1897 return emit_fop_mem(compiler, (op & SLJIT_F32_OP), TMP_FREG1, dst, dstw);
1898 return SLJIT_SUCCESS;
1899 }
1900
1901 static SLJIT_INLINE sljit_s32 sljit_emit_fop1_cmp(struct sljit_compiler *compiler, sljit_s32 op,
1902 sljit_s32 src1, sljit_sw src1w,
1903 sljit_s32 src2, sljit_sw src2w)
1904 {
1905 op ^= SLJIT_F32_OP;
1906
1907 if (src1 & SLJIT_MEM) {
1908 FAIL_IF(emit_fop_mem(compiler, (op & SLJIT_F32_OP) | FPU_LOAD, TMP_FREG1, src1, src1w));
1909 src1 = TMP_FREG1;
1910 }
1911
1912 if (src2 & SLJIT_MEM) {
1913 FAIL_IF(emit_fop_mem(compiler, (op & SLJIT_F32_OP) | FPU_LOAD, TMP_FREG2, src2, src2w));
1914 src2 = TMP_FREG2;
1915 }
1916
1917 FAIL_IF(push_inst(compiler, EMIT_FPU_OPERATION(VCMP_F32, op & SLJIT_F32_OP, src1, src2, 0)));
1918 return push_inst(compiler, VMRS);
1919 }
1920
1921 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fop1(struct sljit_compiler *compiler, sljit_s32 op,
1922 sljit_s32 dst, sljit_sw dstw,
1923 sljit_s32 src, sljit_sw srcw)
1924 {
1925 sljit_s32 dst_r;
1926
1927 CHECK_ERROR();
1928
1929 SLJIT_COMPILE_ASSERT((SLJIT_F32_OP == 0x100), float_transfer_bit_error);
1930 SELECT_FOP1_OPERATION_WITH_CHECKS(compiler, op, dst, dstw, src, srcw);
1931
1932 dst_r = FAST_IS_REG(dst) ? dst : TMP_FREG1;
1933
1934 if (GET_OPCODE(op) != SLJIT_CONV_F64_FROM_F32)
1935 op ^= SLJIT_F32_OP;
1936
1937 if (src & SLJIT_MEM) {
1938 FAIL_IF(emit_fop_mem(compiler, (op & SLJIT_F32_OP) | FPU_LOAD, dst_r, src, srcw));
1939 src = dst_r;
1940 }
1941
1942 switch (GET_OPCODE(op)) {
1943 case SLJIT_MOV_F64:
1944 if (src != dst_r) {
1945 if (dst_r != TMP_FREG1)
1946 FAIL_IF(push_inst(compiler, EMIT_FPU_OPERATION(VMOV_F32, op & SLJIT_F32_OP, dst_r, src, 0)));
1947 else
1948 dst_r = src;
1949 }
1950 break;
1951 case SLJIT_NEG_F64:
1952 FAIL_IF(push_inst(compiler, EMIT_FPU_OPERATION(VNEG_F32, op & SLJIT_F32_OP, dst_r, src, 0)));
1953 break;
1954 case SLJIT_ABS_F64:
1955 FAIL_IF(push_inst(compiler, EMIT_FPU_OPERATION(VABS_F32, op & SLJIT_F32_OP, dst_r, src, 0)));
1956 break;
1957 case SLJIT_CONV_F64_FROM_F32:
1958 FAIL_IF(push_inst(compiler, EMIT_FPU_OPERATION(VCVT_F64_F32, op & SLJIT_F32_OP, dst_r, src, 0)));
1959 op ^= SLJIT_F32_OP;
1960 break;
1961 }
1962
1963 if (dst & SLJIT_MEM)
1964 return emit_fop_mem(compiler, (op & SLJIT_F32_OP), dst_r, dst, dstw);
1965 return SLJIT_SUCCESS;
1966 }
1967
1968 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fop2(struct sljit_compiler *compiler, sljit_s32 op,
1969 sljit_s32 dst, sljit_sw dstw,
1970 sljit_s32 src1, sljit_sw src1w,
1971 sljit_s32 src2, sljit_sw src2w)
1972 {
1973 sljit_s32 dst_r;
1974
1975 CHECK_ERROR();
1976 CHECK(check_sljit_emit_fop2(compiler, op, dst, dstw, src1, src1w, src2, src2w));
1977 ADJUST_LOCAL_OFFSET(dst, dstw);
1978 ADJUST_LOCAL_OFFSET(src1, src1w);
1979 ADJUST_LOCAL_OFFSET(src2, src2w);
1980
1981 op ^= SLJIT_F32_OP;
1982
1983 dst_r = FAST_IS_REG(dst) ? dst : TMP_FREG1;
1984
1985 if (src2 & SLJIT_MEM) {
1986 FAIL_IF(emit_fop_mem(compiler, (op & SLJIT_F32_OP) | FPU_LOAD, TMP_FREG2, src2, src2w));
1987 src2 = TMP_FREG2;
1988 }
1989
1990 if (src1 & SLJIT_MEM) {
1991 FAIL_IF(emit_fop_mem(compiler, (op & SLJIT_F32_OP) | FPU_LOAD, TMP_FREG1, src1, src1w));
1992 src1 = TMP_FREG1;
1993 }
1994
1995 switch (GET_OPCODE(op)) {
1996 case SLJIT_ADD_F64:
1997 FAIL_IF(push_inst(compiler, EMIT_FPU_OPERATION(VADD_F32, op & SLJIT_F32_OP, dst_r, src2, src1)));
1998 break;
1999
2000 case SLJIT_SUB_F64:
2001 FAIL_IF(push_inst(compiler, EMIT_FPU_OPERATION(VSUB_F32, op & SLJIT_F32_OP, dst_r, src2, src1)));
2002 break;
2003
2004 case SLJIT_MUL_F64:
2005 FAIL_IF(push_inst(compiler, EMIT_FPU_OPERATION(VMUL_F32, op & SLJIT_F32_OP, dst_r, src2, src1)));
2006 break;
2007
2008 case SLJIT_DIV_F64:
2009 FAIL_IF(push_inst(compiler, EMIT_FPU_OPERATION(VDIV_F32, op & SLJIT_F32_OP, dst_r, src2, src1)));
2010 break;
2011 }
2012
2013 if (dst_r == TMP_FREG1)
2014 FAIL_IF(emit_fop_mem(compiler, (op & SLJIT_F32_OP), TMP_FREG1, dst, dstw));
2015
2016 return SLJIT_SUCCESS;
2017 }
2018
2019 #undef FPU_LOAD
2020 #undef EMIT_FPU_DATA_TRANSFER
2021 #undef EMIT_FPU_OPERATION
2022
2023 /* --------------------------------------------------------------------- */
2024 /* Other instructions */
2025 /* --------------------------------------------------------------------- */
2026
2027 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fast_enter(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw)
2028 {
2029 CHECK_ERROR();
2030 CHECK(check_sljit_emit_fast_enter(compiler, dst, dstw));
2031 ADJUST_LOCAL_OFFSET(dst, dstw);
2032
2033 SLJIT_ASSERT(reg_map[TMP_REG1] == 14);
2034
2035 if (FAST_IS_REG(dst))
2036 return push_inst(compiler, EMIT_DATA_PROCESS_INS(MOV_DP, 0, dst, SLJIT_UNUSED, RM(TMP_REG1)));
2037
2038 /* Memory. */
2039 return emit_op_mem(compiler, WORD_DATA, TMP_REG1, dst, dstw, TMP_REG2);
2040 }
2041
2042 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fast_return(struct sljit_compiler *compiler, sljit_s32 src, sljit_sw srcw)
2043 {
2044 CHECK_ERROR();
2045 CHECK(check_sljit_emit_fast_return(compiler, src, srcw));
2046 ADJUST_LOCAL_OFFSET(src, srcw);
2047
2048 SLJIT_ASSERT(reg_map[TMP_REG1] == 14);
2049
2050 if (FAST_IS_REG(src))
2051 FAIL_IF(push_inst(compiler, EMIT_DATA_PROCESS_INS(MOV_DP, 0, TMP_REG1, 0, RM(src))));
2052 else if (src & SLJIT_MEM)
2053 FAIL_IF(emit_op_mem(compiler, WORD_DATA | LOAD_DATA, TMP_REG1, src, srcw, TMP_REG2));
2054 else if (src & SLJIT_IMM)
2055 FAIL_IF(load_immediate(compiler, TMP_REG1, srcw));
2056
2057 return push_inst(compiler, BX | RM(TMP_REG1));
2058 }
2059
2060 /* --------------------------------------------------------------------- */
2061 /* Conditional instructions */
2062 /* --------------------------------------------------------------------- */
2063
2064 static sljit_uw get_cc(sljit_s32 type)
2065 {
2066 switch (type) {
2067 case SLJIT_EQUAL:
2068 case SLJIT_MUL_NOT_OVERFLOW:
2069 case SLJIT_EQUAL_F64:
2070 return 0x00000000;
2071
2072 case SLJIT_NOT_EQUAL:
2073 case SLJIT_MUL_OVERFLOW:
2074 case SLJIT_NOT_EQUAL_F64:
2075 return 0x10000000;
2076
2077 case SLJIT_LESS:
2078 case SLJIT_LESS_F64:
2079 return 0x30000000;
2080
2081 case SLJIT_GREATER_EQUAL:
2082 case SLJIT_GREATER_EQUAL_F64:
2083 return 0x20000000;
2084
2085 case SLJIT_GREATER:
2086 case SLJIT_GREATER_F64:
2087 return 0x80000000;
2088
2089 case SLJIT_LESS_EQUAL:
2090 case SLJIT_LESS_EQUAL_F64:
2091 return 0x90000000;
2092
2093 case SLJIT_SIG_LESS:
2094 return 0xb0000000;
2095
2096 case SLJIT_SIG_GREATER_EQUAL:
2097 return 0xa0000000;
2098
2099 case SLJIT_SIG_GREATER:
2100 return 0xc0000000;
2101
2102 case SLJIT_SIG_LESS_EQUAL:
2103 return 0xd0000000;
2104
2105 case SLJIT_OVERFLOW:
2106 case SLJIT_UNORDERED_F64:
2107 return 0x60000000;
2108
2109 case SLJIT_NOT_OVERFLOW:
2110 case SLJIT_ORDERED_F64:
2111 return 0x70000000;
2112
2113 default:
2114 SLJIT_ASSERT(type >= SLJIT_JUMP && type <= SLJIT_CALL3);
2115 return 0xe0000000;
2116 }
2117 }
2118
2119 SLJIT_API_FUNC_ATTRIBUTE struct sljit_label* sljit_emit_label(struct sljit_compiler *compiler)
2120 {
2121 struct sljit_label *label;
2122
2123 CHECK_ERROR_PTR();
2124 CHECK_PTR(check_sljit_emit_label(compiler));
2125
2126 if (compiler->last_label && compiler->last_label->size == compiler->size)
2127 return compiler->last_label;
2128
2129 label = (struct sljit_label*)ensure_abuf(compiler, sizeof(struct sljit_label));
2130 PTR_FAIL_IF(!label);
2131 set_label(label, compiler);
2132 return label;
2133 }
2134
2135 SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_jump(struct sljit_compiler *compiler, sljit_s32 type)
2136 {
2137 struct sljit_jump *jump;
2138
2139 CHECK_ERROR_PTR();
2140 CHECK_PTR(check_sljit_emit_jump(compiler, type));
2141
2142 jump = (struct sljit_jump*)ensure_abuf(compiler, sizeof(struct sljit_jump));
2143 PTR_FAIL_IF(!jump);
2144 set_jump(jump, compiler, type & SLJIT_REWRITABLE_JUMP);
2145 type &= 0xff;
2146
2147 /* In ARM, we don't need to touch the arguments. */
2148 #if (defined SLJIT_CONFIG_ARM_V5 && SLJIT_CONFIG_ARM_V5)
2149 if (type >= SLJIT_FAST_CALL)
2150 PTR_FAIL_IF(prepare_blx(compiler));
2151 PTR_FAIL_IF(push_inst_with_unique_literal(compiler, ((EMIT_DATA_TRANSFER(WORD_DATA | LOAD_DATA, 1, 0,
2152 type <= SLJIT_JUMP ? TMP_PC : TMP_REG2, TMP_PC, 0)) & ~COND_MASK) | get_cc(type), 0));
2153
2154 if (jump->flags & SLJIT_REWRITABLE_JUMP) {
2155 jump->addr = compiler->size;
2156 compiler->patches++;
2157 }
2158
2159 if (type >= SLJIT_FAST_CALL) {
2160 jump->flags |= IS_BL;
2161 PTR_FAIL_IF(emit_blx(compiler));
2162 }
2163
2164 if (!(jump->flags & SLJIT_REWRITABLE_JUMP))
2165 jump->addr = compiler->size;
2166 #else
2167 if (type >= SLJIT_FAST_CALL)
2168 jump->flags |= IS_BL;
2169 PTR_FAIL_IF(emit_imm(compiler, TMP_REG2, 0));
2170 PTR_FAIL_IF(push_inst(compiler, (((type <= SLJIT_JUMP ? BX : BLX) | RM(TMP_REG2)) & ~COND_MASK) | get_cc(type)));
2171 jump->addr = compiler->size;
2172 #endif
2173 return jump;
2174 }
2175
2176 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_ijump(struct sljit_compiler *compiler, sljit_s32 type, sljit_s32 src, sljit_sw srcw)
2177 {
2178 struct sljit_jump *jump;
2179
2180 CHECK_ERROR();
2181 CHECK(check_sljit_emit_ijump(compiler, type, src, srcw));
2182 ADJUST_LOCAL_OFFSET(src, srcw);
2183
2184 /* In ARM, we don't need to touch the arguments. */
2185 if (!(src & SLJIT_IMM)) {
2186 if (FAST_IS_REG(src))
2187 return push_inst(compiler, (type <= SLJIT_JUMP ? BX : BLX) | RM(src));
2188
2189 SLJIT_ASSERT(src & SLJIT_MEM);
2190 FAIL_IF(emit_op_mem(compiler, WORD_DATA | LOAD_DATA, TMP_REG2, src, srcw, TMP_REG2));
2191 return push_inst(compiler, (type <= SLJIT_JUMP ? BX : BLX) | RM(TMP_REG2));
2192 }
2193
2194 jump = (struct sljit_jump*)ensure_abuf(compiler, sizeof(struct sljit_jump));
2195 FAIL_IF(!jump);
2196 set_jump(jump, compiler, JUMP_ADDR | ((type >= SLJIT_FAST_CALL) ? IS_BL : 0));
2197 jump->u.target = srcw;
2198
2199 #if (defined SLJIT_CONFIG_ARM_V5 && SLJIT_CONFIG_ARM_V5)
2200 if (type >= SLJIT_FAST_CALL)
2201 FAIL_IF(prepare_blx(compiler));
2202 FAIL_IF(push_inst_with_unique_literal(compiler, EMIT_DATA_TRANSFER(WORD_DATA | LOAD_DATA, 1, 0, type <= SLJIT_JUMP ? TMP_PC : TMP_REG2, TMP_PC, 0), 0));
2203 if (type >= SLJIT_FAST_CALL)
2204 FAIL_IF(emit_blx(compiler));
2205 #else
2206 FAIL_IF(emit_imm(compiler, TMP_REG2, 0));
2207 FAIL_IF(push_inst(compiler, (type <= SLJIT_JUMP ? BX : BLX) | RM(TMP_REG2)));
2208 #endif
2209 jump->addr = compiler->size;
2210 return SLJIT_SUCCESS;
2211 }
2212
2213 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_flags(struct sljit_compiler *compiler, sljit_s32 op,
2214 sljit_s32 dst, sljit_sw dstw,
2215 sljit_s32 type)
2216 {
2217 sljit_s32 dst_r, flags = GET_ALL_FLAGS(op);
2218 sljit_uw cc, ins;
2219
2220 CHECK_ERROR();
2221 CHECK(check_sljit_emit_op_flags(compiler, op, dst, dstw, type));
2222 ADJUST_LOCAL_OFFSET(dst, dstw);
2223
2224 op = GET_OPCODE(op);
2225 cc = get_cc(type & 0xff);
2226 dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
2227
2228 if (op < SLJIT_ADD) {
2229 FAIL_IF(push_inst(compiler, EMIT_DATA_PROCESS_INS(MOV_DP, 0, dst_r, SLJIT_UNUSED, SRC2_IMM | 0)));
2230 FAIL_IF(push_inst(compiler, (EMIT_DATA_PROCESS_INS(MOV_DP, 0, dst_r, SLJIT_UNUSED, SRC2_IMM | 1) & ~COND_MASK) | cc));
2231 if (dst & SLJIT_MEM)
2232 return emit_op_mem(compiler, WORD_DATA, TMP_REG1, dst, dstw, TMP_REG2);
2233 return SLJIT_SUCCESS;
2234 }
2235
2236 ins = (op == SLJIT_AND ? AND_DP : (op == SLJIT_OR ? ORR_DP : EOR_DP));
2237
2238 if (dst & SLJIT_MEM)
2239 FAIL_IF(emit_op_mem(compiler, WORD_DATA | LOAD_DATA, TMP_REG1, dst, dstw, TMP_REG2));
2240
2241 FAIL_IF(push_inst(compiler, (EMIT_DATA_PROCESS_INS(ins, 0, dst_r, dst_r, SRC2_IMM | 1) & ~COND_MASK) | cc));
2242
2243 if (op == SLJIT_AND)
2244 FAIL_IF(push_inst(compiler, (EMIT_DATA_PROCESS_INS(ins, 0, dst_r, dst_r, SRC2_IMM | 0) & ~COND_MASK) | (cc ^ 0x10000000)));
2245
2246 if (dst & SLJIT_MEM)
2247 FAIL_IF(emit_op_mem(compiler, WORD_DATA, TMP_REG1, dst, dstw, TMP_REG2));
2248
2249 if (flags & SLJIT_SET_Z)
2250 return push_inst(compiler, EMIT_DATA_PROCESS_INS(MOV_DP, SET_FLAGS, TMP_REG2, SLJIT_UNUSED, RM(dst_r)));
2251 return SLJIT_SUCCESS;
2252 }
2253
2254 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_cmov(struct sljit_compiler *compiler, sljit_s32 type,
2255 sljit_s32 dst_reg,
2256 sljit_s32 src, sljit_sw srcw)
2257 {
2258 sljit_uw cc, tmp;
2259
2260 CHECK_ERROR();
2261 CHECK(check_sljit_emit_cmov(compiler, type, dst_reg, src, srcw));
2262
2263 dst_reg &= ~SLJIT_I32_OP;
2264
2265 cc = get_cc(type & 0xff);
2266
2267 if (SLJIT_UNLIKELY(src & SLJIT_IMM)) {
2268 tmp = get_imm(srcw);
2269 if (tmp)
2270 return push_inst(compiler, (EMIT_DATA_PROCESS_INS(MOV_DP, 0, dst_reg, SLJIT_UNUSED, tmp) & ~COND_MASK) | cc);
2271
2272 tmp = get_imm(~srcw);
2273 if (tmp)
2274 return push_inst(compiler, (EMIT_DATA_PROCESS_INS(MVN_DP, 0, dst_reg, SLJIT_UNUSED, tmp) & ~COND_MASK) | cc);
2275
2276 #if (defined SLJIT_CONFIG_ARM_V7 && SLJIT_CONFIG_ARM_V7)
2277 tmp = (sljit_uw) srcw;
2278 FAIL_IF(push_inst(compiler, (MOVW & ~COND_MASK) | cc | RD(dst_reg) | ((tmp << 4) & 0xf0000) | (tmp & 0xfff)));
2279 if (tmp <= 0xffff)
2280 return SLJIT_SUCCESS;
2281 return push_inst(compiler, (MOVT & ~COND_MASK) | cc | RD(dst_reg) | ((tmp >> 12) & 0xf0000) | ((tmp >> 16) & 0xfff));
2282 #else
2283 FAIL_IF(load_immediate(compiler, TMP_REG1, srcw));
2284 src = TMP_REG1;
2285 #endif
2286 }
2287
2288 return push_inst(compiler, (EMIT_DATA_PROCESS_INS(MOV_DP, 0, dst_reg, SLJIT_UNUSED, RM(src)) & ~COND_MASK) | cc);
2289 }
2290
2291 SLJIT_API_FUNC_ATTRIBUTE struct sljit_const* sljit_emit_const(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw, sljit_sw init_value)
2292 {
2293 struct sljit_const *const_;
2294 sljit_s32 reg;
2295
2296 CHECK_ERROR_PTR();
2297 CHECK_PTR(check_sljit_emit_const(compiler, dst, dstw, init_value));
2298 ADJUST_LOCAL_OFFSET(dst, dstw);
2299
2300 const_ = (struct sljit_const*)ensure_abuf(compiler, sizeof(struct sljit_const));
2301 PTR_FAIL_IF(!const_);
2302
2303 reg = SLOW_IS_REG(dst) ? dst : TMP_REG2;
2304
2305 #if (defined SLJIT_CONFIG_ARM_V5 && SLJIT_CONFIG_ARM_V5)
2306 PTR_FAIL_IF(push_inst_with_unique_literal(compiler, EMIT_DATA_TRANSFER(WORD_DATA | LOAD_DATA, 1, 0, reg, TMP_PC, 0), init_value));
2307 compiler->patches++;
2308 #else
2309 PTR_FAIL_IF(emit_imm(compiler, reg, init_value));
2310 #endif
2311 set_const(const_, compiler);
2312
2313 if (dst & SLJIT_MEM)
2314 PTR_FAIL_IF(emit_op_mem(compiler, WORD_DATA, TMP_REG2, dst, dstw, TMP_REG1));
2315 return const_;
2316 }
2317
2318 SLJIT_API_FUNC_ATTRIBUTE void sljit_set_jump_addr(sljit_uw addr, sljit_uw new_target, sljit_sw executable_offset)
2319 {
2320 inline_set_jump_addr(addr, executable_offset, new_target, 1);
2321 }
2322
2323 SLJIT_API_FUNC_ATTRIBUTE void sljit_set_const(sljit_uw addr, sljit_sw new_constant, sljit_sw executable_offset)
2324 {
2325 inline_set_const(addr, executable_offset, new_constant, 1);
2326 }
Platform independent low-level JIT compiler (github mirror, with mob branch)