2 * A64 code generator for TCC
4 * Copyright (c) 2014-2015 Edmund Grimley Evans
6 * Copying and distribution of this file, with or without modification,
7 * are permitted in any medium without royalty provided the copyright
8 * notice and this notice are preserved. This file is offered as-is,
9 * without any warranty.
12 #ifdef TARGET_DEFS_ONLY
14 // Number of registers available to allocator:
15 #define NB_REGS 28 // x0-x18, x30, v0-v7
17 #define TREG_R(x) (x) // x = 0..18
19 #define TREG_F(x) (x + 20) // x = 0..7
21 // Register classes sorted from more general to more precise:
22 #define RC_INT (1 << 0)
23 #define RC_FLOAT (1 << 1)
24 #define RC_R(x) (1 << (2 + (x))) // x = 0..18
25 #define RC_R30 (1 << 21)
26 #define RC_F(x) (1 << (22 + (x))) // x = 0..7
28 #define RC_IRET (RC_R(0)) // int return register class
29 #define RC_FRET (RC_F(0)) // float return register class
31 #define REG_IRET (TREG_R(0)) // int return register number
32 #define REG_FRET (TREG_F(0)) // float return register number
36 #define LDOUBLE_SIZE 16
37 #define LDOUBLE_ALIGN 16
41 #define CHAR_IS_UNSIGNED
43 /* define if return values need to be extended explicitely
44 at caller side (for interfacing with non-TCC compilers) */
46 /******************************************************/
47 #else /* ! TARGET_DEFS_ONLY */
48 /******************************************************/
53 ST_DATA
const char * const target_machine_defs
=
54 #if defined(__APPLE__)
62 ST_DATA
const int reg_classes
[NB_REGS
] = {
82 RC_R30
, // not in RC_INT as we make special use of x30
93 #if defined(CONFIG_TCC_BCHECK)
94 static addr_t func_bound_offset
;
95 static unsigned long func_bound_ind
;
96 ST_DATA
int func_bound_add_epilog
;
99 #define IS_FREG(x) ((x) >= TREG_F(0))
101 static uint32_t intr(int r
)
103 assert(TREG_R(0) <= r
&& r
<= TREG_R30
);
104 return r
< TREG_R30
? r
: 30;
107 static uint32_t fltr(int r
)
109 assert(TREG_F(0) <= r
&& r
<= TREG_F(7));
110 return r
- TREG_F(0);
113 // Add an instruction to text section:
114 ST_FUNC
void o(unsigned int c
)
119 if (ind1
> cur_text_section
->data_allocated
)
120 section_realloc(cur_text_section
, ind1
);
121 write32le(cur_text_section
->data
+ ind
, c
);
125 static int arm64_encode_bimm64(uint64_t x
)
135 if (x
>> 2 == (x
& (((uint64_t)1 << (64 - 2)) - 1)))
136 rep
= 2, x
&= ((uint64_t)1 << 2) - 1;
137 else if (x
>> 4 == (x
& (((uint64_t)1 << (64 - 4)) - 1)))
138 rep
= 4, x
&= ((uint64_t)1 << 4) - 1;
139 else if (x
>> 8 == (x
& (((uint64_t)1 << (64 - 8)) - 1)))
140 rep
= 8, x
&= ((uint64_t)1 << 8) - 1;
141 else if (x
>> 16 == (x
& (((uint64_t)1 << (64 - 16)) - 1)))
142 rep
= 16, x
&= ((uint64_t)1 << 16) - 1;
143 else if (x
>> 32 == (x
& (((uint64_t)1 << (64 - 32)) - 1)))
144 rep
= 32, x
&= ((uint64_t)1 << 32) - 1;
149 if (!(x
& (((uint64_t)1 << 32) - 1))) x
>>= 32, pos
+= 32;
150 if (!(x
& (((uint64_t)1 << 16) - 1))) x
>>= 16, pos
+= 16;
151 if (!(x
& (((uint64_t)1 << 8) - 1))) x
>>= 8, pos
+= 8;
152 if (!(x
& (((uint64_t)1 << 4) - 1))) x
>>= 4, pos
+= 4;
153 if (!(x
& (((uint64_t)1 << 2) - 1))) x
>>= 2, pos
+= 2;
154 if (!(x
& (((uint64_t)1 << 1) - 1))) x
>>= 1, pos
+= 1;
157 if (!(~x
& (((uint64_t)1 << 32) - 1))) x
>>= 32, len
+= 32;
158 if (!(~x
& (((uint64_t)1 << 16) - 1))) x
>>= 16, len
+= 16;
159 if (!(~x
& (((uint64_t)1 << 8) - 1))) x
>>= 8, len
+= 8;
160 if (!(~x
& (((uint64_t)1 << 4) - 1))) x
>>= 4, len
+= 4;
161 if (!(~x
& (((uint64_t)1 << 2) - 1))) x
>>= 2, len
+= 2;
162 if (!(~x
& (((uint64_t)1 << 1) - 1))) x
>>= 1, len
+= 1;
167 pos
= (pos
+ len
) & (rep
- 1);
170 return ((0x1000 & rep
<< 6) | (((rep
- 1) ^ 31) << 1 & 63) |
171 ((rep
- pos
) & (rep
- 1)) << 6 | (len
- 1));
174 static uint32_t arm64_movi(int r
, uint64_t x
)
179 return 0x52800000 | r
| x
<< 5; // movz w(r),#(x)
180 if (!(x
& ~(m
<< 16)))
181 return 0x52a00000 | r
| x
>> 11; // movz w(r),#(x >> 16),lsl #16
182 if (!(x
& ~(m
<< 32)))
183 return 0xd2c00000 | r
| x
>> 27; // movz x(r),#(x >> 32),lsl #32
184 if (!(x
& ~(m
<< 48)))
185 return 0xd2e00000 | r
| x
>> 43; // movz x(r),#(x >> 48),lsl #48
186 if ((x
& ~m
) == m
<< 16)
187 return (0x12800000 | r
|
188 (~x
<< 5 & 0x1fffe0)); // movn w(r),#(~x)
189 if ((x
& ~(m
<< 16)) == m
)
190 return (0x12a00000 | r
|
191 (~x
>> 11 & 0x1fffe0)); // movn w(r),#(~x >> 16),lsl #16
193 return (0x92800000 | r
|
194 (~x
<< 5 & 0x1fffe0)); // movn x(r),#(~x)
196 return (0x92a00000 | r
|
197 (~x
>> 11 & 0x1fffe0)); // movn x(r),#(~x >> 16),lsl #16
199 return (0x92c00000 | r
|
200 (~x
>> 27 & 0x1fffe0)); // movn x(r),#(~x >> 32),lsl #32
202 return (0x92e00000 | r
|
203 (~x
>> 43 & 0x1fffe0)); // movn x(r),#(~x >> 32),lsl #32
204 if (!(x
>> 32) && (e
= arm64_encode_bimm64(x
| x
<< 32)) >= 0)
205 return 0x320003e0 | r
| (uint32_t)e
<< 10; // movi w(r),#(x)
206 if ((e
= arm64_encode_bimm64(x
)) >= 0)
207 return 0xb20003e0 | r
| (uint32_t)e
<< 10; // movi x(r),#(x)
211 static void arm64_movimm(int r
, uint64_t x
)
214 if ((i
= arm64_movi(r
, x
)))
215 o(i
); // a single MOV
217 // MOVZ/MOVN and 1-3 MOVKs
219 uint32_t mov1
= 0xd2800000; // movz
221 for (i
= 0; i
< 64; i
+= 16) {
222 z
+= !(x
>> i
& 0xffff);
223 m
+= !(~x
>> i
& 0xffff);
227 mov1
= 0x92800000; // movn
229 for (i
= 0; i
< 64; i
+= 16)
230 if (x1
>> i
& 0xffff) {
231 o(mov1
| r
| (x1
>> i
& 0xffff) << 5 | i
<< 17);
232 // movz/movn x(r),#(*),lsl #(i)
235 for (i
+= 16; i
< 64; i
+= 16)
236 if (x1
>> i
& 0xffff)
237 o(0xf2800000 | r
| (x
>> i
& 0xffff) << 5 | i
<< 17);
238 // movk x(r),#(*),lsl #(i)
242 // Patch all branches in list pointed to by t to branch to a:
243 ST_FUNC
void gsym_addr(int t_
, int a_
)
248 unsigned char *ptr
= cur_text_section
->data
+ t
;
249 uint32_t next
= read32le(ptr
);
250 if (a
- t
+ 0x8000000 >= 0x10000000)
251 tcc_error("branch out of range");
252 write32le(ptr
, (a
- t
== 4 ? 0xd503201f : // nop
253 0x14000000 | ((a
- t
) >> 2 & 0x3ffffff))); // b
258 static int arm64_type_size(int t
)
261 * case values are in increasing order (from 1 to 11).
262 * which 'may' help compiler optimizers. See tcc.h
264 switch (t
& VT_BTYPE
) {
265 case VT_BYTE
: return 0;
266 case VT_SHORT
: return 1;
267 case VT_INT
: return 2;
268 case VT_LLONG
: return 3;
269 case VT_PTR
: return 3;
270 case VT_FUNC
: return 3;
271 case VT_STRUCT
: return 3;
272 case VT_FLOAT
: return 2;
273 case VT_DOUBLE
: return 3;
274 case VT_LDOUBLE
: return 4;
275 case VT_BOOL
: return 0;
281 static void arm64_spoff(int reg
, uint64_t off
)
283 uint32_t sub
= off
>> 63;
287 o(0x910003e0 | sub
<< 30 | reg
| off
<< 10);
288 // (add|sub) x(reg),sp,#(off)
290 arm64_movimm(30, off
); // use x30 for offset
291 o(0x8b3e63e0 | sub
<< 30 | reg
); // (add|sub) x(reg),sp,x30
295 /* invert 0: return value to use for store/load */
296 /* invert 1: return value to use for arm64_sym */
297 static uint64_t arm64_check_offset(int invert
, int sz_
, uint64_t off
)
300 if (!(off
& ~((uint32_t)0xfff << sz
)) ||
301 (off
< 256 || -off
<= 256))
302 return invert
? off
: 0ul;
303 else if ((off
& ((uint32_t)0xfff << sz
)))
304 return invert
? off
& ((uint32_t)0xfff << sz
)
305 : off
& ~((uint32_t)0xfff << sz
);
306 else if (off
& 0x1ff)
307 return invert
? off
& 0x1ff : off
& ~0x1ff;
309 return invert
? 0ul : off
;
312 static void arm64_ldrx(int sg
, int sz_
, int dst
, int bas
, uint64_t off
)
317 if (!(off
& ~((uint32_t)0xfff << sz
)))
318 o(0x39400000 | dst
| bas
<< 5 | off
<< (10 - sz
) |
319 (uint32_t)!!sg
<< 23 | sz
<< 30); // ldr(*) x(dst),[x(bas),#(off)]
320 else if (off
< 256 || -off
<= 256)
321 o(0x38400000 | dst
| bas
<< 5 | (off
& 511) << 12 |
322 (uint32_t)!!sg
<< 23 | sz
<< 30); // ldur(*) x(dst),[x(bas),#(off)]
324 arm64_movimm(30, off
); // use x30 for offset
325 o(0x38206800 | dst
| bas
<< 5 | (uint32_t)30 << 16 |
326 (uint32_t)(!!sg
+ 1) << 22 | sz
<< 30); // ldr(*) x(dst),[x(bas),x30]
330 static void arm64_ldrv(int sz_
, int dst
, int bas
, uint64_t off
)
333 if (!(off
& ~((uint32_t)0xfff << sz
)))
334 o(0x3d400000 | dst
| bas
<< 5 | off
<< (10 - sz
) |
335 (sz
& 4) << 21 | (sz
& 3) << 30); // ldr (s|d|q)(dst),[x(bas),#(off)]
336 else if (off
< 256 || -off
<= 256)
337 o(0x3c400000 | dst
| bas
<< 5 | (off
& 511) << 12 |
338 (sz
& 4) << 21 | (sz
& 3) << 30); // ldur (s|d|q)(dst),[x(bas),#(off)]
340 arm64_movimm(30, off
); // use x30 for offset
341 o(0x3c606800 | dst
| bas
<< 5 | (uint32_t)30 << 16 |
342 sz
<< 30 | (sz
& 4) << 21); // ldr (s|d|q)(dst),[x(bas),x30]
346 static void arm64_ldrs(int reg_
, int size
)
349 // Use x30 for intermediate value in some cases.
351 default: assert(0); break;
353 /* Can happen with zero size structs */
356 arm64_ldrx(0, 0, reg
, reg
, 0);
359 arm64_ldrx(0, 1, reg
, reg
, 0);
362 arm64_ldrx(0, 1, 30, reg
, 0);
363 arm64_ldrx(0, 0, reg
, reg
, 2);
364 o(0x2a0043c0 | reg
| reg
<< 16); // orr x(reg),x30,x(reg),lsl #16
367 arm64_ldrx(0, 2, reg
, reg
, 0);
370 arm64_ldrx(0, 2, 30, reg
, 0);
371 arm64_ldrx(0, 0, reg
, reg
, 4);
372 o(0xaa0083c0 | reg
| reg
<< 16); // orr x(reg),x30,x(reg),lsl #32
375 arm64_ldrx(0, 2, 30, reg
, 0);
376 arm64_ldrx(0, 1, reg
, reg
, 4);
377 o(0xaa0083c0 | reg
| reg
<< 16); // orr x(reg),x30,x(reg),lsl #32
380 arm64_ldrx(0, 2, 30, reg
, 0);
381 arm64_ldrx(0, 2, reg
, reg
, 3);
382 o(0x53087c00 | reg
| reg
<< 5); // lsr w(reg), w(reg), #8
383 o(0xaa0083c0 | reg
| reg
<< 16); // orr x(reg),x30,x(reg),lsl #32
386 arm64_ldrx(0, 3, reg
, reg
, 0);
389 arm64_ldrx(0, 0, reg
+ 1, reg
, 8);
390 arm64_ldrx(0, 3, reg
, reg
, 0);
393 arm64_ldrx(0, 1, reg
+ 1, reg
, 8);
394 arm64_ldrx(0, 3, reg
, reg
, 0);
397 arm64_ldrx(0, 2, reg
+ 1, reg
, 7);
398 o(0x53087c00 | (reg
+1) | (reg
+1) << 5); // lsr w(reg+1), w(reg+1), #8
399 arm64_ldrx(0, 3, reg
, reg
, 0);
402 arm64_ldrx(0, 2, reg
+ 1, reg
, 8);
403 arm64_ldrx(0, 3, reg
, reg
, 0);
406 arm64_ldrx(0, 3, reg
+ 1, reg
, 5);
407 o(0xd358fc00 | (reg
+1) | (reg
+1) << 5); // lsr x(reg+1), x(reg+1), #24
408 arm64_ldrx(0, 3, reg
, reg
, 0);
411 arm64_ldrx(0, 3, reg
+ 1, reg
, 6);
412 o(0xd350fc00 | (reg
+1) | (reg
+1) << 5); // lsr x(reg+1), x(reg+1), #16
413 arm64_ldrx(0, 3, reg
, reg
, 0);
416 arm64_ldrx(0, 3, reg
+ 1, reg
, 7);
417 o(0xd348fc00 | (reg
+1) | (reg
+1) << 5); // lsr x(reg+1), x(reg+1), #8
418 arm64_ldrx(0, 3, reg
, reg
, 0);
421 o(0xa9400000 | reg
| (reg
+1) << 10 | reg
<< 5);
422 // ldp x(reg),x(reg+1),[x(reg)]
427 static void arm64_strx(int sz_
, int dst
, int bas
, uint64_t off
)
430 if (!(off
& ~((uint32_t)0xfff << sz
)))
431 o(0x39000000 | dst
| bas
<< 5 | off
<< (10 - sz
) | sz
<< 30);
432 // str(*) x(dst),[x(bas],#(off)]
433 else if (off
< 256 || -off
<= 256)
434 o(0x38000000 | dst
| bas
<< 5 | (off
& 511) << 12 | sz
<< 30);
435 // stur(*) x(dst),[x(bas],#(off)]
437 arm64_movimm(30, off
); // use x30 for offset
438 o(0x38206800 | dst
| bas
<< 5 | (uint32_t)30 << 16 | sz
<< 30);
439 // str(*) x(dst),[x(bas),x30]
443 static void arm64_strv(int sz_
, int dst
, int bas
, uint64_t off
)
446 if (!(off
& ~((uint32_t)0xfff << sz
)))
447 o(0x3d000000 | dst
| bas
<< 5 | off
<< (10 - sz
) |
448 (sz
& 4) << 21 | (sz
& 3) << 30); // str (s|d|q)(dst),[x(bas),#(off)]
449 else if (off
< 256 || -off
<= 256)
450 o(0x3c000000 | dst
| bas
<< 5 | (off
& 511) << 12 |
451 (sz
& 4) << 21 | (sz
& 3) << 30); // stur (s|d|q)(dst),[x(bas),#(off)]
453 arm64_movimm(30, off
); // use x30 for offset
454 o(0x3c206800 | dst
| bas
<< 5 | (uint32_t)30 << 16 |
455 sz
<< 30 | (sz
& 4) << 21); // str (s|d|q)(dst),[x(bas),x30]
459 static void arm64_sym(int r
, Sym
*sym
, unsigned long addend
)
461 greloca(cur_text_section
, sym
, ind
, R_AARCH64_ADR_GOT_PAGE
, 0);
462 o(0x90000000 | r
); // adrp xr, #sym
463 greloca(cur_text_section
, sym
, ind
, R_AARCH64_LD64_GOT_LO12_NC
, 0);
464 o(0xf9400000 | r
| (r
<< 5)); // ld xr,[xr, #sym]
466 // add xr, xr, #addend
467 if (addend
& 0xffful
)
468 o(0x91000000 | r
| r
<< 5 | (addend
& 0xfff) << 10);
469 if (addend
> 0xffful
) {
470 // add xr, xr, #addend, lsl #12
471 if (addend
& 0xfff000ul
)
472 o(0x91400000 | r
| r
<< 5 | ((addend
>> 12) & 0xfff) << 10);
473 if (addend
> 0xfffffful
) {
476 o(0xf81f0fe0 | t
); /* str xt, [sp, #-16]! */
477 arm64_movimm(t
, addend
& ~0xfffffful
); // use xt for addent
478 o(0x91000000 | r
| (t
<< 5)); /* add xr, xt, #0 */
479 o(0xf84107e0 | t
); /* ldr xt, [sp], #16 */
485 static void arm64_load_cmp(int r
, SValue
*sv
);
487 ST_FUNC
void load(int r
, SValue
*sv
)
489 int svtt
= sv
->type
.t
;
490 int svr
= sv
->r
& ~VT_BOUNDED
;
491 int svrv
= svr
& VT_VALMASK
;
492 uint64_t svcul
= (uint32_t)sv
->c
.i
;
493 svcul
= svcul
>> 31 & 1 ? svcul
- ((uint64_t)1 << 32) : svcul
;
495 if (svr
== (VT_LOCAL
| VT_LVAL
)) {
497 arm64_ldrv(arm64_type_size(svtt
), fltr(r
), 29, svcul
);
499 arm64_ldrx(!(svtt
& VT_UNSIGNED
), arm64_type_size(svtt
),
504 if (svr
== (VT_CONST
| VT_LVAL
)) {
506 arm64_sym(30, sv
->sym
, // use x30 for address
507 arm64_check_offset(0, arm64_type_size(svtt
), sv
->c
.i
));
509 arm64_movimm (30, sv
->c
.i
);
511 arm64_ldrv(arm64_type_size(svtt
), fltr(r
), 30,
512 arm64_check_offset(1, arm64_type_size(svtt
), sv
->c
.i
));
514 arm64_ldrx(!(svtt
&VT_UNSIGNED
), arm64_type_size(svtt
), intr(r
), 30,
515 arm64_check_offset(1, arm64_type_size(svtt
), sv
->c
.i
));
519 if ((svr
& ~VT_VALMASK
) == VT_LVAL
&& svrv
< VT_CONST
) {
520 if ((svtt
& VT_BTYPE
) != VT_VOID
) {
522 arm64_ldrv(arm64_type_size(svtt
), fltr(r
), intr(svrv
), 0);
524 arm64_ldrx(!(svtt
& VT_UNSIGNED
), arm64_type_size(svtt
),
525 intr(r
), intr(svrv
), 0);
530 if (svr
== (VT_CONST
| VT_LVAL
| VT_SYM
)) {
531 arm64_sym(30, sv
->sym
, // use x30 for address
532 arm64_check_offset(0, arm64_type_size(svtt
), svcul
));
534 arm64_ldrv(arm64_type_size(svtt
), fltr(r
), 30,
535 arm64_check_offset(1, arm64_type_size(svtt
), svcul
));
537 arm64_ldrx(!(svtt
&VT_UNSIGNED
), arm64_type_size(svtt
), intr(r
), 30,
538 arm64_check_offset(1, arm64_type_size(svtt
), svcul
));
542 if (svr
== (VT_CONST
| VT_SYM
)) {
543 arm64_sym(intr(r
), sv
->sym
, svcul
);
547 if (svr
== VT_CONST
) {
548 if ((svtt
& VT_BTYPE
) != VT_VOID
)
549 arm64_movimm(intr(r
), arm64_type_size(svtt
) == 3 ?
550 sv
->c
.i
: (uint32_t)svcul
);
554 if (svr
< VT_CONST
) {
555 if (IS_FREG(r
) && IS_FREG(svr
))
556 if (svtt
== VT_LDOUBLE
)
557 o(0x4ea01c00 | fltr(r
) | fltr(svr
) << 5);
558 // mov v(r).16b,v(svr).16b
560 o(0x1e604000 | fltr(r
) | fltr(svr
) << 5); // fmov d(r),d(svr)
561 else if (!IS_FREG(r
) && !IS_FREG(svr
))
562 o(0xaa0003e0 | intr(r
) | intr(svr
) << 16); // mov x(r),x(svr)
568 if (svr
== VT_LOCAL
) {
570 o(0xd10003a0 | intr(r
) | -svcul
<< 10); // sub x(r),x29,#...
572 arm64_movimm(30, -svcul
); // use x30 for offset
573 o(0xcb0003a0 | intr(r
) | (uint32_t)30 << 16); // sub x(r),x29,x30
578 if (svr
== VT_JMP
|| svr
== VT_JMPI
) {
579 int t
= (svr
== VT_JMPI
);
580 arm64_movimm(intr(r
), t
);
581 o(0x14000002); // b .+8
583 arm64_movimm(intr(r
), t
^ 1);
587 if (svr
== (VT_LLOCAL
| VT_LVAL
)) {
588 arm64_ldrx(0, 3, 30, 29, svcul
); // use x30 for offset
590 arm64_ldrv(arm64_type_size(svtt
), fltr(r
), 30, 0);
592 arm64_ldrx(!(svtt
& VT_UNSIGNED
), arm64_type_size(svtt
),
598 arm64_load_cmp(r
, sv
);
602 printf("load(%x, (%x, %x, %lx))\n", r
, svtt
, sv
->r
, (long)svcul
);
606 ST_FUNC
void store(int r
, SValue
*sv
)
608 int svtt
= sv
->type
.t
;
609 int svr
= sv
->r
& ~VT_BOUNDED
;
610 int svrv
= svr
& VT_VALMASK
;
611 uint64_t svcul
= (uint32_t)sv
->c
.i
;
612 svcul
= svcul
>> 31 & 1 ? svcul
- ((uint64_t)1 << 32) : svcul
;
614 if (svr
== (VT_LOCAL
| VT_LVAL
)) {
616 arm64_strv(arm64_type_size(svtt
), fltr(r
), 29, svcul
);
618 arm64_strx(arm64_type_size(svtt
), intr(r
), 29, svcul
);
622 if (svr
== (VT_CONST
| VT_LVAL
)) {
624 arm64_sym(30, sv
->sym
, // use x30 for address
625 arm64_check_offset(0, arm64_type_size(svtt
), sv
->c
.i
));
627 arm64_movimm (30, sv
->c
.i
);
629 arm64_strv(arm64_type_size(svtt
), fltr(r
), 30,
630 arm64_check_offset(1, arm64_type_size(svtt
), sv
->c
.i
));
632 arm64_strx(arm64_type_size(svtt
), intr(r
), 30,
633 arm64_check_offset(1, arm64_type_size(svtt
), sv
->c
.i
));
637 if ((svr
& ~VT_VALMASK
) == VT_LVAL
&& svrv
< VT_CONST
) {
639 arm64_strv(arm64_type_size(svtt
), fltr(r
), intr(svrv
), 0);
641 arm64_strx(arm64_type_size(svtt
), intr(r
), intr(svrv
), 0);
645 if (svr
== (VT_CONST
| VT_LVAL
| VT_SYM
)) {
646 arm64_sym(30, sv
->sym
, // use x30 for address
647 arm64_check_offset(0, arm64_type_size(svtt
), svcul
));
649 arm64_strv(arm64_type_size(svtt
), fltr(r
), 30,
650 arm64_check_offset(1, arm64_type_size(svtt
), svcul
));
652 arm64_strx(arm64_type_size(svtt
), intr(r
), 30,
653 arm64_check_offset(1, arm64_type_size(svtt
), svcul
));
657 printf("store(%x, (%x, %x, %lx))\n", r
, svtt
, sv
->r
, (long)svcul
);
661 static void arm64_gen_bl_or_b(int b
)
663 if ((vtop
->r
& (VT_VALMASK
| VT_LVAL
)) == VT_CONST
&& (vtop
->r
& VT_SYM
)) {
664 greloca(cur_text_section
, vtop
->sym
, ind
,
665 b
? R_AARCH64_JUMP26
: R_AARCH64_CALL26
, 0);
666 o(0x14000000 | (uint32_t)!b
<< 31); // b/bl .
669 #ifdef CONFIG_TCC_BCHECK
670 vtop
->r
&= ~VT_MUSTBOUND
;
672 o(0xd61f0000 | (uint32_t)!b
<< 21 | intr(gv(RC_R30
)) << 5); // br/blr
676 #if defined(CONFIG_TCC_BCHECK)
678 static void gen_bounds_call(int v
)
680 Sym
*sym
= external_helper_sym(v
);
682 greloca(cur_text_section
, sym
, ind
, R_AARCH64_CALL26
, 0);
686 static void gen_bounds_prolog(void)
688 /* leave some room for bound checking code */
689 func_bound_offset
= lbounds_section
->data_offset
;
690 func_bound_ind
= ind
;
691 func_bound_add_epilog
= 0;
692 o(0xd503201f); /* nop -> mov x0, lbound section pointer */
695 o(0xd503201f); /* nop -> call __bound_local_new */
698 static void gen_bounds_epilog(void)
703 int offset_modified
= func_bound_offset
!= lbounds_section
->data_offset
;
705 if (!offset_modified
&& !func_bound_add_epilog
)
708 /* add end of table info */
709 bounds_ptr
= section_ptr_add(lbounds_section
, sizeof(addr_t
));
712 sym_data
= get_sym_ref(&char_pointer_type
, lbounds_section
,
713 func_bound_offset
, lbounds_section
->data_offset
);
715 /* generate bound local allocation */
716 if (offset_modified
) {
718 ind
= func_bound_ind
;
719 greloca(cur_text_section
, sym_data
, ind
, R_AARCH64_ADR_GOT_PAGE
, 0);
720 o(0x90000000 | 0); // adrp x0, #sym_data
721 greloca(cur_text_section
, sym_data
, ind
, R_AARCH64_LD64_GOT_LO12_NC
, 0);
722 o(0xf9400000 | 0 | (0 << 5)); // ld x0,[x0, #sym_data]
723 gen_bounds_call(TOK___bound_local_new
);
727 /* generate bound check local freeing */
728 o(0xf81f0fe0); /* str x0, [sp, #-16]! */
729 o(0x3c9f0fe0); /* str q0, [sp, #-16]! */
730 greloca(cur_text_section
, sym_data
, ind
, R_AARCH64_ADR_GOT_PAGE
, 0);
731 o(0x90000000 | 0); // adrp x0, #sym_data
732 greloca(cur_text_section
, sym_data
, ind
, R_AARCH64_LD64_GOT_LO12_NC
, 0);
733 o(0xf9400000 | 0 | (0 << 5)); // ld x0,[x0, #sym_data]
734 gen_bounds_call(TOK___bound_local_delete
);
735 o(0x3cc107e0); /* ldr q0, [sp], #16 */
736 o(0xf84107e0); /* ldr x0, [sp], #16 */
740 static int arm64_hfa_aux(CType
*type
, int *fsize
, int num
)
742 if (is_float(type
->t
)) {
743 int a
, n
= type_size(type
, &a
);
744 if (num
>= 4 || (*fsize
&& *fsize
!= n
))
749 else if ((type
->t
& VT_BTYPE
) == VT_STRUCT
) {
750 int is_struct
= 0; // rather than union
752 for (field
= type
->ref
->next
; field
; field
= field
->next
)
759 for (field
= type
->ref
->next
; field
; field
= field
->next
) {
760 if (field
->c
!= (num
- num0
) * *fsize
)
762 num
= arm64_hfa_aux(&field
->type
, fsize
, num
);
766 if (type
->ref
->c
!= (num
- num0
) * *fsize
)
772 for (field
= type
->ref
->next
; field
; field
= field
->next
) {
773 int num1
= arm64_hfa_aux(&field
->type
, fsize
, num0
);
776 num
= num1
< num
? num
: num1
;
778 if (type
->ref
->c
!= (num
- num0
) * *fsize
)
783 else if ((type
->t
& VT_ARRAY
) && ((type
->t
& VT_BTYPE
) != VT_PTR
)) {
787 num1
= arm64_hfa_aux(&type
->ref
->type
, fsize
, num
);
788 if (num1
== -1 || (num1
!= num
&& type
->ref
->c
> 4))
790 num1
= num
+ type
->ref
->c
* (num1
- num
);
798 static int arm64_hfa(CType
*type
, unsigned *fsize
)
800 if ((type
->t
& VT_BTYPE
) == VT_STRUCT
||
801 ((type
->t
& VT_ARRAY
) && ((type
->t
& VT_BTYPE
) != VT_PTR
))) {
803 int n
= arm64_hfa_aux(type
, &sz
, 0);
804 if (0 < n
&& n
<= 4) {
813 static unsigned long arm64_pcs_aux(int n
, CType
**type
, unsigned long *a
)
815 int nx
= 0; // next integer register
816 int nv
= 0; // next vector register
817 unsigned long ns
= 32; // next stack offset
820 for (i
= 0; i
< n
; i
++) {
821 int hfa
= arm64_hfa(type
[i
], 0);
824 if ((type
[i
]->t
& VT_ARRAY
) ||
825 (type
[i
]->t
& VT_BTYPE
) == VT_FUNC
)
828 size
= type_size(type
[i
], &align
);
833 else if (size
> 16) {
834 // B.3: replace with pointer
836 a
[i
] = nx
++ << 1 | 1;
844 else if ((type
[i
]->t
& VT_BTYPE
) == VT_STRUCT
)
846 size
= (size
+ 7) & ~7;
849 if (is_float(type
[i
]->t
) && nv
< 8) {
850 a
[i
] = 16 + (nv
++ << 1);
855 if (hfa
&& nv
+ hfa
<= 8) {
856 a
[i
] = 16 + (nv
<< 1);
864 size
= (size
+ 7) & ~7;
868 if (hfa
|| (type
[i
]->t
& VT_BTYPE
) == VT_LDOUBLE
) {
870 ns
= (ns
+ align
- 1) & -align
;
874 if ((type
[i
]->t
& VT_BTYPE
) == VT_FLOAT
)
878 if (hfa
|| is_float(type
[i
]->t
)) {
885 if ((type
[i
]->t
& VT_BTYPE
) != VT_STRUCT
&& size
<= 8 && nx
< 8) {
895 if ((type
[i
]->t
& VT_BTYPE
) != VT_STRUCT
&& size
== 16 && nx
< 7) {
902 if ((type
[i
]->t
& VT_BTYPE
) == VT_STRUCT
&& size
<= (8 - nx
) * 8) {
904 nx
+= (size
+ 7) >> 3;
913 ns
= (ns
+ align
- 1) & -align
;
916 if ((type
[i
]->t
& VT_BTYPE
) == VT_STRUCT
) {
934 static unsigned long arm64_pcs(int n
, CType
**type
, unsigned long *a
)
939 if ((type
[0]->t
& VT_BTYPE
) == VT_VOID
)
942 arm64_pcs_aux(1, type
, a
);
943 assert(a
[0] == 0 || a
[0] == 1 || a
[0] == 16);
947 stack
= arm64_pcs_aux(n
, type
+ 1, a
+ 1);
951 for (i
= 0; i
<= n
; i
++) {
953 printf("arm64_pcs return: ");
955 printf("arm64_pcs arg %d: ", i
);
956 if (a
[i
] == (unsigned long)-1)
958 else if (a
[i
] == 1 && !i
)
959 printf("X8 pointer\n");
961 printf("X%lu%s\n", a
[i
] / 2, a
[i
] & 1 ? " pointer" : "");
963 printf("V%lu\n", a
[i
] / 2 - 8);
965 printf("stack %lu%s\n",
966 (a
[i
] - 32) & ~1, a
[i
] & 1 ? " pointer" : "");
973 ST_FUNC
void gfunc_call(int nb_args
)
977 unsigned long *a
, *a1
;
981 #ifdef CONFIG_TCC_BCHECK
982 if (tcc_state
->do_bounds_check
)
983 gbound_args(nb_args
);
986 return_type
= &vtop
[-nb_args
].type
.ref
->type
;
987 if ((return_type
->t
& VT_BTYPE
) == VT_STRUCT
)
990 t
= tcc_malloc((nb_args
+ 1) * sizeof(*t
));
991 a
= tcc_malloc((nb_args
+ 1) * sizeof(*a
));
992 a1
= tcc_malloc((nb_args
+ 1) * sizeof(*a1
));
995 for (i
= 0; i
< nb_args
; i
++)
996 t
[nb_args
- i
] = &vtop
[-i
].type
;
998 stack
= arm64_pcs(nb_args
, t
, a
);
1000 // Allocate space for structs replaced by pointer:
1001 for (i
= nb_args
; i
; i
--)
1003 SValue
*arg
= &vtop
[i
- nb_args
];
1004 int align
, size
= type_size(&arg
->type
, &align
);
1005 assert((arg
->type
.t
& VT_BTYPE
) == VT_STRUCT
);
1006 stack
= (stack
+ align
- 1) & -align
;
1011 stack
= (stack
+ 15) >> 4 << 4;
1013 /* fetch cpu flag before generating any code */
1014 if ((vtop
->r
& VT_VALMASK
) == VT_CMP
)
1017 if (stack
>= 0x1000000) // 16Mb
1018 tcc_error("stack size too big %lu", stack
);
1020 o(0xd10003ff | (stack
& 0xfff) << 10); // sub sp,sp,#(n)
1022 o(0xd14003ff | (stack
>> 12) << 10);
1024 // First pass: set all values on stack
1025 for (i
= nb_args
; i
; i
--) {
1026 vpushv(vtop
- nb_args
+ i
);
1029 // struct replaced by pointer
1030 int r
= get_reg(RC_INT
);
1031 arm64_spoff(intr(r
), a1
[i
]);
1032 vset(&vtop
->type
, r
| VT_LVAL
, 0);
1037 r
= get_reg(RC_INT
);
1038 arm64_spoff(intr(r
), a1
[i
]);
1039 arm64_strx(3, intr(r
), 31, (a
[i
] - 32) >> 1 << 1);
1042 else if (a
[i
] >= 32) {
1044 if ((vtop
->type
.t
& VT_BTYPE
) == VT_STRUCT
) {
1045 int r
= get_reg(RC_INT
);
1046 arm64_spoff(intr(r
), a
[i
] - 32);
1047 vset(&vtop
->type
, r
| VT_LVAL
, 0);
1051 else if (is_float(vtop
->type
.t
)) {
1053 arm64_strv(arm64_type_size(vtop
[0].type
.t
),
1054 fltr(vtop
[0].r
), 31, a
[i
] - 32);
1058 arm64_strx(arm64_type_size(vtop
[0].type
.t
),
1059 intr(vtop
[0].r
), 31, a
[i
] - 32);
1066 // Second pass: assign values to registers
1067 for (i
= nb_args
; i
; i
--, vtop
--) {
1068 if (a
[i
] < 16 && !(a
[i
] & 1)) {
1069 // value in general-purpose registers
1070 if ((vtop
->type
.t
& VT_BTYPE
) == VT_STRUCT
) {
1071 int align
, size
= type_size(&vtop
->type
, &align
);
1073 vtop
->type
.t
= VT_PTR
;
1076 arm64_ldrs(a
[i
] / 2, size
);
1083 // struct replaced by pointer in register
1084 arm64_spoff(a
[i
] / 2, a1
[i
]);
1085 else if (a
[i
] < 32) {
1086 // value in floating-point registers
1087 if ((vtop
->type
.t
& VT_BTYPE
) == VT_STRUCT
) {
1088 uint32_t j
, sz
, n
= arm64_hfa(&vtop
->type
, &sz
);
1089 vtop
->type
.t
= VT_PTR
;
1092 for (j
= 0; j
< n
; j
++)
1094 (sz
& 16) << 19 | -(sz
& 8) << 27 | (sz
& 4) << 29 |
1095 (a
[i
] / 2 - 8 + j
) |
1096 j
<< 10); // ldr ([sdq])(*),[x30,#(j * sz)]
1099 gv(RC_F(a
[i
] / 2 - 8));
1103 if ((return_type
->t
& VT_BTYPE
) == VT_STRUCT
) {
1105 // indirect return: set x8 and discard the stack value
1110 // return in registers: keep the address for after the call
1115 arm64_gen_bl_or_b(0);
1118 o(0x910003ff | (stack
& 0xfff) << 10); // add sp,sp,#(n)
1120 o(0x914003ff | (stack
>> 12) << 10);
1123 int rt
= return_type
->t
;
1124 int bt
= rt
& VT_BTYPE
;
1125 if (bt
== VT_STRUCT
&& !(a
[0] & 1)) {
1126 // A struct was returned in registers, so write it out:
1130 int align
, size
= type_size(return_type
, &align
);
1133 o(0xa9000500); // stp x0,x1,[x8]
1135 arm64_strx(size
> 4 ? 3 : size
> 2 ? 2 : size
> 1, 0, 8, 0);
1138 else if (a
[0] == 16) {
1139 uint32_t j
, sz
, n
= arm64_hfa(return_type
, &sz
);
1140 for (j
= 0; j
< n
; j
++)
1142 (sz
& 16) << 19 | -(sz
& 8) << 27 | (sz
& 4) << 29 |
1143 (a
[i
] / 2 - 8 + j
) |
1144 j
<< 10); // str ([sdq])(*),[x8,#(j * sz)]
1154 static unsigned long arm64_func_va_list_stack
;
1155 static int arm64_func_va_list_gr_offs
;
1156 static int arm64_func_va_list_vr_offs
;
1157 static int arm64_func_sub_sp_offset
;
1159 ST_FUNC
void gfunc_prolog(Sym
*func_sym
)
1161 CType
*func_type
= &func_sym
->type
;
1168 func_vc
= 144; // offset of where x8 is stored
1170 for (sym
= func_type
->ref
; sym
; sym
= sym
->next
)
1172 t
= n
? tcc_malloc(n
* sizeof(*t
)) : NULL
;
1173 a
= n
? tcc_malloc(n
* sizeof(*a
)) : NULL
;
1175 for (sym
= func_type
->ref
; sym
; sym
= sym
->next
)
1176 t
[i
++] = &sym
->type
;
1178 arm64_func_va_list_stack
= arm64_pcs(n
- 1, t
, a
);
1180 o(0xa9b27bfd); // stp x29,x30,[sp,#-224]!
1181 o(0xad0087e0); // stp q0,q1,[sp,#16]
1182 o(0xad018fe2); // stp q2,q3,[sp,#48]
1183 o(0xad0297e4); // stp q4,q5,[sp,#80]
1184 o(0xad039fe6); // stp q6,q7,[sp,#112]
1185 o(0xa90923e8); // stp x8,x8,[sp,#144]
1186 o(0xa90a07e0); // stp x0,x1,[sp,#160]
1187 o(0xa90b0fe2); // stp x2,x3,[sp,#176]
1188 o(0xa90c17e4); // stp x4,x5,[sp,#192]
1189 o(0xa90d1fe6); // stp x6,x7,[sp,#208]
1191 arm64_func_va_list_gr_offs
= -64;
1192 arm64_func_va_list_vr_offs
= -128;
1194 for (i
= 1, sym
= func_type
->ref
->next
; sym
; i
++, sym
= sym
->next
) {
1195 int off
= (a
[i
] < 16 ? 160 + a
[i
] / 2 * 8 :
1196 a
[i
] < 32 ? 16 + (a
[i
] - 16) / 2 * 16 :
1197 224 + ((a
[i
] - 32) >> 1 << 1));
1198 sym_push(sym
->v
& ~SYM_FIELD
, &sym
->type
,
1199 (a
[i
] & 1 ? VT_LLOCAL
: VT_LOCAL
) | VT_LVAL
,
1203 int align
, size
= type_size(&sym
->type
, &align
);
1204 arm64_func_va_list_gr_offs
= (a
[i
] / 2 - 7 +
1205 (!(a
[i
] & 1) && size
> 8)) * 8;
1207 else if (a
[i
] < 32) {
1208 uint32_t hfa
= arm64_hfa(&sym
->type
, 0);
1209 arm64_func_va_list_vr_offs
= (a
[i
] / 2 - 16 +
1210 (hfa
? hfa
: 1)) * 16;
1213 // HFAs of float and double need to be written differently:
1214 if (16 <= a
[i
] && a
[i
] < 32 && (sym
->type
.t
& VT_BTYPE
) == VT_STRUCT
) {
1215 uint32_t j
, sz
, k
= arm64_hfa(&sym
->type
, &sz
);
1217 for (j
= 0; j
< k
; j
++) {
1218 o(0x3d0003e0 | -(sz
& 8) << 27 | (sz
& 4) << 29 |
1219 ((a
[i
] - 16) / 2 + j
) | (off
/ sz
+ j
) << 10);
1220 // str ([sdq])(*),[sp,#(j * sz)]
1228 o(0x910003fd); // mov x29,sp
1229 arm64_func_sub_sp_offset
= ind
;
1230 // In gfunc_epilog these will be replaced with code to decrement SP:
1231 o(0xd503201f); // nop
1232 o(0xd503201f); // nop
1234 #ifdef CONFIG_TCC_BCHECK
1235 if (tcc_state
->do_bounds_check
)
1236 gen_bounds_prolog();
1240 ST_FUNC
void gen_va_start(void)
1243 --vtop
; // we don't need the "arg"
1245 r
= intr(gv(RC_INT
));
1247 if (arm64_func_va_list_stack
) {
1248 //xx could use add (immediate) here
1249 arm64_movimm(30, arm64_func_va_list_stack
+ 224);
1250 o(0x8b1e03be); // add x30,x29,x30
1253 o(0x910383be); // add x30,x29,#224
1254 o(0xf900001e | r
<< 5); // str x30,[x(r)]
1256 if (arm64_func_va_list_gr_offs
) {
1257 if (arm64_func_va_list_stack
)
1258 o(0x910383be); // add x30,x29,#224
1259 o(0xf900041e | r
<< 5); // str x30,[x(r),#8]
1262 if (arm64_func_va_list_vr_offs
) {
1263 o(0x910243be); // add x30,x29,#144
1264 o(0xf900081e | r
<< 5); // str x30,[x(r),#16]
1267 arm64_movimm(30, arm64_func_va_list_gr_offs
);
1268 o(0xb900181e | r
<< 5); // str w30,[x(r),#24]
1270 arm64_movimm(30, arm64_func_va_list_vr_offs
);
1271 o(0xb9001c1e | r
<< 5); // str w30,[x(r),#28]
1276 ST_FUNC
void gen_va_arg(CType
*t
)
1278 int align
, size
= type_size(t
, &align
);
1279 unsigned fsize
, hfa
= arm64_hfa(t
, &fsize
);
1282 if (is_float(t
->t
)) {
1288 r0
= intr(gv(RC_INT
));
1289 r1
= get_reg(RC_INT
);
1290 vtop
[0].r
= r1
| VT_LVAL
;
1294 uint32_t n
= size
> 16 ? 8 : (size
+ 7) & -8;
1295 o(0xb940181e | r0
<< 5); // ldr w30,[x(r0),#24] // __gr_offs
1297 assert(0); // this path untested but needed for __uint128_t
1298 o(0x11003fde); // add w30,w30,#15
1299 o(0x121c6fde); // and w30,w30,#-16
1301 o(0x310003c0 | r1
| n
<< 10); // adds w(r1),w30,#(n)
1302 o(0x540000ad); // b.le .+20
1303 o(0xf9400000 | r1
| r0
<< 5); // ldr x(r1),[x(r0)] // __stack
1304 o(0x9100001e | r1
<< 5 | n
<< 10); // add x30,x(r1),#(n)
1305 o(0xf900001e | r0
<< 5); // str x30,[x(r0)] // __stack
1306 o(0x14000004); // b .+16
1307 o(0xb9001800 | r1
| r0
<< 5); // str w(r1),[x(r0),#24] // __gr_offs
1308 o(0xf9400400 | r1
| r0
<< 5); // ldr x(r1),[x(r0),#8] // __gr_top
1309 o(0x8b3ec000 | r1
| r1
<< 5); // add x(r1),x(r1),w30,sxtw
1311 o(0xf9400000 | r1
| r1
<< 5); // ldr x(r1),[x(r1)]
1314 uint32_t rsz
= hfa
<< 4;
1315 uint32_t ssz
= (size
+ 7) & -(uint32_t)8;
1317 o(0xb9401c1e | r0
<< 5); // ldr w30,[x(r0),#28] // __vr_offs
1318 o(0x310003c0 | r1
| rsz
<< 10); // adds w(r1),w30,#(rsz)
1319 b1
= ind
; o(0x5400000d); // b.le lab1
1320 o(0xf9400000 | r1
| r0
<< 5); // ldr x(r1),[x(r0)] // __stack
1322 o(0x91003c00 | r1
| r1
<< 5); // add x(r1),x(r1),#15
1323 o(0x927cec00 | r1
| r1
<< 5); // and x(r1),x(r1),#-16
1325 o(0x9100001e | r1
<< 5 | ssz
<< 10); // add x30,x(r1),#(ssz)
1326 o(0xf900001e | r0
<< 5); // str x30,[x(r0)] // __stack
1327 b2
= ind
; o(0x14000000); // b lab2
1329 write32le(cur_text_section
->data
+ b1
, 0x5400000d | (ind
- b1
) << 3);
1330 o(0xb9001c00 | r1
| r0
<< 5); // str w(r1),[x(r0),#28] // __vr_offs
1331 o(0xf9400800 | r1
| r0
<< 5); // ldr x(r1),[x(r0),#16] // __vr_top
1332 if (hfa
== 1 || fsize
== 16)
1333 o(0x8b3ec000 | r1
| r1
<< 5); // add x(r1),x(r1),w30,sxtw
1335 // We need to change the layout of this HFA.
1336 // Get some space on the stack using global variable "loc":
1337 loc
= (loc
- size
) & -(uint32_t)align
;
1338 o(0x8b3ec000 | 30 | r1
<< 5); // add x30,x(r1),w30,sxtw
1339 arm64_movimm(r1
, loc
);
1340 o(0x8b0003a0 | r1
| r1
<< 16); // add x(r1),x29,x(r1)
1341 o(0x4c402bdc | (uint32_t)fsize
<< 7 |
1342 (uint32_t)(hfa
== 2) << 15 |
1343 (uint32_t)(hfa
== 3) << 14); // ld1 {v28.(4s|2d),...},[x30]
1344 o(0x0d00801c | r1
<< 5 | (fsize
== 8) << 10 |
1345 (uint32_t)(hfa
!= 2) << 13 |
1346 (uint32_t)(hfa
!= 3) << 21); // st(hfa) {v28.(s|d),...}[0],[x(r1)]
1349 write32le(cur_text_section
->data
+ b2
, 0x14000000 | (ind
- b2
) >> 2);
1353 ST_FUNC
int gfunc_sret(CType
*vt
, int variadic
, CType
*ret
,
1354 int *align
, int *regsize
)
1359 ST_FUNC
void gfunc_return(CType
*func_type
)
1361 CType
*t
= func_type
;
1364 arm64_pcs(0, &t
, &a
);
1369 if ((func_type
->t
& VT_BTYPE
) == VT_STRUCT
) {
1370 int align
, size
= type_size(func_type
, &align
);
1373 arm64_ldrs(0, size
);
1379 CType type
= *func_type
;
1381 vset(&type
, VT_LOCAL
| VT_LVAL
, func_vc
);
1388 if ((func_type
->t
& VT_BTYPE
) == VT_STRUCT
) {
1389 uint32_t j
, sz
, n
= arm64_hfa(&vtop
->type
, &sz
);
1392 for (j
= 0; j
< n
; j
++)
1394 (sz
& 16) << 19 | -(sz
& 8) << 27 | (sz
& 4) << 29 |
1395 j
| j
<< 10); // ldr ([sdq])(*),[x0,#(j * sz)]
1406 ST_FUNC
void gfunc_epilog(void)
1408 #ifdef CONFIG_TCC_BCHECK
1409 if (tcc_state
->do_bounds_check
)
1410 gen_bounds_epilog();
1414 // Insert instructions to subtract size of stack frame from SP.
1415 unsigned char *ptr
= cur_text_section
->data
+ arm64_func_sub_sp_offset
;
1416 uint64_t diff
= (-loc
+ 15) & ~15;
1417 if (!(diff
>> 24)) {
1418 if (diff
& 0xfff) // sub sp,sp,#(diff & 0xfff)
1419 write32le(ptr
, 0xd10003ff | (diff
& 0xfff) << 10);
1420 if (diff
>> 12) // sub sp,sp,#(diff >> 12),lsl #12
1421 write32le(ptr
+ 4, 0xd14003ff | (diff
>> 12) << 10);
1424 // In this case we may subtract more than necessary,
1425 // but always less than 17/16 of what we were aiming for.
1428 while (diff
>> 20) {
1429 diff
= (diff
+ 0xffff) >> 16;
1432 while (diff
>> 16) {
1433 diff
= (diff
+ 1) >> 1;
1436 write32le(ptr
, 0xd2800010 | diff
<< 5 | i
<< 21);
1437 // mov x16,#(diff),lsl #(16 * i)
1438 write32le(ptr
+ 4, 0xcb3063ff | j
<< 10);
1439 // sub sp,sp,x16,lsl #(j)
1442 o(0x910003bf); // mov sp,x29
1443 o(0xa8ce7bfd); // ldp x29,x30,[sp],#224
1445 o(0xd65f03c0); // ret
1448 ST_FUNC
void gen_fill_nops(int bytes
)
1451 tcc_error("alignment of code section not multiple of 4");
1453 o(0xd503201f); // nop
1458 // Generate forward branch to label:
1459 ST_FUNC
int gjmp(int t
)
1468 // Generate branch to known address:
1469 ST_FUNC
void gjmp_addr(int a
)
1471 assert(a
- ind
+ 0x8000000 < 0x10000000);
1472 o(0x14000000 | ((a
- ind
) >> 2 & 0x3ffffff));
1475 ST_FUNC
int gjmp_append(int n
, int t
)
1478 /* insert vtop->c jump list in t */
1480 uint32_t n1
= n
, n2
;
1481 while ((n2
= read32le(p
= cur_text_section
->data
+ n1
)))
1489 void arm64_vset_VT_CMP(int op
)
1491 if (op
>= TOK_ULT
&& op
<= TOK_GT
) {
1492 vtop
->cmp_r
= vtop
->r
;
1497 static void arm64_gen_opil(int op
, uint32_t l
);
1499 static void arm64_load_cmp(int r
, SValue
*sv
)
1504 arm64_gen_opil('^', 0);
1512 ST_FUNC
int gjmp_cond(int op
, int t
)
1514 int bt
= vtop
->type
.t
& VT_BTYPE
;
1517 vtop
->r
= vtop
->cmp_r
;
1519 if (bt
== VT_LDOUBLE
) {
1520 uint32_t a
, b
, f
= fltr(gv(RC_FLOAT
));
1521 a
= get_reg(RC_INT
);
1524 b
= get_reg(RC_INT
);
1527 o(0x4e083c00 | a
| f
<< 5); // mov x(a),v(f).d[0]
1528 o(0x4e183c00 | b
| f
<< 5); // mov x(b),v(f).d[1]
1529 o(0xaa000400 | a
| a
<< 5 | b
<< 16); // orr x(a),x(a),x(b),lsl #1
1530 o(0xb4000040 | a
| !!inv
<< 24); // cbz/cbnz x(a),.+8
1533 else if (bt
== VT_FLOAT
|| bt
== VT_DOUBLE
) {
1534 uint32_t a
= fltr(gv(RC_FLOAT
));
1535 o(0x1e202008 | a
<< 5 | (bt
!= VT_FLOAT
) << 22); // fcmp
1536 o(0x54000040 | !!inv
); // b.eq/b.ne .+8
1539 uint32_t ll
= (bt
== VT_PTR
|| bt
== VT_LLONG
);
1540 uint32_t a
= intr(gv(RC_INT
));
1541 o(0x34000040 | a
| !!inv
<< 24 | ll
<< 31); // cbz/cbnz wA,.+8
1546 static int arm64_iconst(uint64_t *val
, SValue
*sv
)
1548 if ((sv
->r
& (VT_VALMASK
| VT_LVAL
| VT_SYM
)) != VT_CONST
)
1552 int bt
= t
& VT_BTYPE
;
1553 *val
= ((bt
== VT_LLONG
|| bt
== VT_PTR
) ? sv
->c
.i
:
1555 (t
& VT_UNSIGNED
? 0 : -(sv
->c
.i
& 0x80000000)));
1560 static int arm64_gen_opic(int op
, uint32_t l
, int rev
, uint64_t val
,
1561 uint32_t x
, uint32_t a
)
1563 if (op
== '-' && !rev
) {
1567 val
= l
? val
: (uint32_t)val
;
1572 uint32_t s
= l
? val
>> 63 : val
>> 31;
1573 val
= s
? -val
: val
;
1574 val
= l
? val
: (uint32_t)val
;
1575 if (!(val
& ~(uint64_t)0xfff))
1576 o(0x11000000 | l
<< 31 | s
<< 30 | x
| a
<< 5 | val
<< 10);
1577 else if (!(val
& ~(uint64_t)0xfff000))
1578 o(0x11400000 | l
<< 31 | s
<< 30 | x
| a
<< 5 | val
>> 12 << 10);
1580 arm64_movimm(30, val
); // use x30
1581 o(0x0b1e0000 | l
<< 31 | s
<< 30 | x
| a
<< 5);
1588 o(0x4b0003e0 | l
<< 31 | x
| a
<< 16); // neg
1589 else if (val
== (l
? (uint64_t)-1 : (uint32_t)-1))
1590 o(0x2a2003e0 | l
<< 31 | x
| a
<< 16); // mvn
1592 arm64_movimm(30, val
); // use x30
1593 o(0x4b0003c0 | l
<< 31 | x
| a
<< 16); // sub
1598 if (val
== -1 || (val
== 0xffffffff && !l
)) {
1599 o(0x2a2003e0 | l
<< 31 | x
| a
<< 16); // mvn
1605 int e
= arm64_encode_bimm64(l
? val
: val
| val
<< 32);
1608 o((op
== '&' ? 0x12000000 :
1609 op
== '|' ? 0x32000000 : 0x52000000) |
1610 l
<< 31 | x
| a
<< 5 | (uint32_t)e
<< 10);
1617 uint32_t n
= 32 << l
;
1618 val
= val
& (n
- 1);
1622 // tcc_warning("shift count >= width of type");
1623 o(0x2a0003e0 | l
<< 31 | a
<< 16);
1626 else if (op
== TOK_SHL
)
1627 o(0x53000000 | l
<< 31 | l
<< 22 | x
| a
<< 5 |
1628 (n
- val
) << 16 | (n
- 1 - val
) << 10); // lsl
1630 o(0x13000000 | (op
== TOK_SHR
) << 30 | l
<< 31 | l
<< 22 |
1631 x
| a
<< 5 | val
<< 16 | (n
- 1) << 10); // lsr/asr
1639 static void arm64_gen_opil(int op
, uint32_t l
)
1643 // Special treatment for operations with a constant operand:
1648 if (arm64_iconst(0, &vtop
[0])) {
1652 if (arm64_iconst(&val
, &vtop
[-1])) {
1654 a
= intr(vtop
[0].r
);
1656 x
= get_reg(RC_INT
);
1658 if (arm64_gen_opic(op
, l
, rev
, val
, intr(x
), a
)) {
1669 gv2(RC_INT
, RC_INT
);
1670 assert(vtop
[-1].r
< VT_CONST
&& vtop
[0].r
< VT_CONST
);
1671 a
= intr(vtop
[-1].r
);
1672 b
= intr(vtop
[0].r
);
1674 x
= get_reg(RC_INT
);
1681 // Use x30 for quotient:
1682 o(0x1ac00c00 | l
<< 31 | 30 | a
<< 5 | b
<< 16); // sdiv
1683 o(0x1b008000 | l
<< 31 | x
| (uint32_t)30 << 5 |
1684 b
<< 16 | a
<< 10); // msub
1687 o(0x0a000000 | l
<< 31 | x
| a
<< 5 | b
<< 16); // and
1690 o(0x1b007c00 | l
<< 31 | x
| a
<< 5 | b
<< 16); // mul
1693 o(0x0b000000 | l
<< 31 | x
| a
<< 5 | b
<< 16); // add
1696 o(0x4b000000 | l
<< 31 | x
| a
<< 5 | b
<< 16); // sub
1699 o(0x1ac00c00 | l
<< 31 | x
| a
<< 5 | b
<< 16); // sdiv
1702 o(0x4a000000 | l
<< 31 | x
| a
<< 5 | b
<< 16); // eor
1705 o(0x2a000000 | l
<< 31 | x
| a
<< 5 | b
<< 16); // orr
1708 o(0x6b00001f | l
<< 31 | a
<< 5 | b
<< 16); // cmp
1709 o(0x1a9f17e0 | x
); // cset wA,eq
1712 o(0x6b00001f | l
<< 31 | a
<< 5 | b
<< 16); // cmp
1713 o(0x1a9fb7e0 | x
); // cset wA,ge
1716 o(0x6b00001f | l
<< 31 | a
<< 5 | b
<< 16); // cmp
1717 o(0x1a9fd7e0 | x
); // cset wA,gt
1720 o(0x6b00001f | l
<< 31 | a
<< 5 | b
<< 16); // cmp
1721 o(0x1a9fc7e0 | x
); // cset wA,le
1724 o(0x6b00001f | l
<< 31 | a
<< 5 | b
<< 16); // cmp
1725 o(0x1a9fa7e0 | x
); // cset wA,lt
1728 o(0x6b00001f | l
<< 31 | a
<< 5 | b
<< 16); // cmp
1729 o(0x1a9f07e0 | x
); // cset wA,ne
1732 o(0x1ac02800 | l
<< 31 | x
| a
<< 5 | b
<< 16); // asr
1735 o(0x1ac02000 | l
<< 31 | x
| a
<< 5 | b
<< 16); // lsl
1738 o(0x1ac02400 | l
<< 31 | x
| a
<< 5 | b
<< 16); // lsr
1742 o(0x1ac00800 | l
<< 31 | x
| a
<< 5 | b
<< 16); // udiv
1745 o(0x6b00001f | l
<< 31 | a
<< 5 | b
<< 16); // cmp
1746 o(0x1a9f37e0 | x
); // cset wA,cs
1749 o(0x6b00001f | l
<< 31 | a
<< 5 | b
<< 16); // cmp
1750 o(0x1a9f97e0 | x
); // cset wA,hi
1753 o(0x6b00001f | l
<< 31 | a
<< 5 | b
<< 16); // cmp
1754 o(0x1a9f27e0 | x
); // cset wA,cc
1757 o(0x6b00001f | l
<< 31 | a
<< 5 | b
<< 16); // cmp
1758 o(0x1a9f87e0 | x
); // cset wA,ls
1761 // Use x30 for quotient:
1762 o(0x1ac00800 | l
<< 31 | 30 | a
<< 5 | b
<< 16); // udiv
1763 o(0x1b008000 | l
<< 31 | x
| (uint32_t)30 << 5 |
1764 b
<< 16 | a
<< 10); // msub
1771 ST_FUNC
void gen_opi(int op
)
1773 arm64_gen_opil(op
, 0);
1774 arm64_vset_VT_CMP(op
);
1777 ST_FUNC
void gen_opl(int op
)
1779 arm64_gen_opil(op
, 1);
1780 arm64_vset_VT_CMP(op
);
1783 ST_FUNC
void gen_opf(int op
)
1785 uint32_t x
, a
, b
, dbl
;
1787 if (vtop
[0].type
.t
== VT_LDOUBLE
) {
1788 CType type
= vtop
[0].type
;
1792 case '*': func
= TOK___multf3
; break;
1793 case '+': func
= TOK___addtf3
; break;
1794 case '-': func
= TOK___subtf3
; break;
1795 case '/': func
= TOK___divtf3
; break;
1796 case TOK_EQ
: func
= TOK___eqtf2
; cond
= 1; break;
1797 case TOK_NE
: func
= TOK___netf2
; cond
= 0; break;
1798 case TOK_LT
: func
= TOK___lttf2
; cond
= 10; break;
1799 case TOK_GE
: func
= TOK___getf2
; cond
= 11; break;
1800 case TOK_LE
: func
= TOK___letf2
; cond
= 12; break;
1801 case TOK_GT
: func
= TOK___gttf2
; cond
= 13; break;
1802 default: assert(0); break;
1804 vpush_helper_func(func
);
1808 vtop
->r
= cond
< 0 ? REG_FRET
: REG_IRET
;
1812 o(0x7100001f); // cmp w0,#0
1813 o(0x1a9f07e0 | (uint32_t)cond
<< 12); // cset w0,(cond)
1818 dbl
= vtop
[0].type
.t
!= VT_FLOAT
;
1819 gv2(RC_FLOAT
, RC_FLOAT
);
1820 assert(vtop
[-1].r
< VT_CONST
&& vtop
[0].r
< VT_CONST
);
1821 a
= fltr(vtop
[-1].r
);
1822 b
= fltr(vtop
[0].r
);
1825 case TOK_EQ
: case TOK_NE
:
1826 case TOK_LT
: case TOK_GE
: case TOK_LE
: case TOK_GT
:
1827 x
= get_reg(RC_INT
);
1833 x
= get_reg(RC_FLOAT
);
1842 o(0x1e200800 | dbl
<< 22 | x
| a
<< 5 | b
<< 16); // fmul
1845 o(0x1e202800 | dbl
<< 22 | x
| a
<< 5 | b
<< 16); // fadd
1848 o(0x1e203800 | dbl
<< 22 | x
| a
<< 5 | b
<< 16); // fsub
1851 o(0x1e201800 | dbl
<< 22 | x
| a
<< 5 | b
<< 16); // fdiv
1854 o(0x1e202000 | dbl
<< 22 | a
<< 5 | b
<< 16); // fcmp
1855 o(0x1a9f17e0 | x
); // cset w(x),eq
1858 o(0x1e202000 | dbl
<< 22 | a
<< 5 | b
<< 16); // fcmp
1859 o(0x1a9fb7e0 | x
); // cset w(x),ge
1862 o(0x1e202000 | dbl
<< 22 | a
<< 5 | b
<< 16); // fcmp
1863 o(0x1a9fd7e0 | x
); // cset w(x),gt
1866 o(0x1e202000 | dbl
<< 22 | a
<< 5 | b
<< 16); // fcmp
1867 o(0x1a9f87e0 | x
); // cset w(x),ls
1870 o(0x1e202000 | dbl
<< 22 | a
<< 5 | b
<< 16); // fcmp
1871 o(0x1a9f57e0 | x
); // cset w(x),mi
1874 o(0x1e202000 | dbl
<< 22 | a
<< 5 | b
<< 16); // fcmp
1875 o(0x1a9f07e0 | x
); // cset w(x),ne
1880 arm64_vset_VT_CMP(op
);
1883 // Generate sign extension from 32 to 64 bits:
1884 ST_FUNC
void gen_cvt_sxtw(void)
1886 uint32_t r
= intr(gv(RC_INT
));
1887 o(0x93407c00 | r
| r
<< 5); // sxtw x(r),w(r)
1890 /* char/short to int conversion */
1891 ST_FUNC
void gen_cvt_csti(int t
)
1893 int r
= intr(gv(RC_INT
));
1895 | ((t
& VT_BTYPE
) == VT_SHORT
) << 13
1896 | (uint32_t)!!(t
& VT_UNSIGNED
) << 30
1897 | r
| r
<< 5); // [su]xt[bh] w(r),w(r)
1900 ST_FUNC
void gen_cvt_itof(int t
)
1902 if (t
== VT_LDOUBLE
) {
1903 int f
= vtop
->type
.t
;
1904 int func
= (f
& VT_BTYPE
) == VT_LLONG
?
1905 (f
& VT_UNSIGNED
? TOK___floatunditf
: TOK___floatditf
) :
1906 (f
& VT_UNSIGNED
? TOK___floatunsitf
: TOK___floatsitf
);
1907 vpush_helper_func(func
);
1916 int d
, n
= intr(gv(RC_INT
));
1917 int s
= !(vtop
->type
.t
& VT_UNSIGNED
);
1918 uint32_t l
= ((vtop
->type
.t
& VT_BTYPE
) == VT_LLONG
);
1920 d
= get_reg(RC_FLOAT
);
1923 o(0x1e220000 | (uint32_t)!s
<< 16 |
1924 (uint32_t)(t
!= VT_FLOAT
) << 22 | fltr(d
) |
1925 l
<< 31 | n
<< 5); // [us]cvtf [sd](d),[wx](n)
1929 ST_FUNC
void gen_cvt_ftoi(int t
)
1931 if ((vtop
->type
.t
& VT_BTYPE
) == VT_LDOUBLE
) {
1932 int func
= (t
& VT_BTYPE
) == VT_LLONG
?
1933 (t
& VT_UNSIGNED
? TOK___fixunstfdi
: TOK___fixtfdi
) :
1934 (t
& VT_UNSIGNED
? TOK___fixunstfsi
: TOK___fixtfsi
);
1935 vpush_helper_func(func
);
1944 int d
, n
= fltr(gv(RC_FLOAT
));
1945 uint32_t l
= ((vtop
->type
.t
& VT_BTYPE
) != VT_FLOAT
);
1947 d
= get_reg(RC_INT
);
1951 (uint32_t)!!(t
& VT_UNSIGNED
) << 16 |
1952 (uint32_t)((t
& VT_BTYPE
) == VT_LLONG
) << 31 | intr(d
) |
1953 l
<< 22 | n
<< 5); // fcvtz[su] [wx](d),[sd](n)
1957 ST_FUNC
void gen_cvt_ftof(int t
)
1959 int f
= vtop
[0].type
.t
& VT_BTYPE
;
1960 assert(t
== VT_FLOAT
|| t
== VT_DOUBLE
|| t
== VT_LDOUBLE
);
1961 assert(f
== VT_FLOAT
|| f
== VT_DOUBLE
|| f
== VT_LDOUBLE
);
1965 if (t
== VT_LDOUBLE
|| f
== VT_LDOUBLE
) {
1966 int func
= (t
== VT_LDOUBLE
) ?
1967 (f
== VT_FLOAT
? TOK___extendsftf2
: TOK___extenddftf2
) :
1968 (t
== VT_FLOAT
? TOK___trunctfsf2
: TOK___trunctfdf2
);
1969 vpush_helper_func(func
);
1979 assert(vtop
[0].r
< VT_CONST
);
1980 a
= fltr(vtop
[0].r
);
1982 x
= get_reg(RC_FLOAT
);
1988 o(0x1e22c000 | x
| a
<< 5); // fcvt d(x),s(a)
1990 o(0x1e624000 | x
| a
<< 5); // fcvt s(x),d(a)
1994 /* increment tcov counter */
1995 ST_FUNC
void gen_increment_tcov (SValue
*sv
)
2000 vtop
->r
= r1
= get_reg(RC_INT
);
2001 r2
= get_reg(RC_INT
);
2002 greloca(cur_text_section
, sv
->sym
, ind
, R_AARCH64_ADR_GOT_PAGE
, 0);
2003 o(0x90000000 | r1
); // adrp r1, #sym
2004 greloca(cur_text_section
, sv
->sym
, ind
, R_AARCH64_LD64_GOT_LO12_NC
, 0);
2005 o(0xf9400000 | r1
| (r1
<< 5)); // ld xr,[xr, #sym]
2006 o(0xf9400000 | (intr(r1
)<<5) | intr(r2
)); // ldr r2, [r1]
2007 o(0x91000400 | (intr(r2
)<<5) | intr(r2
)); // add r2, r2, #1
2008 o(0xf9000000 | (intr(r1
)<<5) | intr(r2
)); // str r2, [r1]
2012 ST_FUNC
void ggoto(void)
2014 arm64_gen_bl_or_b(1);
2018 ST_FUNC
void gen_clear_cache(void)
2020 uint32_t beg
, end
, dsz
, isz
, p
, lab1
, b1
;
2021 gv2(RC_INT
, RC_INT
);
2023 vtop
->r
= get_reg(RC_INT
);
2025 vtop
->r
= get_reg(RC_INT
);
2027 vtop
->r
= get_reg(RC_INT
);
2028 beg
= intr(vtop
[-4].r
); // x0
2029 end
= intr(vtop
[-3].r
); // x1
2030 dsz
= intr(vtop
[-2].r
); // x2
2031 isz
= intr(vtop
[-1].r
); // x3
2032 p
= intr(vtop
[0].r
); // x4
2035 o(0xd53b0020 | isz
); // mrs x(isz),ctr_el0
2036 o(0x52800080 | p
); // mov w(p),#4
2037 o(0x53104c00 | dsz
| isz
<< 5); // ubfx w(dsz),w(isz),#16,#4
2038 o(0x1ac02000 | dsz
| p
<< 5 | dsz
<< 16); // lsl w(dsz),w(p),w(dsz)
2039 o(0x12000c00 | isz
| isz
<< 5); // and w(isz),w(isz),#15
2040 o(0x1ac02000 | isz
| p
<< 5 | isz
<< 16); // lsl w(isz),w(p),w(isz)
2041 o(0x51000400 | p
| dsz
<< 5); // sub w(p),w(dsz),#1
2042 o(0x8a240004 | p
| beg
<< 5 | p
<< 16); // bic x(p),x(beg),x(p)
2043 b1
= ind
; o(0x14000000); // b
2045 o(0xd50b7b20 | p
); // dc cvau,x(p)
2046 o(0x8b000000 | p
| p
<< 5 | dsz
<< 16); // add x(p),x(p),x(dsz)
2047 write32le(cur_text_section
->data
+ b1
, 0x14000000 | (ind
- b1
) >> 2);
2048 o(0xeb00001f | p
<< 5 | end
<< 16); // cmp x(p),x(end)
2049 o(0x54ffffa3 | ((lab1
- ind
) << 3 & 0xffffe0)); // b.cc lab1
2050 o(0xd5033b9f); // dsb ish
2051 o(0x51000400 | p
| isz
<< 5); // sub w(p),w(isz),#1
2052 o(0x8a240004 | p
| beg
<< 5 | p
<< 16); // bic x(p),x(beg),x(p)
2053 b1
= ind
; o(0x14000000); // b
2055 o(0xd50b7520 | p
); // ic ivau,x(p)
2056 o(0x8b000000 | p
| p
<< 5 | isz
<< 16); // add x(p),x(p),x(isz)
2057 write32le(cur_text_section
->data
+ b1
, 0x14000000 | (ind
- b1
) >> 2);
2058 o(0xeb00001f | p
<< 5 | end
<< 16); // cmp x(p),x(end)
2059 o(0x54ffffa3 | ((lab1
- ind
) << 3 & 0xffffe0)); // b.cc lab1
2060 o(0xd5033b9f); // dsb ish
2061 o(0xd5033fdf); // isb
2064 ST_FUNC
void gen_vla_sp_save(int addr
) {
2065 uint32_t r
= intr(get_reg(RC_INT
));
2066 o(0x910003e0 | r
); // mov x(r),sp
2067 arm64_strx(3, r
, 29, addr
);
2070 ST_FUNC
void gen_vla_sp_restore(int addr
) {
2071 // Use x30 because this function can be called when there
2072 // is a live return value in x0 but there is nothing on
2073 // the value stack to prevent get_reg from returning x0.
2075 arm64_ldrx(0, 3, r
, 29, addr
);
2076 o(0x9100001f | r
<< 5); // mov sp,x(r)
2079 ST_FUNC
void gen_vla_alloc(CType
*type
, int align
) {
2081 #if defined(CONFIG_TCC_BCHECK)
2082 if (tcc_state
->do_bounds_check
)
2085 r
= intr(gv(RC_INT
));
2086 #if defined(CONFIG_TCC_BCHECK)
2087 if (tcc_state
->do_bounds_check
)
2088 o(0x91004000 | r
| r
<< 5); // add x(r),x(r),#15+1
2091 o(0x91003c00 | r
| r
<< 5); // add x(r),x(r),#15
2092 o(0x927cec00 | r
| r
<< 5); // bic x(r),x(r),#15
2093 o(0xcb2063ff | r
<< 16); // sub sp,sp,x(r)
2095 #if defined(CONFIG_TCC_BCHECK)
2096 if (tcc_state
->do_bounds_check
) {
2098 vtop
->r
= TREG_R(0);
2099 o(0x910003e0 | vtop
->r
); // mov r0,sp
2101 vpush_helper_func(TOK___bound_new_region
);
2104 func_bound_add_epilog
= 1;
2109 /* end of A64 code generator */
2110 /*************************************************************/
2112 /*************************************************************/