arm64/abi.c

   1 #include "all.h"
   2
   3 typedef struct Abi Abi;
   4 typedef struct Class Class;
   5 typedef struct Insl Insl;
   6 typedef struct Params Params;
   7
   8 enum {
   9         Cstk = 1, /* pass on the stack */
  10         Cptr = 2, /* replaced by a pointer */
  11 };
  12
  13 struct Class {
  14         char class;
  15         char ishfa;
  16         struct {
  17                 char base;
  18                 uchar size;
  19         } hfa;
  20         uint size;
  21         uint align;
  22         Typ *t;
  23         uchar nreg;
  24         uchar ngp;
  25         uchar nfp;
  26         int reg[4];
  27         int cls[4];
  28 };
  29
  30 struct Insl {
  31         Ins i;
  32         Insl *link;
  33 };
  34
  35 struct Params {
  36         uint ngp;
  37         uint nfp;
  38         uint stk;
  39 };
  40
  41 static int gpreg[12] = {R0, R1, R2, R3, R4, R5, R6, R7};
  42 static int fpreg[12] = {V0, V1, V2, V3, V4, V5, V6, V7};
  43 static int store[] = {
  44         [Kw] = Ostorew, [Kl] = Ostorel,
  45         [Ks] = Ostores, [Kd] = Ostored
  46 };
  47
  48 /* layout of call's second argument (RCall)
  49  *
  50  *         13
  51  *  29   14 |    9    5   2  0
  52  *  |0.00|x|x|xxxx|xxxx|xxx|xx|                  range
  53  *        | |    |    |   |  ` gp regs returned (0..2)
  54  *        | |    |    |   ` fp regs returned    (0..4)
  55  *        | |    |    ` gp regs passed          (0..8)
  56  *        | |     ` fp regs passed              (0..8)
  57  *        | ` indirect result register x8 used  (0..1)
  58  *        ` env pointer passed in x9            (0..1)
  59  */
  60
  61 static int
  62 isfloatv(Typ *t, char *cls)
  63 {
  64         Field *f;
  65         uint n;
  66
  67         for (n=0; n<t->nunion; n++)
  68                 for (f=t->fields[n]; f->type != FEnd; f++)
  69                         switch (f->type) {
  70                         case Fs:
  71                                 if (*cls == Kd)
  72                                         return 0;
  73                                 *cls = Ks;
  74                                 break;
  75                         case Fd:
  76                                 if (*cls == Ks)
  77                                         return 0;
  78                                 *cls = Kd;
  79                                 break;
  80                         case FTyp:
  81                                 if (isfloatv(&typ[f->len], cls))
  82                                         break;
  83                                 /* fall through */
  84                         default:
  85                                 return 0;
  86                         }
  87         return 1;
  88 }
  89
  90 static void
  91 typclass(Class *c, Typ *t, int *gp, int *fp)
  92 {
  93         uint64_t sz;
  94         uint n;
  95
  96         sz = (t->size + 7) & -8;
  97         c->t = t;
  98         c->class = 0;
  99         c->ngp = 0;
 100         c->nfp = 0;
 101         c->align = 8;
 102
 103         if (t->align > 3)
 104                 err("alignments larger than 8 are not supported");
 105
 106         if (t->isdark || sz > 16 || sz == 0) {
 107                 /* large structs are replaced by a
 108                  * pointer to some caller-allocated
 109                  * memory */
 110                 c->class |= Cptr;
 111                 c->size = 8;
 112                 c->ngp = 1;
 113                 *c->reg = *gp;
 114                 *c->cls = Kl;
 115                 return;
 116         }
 117
 118         c->size = sz;
 119         c->hfa.base = Kx;
 120         c->ishfa = isfloatv(t, &c->hfa.base);
 121         c->hfa.size = t->size/(KWIDE(c->hfa.base) ? 8 : 4);
 122
 123         if (c->ishfa)
 124                 for (n=0; n<c->hfa.size; n++, c->nfp++) {
 125                         c->reg[n] = *fp++;
 126                         c->cls[n] = c->hfa.base;
 127                 }
 128         else
 129                 for (n=0; n<sz/8; n++, c->ngp++) {
 130                         c->reg[n] = *gp++;
 131                         c->cls[n] = Kl;
 132                 }
 133
 134         c->nreg = n;
 135 }
 136
 137 static void
 138 sttmps(Ref tmp[], int cls[], uint nreg, Ref mem, Fn *fn)
 139 {
 140         uint n;
 141         uint64_t off;
 142         Ref r;
 143
 144         assert(nreg <= 4);
 145         off = 0;
 146         for (n=0; n<nreg; n++) {
 147                 tmp[n] = newtmp("abi", cls[n], fn);
 148                 r = newtmp("abi", Kl, fn);
 149                 emit(store[cls[n]], 0, R, tmp[n], r);
 150                 emit(Oadd, Kl, r, mem, getcon(off, fn));
 151                 off += KWIDE(cls[n]) ? 8 : 4;
 152         }
 153 }
 154
 155 /* todo, may read out of bounds */
 156 static void
 157 ldregs(int reg[], int cls[], int n, Ref mem, Fn *fn)
 158 {
 159         int i;
 160         uint64_t off;
 161         Ref r;
 162
 163         off = 0;
 164         for (i=0; i<n; i++) {
 165                 r = newtmp("abi", Kl, fn);
 166                 emit(Oload, cls[i], TMP(reg[i]), r, R);
 167                 emit(Oadd, Kl, r, mem, getcon(off, fn));
 168                 off += KWIDE(cls[i]) ? 8 : 4;
 169         }
 170 }
 171
 172 static void
 173 selret(Blk *b, Fn *fn)
 174 {
 175         int j, k, cty;
 176         Ref r;
 177         Class cr;
 178
 179         j = b->jmp.type;
 180
 181         if (!isret(j) || j == Jret0)
 182                 return;
 183
 184         r = b->jmp.arg;
 185         b->jmp.type = Jret0;
 186
 187         if (j == Jretc) {
 188                 typclass(&cr, &typ[fn->retty], gpreg, fpreg);
 189                 if (cr.class & Cptr) {
 190                         assert(rtype(fn->retr) == RTmp);
 191                         blit0(fn->retr, r, cr.t->size, fn);
 192                         cty = 0;
 193                 } else {
 194                         ldregs(cr.reg, cr.cls, cr.nreg, r, fn);
 195                         cty = (cr.nfp << 2) | cr.ngp;
 196                 }
 197         } else {
 198                 k = j - Jretw;
 199                 if (KBASE(k) == 0) {
 200                         emit(Ocopy, k, TMP(R0), r, R);
 201                         cty = 1;
 202                 } else {
 203                         emit(Ocopy, k, TMP(V0), r, R);
 204                         cty = 1 << 2;
 205                 }
 206         }
 207
 208         b->jmp.arg = CALL(cty);
 209 }
 210
 211 static int
 212 argsclass(Ins *i0, Ins *i1, Class *carg)
 213 {
 214         int va, envc, ngp, nfp, *gp, *fp;
 215         Class *c;
 216         Ins *i;
 217
 218         va = 0;
 219         envc = 0;
 220         gp = gpreg;
 221         fp = fpreg;
 222         ngp = 8;
 223         nfp = 8;
 224         for (i=i0, c=carg; i<i1; i++, c++)
 225                 switch (i->op) {
 226                 case Oargsb:
 227                 case Oargub:
 228                 case Oparsb:
 229                 case Oparub:
 230                         c->size = 1;
 231                         goto Scalar;
 232                 case Oargsh:
 233                 case Oarguh:
 234                 case Oparsh:
 235                 case Oparuh:
 236                         c->size = 2;
 237                         goto Scalar;
 238                 case Opar:
 239                 case Oarg:
 240                         c->size = 8;
 241                         if (T.apple && !KWIDE(i->cls))
 242                                 c->size = 4;
 243                 Scalar:
 244                         c->align = c->size;
 245                         *c->cls = i->cls;
 246                         if (va) {
 247                                 c->class |= Cstk;
 248                                 c->size = 8;
 249                                 c->align = 8;
 250                                 break;
 251                         }
 252                         if (KBASE(i->cls) == 0 && ngp > 0) {
 253                                 ngp--;
 254                                 *c->reg = *gp++;
 255                                 break;
 256                         }
 257                         if (KBASE(i->cls) == 1 && nfp > 0) {
 258                                 nfp--;
 259                                 *c->reg = *fp++;
 260                                 break;
 261                         }
 262                         c->class |= Cstk;
 263                         break;
 264                 case Oparc:
 265                 case Oargc:
 266                         typclass(c, &typ[i->arg[0].val], gp, fp);
 267                         if (c->ngp <= ngp) {
 268                                 if (c->nfp <= nfp) {
 269                                         ngp -= c->ngp;
 270                                         nfp -= c->nfp;
 271                                         gp += c->ngp;
 272                                         fp += c->nfp;
 273                                         break;
 274                                 } else
 275                                         nfp = 0;
 276                         } else
 277                                 ngp = 0;
 278                         c->class |= Cstk;
 279                         break;
 280                 case Opare:
 281                 case Oarge:
 282                         *c->reg = R9;
 283                         *c->cls = Kl;
 284                         envc = 1;
 285                         break;
 286                 case Oargv:
 287                         va = T.apple != 0;
 288                         break;
 289                 default:
 290                         die("unreachable");
 291                 }
 292
 293         return envc << 14 | (gp-gpreg) << 5 | (fp-fpreg) << 9;
 294 }
 295
 296 bits
 297 arm64_retregs(Ref r, int p[2])
 298 {
 299         bits b;
 300         int ngp, nfp;
 301
 302         assert(rtype(r) == RCall);
 303         ngp = r.val & 3;
 304         nfp = (r.val >> 2) & 7;
 305         if (p) {
 306                 p[0] = ngp;
 307                 p[1] = nfp;
 308         }
 309         b = 0;
 310         while (ngp--)
 311                 b |= BIT(R0+ngp);
 312         while (nfp--)
 313                 b |= BIT(V0+nfp);
 314         return b;
 315 }
 316
 317 bits
 318 arm64_argregs(Ref r, int p[2])
 319 {
 320         bits b;
 321         int ngp, nfp, x8, x9;
 322
 323         assert(rtype(r) == RCall);
 324         ngp = (r.val >> 5) & 15;
 325         nfp = (r.val >> 9) & 15;
 326         x8 = (r.val >> 13) & 1;
 327         x9 = (r.val >> 14) & 1;
 328         if (p) {
 329                 p[0] = ngp + x8 + x9;
 330                 p[1] = nfp;
 331         }
 332         b = 0;
 333         while (ngp--)
 334                 b |= BIT(R0+ngp);
 335         while (nfp--)
 336                 b |= BIT(V0+nfp);
 337         return b | ((bits)x8 << R8) | ((bits)x9 << R9);
 338 }
 339
 340 static void
 341 stkblob(Ref r, Class *c, Fn *fn, Insl **ilp)
 342 {
 343         Insl *il;
 344         int al;
 345         uint64_t sz;
 346
 347         il = alloc(sizeof *il);
 348         al = c->t->align - 2; /* NAlign == 3 */
 349         if (al < 0)
 350                 al = 0;
 351         sz = c->class & Cptr ? c->t->size : c->size;
 352         il->i = (Ins){Oalloc+al, Kl, r, {getcon(sz, fn)}};
 353         il->link = *ilp;
 354         *ilp = il;
 355 }
 356
 357 static uint
 358 align(uint x, uint al)
 359 {
 360         return (x + al-1) & -al;
 361 }
 362
 363 static void
 364 selcall(Fn *fn, Ins *i0, Ins *i1, Insl **ilp)
 365 {
 366         Ins *i;
 367         Class *ca, *c, cr;
 368         int op, cty;
 369         uint n, stk, off;;
 370         Ref r, rstk, tmp[4];
 371
 372         ca = alloc((i1-i0) * sizeof ca[0]);
 373         cty = argsclass(i0, i1, ca);
 374
 375         stk = 0;
 376         for (i=i0, c=ca; i<i1; i++, c++) {
 377                 if (c->class & Cptr) {
 378                         i->arg[0] = newtmp("abi", Kl, fn);
 379                         stkblob(i->arg[0], c, fn, ilp);
 380                         i->op = Oarg;
 381                 }
 382                 if (c->class & Cstk) {
 383                         stk = align(stk, c->align);
 384                         stk += c->size;
 385                 }
 386         }
 387         stk = align(stk, 16);
 388         rstk = getcon(stk, fn);
 389         if (stk)
 390                 emit(Oadd, Kl, TMP(SP), TMP(SP), rstk);
 391
 392         if (!req(i1->arg[1], R)) {
 393                 typclass(&cr, &typ[i1->arg[1].val], gpreg, fpreg);
 394                 stkblob(i1->to, &cr, fn, ilp);
 395                 cty |= (cr.nfp << 2) | cr.ngp;
 396                 if (cr.class & Cptr) {
 397                         /* spill & rega expect calls to be
 398                          * followed by copies from regs,
 399                          * so we emit a dummy
 400                          */
 401                         cty |= 1 << 13 | 1;
 402                         emit(Ocopy, Kw, R, TMP(R0), R);
 403                 } else {
 404                         sttmps(tmp, cr.cls, cr.nreg, i1->to, fn);
 405                         for (n=0; n<cr.nreg; n++) {
 406                                 r = TMP(cr.reg[n]);
 407                                 emit(Ocopy, cr.cls[n], tmp[n], r, R);
 408                         }
 409                 }
 410         } else {
 411                 if (KBASE(i1->cls) == 0) {
 412                         emit(Ocopy, i1->cls, i1->to, TMP(R0), R);
 413                         cty |= 1;
 414                 } else {
 415                         emit(Ocopy, i1->cls, i1->to, TMP(V0), R);
 416                         cty |= 1 << 2;
 417                 }
 418         }
 419
 420         emit(Ocall, 0, R, i1->arg[0], CALL(cty));
 421
 422         if (cty & (1 << 13))
 423                 /* struct return argument */
 424                 emit(Ocopy, Kl, TMP(R8), i1->to, R);
 425
 426         for (i=i0, c=ca; i<i1; i++, c++) {
 427                 if ((c->class & Cstk) != 0)
 428                         continue;
 429                 if (i->op == Oarg || i->op == Oarge)
 430                         emit(Ocopy, *c->cls, TMP(*c->reg), i->arg[0], R);
 431                 if (i->op == Oargc)
 432                         ldregs(c->reg, c->cls, c->nreg, i->arg[1], fn);
 433         }
 434
 435         /* populate the stack */
 436         off = 0;
 437         for (i=i0, c=ca; i<i1; i++, c++) {
 438                 if ((c->class & Cstk) == 0)
 439                         continue;
 440                 off = align(off, c->align);
 441                 if (i->op == Oarg || isargbh(i->op)) {
 442                         r = newtmp("abi", Kl, fn);
 443                         switch (c->size) {
 444                         case 1: op = Ostoreb; break;
 445                         case 2: op = Ostoreh; break;
 446                         case 4:
 447                         case 8: op = store[*c->cls]; break;
 448                         }
 449                         emit(op, 0, R, i->arg[0], r);
 450                         emit(Oadd, Kl, r, TMP(SP), getcon(off, fn));
 451                 }
 452                 if (i->op == Oargc)
 453                         blit(TMP(SP), off, i->arg[1], 0, c->size, fn);
 454                 off += c->size;
 455         }
 456         if (stk)
 457                 emit(Osub, Kl, TMP(SP), TMP(SP), rstk);
 458
 459         for (i=i0, c=ca; i<i1; i++, c++)
 460                 if (c->class & Cptr)
 461                         blit0(i->arg[0], i->arg[1], c->t->size, fn);
 462 }
 463
 464 static Params
 465 selpar(Fn *fn, Ins *i0, Ins *i1)
 466 {
 467         Class *ca, *c, cr;
 468         Insl *il;
 469         Ins *i;
 470         int op, n, cty;
 471         uint off;
 472         Ref r, tmp[16], *t;
 473
 474         ca = alloc((i1-i0) * sizeof ca[0]);
 475         curi = &insb[NIns];
 476
 477         cty = argsclass(i0, i1, ca);
 478         fn->reg = arm64_argregs(CALL(cty), 0);
 479
 480         il = 0;
 481         t = tmp;
 482         for (i=i0, c=ca; i<i1; i++, c++) {
 483                 if (i->op != Oparc || (c->class & (Cptr|Cstk)))
 484                         continue;
 485                 sttmps(t, c->cls, c->nreg, i->to, fn);
 486                 stkblob(i->to, c, fn, &il);
 487                 t += c->nreg;
 488         }
 489         for (; il; il=il->link)
 490                 emiti(il->i);
 491
 492         if (fn->retty >= 0) {
 493                 typclass(&cr, &typ[fn->retty], gpreg, fpreg);
 494                 if (cr.class & Cptr) {
 495                         fn->retr = newtmp("abi", Kl, fn);
 496                         emit(Ocopy, Kl, fn->retr, TMP(R8), R);
 497                         fn->reg |= BIT(R8);
 498                 }
 499         }
 500
 501         t = tmp;
 502         off = 0;
 503         for (i=i0, c=ca; i<i1; i++, c++)
 504                 if (i->op == Oparc && !(c->class & Cptr)) {
 505                         if (c->class & Cstk) {
 506                                 off = align(off, c->align);
 507                                 fn->tmp[i->to.val].slot = -(off+2);
 508                                 off += c->size;
 509                         } else
 510                                 for (n=0; n<c->nreg; n++) {
 511                                         r = TMP(c->reg[n]);
 512                                         emit(Ocopy, c->cls[n], *t++, r, R);
 513                                 }
 514                 } else if (c->class & Cstk) {
 515                         off = align(off, c->align);
 516                         if (isparbh(i->op))
 517                                 op = Oloadsb + (i->op - Oparsb);
 518                         else
 519                                 op = Oload;
 520                         emit(op, *c->cls, i->to, SLOT(-(off+2)), R);
 521                         off += c->size;
 522                 } else {
 523                         emit(Ocopy, *c->cls, i->to, TMP(*c->reg), R);
 524                 }
 525
 526         return (Params){
 527                 .stk = align(off, 8),
 528                 .ngp = (cty >> 5) & 15,
 529                 .nfp = (cty >> 9) & 15
 530         };
 531 }
 532
 533 static Blk *
 534 split(Fn *fn, Blk *b)
 535 {
 536         Blk *bn;
 537
 538         ++fn->nblk;
 539         bn = blknew();
 540         bn->nins = &insb[NIns] - curi;
 541         idup(&bn->ins, curi, bn->nins);
 542         curi = &insb[NIns];
 543         bn->visit = ++b->visit;
 544         (void)!snprintf(bn->name, NString, "%s.%d", b->name, b->visit);
 545         bn->loop = b->loop;
 546         bn->link = b->link;
 547         b->link = bn;
 548         return bn;
 549 }
 550
 551 static void
 552 chpred(Blk *b, Blk *bp, Blk *bp1)
 553 {
 554         Phi *p;
 555         uint a;
 556
 557         for (p=b->phi; p; p=p->link) {
 558                 for (a=0; p->blk[a]!=bp; a++)
 559                         assert(a+1<p->narg);
 560                 p->blk[a] = bp1;
 561         }
 562 }
 563
 564 static void
 565 apple_selvaarg(Fn *fn, Blk *b, Ins *i)
 566 {
 567         Ref ap, stk, stk8, c8;
 568
 569         (void)b;
 570         c8 = getcon(8, fn);
 571         ap = i->arg[0];
 572         stk8 = newtmp("abi", Kl, fn);
 573         stk = newtmp("abi", Kl, fn);
 574
 575         emit(Ostorel, 0, R, stk8, ap);
 576         emit(Oadd, Kl, stk8, stk, c8);
 577         emit(Oload, i->cls, i->to, stk, R);
 578         emit(Oload, Kl, stk, ap, R);
 579 }
 580
 581 static void
 582 arm64_selvaarg(Fn *fn, Blk *b, Ins *i)
 583 {
 584         Ref loc, lreg, lstk, nr, r0, r1, c8, c16, c24, c28, ap;
 585         Blk *b0, *bstk, *breg;
 586         int isgp;
 587
 588         c8 = getcon(8, fn);
 589         c16 = getcon(16, fn);
 590         c24 = getcon(24, fn);
 591         c28 = getcon(28, fn);
 592         ap = i->arg[0];
 593         isgp = KBASE(i->cls) == 0;
 594
 595         /* @b [...]
 596                r0 =l add ap, (24 or 28)
 597                nr =l loadsw r0
 598                r1 =w csltw nr, 0
 599                jnz r1, @breg, @bstk
 600            @breg
 601                r0 =l add ap, (8 or 16)
 602                r1 =l loadl r0
 603                lreg =l add r1, nr
 604                r0 =w add nr, (8 or 16)
 605                r1 =l add ap, (24 or 28)
 606                storew r0, r1
 607            @bstk
 608                lstk =l loadl ap
 609                r0 =l add lstk, 8
 610                storel r0, ap
 611            @b0
 612                %loc =l phi @breg %lreg, @bstk %lstk
 613                i->to =(i->cls) load %loc
 614         */
 615
 616         loc = newtmp("abi", Kl, fn);
 617         emit(Oload, i->cls, i->to, loc, R);
 618         b0 = split(fn, b);
 619         b0->jmp = b->jmp;
 620         b0->s1 = b->s1;
 621         b0->s2 = b->s2;
 622         if (b->s1)
 623                 chpred(b->s1, b, b0);
 624         if (b->s2 && b->s2 != b->s1)
 625                 chpred(b->s2, b, b0);
 626
 627         lreg = newtmp("abi", Kl, fn);
 628         nr = newtmp("abi", Kl, fn);
 629         r0 = newtmp("abi", Kw, fn);
 630         r1 = newtmp("abi", Kl, fn);
 631         emit(Ostorew, Kw, R, r0, r1);
 632         emit(Oadd, Kl, r1, ap, isgp ? c24 : c28);
 633         emit(Oadd, Kw, r0, nr, isgp ? c8 : c16);
 634         r0 = newtmp("abi", Kl, fn);
 635         r1 = newtmp("abi", Kl, fn);
 636         emit(Oadd, Kl, lreg, r1, nr);
 637         emit(Oload, Kl, r1, r0, R);
 638         emit(Oadd, Kl, r0, ap, isgp ? c8 : c16);
 639         breg = split(fn, b);
 640         breg->jmp.type = Jjmp;
 641         breg->s1 = b0;
 642
 643         lstk = newtmp("abi", Kl, fn);
 644         r0 = newtmp("abi", Kl, fn);
 645         emit(Ostorel, Kw, R, r0, ap);
 646         emit(Oadd, Kl, r0, lstk, c8);
 647         emit(Oload, Kl, lstk, ap, R);
 648         bstk = split(fn, b);
 649         bstk->jmp.type = Jjmp;
 650         bstk->s1 = b0;
 651
 652         b0->phi = alloc(sizeof *b0->phi);
 653         *b0->phi = (Phi){
 654                 .cls = Kl, .to = loc,
 655                 .narg = 2,
 656                 .blk = vnew(2, sizeof b0->phi->blk[0], PFn),
 657                 .arg = vnew(2, sizeof b0->phi->arg[0], PFn),
 658         };
 659         b0->phi->blk[0] = bstk;
 660         b0->phi->blk[1] = breg;
 661         b0->phi->arg[0] = lstk;
 662         b0->phi->arg[1] = lreg;
 663         r0 = newtmp("abi", Kl, fn);
 664         r1 = newtmp("abi", Kw, fn);
 665         b->jmp.type = Jjnz;
 666         b->jmp.arg = r1;
 667         b->s1 = breg;
 668         b->s2 = bstk;
 669         emit(Ocmpw+Cislt, Kw, r1, nr, CON_Z);
 670         emit(Oloadsw, Kl, nr, r0, R);
 671         emit(Oadd, Kl, r0, ap, isgp ? c24 : c28);
 672 }
 673
 674 static void
 675 apple_selvastart(Fn *fn, Params p, Ref ap)
 676 {
 677         Ref off, stk, arg;
 678
 679         off = getcon(p.stk, fn);
 680         stk = newtmp("abi", Kl, fn);
 681         arg = newtmp("abi", Kl, fn);
 682
 683         emit(Ostorel, 0, R, arg, ap);
 684         emit(Oadd, Kl, arg, stk, off);
 685         emit(Oaddr, Kl, stk, SLOT(-1), R);
 686 }
 687
 688 static void
 689 arm64_selvastart(Fn *fn, Params p, Ref ap)
 690 {
 691         Ref r0, r1, rsave;
 692
 693         rsave = newtmp("abi", Kl, fn);
 694
 695         r0 = newtmp("abi", Kl, fn);
 696         emit(Ostorel, Kw, R, r0, ap);
 697         emit(Oadd, Kl, r0, rsave, getcon(p.stk + 192, fn));
 698
 699         r0 = newtmp("abi", Kl, fn);
 700         r1 = newtmp("abi", Kl, fn);
 701         emit(Ostorel, Kw, R, r1, r0);
 702         emit(Oadd, Kl, r1, rsave, getcon(64, fn));
 703         emit(Oadd, Kl, r0, ap, getcon(8, fn));
 704
 705         r0 = newtmp("abi", Kl, fn);
 706         r1 = newtmp("abi", Kl, fn);
 707         emit(Ostorel, Kw, R, r1, r0);
 708         emit(Oadd, Kl, r1, rsave, getcon(192, fn));
 709         emit(Oaddr, Kl, rsave, SLOT(-1), R);
 710         emit(Oadd, Kl, r0, ap, getcon(16, fn));
 711
 712         r0 = newtmp("abi", Kl, fn);
 713         emit(Ostorew, Kw, R, getcon((p.ngp-8)*8, fn), r0);
 714         emit(Oadd, Kl, r0, ap, getcon(24, fn));
 715
 716         r0 = newtmp("abi", Kl, fn);
 717         emit(Ostorew, Kw, R, getcon((p.nfp-8)*16, fn), r0);
 718         emit(Oadd, Kl, r0, ap, getcon(28, fn));
 719 }
 720
 721 void
 722 arm64_abi(Fn *fn)
 723 {
 724         Blk *b;
 725         Ins *i, *i0, *ip;
 726         Insl *il;
 727         int n;
 728         Params p;
 729
 730         for (b=fn->start; b; b=b->link)
 731                 b->visit = 0;
 732
 733         /* lower parameters */
 734         for (b=fn->start, i=b->ins; i<&b->ins[b->nins]; i++)
 735                 if (!ispar(i->op))
 736                         break;
 737         p = selpar(fn, b->ins, i);
 738         n = b->nins - (i - b->ins) + (&insb[NIns] - curi);
 739         i0 = alloc(n * sizeof(Ins));
 740         ip = icpy(ip = i0, curi, &insb[NIns] - curi);
 741         ip = icpy(ip, i, &b->ins[b->nins] - i);
 742         b->nins = n;
 743         b->ins = i0;
 744
 745         /* lower calls, returns, and vararg instructions */
 746         il = 0;
 747         b = fn->start;
 748         do {
 749                 if (!(b = b->link))
 750                         b = fn->start; /* do it last */
 751                 if (b->visit)
 752                         continue;
 753                 curi = &insb[NIns];
 754                 selret(b, fn);
 755                 for (i=&b->ins[b->nins]; i!=b->ins;)
 756                         switch ((--i)->op) {
 757                         default:
 758                                 emiti(*i);
 759                                 break;
 760                         case Ocall:
 761                                 for (i0=i; i0>b->ins; i0--)
 762                                         if (!isarg((i0-1)->op))
 763                                                 break;
 764                                 selcall(fn, i0, i, &il);
 765                                 i = i0;
 766                                 break;
 767                         case Ovastart:
 768                                 if (T.apple)
 769                                         apple_selvastart(fn, p, i->arg[0]);
 770                                 else
 771                                         arm64_selvastart(fn, p, i->arg[0]);
 772                                 break;
 773                         case Ovaarg:
 774                                 if (T.apple)
 775                                         apple_selvaarg(fn, b, i);
 776                                 else
 777                                         arm64_selvaarg(fn, b, i);
 778                                 break;
 779                         case Oarg:
 780                         case Oargc:
 781                                 die("unreachable");
 782                         }
 783                 if (b == fn->start)
 784                         for (; il; il=il->link)
 785                                 emiti(il->i);
 786                 b->nins = &insb[NIns] - curi;
 787                 idup(&b->ins, curi, b->nins);
 788         } while (b != fn->start);
 789
 790         if (debug['A']) {
 791                 fprintf(stderr, "\n> After ABI lowering:\n");
 792                 printfn(fn, stderr);
 793         }
 794 }
 795
 796 /* abi0 for apple target; introduces
 797  * necessery sign extension for arg
 798  * passing & returns
 799  */
 800 void
 801 apple_extsb(Fn *fn)
 802 {
 803         Blk *b;
 804         Ins *i0, *i1, *i;
 805         int j, op;
 806         Ref r;
 807
 808         for (b=fn->start; b; b=b->link) {
 809                 curi = &insb[NIns];
 810                 j = b->jmp.type;
 811                 if (isretbh(j)) {
 812                         r = newtmp("abi", Kw, fn);
 813                         op = Oextsb + (j - Jretsb);
 814                         emit(op, Kw, r, b->jmp.arg, R);
 815                         b->jmp.arg = r;
 816                 }
 817                 for (i=&b->ins[b->nins]; i>b->ins;) {
 818                         emiti(*--i);
 819                         if (i->op != Ocall)
 820                                 continue;
 821                         for (i0=i1=i; i0>b->ins; i0--)
 822                                 if (!isarg((i0-1)->op))
 823                                         break;
 824                         for (i=i1; i>i0;) {
 825                                 emiti(*--i);
 826                                 if (isargbh(i->op)) {
 827                                         i->to = newtmp("abi", Kl, fn);
 828                                         curi->arg[0] = i->to;
 829                                 }
 830                         }
 831                         for (i=i1; i>i0;)
 832                                 if (isargbh((--i)->op)) {
 833                                         op = Oextsb + (i->op - Oargsb);
 834                                         emit(op, Kw, i->to, i->arg[0], R);
 835                                 }
 836                 }
 837                 b->nins = &insb[NIns] - curi;
 838                 idup(&b->ins, curi, b->nins);
 839         }
 840
 841         if (debug['A']) {
 842                 fprintf(stderr, "\n> After apple_extsb:\n");
 843                 printfn(fn, stderr);
 844         }
 845 }