src/lj_trace.c

   1 /*
   2 ** Trace management.
   3 ** Copyright (C) 2005-2025 Mike Pall. See Copyright Notice in luajit.h
   4 */
   5
   6 #define lj_trace_c
   7 #define LUA_CORE
   8
   9 #include "lj_obj.h"
  10
  11 #if LJ_HASJIT
  12
  13 #include "lj_gc.h"
  14 #include "lj_err.h"
  15 #include "lj_debug.h"
  16 #include "lj_str.h"
  17 #include "lj_frame.h"
  18 #include "lj_state.h"
  19 #include "lj_bc.h"
  20 #include "lj_ir.h"
  21 #include "lj_jit.h"
  22 #include "lj_iropt.h"
  23 #include "lj_mcode.h"
  24 #include "lj_trace.h"
  25 #include "lj_snap.h"
  26 #include "lj_gdbjit.h"
  27 #include "lj_record.h"
  28 #include "lj_asm.h"
  29 #include "lj_dispatch.h"
  30 #include "lj_vm.h"
  31 #include "lj_vmevent.h"
  32 #include "lj_target.h"
  33 #include "lj_prng.h"
  34
  35 /* -- Error handling ------------------------------------------------------ */
  36
  37 /* Synchronous abort with error message. */
  38 void lj_trace_err(jit_State *J, TraceError e)
  39 {
  40   setnilV(&J->errinfo);  /* No error info. */
  41   setintV(J->L->top++, (int32_t)e);
  42   lj_err_throw(J->L, LUA_ERRRUN);
  43 }
  44
  45 /* Synchronous abort with error message and error info. */
  46 void lj_trace_err_info(jit_State *J, TraceError e)
  47 {
  48   setintV(J->L->top++, (int32_t)e);
  49   lj_err_throw(J->L, LUA_ERRRUN);
  50 }
  51
  52 /* -- Trace management ---------------------------------------------------- */
  53
  54 /* The current trace is first assembled in J->cur. The variable length
  55 ** arrays point to shared, growable buffers (J->irbuf etc.). When trace
  56 ** recording ends successfully, the current trace and its data structures
  57 ** are copied to a new (compact) GCtrace object.
  58 */
  59
  60 /* Find a free trace number. */
  61 static TraceNo trace_findfree(jit_State *J)
  62 {
  63   MSize osz, lim;
  64   if (J->freetrace == 0)
  65     J->freetrace = 1;
  66   for (; J->freetrace < J->sizetrace; J->freetrace++)
  67     if (traceref(J, J->freetrace) == NULL)
  68       return J->freetrace++;
  69   /* Need to grow trace array. */
  70   lim = (MSize)J->param[JIT_P_maxtrace] + 1;
  71   if (lim < 2) lim = 2; else if (lim > 65535) lim = 65535;
  72   osz = J->sizetrace;
  73   if (osz >= lim)
  74     return 0;  /* Too many traces. */
  75   lj_mem_growvec(J->L, J->trace, J->sizetrace, lim, GCRef);
  76   for (; osz < J->sizetrace; osz++)
  77     setgcrefnull(J->trace[osz]);
  78   return J->freetrace;
  79 }
  80
  81 #define TRACE_APPENDVEC(field, szfield, tp) \
  82   T->field = (tp *)p; \
  83   memcpy(p, J->cur.field, J->cur.szfield*sizeof(tp)); \
  84   p += J->cur.szfield*sizeof(tp);
  85
  86 #ifdef LUAJIT_USE_PERFTOOLS
  87 /*
  88 ** Create symbol table of JIT-compiled code. For use with Linux perf tools.
  89 ** Example usage:
  90 **   perf record -f -e cycles luajit test.lua
  91 **   perf report -s symbol
  92 **   rm perf.data /tmp/perf-*.map
  93 */
  94 #include <stdio.h>
  95 #include <unistd.h>
  96
  97 static void perftools_addtrace(GCtrace *T)
  98 {
  99   static FILE *fp;
 100   GCproto *pt = &gcref(T->startpt)->pt;
 101   const BCIns *startpc = mref(T->startpc, const BCIns);
 102   const char *name = proto_chunknamestr(pt);
 103   BCLine lineno;
 104   if (name[0] == '@' || name[0] == '=')
 105     name++;
 106   else
 107     name = "(string)";
 108   lj_assertX(startpc >= proto_bc(pt) && startpc < proto_bc(pt) + pt->sizebc,
 109              "trace PC out of range");
 110   lineno = lj_debug_line(pt, proto_bcpos(pt, startpc));
 111   if (!fp) {
 112     char fname[40];
 113     sprintf(fname, "/tmp/perf-%d.map", getpid());
 114     if (!(fp = fopen(fname, "w"))) return;
 115     setlinebuf(fp);
 116   }
 117   fprintf(fp, "%lx %x TRACE_%d::%s:%u\n",
 118           (long)T->mcode, T->szmcode, T->traceno, name, lineno);
 119 }
 120 #endif
 121
 122 /* Allocate space for copy of T. */
 123 GCtrace * LJ_FASTCALL lj_trace_alloc(lua_State *L, GCtrace *T)
 124 {
 125   size_t sztr = ((sizeof(GCtrace)+7)&~7);
 126   size_t szins = (T->nins-T->nk)*sizeof(IRIns);
 127   size_t sz = sztr + szins +
 128               T->nsnap*sizeof(SnapShot) +
 129               T->nsnapmap*sizeof(SnapEntry);
 130   GCtrace *T2 = lj_mem_newt(L, (MSize)sz, GCtrace);
 131   char *p = (char *)T2 + sztr;
 132   T2->gct = ~LJ_TTRACE;
 133   T2->marked = 0;
 134   T2->traceno = 0;
 135   T2->ir = (IRIns *)p - T->nk;
 136   T2->nins = T->nins;
 137   T2->nk = T->nk;
 138   T2->nsnap = T->nsnap;
 139   T2->nsnapmap = T->nsnapmap;
 140   memcpy(p, T->ir + T->nk, szins);
 141   return T2;
 142 }
 143
 144 /* Save current trace by copying and compacting it. */
 145 static void trace_save(jit_State *J, GCtrace *T)
 146 {
 147   size_t sztr = ((sizeof(GCtrace)+7)&~7);
 148   size_t szins = (J->cur.nins-J->cur.nk)*sizeof(IRIns);
 149   char *p = (char *)T + sztr;
 150   memcpy(T, &J->cur, sizeof(GCtrace));
 151   setgcrefr(T->nextgc, J2G(J)->gc.root);
 152   setgcrefp(J2G(J)->gc.root, T);
 153   newwhite(J2G(J), T);
 154   T->gct = ~LJ_TTRACE;
 155   T->ir = (IRIns *)p - J->cur.nk;  /* The IR has already been copied above. */
 156 #if LJ_ABI_PAUTH
 157   T->mcauth = lj_ptr_sign((ASMFunction)T->mcode, T);
 158 #endif
 159   p += szins;
 160   TRACE_APPENDVEC(snap, nsnap, SnapShot)
 161   TRACE_APPENDVEC(snapmap, nsnapmap, SnapEntry)
 162   J->cur.traceno = 0;
 163   J->curfinal = NULL;
 164   setgcrefp(J->trace[T->traceno], T);
 165   lj_gc_barriertrace(J2G(J), T->traceno);
 166   lj_gdbjit_addtrace(J, T);
 167 #ifdef LUAJIT_USE_PERFTOOLS
 168   perftools_addtrace(T);
 169 #endif
 170 }
 171
 172 void LJ_FASTCALL lj_trace_free(global_State *g, GCtrace *T)
 173 {
 174   jit_State *J = G2J(g);
 175   if (T->traceno) {
 176     lj_gdbjit_deltrace(J, T);
 177     if (T->traceno < J->freetrace)
 178       J->freetrace = T->traceno;
 179     setgcrefnull(J->trace[T->traceno]);
 180   }
 181   lj_mem_free(g, T,
 182     ((sizeof(GCtrace)+7)&~7) + (T->nins-T->nk)*sizeof(IRIns) +
 183     T->nsnap*sizeof(SnapShot) + T->nsnapmap*sizeof(SnapEntry));
 184 }
 185
 186 /* Re-enable compiling a prototype by unpatching any modified bytecode. */
 187 void lj_trace_reenableproto(GCproto *pt)
 188 {
 189   if ((pt->flags & PROTO_ILOOP)) {
 190     BCIns *bc = proto_bc(pt);
 191     BCPos i, sizebc = pt->sizebc;
 192     pt->flags &= ~PROTO_ILOOP;
 193     if (bc_op(bc[0]) == BC_IFUNCF)
 194       setbc_op(&bc[0], BC_FUNCF);
 195     for (i = 1; i < sizebc; i++) {
 196       BCOp op = bc_op(bc[i]);
 197       if (op == BC_IFORL || op == BC_IITERL || op == BC_ILOOP)
 198         setbc_op(&bc[i], (int)op+(int)BC_LOOP-(int)BC_ILOOP);
 199     }
 200   }
 201 }
 202
 203 /* Unpatch the bytecode modified by a root trace. */
 204 static void trace_unpatch(jit_State *J, GCtrace *T)
 205 {
 206   BCOp op = bc_op(T->startins);
 207   BCIns *pc = mref(T->startpc, BCIns);
 208   UNUSED(J);
 209   if (op == BC_JMP)
 210     return;  /* No need to unpatch branches in parent traces (yet). */
 211   switch (bc_op(*pc)) {
 212   case BC_JFORL:
 213     lj_assertJ(traceref(J, bc_d(*pc)) == T, "JFORL references other trace");
 214     *pc = T->startins;
 215     pc += bc_j(T->startins);
 216     lj_assertJ(bc_op(*pc) == BC_JFORI, "FORL does not point to JFORI");
 217     setbc_op(pc, BC_FORI);
 218     break;
 219   case BC_JITERL:
 220   case BC_JLOOP:
 221     lj_assertJ(op == BC_ITERL || op == BC_ITERN || op == BC_LOOP ||
 222                bc_isret(op), "bad original bytecode %d", op);
 223     *pc = T->startins;
 224     break;
 225   case BC_JMP:
 226     lj_assertJ(op == BC_ITERL, "bad original bytecode %d", op);
 227     pc += bc_j(*pc)+2;
 228     if (bc_op(*pc) == BC_JITERL) {
 229       lj_assertJ(traceref(J, bc_d(*pc)) == T, "JITERL references other trace");
 230       *pc = T->startins;
 231     }
 232     break;
 233   case BC_JFUNCF:
 234     lj_assertJ(op == BC_FUNCF, "bad original bytecode %d", op);
 235     *pc = T->startins;
 236     break;
 237   default:  /* Already unpatched. */
 238     break;
 239   }
 240 }
 241
 242 /* Flush a root trace. */
 243 static void trace_flushroot(jit_State *J, GCtrace *T)
 244 {
 245   GCproto *pt = &gcref(T->startpt)->pt;
 246   lj_assertJ(T->root == 0, "not a root trace");
 247   lj_assertJ(pt != NULL, "trace has no prototype");
 248   /* First unpatch any modified bytecode. */
 249   trace_unpatch(J, T);
 250   /* Unlink root trace from chain anchored in prototype. */
 251   if (pt->trace == T->traceno) {  /* Trace is first in chain. Easy. */
 252     pt->trace = T->nextroot;
 253   } else if (pt->trace) {  /* Otherwise search in chain of root traces. */
 254     GCtrace *T2 = traceref(J, pt->trace);
 255     if (T2) {
 256       for (; T2->nextroot; T2 = traceref(J, T2->nextroot))
 257         if (T2->nextroot == T->traceno) {
 258           T2->nextroot = T->nextroot;  /* Unlink from chain. */
 259           break;
 260         }
 261     }
 262   }
 263 }
 264
 265 /* Flush a trace. Only root traces are considered. */
 266 void lj_trace_flush(jit_State *J, TraceNo traceno)
 267 {
 268   if (traceno > 0 && traceno < J->sizetrace) {
 269     GCtrace *T = traceref(J, traceno);
 270     if (T && T->root == 0)
 271       trace_flushroot(J, T);
 272   }
 273 }
 274
 275 /* Flush all traces associated with a prototype. */
 276 void lj_trace_flushproto(global_State *g, GCproto *pt)
 277 {
 278   while (pt->trace != 0)
 279     trace_flushroot(G2J(g), traceref(G2J(g), pt->trace));
 280 }
 281
 282 /* Flush all traces. */
 283 int lj_trace_flushall(lua_State *L)
 284 {
 285   jit_State *J = L2J(L);
 286   ptrdiff_t i;
 287   if ((J2G(J)->hookmask & HOOK_GC))
 288     return 1;
 289   for (i = (ptrdiff_t)J->sizetrace-1; i > 0; i--) {
 290     GCtrace *T = traceref(J, i);
 291     if (T) {
 292       if (T->root == 0)
 293         trace_flushroot(J, T);
 294       lj_gdbjit_deltrace(J, T);
 295       T->traceno = T->link = 0;  /* Blacklist the link for cont_stitch. */
 296       setgcrefnull(J->trace[i]);
 297     }
 298   }
 299   J->cur.traceno = 0;
 300   J->freetrace = 0;
 301   /* Clear penalty cache. */
 302   memset(J->penalty, 0, sizeof(J->penalty));
 303   /* Free the whole machine code and invalidate all exit stub groups. */
 304   lj_mcode_free(J);
 305   memset(J->exitstubgroup, 0, sizeof(J->exitstubgroup));
 306   lj_vmevent_send(L, TRACE,
 307     setstrV(L, L->top++, lj_str_newlit(L, "flush"));
 308   );
 309   return 0;
 310 }
 311
 312 /* Initialize JIT compiler state. */
 313 void lj_trace_initstate(global_State *g)
 314 {
 315   jit_State *J = G2J(g);
 316   TValue *tv;
 317
 318   /* Initialize aligned SIMD constants. */
 319   tv = LJ_KSIMD(J, LJ_KSIMD_ABS);
 320   tv[0].u64 = U64x(7fffffff,ffffffff);
 321   tv[1].u64 = U64x(7fffffff,ffffffff);
 322   tv = LJ_KSIMD(J, LJ_KSIMD_NEG);
 323   tv[0].u64 = U64x(80000000,00000000);
 324   tv[1].u64 = U64x(80000000,00000000);
 325
 326   /* Initialize 32/64 bit constants. */
 327 #if LJ_TARGET_X86ORX64
 328   J->k64[LJ_K64_TOBIT].u64 = U64x(43380000,00000000);
 329 #if LJ_32
 330   J->k64[LJ_K64_M2P64_31].u64 = U64x(c1e00000,00000000);
 331 #endif
 332   J->k64[LJ_K64_2P64].u64 = U64x(43f00000,00000000);
 333   J->k32[LJ_K32_M2P64_31] = LJ_64 ? 0xdf800000 : 0xcf000000;
 334 #endif
 335 #if LJ_TARGET_X86ORX64 || LJ_TARGET_MIPS64
 336   J->k64[LJ_K64_M2P64].u64 = U64x(c3f00000,00000000);
 337 #endif
 338 #if LJ_TARGET_PPC
 339   J->k32[LJ_K32_2P52_2P31] = 0x59800004;
 340   J->k32[LJ_K32_2P52] = 0x59800000;
 341 #endif
 342 #if LJ_TARGET_PPC || LJ_TARGET_MIPS
 343   J->k32[LJ_K32_2P31] = 0x4f000000;
 344 #endif
 345 #if LJ_TARGET_MIPS
 346   J->k64[LJ_K64_2P31].u64 = U64x(41e00000,00000000);
 347 #if LJ_64
 348   J->k64[LJ_K64_2P63].u64 = U64x(43e00000,00000000);
 349   J->k32[LJ_K32_2P63] = 0x5f000000;
 350   J->k32[LJ_K32_M2P64] = 0xdf800000;
 351 #endif
 352 #endif
 353 }
 354
 355 /* Free everything associated with the JIT compiler state. */
 356 void lj_trace_freestate(global_State *g)
 357 {
 358   jit_State *J = G2J(g);
 359 #ifdef LUA_USE_ASSERT
 360   {  /* This assumes all traces have already been freed. */
 361     ptrdiff_t i;
 362     for (i = 1; i < (ptrdiff_t)J->sizetrace; i++)
 363       lj_assertG(i == (ptrdiff_t)J->cur.traceno || traceref(J, i) == NULL,
 364                  "trace still allocated");
 365   }
 366 #endif
 367   lj_mcode_free(J);
 368   lj_mem_freevec(g, J->snapmapbuf, J->sizesnapmap, SnapEntry);
 369   lj_mem_freevec(g, J->snapbuf, J->sizesnap, SnapShot);
 370   lj_mem_freevec(g, J->irbuf + J->irbotlim, J->irtoplim - J->irbotlim, IRIns);
 371   lj_mem_freevec(g, J->trace, J->sizetrace, GCRef);
 372 }
 373
 374 /* -- Penalties and blacklisting ------------------------------------------ */
 375
 376 /* Blacklist a bytecode instruction. */
 377 static void blacklist_pc(GCproto *pt, BCIns *pc)
 378 {
 379   if (bc_op(*pc) == BC_ITERN) {
 380     setbc_op(pc, BC_ITERC);
 381     setbc_op(pc+1+bc_j(pc[1]), BC_JMP);
 382   } else {
 383     setbc_op(pc, (int)bc_op(*pc)+(int)BC_ILOOP-(int)BC_LOOP);
 384     pt->flags |= PROTO_ILOOP;
 385   }
 386 }
 387
 388 /* Penalize a bytecode instruction. */
 389 static void penalty_pc(jit_State *J, GCproto *pt, BCIns *pc, TraceError e)
 390 {
 391   uint32_t i, val = PENALTY_MIN;
 392   for (i = 0; i < PENALTY_SLOTS; i++)
 393     if (mref(J->penalty[i].pc, const BCIns) == pc) {  /* Cache slot found? */
 394       /* First try to bump its hotcount several times. */
 395       val = ((uint32_t)J->penalty[i].val << 1) +
 396             (lj_prng_u64(&J2G(J)->prng) & ((1u<<PENALTY_RNDBITS)-1));
 397       if (val > PENALTY_MAX) {
 398         blacklist_pc(pt, pc);  /* Blacklist it, if that didn't help. */
 399         return;
 400       }
 401       goto setpenalty;
 402     }
 403   /* Assign a new penalty cache slot. */
 404   i = J->penaltyslot;
 405   J->penaltyslot = (J->penaltyslot + 1) & (PENALTY_SLOTS-1);
 406   setmref(J->penalty[i].pc, pc);
 407 setpenalty:
 408   J->penalty[i].val = (uint16_t)val;
 409   J->penalty[i].reason = e;
 410   hotcount_set(J2GG(J), pc+1, val);
 411 }
 412
 413 /* -- Trace compiler state machine ---------------------------------------- */
 414
 415 /* Start tracing. */
 416 static void trace_start(jit_State *J)
 417 {
 418   lua_State *L;
 419   TraceNo traceno;
 420
 421   if ((J->pt->flags & PROTO_NOJIT)) {  /* JIT disabled for this proto? */
 422     if (J->parent == 0 && J->exitno == 0 && bc_op(*J->pc) != BC_ITERN) {
 423       /* Lazy bytecode patching to disable hotcount events. */
 424       lj_assertJ(bc_op(*J->pc) == BC_FORL || bc_op(*J->pc) == BC_ITERL ||
 425                  bc_op(*J->pc) == BC_LOOP || bc_op(*J->pc) == BC_FUNCF,
 426                  "bad hot bytecode %d", bc_op(*J->pc));
 427       setbc_op(J->pc, (int)bc_op(*J->pc)+(int)BC_ILOOP-(int)BC_LOOP);
 428       J->pt->flags |= PROTO_ILOOP;
 429     }
 430     J->state = LJ_TRACE_IDLE;  /* Silently ignored. */
 431     return;
 432   }
 433
 434   /* Ensuring forward progress for BC_ITERN can trigger hotcount again. */
 435   if (!J->parent && bc_op(*J->pc) == BC_JLOOP) {  /* Already compiled. */
 436     J->state = LJ_TRACE_IDLE;  /* Silently ignored. */
 437     return;
 438   }
 439
 440   /* Get a new trace number. */
 441   traceno = trace_findfree(J);
 442   if (LJ_UNLIKELY(traceno == 0)) {  /* No free trace? */
 443     lj_assertJ((J2G(J)->hookmask & HOOK_GC) == 0,
 444                "recorder called from GC hook");
 445     lj_trace_flushall(J->L);
 446     J->state = LJ_TRACE_IDLE;  /* Silently ignored. */
 447     return;
 448   }
 449   setgcrefp(J->trace[traceno], &J->cur);
 450
 451   /* Setup enough of the current trace to be able to send the vmevent. */
 452   memset(&J->cur, 0, sizeof(GCtrace));
 453   J->cur.traceno = traceno;
 454   J->cur.nins = J->cur.nk = REF_BASE;
 455   J->cur.ir = J->irbuf;
 456   J->cur.snap = J->snapbuf;
 457   J->cur.snapmap = J->snapmapbuf;
 458   J->mergesnap = 0;
 459   J->needsnap = 0;
 460   J->bcskip = 0;
 461   J->guardemit.irt = 0;
 462   J->postproc = LJ_POST_NONE;
 463   lj_resetsplit(J);
 464   J->retryrec = 0;
 465   J->ktrace = 0;
 466   setgcref(J->cur.startpt, obj2gco(J->pt));
 467
 468   L = J->L;
 469   lj_vmevent_send(L, TRACE,
 470     setstrV(L, L->top++, lj_str_newlit(L, "start"));
 471     setintV(L->top++, traceno);
 472     setfuncV(L, L->top++, J->fn);
 473     setintV(L->top++, proto_bcpos(J->pt, J->pc));
 474     if (J->parent) {
 475       setintV(L->top++, J->parent);
 476       setintV(L->top++, J->exitno);
 477     } else {
 478       BCOp op = bc_op(*J->pc);
 479       if (op == BC_CALLM || op == BC_CALL || op == BC_ITERC) {
 480         setintV(L->top++, J->exitno);  /* Parent of stitched trace. */
 481         setintV(L->top++, -1);
 482       }
 483     }
 484   );
 485   lj_record_setup(J);
 486 }
 487
 488 /* Stop tracing. */
 489 static void trace_stop(jit_State *J)
 490 {
 491   BCIns *pc = mref(J->cur.startpc, BCIns);
 492   BCOp op = bc_op(J->cur.startins);
 493   GCproto *pt = &gcref(J->cur.startpt)->pt;
 494   TraceNo traceno = J->cur.traceno;
 495   GCtrace *T = J->curfinal;
 496   lua_State *L;
 497
 498   switch (op) {
 499   case BC_FORL:
 500     setbc_op(pc+bc_j(J->cur.startins), BC_JFORI);  /* Patch FORI, too. */
 501     /* fallthrough */
 502   case BC_LOOP:
 503   case BC_ITERL:
 504   case BC_FUNCF:
 505     /* Patch bytecode of starting instruction in root trace. */
 506     setbc_op(pc, (int)op+(int)BC_JLOOP-(int)BC_LOOP);
 507     setbc_d(pc, traceno);
 508   addroot:
 509     /* Add to root trace chain in prototype. */
 510     J->cur.nextroot = pt->trace;
 511     pt->trace = (TraceNo1)traceno;
 512     break;
 513   case BC_ITERN:
 514   case BC_RET:
 515   case BC_RET0:
 516   case BC_RET1:
 517     *pc = BCINS_AD(BC_JLOOP, J->cur.snap[0].nslots, traceno);
 518     goto addroot;
 519   case BC_JMP:
 520     /* Patch exit branch in parent to side trace entry. */
 521     lj_assertJ(J->parent != 0 && J->cur.root != 0, "not a side trace");
 522     lj_asm_patchexit(J, traceref(J, J->parent), J->exitno, J->cur.mcode);
 523     /* Avoid compiling a side trace twice (stack resizing uses parent exit). */
 524     {
 525       SnapShot *snap = &traceref(J, J->parent)->snap[J->exitno];
 526       snap->count = SNAPCOUNT_DONE;
 527       if (J->cur.topslot > snap->topslot) snap->topslot = J->cur.topslot;
 528     }
 529     /* Add to side trace chain in root trace. */
 530     {
 531       GCtrace *root = traceref(J, J->cur.root);
 532       root->nchild++;
 533       J->cur.nextside = root->nextside;
 534       root->nextside = (TraceNo1)traceno;
 535     }
 536     break;
 537   case BC_CALLM:
 538   case BC_CALL:
 539   case BC_ITERC:
 540     /* Trace stitching: patch link of previous trace. */
 541     traceref(J, J->exitno)->link = traceno;
 542     break;
 543   default:
 544     lj_assertJ(0, "bad stop bytecode %d", op);
 545     break;
 546   }
 547
 548   /* Commit new mcode only after all patching is done. */
 549   lj_mcode_commit(J, J->cur.mcode);
 550   J->postproc = LJ_POST_NONE;
 551   trace_save(J, T);
 552
 553   L = J->L;
 554   lj_vmevent_send(L, TRACE,
 555     setstrV(L, L->top++, lj_str_newlit(L, "stop"));
 556     setintV(L->top++, traceno);
 557     setfuncV(L, L->top++, J->fn);
 558   );
 559 }
 560
 561 /* Start a new root trace for down-recursion. */
 562 static int trace_downrec(jit_State *J)
 563 {
 564   /* Restart recording at the return instruction. */
 565   lj_assertJ(J->pt != NULL, "no active prototype");
 566   lj_assertJ(bc_isret(bc_op(*J->pc)), "not at a return bytecode");
 567   if (bc_op(*J->pc) == BC_RETM)
 568     return 0;  /* NYI: down-recursion with RETM. */
 569   J->parent = 0;
 570   J->exitno = 0;
 571   J->state = LJ_TRACE_RECORD;
 572   trace_start(J);
 573   return 1;
 574 }
 575
 576 /* Abort tracing. */
 577 static int trace_abort(jit_State *J)
 578 {
 579   lua_State *L = J->L;
 580   TraceError e = LJ_TRERR_RECERR;
 581   TraceNo traceno;
 582
 583   J->postproc = LJ_POST_NONE;
 584   lj_mcode_abort(J);
 585   if (J->curfinal) {
 586     lj_trace_free(J2G(J), J->curfinal);
 587     J->curfinal = NULL;
 588   }
 589   if (tvisnumber(L->top-1))
 590     e = (TraceError)numberVint(L->top-1);
 591   if (e == LJ_TRERR_MCODELM) {
 592     L->top--;  /* Remove error object */
 593     J->state = LJ_TRACE_ASM;
 594     return 1;  /* Retry ASM with new MCode area. */
 595   }
 596   /* Penalize or blacklist starting bytecode instruction. */
 597   if (J->parent == 0 && !bc_isret(bc_op(J->cur.startins))) {
 598     if (J->exitno == 0) {
 599       BCIns *startpc = mref(J->cur.startpc, BCIns);
 600       if (e == LJ_TRERR_RETRY)
 601         hotcount_set(J2GG(J), startpc+1, 1);  /* Immediate retry. */
 602       else
 603         penalty_pc(J, &gcref(J->cur.startpt)->pt, startpc, e);
 604     } else {
 605       traceref(J, J->exitno)->link = J->exitno;  /* Self-link is blacklisted. */
 606     }
 607   }
 608
 609   /* Is there anything to abort? */
 610   traceno = J->cur.traceno;
 611   if (traceno) {
 612     ptrdiff_t errobj = savestack(L, L->top-1);  /* Stack may be resized. */
 613     J->cur.link = 0;
 614     J->cur.linktype = LJ_TRLINK_NONE;
 615     lj_vmevent_send(L, TRACE,
 616       cTValue *bot = tvref(L->stack)+LJ_FR2;
 617       cTValue *frame;
 618       const BCIns *pc;
 619       BCPos pos = 0;
 620       setstrV(L, L->top++, lj_str_newlit(L, "abort"));
 621       setintV(L->top++, traceno);
 622       /* Find original Lua function call to generate a better error message. */
 623       for (frame = J->L->base-1, pc = J->pc; ; frame = frame_prev(frame)) {
 624         if (isluafunc(frame_func(frame))) {
 625           pos = proto_bcpos(funcproto(frame_func(frame)), pc);
 626           break;
 627         } else if (frame_prev(frame) <= bot) {
 628           break;
 629         } else if (frame_iscont(frame)) {
 630           pc = frame_contpc(frame) - 1;
 631         } else {
 632           pc = frame_pc(frame) - 1;
 633         }
 634       }
 635       setfuncV(L, L->top++, frame_func(frame));
 636       setintV(L->top++, pos);
 637       copyTV(L, L->top++, restorestack(L, errobj));
 638       copyTV(L, L->top++, &J->errinfo);
 639     );
 640     /* Drop aborted trace after the vmevent (which may still access it). */
 641     setgcrefnull(J->trace[traceno]);
 642     if (traceno < J->freetrace)
 643       J->freetrace = traceno;
 644     J->cur.traceno = 0;
 645   }
 646   L->top--;  /* Remove error object */
 647   if (e == LJ_TRERR_DOWNREC)
 648     return trace_downrec(J);
 649   else if (e == LJ_TRERR_MCODEAL)
 650     lj_trace_flushall(L);
 651   return 0;
 652 }
 653
 654 /* Perform pending re-patch of a bytecode instruction. */
 655 static LJ_AINLINE void trace_pendpatch(jit_State *J, int force)
 656 {
 657   if (LJ_UNLIKELY(J->patchpc)) {
 658     if (force || J->bcskip == 0) {
 659       *J->patchpc = J->patchins;
 660       J->patchpc = NULL;
 661     } else {
 662       J->bcskip = 0;
 663     }
 664   }
 665 }
 666
 667 /* State machine for the trace compiler. Protected callback. */
 668 static TValue *trace_state(lua_State *L, lua_CFunction dummy, void *ud)
 669 {
 670   jit_State *J = (jit_State *)ud;
 671   UNUSED(dummy);
 672   do {
 673   retry:
 674     switch (J->state) {
 675     case LJ_TRACE_START:
 676       J->state = LJ_TRACE_RECORD;  /* trace_start() may change state. */
 677       trace_start(J);
 678       lj_dispatch_update(J2G(J));
 679       if (J->state != LJ_TRACE_RECORD_1ST)
 680         break;
 681       /* fallthrough */
 682
 683     case LJ_TRACE_RECORD_1ST:
 684       J->state = LJ_TRACE_RECORD;
 685       /* fallthrough */
 686     case LJ_TRACE_RECORD:
 687       trace_pendpatch(J, 0);
 688       setvmstate(J2G(J), RECORD);
 689       lj_vmevent_send_(L, RECORD,
 690         /* Save/restore state for trace recorder. */
 691         TValue savetv = J2G(J)->tmptv;
 692         TValue savetv2 = J2G(J)->tmptv2;
 693         TraceNo parent = J->parent;
 694         ExitNo exitno = J->exitno;
 695         setintV(L->top++, J->cur.traceno);
 696         setfuncV(L, L->top++, J->fn);
 697         setintV(L->top++, J->pt ? (int32_t)proto_bcpos(J->pt, J->pc) : -1);
 698         setintV(L->top++, J->framedepth);
 699       ,
 700         J2G(J)->tmptv = savetv;
 701         J2G(J)->tmptv2 = savetv2;
 702         J->parent = parent;
 703         J->exitno = exitno;
 704       );
 705       lj_record_ins(J);
 706       break;
 707
 708     case LJ_TRACE_END:
 709       trace_pendpatch(J, 1);
 710       J->loopref = 0;
 711       if ((J->flags & JIT_F_OPT_LOOP) &&
 712           J->cur.link == J->cur.traceno && J->framedepth + J->retdepth == 0) {
 713         setvmstate(J2G(J), OPT);
 714         lj_opt_dce(J);
 715         if (lj_opt_loop(J)) {  /* Loop optimization failed? */
 716           J->cur.link = 0;
 717           J->cur.linktype = LJ_TRLINK_NONE;
 718           J->loopref = J->cur.nins;
 719           J->state = LJ_TRACE_RECORD;  /* Try to continue recording. */
 720           break;
 721         }
 722         J->loopref = J->chain[IR_LOOP];  /* Needed by assembler. */
 723       }
 724       lj_opt_split(J);
 725       lj_opt_sink(J);
 726       if (!J->loopref) J->cur.snap[J->cur.nsnap-1].count = SNAPCOUNT_DONE;
 727       J->state = LJ_TRACE_ASM;
 728       break;
 729
 730     case LJ_TRACE_ASM:
 731       setvmstate(J2G(J), ASM);
 732       lj_asm_trace(J, &J->cur);
 733       trace_stop(J);
 734       setvmstate(J2G(J), INTERP);
 735       J->state = LJ_TRACE_IDLE;
 736       lj_dispatch_update(J2G(J));
 737       return NULL;
 738
 739     default:  /* Trace aborted asynchronously. */
 740       setintV(L->top++, (int32_t)LJ_TRERR_RECERR);
 741       /* fallthrough */
 742     case LJ_TRACE_ERR:
 743       trace_pendpatch(J, 1);
 744       if (trace_abort(J))
 745         goto retry;
 746       setvmstate(J2G(J), INTERP);
 747       J->state = LJ_TRACE_IDLE;
 748       lj_dispatch_update(J2G(J));
 749       return NULL;
 750     }
 751   } while (J->state > LJ_TRACE_RECORD);
 752   return NULL;
 753 }
 754
 755 /* -- Event handling ------------------------------------------------------ */
 756
 757 /* A bytecode instruction is about to be executed. Record it. */
 758 void lj_trace_ins(jit_State *J, const BCIns *pc)
 759 {
 760   /* Note: J->L must already be set. pc is the true bytecode PC here. */
 761   J->pc = pc;
 762   J->fn = curr_func(J->L);
 763   J->pt = isluafunc(J->fn) ? funcproto(J->fn) : NULL;
 764   while (lj_vm_cpcall(J->L, NULL, (void *)J, trace_state) != 0)
 765     J->state = LJ_TRACE_ERR;
 766 }
 767
 768 /* A hotcount triggered. Start recording a root trace. */
 769 void LJ_FASTCALL lj_trace_hot(jit_State *J, const BCIns *pc)
 770 {
 771   /* Note: pc is the interpreter bytecode PC here. It's offset by 1. */
 772   ERRNO_SAVE
 773   /* Reset hotcount. */
 774   hotcount_set(J2GG(J), pc, J->param[JIT_P_hotloop]*HOTCOUNT_LOOP);
 775   /* Only start a new trace if not recording or inside __gc call or vmevent. */
 776   if (J->state == LJ_TRACE_IDLE &&
 777       !(J2G(J)->hookmask & (HOOK_GC|HOOK_VMEVENT))) {
 778     J->parent = 0;  /* Root trace. */
 779     J->exitno = 0;
 780     J->state = LJ_TRACE_START;
 781     lj_trace_ins(J, pc-1);
 782   }
 783   ERRNO_RESTORE
 784 }
 785
 786 /* Check for a hot side exit. If yes, start recording a side trace. */
 787 static void trace_hotside(jit_State *J, const BCIns *pc)
 788 {
 789   SnapShot *snap = &traceref(J, J->parent)->snap[J->exitno];
 790   if (!(J2G(J)->hookmask & (HOOK_GC|HOOK_VMEVENT)) &&
 791       isluafunc(curr_func(J->L)) &&
 792       snap->count != SNAPCOUNT_DONE &&
 793       ++snap->count >= J->param[JIT_P_hotexit]) {
 794     lj_assertJ(J->state == LJ_TRACE_IDLE, "hot side exit while recording");
 795     /* J->parent is non-zero for a side trace. */
 796     J->state = LJ_TRACE_START;
 797     lj_trace_ins(J, pc);
 798   }
 799 }
 800
 801 /* Stitch a new trace to the previous trace. */
 802 void LJ_FASTCALL lj_trace_stitch(jit_State *J, const BCIns *pc)
 803 {
 804   /* Only start a new trace if not recording or inside __gc call or vmevent. */
 805   if (J->state == LJ_TRACE_IDLE &&
 806       !(J2G(J)->hookmask & (HOOK_GC|HOOK_VMEVENT))) {
 807     J->parent = 0;  /* Have to treat it like a root trace. */
 808     /* J->exitno is set to the invoking trace. */
 809     J->state = LJ_TRACE_START;
 810     lj_trace_ins(J, pc);
 811   }
 812 }
 813
 814
 815 /* Tiny struct to pass data to protected call. */
 816 typedef struct ExitDataCP {
 817   jit_State *J;
 818   void *exptr;          /* Pointer to exit state. */
 819   const BCIns *pc;      /* Restart interpreter at this PC. */
 820 } ExitDataCP;
 821
 822 /* Need to protect lj_snap_restore because it may throw. */
 823 static TValue *trace_exit_cp(lua_State *L, lua_CFunction dummy, void *ud)
 824 {
 825   ExitDataCP *exd = (ExitDataCP *)ud;
 826   /* Always catch error here and don't call error function. */
 827   cframe_errfunc(L->cframe) = 0;
 828   cframe_nres(L->cframe) = -2*LUAI_MAXSTACK*(int)sizeof(TValue);
 829   exd->pc = lj_snap_restore(exd->J, exd->exptr);
 830   UNUSED(dummy);
 831   return NULL;
 832 }
 833
 834 #ifndef LUAJIT_DISABLE_VMEVENT
 835 /* Push all registers from exit state. */
 836 static void trace_exit_regs(lua_State *L, ExitState *ex)
 837 {
 838   int32_t i;
 839   setintV(L->top++, RID_NUM_GPR);
 840   setintV(L->top++, RID_NUM_FPR);
 841   for (i = 0; i < RID_NUM_GPR; i++) {
 842     if (sizeof(ex->gpr[i]) == sizeof(int32_t))
 843       setintV(L->top++, (int32_t)ex->gpr[i]);
 844     else
 845       setnumV(L->top++, (lua_Number)ex->gpr[i]);
 846   }
 847 #if !LJ_SOFTFP
 848   for (i = 0; i < RID_NUM_FPR; i++) {
 849     setnumV(L->top, ex->fpr[i]);
 850     if (LJ_UNLIKELY(tvisnan(L->top)))
 851       setnanV(L->top);
 852     L->top++;
 853   }
 854 #endif
 855 }
 856 #endif
 857
 858 #if defined(EXITSTATE_PCREG) || (LJ_UNWIND_JIT && !EXITTRACE_VMSTATE)
 859 /* Determine trace number from pc of exit instruction. */
 860 static TraceNo trace_exit_find(jit_State *J, MCode *pc)
 861 {
 862   TraceNo traceno;
 863   for (traceno = 1; traceno < J->sizetrace; traceno++) {
 864     GCtrace *T = traceref(J, traceno);
 865     if (T && pc >= T->mcode && pc < (MCode *)((char *)T->mcode + T->szmcode))
 866       return traceno;
 867   }
 868   lj_assertJ(0, "bad exit pc");
 869   return 0;
 870 }
 871 #endif
 872
 873 /* A trace exited. Restore interpreter state. */
 874 int LJ_FASTCALL lj_trace_exit(jit_State *J, void *exptr)
 875 {
 876   ERRNO_SAVE
 877   lua_State *L = J->L;
 878   ExitState *ex = (ExitState *)exptr;
 879   ExitDataCP exd;
 880   int errcode, exitcode = J->exitcode;
 881   TValue exiterr;
 882   const BCIns *pc, *retpc;
 883   void *cf;
 884   GCtrace *T;
 885
 886   setnilV(&exiterr);
 887   if (exitcode) {  /* Trace unwound with error code. */
 888     J->exitcode = 0;
 889     copyTV(L, &exiterr, L->top-1);
 890   }
 891
 892 #ifdef EXITSTATE_PCREG
 893   J->parent = trace_exit_find(J, (MCode *)(intptr_t)ex->gpr[EXITSTATE_PCREG]);
 894 #endif
 895   T = traceref(J, J->parent); UNUSED(T);
 896 #ifdef EXITSTATE_CHECKEXIT
 897   if (J->exitno == T->nsnap) {  /* Treat stack check like a parent exit. */
 898     lj_assertJ(T->root != 0, "stack check in root trace");
 899     J->exitno = T->ir[REF_BASE].op2;
 900     J->parent = T->ir[REF_BASE].op1;
 901     T = traceref(J, J->parent);
 902   }
 903 #endif
 904   lj_assertJ(T != NULL && J->exitno < T->nsnap, "bad trace or exit number");
 905   exd.J = J;
 906   exd.exptr = exptr;
 907   errcode = lj_vm_cpcall(L, NULL, &exd, trace_exit_cp);
 908   if (errcode)
 909     return -errcode;  /* Return negated error code. */
 910
 911   if (exitcode) copyTV(L, L->top++, &exiterr);  /* Anchor the error object. */
 912
 913   if (!(LJ_HASPROFILE && (G(L)->hookmask & HOOK_PROFILE)))
 914     lj_vmevent_send(L, TEXIT,
 915       lj_state_checkstack(L, 4+RID_NUM_GPR+RID_NUM_FPR+LUA_MINSTACK);
 916       setintV(L->top++, J->parent);
 917       setintV(L->top++, J->exitno);
 918       trace_exit_regs(L, ex);
 919     );
 920
 921   pc = exd.pc;
 922   cf = cframe_raw(L->cframe);
 923   setcframe_pc(cf, pc);
 924   if (exitcode) {
 925     return -exitcode;
 926   } else if (LJ_HASPROFILE && (G(L)->hookmask & HOOK_PROFILE)) {
 927     /* Just exit to interpreter. */
 928   } else if (G(L)->gc.state == GCSatomic || G(L)->gc.state == GCSfinalize) {
 929     if (!(G(L)->hookmask & HOOK_GC))
 930       lj_gc_step(L);  /* Exited because of GC: drive GC forward. */
 931   } else if ((J->flags & JIT_F_ON)) {
 932     trace_hotside(J, pc);
 933   }
 934   /* Return MULTRES or 0 or -17. */
 935   ERRNO_RESTORE
 936   switch (bc_op(*pc)) {
 937   case BC_CALLM: case BC_CALLMT:
 938     return (int)((BCReg)(L->top - L->base) - bc_a(*pc) - bc_c(*pc) - LJ_FR2);
 939   case BC_RETM:
 940     return (int)((BCReg)(L->top - L->base) + 1 - bc_a(*pc) - bc_d(*pc));
 941   case BC_TSETM:
 942     return (int)((BCReg)(L->top - L->base) + 1 - bc_a(*pc));
 943   case BC_JLOOP:
 944     retpc = &traceref(J, bc_d(*pc))->startins;
 945     if (bc_isret(bc_op(*retpc)) || bc_op(*retpc) == BC_ITERN) {
 946       /* Dispatch to original ins to ensure forward progress. */
 947       if (J->state != LJ_TRACE_RECORD) return -17;
 948       /* Unpatch bytecode when recording. */
 949       J->patchins = *pc;
 950       J->patchpc = (BCIns *)pc;
 951       *J->patchpc = *retpc;
 952       J->bcskip = 1;
 953     }
 954     return 0;
 955   default:
 956     if (bc_op(*pc) >= BC_FUNCF)
 957       return (int)((BCReg)(L->top - L->base) + 1);
 958     return 0;
 959   }
 960 }
 961
 962 #if LJ_UNWIND_JIT
 963 /* Given an mcode address determine trace exit address for unwinding. */
 964 uintptr_t LJ_FASTCALL lj_trace_unwind(jit_State *J, uintptr_t addr, ExitNo *ep)
 965 {
 966 #if EXITTRACE_VMSTATE
 967   TraceNo traceno = J2G(J)->vmstate;
 968 #else
 969   TraceNo traceno = trace_exit_find(J, (MCode *)addr);
 970 #endif
 971   GCtrace *T = traceref(J, traceno);
 972   if (T
 973 #if EXITTRACE_VMSTATE
 974       && addr >= (uintptr_t)T->mcode && addr < (uintptr_t)T->mcode + T->szmcode
 975 #endif
 976      ) {
 977     SnapShot *snap = T->snap;
 978     SnapNo lo = 0, exitno = T->nsnap;
 979     uintptr_t ofs = (uintptr_t)((MCode *)addr - T->mcode);  /* MCode units! */
 980     /* Rightmost binary search for mcode offset to determine exit number. */
 981     do {
 982       SnapNo mid = (lo+exitno) >> 1;
 983       if (ofs < snap[mid].mcofs) exitno = mid; else lo = mid + 1;
 984     } while (lo < exitno);
 985     exitno--;
 986     *ep = exitno;
 987 #ifdef EXITSTUBS_PER_GROUP
 988     return (uintptr_t)exitstub_addr(J, exitno);
 989 #else
 990     return (uintptr_t)exitstub_trace_addr(T, exitno);
 991 #endif
 992   }
 993   /* Cannot correlate addr with trace/exit. This will be fatal. */
 994   lj_assertJ(0, "bad exit pc");
 995   return 0;
 996 }
 997 #endif
 998
 999 #endif