src/lj_err.c

   1 /*
   2 ** Error handling.
   3 ** Copyright (C) 2005-2025 Mike Pall. See Copyright Notice in luajit.h
   4 */
   5
   6 #define lj_err_c
   7 #define LUA_CORE
   8
   9 #include "lj_obj.h"
  10 #include "lj_err.h"
  11 #include "lj_debug.h"
  12 #include "lj_str.h"
  13 #include "lj_func.h"
  14 #include "lj_state.h"
  15 #include "lj_frame.h"
  16 #include "lj_ff.h"
  17 #include "lj_trace.h"
  18 #include "lj_vm.h"
  19 #include "lj_strfmt.h"
  20
  21 /*
  22 ** LuaJIT can either use internal or external frame unwinding:
  23 **
  24 ** - Internal frame unwinding (INT) is free-standing and doesn't require
  25 **   any OS or library support.
  26 **
  27 ** - External frame unwinding (EXT) uses the system-provided unwind handler.
  28 **
  29 ** Pros and Cons:
  30 **
  31 ** - EXT requires unwind tables for *all* functions on the C stack between
  32 **   the pcall/catch and the error/throw. C modules used by Lua code can
  33 **   throw errors, so these need to have unwind tables, too. Transitively
  34 **   this applies to all system libraries used by C modules -- at least
  35 **   when they have callbacks which may throw an error.
  36 **
  37 ** - INT is faster when actually throwing errors, but this happens rarely.
  38 **   Setting up error handlers is zero-cost in any case.
  39 **
  40 ** - INT needs to save *all* callee-saved registers when entering the
  41 **   interpreter. EXT only needs to save those actually used inside the
  42 **   interpreter. JIT-compiled code may need to save some more.
  43 **
  44 ** - EXT provides full interoperability with C++ exceptions. You can throw
  45 **   Lua errors or C++ exceptions through a mix of Lua frames and C++ frames.
  46 **   C++ destructors are called as needed. C++ exceptions caught by pcall
  47 **   are converted to the string "C++ exception". Lua errors can be caught
  48 **   with catch (...) in C++.
  49 **
  50 ** - INT has only limited support for automatically catching C++ exceptions
  51 **   on POSIX systems using DWARF2 stack unwinding. Other systems may use
  52 **   the wrapper function feature. Lua errors thrown through C++ frames
  53 **   cannot be caught by C++ code and C++ destructors are not run.
  54 **
  55 ** - EXT can handle errors from internal helper functions that are called
  56 **   from JIT-compiled code (except for Windows/x86 and 32 bit ARM).
  57 **   INT has no choice but to call the panic handler, if this happens.
  58 **   Note: this is mainly relevant for out-of-memory errors.
  59 **
  60 ** EXT is the default on all systems where the toolchain produces unwind
  61 ** tables by default (*). This is hard-coded and/or detected in src/Makefile.
  62 ** You can thwart the detection with: TARGET_XCFLAGS=-DLUAJIT_UNWIND_INTERNAL
  63 **
  64 ** INT is the default on all other systems.
  65 **
  66 ** EXT can be manually enabled for toolchains that are able to produce
  67 ** conforming unwind tables:
  68 **   "TARGET_XCFLAGS=-funwind-tables -DLUAJIT_UNWIND_EXTERNAL"
  69 ** As explained above, *all* C code used directly or indirectly by LuaJIT
  70 ** must be compiled with -funwind-tables (or -fexceptions). C++ code must
  71 ** *not* be compiled with -fno-exceptions.
  72 **
  73 ** If you're unsure whether error handling inside the VM works correctly,
  74 ** try running this and check whether it prints "OK":
  75 **
  76 **   luajit -e "print(select(2, load('OK')):match('OK'))"
  77 **
  78 ** (*) Originally, toolchains only generated unwind tables for C++ code. For
  79 ** interoperability reasons, this can be manually enabled for plain C code,
  80 ** too (with -funwind-tables). With the introduction of the x64 architecture,
  81 ** the corresponding POSIX and Windows ABIs mandated unwind tables for all
  82 ** code. Over the following years most desktop and server platforms have
  83 ** enabled unwind tables by default on all architectures. OTOH mobile and
  84 ** embedded platforms do not consistently mandate unwind tables.
  85 */
  86
  87 /* -- Error messages ------------------------------------------------------ */
  88
  89 /* Error message strings. */
  90 LJ_DATADEF const char *lj_err_allmsg =
  91 #define ERRDEF(name, msg)       msg "\0"
  92 #include "lj_errmsg.h"
  93 ;
  94
  95 /* -- Internal frame unwinding -------------------------------------------- */
  96
  97 /* Unwind Lua stack and move error message to new top. */
  98 LJ_NOINLINE static void unwindstack(lua_State *L, TValue *top)
  99 {
 100   lj_func_closeuv(L, top);
 101   if (top < L->top-1) {
 102     copyTV(L, top, L->top-1);
 103     L->top = top+1;
 104   }
 105   lj_state_relimitstack(L);
 106 }
 107
 108 /* Unwind until stop frame. Optionally cleanup frames. */
 109 static void *err_unwind(lua_State *L, void *stopcf, int errcode)
 110 {
 111   TValue *frame = L->base-1;
 112   void *cf = L->cframe;
 113   while (cf) {
 114     int32_t nres = cframe_nres(cframe_raw(cf));
 115     if (nres < 0) {  /* C frame without Lua frame? */
 116       TValue *top = restorestack(L, -nres);
 117       if (frame < top) {  /* Frame reached? */
 118         if (errcode) {
 119           L->base = frame+1;
 120           L->cframe = cframe_prev(cf);
 121           unwindstack(L, top);
 122         }
 123         return cf;
 124       }
 125     }
 126     if (frame <= tvref(L->stack)+LJ_FR2)
 127       break;
 128     switch (frame_typep(frame)) {
 129     case FRAME_LUA:  /* Lua frame. */
 130     case FRAME_LUAP:
 131       frame = frame_prevl(frame);
 132       break;
 133     case FRAME_C:  /* C frame. */
 134     unwind_c:
 135 #if LJ_UNWIND_EXT
 136       if (errcode) {
 137         L->base = frame_prevd(frame) + 1;
 138         L->cframe = cframe_prev(cf);
 139         unwindstack(L, frame - LJ_FR2);
 140       } else if (cf != stopcf) {
 141         cf = cframe_prev(cf);
 142         frame = frame_prevd(frame);
 143         break;
 144       }
 145       return NULL;  /* Continue unwinding. */
 146 #else
 147       UNUSED(stopcf);
 148       cf = cframe_prev(cf);
 149       frame = frame_prevd(frame);
 150       break;
 151 #endif
 152     case FRAME_CP:  /* Protected C frame. */
 153       if (cframe_canyield(cf)) {  /* Resume? */
 154         if (errcode) {
 155           hook_leave(G(L));  /* Assumes nobody uses coroutines inside hooks. */
 156           L->cframe = NULL;
 157           L->status = (uint8_t)errcode;
 158         }
 159         return cf;
 160       }
 161       if (errcode) {
 162         L->base = frame_prevd(frame) + 1;
 163         L->cframe = cframe_prev(cf);
 164         unwindstack(L, frame - LJ_FR2);
 165       }
 166       return cf;
 167     case FRAME_CONT:  /* Continuation frame. */
 168       if (frame_iscont_fficb(frame))
 169         goto unwind_c;
 170       /* fallthrough */
 171     case FRAME_VARG:  /* Vararg frame. */
 172       frame = frame_prevd(frame);
 173       break;
 174     case FRAME_PCALL:  /* FF pcall() frame. */
 175     case FRAME_PCALLH:  /* FF pcall() frame inside hook. */
 176       if (errcode) {
 177         global_State *g;
 178         if (errcode == LUA_YIELD) {
 179           frame = frame_prevd(frame);
 180           break;
 181         }
 182         g = G(L);
 183         setgcref(g->cur_L, obj2gco(L));
 184         if (frame_typep(frame) == FRAME_PCALL)
 185           hook_leave(g);
 186         L->base = frame_prevd(frame) + 1;
 187         L->cframe = cf;
 188         unwindstack(L, L->base);
 189       }
 190       return (void *)((intptr_t)cf | CFRAME_UNWIND_FF);
 191     }
 192   }
 193   /* No C frame. */
 194   if (errcode) {
 195     L->base = tvref(L->stack)+1+LJ_FR2;
 196     L->cframe = NULL;
 197     unwindstack(L, L->base);
 198     if (G(L)->panic)
 199       G(L)->panic(L);
 200     exit(EXIT_FAILURE);
 201   }
 202   return L;  /* Anything non-NULL will do. */
 203 }
 204
 205 /* -- External frame unwinding -------------------------------------------- */
 206
 207 #if LJ_ABI_WIN
 208
 209 /*
 210 ** Someone in Redmond owes me several days of my life. A lot of this is
 211 ** undocumented or just plain wrong on MSDN. Some of it can be gathered
 212 ** from 3rd party docs or must be found by trial-and-error. They really
 213 ** don't want you to write your own language-specific exception handler
 214 ** or to interact gracefully with MSVC. :-(
 215 */
 216
 217 #define WIN32_LEAN_AND_MEAN
 218 #include <windows.h>
 219
 220 #if LJ_TARGET_X86
 221 typedef void *UndocumentedDispatcherContext;  /* Unused on x86. */
 222 #else
 223 /* Taken from: http://www.nynaeve.net/?p=99 */
 224 typedef struct UndocumentedDispatcherContext {
 225   ULONG64 ControlPc;
 226   ULONG64 ImageBase;
 227   PRUNTIME_FUNCTION FunctionEntry;
 228   ULONG64 EstablisherFrame;
 229   ULONG64 TargetIp;
 230   PCONTEXT ContextRecord;
 231   void (*LanguageHandler)(void);
 232   PVOID HandlerData;
 233   PUNWIND_HISTORY_TABLE HistoryTable;
 234   ULONG ScopeIndex;
 235   ULONG Fill0;
 236 } UndocumentedDispatcherContext;
 237 #endif
 238
 239 /* Another wild guess. */
 240 extern void __DestructExceptionObject(EXCEPTION_RECORD *rec, int nothrow);
 241
 242 #define LJ_MSVC_EXCODE          ((DWORD)0xe06d7363)
 243 #define LJ_GCC_EXCODE           ((DWORD)0x20474343)
 244
 245 #define LJ_EXCODE               ((DWORD)0xe24c4a00)
 246 #define LJ_EXCODE_MAKE(c)       (LJ_EXCODE | (DWORD)(c))
 247 #define LJ_EXCODE_CHECK(cl)     (((cl) ^ LJ_EXCODE) <= 0xff)
 248 #define LJ_EXCODE_ERRCODE(cl)   ((int)((cl) & 0xff))
 249
 250 /* Windows exception handler for interpreter frame. */
 251 LJ_FUNCA int lj_err_unwind_win(EXCEPTION_RECORD *rec,
 252   void *f, CONTEXT *ctx, UndocumentedDispatcherContext *dispatch)
 253 {
 254 #if LJ_TARGET_X86
 255   void *cf = (char *)f - CFRAME_OFS_SEH;
 256 #elif LJ_TARGET_ARM64
 257   void *cf = (char *)f - CFRAME_SIZE;
 258 #else
 259   void *cf = f;
 260 #endif
 261   lua_State *L = cframe_L(cf);
 262   int errcode = LJ_EXCODE_CHECK(rec->ExceptionCode) ?
 263                 LJ_EXCODE_ERRCODE(rec->ExceptionCode) : LUA_ERRRUN;
 264   if ((rec->ExceptionFlags & 6)) {  /* EH_UNWINDING|EH_EXIT_UNWIND */
 265     if (rec->ExceptionCode == STATUS_LONGJUMP &&
 266         rec->ExceptionRecord &&
 267         LJ_EXCODE_CHECK(rec->ExceptionRecord->ExceptionCode)) {
 268       errcode = LJ_EXCODE_ERRCODE(rec->ExceptionRecord->ExceptionCode);
 269       if ((rec->ExceptionFlags & 0x20)) {  /* EH_TARGET_UNWIND */
 270         /* Unwinding is about to finish; revert the ExceptionCode so that
 271         ** RtlRestoreContext does not try to restore from a _JUMP_BUFFER.
 272         */
 273         rec->ExceptionCode = 0;
 274       }
 275     }
 276     /* Unwind internal frames. */
 277     err_unwind(L, cf, errcode);
 278   } else {
 279     void *cf2 = err_unwind(L, cf, 0);
 280     if (cf2) {  /* We catch it, so start unwinding the upper frames. */
 281 #if !LJ_TARGET_X86
 282       EXCEPTION_RECORD rec2;
 283 #endif
 284       if (rec->ExceptionCode == LJ_MSVC_EXCODE ||
 285           rec->ExceptionCode == LJ_GCC_EXCODE) {
 286 #if !LJ_TARGET_CYGWIN
 287         __DestructExceptionObject(rec, 1);
 288 #endif
 289         setstrV(L, L->top++, lj_err_str(L, LJ_ERR_ERRCPP));
 290       } else if (!LJ_EXCODE_CHECK(rec->ExceptionCode)) {
 291         /* Don't catch access violations etc. */
 292         return 1;  /* ExceptionContinueSearch */
 293       }
 294 #if LJ_TARGET_X86
 295       UNUSED(ctx);
 296       UNUSED(dispatch);
 297       /* Call all handlers for all lower C frames (including ourselves) again
 298       ** with EH_UNWINDING set. Then call the specified function, passing cf
 299       ** and errcode.
 300       */
 301       lj_vm_rtlunwind(cf, (void *)rec,
 302         (cframe_unwind_ff(cf2) && errcode != LUA_YIELD) ?
 303         (void *)lj_vm_unwind_ff : (void *)lj_vm_unwind_c, errcode);
 304       /* lj_vm_rtlunwind does not return. */
 305 #else
 306       if (LJ_EXCODE_CHECK(rec->ExceptionCode)) {
 307         /* For unwind purposes, wrap the EXCEPTION_RECORD in something that
 308         ** looks like a longjmp, so that MSVC will execute C++ destructors in
 309         ** the frames we unwind over. ExceptionInformation[0] should really
 310         ** contain a _JUMP_BUFFER*, but hopefully nobody is looking too closely
 311         ** at this point.
 312         */
 313         rec2.ExceptionCode = STATUS_LONGJUMP;
 314         rec2.ExceptionRecord = rec;
 315         rec2.ExceptionAddress = 0;
 316         rec2.NumberParameters = 1;
 317         rec2.ExceptionInformation[0] = (ULONG_PTR)ctx;
 318         rec = &rec2;
 319       }
 320       /* Unwind the stack and call all handlers for all lower C frames
 321       ** (including ourselves) again with EH_UNWINDING set. Then set
 322       ** stack pointer = f, result = errcode and jump to the specified target.
 323       */
 324       RtlUnwindEx(f, (void *)((cframe_unwind_ff(cf2) && errcode != LUA_YIELD) ?
 325                               lj_vm_unwind_ff_eh :
 326                               lj_vm_unwind_c_eh),
 327                   rec, (void *)(uintptr_t)errcode, dispatch->ContextRecord,
 328                   dispatch->HistoryTable);
 329       /* RtlUnwindEx should never return. */
 330 #endif
 331     }
 332   }
 333   return 1;  /* ExceptionContinueSearch */
 334 }
 335
 336 #if LJ_UNWIND_JIT
 337
 338 #if LJ_TARGET_X64
 339 #define CONTEXT_REG_PC  Rip
 340 #elif LJ_TARGET_ARM64
 341 #define CONTEXT_REG_PC  Pc
 342 #else
 343 #error "NYI: Windows arch-specific unwinder for JIT-compiled code"
 344 #endif
 345
 346 /* Windows unwinder for JIT-compiled code. */
 347 static void err_unwind_win_jit(global_State *g, int errcode)
 348 {
 349   CONTEXT ctx;
 350   UNWIND_HISTORY_TABLE hist;
 351
 352   memset(&hist, 0, sizeof(hist));
 353   RtlCaptureContext(&ctx);
 354   while (1) {
 355     DWORD64 frame, base, addr = ctx.CONTEXT_REG_PC;
 356     void *hdata;
 357     PRUNTIME_FUNCTION func = RtlLookupFunctionEntry(addr, &base, &hist);
 358     if (!func) {  /* Found frame without .pdata: must be JIT-compiled code. */
 359       ExitNo exitno;
 360       uintptr_t stub = lj_trace_unwind(G2J(g), (uintptr_t)(addr - sizeof(MCode)), &exitno);
 361       if (stub) {  /* Jump to side exit to unwind the trace. */
 362         ctx.CONTEXT_REG_PC = stub;
 363         G2J(g)->exitcode = errcode;
 364         RtlRestoreContext(&ctx, NULL);  /* Does not return. */
 365       }
 366       break;
 367     }
 368     RtlVirtualUnwind(UNW_FLAG_NHANDLER, base, addr, func,
 369                      &ctx, &hdata, &frame, NULL);
 370     if (!addr) break;
 371   }
 372   /* Unwinding failed, if we end up here. */
 373 }
 374 #endif
 375
 376 /* Raise Windows exception. */
 377 static void err_raise_ext(global_State *g, int errcode)
 378 {
 379 #if LJ_UNWIND_JIT
 380   if (tvref(g->jit_base)) {
 381     err_unwind_win_jit(g, errcode);
 382     return;  /* Unwinding failed. */
 383   }
 384 #elif LJ_HASJIT
 385   /* Cannot catch on-trace errors for Windows/x86 SEH. Unwind to interpreter. */
 386   setmref(g->jit_base, NULL);
 387 #endif
 388   UNUSED(g);
 389   RaiseException(LJ_EXCODE_MAKE(errcode), 1 /* EH_NONCONTINUABLE */, 0, NULL);
 390 }
 391
 392 #elif !LJ_NO_UNWIND && (defined(__GNUC__) || defined(__clang__))
 393
 394 /*
 395 ** We have to use our own definitions instead of the mandatory (!) unwind.h,
 396 ** since various OS, distros and compilers mess up the header installation.
 397 */
 398
 399 typedef struct _Unwind_Context _Unwind_Context;
 400
 401 #define _URC_OK                 0
 402 #define _URC_FATAL_PHASE2_ERROR 2
 403 #define _URC_FATAL_PHASE1_ERROR 3
 404 #define _URC_HANDLER_FOUND      6
 405 #define _URC_INSTALL_CONTEXT    7
 406 #define _URC_CONTINUE_UNWIND    8
 407 #define _URC_FAILURE            9
 408
 409 #define LJ_UEXCLASS             0x4c55414a49543200ULL   /* LUAJIT2\0 */
 410 #define LJ_UEXCLASS_MAKE(c)     (LJ_UEXCLASS | (uint64_t)(c))
 411 #define LJ_UEXCLASS_CHECK(cl)   (((cl) ^ LJ_UEXCLASS) <= 0xff)
 412 #define LJ_UEXCLASS_ERRCODE(cl) ((int)((cl) & 0xff))
 413
 414 #if !LJ_TARGET_ARM
 415
 416 typedef struct _Unwind_Exception
 417 {
 418   uint64_t exclass;
 419   void (*excleanup)(int, struct _Unwind_Exception *);
 420   uintptr_t p1, p2;
 421 } __attribute__((__aligned__)) _Unwind_Exception;
 422 #define UNWIND_EXCEPTION_TYPE   _Unwind_Exception
 423
 424 extern uintptr_t _Unwind_GetCFA(_Unwind_Context *);
 425 extern void _Unwind_SetGR(_Unwind_Context *, int, uintptr_t);
 426 extern uintptr_t _Unwind_GetIP(_Unwind_Context *);
 427 extern void _Unwind_SetIP(_Unwind_Context *, uintptr_t);
 428 extern void _Unwind_DeleteException(_Unwind_Exception *);
 429 extern int _Unwind_RaiseException(_Unwind_Exception *);
 430
 431 #define _UA_SEARCH_PHASE        1
 432 #define _UA_CLEANUP_PHASE       2
 433 #define _UA_HANDLER_FRAME       4
 434 #define _UA_FORCE_UNWIND        8
 435
 436 /* DWARF2 personality handler referenced from interpreter .eh_frame. */
 437 LJ_FUNCA int lj_err_unwind_dwarf(int version, int actions,
 438   uint64_t uexclass, _Unwind_Exception *uex, _Unwind_Context *ctx)
 439 {
 440   void *cf;
 441   lua_State *L;
 442   if (version != 1)
 443     return _URC_FATAL_PHASE1_ERROR;
 444   cf = (void *)_Unwind_GetCFA(ctx);
 445   L = cframe_L(cf);
 446   if ((actions & _UA_SEARCH_PHASE)) {
 447 #if LJ_UNWIND_EXT
 448     if (err_unwind(L, cf, 0) == NULL)
 449       return _URC_CONTINUE_UNWIND;
 450 #endif
 451     if (!LJ_UEXCLASS_CHECK(uexclass)) {
 452       setstrV(L, L->top++, lj_err_str(L, LJ_ERR_ERRCPP));
 453     }
 454     return _URC_HANDLER_FOUND;
 455   }
 456   if ((actions & _UA_CLEANUP_PHASE)) {
 457     int errcode;
 458     if (LJ_UEXCLASS_CHECK(uexclass)) {
 459       errcode = LJ_UEXCLASS_ERRCODE(uexclass);
 460     } else {
 461       if ((actions & _UA_HANDLER_FRAME))
 462         _Unwind_DeleteException(uex);
 463       errcode = LUA_ERRRUN;
 464     }
 465 #if LJ_UNWIND_EXT
 466     cf = err_unwind(L, cf, errcode);
 467     if ((actions & _UA_FORCE_UNWIND)) {
 468       return _URC_CONTINUE_UNWIND;
 469     } else if (cf) {
 470       ASMFunction ip;
 471       _Unwind_SetGR(ctx, LJ_TARGET_EHRETREG, errcode);
 472       ip = cframe_unwind_ff(cf) ? lj_vm_unwind_ff_eh : lj_vm_unwind_c_eh;
 473       _Unwind_SetIP(ctx, (uintptr_t)lj_ptr_strip(ip));
 474       return _URC_INSTALL_CONTEXT;
 475     }
 476 #if LJ_TARGET_X86ORX64
 477     else if ((actions & _UA_HANDLER_FRAME)) {
 478       /* Workaround for ancient libgcc bug. Still present in RHEL 5.5. :-/
 479       ** Real fix: http://gcc.gnu.org/viewcvs/trunk/gcc/unwind-dw2.c?r1=121165&r2=124837&pathrev=153877&diff_format=h
 480       */
 481       _Unwind_SetGR(ctx, LJ_TARGET_EHRETREG, errcode);
 482       _Unwind_SetIP(ctx, (uintptr_t)lj_vm_unwind_rethrow);
 483       return _URC_INSTALL_CONTEXT;
 484     }
 485 #endif
 486 #else
 487     /* This is not the proper way to escape from the unwinder. We get away with
 488     ** it on non-x64 because the interpreter restores all callee-saved regs.
 489     */
 490     lj_err_throw(L, errcode);
 491 #if LJ_TARGET_X64
 492 #error "Broken build system -- only use the provided Makefiles!"
 493 #endif
 494 #endif
 495   }
 496   return _URC_CONTINUE_UNWIND;
 497 }
 498
 499 #if LJ_UNWIND_EXT && defined(LUA_USE_ASSERT)
 500 struct dwarf_eh_bases { void *tbase, *dbase, *func; };
 501 extern const void *_Unwind_Find_FDE(void *pc, struct dwarf_eh_bases *bases);
 502
 503 /* Verify that external error handling actually has a chance to work. */
 504 void lj_err_verify(void)
 505 {
 506 #if !LJ_TARGET_OSX
 507   /* Check disabled on MacOS due to brilliant software engineering at Apple. */
 508   struct dwarf_eh_bases ehb;
 509   lj_assertX(_Unwind_Find_FDE((void *)lj_err_throw, &ehb), "broken build: external frame unwinding enabled, but missing -funwind-tables");
 510 #endif
 511   /* Check disabled, because of broken Fedora/ARM64. See #722.
 512   lj_assertX(_Unwind_Find_FDE((void *)_Unwind_RaiseException, &ehb), "broken build: external frame unwinding enabled, but system libraries have no unwind tables");
 513   */
 514 }
 515 #endif
 516
 517 #if LJ_UNWIND_JIT
 518 /* DWARF2 personality handler for JIT-compiled code. */
 519 static int err_unwind_jit(int version, int actions,
 520   uint64_t uexclass, _Unwind_Exception *uex, _Unwind_Context *ctx)
 521 {
 522   /* NYI: FFI C++ exception interoperability. */
 523   if (version != 1 || !LJ_UEXCLASS_CHECK(uexclass))
 524     return _URC_FATAL_PHASE1_ERROR;
 525   if ((actions & _UA_SEARCH_PHASE)) {
 526     return _URC_HANDLER_FOUND;
 527   }
 528   if ((actions & _UA_CLEANUP_PHASE)) {
 529     global_State *g = *(global_State **)(uex+1);
 530     ExitNo exitno;
 531     uintptr_t addr = _Unwind_GetIP(ctx);  /* Return address _after_ call. */
 532     uintptr_t stub = lj_trace_unwind(G2J(g), addr - sizeof(MCode), &exitno);
 533     lj_assertG(tvref(g->jit_base), "unexpected throw across mcode frame");
 534     if (stub) {  /* Jump to side exit to unwind the trace. */
 535       G2J(g)->exitcode = LJ_UEXCLASS_ERRCODE(uexclass);
 536 #ifdef LJ_TARGET_MIPS
 537       _Unwind_SetGR(ctx, 4, stub);
 538       _Unwind_SetGR(ctx, 5, exitno);
 539       _Unwind_SetIP(ctx, (uintptr_t)(void *)lj_vm_unwind_stub);
 540 #else
 541       _Unwind_SetIP(ctx, stub);
 542 #endif
 543       return _URC_INSTALL_CONTEXT;
 544     }
 545     return _URC_FATAL_PHASE2_ERROR;
 546   }
 547   return _URC_FATAL_PHASE1_ERROR;
 548 }
 549
 550 /* DWARF2 template frame info for JIT-compiled code.
 551 **
 552 ** After copying the template to the start of the mcode segment,
 553 ** the frame handler function and the code size is patched.
 554 ** The frame handler always installs a new context to jump to the exit,
 555 ** so don't bother to add any unwind opcodes.
 556 */
 557 static const uint8_t err_frame_jit_template[] = {
 558 #if LJ_BE
 559   0,0,0,
 560 #endif
 561   LJ_64 ? 0x1c : 0x14,  /* CIE length. */
 562 #if LJ_LE
 563   0,0,0,
 564 #endif
 565   0,0,0,0, 1, 'z','P','R',0,  /* CIE mark, CIE version, augmentation. */
 566   1, LJ_64 ? 0x78 : 0x7c, LJ_TARGET_EHRAREG,  /* Code/data align, RA. */
 567 #if LJ_64
 568   10, 0, 0,0,0,0,0,0,0,0, 0x1b,  /* Aug. data ABS handler, PCREL|SDATA4 code. */
 569   0,0,0,0,0,  /* Alignment. */
 570 #else
 571   6, 0, 0,0,0,0, 0x1b,  /* Aug. data ABS handler, PCREL|SDATA4 code. */
 572   0,  /* Alignment. */
 573 #endif
 574 #if LJ_BE
 575   0,0,0,
 576 #endif
 577   LJ_64 ? 0x14 : 0x10,  /* FDE length. */
 578   0,0,0,
 579   LJ_64 ? 0x24 : 0x1c,  /* CIE offset. */
 580   0,0,0,
 581   LJ_64 ? 0x14 : 0x10,  /* Code offset. After Final FDE. */
 582 #if LJ_LE
 583   0,0,0,
 584 #endif
 585   0,0,0,0, 0, 0,0,0, /* Code size, augmentation length, alignment. */
 586 #if LJ_64
 587   0,0,0,0,  /* Alignment. */
 588 #endif
 589   0,0,0,0  /* Final FDE. */
 590 };
 591
 592 #define ERR_FRAME_JIT_OFS_HANDLER       0x12
 593 #define ERR_FRAME_JIT_OFS_FDE           (LJ_64 ? 0x20 : 0x18)
 594 #define ERR_FRAME_JIT_OFS_CODE_SIZE     (LJ_64 ? 0x2c : 0x24)
 595 #if LJ_TARGET_OSX
 596 #define ERR_FRAME_JIT_OFS_REGISTER      ERR_FRAME_JIT_OFS_FDE
 597 #else
 598 #define ERR_FRAME_JIT_OFS_REGISTER      0
 599 #endif
 600
 601 extern void __register_frame(const void *);
 602 extern void __deregister_frame(const void *);
 603
 604 uint8_t *lj_err_register_mcode(void *base, size_t sz, uint8_t *info)
 605 {
 606   ASMFunction handler = (ASMFunction)err_unwind_jit;
 607   memcpy(info, err_frame_jit_template, sizeof(err_frame_jit_template));
 608 #if LJ_ABI_PAUTH
 609 #if LJ_TARGET_ARM64
 610   handler = ptrauth_auth_and_resign(handler,
 611     ptrauth_key_function_pointer, 0,
 612     ptrauth_key_process_independent_code, info + ERR_FRAME_JIT_OFS_HANDLER);
 613 #else
 614 #error "missing pointer authentication support for this architecture"
 615 #endif
 616 #endif
 617   memcpy(info + ERR_FRAME_JIT_OFS_HANDLER, &handler, sizeof(handler));
 618   *(uint32_t *)(info + ERR_FRAME_JIT_OFS_CODE_SIZE) =
 619     (uint32_t)(sz - sizeof(err_frame_jit_template) - (info - (uint8_t *)base));
 620   __register_frame(info + ERR_FRAME_JIT_OFS_REGISTER);
 621 #ifdef LUA_USE_ASSERT
 622   {
 623     struct dwarf_eh_bases ehb;
 624     lj_assertX(_Unwind_Find_FDE(info + sizeof(err_frame_jit_template)+1, &ehb),
 625                "bad JIT unwind table registration");
 626   }
 627 #endif
 628   return info + sizeof(err_frame_jit_template);
 629 }
 630
 631 void lj_err_deregister_mcode(void *base, size_t sz, uint8_t *info)
 632 {
 633   UNUSED(base); UNUSED(sz);
 634   __deregister_frame(info + ERR_FRAME_JIT_OFS_REGISTER);
 635 }
 636 #endif
 637
 638 #else /* LJ_TARGET_ARM */
 639
 640 #define _US_VIRTUAL_UNWIND_FRAME        0
 641 #define _US_UNWIND_FRAME_STARTING       1
 642 #define _US_ACTION_MASK                 3
 643 #define _US_FORCE_UNWIND                8
 644
 645 typedef struct _Unwind_Control_Block _Unwind_Control_Block;
 646 #define UNWIND_EXCEPTION_TYPE   _Unwind_Control_Block
 647
 648 struct _Unwind_Control_Block {
 649   uint64_t exclass;
 650   uint32_t misc[20];
 651 };
 652
 653 extern int _Unwind_RaiseException(_Unwind_Control_Block *);
 654 extern int __gnu_unwind_frame(_Unwind_Control_Block *, _Unwind_Context *);
 655 extern int _Unwind_VRS_Set(_Unwind_Context *, int, uint32_t, int, void *);
 656 extern int _Unwind_VRS_Get(_Unwind_Context *, int, uint32_t, int, void *);
 657
 658 static inline uint32_t _Unwind_GetGR(_Unwind_Context *ctx, int r)
 659 {
 660   uint32_t v;
 661   _Unwind_VRS_Get(ctx, 0, r, 0, &v);
 662   return v;
 663 }
 664
 665 static inline void _Unwind_SetGR(_Unwind_Context *ctx, int r, uint32_t v)
 666 {
 667   _Unwind_VRS_Set(ctx, 0, r, 0, &v);
 668 }
 669
 670 extern void lj_vm_unwind_ext(void);
 671
 672 /* ARM unwinder personality handler referenced from interpreter .ARM.extab. */
 673 LJ_FUNCA int lj_err_unwind_arm(int state, _Unwind_Control_Block *ucb,
 674                                _Unwind_Context *ctx)
 675 {
 676   void *cf = (void *)_Unwind_GetGR(ctx, 13);
 677   lua_State *L = cframe_L(cf);
 678   int errcode;
 679
 680   switch ((state & _US_ACTION_MASK)) {
 681   case _US_VIRTUAL_UNWIND_FRAME:
 682     if ((state & _US_FORCE_UNWIND)) break;
 683     return _URC_HANDLER_FOUND;
 684   case _US_UNWIND_FRAME_STARTING:
 685     if (LJ_UEXCLASS_CHECK(ucb->exclass)) {
 686       errcode = LJ_UEXCLASS_ERRCODE(ucb->exclass);
 687     } else {
 688       errcode = LUA_ERRRUN;
 689       setstrV(L, L->top++, lj_err_str(L, LJ_ERR_ERRCPP));
 690     }
 691     cf = err_unwind(L, cf, errcode);
 692     if ((state & _US_FORCE_UNWIND) || cf == NULL) break;
 693     _Unwind_SetGR(ctx, 15, (uint32_t)lj_vm_unwind_ext);
 694     _Unwind_SetGR(ctx, 0, (uint32_t)ucb);
 695     _Unwind_SetGR(ctx, 1, (uint32_t)errcode);
 696     _Unwind_SetGR(ctx, 2, cframe_unwind_ff(cf) ?
 697                             (uint32_t)lj_vm_unwind_ff_eh :
 698                             (uint32_t)lj_vm_unwind_c_eh);
 699     return _URC_INSTALL_CONTEXT;
 700   default:
 701     return _URC_FAILURE;
 702   }
 703   if (__gnu_unwind_frame(ucb, ctx) != _URC_OK)
 704     return _URC_FAILURE;
 705 #ifdef LUA_USE_ASSERT
 706   /* We should never get here unless this is a forced unwind aka backtrace. */
 707   if (_Unwind_GetGR(ctx, 0) == 0xff33aa77) {
 708     _Unwind_SetGR(ctx, 0, 0xff33aa88);
 709   }
 710 #endif
 711   return _URC_CONTINUE_UNWIND;
 712 }
 713
 714 #if LJ_UNWIND_EXT && defined(LUA_USE_ASSERT)
 715 typedef int (*_Unwind_Trace_Fn)(_Unwind_Context *, void *);
 716 extern int _Unwind_Backtrace(_Unwind_Trace_Fn, void *);
 717
 718 static int err_verify_bt(_Unwind_Context *ctx, int *got)
 719 {
 720   if (_Unwind_GetGR(ctx, 0) == 0xff33aa88) { *got = 2; }
 721   else if (*got == 0) { *got = 1; _Unwind_SetGR(ctx, 0, 0xff33aa77); }
 722   return _URC_OK;
 723 }
 724
 725 /* Verify that external error handling actually has a chance to work. */
 726 void lj_err_verify(void)
 727 {
 728   int got = 0;
 729   _Unwind_Backtrace((_Unwind_Trace_Fn)err_verify_bt, &got);
 730   lj_assertX(got == 2, "broken build: external frame unwinding enabled, but missing -funwind-tables");
 731 }
 732 #endif
 733
 734 /*
 735 ** Note: LJ_UNWIND_JIT is not implemented for 32 bit ARM.
 736 **
 737 ** The quirky ARM unwind API doesn't have __register_frame().
 738 ** A potential workaround might involve _Unwind_Backtrace.
 739 ** But most 32 bit ARM targets don't qualify for LJ_UNWIND_EXT, anyway,
 740 ** since they are built without unwind tables by default.
 741 */
 742
 743 #endif /* LJ_TARGET_ARM */
 744
 745
 746 #if LJ_UNWIND_EXT
 747 static __thread struct {
 748   UNWIND_EXCEPTION_TYPE ex;
 749   global_State *g;
 750 } static_uex;
 751
 752 /* Raise external exception. */
 753 static void err_raise_ext(global_State *g, int errcode)
 754 {
 755   memset(&static_uex, 0, sizeof(static_uex));
 756   static_uex.ex.exclass = LJ_UEXCLASS_MAKE(errcode);
 757   static_uex.g = g;
 758   _Unwind_RaiseException(&static_uex.ex);
 759 }
 760
 761 #endif
 762
 763 #endif
 764
 765 /* -- Error handling ------------------------------------------------------ */
 766
 767 /* Throw error. Find catch frame, unwind stack and continue. */
 768 LJ_NOINLINE void LJ_FASTCALL lj_err_throw(lua_State *L, int errcode)
 769 {
 770   global_State *g = G(L);
 771   lj_trace_abort(g);
 772   L->status = LUA_OK;
 773 #if LJ_UNWIND_EXT
 774   err_raise_ext(g, errcode);
 775   /*
 776   ** A return from this function signals a corrupt C stack that cannot be
 777   ** unwound. We have no choice but to call the panic function and exit.
 778   **
 779   ** Usually this is caused by a C function without unwind information.
 780   ** This may happen if you've manually enabled LUAJIT_UNWIND_EXTERNAL
 781   ** and forgot to recompile *every* non-C++ file with -funwind-tables.
 782   */
 783   if (G(L)->panic)
 784     G(L)->panic(L);
 785 #else
 786 #if LJ_HASJIT
 787   setmref(g->jit_base, NULL);
 788 #endif
 789   {
 790     void *cf = err_unwind(L, NULL, errcode);
 791     if (cframe_unwind_ff(cf))
 792       lj_vm_unwind_ff(cframe_raw(cf));
 793     else
 794       lj_vm_unwind_c(cframe_raw(cf), errcode);
 795   }
 796 #endif
 797   exit(EXIT_FAILURE);
 798 }
 799
 800 /* Return string object for error message. */
 801 LJ_NOINLINE GCstr *lj_err_str(lua_State *L, ErrMsg em)
 802 {
 803   return lj_str_newz(L, err2msg(em));
 804 }
 805
 806 /* Out-of-memory error. */
 807 LJ_NOINLINE void lj_err_mem(lua_State *L)
 808 {
 809   if (L->status == LUA_ERRERR+1)  /* Don't touch the stack during lua_open. */
 810     lj_vm_unwind_c(L->cframe, LUA_ERRMEM);
 811   if (LJ_HASJIT) {
 812     TValue *base = tvref(G(L)->jit_base);
 813     if (base) L->base = base;
 814   }
 815   if (curr_funcisL(L)) {
 816     L->top = curr_topL(L);
 817     if (LJ_UNLIKELY(L->top > tvref(L->maxstack))) {
 818       /* The current Lua frame violates the stack. Replace it with a dummy. */
 819       L->top = L->base;
 820       setframe_gc(L->base - 1 - LJ_FR2, obj2gco(L), LJ_TTHREAD);
 821     }
 822   }
 823   setstrV(L, L->top++, lj_err_str(L, LJ_ERR_ERRMEM));
 824   lj_err_throw(L, LUA_ERRMEM);
 825 }
 826
 827 /* Find error function for runtime errors. Requires an extra stack traversal. */
 828 static ptrdiff_t finderrfunc(lua_State *L)
 829 {
 830   cTValue *frame = L->base-1, *bot = tvref(L->stack)+LJ_FR2;
 831   void *cf = L->cframe;
 832   while (frame > bot && cf) {
 833     while (cframe_nres(cframe_raw(cf)) < 0) {  /* cframe without frame? */
 834       if (frame >= restorestack(L, -cframe_nres(cf)))
 835         break;
 836       if (cframe_errfunc(cf) >= 0)  /* Error handler not inherited (-1)? */
 837         return cframe_errfunc(cf);
 838       cf = cframe_prev(cf);  /* Else unwind cframe and continue searching. */
 839       if (cf == NULL)
 840         return 0;
 841     }
 842     switch (frame_typep(frame)) {
 843     case FRAME_LUA:
 844     case FRAME_LUAP:
 845       frame = frame_prevl(frame);
 846       break;
 847     case FRAME_C:
 848       cf = cframe_prev(cf);
 849       /* fallthrough */
 850     case FRAME_VARG:
 851       frame = frame_prevd(frame);
 852       break;
 853     case FRAME_CONT:
 854       if (frame_iscont_fficb(frame))
 855         cf = cframe_prev(cf);
 856       frame = frame_prevd(frame);
 857       break;
 858     case FRAME_CP:
 859       if (cframe_canyield(cf)) return 0;
 860       if (cframe_errfunc(cf) >= 0)
 861         return cframe_errfunc(cf);
 862       cf = cframe_prev(cf);
 863       frame = frame_prevd(frame);
 864       break;
 865     case FRAME_PCALL:
 866     case FRAME_PCALLH:
 867       if (frame_func(frame_prevd(frame))->c.ffid == FF_xpcall)
 868         return savestack(L, frame_prevd(frame)+1);  /* xpcall's errorfunc. */
 869       return 0;
 870     default:
 871       lj_assertL(0, "bad frame type");
 872       return 0;
 873     }
 874   }
 875   return 0;
 876 }
 877
 878 /* Runtime error. */
 879 LJ_NOINLINE void LJ_FASTCALL lj_err_run(lua_State *L)
 880 {
 881   ptrdiff_t ef = (LJ_HASJIT && tvref(G(L)->jit_base)) ? 0 : finderrfunc(L);
 882   if (ef) {
 883     TValue *errfunc, *top;
 884     lj_state_checkstack(L, LUA_MINSTACK * 2);  /* Might raise new error. */
 885     lj_trace_abort(G(L));
 886     errfunc = restorestack(L, ef);
 887     top = L->top;
 888     if (!tvisfunc(errfunc) || L->status == LUA_ERRERR) {
 889       setstrV(L, top-1, lj_err_str(L, LJ_ERR_ERRERR));
 890       lj_err_throw(L, LUA_ERRERR);
 891     }
 892     L->status = LUA_ERRERR;
 893     copyTV(L, top+LJ_FR2, top-1);
 894     copyTV(L, top-1, errfunc);
 895     if (LJ_FR2) setnilV(top++);
 896     L->top = top+1;
 897     lj_vm_call(L, top, 1+1);  /* Stack: |errfunc|msg| -> |msg| */
 898   }
 899   lj_err_throw(L, LUA_ERRRUN);
 900 }
 901
 902 /* Stack overflow error. */
 903 void LJ_FASTCALL lj_err_stkov(lua_State *L)
 904 {
 905   lj_debug_addloc(L, err2msg(LJ_ERR_STKOV), L->base-1, NULL);
 906   lj_err_run(L);
 907 }
 908
 909 #if LJ_HASJIT
 910 /* Rethrow error after doing a trace exit. */
 911 LJ_NOINLINE void LJ_FASTCALL lj_err_trace(lua_State *L, int errcode)
 912 {
 913   if (errcode == LUA_ERRRUN)
 914     lj_err_run(L);
 915   else
 916     lj_err_throw(L, errcode);
 917 }
 918 #endif
 919
 920 /* Formatted runtime error message. */
 921 LJ_NORET LJ_NOINLINE static void err_msgv(lua_State *L, ErrMsg em, ...)
 922 {
 923   const char *msg;
 924   va_list argp;
 925   va_start(argp, em);
 926   if (LJ_HASJIT) {
 927     TValue *base = tvref(G(L)->jit_base);
 928     if (base) L->base = base;
 929   }
 930   if (curr_funcisL(L)) L->top = curr_topL(L);
 931   msg = lj_strfmt_pushvf(L, err2msg(em), argp);
 932   va_end(argp);
 933   lj_debug_addloc(L, msg, L->base-1, NULL);
 934   lj_err_run(L);
 935 }
 936
 937 /* Non-vararg variant for better calling conventions. */
 938 LJ_NOINLINE void lj_err_msg(lua_State *L, ErrMsg em)
 939 {
 940   err_msgv(L, em);
 941 }
 942
 943 /* Lexer error. */
 944 LJ_NOINLINE void lj_err_lex(lua_State *L, GCstr *src, const char *tok,
 945                             BCLine line, ErrMsg em, va_list argp)
 946 {
 947   char buff[LUA_IDSIZE];
 948   const char *msg;
 949   lj_debug_shortname(buff, src, line);
 950   msg = lj_strfmt_pushvf(L, err2msg(em), argp);
 951   msg = lj_strfmt_pushf(L, "%s:%d: %s", buff, line, msg);
 952   if (tok)
 953     lj_strfmt_pushf(L, err2msg(LJ_ERR_XNEAR), msg, tok);
 954   lj_err_throw(L, LUA_ERRSYNTAX);
 955 }
 956
 957 /* Typecheck error for operands. */
 958 LJ_NOINLINE void lj_err_optype(lua_State *L, cTValue *o, ErrMsg opm)
 959 {
 960   const char *tname = lj_typename(o);
 961   const char *opname = err2msg(opm);
 962   if (curr_funcisL(L)) {
 963     GCproto *pt = curr_proto(L);
 964     const BCIns *pc = cframe_Lpc(L) - 1;
 965     const char *oname = NULL;
 966     const char *kind = lj_debug_slotname(pt, pc, (BCReg)(o-L->base), &oname);
 967     if (kind)
 968       err_msgv(L, LJ_ERR_BADOPRT, opname, kind, oname, tname);
 969   }
 970   err_msgv(L, LJ_ERR_BADOPRV, opname, tname);
 971 }
 972
 973 /* Typecheck error for ordered comparisons. */
 974 LJ_NOINLINE void lj_err_comp(lua_State *L, cTValue *o1, cTValue *o2)
 975 {
 976   const char *t1 = lj_typename(o1);
 977   const char *t2 = lj_typename(o2);
 978   err_msgv(L, t1 == t2 ? LJ_ERR_BADCMPV : LJ_ERR_BADCMPT, t1, t2);
 979   /* This assumes the two "boolean" entries are commoned by the C compiler. */
 980 }
 981
 982 /* Typecheck error for __call. */
 983 LJ_NOINLINE void lj_err_optype_call(lua_State *L, TValue *o)
 984 {
 985   /* Gross hack if lua_[p]call or pcall/xpcall fail for a non-callable object:
 986   ** L->base still points to the caller. So add a dummy frame with L instead
 987   ** of a function. See lua_getstack().
 988   */
 989   const BCIns *pc = cframe_Lpc(L);
 990   if (((ptrdiff_t)pc & FRAME_TYPE) != FRAME_LUA) {
 991     const char *tname = lj_typename(o);
 992     setframe_gc(o, obj2gco(L), LJ_TTHREAD);
 993     if (LJ_FR2) o++;
 994     setframe_pc(o, pc);
 995     L->top = L->base = o+1;
 996     err_msgv(L, LJ_ERR_BADCALL, tname);
 997   }
 998   lj_err_optype(L, o, LJ_ERR_OPCALL);
 999 }
1000
1001 /* Error in context of caller. */
1002 LJ_NOINLINE void lj_err_callermsg(lua_State *L, const char *msg)
1003 {
1004   TValue *frame = NULL, *pframe = NULL;
1005   if (!(LJ_HASJIT && tvref(G(L)->jit_base))) {
1006     frame = L->base-1;
1007     if (frame_islua(frame)) {
1008       pframe = frame_prevl(frame);
1009     } else if (frame_iscont(frame)) {
1010       if (frame_iscont_fficb(frame)) {
1011         pframe = frame;
1012         frame = NULL;
1013       } else {
1014         pframe = frame_prevd(frame);
1015 #if LJ_HASFFI
1016         /* Remove frame for FFI metamethods. */
1017         if (frame_func(frame)->c.ffid >= FF_ffi_meta___index &&
1018             frame_func(frame)->c.ffid <= FF_ffi_meta___tostring) {
1019           L->base = pframe+1;
1020           L->top = frame;
1021           setcframe_pc(cframe_raw(L->cframe), frame_contpc(frame));
1022         }
1023 #endif
1024       }
1025     }
1026   }
1027   lj_debug_addloc(L, msg, pframe, frame);
1028   lj_err_run(L);
1029 }
1030
1031 /* Formatted error in context of caller. */
1032 LJ_NOINLINE void lj_err_callerv(lua_State *L, ErrMsg em, ...)
1033 {
1034   const char *msg;
1035   va_list argp;
1036   va_start(argp, em);
1037   msg = lj_strfmt_pushvf(L, err2msg(em), argp);
1038   va_end(argp);
1039   lj_err_callermsg(L, msg);
1040 }
1041
1042 /* Error in context of caller. */
1043 LJ_NOINLINE void lj_err_caller(lua_State *L, ErrMsg em)
1044 {
1045   lj_err_callermsg(L, err2msg(em));
1046 }
1047
1048 /* Argument error message. */
1049 LJ_NORET LJ_NOINLINE static void err_argmsg(lua_State *L, int narg,
1050                                             const char *msg)
1051 {
1052   const char *fname = "?";
1053   const char *ftype = lj_debug_funcname(L, L->base - 1, &fname);
1054   if (narg < 0 && narg > LUA_REGISTRYINDEX)
1055     narg = (int)(L->top - L->base) + narg + 1;
1056   if (ftype && ftype[3] == 'h' && --narg == 0)  /* Check for "method". */
1057     msg = lj_strfmt_pushf(L, err2msg(LJ_ERR_BADSELF), fname, msg);
1058   else
1059     msg = lj_strfmt_pushf(L, err2msg(LJ_ERR_BADARG), narg, fname, msg);
1060   lj_err_callermsg(L, msg);
1061 }
1062
1063 /* Formatted argument error. */
1064 LJ_NOINLINE void lj_err_argv(lua_State *L, int narg, ErrMsg em, ...)
1065 {
1066   const char *msg;
1067   va_list argp;
1068   va_start(argp, em);
1069   msg = lj_strfmt_pushvf(L, err2msg(em), argp);
1070   va_end(argp);
1071   err_argmsg(L, narg, msg);
1072 }
1073
1074 /* Argument error. */
1075 LJ_NOINLINE void lj_err_arg(lua_State *L, int narg, ErrMsg em)
1076 {
1077   err_argmsg(L, narg, err2msg(em));
1078 }
1079
1080 /* Typecheck error for arguments. */
1081 LJ_NOINLINE void lj_err_argtype(lua_State *L, int narg, const char *xname)
1082 {
1083   const char *tname, *msg;
1084   if (narg <= LUA_REGISTRYINDEX) {
1085     if (narg >= LUA_GLOBALSINDEX) {
1086       tname = lj_obj_itypename[~LJ_TTAB];
1087     } else {
1088       GCfunc *fn = curr_func(L);
1089       int idx = LUA_GLOBALSINDEX - narg;
1090       if (idx <= fn->c.nupvalues)
1091         tname = lj_typename(&fn->c.upvalue[idx-1]);
1092       else
1093         tname = lj_obj_typename[0];
1094     }
1095   } else {
1096     TValue *o = narg < 0 ? L->top + narg : L->base + narg-1;
1097     tname = o < L->top ? lj_typename(o) : lj_obj_typename[0];
1098   }
1099   msg = lj_strfmt_pushf(L, err2msg(LJ_ERR_BADTYPE), xname, tname);
1100   err_argmsg(L, narg, msg);
1101 }
1102
1103 /* Typecheck error for arguments. */
1104 LJ_NOINLINE void lj_err_argt(lua_State *L, int narg, int tt)
1105 {
1106   lj_err_argtype(L, narg, lj_obj_typename[tt+1]);
1107 }
1108
1109 /* -- Public error handling API ------------------------------------------- */
1110
1111 LUA_API lua_CFunction lua_atpanic(lua_State *L, lua_CFunction panicf)
1112 {
1113   lua_CFunction old = G(L)->panic;
1114   G(L)->panic = panicf;
1115   return old;
1116 }
1117
1118 /* Forwarders for the public API (C calling convention and no LJ_NORET). */
1119 LUA_API int lua_error(lua_State *L)
1120 {
1121   lj_err_run(L);
1122   return 0;  /* unreachable */
1123 }
1124
1125 LUALIB_API int luaL_argerror(lua_State *L, int narg, const char *msg)
1126 {
1127   err_argmsg(L, narg, msg);
1128   return 0;  /* unreachable */
1129 }
1130
1131 LUALIB_API int luaL_typerror(lua_State *L, int narg, const char *xname)
1132 {
1133   lj_err_argtype(L, narg, xname);
1134   return 0;  /* unreachable */
1135 }
1136
1137 LUALIB_API void luaL_where(lua_State *L, int level)
1138 {
1139   int size;
1140   cTValue *frame = lj_debug_frame(L, level, &size);
1141   lj_debug_addloc(L, "", frame, size ? frame+size : NULL);
1142 }
1143
1144 LUALIB_API int luaL_error(lua_State *L, const char *fmt, ...)
1145 {
1146   const char *msg;
1147   va_list argp;
1148   va_start(argp, fmt);
1149   msg = lj_strfmt_pushvf(L, fmt, argp);
1150   va_end(argp);
1151   lj_err_callermsg(L, msg);
1152   return 0;  /* unreachable */
1153 }
1154