src/lj_obj.h

   1 /*
   2 ** LuaJIT VM tags, values and objects.
   3 ** Copyright (C) 2005-2025 Mike Pall. See Copyright Notice in luajit.h
   4 **
   5 ** Portions taken verbatim or adapted from the Lua interpreter.
   6 ** Copyright (C) 1994-2008 Lua.org, PUC-Rio. See Copyright Notice in lua.h
   7 */
   8
   9 #ifndef _LJ_OBJ_H
  10 #define _LJ_OBJ_H
  11
  12 #include "lua.h"
  13 #include "lj_def.h"
  14 #include "lj_arch.h"
  15
  16 /* -- Memory references --------------------------------------------------- */
  17
  18 /* Memory and GC object sizes. */
  19 typedef uint32_t MSize;
  20 #if LJ_GC64
  21 typedef uint64_t GCSize;
  22 #else
  23 typedef uint32_t GCSize;
  24 #endif
  25
  26 /* Memory reference */
  27 typedef struct MRef {
  28 #if LJ_GC64
  29   uint64_t ptr64;       /* True 64 bit pointer. */
  30 #else
  31   uint32_t ptr32;       /* Pseudo 32 bit pointer. */
  32 #endif
  33 } MRef;
  34
  35 #if LJ_GC64
  36 #define mref(r, t)      ((t *)(void *)(r).ptr64)
  37 #define mrefu(r)        ((r).ptr64)
  38
  39 #define setmref(r, p)   ((r).ptr64 = (uint64_t)(void *)(p))
  40 #define setmrefu(r, u)  ((r).ptr64 = (uint64_t)(u))
  41 #define setmrefr(r, v)  ((r).ptr64 = (v).ptr64)
  42 #else
  43 #define mref(r, t)      ((t *)(void *)(uintptr_t)(r).ptr32)
  44 #define mrefu(r)        ((r).ptr32)
  45
  46 #define setmref(r, p)   ((r).ptr32 = (uint32_t)(uintptr_t)(void *)(p))
  47 #define setmrefu(r, u)  ((r).ptr32 = (uint32_t)(u))
  48 #define setmrefr(r, v)  ((r).ptr32 = (v).ptr32)
  49 #endif
  50
  51 /* -- GC object references ------------------------------------------------ */
  52
  53 /* GCobj reference */
  54 typedef struct GCRef {
  55 #if LJ_GC64
  56   uint64_t gcptr64;     /* True 64 bit pointer. */
  57 #else
  58   uint32_t gcptr32;     /* Pseudo 32 bit pointer. */
  59 #endif
  60 } GCRef;
  61
  62 /* Common GC header for all collectable objects. */
  63 #define GCHeader        GCRef nextgc; uint8_t marked; uint8_t gct
  64 /* This occupies 6 bytes, so use the next 2 bytes for non-32 bit fields. */
  65
  66 #if LJ_GC64
  67 #define gcref(r)        ((GCobj *)(r).gcptr64)
  68 #define gcrefp(r, t)    ((t *)(void *)(r).gcptr64)
  69 #define gcrefu(r)       ((r).gcptr64)
  70 #define gcrefeq(r1, r2) ((r1).gcptr64 == (r2).gcptr64)
  71
  72 #define setgcref(r, gc) ((r).gcptr64 = (uint64_t)&(gc)->gch)
  73 #define setgcreft(r, gc, it) \
  74   (r).gcptr64 = (uint64_t)&(gc)->gch | (((uint64_t)(it)) << 47)
  75 #define setgcrefp(r, p) ((r).gcptr64 = (uint64_t)(p))
  76 #define setgcrefnull(r) ((r).gcptr64 = 0)
  77 #define setgcrefr(r, v) ((r).gcptr64 = (v).gcptr64)
  78 #else
  79 #define gcref(r)        ((GCobj *)(uintptr_t)(r).gcptr32)
  80 #define gcrefp(r, t)    ((t *)(void *)(uintptr_t)(r).gcptr32)
  81 #define gcrefu(r)       ((r).gcptr32)
  82 #define gcrefeq(r1, r2) ((r1).gcptr32 == (r2).gcptr32)
  83
  84 #define setgcref(r, gc) ((r).gcptr32 = (uint32_t)(uintptr_t)&(gc)->gch)
  85 #define setgcrefp(r, p) ((r).gcptr32 = (uint32_t)(uintptr_t)(p))
  86 #define setgcrefnull(r) ((r).gcptr32 = 0)
  87 #define setgcrefr(r, v) ((r).gcptr32 = (v).gcptr32)
  88 #endif
  89
  90 #define gcnext(gc)      (gcref((gc)->gch.nextgc))
  91
  92 /* IMPORTANT NOTE:
  93 **
  94 ** All uses of the setgcref* macros MUST be accompanied with a write barrier.
  95 **
  96 ** This is to ensure the integrity of the incremental GC. The invariant
  97 ** to preserve is that a black object never points to a white object.
  98 ** I.e. never store a white object into a field of a black object.
  99 **
 100 ** It's ok to LEAVE OUT the write barrier ONLY in the following cases:
 101 ** - The source is not a GC object (NULL).
 102 ** - The target is a GC root. I.e. everything in global_State.
 103 ** - The target is a lua_State field (threads are never black).
 104 ** - The target is a stack slot, see setgcV et al.
 105 ** - The target is an open upvalue, i.e. pointing to a stack slot.
 106 ** - The target is a newly created object (i.e. marked white). But make
 107 **   sure nothing invokes the GC inbetween.
 108 ** - The target and the source are the same object (self-reference).
 109 ** - The target already contains the object (e.g. moving elements around).
 110 **
 111 ** The most common case is a store to a stack slot. All other cases where
 112 ** a barrier has been omitted are annotated with a NOBARRIER comment.
 113 **
 114 ** The same logic applies for stores to table slots (array part or hash
 115 ** part). ALL uses of lj_tab_set* require a barrier for the stored value
 116 ** *and* the stored key, based on the above rules. In practice this means
 117 ** a barrier is needed if *either* of the key or value are a GC object.
 118 **
 119 ** It's ok to LEAVE OUT the write barrier in the following special cases:
 120 ** - The stored value is nil. The key doesn't matter because it's either
 121 **   not resurrected or lj_tab_newkey() will take care of the key barrier.
 122 ** - The key doesn't matter if the *previously* stored value is guaranteed
 123 **   to be non-nil (because the key is kept alive in the table).
 124 ** - The key doesn't matter if it's guaranteed not to be part of the table,
 125 **   since lj_tab_newkey() takes care of the key barrier. This applies
 126 **   trivially to new tables, but watch out for resurrected keys. Storing
 127 **   a nil value leaves the key in the table!
 128 **
 129 ** In case of doubt use lj_gc_anybarriert() as it's rather cheap. It's used
 130 ** by the interpreter for all table stores.
 131 **
 132 ** Note: In contrast to Lua's GC, LuaJIT's GC does *not* specially mark
 133 ** dead keys in tables. The reference is left in, but it's guaranteed to
 134 ** be never dereferenced as long as the value is nil. It's ok if the key is
 135 ** freed or if any object subsequently gets the same address.
 136 **
 137 ** Not destroying dead keys helps to keep key hash slots stable. This avoids
 138 ** specialization back-off for HREFK when a value flips between nil and
 139 ** non-nil and the GC gets in the way. It also allows safely hoisting
 140 ** HREF/HREFK across GC steps. Dead keys are only removed if a table is
 141 ** resized (i.e. by NEWREF) and xREF must not be CSEd across a resize.
 142 **
 143 ** The trade-off is that a write barrier for tables must take the key into
 144 ** account, too. Implicitly resurrecting the key by storing a non-nil value
 145 ** may invalidate the incremental GC invariant.
 146 */
 147
 148 /* -- Common type definitions --------------------------------------------- */
 149
 150 /* Types for handling bytecodes. Need this here, details in lj_bc.h. */
 151 typedef uint32_t BCIns;  /* Bytecode instruction. */
 152 typedef uint32_t BCPos;  /* Bytecode position. */
 153 typedef uint32_t BCReg;  /* Bytecode register. */
 154 typedef int32_t BCLine;  /* Bytecode line number. */
 155
 156 /* Internal assembler functions. Never call these directly from C. */
 157 typedef void (*ASMFunction)(void);
 158
 159 /* Resizable string buffer. Need this here, details in lj_buf.h. */
 160 #define SBufHeader      char *w, *e, *b; MRef L
 161 typedef struct SBuf {
 162   SBufHeader;
 163 } SBuf;
 164
 165 /* -- Tags and values ----------------------------------------------------- */
 166
 167 /* Frame link. */
 168 typedef union {
 169   int32_t ftsz;         /* Frame type and size of previous frame. */
 170   MRef pcr;             /* Or PC for Lua frames. */
 171 } FrameLink;
 172
 173 /* Tagged value. */
 174 typedef LJ_ALIGN(8) union TValue {
 175   uint64_t u64;         /* 64 bit pattern overlaps number. */
 176   lua_Number n;         /* Number object overlaps split tag/value object. */
 177 #if LJ_GC64
 178   GCRef gcr;            /* GCobj reference with tag. */
 179   int64_t it64;
 180   struct {
 181     LJ_ENDIAN_LOHI(
 182       int32_t i;        /* Integer value. */
 183     , uint32_t it;      /* Internal object tag. Must overlap MSW of number. */
 184     )
 185   };
 186 #else
 187   struct {
 188     LJ_ENDIAN_LOHI(
 189       union {
 190         GCRef gcr;      /* GCobj reference (if any). */
 191         int32_t i;      /* Integer value. */
 192       };
 193     , uint32_t it;      /* Internal object tag. Must overlap MSW of number. */
 194     )
 195   };
 196 #endif
 197 #if LJ_FR2
 198   int64_t ftsz;         /* Frame type and size of previous frame, or PC. */
 199 #else
 200   struct {
 201     LJ_ENDIAN_LOHI(
 202       GCRef func;       /* Function for next frame (or dummy L). */
 203     , FrameLink tp;     /* Link to previous frame. */
 204     )
 205   } fr;
 206 #endif
 207   struct {
 208     LJ_ENDIAN_LOHI(
 209       uint32_t lo;      /* Lower 32 bits of number. */
 210     , uint32_t hi;      /* Upper 32 bits of number. */
 211     )
 212   } u32;
 213 } TValue;
 214
 215 typedef const TValue cTValue;
 216
 217 #define tvref(r)        (mref(r, TValue))
 218
 219 /* More external and GCobj tags for internal objects. */
 220 #define LAST_TT         LUA_TTHREAD
 221 #define LUA_TPROTO      (LAST_TT+1)
 222 #define LUA_TCDATA      (LAST_TT+2)
 223
 224 /* Internal object tags.
 225 **
 226 ** Format for 32 bit GC references (!LJ_GC64):
 227 **
 228 ** Internal tags overlap the MSW of a number object (must be a double).
 229 ** Interpreted as a double these are special NaNs. The FPU only generates
 230 ** one type of NaN (0xfff8_0000_0000_0000). So MSWs > 0xfff80000 are available
 231 ** for use as internal tags. Small negative numbers are used to shorten the
 232 ** encoding of type comparisons (reg/mem against sign-ext. 8 bit immediate).
 233 **
 234 **                  ---MSW---.---LSW---
 235 ** primitive types |  itype  |         |
 236 ** lightuserdata   |  itype  |  void * |  (32 bit platforms)
 237 ** lightuserdata   |ffff|seg|    ofs   |  (64 bit platforms)
 238 ** GC objects      |  itype  |  GCRef  |
 239 ** int (LJ_DUALNUM)|  itype  |   int   |
 240 ** number           -------double------
 241 **
 242 ** Format for 64 bit GC references (LJ_GC64):
 243 **
 244 ** The upper 13 bits must be 1 (0xfff8...) for a special NaN. The next
 245 ** 4 bits hold the internal tag. The lowest 47 bits either hold a pointer,
 246 ** a zero-extended 32 bit integer or all bits set to 1 for primitive types.
 247 **
 248 **                     ------MSW------.------LSW------
 249 ** primitive types    |1..1|itype|1..................1|
 250 ** GC objects         |1..1|itype|-------GCRef--------|
 251 ** lightuserdata      |1..1|itype|seg|------ofs-------|
 252 ** int (LJ_DUALNUM)   |1..1|itype|0..0|-----int-------|
 253 ** number              ------------double-------------
 254 **
 255 ** ORDER LJ_T
 256 ** Primitive types nil/false/true must be first, lightuserdata next.
 257 ** GC objects are at the end, table/userdata must be lowest.
 258 ** Also check lj_ir.h for similar ordering constraints.
 259 */
 260 #define LJ_TNIL                 (~0u)
 261 #define LJ_TFALSE               (~1u)
 262 #define LJ_TTRUE                (~2u)
 263 #define LJ_TLIGHTUD             (~3u)
 264 #define LJ_TSTR                 (~4u)
 265 #define LJ_TUPVAL               (~5u)
 266 #define LJ_TTHREAD              (~6u)
 267 #define LJ_TPROTO               (~7u)
 268 #define LJ_TFUNC                (~8u)
 269 #define LJ_TTRACE               (~9u)
 270 #define LJ_TCDATA               (~10u)
 271 #define LJ_TTAB                 (~11u)
 272 #define LJ_TUDATA               (~12u)
 273 /* This is just the canonical number type used in some places. */
 274 #define LJ_TNUMX                (~13u)
 275
 276 /* Integers have itype == LJ_TISNUM doubles have itype < LJ_TISNUM */
 277 #if LJ_64 && !LJ_GC64
 278 #define LJ_TISNUM               0xfffeffffu
 279 #else
 280 #define LJ_TISNUM               LJ_TNUMX
 281 #endif
 282 #define LJ_TISTRUECOND          LJ_TFALSE
 283 #define LJ_TISPRI               LJ_TTRUE
 284 #define LJ_TISGCV               (LJ_TSTR+1)
 285 #define LJ_TISTABUD             LJ_TTAB
 286
 287 /* Type marker for slot holding a traversal index. Must be lightuserdata. */
 288 #define LJ_KEYINDEX             0xfffe7fffu
 289
 290 #if LJ_GC64
 291 #define LJ_GCVMASK              (((uint64_t)1 << 47) - 1)
 292 #endif
 293
 294 #if LJ_64
 295 /* To stay within 47 bits, lightuserdata is segmented. */
 296 #define LJ_LIGHTUD_BITS_SEG     8
 297 #define LJ_LIGHTUD_BITS_LO      (47 - LJ_LIGHTUD_BITS_SEG)
 298 #endif
 299
 300 /* -- String object ------------------------------------------------------- */
 301
 302 typedef uint32_t StrHash;       /* String hash value. */
 303 typedef uint32_t StrID;         /* String ID. */
 304
 305 /* String object header. String payload follows. */
 306 typedef struct GCstr {
 307   GCHeader;
 308   uint8_t reserved;     /* Used by lexer for fast lookup of reserved words. */
 309   uint8_t hashalg;      /* Hash algorithm. */
 310   StrID sid;            /* Interned string ID. */
 311   StrHash hash;         /* Hash of string. */
 312   MSize len;            /* Size of string. */
 313 } GCstr;
 314
 315 #define strref(r)       (&gcref((r))->str)
 316 #define strdata(s)      ((const char *)((s)+1))
 317 #define strdatawr(s)    ((char *)((s)+1))
 318 #define strVdata(o)     strdata(strV(o))
 319
 320 /* -- Userdata object ----------------------------------------------------- */
 321
 322 /* Userdata object. Payload follows. */
 323 typedef struct GCudata {
 324   GCHeader;
 325   uint8_t udtype;       /* Userdata type. */
 326   uint8_t unused2;
 327   GCRef env;            /* Should be at same offset in GCfunc. */
 328   MSize len;            /* Size of payload. */
 329   GCRef metatable;      /* Must be at same offset in GCtab. */
 330   uint32_t align1;      /* To force 8 byte alignment of the payload. */
 331 } GCudata;
 332
 333 /* Userdata types. */
 334 enum {
 335   UDTYPE_USERDATA,      /* Regular userdata. */
 336   UDTYPE_IO_FILE,       /* I/O library FILE. */
 337   UDTYPE_FFI_CLIB,      /* FFI C library namespace. */
 338   UDTYPE_BUFFER,        /* String buffer. */
 339   UDTYPE__MAX
 340 };
 341
 342 #define uddata(u)       ((void *)((u)+1))
 343 #define sizeudata(u)    (sizeof(struct GCudata)+(u)->len)
 344
 345 /* -- C data object ------------------------------------------------------- */
 346
 347 /* C data object. Payload follows. */
 348 typedef struct GCcdata {
 349   GCHeader;
 350   uint16_t ctypeid;     /* C type ID. */
 351 } GCcdata;
 352
 353 /* Prepended to variable-sized or realigned C data objects. */
 354 typedef struct GCcdataVar {
 355   uint16_t offset;      /* Offset to allocated memory (relative to GCcdata). */
 356   uint16_t extra;       /* Extra space allocated (incl. GCcdata + GCcdatav). */
 357   MSize len;            /* Size of payload. */
 358 } GCcdataVar;
 359
 360 #define cdataptr(cd)    ((void *)((cd)+1))
 361 #define cdataisv(cd)    ((cd)->marked & 0x80)
 362 #define cdatav(cd)      ((GCcdataVar *)((char *)(cd) - sizeof(GCcdataVar)))
 363 #define cdatavlen(cd)   check_exp(cdataisv(cd), cdatav(cd)->len)
 364 #define sizecdatav(cd)  (cdatavlen(cd) + cdatav(cd)->extra)
 365 #define memcdatav(cd)   ((void *)((char *)(cd) - cdatav(cd)->offset))
 366
 367 /* -- Prototype object ---------------------------------------------------- */
 368
 369 #define SCALE_NUM_GCO   ((int32_t)sizeof(lua_Number)/sizeof(GCRef))
 370 #define round_nkgc(n)   (((n) + SCALE_NUM_GCO-1) & ~(SCALE_NUM_GCO-1))
 371
 372 typedef struct GCproto {
 373   GCHeader;
 374   uint8_t numparams;    /* Number of parameters. */
 375   uint8_t framesize;    /* Fixed frame size. */
 376   MSize sizebc;         /* Number of bytecode instructions. */
 377 #if LJ_GC64
 378   uint32_t unused_gc64;
 379 #endif
 380   GCRef gclist;
 381   MRef k;               /* Split constant array (points to the middle). */
 382   MRef uv;              /* Upvalue list. local slot|0x8000 or parent uv idx. */
 383   MSize sizekgc;        /* Number of collectable constants. */
 384   MSize sizekn;         /* Number of lua_Number constants. */
 385   MSize sizept;         /* Total size including colocated arrays. */
 386   uint8_t sizeuv;       /* Number of upvalues. */
 387   uint8_t flags;        /* Miscellaneous flags (see below). */
 388   uint16_t trace;       /* Anchor for chain of root traces. */
 389   /* ------ The following fields are for debugging/tracebacks only ------ */
 390   GCRef chunkname;      /* Name of the chunk this function was defined in. */
 391   BCLine firstline;     /* First line of the function definition. */
 392   BCLine numline;       /* Number of lines for the function definition. */
 393   MRef lineinfo;        /* Compressed map from bytecode ins. to source line. */
 394   MRef uvinfo;          /* Upvalue names. */
 395   MRef varinfo;         /* Names and compressed extents of local variables. */
 396 } GCproto;
 397
 398 /* Flags for prototype. */
 399 #define PROTO_CHILD             0x01    /* Has child prototypes. */
 400 #define PROTO_VARARG            0x02    /* Vararg function. */
 401 #define PROTO_FFI               0x04    /* Uses BC_KCDATA for FFI datatypes. */
 402 #define PROTO_NOJIT             0x08    /* JIT disabled for this function. */
 403 #define PROTO_ILOOP             0x10    /* Patched bytecode with ILOOP etc. */
 404 /* Only used during parsing. */
 405 #define PROTO_HAS_RETURN        0x20    /* Already emitted a return. */
 406 #define PROTO_FIXUP_RETURN      0x40    /* Need to fixup emitted returns. */
 407 /* Top bits used for counting created closures. */
 408 #define PROTO_CLCOUNT           0x20    /* Base of saturating 3 bit counter. */
 409 #define PROTO_CLC_BITS          3
 410 #define PROTO_CLC_POLY          (3*PROTO_CLCOUNT)  /* Polymorphic threshold. */
 411
 412 #define PROTO_UV_LOCAL          0x8000  /* Upvalue for local slot. */
 413 #define PROTO_UV_IMMUTABLE      0x4000  /* Immutable upvalue. */
 414
 415 #define proto_kgc(pt, idx) \
 416   check_exp((uintptr_t)(intptr_t)(idx) >= ~(uintptr_t)(pt)->sizekgc+1u, \
 417             gcref(mref((pt)->k, GCRef)[(idx)]))
 418 #define proto_knumtv(pt, idx) \
 419   check_exp((uintptr_t)(idx) < (pt)->sizekn, &mref((pt)->k, TValue)[(idx)])
 420 #define proto_bc(pt)            ((BCIns *)((char *)(pt) + sizeof(GCproto)))
 421 #define proto_bcpos(pt, pc)     ((BCPos)((pc) - proto_bc(pt)))
 422 #define proto_uv(pt)            (mref((pt)->uv, uint16_t))
 423
 424 #define proto_chunkname(pt)     (strref((pt)->chunkname))
 425 #define proto_chunknamestr(pt)  (strdata(proto_chunkname((pt))))
 426 #define proto_lineinfo(pt)      (mref((pt)->lineinfo, const void))
 427 #define proto_uvinfo(pt)        (mref((pt)->uvinfo, const uint8_t))
 428 #define proto_varinfo(pt)       (mref((pt)->varinfo, const uint8_t))
 429
 430 /* -- Upvalue object ------------------------------------------------------ */
 431
 432 typedef struct GCupval {
 433   GCHeader;
 434   uint8_t closed;       /* Set if closed (i.e. uv->v == &uv->u.value). */
 435   uint8_t immutable;    /* Immutable value. */
 436   union {
 437     TValue tv;          /* If closed: the value itself. */
 438     struct {            /* If open: double linked list, anchored at thread. */
 439       GCRef prev;
 440       GCRef next;
 441     };
 442   };
 443   MRef v;               /* Points to stack slot (open) or above (closed). */
 444   uint32_t dhash;       /* Disambiguation hash: dh1 != dh2 => cannot alias. */
 445 } GCupval;
 446
 447 #define uvprev(uv_)     (&gcref((uv_)->prev)->uv)
 448 #define uvnext(uv_)     (&gcref((uv_)->next)->uv)
 449 #define uvval(uv_)      (mref((uv_)->v, TValue))
 450
 451 /* -- Function object (closures) ------------------------------------------ */
 452
 453 /* Common header for functions. env should be at same offset in GCudata. */
 454 #define GCfuncHeader \
 455   GCHeader; uint8_t ffid; uint8_t nupvalues; \
 456   GCRef env; GCRef gclist; MRef pc
 457
 458 typedef struct GCfuncC {
 459   GCfuncHeader;
 460   lua_CFunction f;      /* C function to be called. */
 461   TValue upvalue[1];    /* Array of upvalues (TValue). */
 462 } GCfuncC;
 463
 464 typedef struct GCfuncL {
 465   GCfuncHeader;
 466   GCRef uvptr[1];       /* Array of _pointers_ to upvalue objects (GCupval). */
 467 } GCfuncL;
 468
 469 typedef union GCfunc {
 470   GCfuncC c;
 471   GCfuncL l;
 472 } GCfunc;
 473
 474 #define FF_LUA          0
 475 #define FF_C            1
 476 #define isluafunc(fn)   ((fn)->c.ffid == FF_LUA)
 477 #define iscfunc(fn)     ((fn)->c.ffid == FF_C)
 478 #define isffunc(fn)     ((fn)->c.ffid > FF_C)
 479 #define funcproto(fn) \
 480   check_exp(isluafunc(fn), (GCproto *)(mref((fn)->l.pc, char)-sizeof(GCproto)))
 481 #define sizeCfunc(n)    (sizeof(GCfuncC)-sizeof(TValue)+sizeof(TValue)*(n))
 482 #define sizeLfunc(n)    (sizeof(GCfuncL)-sizeof(GCRef)+sizeof(GCRef)*(n))
 483
 484 /* -- Table object -------------------------------------------------------- */
 485
 486 /* Hash node. */
 487 typedef struct Node {
 488   TValue val;           /* Value object. Must be first field. */
 489   TValue key;           /* Key object. */
 490   MRef next;            /* Hash chain. */
 491 #if !LJ_GC64
 492   MRef freetop;         /* Top of free elements (stored in t->node[0]). */
 493 #endif
 494 } Node;
 495
 496 LJ_STATIC_ASSERT(offsetof(Node, val) == 0);
 497
 498 typedef struct GCtab {
 499   GCHeader;
 500   uint8_t nomm;         /* Negative cache for fast metamethods. */
 501   int8_t colo;          /* Array colocation. */
 502   MRef array;           /* Array part. */
 503   GCRef gclist;
 504   GCRef metatable;      /* Must be at same offset in GCudata. */
 505   MRef node;            /* Hash part. */
 506   uint32_t asize;       /* Size of array part (keys [0, asize-1]). */
 507   uint32_t hmask;       /* Hash part mask (size of hash part - 1). */
 508 #if LJ_GC64
 509   MRef freetop;         /* Top of free elements. */
 510 #endif
 511 } GCtab;
 512
 513 #define sizetabcolo(n)  ((n)*sizeof(TValue) + sizeof(GCtab))
 514 #define tabref(r)       ((GCtab *)gcref((r)))
 515 #define noderef(r)      (mref((r), Node))
 516 #define nextnode(n)     (mref((n)->next, Node))
 517 #if LJ_GC64
 518 #define getfreetop(t, n)        (noderef((t)->freetop))
 519 #define setfreetop(t, n, v)     (setmref((t)->freetop, (v)))
 520 #else
 521 #define getfreetop(t, n)        (noderef((n)->freetop))
 522 #define setfreetop(t, n, v)     (setmref((n)->freetop, (v)))
 523 #endif
 524
 525 /* -- State objects ------------------------------------------------------- */
 526
 527 /* VM states. */
 528 enum {
 529   LJ_VMST_INTERP,       /* Interpreter. */
 530   LJ_VMST_C,            /* C function. */
 531   LJ_VMST_GC,           /* Garbage collector. */
 532   LJ_VMST_EXIT,         /* Trace exit handler. */
 533   LJ_VMST_RECORD,       /* Trace recorder. */
 534   LJ_VMST_OPT,          /* Optimizer. */
 535   LJ_VMST_ASM,          /* Assembler. */
 536   LJ_VMST__MAX
 537 };
 538
 539 #define setvmstate(g, st)       ((g)->vmstate = ~LJ_VMST_##st)
 540
 541 /* Metamethods. ORDER MM */
 542 #ifdef LJ_HASFFI
 543 #define MMDEF_FFI(_) _(new)
 544 #else
 545 #define MMDEF_FFI(_)
 546 #endif
 547
 548 #if LJ_52 || LJ_HASFFI
 549 #define MMDEF_PAIRS(_) _(pairs) _(ipairs)
 550 #else
 551 #define MMDEF_PAIRS(_)
 552 #define MM_pairs        255
 553 #define MM_ipairs       255
 554 #endif
 555
 556 #define MMDEF(_) \
 557   _(index) _(newindex) _(gc) _(mode) _(eq) _(len) \
 558   /* Only the above (fast) metamethods are negative cached (max. 8). */ \
 559   _(lt) _(le) _(concat) _(call) \
 560   /* The following must be in ORDER ARITH. */ \
 561   _(add) _(sub) _(mul) _(div) _(mod) _(pow) _(unm) \
 562   /* The following are used in the standard libraries. */ \
 563   _(metatable) _(tostring) MMDEF_FFI(_) MMDEF_PAIRS(_)
 564
 565 typedef enum {
 566 #define MMENUM(name)    MM_##name,
 567 MMDEF(MMENUM)
 568 #undef MMENUM
 569   MM__MAX,
 570   MM____ = MM__MAX,
 571   MM_FAST = MM_len
 572 } MMS;
 573
 574 /* GC root IDs. */
 575 typedef enum {
 576   GCROOT_MMNAME,        /* Metamethod names. */
 577   GCROOT_MMNAME_LAST = GCROOT_MMNAME + MM__MAX-1,
 578   GCROOT_BASEMT,        /* Metatables for base types. */
 579   GCROOT_BASEMT_NUM = GCROOT_BASEMT + ~LJ_TNUMX,
 580   GCROOT_IO_INPUT,      /* Userdata for default I/O input file. */
 581   GCROOT_IO_OUTPUT,     /* Userdata for default I/O output file. */
 582 #if LJ_HASFFI
 583   GCROOT_FFI_FIN,       /* FFI finalizer table. */
 584 #endif
 585   GCROOT_MAX
 586 } GCRootID;
 587
 588 #define basemt_it(g, it)        ((g)->gcroot[GCROOT_BASEMT+~(it)])
 589 #define basemt_obj(g, o)        ((g)->gcroot[GCROOT_BASEMT+itypemap(o)])
 590 #define mmname_str(g, mm)       (strref((g)->gcroot[GCROOT_MMNAME+(mm)]))
 591
 592 /* Garbage collector state. */
 593 typedef struct GCState {
 594   GCSize total;         /* Memory currently allocated. */
 595   GCSize threshold;     /* Memory threshold. */
 596   uint8_t currentwhite; /* Current white color. */
 597   uint8_t state;        /* GC state. */
 598   uint8_t unused0;
 599 #if LJ_64
 600   uint8_t lightudnum;   /* Number of lightuserdata segments - 1. */
 601 #else
 602   uint8_t unused1;
 603 #endif
 604   MSize sweepstr;       /* Sweep position in string table. */
 605   GCRef root;           /* List of all collectable objects. */
 606   MRef sweep;           /* Sweep position in root list. */
 607   GCRef gray;           /* List of gray objects. */
 608   GCRef grayagain;      /* List of objects for atomic traversal. */
 609   GCRef weak;           /* List of weak tables (to be cleared). */
 610   GCRef mmudata;        /* List of userdata (to be finalized). */
 611   GCSize debt;          /* Debt (how much GC is behind schedule). */
 612   GCSize estimate;      /* Estimate of memory actually in use. */
 613   MSize stepmul;        /* Incremental GC step granularity. */
 614   MSize pause;          /* Pause between successive GC cycles. */
 615 #if LJ_64
 616   MRef lightudseg;      /* Upper bits of lightuserdata segments. */
 617 #endif
 618 } GCState;
 619
 620 /* String interning state. */
 621 typedef struct StrInternState {
 622   GCRef *tab;           /* String hash table anchors. */
 623   MSize mask;           /* String hash mask (size of hash table - 1). */
 624   MSize num;            /* Number of strings in hash table. */
 625   StrID id;             /* Next string ID. */
 626   uint8_t idreseed;     /* String ID reseed counter. */
 627   uint8_t second;       /* String interning table uses secondary hashing. */
 628   uint8_t unused1;
 629   uint8_t unused2;
 630   LJ_ALIGN(8) uint64_t seed;    /* Random string seed. */
 631 } StrInternState;
 632
 633 /* Global state, shared by all threads of a Lua universe. */
 634 typedef struct global_State {
 635   lua_Alloc allocf;     /* Memory allocator. */
 636   void *allocd;         /* Memory allocator data. */
 637   GCState gc;           /* Garbage collector. */
 638   GCstr strempty;       /* Empty string. */
 639   uint8_t stremptyz;    /* Zero terminator of empty string. */
 640   uint8_t hookmask;     /* Hook mask. */
 641   uint8_t dispatchmode; /* Dispatch mode. */
 642   uint8_t vmevmask;     /* VM event mask. */
 643   StrInternState str;   /* String interning. */
 644   volatile int32_t vmstate;  /* VM state or current JIT code trace number. */
 645   GCRef mainthref;      /* Link to main thread. */
 646   SBuf tmpbuf;          /* Temporary string buffer. */
 647   TValue tmptv, tmptv2; /* Temporary TValues. */
 648   Node nilnode;         /* Fallback 1-element hash part (nil key and value). */
 649   TValue registrytv;    /* Anchor for registry. */
 650   GCupval uvhead;       /* Head of double-linked list of all open upvalues. */
 651   int32_t hookcount;    /* Instruction hook countdown. */
 652   int32_t hookcstart;   /* Start count for instruction hook counter. */
 653   lua_Hook hookf;       /* Hook function. */
 654   lua_CFunction wrapf;  /* Wrapper for C function calls. */
 655   lua_CFunction panic;  /* Called as a last resort for errors. */
 656   BCIns bc_cfunc_int;   /* Bytecode for internal C function calls. */
 657   BCIns bc_cfunc_ext;   /* Bytecode for external C function calls. */
 658   GCRef cur_L;          /* Currently executing lua_State. */
 659   MRef jit_base;        /* Current JIT code L->base or NULL. */
 660   MRef ctype_state;     /* Pointer to C type state. */
 661   PRNGState prng;       /* Global PRNG state. */
 662   GCRef gcroot[GCROOT_MAX];  /* GC roots. */
 663 } global_State;
 664
 665 #define mainthread(g)   (&gcref(g->mainthref)->th)
 666 #define niltv(L) \
 667   check_exp(tvisnil(&G(L)->nilnode.val), &G(L)->nilnode.val)
 668 #define niltvg(g) \
 669   check_exp(tvisnil(&(g)->nilnode.val), &(g)->nilnode.val)
 670
 671 /* Hook management. Hook event masks are defined in lua.h. */
 672 #define HOOK_EVENTMASK          0x0f
 673 #define HOOK_ACTIVE             0x10
 674 #define HOOK_ACTIVE_SHIFT       4
 675 #define HOOK_VMEVENT            0x20
 676 #define HOOK_GC                 0x40
 677 #define HOOK_PROFILE            0x80
 678 #define hook_active(g)          ((g)->hookmask & HOOK_ACTIVE)
 679 #define hook_enter(g)           ((g)->hookmask |= HOOK_ACTIVE)
 680 #define hook_entergc(g) \
 681   ((g)->hookmask = ((g)->hookmask | (HOOK_ACTIVE|HOOK_GC)) & ~HOOK_PROFILE)
 682 #define hook_vmevent(g)         ((g)->hookmask |= (HOOK_ACTIVE|HOOK_VMEVENT))
 683 #define hook_leave(g)           ((g)->hookmask &= ~HOOK_ACTIVE)
 684 #define hook_save(g)            ((g)->hookmask & ~HOOK_EVENTMASK)
 685 #define hook_restore(g, h) \
 686   ((g)->hookmask = ((g)->hookmask & HOOK_EVENTMASK) | (h))
 687
 688 /* Per-thread state object. */
 689 struct lua_State {
 690   GCHeader;
 691   uint8_t dummy_ffid;   /* Fake FF_C for curr_funcisL() on dummy frames. */
 692   uint8_t status;       /* Thread status. */
 693   MRef glref;           /* Link to global state. */
 694   GCRef gclist;         /* GC chain. */
 695   TValue *base;         /* Base of currently executing function. */
 696   TValue *top;          /* First free slot in the stack. */
 697   MRef maxstack;        /* Last free slot in the stack. */
 698   MRef stack;           /* Stack base. */
 699   GCRef openupval;      /* List of open upvalues in the stack. */
 700   GCRef env;            /* Thread environment (table of globals). */
 701   void *cframe;         /* End of C stack frame chain. */
 702   MSize stacksize;      /* True stack size (incl. LJ_STACK_EXTRA). */
 703 };
 704
 705 #define G(L)                    (mref(L->glref, global_State))
 706 #define registry(L)             (&G(L)->registrytv)
 707
 708 /* Macros to access the currently executing (Lua) function. */
 709 #if LJ_GC64
 710 #define curr_func(L)            (&gcval(L->base-2)->fn)
 711 #elif LJ_FR2
 712 #define curr_func(L)            (&gcref((L->base-2)->gcr)->fn)
 713 #else
 714 #define curr_func(L)            (&gcref((L->base-1)->fr.func)->fn)
 715 #endif
 716 #define curr_funcisL(L)         (isluafunc(curr_func(L)))
 717 #define curr_proto(L)           (funcproto(curr_func(L)))
 718 #define curr_topL(L)            (L->base + curr_proto(L)->framesize)
 719 #define curr_top(L)             (curr_funcisL(L) ? curr_topL(L) : L->top)
 720
 721 #if defined(LUA_USE_ASSERT) || defined(LUA_USE_APICHECK)
 722 LJ_FUNC_NORET void lj_assert_fail(global_State *g, const char *file, int line,
 723                                   const char *func, const char *fmt, ...);
 724 #endif
 725
 726 /* -- GC object definition and conversions -------------------------------- */
 727
 728 /* GC header for generic access to common fields of GC objects. */
 729 typedef struct GChead {
 730   GCHeader;
 731   uint8_t unused1;
 732   uint8_t unused2;
 733   GCRef env;
 734   GCRef gclist;
 735   GCRef metatable;
 736 } GChead;
 737
 738 /* The env field SHOULD be at the same offset for all GC objects. */
 739 LJ_STATIC_ASSERT(offsetof(GChead, env) == offsetof(GCfuncL, env));
 740 LJ_STATIC_ASSERT(offsetof(GChead, env) == offsetof(GCudata, env));
 741
 742 /* The metatable field MUST be at the same offset for all GC objects. */
 743 LJ_STATIC_ASSERT(offsetof(GChead, metatable) == offsetof(GCtab, metatable));
 744 LJ_STATIC_ASSERT(offsetof(GChead, metatable) == offsetof(GCudata, metatable));
 745
 746 /* The gclist field MUST be at the same offset for all GC objects. */
 747 LJ_STATIC_ASSERT(offsetof(GChead, gclist) == offsetof(lua_State, gclist));
 748 LJ_STATIC_ASSERT(offsetof(GChead, gclist) == offsetof(GCproto, gclist));
 749 LJ_STATIC_ASSERT(offsetof(GChead, gclist) == offsetof(GCfuncL, gclist));
 750 LJ_STATIC_ASSERT(offsetof(GChead, gclist) == offsetof(GCtab, gclist));
 751
 752 typedef union GCobj {
 753   GChead gch;
 754   GCstr str;
 755   GCupval uv;
 756   lua_State th;
 757   GCproto pt;
 758   GCfunc fn;
 759   GCcdata cd;
 760   GCtab tab;
 761   GCudata ud;
 762 } GCobj;
 763
 764 /* Macros to convert a GCobj pointer into a specific value. */
 765 #define gco2str(o)      check_exp((o)->gch.gct == ~LJ_TSTR, &(o)->str)
 766 #define gco2uv(o)       check_exp((o)->gch.gct == ~LJ_TUPVAL, &(o)->uv)
 767 #define gco2th(o)       check_exp((o)->gch.gct == ~LJ_TTHREAD, &(o)->th)
 768 #define gco2pt(o)       check_exp((o)->gch.gct == ~LJ_TPROTO, &(o)->pt)
 769 #define gco2func(o)     check_exp((o)->gch.gct == ~LJ_TFUNC, &(o)->fn)
 770 #define gco2cd(o)       check_exp((o)->gch.gct == ~LJ_TCDATA, &(o)->cd)
 771 #define gco2tab(o)      check_exp((o)->gch.gct == ~LJ_TTAB, &(o)->tab)
 772 #define gco2ud(o)       check_exp((o)->gch.gct == ~LJ_TUDATA, &(o)->ud)
 773
 774 /* Macro to convert any collectable object into a GCobj pointer. */
 775 #define obj2gco(v)      ((GCobj *)(v))
 776
 777 /* -- TValue getters/setters ---------------------------------------------- */
 778
 779 /* Macros to test types. */
 780 #if LJ_GC64
 781 #define itype(o)        ((uint32_t)((o)->it64 >> 47))
 782 #define tvisnil(o)      ((o)->it64 == -1)
 783 #else
 784 #define itype(o)        ((o)->it)
 785 #define tvisnil(o)      (itype(o) == LJ_TNIL)
 786 #endif
 787 #define tvisfalse(o)    (itype(o) == LJ_TFALSE)
 788 #define tvistrue(o)     (itype(o) == LJ_TTRUE)
 789 #define tvisbool(o)     (tvisfalse(o) || tvistrue(o))
 790 #if LJ_64 && !LJ_GC64
 791 #define tvislightud(o)  (((int32_t)itype(o) >> 15) == -2)
 792 #else
 793 #define tvislightud(o)  (itype(o) == LJ_TLIGHTUD)
 794 #endif
 795 #define tvisstr(o)      (itype(o) == LJ_TSTR)
 796 #define tvisfunc(o)     (itype(o) == LJ_TFUNC)
 797 #define tvisthread(o)   (itype(o) == LJ_TTHREAD)
 798 #define tvisproto(o)    (itype(o) == LJ_TPROTO)
 799 #define tviscdata(o)    (itype(o) == LJ_TCDATA)
 800 #define tvistab(o)      (itype(o) == LJ_TTAB)
 801 #define tvisudata(o)    (itype(o) == LJ_TUDATA)
 802 #define tvisnumber(o)   (itype(o) <= LJ_TISNUM)
 803 #define tvisint(o)      (LJ_DUALNUM && itype(o) == LJ_TISNUM)
 804 #define tvisnum(o)      (itype(o) < LJ_TISNUM)
 805
 806 #define tvistruecond(o) (itype(o) < LJ_TISTRUECOND)
 807 #define tvispri(o)      (itype(o) >= LJ_TISPRI)
 808 #define tvistabud(o)    (itype(o) <= LJ_TISTABUD)  /* && !tvisnum() */
 809 #define tvisgcv(o)      ((itype(o) - LJ_TISGCV) > (LJ_TNUMX - LJ_TISGCV))
 810
 811 /* Special macros to test numbers for NaN, +0, -0, +1 and raw equality. */
 812 #define tvisnan(o)      ((o)->n != (o)->n)
 813 #if LJ_64
 814 #define tviszero(o)     (((o)->u64 << 1) == 0)
 815 #else
 816 #define tviszero(o)     (((o)->u32.lo | ((o)->u32.hi << 1)) == 0)
 817 #endif
 818 #define tvispzero(o)    ((o)->u64 == 0)
 819 #define tvismzero(o)    ((o)->u64 == U64x(80000000,00000000))
 820 #define tvispone(o)     ((o)->u64 == U64x(3ff00000,00000000))
 821 #define rawnumequal(o1, o2)     ((o1)->u64 == (o2)->u64)
 822
 823 /* Macros to convert type ids. */
 824 #if LJ_64 && !LJ_GC64
 825 #define itypemap(o) \
 826   (tvisnumber(o) ? ~LJ_TNUMX : tvislightud(o) ? ~LJ_TLIGHTUD : ~itype(o))
 827 #else
 828 #define itypemap(o)     (tvisnumber(o) ? ~LJ_TNUMX : ~itype(o))
 829 #endif
 830
 831 /* Macros to get tagged values. */
 832 #if LJ_GC64
 833 #define gcval(o)        ((GCobj *)(gcrefu((o)->gcr) & LJ_GCVMASK))
 834 #else
 835 #define gcval(o)        (gcref((o)->gcr))
 836 #endif
 837 #define boolV(o)        check_exp(tvisbool(o), (LJ_TFALSE - itype(o)))
 838 #if LJ_64
 839 #define lightudseg(u) \
 840   (((u) >> LJ_LIGHTUD_BITS_LO) & ((1 << LJ_LIGHTUD_BITS_SEG)-1))
 841 #define lightudlo(u) \
 842   ((u) & (((uint64_t)1 << LJ_LIGHTUD_BITS_LO) - 1))
 843 #define lightudup(p) \
 844   ((uint32_t)(((p) >> LJ_LIGHTUD_BITS_LO) << (LJ_LIGHTUD_BITS_LO-32)))
 845 static LJ_AINLINE void *lightudV(global_State *g, cTValue *o)
 846 {
 847   uint64_t u = o->u64;
 848   uint64_t seg = lightudseg(u);
 849   uint32_t *segmap = mref(g->gc.lightudseg, uint32_t);
 850   lj_assertG(tvislightud(o), "lightuserdata expected");
 851   if (seg == (1 << LJ_LIGHTUD_BITS_SEG)-1) return NULL;
 852   lj_assertG(seg <= g->gc.lightudnum, "bad lightuserdata segment %d", seg);
 853   return (void *)(((uint64_t)segmap[seg] << 32) | lightudlo(u));
 854 }
 855 #else
 856 #define lightudV(g, o)  check_exp(tvislightud(o), gcrefp((o)->gcr, void))
 857 #endif
 858 #define gcV(o)          check_exp(tvisgcv(o), gcval(o))
 859 #define strV(o)         check_exp(tvisstr(o), &gcval(o)->str)
 860 #define funcV(o)        check_exp(tvisfunc(o), &gcval(o)->fn)
 861 #define threadV(o)      check_exp(tvisthread(o), &gcval(o)->th)
 862 #define protoV(o)       check_exp(tvisproto(o), &gcval(o)->pt)
 863 #define cdataV(o)       check_exp(tviscdata(o), &gcval(o)->cd)
 864 #define tabV(o)         check_exp(tvistab(o), &gcval(o)->tab)
 865 #define udataV(o)       check_exp(tvisudata(o), &gcval(o)->ud)
 866 #define numV(o)         check_exp(tvisnum(o), (o)->n)
 867 #define intV(o)         check_exp(tvisint(o), (int32_t)(o)->i)
 868
 869 /* Macros to set tagged values. */
 870 #if LJ_GC64
 871 #define setitype(o, i)          ((o)->it = ((i) << 15))
 872 #define setnilV(o)              ((o)->it64 = -1)
 873 #define setpriV(o, x)           ((o)->it64 = (int64_t)~((uint64_t)~(x)<<47))
 874 #define setboolV(o, x)          ((o)->it64 = (int64_t)~((uint64_t)((x)+1)<<47))
 875 #else
 876 #define setitype(o, i)          ((o)->it = (i))
 877 #define setnilV(o)              ((o)->it = LJ_TNIL)
 878 #define setboolV(o, x)          ((o)->it = LJ_TFALSE-(uint32_t)(x))
 879 #define setpriV(o, i)           (setitype((o), (i)))
 880 #endif
 881
 882 static LJ_AINLINE void setrawlightudV(TValue *o, void *p)
 883 {
 884 #if LJ_GC64
 885   o->u64 = (uint64_t)p | (((uint64_t)LJ_TLIGHTUD) << 47);
 886 #elif LJ_64
 887   o->u64 = (uint64_t)p | (((uint64_t)0xffff) << 48);
 888 #else
 889   setgcrefp(o->gcr, p); setitype(o, LJ_TLIGHTUD);
 890 #endif
 891 }
 892
 893 #if LJ_FR2 || LJ_32
 894 #define contptr(f)              ((void *)(f))
 895 #define setcont(o, f)           ((o)->u64 = (uint64_t)(uintptr_t)contptr(f))
 896 #else
 897 #define contptr(f) \
 898   ((void *)(uintptr_t)(uint32_t)((intptr_t)(f) - (intptr_t)lj_vm_asm_begin))
 899 #define setcont(o, f) \
 900   ((o)->u64 = (uint64_t)(void *)(f) - (uint64_t)lj_vm_asm_begin)
 901 #endif
 902
 903 static LJ_AINLINE void checklivetv(lua_State *L, TValue *o, const char *msg)
 904 {
 905   UNUSED(L); UNUSED(o); UNUSED(msg);
 906 #if LUA_USE_ASSERT
 907   if (tvisgcv(o)) {
 908     lj_assertL(~itype(o) == gcval(o)->gch.gct,
 909                "mismatch of TValue type %d vs GC type %d",
 910                ~itype(o), gcval(o)->gch.gct);
 911     /* Copy of isdead check from lj_gc.h to avoid circular include. */
 912     lj_assertL(!(gcval(o)->gch.marked & (G(L)->gc.currentwhite ^ 3) & 3), msg);
 913   }
 914 #endif
 915 }
 916
 917 static LJ_AINLINE void setgcVraw(TValue *o, GCobj *v, uint32_t itype)
 918 {
 919 #if LJ_GC64
 920   setgcreft(o->gcr, v, itype);
 921 #else
 922   setgcref(o->gcr, v); setitype(o, itype);
 923 #endif
 924 }
 925
 926 static LJ_AINLINE void setgcV(lua_State *L, TValue *o, GCobj *v, uint32_t it)
 927 {
 928   setgcVraw(o, v, it);
 929   checklivetv(L, o, "store to dead GC object");
 930 }
 931
 932 #define define_setV(name, type, tag) \
 933 static LJ_AINLINE void name(lua_State *L, TValue *o, const type *v) \
 934 { \
 935   setgcV(L, o, obj2gco(v), tag); \
 936 }
 937 define_setV(setstrV, GCstr, LJ_TSTR)
 938 define_setV(setthreadV, lua_State, LJ_TTHREAD)
 939 define_setV(setprotoV, GCproto, LJ_TPROTO)
 940 define_setV(setfuncV, GCfunc, LJ_TFUNC)
 941 define_setV(setcdataV, GCcdata, LJ_TCDATA)
 942 define_setV(settabV, GCtab, LJ_TTAB)
 943 define_setV(setudataV, GCudata, LJ_TUDATA)
 944
 945 #define setnumV(o, x)           ((o)->n = (x))
 946 #define setnanV(o)              ((o)->u64 = U64x(fff80000,00000000))
 947 #define setpinfV(o)             ((o)->u64 = U64x(7ff00000,00000000))
 948 #define setminfV(o)             ((o)->u64 = U64x(fff00000,00000000))
 949
 950 static LJ_AINLINE void setintV(TValue *o, int32_t i)
 951 {
 952 #if LJ_DUALNUM
 953   o->i = (uint32_t)i; setitype(o, LJ_TISNUM);
 954 #else
 955   o->n = (lua_Number)i;
 956 #endif
 957 }
 958
 959 static LJ_AINLINE void setint64V(TValue *o, int64_t i)
 960 {
 961   if (LJ_DUALNUM && LJ_LIKELY(i == (int64_t)(int32_t)i))
 962     setintV(o, (int32_t)i);
 963   else
 964     setnumV(o, (lua_Number)i);
 965 }
 966
 967 #if LJ_64
 968 #define setintptrV(o, i)        setint64V((o), (i))
 969 #else
 970 #define setintptrV(o, i)        setintV((o), (i))
 971 #endif
 972
 973 /* Copy tagged values. */
 974 static LJ_AINLINE void copyTV(lua_State *L, TValue *o1, const TValue *o2)
 975 {
 976   *o1 = *o2;
 977   checklivetv(L, o1, "copy of dead GC object");
 978 }
 979
 980 /* -- Number to integer conversion ---------------------------------------- */
 981
 982 #if LJ_SOFTFP
 983 LJ_ASMF int32_t lj_vm_tobit(double x);
 984 #if LJ_TARGET_MIPS64
 985 LJ_ASMF int32_t lj_vm_tointg(double x);
 986 #endif
 987 #endif
 988
 989 static LJ_AINLINE int32_t lj_num2bit(lua_Number n)
 990 {
 991 #if LJ_SOFTFP
 992   return lj_vm_tobit(n);
 993 #else
 994   TValue o;
 995   o.n = n + 6755399441055744.0;  /* 2^52 + 2^51 */
 996   return (int32_t)o.u32.lo;
 997 #endif
 998 }
 999
1000 #define lj_num2int(n)   ((int32_t)(n))
1001
1002 /*
1003 ** This must match the JIT backend behavior. In particular for archs
1004 ** that don't have a common hardware instruction for this conversion.
1005 ** Note that signed FP to unsigned int conversions have an undefined
1006 ** result and should never be relied upon in portable FFI code.
1007 ** See also: C99 or C11 standard, 6.3.1.4, footnote of (1).
1008 */
1009 static LJ_AINLINE uint64_t lj_num2u64(lua_Number n)
1010 {
1011 #if LJ_TARGET_X86ORX64 || LJ_TARGET_MIPS
1012   int64_t i = (int64_t)n;
1013   if (i < 0) i = (int64_t)(n - 18446744073709551616.0);
1014   return (uint64_t)i;
1015 #else
1016   return (uint64_t)n;
1017 #endif
1018 }
1019
1020 static LJ_AINLINE int32_t numberVint(cTValue *o)
1021 {
1022   if (LJ_LIKELY(tvisint(o)))
1023     return intV(o);
1024   else
1025     return lj_num2int(numV(o));
1026 }
1027
1028 static LJ_AINLINE lua_Number numberVnum(cTValue *o)
1029 {
1030   if (LJ_UNLIKELY(tvisint(o)))
1031     return (lua_Number)intV(o);
1032   else
1033     return numV(o);
1034 }
1035
1036 /* -- Miscellaneous object handling --------------------------------------- */
1037
1038 /* Names and maps for internal and external object tags. */
1039 LJ_DATA const char *const lj_obj_typename[1+LUA_TCDATA+1];
1040 LJ_DATA const char *const lj_obj_itypename[~LJ_TNUMX+1];
1041
1042 #define lj_typename(o)  (lj_obj_itypename[itypemap(o)])
1043
1044 /* Compare two objects without calling metamethods. */
1045 LJ_FUNC int LJ_FASTCALL lj_obj_equal(cTValue *o1, cTValue *o2);
1046 LJ_FUNC const void * LJ_FASTCALL lj_obj_ptr(global_State *g, cTValue *o);
1047
1048 #if LJ_ABI_PAUTH
1049 #if LJ_TARGET_ARM64
1050 #include <ptrauth.h>
1051 #define lj_ptr_sign(ptr, ctx) \
1052   ptrauth_sign_unauthenticated((ptr), ptrauth_key_function_pointer, (ctx))
1053 #define lj_ptr_strip(ptr) ptrauth_strip((ptr), ptrauth_key_function_pointer)
1054 #else
1055 #error "No support for pointer authentication for this architecture"
1056 #endif
1057 #else
1058 #define lj_ptr_sign(ptr, ctx) (ptr)
1059 #define lj_ptr_strip(ptr) (ptr)
1060 #endif
1061
1062 #endif