src/backend/jit/llvm/llvmjit.c

   1 /*-------------------------------------------------------------------------
   2  *
   3  * llvmjit.c
   4  *        Core part of the LLVM JIT provider.
   5  *
   6  * Copyright (c) 2016-2025, PostgreSQL Global Development Group
   7  *
   8  * IDENTIFICATION
   9  *        src/backend/jit/llvm/llvmjit.c
  10  *
  11  *-------------------------------------------------------------------------
  12  */
  13
  14 #include "postgres.h"
  15
  16 #include <llvm-c/Analysis.h>
  17 #include <llvm-c/BitReader.h>
  18 #include <llvm-c/BitWriter.h>
  19 #include <llvm-c/Core.h>
  20 #include <llvm-c/ExecutionEngine.h>
  21 #if LLVM_VERSION_MAJOR > 16
  22 #include <llvm-c/Transforms/PassBuilder.h>
  23 #endif
  24 #include <llvm-c/Orc.h>
  25 #include <llvm-c/OrcEE.h>
  26 #include <llvm-c/LLJIT.h>
  27 #include <llvm-c/Support.h>
  28 #include <llvm-c/Target.h>
  29 #if LLVM_VERSION_MAJOR < 17
  30 #include <llvm-c/Transforms/IPO.h>
  31 #include <llvm-c/Transforms/PassManagerBuilder.h>
  32 #include <llvm-c/Transforms/Scalar.h>
  33 #include <llvm-c/Transforms/Utils.h>
  34 #endif
  35
  36 #include "jit/llvmjit.h"
  37 #include "jit/llvmjit_backport.h"
  38 #include "jit/llvmjit_emit.h"
  39 #include "miscadmin.h"
  40 #include "portability/instr_time.h"
  41 #include "storage/ipc.h"
  42 #include "utils/memutils.h"
  43 #include "utils/resowner.h"
  44
  45 #define LLVMJIT_LLVM_CONTEXT_REUSE_MAX 100
  46
  47 /* Handle of a module emitted via ORC JIT */
  48 typedef struct LLVMJitHandle
  49 {
  50         LLVMOrcLLJITRef lljit;
  51         LLVMOrcResourceTrackerRef resource_tracker;
  52 } LLVMJitHandle;
  53
  54
  55 /* types & functions commonly needed for JITing */
  56 LLVMTypeRef TypeSizeT;
  57 LLVMTypeRef TypeParamBool;
  58 LLVMTypeRef TypeStorageBool;
  59 LLVMTypeRef TypePGFunction;
  60 LLVMTypeRef StructNullableDatum;
  61 LLVMTypeRef StructHeapTupleData;
  62 LLVMTypeRef StructMinimalTupleData;
  63 LLVMTypeRef StructTupleDescData;
  64 LLVMTypeRef StructTupleTableSlot;
  65 LLVMTypeRef StructHeapTupleHeaderData;
  66 LLVMTypeRef StructHeapTupleTableSlot;
  67 LLVMTypeRef StructMinimalTupleTableSlot;
  68 LLVMTypeRef StructMemoryContextData;
  69 LLVMTypeRef StructFunctionCallInfoData;
  70 LLVMTypeRef StructExprContext;
  71 LLVMTypeRef StructExprEvalStep;
  72 LLVMTypeRef StructExprState;
  73 LLVMTypeRef StructAggState;
  74 LLVMTypeRef StructAggStatePerGroupData;
  75 LLVMTypeRef StructAggStatePerTransData;
  76 LLVMTypeRef StructPlanState;
  77
  78 LLVMValueRef AttributeTemplate;
  79 LLVMValueRef ExecEvalSubroutineTemplate;
  80 LLVMValueRef ExecEvalBoolSubroutineTemplate;
  81
  82 static LLVMModuleRef llvm_types_module = NULL;
  83
  84 static bool llvm_session_initialized = false;
  85 static size_t llvm_generation = 0;
  86
  87 /* number of LLVMJitContexts that currently are in use */
  88 static size_t llvm_jit_context_in_use_count = 0;
  89
  90 /* how many times has the current LLVMContextRef been used */
  91 static size_t llvm_llvm_context_reuse_count = 0;
  92 static const char *llvm_triple = NULL;
  93 static const char *llvm_layout = NULL;
  94 static LLVMContextRef llvm_context;
  95
  96
  97 static LLVMTargetRef llvm_targetref;
  98 static LLVMOrcThreadSafeContextRef llvm_ts_context;
  99 static LLVMOrcLLJITRef llvm_opt0_orc;
 100 static LLVMOrcLLJITRef llvm_opt3_orc;
 101
 102
 103 static void llvm_release_context(JitContext *context);
 104 static void llvm_session_initialize(void);
 105 static void llvm_shutdown(int code, Datum arg);
 106 static void llvm_compile_module(LLVMJitContext *context);
 107 static void llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module);
 108
 109 static void llvm_create_types(void);
 110 static void llvm_set_target(void);
 111 static void llvm_recreate_llvm_context(void);
 112 static uint64_t llvm_resolve_symbol(const char *name, void *ctx);
 113
 114 static LLVMOrcLLJITRef llvm_create_jit_instance(LLVMTargetMachineRef tm);
 115 static char *llvm_error_message(LLVMErrorRef error);
 116
 117 /* ResourceOwner callbacks to hold JitContexts  */
 118 static void ResOwnerReleaseJitContext(Datum res);
 119
 120 static const ResourceOwnerDesc jit_resowner_desc =
 121 {
 122         .name = "LLVM JIT context",
 123         .release_phase = RESOURCE_RELEASE_BEFORE_LOCKS,
 124         .release_priority = RELEASE_PRIO_JIT_CONTEXTS,
 125         .ReleaseResource = ResOwnerReleaseJitContext,
 126         .DebugPrint = NULL                      /* the default message is fine */
 127 };
 128
 129 /* Convenience wrappers over ResourceOwnerRemember/Forget */
 130 static inline void
 131 ResourceOwnerRememberJIT(ResourceOwner owner, LLVMJitContext *handle)
 132 {
 133         ResourceOwnerRemember(owner, PointerGetDatum(handle), &jit_resowner_desc);
 134 }
 135 static inline void
 136 ResourceOwnerForgetJIT(ResourceOwner owner, LLVMJitContext *handle)
 137 {
 138         ResourceOwnerForget(owner, PointerGetDatum(handle), &jit_resowner_desc);
 139 }
 140
 141 PG_MODULE_MAGIC;
 142
 143
 144 /*
 145  * Initialize LLVM JIT provider.
 146  */
 147 void
 148 _PG_jit_provider_init(JitProviderCallbacks *cb)
 149 {
 150         cb->reset_after_error = llvm_reset_after_error;
 151         cb->release_context = llvm_release_context;
 152         cb->compile_expr = llvm_compile_expr;
 153 }
 154
 155
 156 /*
 157  * Every now and then create a new LLVMContextRef. Unfortunately, during every
 158  * round of inlining, types may "leak" (they can still be found/used via the
 159  * context, but new types will be created the next time in inlining is
 160  * performed). To prevent that from slowly accumulating problematic amounts of
 161  * memory, recreate the LLVMContextRef we use. We don't want to do so too
 162  * often, as that implies some overhead (particularly re-loading the module
 163  * summaries / modules is fairly expensive). A future TODO would be to make
 164  * this more finegrained and only drop/recreate the LLVMContextRef when we know
 165  * there has been inlining. If we can get the size of the context from LLVM
 166  * then that might be a better way to determine when to drop/recreate rather
 167  * then the usagecount heuristic currently employed.
 168  */
 169 static void
 170 llvm_recreate_llvm_context(void)
 171 {
 172         if (!llvm_context)
 173                 elog(ERROR, "Trying to recreate a non-existing context");
 174
 175         /*
 176          * We can only safely recreate the LLVM context if no other code is being
 177          * JITed, otherwise we'd release the types in use for that.
 178          */
 179         if (llvm_jit_context_in_use_count > 0)
 180         {
 181                 llvm_llvm_context_reuse_count++;
 182                 return;
 183         }
 184
 185         if (llvm_llvm_context_reuse_count <= LLVMJIT_LLVM_CONTEXT_REUSE_MAX)
 186         {
 187                 llvm_llvm_context_reuse_count++;
 188                 return;
 189         }
 190
 191         /*
 192          * Need to reset the modules that the inlining code caches before
 193          * disposing of the context. LLVM modules exist within a specific LLVM
 194          * context, therefore disposing of the context before resetting the cache
 195          * would lead to dangling pointers to modules.
 196          */
 197         llvm_inline_reset_caches();
 198
 199         LLVMContextDispose(llvm_context);
 200         llvm_context = LLVMContextCreate();
 201         llvm_llvm_context_reuse_count = 0;
 202
 203         /*
 204          * Re-build cached type information, so code generation code can rely on
 205          * that information to be present (also prevents the variables to be
 206          * dangling references).
 207          */
 208         llvm_create_types();
 209 }
 210
 211
 212 /*
 213  * Create a context for JITing work.
 214  *
 215  * The context, including subsidiary resources, will be cleaned up either when
 216  * the context is explicitly released, or when the lifetime of
 217  * CurrentResourceOwner ends (usually the end of the current [sub]xact).
 218  */
 219 LLVMJitContext *
 220 llvm_create_context(int jitFlags)
 221 {
 222         LLVMJitContext *context;
 223
 224         llvm_assert_in_fatal_section();
 225
 226         llvm_session_initialize();
 227
 228         llvm_recreate_llvm_context();
 229
 230         ResourceOwnerEnlarge(CurrentResourceOwner);
 231
 232         context = MemoryContextAllocZero(TopMemoryContext,
 233                                                                          sizeof(LLVMJitContext));
 234         context->base.flags = jitFlags;
 235
 236         /* ensure cleanup */
 237         context->resowner = CurrentResourceOwner;
 238         ResourceOwnerRememberJIT(CurrentResourceOwner, context);
 239
 240         llvm_jit_context_in_use_count++;
 241
 242         return context;
 243 }
 244
 245 /*
 246  * Release resources required by one llvm context.
 247  */
 248 static void
 249 llvm_release_context(JitContext *context)
 250 {
 251         LLVMJitContext *llvm_jit_context = (LLVMJitContext *) context;
 252         ListCell   *lc;
 253
 254         /*
 255          * Consider as cleaned up even if we skip doing so below, that way we can
 256          * verify the tracking is correct (see llvm_shutdown()).
 257          */
 258         llvm_jit_context_in_use_count--;
 259
 260         /*
 261          * When this backend is exiting, don't clean up LLVM. As an error might
 262          * have occurred from within LLVM, we do not want to risk reentering. All
 263          * resource cleanup is going to happen through process exit.
 264          */
 265         if (proc_exit_inprogress)
 266                 return;
 267
 268         llvm_enter_fatal_on_oom();
 269
 270         if (llvm_jit_context->module)
 271         {
 272                 LLVMDisposeModule(llvm_jit_context->module);
 273                 llvm_jit_context->module = NULL;
 274         }
 275
 276         foreach(lc, llvm_jit_context->handles)
 277         {
 278                 LLVMJitHandle *jit_handle = (LLVMJitHandle *) lfirst(lc);
 279
 280                 {
 281                         LLVMOrcExecutionSessionRef ee;
 282                         LLVMOrcSymbolStringPoolRef sp;
 283
 284                         LLVMOrcResourceTrackerRemove(jit_handle->resource_tracker);
 285                         LLVMOrcReleaseResourceTracker(jit_handle->resource_tracker);
 286
 287                         /*
 288                          * Without triggering cleanup of the string pool, we'd leak
 289                          * memory. It'd be sufficient to do this far less often, but in
 290                          * experiments the required time was small enough to just always
 291                          * do it.
 292                          */
 293                         ee = LLVMOrcLLJITGetExecutionSession(jit_handle->lljit);
 294                         sp = LLVMOrcExecutionSessionGetSymbolStringPool(ee);
 295                         LLVMOrcSymbolStringPoolClearDeadEntries(sp);
 296                 }
 297
 298                 pfree(jit_handle);
 299         }
 300         list_free(llvm_jit_context->handles);
 301         llvm_jit_context->handles = NIL;
 302
 303         llvm_leave_fatal_on_oom();
 304
 305         if (llvm_jit_context->resowner)
 306                 ResourceOwnerForgetJIT(llvm_jit_context->resowner, llvm_jit_context);
 307 }
 308
 309 /*
 310  * Return module which may be modified, e.g. by creating new functions.
 311  */
 312 LLVMModuleRef
 313 llvm_mutable_module(LLVMJitContext *context)
 314 {
 315         llvm_assert_in_fatal_section();
 316
 317         /*
 318          * If there's no in-progress module, create a new one.
 319          */
 320         if (!context->module)
 321         {
 322                 context->compiled = false;
 323                 context->module_generation = llvm_generation++;
 324                 context->module = LLVMModuleCreateWithNameInContext("pg", llvm_context);
 325                 LLVMSetTarget(context->module, llvm_triple);
 326                 LLVMSetDataLayout(context->module, llvm_layout);
 327         }
 328
 329         return context->module;
 330 }
 331
 332 /*
 333  * Expand function name to be non-conflicting. This should be used by code
 334  * generating code, when adding new externally visible function definitions to
 335  * a Module.
 336  */
 337 char *
 338 llvm_expand_funcname(struct LLVMJitContext *context, const char *basename)
 339 {
 340         Assert(context->module != NULL);
 341
 342         context->base.instr.created_functions++;
 343
 344         /*
 345          * Previously we used dots to separate, but turns out some tools, e.g.
 346          * GDB, don't like that and truncate name.
 347          */
 348         return psprintf("%s_%zu_%d",
 349                                         basename,
 350                                         context->module_generation,
 351                                         context->counter++);
 352 }
 353
 354 /*
 355  * Return pointer to function funcname, which has to exist. If there's pending
 356  * code to be optimized and emitted, do so first.
 357  */
 358 void *
 359 llvm_get_function(LLVMJitContext *context, const char *funcname)
 360 {
 361         ListCell   *lc;
 362
 363         llvm_assert_in_fatal_section();
 364
 365         /*
 366          * If there is a pending / not emitted module, compile and emit now.
 367          * Otherwise we might not find the [correct] function.
 368          */
 369         if (!context->compiled)
 370         {
 371                 llvm_compile_module(context);
 372         }
 373
 374         /*
 375          * ORC's symbol table is of *unmangled* symbols. Therefore we don't need
 376          * to mangle here.
 377          */
 378
 379         foreach(lc, context->handles)
 380         {
 381                 LLVMJitHandle *handle = (LLVMJitHandle *) lfirst(lc);
 382                 instr_time      starttime;
 383                 instr_time      endtime;
 384                 LLVMErrorRef error;
 385                 LLVMOrcJITTargetAddress addr;
 386
 387                 INSTR_TIME_SET_CURRENT(starttime);
 388
 389                 addr = 0;
 390                 error = LLVMOrcLLJITLookup(handle->lljit, &addr, funcname);
 391                 if (error)
 392                         elog(ERROR, "failed to look up symbol \"%s\": %s",
 393                                  funcname, llvm_error_message(error));
 394
 395                 /*
 396                  * LLJIT only actually emits code the first time a symbol is
 397                  * referenced. Thus add lookup time to emission time. That's counting
 398                  * a bit more than with older LLVM versions, but unlikely to ever
 399                  * matter.
 400                  */
 401                 INSTR_TIME_SET_CURRENT(endtime);
 402                 INSTR_TIME_ACCUM_DIFF(context->base.instr.emission_counter,
 403                                                           endtime, starttime);
 404
 405                 if (addr)
 406                         return (void *) (uintptr_t) addr;
 407         }
 408
 409         elog(ERROR, "failed to JIT: %s", funcname);
 410
 411         return NULL;
 412 }
 413
 414 /*
 415  * Return type of a variable in llvmjit_types.c. This is useful to keep types
 416  * in sync between plain C and JIT related code.
 417  */
 418 LLVMTypeRef
 419 llvm_pg_var_type(const char *varname)
 420 {
 421         LLVMValueRef v_srcvar;
 422         LLVMTypeRef typ;
 423
 424         /* this'll return a *pointer* to the global */
 425         v_srcvar = LLVMGetNamedGlobal(llvm_types_module, varname);
 426         if (!v_srcvar)
 427                 elog(ERROR, "variable %s not in llvmjit_types.c", varname);
 428
 429         typ = LLVMGlobalGetValueType(v_srcvar);
 430
 431         return typ;
 432 }
 433
 434 /*
 435  * Return function type of a variable in llvmjit_types.c. This is useful to
 436  * keep function types in sync between C and JITed code.
 437  */
 438 LLVMTypeRef
 439 llvm_pg_var_func_type(const char *varname)
 440 {
 441         LLVMValueRef v_srcvar;
 442         LLVMTypeRef typ;
 443
 444         v_srcvar = LLVMGetNamedFunction(llvm_types_module, varname);
 445         if (!v_srcvar)
 446                 elog(ERROR, "function %s not in llvmjit_types.c", varname);
 447
 448         typ = LLVMGetFunctionType(v_srcvar);
 449
 450         return typ;
 451 }
 452
 453 /*
 454  * Return declaration for a function referenced in llvmjit_types.c, adding it
 455  * to the module if necessary.
 456  *
 457  * This is used to make functions discovered via llvm_create_types() known to
 458  * the module that's currently being worked on.
 459  */
 460 LLVMValueRef
 461 llvm_pg_func(LLVMModuleRef mod, const char *funcname)
 462 {
 463         LLVMValueRef v_srcfn;
 464         LLVMValueRef v_fn;
 465
 466         /* don't repeatedly add function */
 467         v_fn = LLVMGetNamedFunction(mod, funcname);
 468         if (v_fn)
 469                 return v_fn;
 470
 471         v_srcfn = LLVMGetNamedFunction(llvm_types_module, funcname);
 472
 473         if (!v_srcfn)
 474                 elog(ERROR, "function %s not in llvmjit_types.c", funcname);
 475
 476         v_fn = LLVMAddFunction(mod,
 477                                                    funcname,
 478                                                    LLVMGetFunctionType(v_srcfn));
 479         llvm_copy_attributes(v_srcfn, v_fn);
 480
 481         return v_fn;
 482 }
 483
 484 /*
 485  * Copy attributes from one function to another, for a specific index (an
 486  * index can reference return value, function and parameter attributes).
 487  */
 488 static void
 489 llvm_copy_attributes_at_index(LLVMValueRef v_from, LLVMValueRef v_to, uint32 index)
 490 {
 491         int                     num_attributes;
 492         LLVMAttributeRef *attrs;
 493
 494         num_attributes = LLVMGetAttributeCountAtIndex(v_from, index);
 495
 496         if (num_attributes == 0)
 497                 return;
 498
 499         attrs = palloc(sizeof(LLVMAttributeRef) * num_attributes);
 500         LLVMGetAttributesAtIndex(v_from, index, attrs);
 501
 502         for (int attno = 0; attno < num_attributes; attno++)
 503                 LLVMAddAttributeAtIndex(v_to, index, attrs[attno]);
 504
 505         pfree(attrs);
 506 }
 507
 508 /*
 509  * Copy all attributes from one function to another. I.e. function, return and
 510  * parameters will be copied.
 511  */
 512 void
 513 llvm_copy_attributes(LLVMValueRef v_from, LLVMValueRef v_to)
 514 {
 515         uint32          param_count;
 516
 517         /* copy function attributes */
 518         llvm_copy_attributes_at_index(v_from, v_to, LLVMAttributeFunctionIndex);
 519
 520         if (LLVMGetTypeKind(LLVMGetFunctionReturnType(v_to)) != LLVMVoidTypeKind)
 521         {
 522                 /* and the return value attributes */
 523                 llvm_copy_attributes_at_index(v_from, v_to, LLVMAttributeReturnIndex);
 524         }
 525
 526         /* and each function parameter's attribute */
 527         param_count = LLVMCountParams(v_from);
 528
 529         for (int paramidx = 1; paramidx <= param_count; paramidx++)
 530                 llvm_copy_attributes_at_index(v_from, v_to, paramidx);
 531 }
 532
 533 /*
 534  * Return a callable LLVMValueRef for fcinfo.
 535  */
 536 LLVMValueRef
 537 llvm_function_reference(LLVMJitContext *context,
 538                                                 LLVMBuilderRef builder,
 539                                                 LLVMModuleRef mod,
 540                                                 FunctionCallInfo fcinfo)
 541 {
 542         char       *modname;
 543         char       *basename;
 544         char       *funcname;
 545
 546         LLVMValueRef v_fn;
 547
 548         fmgr_symbol(fcinfo->flinfo->fn_oid, &modname, &basename);
 549
 550         if (modname != NULL && basename != NULL)
 551         {
 552                 /* external function in loadable library */
 553                 funcname = psprintf("pgextern.%s.%s", modname, basename);
 554         }
 555         else if (basename != NULL)
 556         {
 557                 /* internal function */
 558                 funcname = pstrdup(basename);
 559         }
 560         else
 561         {
 562                 /*
 563                  * Function we don't know to handle, return pointer. We do so by
 564                  * creating a global constant containing a pointer to the function.
 565                  * Makes IR more readable.
 566                  */
 567                 LLVMValueRef v_fn_addr;
 568
 569                 funcname = psprintf("pgoidextern.%u",
 570                                                         fcinfo->flinfo->fn_oid);
 571                 v_fn = LLVMGetNamedGlobal(mod, funcname);
 572                 if (v_fn != 0)
 573                         return l_load(builder, TypePGFunction, v_fn, "");
 574
 575                 v_fn_addr = l_ptr_const(fcinfo->flinfo->fn_addr, TypePGFunction);
 576
 577                 v_fn = LLVMAddGlobal(mod, TypePGFunction, funcname);
 578                 LLVMSetInitializer(v_fn, v_fn_addr);
 579                 LLVMSetGlobalConstant(v_fn, true);
 580                 LLVMSetLinkage(v_fn, LLVMPrivateLinkage);
 581                 LLVMSetUnnamedAddr(v_fn, true);
 582
 583                 return l_load(builder, TypePGFunction, v_fn, "");
 584         }
 585
 586         /* check if function already has been added */
 587         v_fn = LLVMGetNamedFunction(mod, funcname);
 588         if (v_fn != 0)
 589                 return v_fn;
 590
 591         v_fn = LLVMAddFunction(mod, funcname, LLVMGetFunctionType(AttributeTemplate));
 592
 593         return v_fn;
 594 }
 595
 596 /*
 597  * Optimize code in module using the flags set in context.
 598  */
 599 static void
 600 llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module)
 601 {
 602 #if LLVM_VERSION_MAJOR < 17
 603         LLVMPassManagerBuilderRef llvm_pmb;
 604         LLVMPassManagerRef llvm_mpm;
 605         LLVMPassManagerRef llvm_fpm;
 606         LLVMValueRef func;
 607         int                     compile_optlevel;
 608
 609         if (context->base.flags & PGJIT_OPT3)
 610                 compile_optlevel = 3;
 611         else
 612                 compile_optlevel = 0;
 613
 614         /*
 615          * Have to create a new pass manager builder every pass through, as the
 616          * inliner has some per-builder state. Otherwise one ends up only inlining
 617          * a function the first time though.
 618          */
 619         llvm_pmb = LLVMPassManagerBuilderCreate();
 620         LLVMPassManagerBuilderSetOptLevel(llvm_pmb, compile_optlevel);
 621         llvm_fpm = LLVMCreateFunctionPassManagerForModule(module);
 622
 623         if (context->base.flags & PGJIT_OPT3)
 624         {
 625                 /* TODO: Unscientifically determined threshold */
 626                 LLVMPassManagerBuilderUseInlinerWithThreshold(llvm_pmb, 512);
 627         }
 628         else
 629         {
 630                 /* we rely on mem2reg heavily, so emit even in the O0 case */
 631                 LLVMAddPromoteMemoryToRegisterPass(llvm_fpm);
 632         }
 633
 634         LLVMPassManagerBuilderPopulateFunctionPassManager(llvm_pmb, llvm_fpm);
 635
 636         /*
 637          * Do function level optimization. This could be moved to the point where
 638          * functions are emitted, to reduce memory usage a bit.
 639          */
 640         LLVMInitializeFunctionPassManager(llvm_fpm);
 641         for (func = LLVMGetFirstFunction(context->module);
 642                  func != NULL;
 643                  func = LLVMGetNextFunction(func))
 644                 LLVMRunFunctionPassManager(llvm_fpm, func);
 645         LLVMFinalizeFunctionPassManager(llvm_fpm);
 646         LLVMDisposePassManager(llvm_fpm);
 647
 648         /*
 649          * Perform module level optimization. We do so even in the non-optimized
 650          * case, so always-inline functions etc get inlined. It's cheap enough.
 651          */
 652         llvm_mpm = LLVMCreatePassManager();
 653         LLVMPassManagerBuilderPopulateModulePassManager(llvm_pmb,
 654                                                                                                         llvm_mpm);
 655         /* always use always-inliner pass */
 656         if (!(context->base.flags & PGJIT_OPT3))
 657                 LLVMAddAlwaysInlinerPass(llvm_mpm);
 658         /* if doing inlining, but no expensive optimization, add inlining pass */
 659         if (context->base.flags & PGJIT_INLINE
 660                 && !(context->base.flags & PGJIT_OPT3))
 661                 LLVMAddFunctionInliningPass(llvm_mpm);
 662         LLVMRunPassManager(llvm_mpm, context->module);
 663         LLVMDisposePassManager(llvm_mpm);
 664
 665         LLVMPassManagerBuilderDispose(llvm_pmb);
 666 #else
 667         LLVMPassBuilderOptionsRef options;
 668         LLVMErrorRef err;
 669         const char *passes;
 670
 671         if (context->base.flags & PGJIT_OPT3)
 672                 passes = "default<O3>";
 673         else
 674                 passes = "default<O0>,mem2reg";
 675
 676         options = LLVMCreatePassBuilderOptions();
 677
 678 #ifdef LLVM_PASS_DEBUG
 679         LLVMPassBuilderOptionsSetDebugLogging(options, 1);
 680 #endif
 681
 682         /* In assertion builds, run the LLVM verify pass. */
 683 #ifdef USE_ASSERT_CHECKING
 684         LLVMPassBuilderOptionsSetVerifyEach(options, true);
 685 #endif
 686
 687         LLVMPassBuilderOptionsSetInlinerThreshold(options, 512);
 688
 689         err = LLVMRunPasses(module, passes, NULL, options);
 690
 691         if (err)
 692                 elog(ERROR, "failed to JIT module: %s", llvm_error_message(err));
 693
 694         LLVMDisposePassBuilderOptions(options);
 695 #endif
 696 }
 697
 698 /*
 699  * Emit code for the currently pending module.
 700  */
 701 static void
 702 llvm_compile_module(LLVMJitContext *context)
 703 {
 704         LLVMJitHandle *handle;
 705         MemoryContext oldcontext;
 706         instr_time      starttime;
 707         instr_time      endtime;
 708         LLVMOrcLLJITRef compile_orc;
 709
 710         if (context->base.flags & PGJIT_OPT3)
 711                 compile_orc = llvm_opt3_orc;
 712         else
 713                 compile_orc = llvm_opt0_orc;
 714
 715         /* perform inlining */
 716         if (context->base.flags & PGJIT_INLINE)
 717         {
 718                 INSTR_TIME_SET_CURRENT(starttime);
 719                 llvm_inline(context->module);
 720                 INSTR_TIME_SET_CURRENT(endtime);
 721                 INSTR_TIME_ACCUM_DIFF(context->base.instr.inlining_counter,
 722                                                           endtime, starttime);
 723         }
 724
 725         if (jit_dump_bitcode)
 726         {
 727                 char       *filename;
 728
 729                 filename = psprintf("%d.%zu.bc",
 730                                                         MyProcPid,
 731                                                         context->module_generation);
 732                 LLVMWriteBitcodeToFile(context->module, filename);
 733                 pfree(filename);
 734         }
 735
 736
 737         /* optimize according to the chosen optimization settings */
 738         INSTR_TIME_SET_CURRENT(starttime);
 739         llvm_optimize_module(context, context->module);
 740         INSTR_TIME_SET_CURRENT(endtime);
 741         INSTR_TIME_ACCUM_DIFF(context->base.instr.optimization_counter,
 742                                                   endtime, starttime);
 743
 744         if (jit_dump_bitcode)
 745         {
 746                 char       *filename;
 747
 748                 filename = psprintf("%d.%zu.optimized.bc",
 749                                                         MyProcPid,
 750                                                         context->module_generation);
 751                 LLVMWriteBitcodeToFile(context->module, filename);
 752                 pfree(filename);
 753         }
 754
 755         handle = (LLVMJitHandle *)
 756                 MemoryContextAlloc(TopMemoryContext, sizeof(LLVMJitHandle));
 757
 758         /*
 759          * Emit the code. Note that this can, depending on the optimization
 760          * settings, take noticeable resources as code emission executes low-level
 761          * instruction combining/selection passes etc. Without optimization a
 762          * faster instruction selection mechanism is used.
 763          */
 764         INSTR_TIME_SET_CURRENT(starttime);
 765         {
 766                 LLVMOrcThreadSafeModuleRef ts_module;
 767                 LLVMErrorRef error;
 768                 LLVMOrcJITDylibRef jd = LLVMOrcLLJITGetMainJITDylib(compile_orc);
 769
 770                 ts_module = LLVMOrcCreateNewThreadSafeModule(context->module, llvm_ts_context);
 771
 772                 handle->lljit = compile_orc;
 773                 handle->resource_tracker = LLVMOrcJITDylibCreateResourceTracker(jd);
 774
 775                 /*
 776                  * NB: This doesn't actually emit code. That happens lazily the first
 777                  * time a symbol defined in the module is requested. Due to that
 778                  * llvm_get_function() also accounts for emission time.
 779                  */
 780
 781                 context->module = NULL; /* will be owned by LLJIT */
 782                 error = LLVMOrcLLJITAddLLVMIRModuleWithRT(compile_orc,
 783                                                                                                   handle->resource_tracker,
 784                                                                                                   ts_module);
 785
 786                 if (error)
 787                         elog(ERROR, "failed to JIT module: %s",
 788                                  llvm_error_message(error));
 789
 790                 /* LLVMOrcLLJITAddLLVMIRModuleWithRT takes ownership of the module */
 791         }
 792
 793         INSTR_TIME_SET_CURRENT(endtime);
 794         INSTR_TIME_ACCUM_DIFF(context->base.instr.emission_counter,
 795                                                   endtime, starttime);
 796
 797         context->module = NULL;
 798         context->compiled = true;
 799
 800         /* remember emitted code for cleanup and lookups */
 801         oldcontext = MemoryContextSwitchTo(TopMemoryContext);
 802         context->handles = lappend(context->handles, handle);
 803         MemoryContextSwitchTo(oldcontext);
 804
 805         ereport(DEBUG1,
 806                         (errmsg_internal("time to inline: %.3fs, opt: %.3fs, emit: %.3fs",
 807                                                          INSTR_TIME_GET_DOUBLE(context->base.instr.inlining_counter),
 808                                                          INSTR_TIME_GET_DOUBLE(context->base.instr.optimization_counter),
 809                                                          INSTR_TIME_GET_DOUBLE(context->base.instr.emission_counter)),
 810                          errhidestmt(true),
 811                          errhidecontext(true)));
 812 }
 813
 814 /*
 815  * Per session initialization.
 816  */
 817 static void
 818 llvm_session_initialize(void)
 819 {
 820         MemoryContext oldcontext;
 821         char       *error = NULL;
 822         char       *cpu = NULL;
 823         char       *features = NULL;
 824         LLVMTargetMachineRef opt0_tm;
 825         LLVMTargetMachineRef opt3_tm;
 826
 827         if (llvm_session_initialized)
 828                 return;
 829
 830         oldcontext = MemoryContextSwitchTo(TopMemoryContext);
 831
 832         LLVMInitializeNativeTarget();
 833         LLVMInitializeNativeAsmPrinter();
 834         LLVMInitializeNativeAsmParser();
 835
 836         if (llvm_context == NULL)
 837         {
 838                 llvm_context = LLVMContextCreate();
 839
 840                 llvm_jit_context_in_use_count = 0;
 841                 llvm_llvm_context_reuse_count = 0;
 842         }
 843
 844         /*
 845          * Synchronize types early, as that also includes inferring the target
 846          * triple.
 847          */
 848         llvm_create_types();
 849
 850         /*
 851          * Extract target information from loaded module.
 852          */
 853         llvm_set_target();
 854
 855         if (LLVMGetTargetFromTriple(llvm_triple, &llvm_targetref, &error) != 0)
 856         {
 857                 elog(FATAL, "failed to query triple %s", error);
 858         }
 859
 860         /*
 861          * We want the generated code to use all available features. Therefore
 862          * grab the host CPU string and detect features of the current CPU. The
 863          * latter is needed because some CPU architectures default to enabling
 864          * features not all CPUs have (weird, huh).
 865          */
 866         cpu = LLVMGetHostCPUName();
 867         features = LLVMGetHostCPUFeatures();
 868         elog(DEBUG2, "LLVMJIT detected CPU \"%s\", with features \"%s\"",
 869                  cpu, features);
 870
 871         opt0_tm =
 872                 LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, features,
 873                                                                 LLVMCodeGenLevelNone,
 874                                                                 LLVMRelocDefault,
 875                                                                 LLVMCodeModelJITDefault);
 876         opt3_tm =
 877                 LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, features,
 878                                                                 LLVMCodeGenLevelAggressive,
 879                                                                 LLVMRelocDefault,
 880                                                                 LLVMCodeModelJITDefault);
 881
 882         LLVMDisposeMessage(cpu);
 883         cpu = NULL;
 884         LLVMDisposeMessage(features);
 885         features = NULL;
 886
 887         /* force symbols in main binary to be loaded */
 888         LLVMLoadLibraryPermanently(NULL);
 889
 890         {
 891                 llvm_ts_context = LLVMOrcCreateNewThreadSafeContext();
 892
 893                 llvm_opt0_orc = llvm_create_jit_instance(opt0_tm);
 894                 opt0_tm = 0;
 895
 896                 llvm_opt3_orc = llvm_create_jit_instance(opt3_tm);
 897                 opt3_tm = 0;
 898         }
 899
 900         on_proc_exit(llvm_shutdown, 0);
 901
 902         llvm_session_initialized = true;
 903
 904         MemoryContextSwitchTo(oldcontext);
 905 }
 906
 907 static void
 908 llvm_shutdown(int code, Datum arg)
 909 {
 910         /*
 911          * If llvm_shutdown() is reached while in a fatal-on-oom section an error
 912          * has occurred in the middle of LLVM code. It is not safe to call back
 913          * into LLVM (which is why a FATAL error was thrown).
 914          *
 915          * We do need to shutdown LLVM in other shutdown cases, otherwise e.g.
 916          * profiling data won't be written out.
 917          */
 918         if (llvm_in_fatal_on_oom())
 919         {
 920                 Assert(proc_exit_inprogress);
 921                 return;
 922         }
 923
 924         if (llvm_jit_context_in_use_count != 0)
 925                 elog(PANIC, "LLVMJitContext in use count not 0 at exit (is %zu)",
 926                          llvm_jit_context_in_use_count);
 927
 928         {
 929                 if (llvm_opt3_orc)
 930                 {
 931                         LLVMOrcDisposeLLJIT(llvm_opt3_orc);
 932                         llvm_opt3_orc = NULL;
 933                 }
 934                 if (llvm_opt0_orc)
 935                 {
 936                         LLVMOrcDisposeLLJIT(llvm_opt0_orc);
 937                         llvm_opt0_orc = NULL;
 938                 }
 939                 if (llvm_ts_context)
 940                 {
 941                         LLVMOrcDisposeThreadSafeContext(llvm_ts_context);
 942                         llvm_ts_context = NULL;
 943                 }
 944         }
 945 }
 946
 947 /* helper for llvm_create_types, returning a function's return type */
 948 static LLVMTypeRef
 949 load_return_type(LLVMModuleRef mod, const char *name)
 950 {
 951         LLVMValueRef value;
 952         LLVMTypeRef typ;
 953
 954         /* this'll return a *pointer* to the function */
 955         value = LLVMGetNamedFunction(mod, name);
 956         if (!value)
 957                 elog(ERROR, "function %s is unknown", name);
 958
 959         typ = LLVMGetFunctionReturnType(value); /* in llvmjit_wrap.cpp */
 960
 961         return typ;
 962 }
 963
 964 /*
 965  * Load triple & layout from clang emitted file so we're guaranteed to be
 966  * compatible.
 967  */
 968 static void
 969 llvm_set_target(void)
 970 {
 971         if (!llvm_types_module)
 972                 elog(ERROR, "failed to extract target information, llvmjit_types.c not loaded");
 973
 974         if (llvm_triple == NULL)
 975                 llvm_triple = pstrdup(LLVMGetTarget(llvm_types_module));
 976
 977         if (llvm_layout == NULL)
 978                 llvm_layout = pstrdup(LLVMGetDataLayoutStr(llvm_types_module));
 979 }
 980
 981 /*
 982  * Load required information, types, function signatures from llvmjit_types.c
 983  * and make them available in global variables.
 984  *
 985  * Those global variables are then used while emitting code.
 986  */
 987 static void
 988 llvm_create_types(void)
 989 {
 990         char            path[MAXPGPATH];
 991         LLVMMemoryBufferRef buf;
 992         char       *msg;
 993
 994         snprintf(path, MAXPGPATH, "%s/%s", pkglib_path, "llvmjit_types.bc");
 995
 996         /* open file */
 997         if (LLVMCreateMemoryBufferWithContentsOfFile(path, &buf, &msg))
 998         {
 999                 elog(ERROR, "LLVMCreateMemoryBufferWithContentsOfFile(%s) failed: %s",
1000                          path, msg);
1001         }
1002
1003         /* eagerly load contents, going to need it all */
1004         if (LLVMParseBitcodeInContext2(llvm_context, buf, &llvm_types_module))
1005         {
1006                 elog(ERROR, "LLVMParseBitcodeInContext2 of %s failed", path);
1007         }
1008         LLVMDisposeMemoryBuffer(buf);
1009
1010         TypeSizeT = llvm_pg_var_type("TypeSizeT");
1011         TypeParamBool = load_return_type(llvm_types_module, "FunctionReturningBool");
1012         TypeStorageBool = llvm_pg_var_type("TypeStorageBool");
1013         TypePGFunction = llvm_pg_var_type("TypePGFunction");
1014         StructNullableDatum = llvm_pg_var_type("StructNullableDatum");
1015         StructExprContext = llvm_pg_var_type("StructExprContext");
1016         StructExprEvalStep = llvm_pg_var_type("StructExprEvalStep");
1017         StructExprState = llvm_pg_var_type("StructExprState");
1018         StructFunctionCallInfoData = llvm_pg_var_type("StructFunctionCallInfoData");
1019         StructMemoryContextData = llvm_pg_var_type("StructMemoryContextData");
1020         StructTupleTableSlot = llvm_pg_var_type("StructTupleTableSlot");
1021         StructHeapTupleTableSlot = llvm_pg_var_type("StructHeapTupleTableSlot");
1022         StructMinimalTupleTableSlot = llvm_pg_var_type("StructMinimalTupleTableSlot");
1023         StructHeapTupleData = llvm_pg_var_type("StructHeapTupleData");
1024         StructHeapTupleHeaderData = llvm_pg_var_type("StructHeapTupleHeaderData");
1025         StructTupleDescData = llvm_pg_var_type("StructTupleDescData");
1026         StructAggState = llvm_pg_var_type("StructAggState");
1027         StructAggStatePerGroupData = llvm_pg_var_type("StructAggStatePerGroupData");
1028         StructAggStatePerTransData = llvm_pg_var_type("StructAggStatePerTransData");
1029         StructPlanState = llvm_pg_var_type("StructPlanState");
1030         StructMinimalTupleData = llvm_pg_var_type("StructMinimalTupleData");
1031
1032         AttributeTemplate = LLVMGetNamedFunction(llvm_types_module, "AttributeTemplate");
1033         ExecEvalSubroutineTemplate = LLVMGetNamedFunction(llvm_types_module, "ExecEvalSubroutineTemplate");
1034         ExecEvalBoolSubroutineTemplate = LLVMGetNamedFunction(llvm_types_module, "ExecEvalBoolSubroutineTemplate");
1035 }
1036
1037 /*
1038  * Split a symbol into module / function parts.  If the function is in the
1039  * main binary (or an external library) *modname will be NULL.
1040  */
1041 void
1042 llvm_split_symbol_name(const char *name, char **modname, char **funcname)
1043 {
1044         *modname = NULL;
1045         *funcname = NULL;
1046
1047         /*
1048          * Module function names are pgextern.$module.$funcname
1049          */
1050         if (strncmp(name, "pgextern.", strlen("pgextern.")) == 0)
1051         {
1052                 /*
1053                  * Symbol names cannot contain a ., therefore we can split based on
1054                  * first and last occurrence of one.
1055                  */
1056                 *funcname = rindex(name, '.');
1057                 (*funcname)++;                  /* jump over . */
1058
1059                 *modname = pnstrdup(name + strlen("pgextern."),
1060                                                         *funcname - name - strlen("pgextern.") - 1);
1061                 Assert(funcname);
1062
1063                 *funcname = pstrdup(*funcname);
1064         }
1065         else
1066         {
1067                 *modname = NULL;
1068                 *funcname = pstrdup(name);
1069         }
1070 }
1071
1072 /*
1073  * Attempt to resolve symbol, so LLVM can emit a reference to it.
1074  */
1075 static uint64_t
1076 llvm_resolve_symbol(const char *symname, void *ctx)
1077 {
1078         uintptr_t       addr;
1079         char       *funcname;
1080         char       *modname;
1081
1082         /*
1083          * macOS prefixes all object level symbols with an underscore. But neither
1084          * dlsym() nor PG's inliner expect that. So undo.
1085          */
1086 #if defined(__darwin__)
1087         if (symname[0] != '_')
1088                 elog(ERROR, "expected prefixed symbol name, but got \"%s\"", symname);
1089         symname++;
1090 #endif
1091
1092         llvm_split_symbol_name(symname, &modname, &funcname);
1093
1094         /* functions that aren't resolved to names shouldn't ever get here */
1095         Assert(funcname);
1096
1097         if (modname)
1098                 addr = (uintptr_t) load_external_function(modname, funcname,
1099                                                                                                   true, NULL);
1100         else
1101                 addr = (uintptr_t) LLVMSearchForAddressOfSymbol(symname);
1102
1103         pfree(funcname);
1104         if (modname)
1105                 pfree(modname);
1106
1107         /* let LLVM will error out - should never happen */
1108         if (!addr)
1109                 elog(WARNING, "failed to resolve name %s", symname);
1110
1111         return (uint64_t) addr;
1112 }
1113
1114 static LLVMErrorRef
1115 llvm_resolve_symbols(LLVMOrcDefinitionGeneratorRef GeneratorObj, void *Ctx,
1116                                          LLVMOrcLookupStateRef *LookupState, LLVMOrcLookupKind Kind,
1117                                          LLVMOrcJITDylibRef JD, LLVMOrcJITDylibLookupFlags JDLookupFlags,
1118                                          LLVMOrcCLookupSet LookupSet, size_t LookupSetSize)
1119 {
1120 #if LLVM_VERSION_MAJOR > 14
1121         LLVMOrcCSymbolMapPairs symbols = palloc0(sizeof(LLVMOrcCSymbolMapPair) * LookupSetSize);
1122 #else
1123         LLVMOrcCSymbolMapPairs symbols = palloc0(sizeof(LLVMJITCSymbolMapPair) * LookupSetSize);
1124 #endif
1125         LLVMErrorRef error;
1126         LLVMOrcMaterializationUnitRef mu;
1127
1128         for (int i = 0; i < LookupSetSize; i++)
1129         {
1130                 const char *name = LLVMOrcSymbolStringPoolEntryStr(LookupSet[i].Name);
1131
1132                 LLVMOrcRetainSymbolStringPoolEntry(LookupSet[i].Name);
1133                 symbols[i].Name = LookupSet[i].Name;
1134                 symbols[i].Sym.Address = llvm_resolve_symbol(name, NULL);
1135                 symbols[i].Sym.Flags.GenericFlags = LLVMJITSymbolGenericFlagsExported;
1136         }
1137
1138         mu = LLVMOrcAbsoluteSymbols(symbols, LookupSetSize);
1139         error = LLVMOrcJITDylibDefine(JD, mu);
1140         if (error != LLVMErrorSuccess)
1141                 LLVMOrcDisposeMaterializationUnit(mu);
1142
1143         pfree(symbols);
1144
1145         return error;
1146 }
1147
1148 /*
1149  * We cannot throw errors through LLVM (without causing a FATAL at least), so
1150  * just use WARNING here. That's OK anyway, as the error is also reported at
1151  * the top level action (with less detail) and there might be multiple
1152  * invocations of errors with details.
1153  *
1154  * This doesn't really happen during normal operation, but in cases like
1155  * symbol resolution breakage. So just using elog(WARNING) is fine.
1156  */
1157 static void
1158 llvm_log_jit_error(void *ctx, LLVMErrorRef error)
1159 {
1160         elog(WARNING, "error during JITing: %s",
1161                  llvm_error_message(error));
1162 }
1163
1164 /*
1165  * Create our own object layer, so we can add event listeners.
1166  */
1167 static LLVMOrcObjectLayerRef
1168 llvm_create_object_layer(void *Ctx, LLVMOrcExecutionSessionRef ES, const char *Triple)
1169 {
1170 #ifdef USE_LLVM_BACKPORT_SECTION_MEMORY_MANAGER
1171         LLVMOrcObjectLayerRef objlayer =
1172                 LLVMOrcCreateRTDyldObjectLinkingLayerWithSafeSectionMemoryManager(ES);
1173 #else
1174         LLVMOrcObjectLayerRef objlayer =
1175                 LLVMOrcCreateRTDyldObjectLinkingLayerWithSectionMemoryManager(ES);
1176 #endif
1177
1178
1179 #if defined(HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER) && HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER
1180         if (jit_debugging_support)
1181         {
1182                 LLVMJITEventListenerRef l = LLVMCreateGDBRegistrationListener();
1183
1184                 LLVMOrcRTDyldObjectLinkingLayerRegisterJITEventListener(objlayer, l);
1185         }
1186 #endif
1187
1188 #if defined(HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER) && HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER
1189         if (jit_profiling_support)
1190         {
1191                 LLVMJITEventListenerRef l = LLVMCreatePerfJITEventListener();
1192
1193                 LLVMOrcRTDyldObjectLinkingLayerRegisterJITEventListener(objlayer, l);
1194         }
1195 #endif
1196
1197         return objlayer;
1198 }
1199
1200 /*
1201  * Create LLJIT instance, using the passed in target machine. Note that the
1202  * target machine afterwards is owned by the LLJIT instance.
1203  */
1204 static LLVMOrcLLJITRef
1205 llvm_create_jit_instance(LLVMTargetMachineRef tm)
1206 {
1207         LLVMOrcLLJITRef lljit;
1208         LLVMOrcJITTargetMachineBuilderRef tm_builder;
1209         LLVMOrcLLJITBuilderRef lljit_builder;
1210         LLVMErrorRef error;
1211         LLVMOrcDefinitionGeneratorRef main_gen;
1212         LLVMOrcDefinitionGeneratorRef ref_gen;
1213
1214         lljit_builder = LLVMOrcCreateLLJITBuilder();
1215         tm_builder = LLVMOrcJITTargetMachineBuilderCreateFromTargetMachine(tm);
1216         LLVMOrcLLJITBuilderSetJITTargetMachineBuilder(lljit_builder, tm_builder);
1217
1218         LLVMOrcLLJITBuilderSetObjectLinkingLayerCreator(lljit_builder,
1219                                                                                                         llvm_create_object_layer,
1220                                                                                                         NULL);
1221
1222         error = LLVMOrcCreateLLJIT(&lljit, lljit_builder);
1223         if (error)
1224                 elog(ERROR, "failed to create lljit instance: %s",
1225                          llvm_error_message(error));
1226
1227         LLVMOrcExecutionSessionSetErrorReporter(LLVMOrcLLJITGetExecutionSession(lljit),
1228                                                                                         llvm_log_jit_error, NULL);
1229
1230         /*
1231          * Symbol resolution support for symbols in the postgres binary /
1232          * libraries already loaded.
1233          */
1234         error = LLVMOrcCreateDynamicLibrarySearchGeneratorForProcess(&main_gen,
1235                                                                                                                                  LLVMOrcLLJITGetGlobalPrefix(lljit),
1236                                                                                                                                  0, NULL);
1237         if (error)
1238                 elog(ERROR, "failed to create generator: %s",
1239                          llvm_error_message(error));
1240         LLVMOrcJITDylibAddGenerator(LLVMOrcLLJITGetMainJITDylib(lljit), main_gen);
1241
1242         /*
1243          * Symbol resolution support for "special" functions, e.g. a call into an
1244          * SQL callable function.
1245          */
1246 #if LLVM_VERSION_MAJOR > 14
1247         ref_gen = LLVMOrcCreateCustomCAPIDefinitionGenerator(llvm_resolve_symbols, NULL, NULL);
1248 #else
1249         ref_gen = LLVMOrcCreateCustomCAPIDefinitionGenerator(llvm_resolve_symbols, NULL);
1250 #endif
1251         LLVMOrcJITDylibAddGenerator(LLVMOrcLLJITGetMainJITDylib(lljit), ref_gen);
1252
1253         return lljit;
1254 }
1255
1256 static char *
1257 llvm_error_message(LLVMErrorRef error)
1258 {
1259         char       *orig = LLVMGetErrorMessage(error);
1260         char       *msg = pstrdup(orig);
1261
1262         LLVMDisposeErrorMessage(orig);
1263
1264         return msg;
1265 }
1266
1267 /*
1268  * ResourceOwner callbacks
1269  */
1270 static void
1271 ResOwnerReleaseJitContext(Datum res)
1272 {
1273         LLVMJitContext *context = (LLVMJitContext *) DatumGetPointer(res);
1274
1275         context->resowner = NULL;
1276         jit_release_context(&context->base);
1277 }