llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp

   1 //===---- IndirectionUtils.cpp - Utilities for call indirection in Orc ----===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "llvm/ExecutionEngine/Orc/IndirectionUtils.h"
  10 #include "llvm/ExecutionEngine/JITLink/x86_64.h"
  11 #include "llvm/ExecutionEngine/Orc/OrcABISupport.h"
  12 #include "llvm/IR/IRBuilder.h"
  13 #include "llvm/IR/Module.h"
  14 #include "llvm/MC/MCDisassembler/MCDisassembler.h"
  15 #include "llvm/MC/MCInstrAnalysis.h"
  16 #include "llvm/TargetParser/Triple.h"
  17 #include "llvm/Transforms/Utils/Cloning.h"
  18
  19 #define DEBUG_TYPE "orc"
  20
  21 using namespace llvm;
  22 using namespace llvm::orc;
  23
  24 namespace {
  25
  26 class CompileCallbackMaterializationUnit : public orc::MaterializationUnit {
  27 public:
  28   using CompileFunction = JITCompileCallbackManager::CompileFunction;
  29
  30   CompileCallbackMaterializationUnit(SymbolStringPtr Name,
  31                                      CompileFunction Compile)
  32       : MaterializationUnit(Interface(
  33             SymbolFlagsMap({{Name, JITSymbolFlags::Exported}}), nullptr)),
  34         Name(std::move(Name)), Compile(std::move(Compile)) {}
  35
  36   StringRef getName() const override { return "<Compile Callbacks>"; }
  37
  38 private:
  39   void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
  40     SymbolMap Result;
  41     Result[Name] = {Compile(), JITSymbolFlags::Exported};
  42     // No dependencies, so these calls cannot fail.
  43     cantFail(R->notifyResolved(Result));
  44     cantFail(R->notifyEmitted({}));
  45   }
  46
  47   void discard(const JITDylib &JD, const SymbolStringPtr &Name) override {
  48     llvm_unreachable("Discard should never occur on a LMU?");
  49   }
  50
  51   SymbolStringPtr Name;
  52   CompileFunction Compile;
  53 };
  54
  55 } // namespace
  56
  57 namespace llvm {
  58 namespace orc {
  59
  60 TrampolinePool::~TrampolinePool() = default;
  61 void IndirectStubsManager::anchor() {}
  62
  63 Expected<ExecutorAddr>
  64 JITCompileCallbackManager::getCompileCallback(CompileFunction Compile) {
  65   if (auto TrampolineAddr = TP->getTrampoline()) {
  66     auto CallbackName =
  67         ES.intern(std::string("cc") + std::to_string(++NextCallbackId));
  68
  69     std::lock_guard<std::mutex> Lock(CCMgrMutex);
  70     AddrToSymbol[*TrampolineAddr] = CallbackName;
  71     cantFail(
  72         CallbacksJD.define(std::make_unique<CompileCallbackMaterializationUnit>(
  73             std::move(CallbackName), std::move(Compile))));
  74     return *TrampolineAddr;
  75   } else
  76     return TrampolineAddr.takeError();
  77 }
  78
  79 ExecutorAddr
  80 JITCompileCallbackManager::executeCompileCallback(ExecutorAddr TrampolineAddr) {
  81   SymbolStringPtr Name;
  82
  83   {
  84     std::unique_lock<std::mutex> Lock(CCMgrMutex);
  85     auto I = AddrToSymbol.find(TrampolineAddr);
  86
  87     // If this address is not associated with a compile callback then report an
  88     // error to the execution session and return ErrorHandlerAddress to the
  89     // callee.
  90     if (I == AddrToSymbol.end()) {
  91       Lock.unlock();
  92       ES.reportError(
  93           make_error<StringError>("No compile callback for trampoline at " +
  94                                       formatv("{0:x}", TrampolineAddr),
  95                                   inconvertibleErrorCode()));
  96       return ErrorHandlerAddress;
  97     } else
  98       Name = I->second;
  99   }
 100
 101   if (auto Sym =
 102           ES.lookup(makeJITDylibSearchOrder(
 103                         &CallbacksJD, JITDylibLookupFlags::MatchAllSymbols),
 104                     Name))
 105     return Sym->getAddress();
 106   else {
 107     llvm::dbgs() << "Didn't find callback.\n";
 108     // If anything goes wrong materializing Sym then report it to the session
 109     // and return the ErrorHandlerAddress;
 110     ES.reportError(Sym.takeError());
 111     return ErrorHandlerAddress;
 112   }
 113 }
 114
 115 Error IndirectStubsManager::redirect(JITDylib &JD, const SymbolMap &NewDests) {
 116   for (auto &[Name, Dest] : NewDests)
 117     if (auto Err = updatePointer(*Name, Dest.getAddress()))
 118       return Err;
 119   return Error::success();
 120 }
 121
 122 void IndirectStubsManager::emitRedirectableSymbols(
 123     std::unique_ptr<MaterializationResponsibility> MR, SymbolMap InitialDests) {
 124   StubInitsMap StubInits;
 125   for (auto &[Name, Dest] : InitialDests)
 126     StubInits[*Name] = {Dest.getAddress(), Dest.getFlags()};
 127   if (auto Err = createStubs(StubInits)) {
 128     MR->getExecutionSession().reportError(std::move(Err));
 129     return MR->failMaterialization();
 130   }
 131   SymbolMap Stubs;
 132   for (auto &[Name, Dest] : InitialDests) {
 133     auto StubSym = findStub(*Name, false);
 134     assert(StubSym.getAddress() && "Stub symbol should be present");
 135     Stubs[Name] = StubSym;
 136   }
 137   if (auto Err = MR->notifyResolved(Stubs)) {
 138     MR->getExecutionSession().reportError(std::move(Err));
 139     return MR->failMaterialization();
 140   }
 141   if (auto Err = MR->notifyEmitted({})) {
 142     MR->getExecutionSession().reportError(std::move(Err));
 143     return MR->failMaterialization();
 144   }
 145 }
 146
 147 Expected<std::unique_ptr<JITCompileCallbackManager>>
 148 createLocalCompileCallbackManager(const Triple &T, ExecutionSession &ES,
 149                                   ExecutorAddr ErrorHandlerAddress) {
 150   switch (T.getArch()) {
 151   default:
 152     return make_error<StringError>(
 153         std::string("No callback manager available for ") + T.str(),
 154         inconvertibleErrorCode());
 155   case Triple::aarch64:
 156   case Triple::aarch64_32: {
 157     typedef orc::LocalJITCompileCallbackManager<orc::OrcAArch64> CCMgrT;
 158     return CCMgrT::Create(ES, ErrorHandlerAddress);
 159     }
 160
 161     case Triple::x86: {
 162       typedef orc::LocalJITCompileCallbackManager<orc::OrcI386> CCMgrT;
 163       return CCMgrT::Create(ES, ErrorHandlerAddress);
 164     }
 165
 166     case Triple::loongarch64: {
 167       typedef orc::LocalJITCompileCallbackManager<orc::OrcLoongArch64> CCMgrT;
 168       return CCMgrT::Create(ES, ErrorHandlerAddress);
 169     }
 170
 171     case Triple::mips: {
 172       typedef orc::LocalJITCompileCallbackManager<orc::OrcMips32Be> CCMgrT;
 173       return CCMgrT::Create(ES, ErrorHandlerAddress);
 174     }
 175     case Triple::mipsel: {
 176       typedef orc::LocalJITCompileCallbackManager<orc::OrcMips32Le> CCMgrT;
 177       return CCMgrT::Create(ES, ErrorHandlerAddress);
 178     }
 179
 180     case Triple::mips64:
 181     case Triple::mips64el: {
 182       typedef orc::LocalJITCompileCallbackManager<orc::OrcMips64> CCMgrT;
 183       return CCMgrT::Create(ES, ErrorHandlerAddress);
 184     }
 185
 186     case Triple::riscv64: {
 187       typedef orc::LocalJITCompileCallbackManager<orc::OrcRiscv64> CCMgrT;
 188       return CCMgrT::Create(ES, ErrorHandlerAddress);
 189     }
 190
 191     case Triple::x86_64: {
 192       if (T.getOS() == Triple::OSType::Win32) {
 193         typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_Win32> CCMgrT;
 194         return CCMgrT::Create(ES, ErrorHandlerAddress);
 195       } else {
 196         typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_SysV> CCMgrT;
 197         return CCMgrT::Create(ES, ErrorHandlerAddress);
 198       }
 199     }
 200
 201   }
 202 }
 203
 204 std::function<std::unique_ptr<IndirectStubsManager>()>
 205 createLocalIndirectStubsManagerBuilder(const Triple &T) {
 206   switch (T.getArch()) {
 207     default:
 208       return [](){
 209         return std::make_unique<
 210                        orc::LocalIndirectStubsManager<orc::OrcGenericABI>>();
 211       };
 212
 213     case Triple::aarch64:
 214     case Triple::aarch64_32:
 215       return [](){
 216         return std::make_unique<
 217                        orc::LocalIndirectStubsManager<orc::OrcAArch64>>();
 218       };
 219
 220     case Triple::x86:
 221       return [](){
 222         return std::make_unique<
 223                        orc::LocalIndirectStubsManager<orc::OrcI386>>();
 224       };
 225
 226     case Triple::loongarch64:
 227       return []() {
 228         return std::make_unique<
 229             orc::LocalIndirectStubsManager<orc::OrcLoongArch64>>();
 230       };
 231
 232     case Triple::mips:
 233       return [](){
 234           return std::make_unique<
 235                       orc::LocalIndirectStubsManager<orc::OrcMips32Be>>();
 236       };
 237
 238     case Triple::mipsel:
 239       return [](){
 240           return std::make_unique<
 241                       orc::LocalIndirectStubsManager<orc::OrcMips32Le>>();
 242       };
 243
 244     case Triple::mips64:
 245     case Triple::mips64el:
 246       return [](){
 247           return std::make_unique<
 248                       orc::LocalIndirectStubsManager<orc::OrcMips64>>();
 249       };
 250
 251     case Triple::riscv64:
 252       return []() {
 253         return std::make_unique<
 254             orc::LocalIndirectStubsManager<orc::OrcRiscv64>>();
 255       };
 256
 257     case Triple::x86_64:
 258       if (T.getOS() == Triple::OSType::Win32) {
 259         return [](){
 260           return std::make_unique<
 261                      orc::LocalIndirectStubsManager<orc::OrcX86_64_Win32>>();
 262         };
 263       } else {
 264         return [](){
 265           return std::make_unique<
 266                      orc::LocalIndirectStubsManager<orc::OrcX86_64_SysV>>();
 267         };
 268       }
 269
 270   }
 271 }
 272
 273 Constant* createIRTypedAddress(FunctionType &FT, ExecutorAddr Addr) {
 274   Constant *AddrIntVal =
 275     ConstantInt::get(Type::getInt64Ty(FT.getContext()), Addr.getValue());
 276   Constant *AddrPtrVal =
 277     ConstantExpr::getIntToPtr(AddrIntVal, PointerType::get(&FT, 0));
 278   return AddrPtrVal;
 279 }
 280
 281 GlobalVariable* createImplPointer(PointerType &PT, Module &M,
 282                                   const Twine &Name, Constant *Initializer) {
 283   auto IP = new GlobalVariable(M, &PT, false, GlobalValue::ExternalLinkage,
 284                                Initializer, Name, nullptr,
 285                                GlobalValue::NotThreadLocal, 0, true);
 286   IP->setVisibility(GlobalValue::HiddenVisibility);
 287   return IP;
 288 }
 289
 290 void makeStub(Function &F, Value &ImplPointer) {
 291   assert(F.isDeclaration() && "Can't turn a definition into a stub.");
 292   assert(F.getParent() && "Function isn't in a module.");
 293   Module &M = *F.getParent();
 294   BasicBlock *EntryBlock = BasicBlock::Create(M.getContext(), "entry", &F);
 295   IRBuilder<> Builder(EntryBlock);
 296   LoadInst *ImplAddr = Builder.CreateLoad(F.getType(), &ImplPointer);
 297   std::vector<Value*> CallArgs;
 298   for (auto &A : F.args())
 299     CallArgs.push_back(&A);
 300   CallInst *Call = Builder.CreateCall(F.getFunctionType(), ImplAddr, CallArgs);
 301   Call->setTailCall();
 302   Call->setAttributes(F.getAttributes());
 303   if (F.getReturnType()->isVoidTy())
 304     Builder.CreateRetVoid();
 305   else
 306     Builder.CreateRet(Call);
 307 }
 308
 309 std::vector<GlobalValue *> SymbolLinkagePromoter::operator()(Module &M) {
 310   std::vector<GlobalValue *> PromotedGlobals;
 311
 312   for (auto &GV : M.global_values()) {
 313     bool Promoted = true;
 314
 315     // Rename if necessary.
 316     if (!GV.hasName())
 317       GV.setName("__orc_anon." + Twine(NextId++));
 318     else if (GV.getName().starts_with("\01L"))
 319       GV.setName("__" + GV.getName().substr(1) + "." + Twine(NextId++));
 320     else if (GV.hasLocalLinkage())
 321       GV.setName("__orc_lcl." + GV.getName() + "." + Twine(NextId++));
 322     else
 323       Promoted = false;
 324
 325     if (GV.hasLocalLinkage()) {
 326       GV.setLinkage(GlobalValue::ExternalLinkage);
 327       GV.setVisibility(GlobalValue::HiddenVisibility);
 328       Promoted = true;
 329     }
 330     GV.setUnnamedAddr(GlobalValue::UnnamedAddr::None);
 331
 332     if (Promoted)
 333       PromotedGlobals.push_back(&GV);
 334   }
 335
 336   return PromotedGlobals;
 337 }
 338
 339 Function* cloneFunctionDecl(Module &Dst, const Function &F,
 340                             ValueToValueMapTy *VMap) {
 341   Function *NewF =
 342     Function::Create(cast<FunctionType>(F.getValueType()),
 343                      F.getLinkage(), F.getName(), &Dst);
 344   NewF->copyAttributesFrom(&F);
 345
 346   if (VMap) {
 347     (*VMap)[&F] = NewF;
 348     auto NewArgI = NewF->arg_begin();
 349     for (auto ArgI = F.arg_begin(), ArgE = F.arg_end(); ArgI != ArgE;
 350          ++ArgI, ++NewArgI)
 351       (*VMap)[&*ArgI] = &*NewArgI;
 352   }
 353
 354   return NewF;
 355 }
 356
 357 GlobalVariable* cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV,
 358                                         ValueToValueMapTy *VMap) {
 359   GlobalVariable *NewGV = new GlobalVariable(
 360       Dst, GV.getValueType(), GV.isConstant(),
 361       GV.getLinkage(), nullptr, GV.getName(), nullptr,
 362       GV.getThreadLocalMode(), GV.getType()->getAddressSpace());
 363   NewGV->copyAttributesFrom(&GV);
 364   if (VMap)
 365     (*VMap)[&GV] = NewGV;
 366   return NewGV;
 367 }
 368
 369 GlobalAlias* cloneGlobalAliasDecl(Module &Dst, const GlobalAlias &OrigA,
 370                                   ValueToValueMapTy &VMap) {
 371   assert(OrigA.getAliasee() && "Original alias doesn't have an aliasee?");
 372   auto *NewA = GlobalAlias::create(OrigA.getValueType(),
 373                                    OrigA.getType()->getPointerAddressSpace(),
 374                                    OrigA.getLinkage(), OrigA.getName(), &Dst);
 375   NewA->copyAttributesFrom(&OrigA);
 376   VMap[&OrigA] = NewA;
 377   return NewA;
 378 }
 379
 380 Error addFunctionPointerRelocationsToCurrentSymbol(jitlink::Symbol &Sym,
 381                                                    jitlink::LinkGraph &G,
 382                                                    MCDisassembler &Disassembler,
 383                                                    MCInstrAnalysis &MIA) {
 384   // AArch64 appears to already come with the necessary relocations. Among other
 385   // architectures, only x86_64 is currently implemented here.
 386   if (G.getTargetTriple().getArch() != Triple::x86_64)
 387     return Error::success();
 388
 389   raw_null_ostream CommentStream;
 390   auto &STI = Disassembler.getSubtargetInfo();
 391
 392   // Determine the function bounds
 393   auto &B = Sym.getBlock();
 394   assert(!B.isZeroFill() && "expected content block");
 395   auto SymAddress = Sym.getAddress();
 396   auto SymStartInBlock =
 397       (const uint8_t *)B.getContent().data() + Sym.getOffset();
 398   auto SymSize = Sym.getSize() ? Sym.getSize() : B.getSize() - Sym.getOffset();
 399   auto Content = ArrayRef(SymStartInBlock, SymSize);
 400
 401   LLVM_DEBUG(dbgs() << "Adding self-relocations to " << Sym.getName() << "\n");
 402
 403   SmallDenseSet<uintptr_t, 8> ExistingRelocations;
 404   for (auto &E : B.edges()) {
 405     if (E.isRelocation())
 406       ExistingRelocations.insert(E.getOffset());
 407   }
 408
 409   size_t I = 0;
 410   while (I < Content.size()) {
 411     MCInst Instr;
 412     uint64_t InstrSize = 0;
 413     uint64_t InstrStart = SymAddress.getValue() + I;
 414     auto DecodeStatus = Disassembler.getInstruction(
 415         Instr, InstrSize, Content.drop_front(I), InstrStart, CommentStream);
 416     if (DecodeStatus != MCDisassembler::Success) {
 417       LLVM_DEBUG(dbgs() << "Aborting due to disassembly failure at address "
 418                         << InstrStart);
 419       return make_error<StringError>(
 420           formatv("failed to disassemble at address {0:x16}", InstrStart),
 421           inconvertibleErrorCode());
 422     }
 423     // Advance to the next instruction.
 424     I += InstrSize;
 425
 426     // Check for a PC-relative address equal to the symbol itself.
 427     auto PCRelAddr =
 428         MIA.evaluateMemoryOperandAddress(Instr, &STI, InstrStart, InstrSize);
 429     if (!PCRelAddr || *PCRelAddr != SymAddress.getValue())
 430       continue;
 431
 432     auto RelocOffInInstr =
 433         MIA.getMemoryOperandRelocationOffset(Instr, InstrSize);
 434     if (!RelocOffInInstr || InstrSize - *RelocOffInInstr != 4) {
 435       LLVM_DEBUG(dbgs() << "Skipping unknown self-relocation at "
 436                         << InstrStart);
 437       continue;
 438     }
 439
 440     auto RelocOffInBlock = orc::ExecutorAddr(InstrStart) + *RelocOffInInstr -
 441                            SymAddress + Sym.getOffset();
 442     if (ExistingRelocations.contains(RelocOffInBlock))
 443       continue;
 444
 445     LLVM_DEBUG(dbgs() << "Adding delta32 self-relocation at " << InstrStart);
 446     B.addEdge(jitlink::x86_64::Delta32, RelocOffInBlock, Sym, /*Addend=*/-4);
 447   }
 448   return Error::success();
 449 }
 450
 451 } // End namespace orc.
 452 } // End namespace llvm.