llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp

   1 //===---- IndirectionUtils.cpp - Utilities for call indirection in Orc ----===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "llvm/ExecutionEngine/Orc/IndirectionUtils.h"
  10 #include "llvm/ADT/STLExtras.h"
  11 #include "llvm/ADT/Triple.h"
  12 #include "llvm/ExecutionEngine/Orc/OrcABISupport.h"
  13 #include "llvm/IR/IRBuilder.h"
  14 #include "llvm/Support/Format.h"
  15 #include "llvm/Transforms/Utils/Cloning.h"
  16 #include <sstream>
  17
  18 using namespace llvm;
  19 using namespace llvm::orc;
  20
  21 namespace {
  22
  23 class CompileCallbackMaterializationUnit : public orc::MaterializationUnit {
  24 public:
  25   using CompileFunction = JITCompileCallbackManager::CompileFunction;
  26
  27   CompileCallbackMaterializationUnit(SymbolStringPtr Name,
  28                                      CompileFunction Compile)
  29       : MaterializationUnit(SymbolFlagsMap({{Name, JITSymbolFlags::Exported}}),
  30                             nullptr),
  31         Name(std::move(Name)), Compile(std::move(Compile)) {}
  32
  33   StringRef getName() const override { return "<Compile Callbacks>"; }
  34
  35 private:
  36   void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
  37     SymbolMap Result;
  38     Result[Name] = JITEvaluatedSymbol(Compile(), JITSymbolFlags::Exported);
  39     // No dependencies, so these calls cannot fail.
  40     cantFail(R->notifyResolved(Result));
  41     cantFail(R->notifyEmitted());
  42   }
  43
  44   void discard(const JITDylib &JD, const SymbolStringPtr &Name) override {
  45     llvm_unreachable("Discard should never occur on a LMU?");
  46   }
  47
  48   SymbolStringPtr Name;
  49   CompileFunction Compile;
  50 };
  51
  52 } // namespace
  53
  54 namespace llvm {
  55 namespace orc {
  56
  57 TrampolinePool::~TrampolinePool() {}
  58 void IndirectStubsManager::anchor() {}
  59
  60 Expected<JITTargetAddress>
  61 JITCompileCallbackManager::getCompileCallback(CompileFunction Compile) {
  62   if (auto TrampolineAddr = TP->getTrampoline()) {
  63     auto CallbackName =
  64         ES.intern(std::string("cc") + std::to_string(++NextCallbackId));
  65
  66     std::lock_guard<std::mutex> Lock(CCMgrMutex);
  67     AddrToSymbol[*TrampolineAddr] = CallbackName;
  68     cantFail(
  69         CallbacksJD.define(std::make_unique<CompileCallbackMaterializationUnit>(
  70             std::move(CallbackName), std::move(Compile))));
  71     return *TrampolineAddr;
  72   } else
  73     return TrampolineAddr.takeError();
  74 }
  75
  76 JITTargetAddress JITCompileCallbackManager::executeCompileCallback(
  77     JITTargetAddress TrampolineAddr) {
  78   SymbolStringPtr Name;
  79
  80   {
  81     std::unique_lock<std::mutex> Lock(CCMgrMutex);
  82     auto I = AddrToSymbol.find(TrampolineAddr);
  83
  84     // If this address is not associated with a compile callback then report an
  85     // error to the execution session and return ErrorHandlerAddress to the
  86     // callee.
  87     if (I == AddrToSymbol.end()) {
  88       Lock.unlock();
  89       std::string ErrMsg;
  90       {
  91         raw_string_ostream ErrMsgStream(ErrMsg);
  92         ErrMsgStream << "No compile callback for trampoline at "
  93                      << format("0x%016" PRIx64, TrampolineAddr);
  94       }
  95       ES.reportError(
  96           make_error<StringError>(std::move(ErrMsg), inconvertibleErrorCode()));
  97       return ErrorHandlerAddress;
  98     } else
  99       Name = I->second;
 100   }
 101
 102   if (auto Sym =
 103           ES.lookup(makeJITDylibSearchOrder(
 104                         &CallbacksJD, JITDylibLookupFlags::MatchAllSymbols),
 105                     Name))
 106     return Sym->getAddress();
 107   else {
 108     llvm::dbgs() << "Didn't find callback.\n";
 109     // If anything goes wrong materializing Sym then report it to the session
 110     // and return the ErrorHandlerAddress;
 111     ES.reportError(Sym.takeError());
 112     return ErrorHandlerAddress;
 113   }
 114 }
 115
 116 Expected<std::unique_ptr<JITCompileCallbackManager>>
 117 createLocalCompileCallbackManager(const Triple &T, ExecutionSession &ES,
 118                                   JITTargetAddress ErrorHandlerAddress) {
 119   switch (T.getArch()) {
 120   default:
 121     return make_error<StringError>(
 122         std::string("No callback manager available for ") + T.str(),
 123         inconvertibleErrorCode());
 124   case Triple::aarch64:
 125   case Triple::aarch64_32: {
 126     typedef orc::LocalJITCompileCallbackManager<orc::OrcAArch64> CCMgrT;
 127     return CCMgrT::Create(ES, ErrorHandlerAddress);
 128     }
 129
 130     case Triple::x86: {
 131       typedef orc::LocalJITCompileCallbackManager<orc::OrcI386> CCMgrT;
 132       return CCMgrT::Create(ES, ErrorHandlerAddress);
 133     }
 134
 135     case Triple::mips: {
 136       typedef orc::LocalJITCompileCallbackManager<orc::OrcMips32Be> CCMgrT;
 137       return CCMgrT::Create(ES, ErrorHandlerAddress);
 138     }
 139     case Triple::mipsel: {
 140       typedef orc::LocalJITCompileCallbackManager<orc::OrcMips32Le> CCMgrT;
 141       return CCMgrT::Create(ES, ErrorHandlerAddress);
 142     }
 143
 144     case Triple::mips64:
 145     case Triple::mips64el: {
 146       typedef orc::LocalJITCompileCallbackManager<orc::OrcMips64> CCMgrT;
 147       return CCMgrT::Create(ES, ErrorHandlerAddress);
 148     }
 149
 150     case Triple::x86_64: {
 151       if (T.getOS() == Triple::OSType::Win32) {
 152         typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_Win32> CCMgrT;
 153         return CCMgrT::Create(ES, ErrorHandlerAddress);
 154       } else {
 155         typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_SysV> CCMgrT;
 156         return CCMgrT::Create(ES, ErrorHandlerAddress);
 157       }
 158     }
 159
 160   }
 161 }
 162
 163 std::function<std::unique_ptr<IndirectStubsManager>()>
 164 createLocalIndirectStubsManagerBuilder(const Triple &T) {
 165   switch (T.getArch()) {
 166     default:
 167       return [](){
 168         return std::make_unique<
 169                        orc::LocalIndirectStubsManager<orc::OrcGenericABI>>();
 170       };
 171
 172     case Triple::aarch64:
 173     case Triple::aarch64_32:
 174       return [](){
 175         return std::make_unique<
 176                        orc::LocalIndirectStubsManager<orc::OrcAArch64>>();
 177       };
 178
 179     case Triple::x86:
 180       return [](){
 181         return std::make_unique<
 182                        orc::LocalIndirectStubsManager<orc::OrcI386>>();
 183       };
 184
 185     case Triple::mips:
 186       return [](){
 187           return std::make_unique<
 188                       orc::LocalIndirectStubsManager<orc::OrcMips32Be>>();
 189       };
 190
 191     case Triple::mipsel:
 192       return [](){
 193           return std::make_unique<
 194                       orc::LocalIndirectStubsManager<orc::OrcMips32Le>>();
 195       };
 196
 197     case Triple::mips64:
 198     case Triple::mips64el:
 199       return [](){
 200           return std::make_unique<
 201                       orc::LocalIndirectStubsManager<orc::OrcMips64>>();
 202       };
 203
 204     case Triple::x86_64:
 205       if (T.getOS() == Triple::OSType::Win32) {
 206         return [](){
 207           return std::make_unique<
 208                      orc::LocalIndirectStubsManager<orc::OrcX86_64_Win32>>();
 209         };
 210       } else {
 211         return [](){
 212           return std::make_unique<
 213                      orc::LocalIndirectStubsManager<orc::OrcX86_64_SysV>>();
 214         };
 215       }
 216
 217   }
 218 }
 219
 220 Constant* createIRTypedAddress(FunctionType &FT, JITTargetAddress Addr) {
 221   Constant *AddrIntVal =
 222     ConstantInt::get(Type::getInt64Ty(FT.getContext()), Addr);
 223   Constant *AddrPtrVal =
 224     ConstantExpr::getCast(Instruction::IntToPtr, AddrIntVal,
 225                           PointerType::get(&FT, 0));
 226   return AddrPtrVal;
 227 }
 228
 229 GlobalVariable* createImplPointer(PointerType &PT, Module &M,
 230                                   const Twine &Name, Constant *Initializer) {
 231   auto IP = new GlobalVariable(M, &PT, false, GlobalValue::ExternalLinkage,
 232                                Initializer, Name, nullptr,
 233                                GlobalValue::NotThreadLocal, 0, true);
 234   IP->setVisibility(GlobalValue::HiddenVisibility);
 235   return IP;
 236 }
 237
 238 void makeStub(Function &F, Value &ImplPointer) {
 239   assert(F.isDeclaration() && "Can't turn a definition into a stub.");
 240   assert(F.getParent() && "Function isn't in a module.");
 241   Module &M = *F.getParent();
 242   BasicBlock *EntryBlock = BasicBlock::Create(M.getContext(), "entry", &F);
 243   IRBuilder<> Builder(EntryBlock);
 244   LoadInst *ImplAddr = Builder.CreateLoad(F.getType(), &ImplPointer);
 245   std::vector<Value*> CallArgs;
 246   for (auto &A : F.args())
 247     CallArgs.push_back(&A);
 248   CallInst *Call = Builder.CreateCall(F.getFunctionType(), ImplAddr, CallArgs);
 249   Call->setTailCall();
 250   Call->setAttributes(F.getAttributes());
 251   if (F.getReturnType()->isVoidTy())
 252     Builder.CreateRetVoid();
 253   else
 254     Builder.CreateRet(Call);
 255 }
 256
 257 std::vector<GlobalValue *> SymbolLinkagePromoter::operator()(Module &M) {
 258   std::vector<GlobalValue *> PromotedGlobals;
 259
 260   for (auto &GV : M.global_values()) {
 261     bool Promoted = true;
 262
 263     // Rename if necessary.
 264     if (!GV.hasName())
 265       GV.setName("__orc_anon." + Twine(NextId++));
 266     else if (GV.getName().startswith("\01L"))
 267       GV.setName("__" + GV.getName().substr(1) + "." + Twine(NextId++));
 268     else if (GV.hasLocalLinkage())
 269       GV.setName("__orc_lcl." + GV.getName() + "." + Twine(NextId++));
 270     else
 271       Promoted = false;
 272
 273     if (GV.hasLocalLinkage()) {
 274       GV.setLinkage(GlobalValue::ExternalLinkage);
 275       GV.setVisibility(GlobalValue::HiddenVisibility);
 276       Promoted = true;
 277     }
 278     GV.setUnnamedAddr(GlobalValue::UnnamedAddr::None);
 279
 280     if (Promoted)
 281       PromotedGlobals.push_back(&GV);
 282   }
 283
 284   return PromotedGlobals;
 285 }
 286
 287 Function* cloneFunctionDecl(Module &Dst, const Function &F,
 288                             ValueToValueMapTy *VMap) {
 289   Function *NewF =
 290     Function::Create(cast<FunctionType>(F.getValueType()),
 291                      F.getLinkage(), F.getName(), &Dst);
 292   NewF->copyAttributesFrom(&F);
 293
 294   if (VMap) {
 295     (*VMap)[&F] = NewF;
 296     auto NewArgI = NewF->arg_begin();
 297     for (auto ArgI = F.arg_begin(), ArgE = F.arg_end(); ArgI != ArgE;
 298          ++ArgI, ++NewArgI)
 299       (*VMap)[&*ArgI] = &*NewArgI;
 300   }
 301
 302   return NewF;
 303 }
 304
 305 void moveFunctionBody(Function &OrigF, ValueToValueMapTy &VMap,
 306                       ValueMaterializer *Materializer,
 307                       Function *NewF) {
 308   assert(!OrigF.isDeclaration() && "Nothing to move");
 309   if (!NewF)
 310     NewF = cast<Function>(VMap[&OrigF]);
 311   else
 312     assert(VMap[&OrigF] == NewF && "Incorrect function mapping in VMap.");
 313   assert(NewF && "Function mapping missing from VMap.");
 314   assert(NewF->getParent() != OrigF.getParent() &&
 315          "moveFunctionBody should only be used to move bodies between "
 316          "modules.");
 317
 318   SmallVector<ReturnInst *, 8> Returns; // Ignore returns cloned.
 319   CloneFunctionInto(NewF, &OrigF, VMap,
 320                     CloneFunctionChangeType::DifferentModule, Returns, "",
 321                     nullptr, nullptr, Materializer);
 322   OrigF.deleteBody();
 323 }
 324
 325 GlobalVariable* cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV,
 326                                         ValueToValueMapTy *VMap) {
 327   GlobalVariable *NewGV = new GlobalVariable(
 328       Dst, GV.getValueType(), GV.isConstant(),
 329       GV.getLinkage(), nullptr, GV.getName(), nullptr,
 330       GV.getThreadLocalMode(), GV.getType()->getAddressSpace());
 331   NewGV->copyAttributesFrom(&GV);
 332   if (VMap)
 333     (*VMap)[&GV] = NewGV;
 334   return NewGV;
 335 }
 336
 337 void moveGlobalVariableInitializer(GlobalVariable &OrigGV,
 338                                    ValueToValueMapTy &VMap,
 339                                    ValueMaterializer *Materializer,
 340                                    GlobalVariable *NewGV) {
 341   assert(OrigGV.hasInitializer() && "Nothing to move");
 342   if (!NewGV)
 343     NewGV = cast<GlobalVariable>(VMap[&OrigGV]);
 344   else
 345     assert(VMap[&OrigGV] == NewGV &&
 346            "Incorrect global variable mapping in VMap.");
 347   assert(NewGV->getParent() != OrigGV.getParent() &&
 348          "moveGlobalVariableInitializer should only be used to move "
 349          "initializers between modules");
 350
 351   NewGV->setInitializer(MapValue(OrigGV.getInitializer(), VMap, RF_None,
 352                                  nullptr, Materializer));
 353 }
 354
 355 GlobalAlias* cloneGlobalAliasDecl(Module &Dst, const GlobalAlias &OrigA,
 356                                   ValueToValueMapTy &VMap) {
 357   assert(OrigA.getAliasee() && "Original alias doesn't have an aliasee?");
 358   auto *NewA = GlobalAlias::create(OrigA.getValueType(),
 359                                    OrigA.getType()->getPointerAddressSpace(),
 360                                    OrigA.getLinkage(), OrigA.getName(), &Dst);
 361   NewA->copyAttributesFrom(&OrigA);
 362   VMap[&OrigA] = NewA;
 363   return NewA;
 364 }
 365
 366 void cloneModuleFlagsMetadata(Module &Dst, const Module &Src,
 367                               ValueToValueMapTy &VMap) {
 368   auto *MFs = Src.getModuleFlagsMetadata();
 369   if (!MFs)
 370     return;
 371   for (auto *MF : MFs->operands())
 372     Dst.addModuleFlag(MapMetadata(MF, VMap));
 373 }
 374
 375 } // End namespace orc.
 376 } // End namespace llvm.