llvm/lib/ExecutionEngine/JITLink/JITLink.cpp

   1 //===------------- JITLink.cpp - Core Run-time JIT linker APIs ------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "llvm/ExecutionEngine/JITLink/JITLink.h"
  10
  11 #include "llvm/ADT/StringExtras.h"
  12 #include "llvm/BinaryFormat/Magic.h"
  13 #include "llvm/ExecutionEngine/JITLink/COFF.h"
  14 #include "llvm/ExecutionEngine/JITLink/ELF.h"
  15 #include "llvm/ExecutionEngine/JITLink/MachO.h"
  16 #include "llvm/ExecutionEngine/JITLink/aarch64.h"
  17 #include "llvm/ExecutionEngine/JITLink/i386.h"
  18 #include "llvm/ExecutionEngine/JITLink/loongarch.h"
  19 #include "llvm/ExecutionEngine/JITLink/x86_64.h"
  20 #include "llvm/Support/Format.h"
  21 #include "llvm/Support/MemoryBuffer.h"
  22 #include "llvm/Support/raw_ostream.h"
  23
  24 using namespace llvm;
  25 using namespace llvm::object;
  26
  27 #define DEBUG_TYPE "jitlink"
  28
  29 namespace {
  30
  31 enum JITLinkErrorCode { GenericJITLinkError = 1 };
  32
  33 // FIXME: This class is only here to support the transition to llvm::Error. It
  34 // will be removed once this transition is complete. Clients should prefer to
  35 // deal with the Error value directly, rather than converting to error_code.
  36 class JITLinkerErrorCategory : public std::error_category {
  37 public:
  38   const char *name() const noexcept override { return "runtimedyld"; }
  39
  40   std::string message(int Condition) const override {
  41     switch (static_cast<JITLinkErrorCode>(Condition)) {
  42     case GenericJITLinkError:
  43       return "Generic JITLink error";
  44     }
  45     llvm_unreachable("Unrecognized JITLinkErrorCode");
  46   }
  47 };
  48
  49 } // namespace
  50
  51 namespace llvm {
  52 namespace jitlink {
  53
  54 char JITLinkError::ID = 0;
  55
  56 void JITLinkError::log(raw_ostream &OS) const { OS << ErrMsg; }
  57
  58 std::error_code JITLinkError::convertToErrorCode() const {
  59   static JITLinkerErrorCategory TheJITLinkerErrorCategory;
  60   return std::error_code(GenericJITLinkError, TheJITLinkerErrorCategory);
  61 }
  62
  63 const char *getGenericEdgeKindName(Edge::Kind K) {
  64   switch (K) {
  65   case Edge::Invalid:
  66     return "INVALID RELOCATION";
  67   case Edge::KeepAlive:
  68     return "Keep-Alive";
  69   default:
  70     return "<Unrecognized edge kind>";
  71   }
  72 }
  73
  74 const char *getLinkageName(Linkage L) {
  75   switch (L) {
  76   case Linkage::Strong:
  77     return "strong";
  78   case Linkage::Weak:
  79     return "weak";
  80   }
  81   llvm_unreachable("Unrecognized llvm.jitlink.Linkage enum");
  82 }
  83
  84 const char *getScopeName(Scope S) {
  85   switch (S) {
  86   case Scope::Default:
  87     return "default";
  88   case Scope::Hidden:
  89     return "hidden";
  90   case Scope::Local:
  91     return "local";
  92   }
  93   llvm_unreachable("Unrecognized llvm.jitlink.Scope enum");
  94 }
  95
  96 bool isCStringBlock(Block &B) {
  97   if (B.getSize() == 0) // Empty blocks are not valid C-strings.
  98     return false;
  99
 100   // Zero-fill blocks of size one are valid empty strings.
 101   if (B.isZeroFill())
 102     return B.getSize() == 1;
 103
 104   for (size_t I = 0; I != B.getSize() - 1; ++I)
 105     if (B.getContent()[I] == '\0')
 106       return false;
 107
 108   return B.getContent()[B.getSize() - 1] == '\0';
 109 }
 110
 111 raw_ostream &operator<<(raw_ostream &OS, const Block &B) {
 112   return OS << B.getAddress() << " -- " << (B.getAddress() + B.getSize())
 113             << ": "
 114             << "size = " << formatv("{0:x8}", B.getSize()) << ", "
 115             << (B.isZeroFill() ? "zero-fill" : "content")
 116             << ", align = " << B.getAlignment()
 117             << ", align-ofs = " << B.getAlignmentOffset()
 118             << ", section = " << B.getSection().getName();
 119 }
 120
 121 raw_ostream &operator<<(raw_ostream &OS, const Symbol &Sym) {
 122   OS << Sym.getAddress() << " (" << (Sym.isDefined() ? "block" : "addressable")
 123      << " + " << formatv("{0:x8}", Sym.getOffset())
 124      << "): size: " << formatv("{0:x8}", Sym.getSize())
 125      << ", linkage: " << formatv("{0:6}", getLinkageName(Sym.getLinkage()))
 126      << ", scope: " << formatv("{0:8}", getScopeName(Sym.getScope())) << ", "
 127      << (Sym.isLive() ? "live" : "dead") << "  -   "
 128      << (Sym.hasName() ? Sym.getName() : "<anonymous symbol>");
 129   return OS;
 130 }
 131
 132 void printEdge(raw_ostream &OS, const Block &B, const Edge &E,
 133                StringRef EdgeKindName) {
 134   OS << "edge@" << B.getAddress() + E.getOffset() << ": " << B.getAddress()
 135      << " + " << formatv("{0:x}", E.getOffset()) << " -- " << EdgeKindName
 136      << " -> ";
 137
 138   auto &TargetSym = E.getTarget();
 139   if (TargetSym.hasName())
 140     OS << TargetSym.getName();
 141   else {
 142     auto &TargetBlock = TargetSym.getBlock();
 143     auto &TargetSec = TargetBlock.getSection();
 144     orc::ExecutorAddr SecAddress(~uint64_t(0));
 145     for (auto *B : TargetSec.blocks())
 146       if (B->getAddress() < SecAddress)
 147         SecAddress = B->getAddress();
 148
 149     orc::ExecutorAddrDiff SecDelta = TargetSym.getAddress() - SecAddress;
 150     OS << TargetSym.getAddress() << " (section " << TargetSec.getName();
 151     if (SecDelta)
 152       OS << " + " << formatv("{0:x}", SecDelta);
 153     OS << " / block " << TargetBlock.getAddress();
 154     if (TargetSym.getOffset())
 155       OS << " + " << formatv("{0:x}", TargetSym.getOffset());
 156     OS << ")";
 157   }
 158
 159   if (E.getAddend() != 0)
 160     OS << " + " << E.getAddend();
 161 }
 162
 163 Section::~Section() {
 164   for (auto *Sym : Symbols)
 165     Sym->~Symbol();
 166   for (auto *B : Blocks)
 167     B->~Block();
 168 }
 169
 170 Block &LinkGraph::splitBlock(Block &B, size_t SplitIndex,
 171                              SplitBlockCache *Cache) {
 172
 173   assert(SplitIndex > 0 && "splitBlock can not be called with SplitIndex == 0");
 174
 175   // If the split point covers all of B then just return B.
 176   if (SplitIndex == B.getSize())
 177     return B;
 178
 179   assert(SplitIndex < B.getSize() && "SplitIndex out of range");
 180
 181   // Create the new block covering [ 0, SplitIndex ).
 182   auto &NewBlock =
 183       B.isZeroFill()
 184           ? createZeroFillBlock(B.getSection(), SplitIndex, B.getAddress(),
 185                                 B.getAlignment(), B.getAlignmentOffset())
 186           : createContentBlock(
 187                 B.getSection(), B.getContent().slice(0, SplitIndex),
 188                 B.getAddress(), B.getAlignment(), B.getAlignmentOffset());
 189
 190   // Modify B to cover [ SplitIndex, B.size() ).
 191   B.setAddress(B.getAddress() + SplitIndex);
 192   B.setContent(B.getContent().slice(SplitIndex));
 193   B.setAlignmentOffset((B.getAlignmentOffset() + SplitIndex) %
 194                        B.getAlignment());
 195
 196   // Handle edge transfer/update.
 197   {
 198     // Copy edges to NewBlock (recording their iterators so that we can remove
 199     // them from B), and update of Edges remaining on B.
 200     std::vector<Block::edge_iterator> EdgesToRemove;
 201     for (auto I = B.edges().begin(); I != B.edges().end();) {
 202       if (I->getOffset() < SplitIndex) {
 203         NewBlock.addEdge(*I);
 204         I = B.removeEdge(I);
 205       } else {
 206         I->setOffset(I->getOffset() - SplitIndex);
 207         ++I;
 208       }
 209     }
 210   }
 211
 212   // Handle symbol transfer/update.
 213   {
 214     // Initialize the symbols cache if necessary.
 215     SplitBlockCache LocalBlockSymbolsCache;
 216     if (!Cache)
 217       Cache = &LocalBlockSymbolsCache;
 218     if (*Cache == std::nullopt) {
 219       *Cache = SplitBlockCache::value_type();
 220       for (auto *Sym : B.getSection().symbols())
 221         if (&Sym->getBlock() == &B)
 222           (*Cache)->push_back(Sym);
 223
 224       llvm::sort(**Cache, [](const Symbol *LHS, const Symbol *RHS) {
 225         return LHS->getOffset() > RHS->getOffset();
 226       });
 227     }
 228     auto &BlockSymbols = **Cache;
 229
 230     // Transfer all symbols with offset less than SplitIndex to NewBlock.
 231     while (!BlockSymbols.empty() &&
 232            BlockSymbols.back()->getOffset() < SplitIndex) {
 233       auto *Sym = BlockSymbols.back();
 234       // If the symbol extends beyond the split, update the size to be within
 235       // the new block.
 236       if (Sym->getOffset() + Sym->getSize() > SplitIndex)
 237         Sym->setSize(SplitIndex - Sym->getOffset());
 238       Sym->setBlock(NewBlock);
 239       BlockSymbols.pop_back();
 240     }
 241
 242     // Update offsets for all remaining symbols in B.
 243     for (auto *Sym : BlockSymbols)
 244       Sym->setOffset(Sym->getOffset() - SplitIndex);
 245   }
 246
 247   return NewBlock;
 248 }
 249
 250 void LinkGraph::dump(raw_ostream &OS) {
 251   DenseMap<Block *, std::vector<Symbol *>> BlockSymbols;
 252
 253   // Map from blocks to the symbols pointing at them.
 254   for (auto *Sym : defined_symbols())
 255     BlockSymbols[&Sym->getBlock()].push_back(Sym);
 256
 257   // For each block, sort its symbols by something approximating
 258   // relevance.
 259   for (auto &KV : BlockSymbols)
 260     llvm::sort(KV.second, [](const Symbol *LHS, const Symbol *RHS) {
 261       if (LHS->getOffset() != RHS->getOffset())
 262         return LHS->getOffset() < RHS->getOffset();
 263       if (LHS->getLinkage() != RHS->getLinkage())
 264         return LHS->getLinkage() < RHS->getLinkage();
 265       if (LHS->getScope() != RHS->getScope())
 266         return LHS->getScope() < RHS->getScope();
 267       if (LHS->hasName()) {
 268         if (!RHS->hasName())
 269           return true;
 270         return LHS->getName() < RHS->getName();
 271       }
 272       return false;
 273     });
 274
 275   for (auto &Sec : sections()) {
 276     OS << "section " << Sec.getName() << ":\n\n";
 277
 278     std::vector<Block *> SortedBlocks;
 279     llvm::copy(Sec.blocks(), std::back_inserter(SortedBlocks));
 280     llvm::sort(SortedBlocks, [](const Block *LHS, const Block *RHS) {
 281       return LHS->getAddress() < RHS->getAddress();
 282     });
 283
 284     for (auto *B : SortedBlocks) {
 285       OS << "  block " << B->getAddress()
 286          << " size = " << formatv("{0:x8}", B->getSize())
 287          << ", align = " << B->getAlignment()
 288          << ", alignment-offset = " << B->getAlignmentOffset();
 289       if (B->isZeroFill())
 290         OS << ", zero-fill";
 291       OS << "\n";
 292
 293       auto BlockSymsI = BlockSymbols.find(B);
 294       if (BlockSymsI != BlockSymbols.end()) {
 295         OS << "    symbols:\n";
 296         auto &Syms = BlockSymsI->second;
 297         for (auto *Sym : Syms)
 298           OS << "      " << *Sym << "\n";
 299       } else
 300         OS << "    no symbols\n";
 301
 302       if (!B->edges_empty()) {
 303         OS << "    edges:\n";
 304         std::vector<Edge> SortedEdges;
 305         llvm::copy(B->edges(), std::back_inserter(SortedEdges));
 306         llvm::sort(SortedEdges, [](const Edge &LHS, const Edge &RHS) {
 307           return LHS.getOffset() < RHS.getOffset();
 308         });
 309         for (auto &E : SortedEdges) {
 310           OS << "      " << B->getFixupAddress(E) << " (block + "
 311              << formatv("{0:x8}", E.getOffset()) << "), addend = ";
 312           if (E.getAddend() >= 0)
 313             OS << formatv("+{0:x8}", E.getAddend());
 314           else
 315             OS << formatv("-{0:x8}", -E.getAddend());
 316           OS << ", kind = " << getEdgeKindName(E.getKind()) << ", target = ";
 317           if (E.getTarget().hasName())
 318             OS << E.getTarget().getName();
 319           else
 320             OS << "addressable@"
 321                << formatv("{0:x16}", E.getTarget().getAddress()) << "+"
 322                << formatv("{0:x8}", E.getTarget().getOffset());
 323           OS << "\n";
 324         }
 325       } else
 326         OS << "    no edges\n";
 327       OS << "\n";
 328     }
 329   }
 330
 331   OS << "Absolute symbols:\n";
 332   if (!absolute_symbols().empty()) {
 333     for (auto *Sym : absolute_symbols())
 334       OS << "  " << Sym->getAddress() << ": " << *Sym << "\n";
 335   } else
 336     OS << "  none\n";
 337
 338   OS << "\nExternal symbols:\n";
 339   if (!external_symbols().empty()) {
 340     for (auto *Sym : external_symbols())
 341       OS << "  " << Sym->getAddress() << ": " << *Sym << "\n";
 342   } else
 343     OS << "  none\n";
 344 }
 345
 346 raw_ostream &operator<<(raw_ostream &OS, const SymbolLookupFlags &LF) {
 347   switch (LF) {
 348   case SymbolLookupFlags::RequiredSymbol:
 349     return OS << "RequiredSymbol";
 350   case SymbolLookupFlags::WeaklyReferencedSymbol:
 351     return OS << "WeaklyReferencedSymbol";
 352   }
 353   llvm_unreachable("Unrecognized lookup flags");
 354 }
 355
 356 void JITLinkAsyncLookupContinuation::anchor() {}
 357
 358 JITLinkContext::~JITLinkContext() = default;
 359
 360 bool JITLinkContext::shouldAddDefaultTargetPasses(const Triple &TT) const {
 361   return true;
 362 }
 363
 364 LinkGraphPassFunction JITLinkContext::getMarkLivePass(const Triple &TT) const {
 365   return LinkGraphPassFunction();
 366 }
 367
 368 Error JITLinkContext::modifyPassConfig(LinkGraph &G,
 369                                        PassConfiguration &Config) {
 370   return Error::success();
 371 }
 372
 373 Error markAllSymbolsLive(LinkGraph &G) {
 374   for (auto *Sym : G.defined_symbols())
 375     Sym->setLive(true);
 376   return Error::success();
 377 }
 378
 379 Error makeTargetOutOfRangeError(const LinkGraph &G, const Block &B,
 380                                 const Edge &E) {
 381   std::string ErrMsg;
 382   {
 383     raw_string_ostream ErrStream(ErrMsg);
 384     Section &Sec = B.getSection();
 385     ErrStream << "In graph " << G.getName() << ", section " << Sec.getName()
 386               << ": relocation target ";
 387     if (E.getTarget().hasName()) {
 388       ErrStream << "\"" << E.getTarget().getName() << "\"";
 389     } else
 390       ErrStream << E.getTarget().getBlock().getSection().getName() << " + "
 391                 << formatv("{0:x}", E.getOffset());
 392     ErrStream << " at address " << formatv("{0:x}", E.getTarget().getAddress())
 393               << " is out of range of " << G.getEdgeKindName(E.getKind())
 394               << " fixup at " << formatv("{0:x}", B.getFixupAddress(E)) << " (";
 395
 396     Symbol *BestSymbolForBlock = nullptr;
 397     for (auto *Sym : Sec.symbols())
 398       if (&Sym->getBlock() == &B && Sym->hasName() && Sym->getOffset() == 0 &&
 399           (!BestSymbolForBlock ||
 400            Sym->getScope() < BestSymbolForBlock->getScope() ||
 401            Sym->getLinkage() < BestSymbolForBlock->getLinkage()))
 402         BestSymbolForBlock = Sym;
 403
 404     if (BestSymbolForBlock)
 405       ErrStream << BestSymbolForBlock->getName() << ", ";
 406     else
 407       ErrStream << "<anonymous block> @ ";
 408
 409     ErrStream << formatv("{0:x}", B.getAddress()) << " + "
 410               << formatv("{0:x}", E.getOffset()) << ")";
 411   }
 412   return make_error<JITLinkError>(std::move(ErrMsg));
 413 }
 414
 415 Error makeAlignmentError(llvm::orc::ExecutorAddr Loc, uint64_t Value, int N,
 416                          const Edge &E) {
 417   return make_error<JITLinkError>("0x" + llvm::utohexstr(Loc.getValue()) +
 418                                   " improper alignment for relocation " +
 419                                   formatv("{0:d}", E.getKind()) + ": 0x" +
 420                                   llvm::utohexstr(Value) +
 421                                   " is not aligned to " + Twine(N) + " bytes");
 422 }
 423
 424 AnonymousPointerCreator getAnonymousPointerCreator(const Triple &TT) {
 425   switch (TT.getArch()) {
 426   case Triple::aarch64:
 427     return aarch64::createAnonymousPointer;
 428   case Triple::x86_64:
 429     return x86_64::createAnonymousPointer;
 430   case Triple::x86:
 431     return i386::createAnonymousPointer;
 432   case Triple::loongarch32:
 433   case Triple::loongarch64:
 434     return loongarch::createAnonymousPointer;
 435   default:
 436     return nullptr;
 437   }
 438 }
 439
 440 PointerJumpStubCreator getPointerJumpStubCreator(const Triple &TT) {
 441   switch (TT.getArch()) {
 442   case Triple::aarch64:
 443     return aarch64::createAnonymousPointerJumpStub;
 444   case Triple::x86_64:
 445     return x86_64::createAnonymousPointerJumpStub;
 446   case Triple::x86:
 447     return i386::createAnonymousPointerJumpStub;
 448   case Triple::loongarch32:
 449   case Triple::loongarch64:
 450     return loongarch::createAnonymousPointerJumpStub;
 451   default:
 452     return nullptr;
 453   }
 454 }
 455
 456 Expected<std::unique_ptr<LinkGraph>>
 457 createLinkGraphFromObject(MemoryBufferRef ObjectBuffer) {
 458   auto Magic = identify_magic(ObjectBuffer.getBuffer());
 459   switch (Magic) {
 460   case file_magic::macho_object:
 461     return createLinkGraphFromMachOObject(ObjectBuffer);
 462   case file_magic::elf_relocatable:
 463     return createLinkGraphFromELFObject(ObjectBuffer);
 464   case file_magic::coff_object:
 465     return createLinkGraphFromCOFFObject(ObjectBuffer);
 466   default:
 467     return make_error<JITLinkError>("Unsupported file format");
 468   };
 469 }
 470
 471 std::unique_ptr<LinkGraph> absoluteSymbolsLinkGraph(const Triple &TT,
 472                                                     orc::SymbolMap Symbols) {
 473   unsigned PointerSize;
 474   endianness Endianness =
 475       TT.isLittleEndian() ? endianness::little : endianness::big;
 476   switch (TT.getArch()) {
 477   case Triple::aarch64:
 478   case llvm::Triple::riscv64:
 479   case Triple::x86_64:
 480     PointerSize = 8;
 481     break;
 482   case llvm::Triple::arm:
 483   case llvm::Triple::riscv32:
 484   case llvm::Triple::x86:
 485     PointerSize = 4;
 486     break;
 487   default:
 488     llvm::report_fatal_error("unhandled target architecture");
 489   }
 490
 491   static std::atomic<uint64_t> Counter = {0};
 492   auto Index = Counter.fetch_add(1, std::memory_order_relaxed);
 493   auto G = std::make_unique<LinkGraph>(
 494       "<Absolute Symbols " + std::to_string(Index) + ">", TT, PointerSize,
 495       Endianness, /*GetEdgeKindName=*/nullptr);
 496   for (auto &[Name, Def] : Symbols) {
 497     auto &Sym =
 498         G->addAbsoluteSymbol(*Name, Def.getAddress(), /*Size=*/0,
 499                              Linkage::Strong, Scope::Default, /*IsLive=*/true);
 500     Sym.setCallable(Def.getFlags().isCallable());
 501   }
 502
 503   return G;
 504 }
 505
 506 void link(std::unique_ptr<LinkGraph> G, std::unique_ptr<JITLinkContext> Ctx) {
 507   switch (G->getTargetTriple().getObjectFormat()) {
 508   case Triple::MachO:
 509     return link_MachO(std::move(G), std::move(Ctx));
 510   case Triple::ELF:
 511     return link_ELF(std::move(G), std::move(Ctx));
 512   case Triple::COFF:
 513     return link_COFF(std::move(G), std::move(Ctx));
 514   default:
 515     Ctx->notifyFailed(make_error<JITLinkError>("Unsupported object format"));
 516   };
 517 }
 518
 519 } // end namespace jitlink
 520 } // end namespace llvm