llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp

   1 //===---- ELF_x86_64.cpp -JIT linker implementation for ELF/x86-64 ----===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // ELF/x86-64 jit-link implementation.
  10 //
  11 //===----------------------------------------------------------------------===//
  12
  13 #include "llvm/ExecutionEngine/JITLink/ELF_x86_64.h"
  14 #include "llvm/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.h"
  15 #include "llvm/ExecutionEngine/JITLink/JITLink.h"
  16 #include "llvm/ExecutionEngine/JITLink/TableManager.h"
  17 #include "llvm/ExecutionEngine/JITLink/x86_64.h"
  18 #include "llvm/Object/ELFObjectFile.h"
  19 #include "llvm/Support/Endian.h"
  20
  21 #include "DefineExternalSectionStartAndEndSymbols.h"
  22 #include "EHFrameSupportImpl.h"
  23 #include "ELFLinkGraphBuilder.h"
  24 #include "JITLinkGeneric.h"
  25
  26 #define DEBUG_TYPE "jitlink"
  27
  28 using namespace llvm;
  29 using namespace llvm::jitlink;
  30
  31 namespace {
  32
  33 constexpr StringRef ELFGOTSymbolName = "_GLOBAL_OFFSET_TABLE_";
  34 constexpr StringRef ELFTLSInfoSectionName = "$__TLSINFO";
  35
  36 class TLSInfoTableManager_ELF_x86_64
  37     : public TableManager<TLSInfoTableManager_ELF_x86_64> {
  38 public:
  39   static const uint8_t TLSInfoEntryContent[16];
  40
  41   static StringRef getSectionName() { return ELFTLSInfoSectionName; }
  42
  43   bool visitEdge(LinkGraph &G, Block *B, Edge &E) {
  44     if (E.getKind() == x86_64::RequestTLSDescInGOTAndTransformToDelta32) {
  45       LLVM_DEBUG({
  46         dbgs() << "  Fixing " << G.getEdgeKindName(E.getKind()) << " edge at "
  47                << formatv("{0:x}", B->getFixupAddress(E)) << " ("
  48                << formatv("{0:x}", B->getAddress()) << " + "
  49                << formatv("{0:x}", E.getOffset()) << ")\n";
  50       });
  51       E.setKind(x86_64::Delta32);
  52       E.setTarget(getEntryForTarget(G, E.getTarget()));
  53       return true;
  54     }
  55     return false;
  56   }
  57
  58   Symbol &createEntry(LinkGraph &G, Symbol &Target) {
  59     // the TLS Info entry's key value will be written by the fixTLVSectionByName
  60     // pass, so create mutable content.
  61     auto &TLSInfoEntry = G.createMutableContentBlock(
  62         getTLSInfoSection(G), G.allocateContent(getTLSInfoEntryContent()),
  63         orc::ExecutorAddr(), 8, 0);
  64     TLSInfoEntry.addEdge(x86_64::Pointer64, 8, Target, 0);
  65     return G.addAnonymousSymbol(TLSInfoEntry, 0, 16, false, false);
  66   }
  67
  68 private:
  69   Section &getTLSInfoSection(LinkGraph &G) {
  70     if (!TLSInfoTable)
  71       TLSInfoTable =
  72           &G.createSection(ELFTLSInfoSectionName, orc::MemProt::Read);
  73     return *TLSInfoTable;
  74   }
  75
  76   ArrayRef<char> getTLSInfoEntryContent() const {
  77     return {reinterpret_cast<const char *>(TLSInfoEntryContent),
  78             sizeof(TLSInfoEntryContent)};
  79   }
  80
  81   Section *TLSInfoTable = nullptr;
  82 };
  83
  84 const uint8_t TLSInfoTableManager_ELF_x86_64::TLSInfoEntryContent[16] = {
  85     0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /*pthread key */
  86     0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00  /*data address*/
  87 };
  88
  89 Error buildTables_ELF_x86_64(LinkGraph &G) {
  90   LLVM_DEBUG(dbgs() << "Visiting edges in graph:\n");
  91
  92   x86_64::GOTTableManager GOT;
  93   x86_64::PLTTableManager PLT(GOT);
  94   TLSInfoTableManager_ELF_x86_64 TLSInfo;
  95   visitExistingEdges(G, GOT, PLT, TLSInfo);
  96   return Error::success();
  97 }
  98 } // namespace
  99
 100 namespace llvm {
 101 namespace jitlink {
 102
 103 class ELFLinkGraphBuilder_x86_64 : public ELFLinkGraphBuilder<object::ELF64LE> {
 104 private:
 105   using ELFT = object::ELF64LE;
 106
 107   Error addRelocations() override {
 108     LLVM_DEBUG(dbgs() << "Processing relocations:\n");
 109
 110     using Base = ELFLinkGraphBuilder<ELFT>;
 111     using Self = ELFLinkGraphBuilder_x86_64;
 112     for (const auto &RelSect : Base::Sections) {
 113       // Validate the section to read relocation entries from.
 114       if (RelSect.sh_type == ELF::SHT_REL)
 115         return make_error<StringError>(
 116             "No SHT_REL in valid x64 ELF object files",
 117             inconvertibleErrorCode());
 118
 119       if (Error Err = Base::forEachRelaRelocation(RelSect, this,
 120                                                   &Self::addSingleRelocation))
 121         return Err;
 122     }
 123
 124     return Error::success();
 125   }
 126
 127   Error addSingleRelocation(const typename ELFT::Rela &Rel,
 128                             const typename ELFT::Shdr &FixupSection,
 129                             Block &BlockToFix) {
 130     using Base = ELFLinkGraphBuilder<ELFT>;
 131
 132     auto ELFReloc = Rel.getType(false);
 133
 134     // R_X86_64_NONE is a no-op.
 135     if (LLVM_UNLIKELY(ELFReloc == ELF::R_X86_64_NONE))
 136       return Error::success();
 137
 138     uint32_t SymbolIndex = Rel.getSymbol(false);
 139     auto ObjSymbol = Base::Obj.getRelocationSymbol(Rel, Base::SymTabSec);
 140     if (!ObjSymbol)
 141       return ObjSymbol.takeError();
 142
 143     Symbol *GraphSymbol = Base::getGraphSymbol(SymbolIndex);
 144     if (!GraphSymbol)
 145       return make_error<StringError>(
 146           formatv("Could not find symbol at given index, did you add it to "
 147                   "JITSymbolTable? index: {0}, shndx: {1} Size of table: {2}",
 148                   SymbolIndex, (*ObjSymbol)->st_shndx,
 149                   Base::GraphSymbols.size()),
 150           inconvertibleErrorCode());
 151
 152     // Validate the relocation kind.
 153     int64_t Addend = Rel.r_addend;
 154     Edge::Kind Kind = Edge::Invalid;
 155
 156     switch (ELFReloc) {
 157     case ELF::R_X86_64_PC32:
 158     case ELF::R_X86_64_GOTPC32:
 159       Kind = x86_64::Delta32;
 160       break;
 161     case ELF::R_X86_64_PC64:
 162     case ELF::R_X86_64_GOTPC64:
 163       Kind = x86_64::Delta64;
 164       break;
 165     case ELF::R_X86_64_32:
 166       Kind = x86_64::Pointer32;
 167       break;
 168     case ELF::R_X86_64_16:
 169       Kind = x86_64::Pointer16;
 170       break;
 171     case ELF::R_X86_64_8:
 172       Kind = x86_64::Pointer8;
 173       break;
 174     case ELF::R_X86_64_32S:
 175       Kind = x86_64::Pointer32Signed;
 176       break;
 177     case ELF::R_X86_64_64:
 178       Kind = x86_64::Pointer64;
 179       break;
 180     case ELF::R_X86_64_GOTPCREL:
 181       Kind = x86_64::RequestGOTAndTransformToDelta32;
 182       break;
 183     case ELF::R_X86_64_REX_GOTPCRELX:
 184       Kind = x86_64::RequestGOTAndTransformToPCRel32GOTLoadREXRelaxable;
 185       Addend = 0;
 186       break;
 187     case ELF::R_X86_64_TLSGD:
 188       Kind = x86_64::RequestTLSDescInGOTAndTransformToDelta32;
 189       break;
 190     case ELF::R_X86_64_GOTPCRELX:
 191       Kind = x86_64::RequestGOTAndTransformToPCRel32GOTLoadRelaxable;
 192       Addend = 0;
 193       break;
 194     case ELF::R_X86_64_GOTPCREL64:
 195       Kind = x86_64::RequestGOTAndTransformToDelta64;
 196       break;
 197     case ELF::R_X86_64_GOT64:
 198       Kind = x86_64::RequestGOTAndTransformToDelta64FromGOT;
 199       break;
 200     case ELF::R_X86_64_GOTOFF64:
 201       Kind = x86_64::Delta64FromGOT;
 202       break;
 203     case ELF::R_X86_64_PLT32:
 204       Kind = x86_64::BranchPCRel32;
 205       // BranchPCRel32 implicitly handles the '-4' PC adjustment, so we have to
 206       // adjust the addend by '+4' to compensate.
 207       Addend += 4;
 208       break;
 209     default:
 210       return make_error<JITLinkError>(
 211           "In " + G->getName() + ": Unsupported x86-64 relocation type " +
 212           object::getELFRelocationTypeName(ELF::EM_X86_64, ELFReloc));
 213     }
 214
 215     auto FixupAddress = orc::ExecutorAddr(FixupSection.sh_addr) + Rel.r_offset;
 216     Edge::OffsetT Offset = FixupAddress - BlockToFix.getAddress();
 217     Edge GE(Kind, Offset, *GraphSymbol, Addend);
 218     LLVM_DEBUG({
 219       dbgs() << "    ";
 220       printEdge(dbgs(), BlockToFix, GE, x86_64::getEdgeKindName(Kind));
 221       dbgs() << "\n";
 222     });
 223
 224     BlockToFix.addEdge(std::move(GE));
 225     return Error::success();
 226   }
 227
 228 public:
 229   ELFLinkGraphBuilder_x86_64(StringRef FileName,
 230                              const object::ELFFile<object::ELF64LE> &Obj,
 231                              SubtargetFeatures Features)
 232       : ELFLinkGraphBuilder(Obj, Triple("x86_64-unknown-linux"),
 233                             std::move(Features), FileName,
 234                             x86_64::getEdgeKindName) {}
 235 };
 236
 237 class ELFJITLinker_x86_64 : public JITLinker<ELFJITLinker_x86_64> {
 238   friend class JITLinker<ELFJITLinker_x86_64>;
 239
 240 public:
 241   ELFJITLinker_x86_64(std::unique_ptr<JITLinkContext> Ctx,
 242                       std::unique_ptr<LinkGraph> G,
 243                       PassConfiguration PassConfig)
 244       : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) {
 245
 246     if (shouldAddDefaultTargetPasses(getGraph().getTargetTriple()))
 247       getPassConfig().PostAllocationPasses.push_back(
 248           [this](LinkGraph &G) { return getOrCreateGOTSymbol(G); });
 249   }
 250
 251 private:
 252   Symbol *GOTSymbol = nullptr;
 253
 254   Error getOrCreateGOTSymbol(LinkGraph &G) {
 255     auto DefineExternalGOTSymbolIfPresent =
 256         createDefineExternalSectionStartAndEndSymbolsPass(
 257             [&](LinkGraph &LG, Symbol &Sym) -> SectionRangeSymbolDesc {
 258               if (Sym.getName() == ELFGOTSymbolName)
 259                 if (auto *GOTSection = G.findSectionByName(
 260                         x86_64::GOTTableManager::getSectionName())) {
 261                   GOTSymbol = &Sym;
 262                   return {*GOTSection, true};
 263                 }
 264               return {};
 265             });
 266
 267     // Try to attach _GLOBAL_OFFSET_TABLE_ to the GOT if it's defined as an
 268     // external.
 269     if (auto Err = DefineExternalGOTSymbolIfPresent(G))
 270       return Err;
 271
 272     // If we succeeded then we're done.
 273     if (GOTSymbol)
 274       return Error::success();
 275
 276     // Otherwise look for a GOT section: If it already has a start symbol we'll
 277     // record it, otherwise we'll create our own.
 278     // If there's a GOT section but we didn't find an external GOT symbol...
 279     if (auto *GOTSection =
 280             G.findSectionByName(x86_64::GOTTableManager::getSectionName())) {
 281
 282       // Check for an existing defined symbol.
 283       for (auto *Sym : GOTSection->symbols())
 284         if (Sym->getName() == ELFGOTSymbolName) {
 285           GOTSymbol = Sym;
 286           return Error::success();
 287         }
 288
 289       // If there's no defined symbol then create one.
 290       SectionRange SR(*GOTSection);
 291       if (SR.empty())
 292         GOTSymbol =
 293             &G.addAbsoluteSymbol(ELFGOTSymbolName, orc::ExecutorAddr(), 0,
 294                                  Linkage::Strong, Scope::Local, true);
 295       else
 296         GOTSymbol =
 297             &G.addDefinedSymbol(*SR.getFirstBlock(), 0, ELFGOTSymbolName, 0,
 298                                 Linkage::Strong, Scope::Local, false, true);
 299     }
 300
 301     // If we still haven't found a GOT symbol then double check the externals.
 302     // We may have a GOT-relative reference but no GOT section, in which case
 303     // we just need to point the GOT symbol at some address in this graph.
 304     if (!GOTSymbol) {
 305       for (auto *Sym : G.external_symbols()) {
 306         if (Sym->getName() == ELFGOTSymbolName) {
 307           auto Blocks = G.blocks();
 308           if (!Blocks.empty()) {
 309             G.makeAbsolute(*Sym, (*Blocks.begin())->getAddress());
 310             GOTSymbol = Sym;
 311             break;
 312           }
 313         }
 314       }
 315     }
 316
 317     return Error::success();
 318   }
 319
 320   Error applyFixup(LinkGraph &G, Block &B, const Edge &E) const {
 321     return x86_64::applyFixup(G, B, E, GOTSymbol);
 322   }
 323 };
 324
 325 Expected<std::unique_ptr<LinkGraph>>
 326 createLinkGraphFromELFObject_x86_64(MemoryBufferRef ObjectBuffer) {
 327   LLVM_DEBUG({
 328     dbgs() << "Building jitlink graph for new input "
 329            << ObjectBuffer.getBufferIdentifier() << "...\n";
 330   });
 331
 332   auto ELFObj = object::ObjectFile::createELFObjectFile(ObjectBuffer);
 333   if (!ELFObj)
 334     return ELFObj.takeError();
 335
 336   auto Features = (*ELFObj)->getFeatures();
 337   if (!Features)
 338     return Features.takeError();
 339
 340   auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF64LE>>(**ELFObj);
 341   return ELFLinkGraphBuilder_x86_64((*ELFObj)->getFileName(),
 342                                     ELFObjFile.getELFFile(),
 343                                     std::move(*Features))
 344       .buildGraph();
 345 }
 346
 347 static SectionRangeSymbolDesc
 348 identifyELFSectionStartAndEndSymbols(LinkGraph &G, Symbol &Sym) {
 349   constexpr StringRef StartSymbolPrefix = "__start";
 350   constexpr StringRef EndSymbolPrefix = "__end";
 351
 352   auto SymName = Sym.getName();
 353   if (SymName.startswith(StartSymbolPrefix)) {
 354     if (auto *Sec =
 355             G.findSectionByName(SymName.drop_front(StartSymbolPrefix.size())))
 356       return {*Sec, true};
 357   } else if (SymName.startswith(EndSymbolPrefix)) {
 358     if (auto *Sec =
 359             G.findSectionByName(SymName.drop_front(EndSymbolPrefix.size())))
 360       return {*Sec, false};
 361   }
 362   return {};
 363 }
 364
 365 void link_ELF_x86_64(std::unique_ptr<LinkGraph> G,
 366                      std::unique_ptr<JITLinkContext> Ctx) {
 367   PassConfiguration Config;
 368
 369   if (Ctx->shouldAddDefaultTargetPasses(G->getTargetTriple())) {
 370
 371     Config.PrePrunePasses.push_back(DWARFRecordSectionSplitter(".eh_frame"));
 372     Config.PrePrunePasses.push_back(EHFrameEdgeFixer(
 373         ".eh_frame", x86_64::PointerSize, x86_64::Pointer32, x86_64::Pointer64,
 374         x86_64::Delta32, x86_64::Delta64, x86_64::NegDelta32));
 375     Config.PrePrunePasses.push_back(EHFrameNullTerminator(".eh_frame"));
 376
 377     // Construct a JITLinker and run the link function.
 378     // Add a mark-live pass.
 379     if (auto MarkLive = Ctx->getMarkLivePass(G->getTargetTriple()))
 380       Config.PrePrunePasses.push_back(std::move(MarkLive));
 381     else
 382       Config.PrePrunePasses.push_back(markAllSymbolsLive);
 383
 384     // Add an in-place GOT/Stubs/TLSInfoEntry build pass.
 385     Config.PostPrunePasses.push_back(buildTables_ELF_x86_64);
 386
 387     // Resolve any external section start / end symbols.
 388     Config.PostAllocationPasses.push_back(
 389         createDefineExternalSectionStartAndEndSymbolsPass(
 390             identifyELFSectionStartAndEndSymbols));
 391
 392     // Add GOT/Stubs optimizer pass.
 393     Config.PreFixupPasses.push_back(x86_64::optimizeGOTAndStubAccesses);
 394   }
 395
 396   if (auto Err = Ctx->modifyPassConfig(*G, Config))
 397     return Ctx->notifyFailed(std::move(Err));
 398
 399   ELFJITLinker_x86_64::link(std::move(Ctx), std::move(G), std::move(Config));
 400 }
 401 } // end namespace jitlink
 402 } // end namespace llvm