llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp

   1 //===---- ELF_x86_64.cpp -JIT linker implementation for ELF/x86-64 ----===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // ELF/x86-64 jit-link implementation.
  10 //
  11 //===----------------------------------------------------------------------===//
  12
  13 #include "llvm/ExecutionEngine/JITLink/ELF_x86_64.h"
  14 #include "llvm/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.h"
  15 #include "llvm/ExecutionEngine/JITLink/JITLink.h"
  16 #include "llvm/ExecutionEngine/JITLink/TableManager.h"
  17 #include "llvm/ExecutionEngine/JITLink/x86_64.h"
  18 #include "llvm/Object/ELFObjectFile.h"
  19
  20 #include "DefineExternalSectionStartAndEndSymbols.h"
  21 #include "EHFrameSupportImpl.h"
  22 #include "ELFLinkGraphBuilder.h"
  23 #include "JITLinkGeneric.h"
  24
  25 #define DEBUG_TYPE "jitlink"
  26
  27 using namespace llvm;
  28 using namespace llvm::jitlink;
  29
  30 namespace {
  31
  32 constexpr StringRef ELFGOTSymbolName = "_GLOBAL_OFFSET_TABLE_";
  33 constexpr StringRef ELFTLSInfoSectionName = "$__TLSINFO";
  34
  35 class TLSInfoTableManager_ELF_x86_64
  36     : public TableManager<TLSInfoTableManager_ELF_x86_64> {
  37 public:
  38   static const uint8_t TLSInfoEntryContent[16];
  39
  40   static StringRef getSectionName() { return ELFTLSInfoSectionName; }
  41
  42   bool visitEdge(LinkGraph &G, Block *B, Edge &E) {
  43     if (E.getKind() == x86_64::RequestTLSDescInGOTAndTransformToDelta32) {
  44       LLVM_DEBUG({
  45         dbgs() << "  Fixing " << G.getEdgeKindName(E.getKind()) << " edge at "
  46                << formatv("{0:x}", B->getFixupAddress(E)) << " ("
  47                << formatv("{0:x}", B->getAddress()) << " + "
  48                << formatv("{0:x}", E.getOffset()) << ")\n";
  49       });
  50       E.setKind(x86_64::Delta32);
  51       E.setTarget(getEntryForTarget(G, E.getTarget()));
  52       return true;
  53     }
  54     return false;
  55   }
  56
  57   Symbol &createEntry(LinkGraph &G, Symbol &Target) {
  58     // the TLS Info entry's key value will be written by the fixTLVSectionByName
  59     // pass, so create mutable content.
  60     auto &TLSInfoEntry = G.createMutableContentBlock(
  61         getTLSInfoSection(G), G.allocateContent(getTLSInfoEntryContent()),
  62         orc::ExecutorAddr(), 8, 0);
  63     TLSInfoEntry.addEdge(x86_64::Pointer64, 8, Target, 0);
  64     return G.addAnonymousSymbol(TLSInfoEntry, 0, 16, false, false);
  65   }
  66
  67 private:
  68   Section &getTLSInfoSection(LinkGraph &G) {
  69     if (!TLSInfoTable)
  70       TLSInfoTable =
  71           &G.createSection(ELFTLSInfoSectionName, orc::MemProt::Read);
  72     return *TLSInfoTable;
  73   }
  74
  75   ArrayRef<char> getTLSInfoEntryContent() const {
  76     return {reinterpret_cast<const char *>(TLSInfoEntryContent),
  77             sizeof(TLSInfoEntryContent)};
  78   }
  79
  80   Section *TLSInfoTable = nullptr;
  81 };
  82
  83 const uint8_t TLSInfoTableManager_ELF_x86_64::TLSInfoEntryContent[16] = {
  84     0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /*pthread key */
  85     0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00  /*data address*/
  86 };
  87
  88 Error buildTables_ELF_x86_64(LinkGraph &G) {
  89   LLVM_DEBUG(dbgs() << "Visiting edges in graph:\n");
  90
  91   x86_64::GOTTableManager GOT;
  92   x86_64::PLTTableManager PLT(GOT);
  93   TLSInfoTableManager_ELF_x86_64 TLSInfo;
  94   visitExistingEdges(G, GOT, PLT, TLSInfo);
  95   return Error::success();
  96 }
  97 } // namespace
  98
  99 namespace llvm {
 100 namespace jitlink {
 101
 102 class ELFLinkGraphBuilder_x86_64 : public ELFLinkGraphBuilder<object::ELF64LE> {
 103 private:
 104   using ELFT = object::ELF64LE;
 105
 106   Error addRelocations() override {
 107     LLVM_DEBUG(dbgs() << "Processing relocations:\n");
 108
 109     using Base = ELFLinkGraphBuilder<ELFT>;
 110     using Self = ELFLinkGraphBuilder_x86_64;
 111     for (const auto &RelSect : Base::Sections) {
 112       // Validate the section to read relocation entries from.
 113       if (RelSect.sh_type == ELF::SHT_REL)
 114         return make_error<StringError>(
 115             "No SHT_REL in valid x64 ELF object files",
 116             inconvertibleErrorCode());
 117
 118       if (Error Err = Base::forEachRelaRelocation(RelSect, this,
 119                                                   &Self::addSingleRelocation))
 120         return Err;
 121     }
 122
 123     return Error::success();
 124   }
 125
 126   Error addSingleRelocation(const typename ELFT::Rela &Rel,
 127                             const typename ELFT::Shdr &FixupSection,
 128                             Block &BlockToFix) {
 129     using Base = ELFLinkGraphBuilder<ELFT>;
 130
 131     auto ELFReloc = Rel.getType(false);
 132
 133     // R_X86_64_NONE is a no-op.
 134     if (LLVM_UNLIKELY(ELFReloc == ELF::R_X86_64_NONE))
 135       return Error::success();
 136
 137     uint32_t SymbolIndex = Rel.getSymbol(false);
 138     auto ObjSymbol = Base::Obj.getRelocationSymbol(Rel, Base::SymTabSec);
 139     if (!ObjSymbol)
 140       return ObjSymbol.takeError();
 141
 142     Symbol *GraphSymbol = Base::getGraphSymbol(SymbolIndex);
 143     if (!GraphSymbol)
 144       return make_error<StringError>(
 145           formatv("Could not find symbol at given index, did you add it to "
 146                   "JITSymbolTable? index: {0}, shndx: {1} Size of table: {2}",
 147                   SymbolIndex, (*ObjSymbol)->st_shndx,
 148                   Base::GraphSymbols.size()),
 149           inconvertibleErrorCode());
 150
 151     // Validate the relocation kind.
 152     int64_t Addend = Rel.r_addend;
 153     Edge::Kind Kind = Edge::Invalid;
 154
 155     switch (ELFReloc) {
 156     case ELF::R_X86_64_PC32:
 157     case ELF::R_X86_64_GOTPC32:
 158       Kind = x86_64::Delta32;
 159       break;
 160     case ELF::R_X86_64_PC64:
 161     case ELF::R_X86_64_GOTPC64:
 162       Kind = x86_64::Delta64;
 163       break;
 164     case ELF::R_X86_64_32:
 165       Kind = x86_64::Pointer32;
 166       break;
 167     case ELF::R_X86_64_16:
 168       Kind = x86_64::Pointer16;
 169       break;
 170     case ELF::R_X86_64_8:
 171       Kind = x86_64::Pointer8;
 172       break;
 173     case ELF::R_X86_64_32S:
 174       Kind = x86_64::Pointer32Signed;
 175       break;
 176     case ELF::R_X86_64_64:
 177       Kind = x86_64::Pointer64;
 178       break;
 179     case ELF::R_X86_64_GOTPCREL:
 180       Kind = x86_64::RequestGOTAndTransformToDelta32;
 181       break;
 182     case ELF::R_X86_64_REX_GOTPCRELX:
 183       Kind = x86_64::RequestGOTAndTransformToPCRel32GOTLoadREXRelaxable;
 184       Addend = 0;
 185       break;
 186     case ELF::R_X86_64_TLSGD:
 187       Kind = x86_64::RequestTLSDescInGOTAndTransformToDelta32;
 188       break;
 189     case ELF::R_X86_64_GOTPCRELX:
 190       Kind = x86_64::RequestGOTAndTransformToPCRel32GOTLoadRelaxable;
 191       Addend = 0;
 192       break;
 193     case ELF::R_X86_64_GOTPCREL64:
 194       Kind = x86_64::RequestGOTAndTransformToDelta64;
 195       break;
 196     case ELF::R_X86_64_GOT64:
 197       Kind = x86_64::RequestGOTAndTransformToDelta64FromGOT;
 198       break;
 199     case ELF::R_X86_64_GOTOFF64:
 200       Kind = x86_64::Delta64FromGOT;
 201       break;
 202     case ELF::R_X86_64_PLT32:
 203       Kind = x86_64::BranchPCRel32;
 204       // BranchPCRel32 implicitly handles the '-4' PC adjustment, so we have to
 205       // adjust the addend by '+4' to compensate.
 206       Addend += 4;
 207       break;
 208     default:
 209       return make_error<JITLinkError>(
 210           "In " + G->getName() + ": Unsupported x86-64 relocation type " +
 211           object::getELFRelocationTypeName(ELF::EM_X86_64, ELFReloc));
 212     }
 213
 214     auto FixupAddress = orc::ExecutorAddr(FixupSection.sh_addr) + Rel.r_offset;
 215     Edge::OffsetT Offset = FixupAddress - BlockToFix.getAddress();
 216     Edge GE(Kind, Offset, *GraphSymbol, Addend);
 217     LLVM_DEBUG({
 218       dbgs() << "    ";
 219       printEdge(dbgs(), BlockToFix, GE, x86_64::getEdgeKindName(Kind));
 220       dbgs() << "\n";
 221     });
 222
 223     BlockToFix.addEdge(std::move(GE));
 224     return Error::success();
 225   }
 226
 227 public:
 228   ELFLinkGraphBuilder_x86_64(StringRef FileName,
 229                              const object::ELFFile<object::ELF64LE> &Obj,
 230                              SubtargetFeatures Features)
 231       : ELFLinkGraphBuilder(Obj, Triple("x86_64-unknown-linux"),
 232                             std::move(Features), FileName,
 233                             x86_64::getEdgeKindName) {}
 234 };
 235
 236 class ELFJITLinker_x86_64 : public JITLinker<ELFJITLinker_x86_64> {
 237   friend class JITLinker<ELFJITLinker_x86_64>;
 238
 239 public:
 240   ELFJITLinker_x86_64(std::unique_ptr<JITLinkContext> Ctx,
 241                       std::unique_ptr<LinkGraph> G,
 242                       PassConfiguration PassConfig)
 243       : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) {
 244
 245     if (shouldAddDefaultTargetPasses(getGraph().getTargetTriple()))
 246       getPassConfig().PostAllocationPasses.push_back(
 247           [this](LinkGraph &G) { return getOrCreateGOTSymbol(G); });
 248   }
 249
 250 private:
 251   Symbol *GOTSymbol = nullptr;
 252
 253   Error getOrCreateGOTSymbol(LinkGraph &G) {
 254     auto DefineExternalGOTSymbolIfPresent =
 255         createDefineExternalSectionStartAndEndSymbolsPass(
 256             [&](LinkGraph &LG, Symbol &Sym) -> SectionRangeSymbolDesc {
 257               if (Sym.getName() == ELFGOTSymbolName)
 258                 if (auto *GOTSection = G.findSectionByName(
 259                         x86_64::GOTTableManager::getSectionName())) {
 260                   GOTSymbol = &Sym;
 261                   return {*GOTSection, true};
 262                 }
 263               return {};
 264             });
 265
 266     // Try to attach _GLOBAL_OFFSET_TABLE_ to the GOT if it's defined as an
 267     // external.
 268     if (auto Err = DefineExternalGOTSymbolIfPresent(G))
 269       return Err;
 270
 271     // If we succeeded then we're done.
 272     if (GOTSymbol)
 273       return Error::success();
 274
 275     // Otherwise look for a GOT section: If it already has a start symbol we'll
 276     // record it, otherwise we'll create our own.
 277     // If there's a GOT section but we didn't find an external GOT symbol...
 278     if (auto *GOTSection =
 279             G.findSectionByName(x86_64::GOTTableManager::getSectionName())) {
 280
 281       // Check for an existing defined symbol.
 282       for (auto *Sym : GOTSection->symbols())
 283         if (Sym->getName() == ELFGOTSymbolName) {
 284           GOTSymbol = Sym;
 285           return Error::success();
 286         }
 287
 288       // If there's no defined symbol then create one.
 289       SectionRange SR(*GOTSection);
 290       if (SR.empty())
 291         GOTSymbol =
 292             &G.addAbsoluteSymbol(ELFGOTSymbolName, orc::ExecutorAddr(), 0,
 293                                  Linkage::Strong, Scope::Local, true);
 294       else
 295         GOTSymbol =
 296             &G.addDefinedSymbol(*SR.getFirstBlock(), 0, ELFGOTSymbolName, 0,
 297                                 Linkage::Strong, Scope::Local, false, true);
 298     }
 299
 300     // If we still haven't found a GOT symbol then double check the externals.
 301     // We may have a GOT-relative reference but no GOT section, in which case
 302     // we just need to point the GOT symbol at some address in this graph.
 303     if (!GOTSymbol) {
 304       for (auto *Sym : G.external_symbols()) {
 305         if (Sym->getName() == ELFGOTSymbolName) {
 306           auto Blocks = G.blocks();
 307           if (!Blocks.empty()) {
 308             G.makeAbsolute(*Sym, (*Blocks.begin())->getAddress());
 309             GOTSymbol = Sym;
 310             break;
 311           }
 312         }
 313       }
 314     }
 315
 316     return Error::success();
 317   }
 318
 319   Error applyFixup(LinkGraph &G, Block &B, const Edge &E) const {
 320     return x86_64::applyFixup(G, B, E, GOTSymbol);
 321   }
 322 };
 323
 324 Expected<std::unique_ptr<LinkGraph>>
 325 createLinkGraphFromELFObject_x86_64(MemoryBufferRef ObjectBuffer) {
 326   LLVM_DEBUG({
 327     dbgs() << "Building jitlink graph for new input "
 328            << ObjectBuffer.getBufferIdentifier() << "...\n";
 329   });
 330
 331   auto ELFObj = object::ObjectFile::createELFObjectFile(ObjectBuffer);
 332   if (!ELFObj)
 333     return ELFObj.takeError();
 334
 335   auto Features = (*ELFObj)->getFeatures();
 336   if (!Features)
 337     return Features.takeError();
 338
 339   auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF64LE>>(**ELFObj);
 340   return ELFLinkGraphBuilder_x86_64((*ELFObj)->getFileName(),
 341                                     ELFObjFile.getELFFile(),
 342                                     std::move(*Features))
 343       .buildGraph();
 344 }
 345
 346 static SectionRangeSymbolDesc
 347 identifyELFSectionStartAndEndSymbols(LinkGraph &G, Symbol &Sym) {
 348   constexpr StringRef StartSymbolPrefix = "__start";
 349   constexpr StringRef EndSymbolPrefix = "__end";
 350
 351   auto SymName = Sym.getName();
 352   if (SymName.starts_with(StartSymbolPrefix)) {
 353     if (auto *Sec =
 354             G.findSectionByName(SymName.drop_front(StartSymbolPrefix.size())))
 355       return {*Sec, true};
 356   } else if (SymName.starts_with(EndSymbolPrefix)) {
 357     if (auto *Sec =
 358             G.findSectionByName(SymName.drop_front(EndSymbolPrefix.size())))
 359       return {*Sec, false};
 360   }
 361   return {};
 362 }
 363
 364 void link_ELF_x86_64(std::unique_ptr<LinkGraph> G,
 365                      std::unique_ptr<JITLinkContext> Ctx) {
 366   PassConfiguration Config;
 367
 368   if (Ctx->shouldAddDefaultTargetPasses(G->getTargetTriple())) {
 369
 370     Config.PrePrunePasses.push_back(DWARFRecordSectionSplitter(".eh_frame"));
 371     Config.PrePrunePasses.push_back(EHFrameEdgeFixer(
 372         ".eh_frame", x86_64::PointerSize, x86_64::Pointer32, x86_64::Pointer64,
 373         x86_64::Delta32, x86_64::Delta64, x86_64::NegDelta32));
 374     Config.PrePrunePasses.push_back(EHFrameNullTerminator(".eh_frame"));
 375
 376     // Construct a JITLinker and run the link function.
 377     // Add a mark-live pass.
 378     if (auto MarkLive = Ctx->getMarkLivePass(G->getTargetTriple()))
 379       Config.PrePrunePasses.push_back(std::move(MarkLive));
 380     else
 381       Config.PrePrunePasses.push_back(markAllSymbolsLive);
 382
 383     // Add an in-place GOT/Stubs/TLSInfoEntry build pass.
 384     Config.PostPrunePasses.push_back(buildTables_ELF_x86_64);
 385
 386     // Resolve any external section start / end symbols.
 387     Config.PostAllocationPasses.push_back(
 388         createDefineExternalSectionStartAndEndSymbolsPass(
 389             identifyELFSectionStartAndEndSymbols));
 390
 391     // Add GOT/Stubs optimizer pass.
 392     Config.PreFixupPasses.push_back(x86_64::optimizeGOTAndStubAccesses);
 393   }
 394
 395   if (auto Err = Ctx->modifyPassConfig(*G, Config))
 396     return Ctx->notifyFailed(std::move(Err));
 397
 398   ELFJITLinker_x86_64::link(std::move(Ctx), std::move(G), std::move(Config));
 399 }
 400 } // end namespace jitlink
 401 } // end namespace llvm