llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp

   1 //===---- ELF_x86_64.cpp -JIT linker implementation for ELF/x86-64 ----===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // ELF/x86-64 jit-link implementation.
  10 //
  11 //===----------------------------------------------------------------------===//
  12
  13 #include "llvm/ExecutionEngine/JITLink/ELF_x86_64.h"
  14 #include "llvm/ExecutionEngine/JITLink/JITLink.h"
  15 #include "llvm/ExecutionEngine/JITLink/x86_64.h"
  16 #include "llvm/Object/ELFObjectFile.h"
  17 #include "llvm/Support/Endian.h"
  18
  19 #include "DefineExternalSectionStartAndEndSymbols.h"
  20 #include "EHFrameSupportImpl.h"
  21 #include "ELFLinkGraphBuilder.h"
  22 #include "JITLinkGeneric.h"
  23 #include "PerGraphGOTAndPLTStubsBuilder.h"
  24
  25 #define DEBUG_TYPE "jitlink"
  26
  27 using namespace llvm;
  28 using namespace llvm::jitlink;
  29 using namespace llvm::jitlink::ELF_x86_64_Edges;
  30
  31 namespace {
  32
  33 constexpr StringRef ELFGOTSectionName = "$__GOT";
  34 constexpr StringRef ELFGOTSymbolName = "_GLOBAL_OFFSET_TABLE_";
  35
  36 class PerGraphGOTAndPLTStubsBuilder_ELF_x86_64
  37     : public PerGraphGOTAndPLTStubsBuilder<
  38           PerGraphGOTAndPLTStubsBuilder_ELF_x86_64> {
  39 public:
  40   static const uint8_t NullGOTEntryContent[8];
  41   static const uint8_t StubContent[6];
  42
  43   using PerGraphGOTAndPLTStubsBuilder<
  44       PerGraphGOTAndPLTStubsBuilder_ELF_x86_64>::PerGraphGOTAndPLTStubsBuilder;
  45
  46   bool isGOTEdgeToFix(Edge &E) const {
  47     if (E.getKind() == x86_64::Delta64FromGOT) {
  48       // We need to make sure that the GOT section exists, but don't otherwise
  49       // need to fix up this edge.
  50       getGOTSection();
  51       return false;
  52     }
  53     return E.getKind() == x86_64::RequestGOTAndTransformToDelta32 ||
  54            E.getKind() == x86_64::RequestGOTAndTransformToDelta64 ||
  55            E.getKind() ==
  56                x86_64::RequestGOTAndTransformToPCRel32GOTLoadREXRelaxable ||
  57            E.getKind() == x86_64::RequestGOTAndTransformToDelta64FromGOT ||
  58            E.getKind() ==
  59                x86_64::RequestGOTAndTransformToPCRel32GOTLoadRelaxable;
  60   }
  61
  62   Symbol &createGOTEntry(Symbol &Target) {
  63     auto &GOTEntryBlock = G.createContentBlock(
  64         getGOTSection(), getGOTEntryBlockContent(), 0, 8, 0);
  65     GOTEntryBlock.addEdge(x86_64::Pointer64, 0, Target, 0);
  66     return G.addAnonymousSymbol(GOTEntryBlock, 0, 8, false, false);
  67   }
  68
  69   void fixGOTEdge(Edge &E, Symbol &GOTEntry) {
  70     // If this is a PCRel32GOT/PCRel64GOT then change it to an ordinary
  71     // PCRel32/PCRel64. If it is a PCRel32GOTLoad then leave it as-is for now:
  72     // We will use the kind to check for GOT optimization opportunities in the
  73     // optimizeMachO_x86_64_GOTAndStubs pass below.
  74     // If it's a GOT64 leave it as is.
  75     switch (E.getKind()) {
  76     case x86_64::RequestGOTAndTransformToPCRel32GOTLoadREXRelaxable:
  77       E.setKind(x86_64::PCRel32GOTLoadREXRelaxable);
  78       break;
  79     case x86_64::RequestGOTAndTransformToPCRel32GOTLoadRelaxable:
  80       E.setKind(x86_64::PCRel32GOTLoadRelaxable);
  81       break;
  82     case x86_64::RequestGOTAndTransformToDelta64:
  83       E.setKind(x86_64::Delta64);
  84       break;
  85     case x86_64::RequestGOTAndTransformToDelta64FromGOT:
  86       E.setKind(x86_64::Delta64FromGOT);
  87       break;
  88     case x86_64::RequestGOTAndTransformToDelta32:
  89       E.setKind(x86_64::Delta32);
  90       break;
  91     default:
  92       llvm_unreachable("Unexpected GOT edge kind");
  93     }
  94
  95     E.setTarget(GOTEntry);
  96     // Leave the edge addend as-is.
  97   }
  98
  99   bool isExternalBranchEdge(Edge &E) {
 100     return E.getKind() == x86_64::BranchPCRel32 && !E.getTarget().isDefined();
 101   }
 102
 103   Symbol &createPLTStub(Symbol &Target) {
 104     auto &StubContentBlock =
 105         G.createContentBlock(getStubsSection(), getStubBlockContent(), 0, 1, 0);
 106     // Re-use GOT entries for stub targets.
 107     auto &GOTEntrySymbol = getGOTEntry(Target);
 108     StubContentBlock.addEdge(x86_64::Delta32, 2, GOTEntrySymbol, -4);
 109     return G.addAnonymousSymbol(StubContentBlock, 0, 6, true, false);
 110   }
 111
 112   void fixPLTEdge(Edge &E, Symbol &Stub) {
 113     assert(E.getKind() == x86_64::BranchPCRel32 && "Not a Branch32 edge?");
 114
 115     // Set the edge kind to Branch32ToPtrJumpStubBypassable to enable it to be
 116     // optimized when the target is in-range.
 117     E.setKind(x86_64::BranchPCRel32ToPtrJumpStubBypassable);
 118     E.setTarget(Stub);
 119   }
 120
 121 private:
 122   Section &getGOTSection() const {
 123     if (!GOTSection)
 124       GOTSection = &G.createSection(ELFGOTSectionName, sys::Memory::MF_READ);
 125     return *GOTSection;
 126   }
 127
 128   Section &getStubsSection() const {
 129     if (!StubsSection) {
 130       auto StubsProt = static_cast<sys::Memory::ProtectionFlags>(
 131           sys::Memory::MF_READ | sys::Memory::MF_EXEC);
 132       StubsSection = &G.createSection("$__STUBS", StubsProt);
 133     }
 134     return *StubsSection;
 135   }
 136
 137   ArrayRef<char> getGOTEntryBlockContent() {
 138     return {reinterpret_cast<const char *>(NullGOTEntryContent),
 139             sizeof(NullGOTEntryContent)};
 140   }
 141
 142   ArrayRef<char> getStubBlockContent() {
 143     return {reinterpret_cast<const char *>(StubContent), sizeof(StubContent)};
 144   }
 145
 146   mutable Section *GOTSection = nullptr;
 147   mutable Section *StubsSection = nullptr;
 148 };
 149
 150 } // namespace
 151
 152 const uint8_t PerGraphGOTAndPLTStubsBuilder_ELF_x86_64::NullGOTEntryContent[8] =
 153     {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
 154 const uint8_t PerGraphGOTAndPLTStubsBuilder_ELF_x86_64::StubContent[6] = {
 155     0xFF, 0x25, 0x00, 0x00, 0x00, 0x00};
 156
 157 static const char *getELFX86_64RelocName(uint32_t Type) {
 158   switch (Type) {
 159 #define ELF_RELOC(Name, Number)                                                \
 160   case Number:                                                                 \
 161     return #Name;
 162 #include "llvm/BinaryFormat/ELFRelocs/x86_64.def"
 163 #undef ELF_RELOC
 164   }
 165   return "Unrecognized ELF/x86-64 relocation type";
 166 }
 167
 168 namespace llvm {
 169 namespace jitlink {
 170
 171 // This should become a template as the ELFFile is so a lot of this could become
 172 // generic
 173 class ELFLinkGraphBuilder_x86_64 : public ELFLinkGraphBuilder<object::ELF64LE> {
 174 private:
 175
 176   static Expected<ELF_x86_64_Edges::ELFX86RelocationKind>
 177   getRelocationKind(const uint32_t Type) {
 178     switch (Type) {
 179     case ELF::R_X86_64_PC32:
 180       return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32;
 181     case ELF::R_X86_64_PC64:
 182     case ELF::R_X86_64_GOTPC64:
 183       return ELF_x86_64_Edges::ELFX86RelocationKind::Delta64;
 184     case ELF::R_X86_64_64:
 185       return ELF_x86_64_Edges::ELFX86RelocationKind::Pointer64;
 186     case ELF::R_X86_64_GOTPCREL:
 187       return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32GOTLoad;
 188     case ELF::R_X86_64_GOTPCRELX:
 189       return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32GOTLoadRelaxable;
 190     case ELF::R_X86_64_REX_GOTPCRELX:
 191       return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32REXGOTLoadRelaxable;
 192     case ELF::R_X86_64_GOTPCREL64:
 193       return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel64GOT;
 194     case ELF::R_X86_64_GOT64:
 195       return ELF_x86_64_Edges::ELFX86RelocationKind::GOT64;
 196     case ELF::R_X86_64_GOTOFF64:
 197       return ELF_x86_64_Edges::ELFX86RelocationKind::GOTOFF64;
 198     case ELF::R_X86_64_PLT32:
 199       return ELF_x86_64_Edges::ELFX86RelocationKind::Branch32;
 200     }
 201     return make_error<JITLinkError>("Unsupported x86-64 relocation type " +
 202                                     formatv("{0:d}: ", Type) +
 203                                     getELFX86_64RelocName(Type));
 204   }
 205
 206   Error addRelocations() override {
 207     LLVM_DEBUG(dbgs() << "Adding relocations\n");
 208     // TODO a partern is forming of iterate some sections but only give me
 209     // ones I am interested, i should abstract that concept some where
 210     for (auto &SecRef : Sections) {
 211       if (SecRef.sh_type != ELF::SHT_RELA && SecRef.sh_type != ELF::SHT_REL)
 212         continue;
 213       // TODO can the elf obj file do this for me?
 214       if (SecRef.sh_type == ELF::SHT_REL)
 215         return make_error<llvm::StringError>("Shouldn't have REL in x64",
 216                                              llvm::inconvertibleErrorCode());
 217
 218       auto RelSectName = Obj.getSectionName(SecRef);
 219       if (!RelSectName)
 220         return RelSectName.takeError();
 221
 222       LLVM_DEBUG({
 223         dbgs() << "Adding relocations from section " << *RelSectName << "\n";
 224       });
 225
 226       auto UpdateSection = Obj.getSection(SecRef.sh_info);
 227       if (!UpdateSection)
 228         return UpdateSection.takeError();
 229
 230       auto UpdateSectionName = Obj.getSectionName(**UpdateSection);
 231       if (!UpdateSectionName)
 232         return UpdateSectionName.takeError();
 233
 234       // Don't process relocations for debug sections.
 235       if (isDwarfSection(*UpdateSectionName)) {
 236         LLVM_DEBUG({
 237           dbgs() << "  Target is dwarf section " << *UpdateSectionName
 238                  << ". Skipping.\n";
 239         });
 240         continue;
 241       } else
 242         LLVM_DEBUG({
 243           dbgs() << "  For target section " << *UpdateSectionName << "\n";
 244         });
 245
 246       auto JITSection = G->findSectionByName(*UpdateSectionName);
 247       if (!JITSection)
 248         return make_error<llvm::StringError>(
 249             "Refencing a a section that wasn't added to graph" +
 250                 *UpdateSectionName,
 251             llvm::inconvertibleErrorCode());
 252
 253       auto Relocations = Obj.relas(SecRef);
 254       if (!Relocations)
 255         return Relocations.takeError();
 256
 257       for (const auto &Rela : *Relocations) {
 258         auto Type = Rela.getType(false);
 259
 260         LLVM_DEBUG({
 261           dbgs() << "Relocation Type: " << Type << "\n"
 262                  << "Name: " << Obj.getRelocationTypeName(Type) << "\n";
 263         });
 264         auto SymbolIndex = Rela.getSymbol(false);
 265         auto Symbol = Obj.getRelocationSymbol(Rela, SymTabSec);
 266         if (!Symbol)
 267           return Symbol.takeError();
 268
 269         auto BlockToFix = *(JITSection->blocks().begin());
 270         auto *TargetSymbol = getGraphSymbol(SymbolIndex);
 271
 272         if (!TargetSymbol) {
 273           return make_error<llvm::StringError>(
 274               "Could not find symbol at given index, did you add it to "
 275               "JITSymbolTable? index: " +
 276                   std::to_string(SymbolIndex) +
 277                   ", shndx: " + std::to_string((*Symbol)->st_shndx) +
 278                   " Size of table: " + std::to_string(GraphSymbols.size()),
 279               llvm::inconvertibleErrorCode());
 280         }
 281         int64_t Addend = Rela.r_addend;
 282         JITTargetAddress FixupAddress =
 283             (*UpdateSection)->sh_addr + Rela.r_offset;
 284
 285         LLVM_DEBUG({
 286           dbgs() << "Processing relocation at "
 287                  << format("0x%016" PRIx64, FixupAddress) << "\n";
 288         });
 289         auto ELFRelocKind = getRelocationKind(Type);
 290         if (!ELFRelocKind)
 291           return ELFRelocKind.takeError();
 292
 293         Edge::Kind Kind = Edge::Invalid;
 294         switch (*ELFRelocKind) {
 295         case PCRel32:
 296           Kind = x86_64::Delta32;
 297           break;
 298         case Delta64:
 299           Kind = x86_64::Delta64;
 300           break;
 301         case Pointer64:
 302           Kind = x86_64::Pointer64;
 303           break;
 304         case PCRel32GOTLoad: {
 305           Kind = x86_64::RequestGOTAndTransformToDelta32;
 306           break;
 307         }
 308         case PCRel32REXGOTLoadRelaxable: {
 309           Kind = x86_64::RequestGOTAndTransformToPCRel32GOTLoadREXRelaxable;
 310           Addend = 0;
 311           break;
 312         }
 313         case PCRel32GOTLoadRelaxable: {
 314           Kind = x86_64::RequestGOTAndTransformToPCRel32GOTLoadRelaxable;
 315           Addend = 0;
 316           break;
 317         }
 318         case PCRel64GOT: {
 319           Kind = x86_64::RequestGOTAndTransformToDelta64;
 320           break;
 321         }
 322         case GOT64: {
 323           Kind = x86_64::RequestGOTAndTransformToDelta64FromGOT;
 324           break;
 325         }
 326         case GOTOFF64: {
 327           Kind = x86_64::Delta64FromGOT;
 328           break;
 329         }
 330         case Branch32: {
 331           Kind = x86_64::BranchPCRel32;
 332           Addend = 0;
 333           break;
 334         }
 335         }
 336
 337         LLVM_DEBUG({
 338           Edge GE(Kind, FixupAddress - BlockToFix->getAddress(), *TargetSymbol,
 339                   Addend);
 340           printEdge(dbgs(), *BlockToFix, GE, getELFX86RelocationKindName(Kind));
 341           dbgs() << "\n";
 342         });
 343         BlockToFix->addEdge(Kind, FixupAddress - BlockToFix->getAddress(),
 344                             *TargetSymbol, Addend);
 345       }
 346     }
 347     return Error::success();
 348   }
 349
 350 public:
 351   ELFLinkGraphBuilder_x86_64(StringRef FileName,
 352                              const object::ELFFile<object::ELF64LE> &Obj)
 353       : ELFLinkGraphBuilder(Obj, Triple("x86_64-unknown-linux"), FileName,
 354                             x86_64::getEdgeKindName) {}
 355 };
 356
 357 class ELFJITLinker_x86_64 : public JITLinker<ELFJITLinker_x86_64> {
 358   friend class JITLinker<ELFJITLinker_x86_64>;
 359
 360 public:
 361   ELFJITLinker_x86_64(std::unique_ptr<JITLinkContext> Ctx,
 362                       std::unique_ptr<LinkGraph> G,
 363                       PassConfiguration PassConfig)
 364       : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) {
 365     getPassConfig().PostAllocationPasses.push_back(
 366         [this](LinkGraph &G) { return getOrCreateGOTSymbol(G); });
 367   }
 368
 369 private:
 370   Symbol *GOTSymbol = nullptr;
 371
 372   Error getOrCreateGOTSymbol(LinkGraph &G) {
 373     auto DefineExternalGOTSymbolIfPresent =
 374         createDefineExternalSectionStartAndEndSymbolsPass(
 375             [&](LinkGraph &LG, Symbol &Sym) -> SectionRangeSymbolDesc {
 376               if (Sym.getName() == ELFGOTSymbolName)
 377                 if (auto *GOTSection = G.findSectionByName(ELFGOTSectionName)) {
 378                   GOTSymbol = &Sym;
 379                   return {*GOTSection, true};
 380                 }
 381               return {};
 382             });
 383
 384     // Try to attach _GLOBAL_OFFSET_TABLE_ to the GOT if it's defined as an
 385     // external.
 386     if (auto Err = DefineExternalGOTSymbolIfPresent(G))
 387       return Err;
 388
 389     // If we succeeded then we're done.
 390     if (GOTSymbol)
 391       return Error::success();
 392
 393     // Otherwise look for a GOT section: If it already has a start symbol we'll
 394     // record it, otherwise we'll create our own.
 395     // If there's a GOT section but we didn't find an external GOT symbol...
 396     if (auto *GOTSection = G.findSectionByName(ELFGOTSectionName)) {
 397
 398       // Check for an existing defined symbol.
 399       for (auto *Sym : GOTSection->symbols())
 400         if (Sym->getName() == ELFGOTSymbolName) {
 401           GOTSymbol = Sym;
 402           return Error::success();
 403         }
 404
 405       // If there's no defined symbol then create one.
 406       SectionRange SR(*GOTSection);
 407       if (SR.empty())
 408         GOTSymbol = &G.addAbsoluteSymbol(ELFGOTSymbolName, 0, 0,
 409                                          Linkage::Strong, Scope::Local, true);
 410       else
 411         GOTSymbol =
 412             &G.addDefinedSymbol(*SR.getFirstBlock(), 0, ELFGOTSymbolName, 0,
 413                                 Linkage::Strong, Scope::Local, false, true);
 414     }
 415
 416     return Error::success();
 417   }
 418
 419   Error applyFixup(LinkGraph &G, Block &B, const Edge &E) const {
 420     return x86_64::applyFixup(G, B, E, GOTSymbol);
 421   }
 422 };
 423
 424 Expected<std::unique_ptr<LinkGraph>>
 425 createLinkGraphFromELFObject_x86_64(MemoryBufferRef ObjectBuffer) {
 426   LLVM_DEBUG({
 427     dbgs() << "Building jitlink graph for new input "
 428            << ObjectBuffer.getBufferIdentifier() << "...\n";
 429   });
 430
 431   auto ELFObj = object::ObjectFile::createELFObjectFile(ObjectBuffer);
 432   if (!ELFObj)
 433     return ELFObj.takeError();
 434
 435   auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF64LE>>(**ELFObj);
 436   return ELFLinkGraphBuilder_x86_64((*ELFObj)->getFileName(),
 437                                     ELFObjFile.getELFFile())
 438       .buildGraph();
 439 }
 440
 441 static SectionRangeSymbolDesc
 442 identifyELFSectionStartAndEndSymbols(LinkGraph &G, Symbol &Sym) {
 443   constexpr StringRef StartSymbolPrefix = "__start";
 444   constexpr StringRef EndSymbolPrefix = "__end";
 445
 446   auto SymName = Sym.getName();
 447   if (SymName.startswith(StartSymbolPrefix)) {
 448     if (auto *Sec =
 449             G.findSectionByName(SymName.drop_front(StartSymbolPrefix.size())))
 450       return {*Sec, true};
 451   } else if (SymName.startswith(EndSymbolPrefix)) {
 452     if (auto *Sec =
 453             G.findSectionByName(SymName.drop_front(EndSymbolPrefix.size())))
 454       return {*Sec, false};
 455   }
 456   return {};
 457 }
 458
 459 void link_ELF_x86_64(std::unique_ptr<LinkGraph> G,
 460                      std::unique_ptr<JITLinkContext> Ctx) {
 461   PassConfiguration Config;
 462
 463   if (Ctx->shouldAddDefaultTargetPasses(G->getTargetTriple())) {
 464
 465     Config.PrePrunePasses.push_back(EHFrameSplitter(".eh_frame"));
 466     Config.PrePrunePasses.push_back(
 467         EHFrameEdgeFixer(".eh_frame", x86_64::PointerSize, x86_64::Delta64,
 468                          x86_64::Delta32, x86_64::NegDelta32));
 469     Config.PrePrunePasses.push_back(EHFrameNullTerminator(".eh_frame"));
 470
 471     // Construct a JITLinker and run the link function.
 472     // Add a mark-live pass.
 473     if (auto MarkLive = Ctx->getMarkLivePass(G->getTargetTriple()))
 474       Config.PrePrunePasses.push_back(std::move(MarkLive));
 475     else
 476       Config.PrePrunePasses.push_back(markAllSymbolsLive);
 477
 478     // Add an in-place GOT/Stubs pass.
 479     Config.PostPrunePasses.push_back(
 480         PerGraphGOTAndPLTStubsBuilder_ELF_x86_64::asPass);
 481
 482     // Resolve any external section start / end symbols.
 483     Config.PostAllocationPasses.push_back(
 484         createDefineExternalSectionStartAndEndSymbolsPass(
 485             identifyELFSectionStartAndEndSymbols));
 486
 487     // Add GOT/Stubs optimizer pass.
 488     Config.PreFixupPasses.push_back(x86_64::optimize_x86_64_GOTAndStubs);
 489   }
 490
 491   if (auto Err = Ctx->modifyPassConfig(*G, Config))
 492     return Ctx->notifyFailed(std::move(Err));
 493
 494   ELFJITLinker_x86_64::link(std::move(Ctx), std::move(G), std::move(Config));
 495 }
 496 const char *getELFX86RelocationKindName(Edge::Kind R) {
 497   switch (R) {
 498   case Branch32:
 499     return "Branch32";
 500   case Pointer64:
 501     return "Pointer64";
 502   case PCRel32:
 503     return "PCRel32";
 504   case PCRel32GOTLoad:
 505     return "PCRel32GOTLoad";
 506   case PCRel32GOTLoadRelaxable:
 507     return "PCRel32GOTLoadRelaxable";
 508   case PCRel32REXGOTLoadRelaxable:
 509     return "PCRel32REXGOTLoad";
 510   case PCRel64GOT:
 511     return "PCRel64GOT";
 512   case Delta64:
 513     return "Delta64";
 514   case GOT64:
 515     return "GOT64";
 516   case GOTOFF64:
 517     return "GOTOFF64";
 518   }
 519   return getGenericEdgeKindName(static_cast<Edge::Kind>(R));
 520 }
 521 } // end namespace jitlink
 522 } // end namespace llvm