bolt/lib/Rewrite/ExecutableFileMemoryManager.cpp

   1 //===- bolt/Rewrite/ExecutableFileMemoryManager.cpp -----------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "bolt/Rewrite/ExecutableFileMemoryManager.h"
  10 #include "bolt/Rewrite/JITLinkLinker.h"
  11 #include "bolt/Rewrite/RewriteInstance.h"
  12 #include "llvm/ExecutionEngine/JITLink/JITLink.h"
  13 #include "llvm/Support/MemAlloc.h"
  14
  15 #undef  DEBUG_TYPE
  16 #define DEBUG_TYPE "efmm"
  17
  18 using namespace llvm;
  19 using namespace object;
  20 using namespace bolt;
  21
  22 namespace llvm {
  23
  24 namespace bolt {
  25
  26 namespace {
  27
  28 SmallVector<jitlink::Section *> orderedSections(jitlink::LinkGraph &G) {
  29   SmallVector<jitlink::Section *> Sections(
  30       llvm::map_range(G.sections(), [](auto &S) { return &S; }));
  31   llvm::sort(Sections, [](const auto *LHS, const auto *RHS) {
  32     return LHS->getOrdinal() < RHS->getOrdinal();
  33   });
  34   return Sections;
  35 }
  36
  37 size_t sectionAlignment(const jitlink::Section &Section) {
  38   assert(!Section.empty() && "Cannot get alignment for empty section");
  39   return JITLinkLinker::orderedBlocks(Section).front()->getAlignment();
  40 }
  41
  42 StringRef sectionName(const jitlink::Section &Section,
  43                       const BinaryContext &BC) {
  44   auto Name = Section.getName();
  45
  46   if (BC.isMachO()) {
  47     // JITLink "normalizes" section names as "SegmentName,SectionName" on
  48     // Mach-O. BOLT internally refers to sections just by the section name so
  49     // strip-off the segment name.
  50     auto SegmentEnd = Name.find(',');
  51     assert(SegmentEnd != StringRef::npos && "Mach-O segment not found");
  52     Name = Name.substr(SegmentEnd + 1);
  53   }
  54
  55   return Name;
  56 }
  57
  58 struct SectionAllocInfo {
  59   void *Address;
  60   size_t Size;
  61   size_t Alignment;
  62 };
  63
  64 struct AllocInfo {
  65   SmallVector<SectionAllocInfo, 8> AllocatedSections;
  66
  67   ~AllocInfo() {
  68     for (auto &Section : AllocatedSections)
  69       deallocate_buffer(Section.Address, Section.Size, Section.Alignment);
  70   }
  71
  72   SectionAllocInfo allocateSection(const jitlink::Section &Section) {
  73     auto Size = JITLinkLinker::sectionSize(Section);
  74     auto Alignment = sectionAlignment(Section);
  75     auto *Buf = allocate_buffer(Size, Alignment);
  76     SectionAllocInfo Alloc{Buf, Size, Alignment};
  77     AllocatedSections.push_back(Alloc);
  78     return Alloc;
  79   }
  80 };
  81
  82 struct BOLTInFlightAlloc : ExecutableFileMemoryManager::InFlightAlloc {
  83   // Even though this is passed using a raw pointer in FinalizedAlloc, we keep
  84   // it in a unique_ptr as long as possible to enjoy automatic cleanup when
  85   // something goes wrong.
  86   std::unique_ptr<AllocInfo> Alloc;
  87
  88 public:
  89   BOLTInFlightAlloc(std::unique_ptr<AllocInfo> Alloc)
  90       : Alloc(std::move(Alloc)) {}
  91
  92   virtual void abandon(OnAbandonedFunction OnAbandoned) override {
  93     OnAbandoned(Error::success());
  94   }
  95
  96   virtual void finalize(OnFinalizedFunction OnFinalized) override {
  97     OnFinalized(ExecutableFileMemoryManager::FinalizedAlloc(
  98         orc::ExecutorAddr::fromPtr(Alloc.release())));
  99   }
 100 };
 101
 102 } // anonymous namespace
 103
 104 void ExecutableFileMemoryManager::updateSection(
 105     const jitlink::Section &JLSection, uint8_t *Contents, size_t Size,
 106     size_t Alignment) {
 107   auto SectionID = JLSection.getName();
 108   auto SectionName = sectionName(JLSection, BC);
 109   auto Prot = JLSection.getMemProt();
 110   auto IsCode = (Prot & orc::MemProt::Exec) != orc::MemProt::None;
 111   auto IsReadOnly = (Prot & orc::MemProt::Write) == orc::MemProt::None;
 112
 113   // Register a debug section as a note section.
 114   if (!ObjectsLoaded && RewriteInstance::isDebugSection(SectionName)) {
 115     BinarySection &Section =
 116         BC.registerOrUpdateNoteSection(SectionName, Contents, Size, Alignment);
 117     Section.setSectionID(SectionID);
 118     assert(!Section.isAllocatable() && "note sections cannot be allocatable");
 119     return;
 120   }
 121
 122   if (!IsCode && (SectionName == ".strtab" || SectionName == ".symtab" ||
 123                   SectionName == "" || SectionName.starts_with(".rela.")))
 124     return;
 125
 126   SmallVector<char, 256> Buf;
 127   if (ObjectsLoaded > 0) {
 128     if (BC.isELF()) {
 129       SectionName = (Twine(SectionName) + ".bolt.extra." + Twine(ObjectsLoaded))
 130                         .toStringRef(Buf);
 131     } else if (BC.isMachO()) {
 132       assert((SectionName == "__text" || SectionName == "__data" ||
 133               SectionName == "__fini" || SectionName == "__setup" ||
 134               SectionName == "__cstring" || SectionName == "__literal16") &&
 135              "Unexpected section in the instrumentation library");
 136       // Sections coming from the instrumentation runtime are prefixed with "I".
 137       SectionName = ("I" + Twine(SectionName)).toStringRef(Buf);
 138     }
 139   }
 140
 141   BinarySection *Section = nullptr;
 142   if (!OrgSecPrefix.empty() && SectionName.starts_with(OrgSecPrefix)) {
 143     // Update the original section contents.
 144     ErrorOr<BinarySection &> OrgSection =
 145         BC.getUniqueSectionByName(SectionName.substr(OrgSecPrefix.length()));
 146     assert(OrgSection && OrgSection->isAllocatable() &&
 147            "Original section must exist and be allocatable.");
 148
 149     Section = &OrgSection.get();
 150     Section->updateContents(Contents, Size);
 151   } else {
 152     // If the input contains a section with the section name, rename it in the
 153     // output file to avoid the section name conflict and emit the new section
 154     // under a unique internal name.
 155     ErrorOr<BinarySection &> OrgSection =
 156         BC.getUniqueSectionByName(SectionName);
 157     bool UsePrefix = false;
 158     if (OrgSection && OrgSection->hasSectionRef()) {
 159       OrgSection->setOutputName(OrgSecPrefix + SectionName);
 160       UsePrefix = true;
 161     }
 162
 163     // Register the new section under a unique name to avoid name collision with
 164     // sections in the input file.
 165     BinarySection &NewSection = BC.registerOrUpdateSection(
 166         UsePrefix ? NewSecPrefix + SectionName : SectionName, ELF::SHT_PROGBITS,
 167         BinarySection::getFlags(IsReadOnly, IsCode, true), Contents, Size,
 168         Alignment);
 169     if (UsePrefix)
 170       NewSection.setOutputName(SectionName);
 171     Section = &NewSection;
 172   }
 173
 174   LLVM_DEBUG({
 175     dbgs() << "BOLT: allocating "
 176            << (IsCode ? "code" : (IsReadOnly ? "read-only data" : "data"))
 177            << " section : " << Section->getOutputName() << " ("
 178            << Section->getName() << ")"
 179            << " with size " << Size << ", alignment " << Alignment << " at "
 180            << Contents << ", ID = " << SectionID << "\n";
 181   });
 182
 183   Section->setSectionID(SectionID);
 184 }
 185
 186 void ExecutableFileMemoryManager::allocate(const jitlink::JITLinkDylib *JD,
 187                                            jitlink::LinkGraph &G,
 188                                            OnAllocatedFunction OnAllocated) {
 189   auto Alloc = std::make_unique<AllocInfo>();
 190
 191   for (auto *Section : orderedSections(G)) {
 192     if (Section->empty())
 193       continue;
 194
 195     auto SectionAlloc = Alloc->allocateSection(*Section);
 196     updateSection(*Section, static_cast<uint8_t *>(SectionAlloc.Address),
 197                   SectionAlloc.Size, SectionAlloc.Alignment);
 198
 199     size_t CurrentOffset = 0;
 200     auto *Buf = static_cast<char *>(SectionAlloc.Address);
 201     for (auto *Block : JITLinkLinker::orderedBlocks(*Section)) {
 202       CurrentOffset = jitlink::alignToBlock(CurrentOffset, *Block);
 203       auto BlockSize = Block->getSize();
 204       auto *BlockBuf = Buf + CurrentOffset;
 205
 206       if (Block->isZeroFill())
 207         std::memset(BlockBuf, 0, BlockSize);
 208       else
 209         std::memcpy(BlockBuf, Block->getContent().data(), BlockSize);
 210
 211       Block->setMutableContent({BlockBuf, Block->getSize()});
 212       CurrentOffset += BlockSize;
 213     }
 214   }
 215
 216   OnAllocated(std::make_unique<BOLTInFlightAlloc>(std::move(Alloc)));
 217 }
 218
 219 void ExecutableFileMemoryManager::deallocate(
 220     std::vector<FinalizedAlloc> Allocs, OnDeallocatedFunction OnDeallocated) {
 221   for (auto &Alloc : Allocs)
 222     delete Alloc.release().toPtr<AllocInfo *>();
 223
 224   OnDeallocated(Error::success());
 225 }
 226
 227 } // namespace bolt
 228
 229 } // namespace llvm