bolt/lib/RuntimeLibs/InstrumentationRuntimeLibrary.cpp

   1 //===- bolt/RuntimeLibs/InstrumentationRuntimeLibrary.cpp -----------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file implements the InstrumentationRuntimeLibrary class.
  10 //
  11 //===----------------------------------------------------------------------===//
  12
  13 #include "bolt/RuntimeLibs/InstrumentationRuntimeLibrary.h"
  14 #include "bolt/Core/BinaryFunction.h"
  15 #include "bolt/Core/JumpTable.h"
  16 #include "bolt/Core/Linker.h"
  17 #include "bolt/Utils/CommandLineOpts.h"
  18 #include "llvm/MC/MCStreamer.h"
  19 #include "llvm/Support/Alignment.h"
  20 #include "llvm/Support/CommandLine.h"
  21
  22 using namespace llvm;
  23 using namespace bolt;
  24
  25 namespace opts {
  26
  27 cl::opt<std::string> RuntimeInstrumentationLib(
  28     "runtime-instrumentation-lib",
  29     cl::desc("specify file name of the runtime instrumentation library"),
  30     cl::init("libbolt_rt_instr.a"), cl::cat(BoltOptCategory));
  31
  32 extern cl::opt<bool> InstrumentationFileAppendPID;
  33 extern cl::opt<bool> ConservativeInstrumentation;
  34 extern cl::opt<std::string> InstrumentationFilename;
  35 extern cl::opt<std::string> InstrumentationBinpath;
  36 extern cl::opt<uint32_t> InstrumentationSleepTime;
  37 extern cl::opt<bool> InstrumentationNoCountersClear;
  38 extern cl::opt<bool> InstrumentationWaitForks;
  39 extern cl::opt<JumpTableSupportLevel> JumpTables;
  40
  41 } // namespace opts
  42
  43 void InstrumentationRuntimeLibrary::adjustCommandLineOptions(
  44     const BinaryContext &BC) const {
  45   if (!BC.HasRelocations) {
  46     errs() << "BOLT-ERROR: instrumentation runtime libraries require "
  47               "relocations\n";
  48     exit(1);
  49   }
  50   if (opts::JumpTables != JTS_MOVE) {
  51     opts::JumpTables = JTS_MOVE;
  52     outs() << "BOLT-INFO: forcing -jump-tables=move for instrumentation\n";
  53   }
  54   if (!BC.StartFunctionAddress) {
  55     errs() << "BOLT-ERROR: instrumentation runtime libraries require a known "
  56               "entry point of "
  57               "the input binary\n";
  58     exit(1);
  59   }
  60   if (!BC.FiniFunctionAddress && !BC.IsStaticExecutable) {
  61     errs() << "BOLT-ERROR: input binary lacks DT_FINI entry in the dynamic "
  62               "section but instrumentation currently relies on patching "
  63               "DT_FINI to write the profile\n";
  64     exit(1);
  65   }
  66
  67   if ((opts::InstrumentationWaitForks || opts::InstrumentationSleepTime) &&
  68       opts::InstrumentationFileAppendPID) {
  69     errs()
  70         << "BOLT-ERROR: instrumentation-file-append-pid is not compatible with "
  71            "instrumentation-sleep-time and instrumentation-wait-forks. If you "
  72            "want a separate profile for each fork, it can only be dumped in "
  73            "the end of process when instrumentation-file-append-pid is used.\n";
  74     exit(1);
  75   }
  76 }
  77
  78 void InstrumentationRuntimeLibrary::emitBinary(BinaryContext &BC,
  79                                                MCStreamer &Streamer) {
  80   MCSection *Section = BC.isELF()
  81                            ? static_cast<MCSection *>(BC.Ctx->getELFSection(
  82                                  ".bolt.instr.counters", ELF::SHT_PROGBITS,
  83                                  BinarySection::getFlags(/*IsReadOnly=*/false,
  84                                                          /*IsText=*/false,
  85                                                          /*IsAllocatable=*/true)
  86
  87                                      ))
  88                            : static_cast<MCSection *>(BC.Ctx->getMachOSection(
  89                                  "__BOLT", "__counters", MachO::S_REGULAR,
  90                                  SectionKind::getData()));
  91
  92   if (BC.IsStaticExecutable && !opts::InstrumentationSleepTime) {
  93     errs() << "BOLT-ERROR: instrumentation of static binary currently does not "
  94               "support profile output on binary finalization, so it "
  95               "requires -instrumentation-sleep-time=N (N>0) usage\n";
  96     exit(1);
  97   }
  98
  99   Section->setAlignment(llvm::Align(BC.RegularPageSize));
 100   Streamer.switchSection(Section);
 101
 102   // EmitOffset is used to determine padding size for data alignment
 103   uint64_t EmitOffset = 0;
 104
 105   auto emitLabel = [&Streamer](MCSymbol *Symbol, bool IsGlobal = true) {
 106     Streamer.emitLabel(Symbol);
 107     if (IsGlobal)
 108       Streamer.emitSymbolAttribute(Symbol, MCSymbolAttr::MCSA_Global);
 109   };
 110
 111   auto emitLabelByName = [&BC, emitLabel](StringRef Name,
 112                                           bool IsGlobal = true) {
 113     MCSymbol *Symbol = BC.Ctx->getOrCreateSymbol(Name);
 114     emitLabel(Symbol, IsGlobal);
 115   };
 116
 117   auto emitPadding = [&Streamer, &EmitOffset](unsigned Size) {
 118     const uint64_t Padding = alignTo(EmitOffset, Size) - EmitOffset;
 119     if (Padding) {
 120       Streamer.emitFill(Padding, 0);
 121       EmitOffset += Padding;
 122     }
 123   };
 124
 125   auto emitDataSize = [&EmitOffset](unsigned Size) { EmitOffset += Size; };
 126
 127   auto emitDataPadding = [emitPadding, emitDataSize](unsigned Size) {
 128     emitPadding(Size);
 129     emitDataSize(Size);
 130   };
 131
 132   auto emitFill = [&Streamer, emitDataSize,
 133                    emitLabel](unsigned Size, MCSymbol *Symbol = nullptr,
 134                               uint8_t Byte = 0) {
 135     emitDataSize(Size);
 136     if (Symbol)
 137       emitLabel(Symbol, /*IsGlobal*/ false);
 138     Streamer.emitFill(Size, Byte);
 139   };
 140
 141   auto emitValue = [&BC, &Streamer, emitDataPadding,
 142                     emitLabel](MCSymbol *Symbol, const MCExpr *Value) {
 143     const unsigned Psize = BC.AsmInfo->getCodePointerSize();
 144     emitDataPadding(Psize);
 145     emitLabel(Symbol);
 146     if (Value)
 147       Streamer.emitValue(Value, Psize);
 148     else
 149       Streamer.emitFill(Psize, 0);
 150   };
 151
 152   auto emitIntValue = [&Streamer, emitDataPadding, emitLabelByName](
 153                           StringRef Name, uint64_t Value, unsigned Size = 4) {
 154     emitDataPadding(Size);
 155     emitLabelByName(Name);
 156     Streamer.emitIntValue(Value, Size);
 157   };
 158
 159   auto emitString = [&Streamer, emitDataSize, emitLabelByName,
 160                      emitFill](StringRef Name, StringRef Contents) {
 161     emitDataSize(Contents.size());
 162     emitLabelByName(Name);
 163     Streamer.emitBytes(Contents);
 164     emitFill(1);
 165   };
 166
 167   // All of the following symbols will be exported as globals to be used by the
 168   // instrumentation runtime library to dump the instrumentation data to disk.
 169   // Label marking start of the memory region containing instrumentation
 170   // counters, total vector size is Counters.size() 8-byte counters
 171   emitLabelByName("__bolt_instr_locations");
 172   for (MCSymbol *const &Label : Summary->Counters)
 173     emitFill(sizeof(uint64_t), Label);
 174
 175   emitPadding(BC.RegularPageSize);
 176   emitIntValue("__bolt_instr_sleep_time", opts::InstrumentationSleepTime);
 177   emitIntValue("__bolt_instr_no_counters_clear",
 178                !!opts::InstrumentationNoCountersClear, 1);
 179   emitIntValue("__bolt_instr_conservative", !!opts::ConservativeInstrumentation,
 180                1);
 181   emitIntValue("__bolt_instr_wait_forks", !!opts::InstrumentationWaitForks, 1);
 182   emitIntValue("__bolt_num_counters", Summary->Counters.size());
 183   emitValue(Summary->IndCallCounterFuncPtr, nullptr);
 184   emitValue(Summary->IndTailCallCounterFuncPtr, nullptr);
 185   emitIntValue("__bolt_instr_num_ind_calls",
 186                Summary->IndCallDescriptions.size());
 187   emitIntValue("__bolt_instr_num_ind_targets",
 188                Summary->IndCallTargetDescriptions.size());
 189   emitIntValue("__bolt_instr_num_funcs", Summary->FunctionDescriptions.size());
 190   emitString("__bolt_instr_filename", opts::InstrumentationFilename);
 191   emitString("__bolt_instr_binpath", opts::InstrumentationBinpath);
 192   emitIntValue("__bolt_instr_use_pid", !!opts::InstrumentationFileAppendPID, 1);
 193
 194   if (BC.isMachO()) {
 195     MCSection *TablesSection = BC.Ctx->getMachOSection(
 196         "__BOLT", "__tables", MachO::S_REGULAR, SectionKind::getData());
 197     TablesSection->setAlignment(llvm::Align(BC.RegularPageSize));
 198     Streamer.switchSection(TablesSection);
 199     emitString("__bolt_instr_tables", buildTables(BC));
 200   }
 201 }
 202
 203 void InstrumentationRuntimeLibrary::link(
 204     BinaryContext &BC, StringRef ToolPath, BOLTLinker &Linker,
 205     BOLTLinker::SectionsMapper MapSections) {
 206   std::string LibPath = getLibPath(ToolPath, opts::RuntimeInstrumentationLib);
 207   loadLibrary(LibPath, Linker, MapSections);
 208
 209   if (BC.isMachO())
 210     return;
 211
 212   RuntimeFiniAddress = Linker.lookupSymbol("__bolt_instr_fini").value_or(0);
 213   if (!RuntimeFiniAddress) {
 214     errs() << "BOLT-ERROR: instrumentation library does not define "
 215               "__bolt_instr_fini: "
 216            << LibPath << "\n";
 217     exit(1);
 218   }
 219   RuntimeStartAddress = Linker.lookupSymbol("__bolt_instr_start").value_or(0);
 220   if (!RuntimeStartAddress) {
 221     errs() << "BOLT-ERROR: instrumentation library does not define "
 222               "__bolt_instr_start: "
 223            << LibPath << "\n";
 224     exit(1);
 225   }
 226   outs() << "BOLT-INFO: output linked against instrumentation runtime "
 227             "library, lib entry point is 0x"
 228          << Twine::utohexstr(RuntimeFiniAddress) << "\n";
 229   outs() << "BOLT-INFO: clear procedure is 0x"
 230          << Twine::utohexstr(
 231                 Linker.lookupSymbol("__bolt_instr_clear_counters").value_or(0))
 232          << "\n";
 233
 234   emitTablesAsELFNote(BC);
 235 }
 236
 237 std::string InstrumentationRuntimeLibrary::buildTables(BinaryContext &BC) {
 238   std::string TablesStr;
 239   raw_string_ostream OS(TablesStr);
 240
 241   // This is sync'ed with runtime/instr.cpp:readDescriptions()
 242   auto getOutputAddress = [](const BinaryFunction &Func,
 243                              uint64_t Offset) -> uint64_t {
 244     return Offset == 0
 245                ? Func.getOutputAddress()
 246                : Func.translateInputToOutputAddress(Func.getAddress() + Offset);
 247   };
 248
 249   // Indirect targets need to be sorted for fast lookup during runtime
 250   llvm::sort(Summary->IndCallTargetDescriptions,
 251              [&](const IndCallTargetDescription &A,
 252                  const IndCallTargetDescription &B) {
 253                return getOutputAddress(*A.Target, A.ToLoc.Offset) <
 254                       getOutputAddress(*B.Target, B.ToLoc.Offset);
 255              });
 256
 257   // Start of the vector with descriptions (one CounterDescription for each
 258   // counter), vector size is Counters.size() CounterDescription-sized elmts
 259   const size_t IDSize =
 260       Summary->IndCallDescriptions.size() * sizeof(IndCallDescription);
 261   OS.write(reinterpret_cast<const char *>(&IDSize), 4);
 262   for (const IndCallDescription &Desc : Summary->IndCallDescriptions) {
 263     OS.write(reinterpret_cast<const char *>(&Desc.FromLoc.FuncString), 4);
 264     OS.write(reinterpret_cast<const char *>(&Desc.FromLoc.Offset), 4);
 265   }
 266
 267   const size_t ITDSize = Summary->IndCallTargetDescriptions.size() *
 268                          sizeof(IndCallTargetDescription);
 269   OS.write(reinterpret_cast<const char *>(&ITDSize), 4);
 270   for (const IndCallTargetDescription &Desc :
 271        Summary->IndCallTargetDescriptions) {
 272     OS.write(reinterpret_cast<const char *>(&Desc.ToLoc.FuncString), 4);
 273     OS.write(reinterpret_cast<const char *>(&Desc.ToLoc.Offset), 4);
 274     uint64_t TargetFuncAddress =
 275         getOutputAddress(*Desc.Target, Desc.ToLoc.Offset);
 276     OS.write(reinterpret_cast<const char *>(&TargetFuncAddress), 8);
 277   }
 278
 279   uint32_t FuncDescSize = Summary->getFDSize();
 280   OS.write(reinterpret_cast<const char *>(&FuncDescSize), 4);
 281   for (const FunctionDescription &Desc : Summary->FunctionDescriptions) {
 282     const size_t LeafNum = Desc.LeafNodes.size();
 283     OS.write(reinterpret_cast<const char *>(&LeafNum), 4);
 284     for (const InstrumentedNode &LeafNode : Desc.LeafNodes) {
 285       OS.write(reinterpret_cast<const char *>(&LeafNode.Node), 4);
 286       OS.write(reinterpret_cast<const char *>(&LeafNode.Counter), 4);
 287     }
 288     const size_t EdgesNum = Desc.Edges.size();
 289     OS.write(reinterpret_cast<const char *>(&EdgesNum), 4);
 290     for (const EdgeDescription &Edge : Desc.Edges) {
 291       OS.write(reinterpret_cast<const char *>(&Edge.FromLoc.FuncString), 4);
 292       OS.write(reinterpret_cast<const char *>(&Edge.FromLoc.Offset), 4);
 293       OS.write(reinterpret_cast<const char *>(&Edge.FromNode), 4);
 294       OS.write(reinterpret_cast<const char *>(&Edge.ToLoc.FuncString), 4);
 295       OS.write(reinterpret_cast<const char *>(&Edge.ToLoc.Offset), 4);
 296       OS.write(reinterpret_cast<const char *>(&Edge.ToNode), 4);
 297       OS.write(reinterpret_cast<const char *>(&Edge.Counter), 4);
 298     }
 299     const size_t CallsNum = Desc.Calls.size();
 300     OS.write(reinterpret_cast<const char *>(&CallsNum), 4);
 301     for (const CallDescription &Call : Desc.Calls) {
 302       OS.write(reinterpret_cast<const char *>(&Call.FromLoc.FuncString), 4);
 303       OS.write(reinterpret_cast<const char *>(&Call.FromLoc.Offset), 4);
 304       OS.write(reinterpret_cast<const char *>(&Call.FromNode), 4);
 305       OS.write(reinterpret_cast<const char *>(&Call.ToLoc.FuncString), 4);
 306       OS.write(reinterpret_cast<const char *>(&Call.ToLoc.Offset), 4);
 307       OS.write(reinterpret_cast<const char *>(&Call.Counter), 4);
 308       uint64_t TargetFuncAddress =
 309           getOutputAddress(*Call.Target, Call.ToLoc.Offset);
 310       OS.write(reinterpret_cast<const char *>(&TargetFuncAddress), 8);
 311     }
 312     const size_t EntryNum = Desc.EntryNodes.size();
 313     OS.write(reinterpret_cast<const char *>(&EntryNum), 4);
 314     for (const EntryNode &EntryNode : Desc.EntryNodes) {
 315       OS.write(reinterpret_cast<const char *>(&EntryNode.Node), 8);
 316       uint64_t TargetFuncAddress =
 317           getOutputAddress(*Desc.Function, EntryNode.Address);
 318       OS.write(reinterpret_cast<const char *>(&TargetFuncAddress), 8);
 319     }
 320   }
 321   // Our string table lives immediately after descriptions vector
 322   OS << Summary->StringTable;
 323   OS.flush();
 324
 325   return TablesStr;
 326 }
 327
 328 void InstrumentationRuntimeLibrary::emitTablesAsELFNote(BinaryContext &BC) {
 329   std::string TablesStr = buildTables(BC);
 330   const std::string BoltInfo = BinarySection::encodeELFNote(
 331       "BOLT", TablesStr, BinarySection::NT_BOLT_INSTRUMENTATION_TABLES);
 332   BC.registerOrUpdateNoteSection(".bolt.instr.tables", copyByteArray(BoltInfo),
 333                                  BoltInfo.size(),
 334                                  /*Alignment=*/1,
 335                                  /*IsReadOnly=*/true, ELF::SHT_NOTE);
 336 }