bolt/lib/Core/BinaryEmitter.cpp

   1 //===- bolt/Core/BinaryEmitter.cpp - Emit code and data -------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file implements the collection of functions and classes used for
  10 // emission of code and data into object/binary file.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "bolt/Core/BinaryEmitter.h"
  15 #include "bolt/Core/BinaryContext.h"
  16 #include "bolt/Core/BinaryFunction.h"
  17 #include "bolt/Core/DebugData.h"
  18 #include "bolt/Core/FunctionLayout.h"
  19 #include "bolt/Utils/CommandLineOpts.h"
  20 #include "bolt/Utils/Utils.h"
  21 #include "llvm/DebugInfo/DWARF/DWARFCompileUnit.h"
  22 #include "llvm/MC/MCSection.h"
  23 #include "llvm/MC/MCStreamer.h"
  24 #include "llvm/Support/CommandLine.h"
  25 #include "llvm/Support/LEB128.h"
  26 #include "llvm/Support/SMLoc.h"
  27
  28 #define DEBUG_TYPE "bolt"
  29
  30 using namespace llvm;
  31 using namespace bolt;
  32
  33 namespace opts {
  34
  35 extern cl::opt<JumpTableSupportLevel> JumpTables;
  36 extern cl::opt<bool> PreserveBlocksAlignment;
  37
  38 cl::opt<bool> AlignBlocks("align-blocks", cl::desc("align basic blocks"),
  39                           cl::cat(BoltOptCategory));
  40
  41 static cl::list<std::string>
  42 BreakFunctionNames("break-funcs",
  43   cl::CommaSeparated,
  44   cl::desc("list of functions to core dump on (debugging)"),
  45   cl::value_desc("func1,func2,func3,..."),
  46   cl::Hidden,
  47   cl::cat(BoltCategory));
  48
  49 cl::list<std::string>
  50     FunctionPadSpec("pad-funcs", cl::CommaSeparated,
  51                     cl::desc("list of functions to pad with amount of bytes"),
  52                     cl::value_desc("func1:pad1,func2:pad2,func3:pad3,..."),
  53                     cl::Hidden, cl::cat(BoltCategory));
  54
  55 cl::list<std::string> FunctionPadBeforeSpec(
  56     "pad-funcs-before", cl::CommaSeparated,
  57     cl::desc("list of functions to pad with amount of bytes"),
  58     cl::value_desc("func1:pad1,func2:pad2,func3:pad3,..."), cl::Hidden,
  59     cl::cat(BoltCategory));
  60
  61 static cl::opt<bool> MarkFuncs(
  62     "mark-funcs",
  63     cl::desc("mark function boundaries with break instruction to make "
  64              "sure we accidentally don't cross them"),
  65     cl::ReallyHidden, cl::cat(BoltCategory));
  66
  67 static cl::opt<bool> PrintJumpTables("print-jump-tables",
  68                                      cl::desc("print jump tables"), cl::Hidden,
  69                                      cl::cat(BoltCategory));
  70
  71 static cl::opt<bool>
  72 X86AlignBranchBoundaryHotOnly("x86-align-branch-boundary-hot-only",
  73   cl::desc("only apply branch boundary alignment in hot code"),
  74   cl::init(true),
  75   cl::cat(BoltOptCategory));
  76
  77 size_t padFunction(const cl::list<std::string> &Spec,
  78                    const BinaryFunction &Function) {
  79   static std::map<std::string, size_t> FunctionPadding;
  80
  81   if (FunctionPadding.empty() && !Spec.empty()) {
  82     for (const std::string &Spec : Spec) {
  83       size_t N = Spec.find(':');
  84       if (N == std::string::npos)
  85         continue;
  86       std::string Name = Spec.substr(0, N);
  87       size_t Padding = std::stoull(Spec.substr(N + 1));
  88       FunctionPadding[Name] = Padding;
  89     }
  90   }
  91
  92   for (auto &FPI : FunctionPadding) {
  93     std::string Name = FPI.first;
  94     size_t Padding = FPI.second;
  95     if (Function.hasNameRegex(Name))
  96       return Padding;
  97   }
  98
  99   return 0;
 100 }
 101
 102 } // namespace opts
 103
 104 namespace {
 105 using JumpTable = bolt::JumpTable;
 106
 107 class BinaryEmitter {
 108 private:
 109   BinaryEmitter(const BinaryEmitter &) = delete;
 110   BinaryEmitter &operator=(const BinaryEmitter &) = delete;
 111
 112   MCStreamer &Streamer;
 113   BinaryContext &BC;
 114
 115 public:
 116   BinaryEmitter(MCStreamer &Streamer, BinaryContext &BC)
 117       : Streamer(Streamer), BC(BC) {}
 118
 119   /// Emit all code and data.
 120   void emitAll(StringRef OrgSecPrefix);
 121
 122   /// Emit function code. The caller is responsible for emitting function
 123   /// symbol(s) and setting the section to emit the code to.
 124   void emitFunctionBody(BinaryFunction &BF, FunctionFragment &FF,
 125                         bool EmitCodeOnly = false);
 126
 127 private:
 128   /// Emit function code.
 129   void emitFunctions();
 130
 131   /// Emit a single function.
 132   bool emitFunction(BinaryFunction &BF, FunctionFragment &FF);
 133
 134   /// Helper for emitFunctionBody to write data inside a function
 135   /// (used for AArch64)
 136   void emitConstantIslands(BinaryFunction &BF, bool EmitColdPart,
 137                            BinaryFunction *OnBehalfOf = nullptr);
 138
 139   /// Emit jump tables for the function.
 140   void emitJumpTables(const BinaryFunction &BF);
 141
 142   /// Emit jump table data. Callee supplies sections for the data.
 143   void emitJumpTable(const JumpTable &JT, MCSection *HotSection,
 144                      MCSection *ColdSection);
 145
 146   void emitCFIInstruction(const MCCFIInstruction &Inst) const;
 147
 148   /// Emit exception handling ranges for the function fragment.
 149   void emitLSDA(BinaryFunction &BF, const FunctionFragment &FF);
 150
 151   /// Emit line number information corresponding to \p NewLoc. \p PrevLoc
 152   /// provides a context for de-duplication of line number info.
 153   /// \p FirstInstr indicates if \p NewLoc represents the first instruction
 154   /// in a sequence, such as a function fragment.
 155   ///
 156   /// If \p NewLoc location matches \p PrevLoc, no new line number entry will be
 157   /// created and the function will return \p PrevLoc while \p InstrLabel will
 158   /// be ignored. Otherwise, the caller should use \p InstrLabel to mark the
 159   /// corresponding instruction by emitting \p InstrLabel before it.
 160   /// If \p InstrLabel is set by the caller, its value will be used with \p
 161   /// \p NewLoc. If it was nullptr on entry, it will be populated with a pointer
 162   /// to a new temp symbol used with \p NewLoc.
 163   ///
 164   /// Return new current location which is either \p NewLoc or \p PrevLoc.
 165   SMLoc emitLineInfo(const BinaryFunction &BF, SMLoc NewLoc, SMLoc PrevLoc,
 166                      bool FirstInstr, MCSymbol *&InstrLabel);
 167
 168   /// Use \p FunctionEndSymbol to mark the end of the line info sequence.
 169   /// Note that it does not automatically result in the insertion of the EOS
 170   /// marker in the line table program, but provides one to the DWARF generator
 171   /// when it needs it.
 172   void emitLineInfoEnd(const BinaryFunction &BF, MCSymbol *FunctionEndSymbol);
 173
 174   /// Emit debug line info for unprocessed functions from CUs that include
 175   /// emitted functions.
 176   void emitDebugLineInfoForOriginalFunctions();
 177
 178   /// Emit debug line for CUs that were not modified.
 179   void emitDebugLineInfoForUnprocessedCUs();
 180
 181   /// Emit data sections that have code references in them.
 182   void emitDataSections(StringRef OrgSecPrefix);
 183 };
 184
 185 } // anonymous namespace
 186
 187 void BinaryEmitter::emitAll(StringRef OrgSecPrefix) {
 188   Streamer.initSections(false, *BC.STI);
 189   Streamer.setUseAssemblerInfoForParsing(false);
 190
 191   if (opts::UpdateDebugSections && BC.isELF()) {
 192     // Force the emission of debug line info into allocatable section to ensure
 193     // JITLink will process it.
 194     //
 195     // NB: on MachO all sections are required for execution, hence no need
 196     //     to change flags/attributes.
 197     MCSectionELF *ELFDwarfLineSection =
 198         static_cast<MCSectionELF *>(BC.MOFI->getDwarfLineSection());
 199     ELFDwarfLineSection->setFlags(ELF::SHF_ALLOC);
 200     MCSectionELF *ELFDwarfLineStrSection =
 201         static_cast<MCSectionELF *>(BC.MOFI->getDwarfLineStrSection());
 202     ELFDwarfLineStrSection->setFlags(ELF::SHF_ALLOC);
 203   }
 204
 205   if (RuntimeLibrary *RtLibrary = BC.getRuntimeLibrary())
 206     RtLibrary->emitBinary(BC, Streamer);
 207
 208   BC.getTextSection()->setAlignment(Align(opts::AlignText));
 209
 210   emitFunctions();
 211
 212   if (opts::UpdateDebugSections) {
 213     emitDebugLineInfoForOriginalFunctions();
 214     DwarfLineTable::emit(BC, Streamer);
 215   }
 216
 217   emitDataSections(OrgSecPrefix);
 218
 219   // TODO Enable for Mach-O once BinaryContext::getDataSection supports it.
 220   if (BC.isELF())
 221     AddressMap::emit(Streamer, BC);
 222   Streamer.setUseAssemblerInfoForParsing(true);
 223 }
 224
 225 void BinaryEmitter::emitFunctions() {
 226   auto emit = [&](const std::vector<BinaryFunction *> &Functions) {
 227     const bool HasProfile = BC.NumProfiledFuncs > 0;
 228     const bool OriginalAllowAutoPadding = Streamer.getAllowAutoPadding();
 229     for (BinaryFunction *Function : Functions) {
 230       if (!BC.shouldEmit(*Function))
 231         continue;
 232
 233       LLVM_DEBUG(dbgs() << "BOLT: generating code for function \"" << *Function
 234                         << "\" : " << Function->getFunctionNumber() << '\n');
 235
 236       // Was any part of the function emitted.
 237       bool Emitted = false;
 238
 239       // Turn off Intel JCC Erratum mitigation for cold code if requested
 240       if (HasProfile && opts::X86AlignBranchBoundaryHotOnly &&
 241           !Function->hasValidProfile())
 242         Streamer.setAllowAutoPadding(false);
 243
 244       FunctionLayout &Layout = Function->getLayout();
 245       Emitted |= emitFunction(*Function, Layout.getMainFragment());
 246
 247       if (Function->isSplit()) {
 248         if (opts::X86AlignBranchBoundaryHotOnly)
 249           Streamer.setAllowAutoPadding(false);
 250
 251         assert((Layout.fragment_size() == 1 || Function->isSimple()) &&
 252                "Only simple functions can have fragments");
 253         for (FunctionFragment &FF : Layout.getSplitFragments()) {
 254           // Skip empty fragments so no symbols and sections for empty fragments
 255           // are generated
 256           if (FF.empty() && !Function->hasConstantIsland())
 257             continue;
 258           Emitted |= emitFunction(*Function, FF);
 259         }
 260       }
 261
 262       Streamer.setAllowAutoPadding(OriginalAllowAutoPadding);
 263
 264       if (Emitted)
 265         Function->setEmitted(/*KeepCFG=*/opts::PrintCacheMetrics);
 266     }
 267   };
 268
 269   // Mark the start of hot text.
 270   if (opts::HotText) {
 271     Streamer.switchSection(BC.getTextSection());
 272     Streamer.emitLabel(BC.getHotTextStartSymbol());
 273   }
 274
 275   // Emit functions in sorted order.
 276   std::vector<BinaryFunction *> SortedFunctions = BC.getSortedFunctions();
 277   emit(SortedFunctions);
 278
 279   // Emit functions added by BOLT.
 280   emit(BC.getInjectedBinaryFunctions());
 281
 282   // Mark the end of hot text.
 283   if (opts::HotText) {
 284     if (BC.HasWarmSection)
 285       Streamer.switchSection(BC.getCodeSection(BC.getWarmCodeSectionName()));
 286     else
 287       Streamer.switchSection(BC.getTextSection());
 288     Streamer.emitLabel(BC.getHotTextEndSymbol());
 289   }
 290 }
 291
 292 bool BinaryEmitter::emitFunction(BinaryFunction &Function,
 293                                  FunctionFragment &FF) {
 294   if (Function.size() == 0 && !Function.hasIslandsInfo())
 295     return false;
 296
 297   if (Function.getState() == BinaryFunction::State::Empty)
 298     return false;
 299
 300   // Avoid emitting function without instructions when overwriting the original
 301   // function in-place. Otherwise, emit the empty function to define the symbol.
 302   if (!BC.HasRelocations && !Function.hasNonPseudoInstructions())
 303     return false;
 304
 305   MCSection *Section =
 306       BC.getCodeSection(Function.getCodeSectionName(FF.getFragmentNum()));
 307   Streamer.switchSection(Section);
 308   Section->setHasInstructions(true);
 309   BC.Ctx->addGenDwarfSection(Section);
 310
 311   if (BC.HasRelocations) {
 312     // Set section alignment to at least maximum possible object alignment.
 313     // We need this to support LongJmp and other passes that calculates
 314     // tentative layout.
 315     Section->ensureMinAlignment(Align(opts::AlignFunctions));
 316
 317     Streamer.emitCodeAlignment(Function.getMinAlign(), &*BC.STI);
 318     uint16_t MaxAlignBytes = FF.isSplitFragment()
 319                                  ? Function.getMaxColdAlignmentBytes()
 320                                  : Function.getMaxAlignmentBytes();
 321     if (MaxAlignBytes > 0)
 322       Streamer.emitCodeAlignment(Function.getAlign(), &*BC.STI, MaxAlignBytes);
 323   } else {
 324     Streamer.emitCodeAlignment(Function.getAlign(), &*BC.STI);
 325   }
 326
 327   if (size_t Padding =
 328           opts::padFunction(opts::FunctionPadBeforeSpec, Function)) {
 329     // Handle padFuncsBefore after the above alignment logic but before
 330     // symbol addresses are decided.
 331     if (!BC.HasRelocations) {
 332       BC.errs() << "BOLT-ERROR: -pad-before-funcs is not supported in "
 333                 << "non-relocation mode\n";
 334       exit(1);
 335     }
 336
 337     // Preserve Function.getMinAlign().
 338     if (!isAligned(Function.getMinAlign(), Padding)) {
 339       BC.errs() << "BOLT-ERROR: user-requested " << Padding
 340                 << " padding bytes before function " << Function
 341                 << " is not a multiple of the minimum function alignment ("
 342                 << Function.getMinAlign().value() << ").\n";
 343       exit(1);
 344     }
 345
 346     LLVM_DEBUG(dbgs() << "BOLT-DEBUG: padding before function " << Function
 347                       << " with " << Padding << " bytes\n");
 348
 349     // Since the padding is not executed, it can be null bytes.
 350     Streamer.emitFill(Padding, 0);
 351   }
 352
 353   MCContext &Context = Streamer.getContext();
 354   const MCAsmInfo *MAI = Context.getAsmInfo();
 355
 356   MCSymbol *const StartSymbol = Function.getSymbol(FF.getFragmentNum());
 357
 358   // Emit all symbols associated with the main function entry.
 359   if (FF.isMainFragment()) {
 360     for (MCSymbol *Symbol : Function.getSymbols()) {
 361       Streamer.emitSymbolAttribute(Symbol, MCSA_ELF_TypeFunction);
 362       Streamer.emitLabel(Symbol);
 363     }
 364   } else {
 365     Streamer.emitSymbolAttribute(StartSymbol, MCSA_ELF_TypeFunction);
 366     Streamer.emitLabel(StartSymbol);
 367   }
 368
 369   // Emit CFI start
 370   if (Function.hasCFI()) {
 371     Streamer.emitCFIStartProc(/*IsSimple=*/false);
 372     if (Function.getPersonalityFunction() != nullptr)
 373       Streamer.emitCFIPersonality(Function.getPersonalityFunction(),
 374                                   Function.getPersonalityEncoding());
 375     MCSymbol *LSDASymbol = Function.getLSDASymbol(FF.getFragmentNum());
 376     if (LSDASymbol)
 377       Streamer.emitCFILsda(LSDASymbol, BC.LSDAEncoding);
 378     else
 379       Streamer.emitCFILsda(0, dwarf::DW_EH_PE_omit);
 380     // Emit CFI instructions relative to the CIE
 381     for (const MCCFIInstruction &CFIInstr : Function.cie()) {
 382       // Only write CIE CFI insns that LLVM will not already emit
 383       const std::vector<MCCFIInstruction> &FrameInstrs =
 384           MAI->getInitialFrameState();
 385       if (!llvm::is_contained(FrameInstrs, CFIInstr))
 386         emitCFIInstruction(CFIInstr);
 387     }
 388   }
 389
 390   assert((Function.empty() || !(*Function.begin()).isCold()) &&
 391          "first basic block should never be cold");
 392
 393   // Emit UD2 at the beginning if requested by user.
 394   if (!opts::BreakFunctionNames.empty()) {
 395     for (std::string &Name : opts::BreakFunctionNames) {
 396       if (Function.hasNameRegex(Name)) {
 397         Streamer.emitIntValue(0x0B0F, 2); // UD2: 0F 0B
 398         break;
 399       }
 400     }
 401   }
 402
 403   // Emit code.
 404   emitFunctionBody(Function, FF, /*EmitCodeOnly=*/false);
 405
 406   // Emit padding if requested.
 407   if (size_t Padding = opts::padFunction(opts::FunctionPadSpec, Function)) {
 408     LLVM_DEBUG(dbgs() << "BOLT-DEBUG: padding function " << Function << " with "
 409                       << Padding << " bytes\n");
 410     Streamer.emitFill(Padding, MAI->getTextAlignFillValue());
 411   }
 412
 413   if (opts::MarkFuncs)
 414     Streamer.emitBytes(BC.MIB->getTrapFillValue());
 415
 416   // Emit CFI end
 417   if (Function.hasCFI())
 418     Streamer.emitCFIEndProc();
 419
 420   MCSymbol *EndSymbol = Function.getFunctionEndLabel(FF.getFragmentNum());
 421   Streamer.emitLabel(EndSymbol);
 422
 423   if (MAI->hasDotTypeDotSizeDirective()) {
 424     const MCExpr *SizeExpr = MCBinaryExpr::createSub(
 425         MCSymbolRefExpr::create(EndSymbol, Context),
 426         MCSymbolRefExpr::create(StartSymbol, Context), Context);
 427     Streamer.emitELFSize(StartSymbol, SizeExpr);
 428   }
 429
 430   if (opts::UpdateDebugSections && Function.getDWARFUnit())
 431     emitLineInfoEnd(Function, EndSymbol);
 432
 433   // Exception handling info for the function.
 434   emitLSDA(Function, FF);
 435
 436   if (FF.isMainFragment() && opts::JumpTables > JTS_NONE)
 437     emitJumpTables(Function);
 438
 439   return true;
 440 }
 441
 442 void BinaryEmitter::emitFunctionBody(BinaryFunction &BF, FunctionFragment &FF,
 443                                      bool EmitCodeOnly) {
 444   if (!EmitCodeOnly && FF.isSplitFragment() && BF.hasConstantIsland()) {
 445     assert(BF.getLayout().isHotColdSplit() &&
 446            "Constant island support only with hot/cold split");
 447     BF.duplicateConstantIslands();
 448   }
 449
 450   // Track the first emitted instruction with debug info.
 451   bool FirstInstr = true;
 452   for (BinaryBasicBlock *const BB : FF) {
 453     if ((opts::AlignBlocks || opts::PreserveBlocksAlignment) &&
 454         BB->getAlignment() > 1)
 455       Streamer.emitCodeAlignment(BB->getAlign(), &*BC.STI,
 456                                  BB->getAlignmentMaxBytes());
 457     Streamer.emitLabel(BB->getLabel());
 458     if (!EmitCodeOnly) {
 459       if (MCSymbol *EntrySymbol = BF.getSecondaryEntryPointSymbol(*BB))
 460         Streamer.emitLabel(EntrySymbol);
 461     }
 462
 463     SMLoc LastLocSeen;
 464     for (auto I = BB->begin(), E = BB->end(); I != E; ++I) {
 465       MCInst &Instr = *I;
 466
 467       if (EmitCodeOnly && BC.MIB->isPseudo(Instr))
 468         continue;
 469
 470       // Handle pseudo instructions.
 471       if (BC.MIB->isCFI(Instr)) {
 472         emitCFIInstruction(*BF.getCFIFor(Instr));
 473         continue;
 474       }
 475
 476       if (!EmitCodeOnly) {
 477         // A symbol to be emitted before the instruction to mark its location.
 478         MCSymbol *InstrLabel = BC.MIB->getInstLabel(Instr);
 479
 480         if (opts::UpdateDebugSections && BF.getDWARFUnit()) {
 481           LastLocSeen = emitLineInfo(BF, Instr.getLoc(), LastLocSeen,
 482                                      FirstInstr, InstrLabel);
 483           FirstInstr = false;
 484         }
 485
 486         // Prepare to tag this location with a label if we need to keep track of
 487         // the location of calls/returns for BOLT address translation maps
 488         if (BF.requiresAddressTranslation() && BC.MIB->getOffset(Instr)) {
 489           const uint32_t Offset = *BC.MIB->getOffset(Instr);
 490           if (!InstrLabel)
 491             InstrLabel = BC.Ctx->createTempSymbol();
 492           BB->getLocSyms().emplace_back(Offset, InstrLabel);
 493         }
 494
 495         if (InstrLabel)
 496           Streamer.emitLabel(InstrLabel);
 497       }
 498
 499       // Emit sized NOPs via MCAsmBackend::writeNopData() interface on x86.
 500       // This is a workaround for invalid NOPs handling by asm/disasm layer.
 501       if (BC.isX86() && BC.MIB->isNoop(Instr)) {
 502         if (std::optional<uint32_t> Size = BC.MIB->getSize(Instr)) {
 503           SmallString<15> Code;
 504           raw_svector_ostream VecOS(Code);
 505           BC.MAB->writeNopData(VecOS, *Size, BC.STI.get());
 506           Streamer.emitBytes(Code);
 507           continue;
 508         }
 509       }
 510
 511       Streamer.emitInstruction(Instr, *BC.STI);
 512     }
 513   }
 514
 515   if (!EmitCodeOnly)
 516     emitConstantIslands(BF, FF.isSplitFragment());
 517 }
 518
 519 void BinaryEmitter::emitConstantIslands(BinaryFunction &BF, bool EmitColdPart,
 520                                         BinaryFunction *OnBehalfOf) {
 521   if (!BF.hasIslandsInfo())
 522     return;
 523
 524   BinaryFunction::IslandInfo &Islands = BF.getIslandInfo();
 525   if (Islands.DataOffsets.empty() && Islands.Dependency.empty())
 526     return;
 527
 528   // AArch64 requires CI to be aligned to 8 bytes due to access instructions
 529   // restrictions. E.g. the ldr with imm, where imm must be aligned to 8 bytes.
 530   const uint16_t Alignment = OnBehalfOf
 531                                  ? OnBehalfOf->getConstantIslandAlignment()
 532                                  : BF.getConstantIslandAlignment();
 533   Streamer.emitCodeAlignment(Align(Alignment), &*BC.STI);
 534
 535   if (!OnBehalfOf) {
 536     if (!EmitColdPart)
 537       Streamer.emitLabel(BF.getFunctionConstantIslandLabel());
 538     else
 539       Streamer.emitLabel(BF.getFunctionColdConstantIslandLabel());
 540   }
 541
 542   assert((!OnBehalfOf || Islands.Proxies[OnBehalfOf].size() > 0) &&
 543          "spurious OnBehalfOf constant island emission");
 544
 545   assert(!BF.isInjected() &&
 546          "injected functions should not have constant islands");
 547   // Raw contents of the function.
 548   StringRef SectionContents = BF.getOriginSection()->getContents();
 549
 550   // Raw contents of the function.
 551   StringRef FunctionContents = SectionContents.substr(
 552       BF.getAddress() - BF.getOriginSection()->getAddress(), BF.getMaxSize());
 553
 554   if (opts::Verbosity && !OnBehalfOf)
 555     BC.outs() << "BOLT-INFO: emitting constant island for function " << BF
 556               << "\n";
 557
 558   // We split the island into smaller blocks and output labels between them.
 559   auto IS = Islands.Offsets.begin();
 560   for (auto DataIter = Islands.DataOffsets.begin();
 561        DataIter != Islands.DataOffsets.end(); ++DataIter) {
 562     uint64_t FunctionOffset = *DataIter;
 563     uint64_t EndOffset = 0ULL;
 564
 565     // Determine size of this data chunk
 566     auto NextData = std::next(DataIter);
 567     auto CodeIter = Islands.CodeOffsets.lower_bound(*DataIter);
 568     if (CodeIter == Islands.CodeOffsets.end() &&
 569         NextData == Islands.DataOffsets.end())
 570       EndOffset = BF.getMaxSize();
 571     else if (CodeIter == Islands.CodeOffsets.end())
 572       EndOffset = *NextData;
 573     else if (NextData == Islands.DataOffsets.end())
 574       EndOffset = *CodeIter;
 575     else
 576       EndOffset = (*CodeIter > *NextData) ? *NextData : *CodeIter;
 577
 578     if (FunctionOffset == EndOffset)
 579       continue; // Size is zero, nothing to emit
 580
 581     auto emitCI = [&](uint64_t &FunctionOffset, uint64_t EndOffset) {
 582       if (FunctionOffset >= EndOffset)
 583         return;
 584
 585       for (auto It = Islands.Relocations.lower_bound(FunctionOffset);
 586            It != Islands.Relocations.end(); ++It) {
 587         if (It->first >= EndOffset)
 588           break;
 589
 590         const Relocation &Relocation = It->second;
 591         if (FunctionOffset < Relocation.Offset) {
 592           Streamer.emitBytes(
 593               FunctionContents.slice(FunctionOffset, Relocation.Offset));
 594           FunctionOffset = Relocation.Offset;
 595         }
 596
 597         LLVM_DEBUG(
 598             dbgs() << "BOLT-DEBUG: emitting constant island relocation"
 599                    << " for " << BF << " at offset 0x"
 600                    << Twine::utohexstr(Relocation.Offset) << " with size "
 601                    << Relocation::getSizeForType(Relocation.Type) << '\n');
 602
 603         FunctionOffset += Relocation.emit(&Streamer);
 604       }
 605
 606       assert(FunctionOffset <= EndOffset && "overflow error");
 607       if (FunctionOffset < EndOffset) {
 608         Streamer.emitBytes(FunctionContents.slice(FunctionOffset, EndOffset));
 609         FunctionOffset = EndOffset;
 610       }
 611     };
 612
 613     // Emit labels, relocs and data
 614     while (IS != Islands.Offsets.end() && IS->first < EndOffset) {
 615       auto NextLabelOffset =
 616           IS == Islands.Offsets.end() ? EndOffset : IS->first;
 617       auto NextStop = std::min(NextLabelOffset, EndOffset);
 618       assert(NextStop <= EndOffset && "internal overflow error");
 619       emitCI(FunctionOffset, NextStop);
 620       if (IS != Islands.Offsets.end() && FunctionOffset == IS->first) {
 621         // This is a slightly complex code to decide which label to emit. We
 622         // have 4 cases to handle: regular symbol, cold symbol, regular or cold
 623         // symbol being emitted on behalf of an external function.
 624         if (!OnBehalfOf) {
 625           if (!EmitColdPart) {
 626             LLVM_DEBUG(dbgs() << "BOLT-DEBUG: emitted label "
 627                               << IS->second->getName() << " at offset 0x"
 628                               << Twine::utohexstr(IS->first) << '\n');
 629             if (IS->second->isUndefined())
 630               Streamer.emitLabel(IS->second);
 631             else
 632               assert(BF.hasName(std::string(IS->second->getName())));
 633           } else if (Islands.ColdSymbols.count(IS->second) != 0) {
 634             LLVM_DEBUG(dbgs()
 635                        << "BOLT-DEBUG: emitted label "
 636                        << Islands.ColdSymbols[IS->second]->getName() << '\n');
 637             if (Islands.ColdSymbols[IS->second]->isUndefined())
 638               Streamer.emitLabel(Islands.ColdSymbols[IS->second]);
 639           }
 640         } else {
 641           if (!EmitColdPart) {
 642             if (MCSymbol *Sym = Islands.Proxies[OnBehalfOf][IS->second]) {
 643               LLVM_DEBUG(dbgs() << "BOLT-DEBUG: emitted label "
 644                                 << Sym->getName() << '\n');
 645               Streamer.emitLabel(Sym);
 646             }
 647           } else if (MCSymbol *Sym =
 648                          Islands.ColdProxies[OnBehalfOf][IS->second]) {
 649             LLVM_DEBUG(dbgs() << "BOLT-DEBUG: emitted label " << Sym->getName()
 650                               << '\n');
 651             Streamer.emitLabel(Sym);
 652           }
 653         }
 654         ++IS;
 655       }
 656     }
 657     assert(FunctionOffset <= EndOffset && "overflow error");
 658     emitCI(FunctionOffset, EndOffset);
 659   }
 660   assert(IS == Islands.Offsets.end() && "some symbols were not emitted!");
 661
 662   if (OnBehalfOf)
 663     return;
 664   // Now emit constant islands from other functions that we may have used in
 665   // this function.
 666   for (BinaryFunction *ExternalFunc : Islands.Dependency)
 667     emitConstantIslands(*ExternalFunc, EmitColdPart, &BF);
 668 }
 669
 670 SMLoc BinaryEmitter::emitLineInfo(const BinaryFunction &BF, SMLoc NewLoc,
 671                                   SMLoc PrevLoc, bool FirstInstr,
 672                                   MCSymbol *&InstrLabel) {
 673   DWARFUnit *FunctionCU = BF.getDWARFUnit();
 674   const DWARFDebugLine::LineTable *FunctionLineTable = BF.getDWARFLineTable();
 675   assert(FunctionCU && "cannot emit line info for function without CU");
 676
 677   DebugLineTableRowRef RowReference = DebugLineTableRowRef::fromSMLoc(NewLoc);
 678
 679   // Check if no new line info needs to be emitted.
 680   if (RowReference == DebugLineTableRowRef::NULL_ROW ||
 681       NewLoc.getPointer() == PrevLoc.getPointer())
 682     return PrevLoc;
 683
 684   unsigned CurrentFilenum = 0;
 685   const DWARFDebugLine::LineTable *CurrentLineTable = FunctionLineTable;
 686
 687   // If the CU id from the current instruction location does not
 688   // match the CU id from the current function, it means that we
 689   // have come across some inlined code.  We must look up the CU
 690   // for the instruction's original function and get the line table
 691   // from that.
 692   const uint64_t FunctionUnitIndex = FunctionCU->getOffset();
 693   const uint32_t CurrentUnitIndex = RowReference.DwCompileUnitIndex;
 694   if (CurrentUnitIndex != FunctionUnitIndex) {
 695     CurrentLineTable = BC.DwCtx->getLineTableForUnit(
 696         BC.DwCtx->getCompileUnitForOffset(CurrentUnitIndex));
 697     // Add filename from the inlined function to the current CU.
 698     CurrentFilenum = BC.addDebugFilenameToUnit(
 699         FunctionUnitIndex, CurrentUnitIndex,
 700         CurrentLineTable->Rows[RowReference.RowIndex - 1].File);
 701   }
 702
 703   const DWARFDebugLine::Row &CurrentRow =
 704       CurrentLineTable->Rows[RowReference.RowIndex - 1];
 705   if (!CurrentFilenum)
 706     CurrentFilenum = CurrentRow.File;
 707
 708   unsigned Flags = (DWARF2_FLAG_IS_STMT * CurrentRow.IsStmt) |
 709                    (DWARF2_FLAG_BASIC_BLOCK * CurrentRow.BasicBlock) |
 710                    (DWARF2_FLAG_PROLOGUE_END * CurrentRow.PrologueEnd) |
 711                    (DWARF2_FLAG_EPILOGUE_BEGIN * CurrentRow.EpilogueBegin);
 712
 713   // Always emit is_stmt at the beginning of function fragment.
 714   if (FirstInstr)
 715     Flags |= DWARF2_FLAG_IS_STMT;
 716
 717   BC.Ctx->setCurrentDwarfLoc(CurrentFilenum, CurrentRow.Line, CurrentRow.Column,
 718                              Flags, CurrentRow.Isa, CurrentRow.Discriminator);
 719   const MCDwarfLoc &DwarfLoc = BC.Ctx->getCurrentDwarfLoc();
 720   BC.Ctx->clearDwarfLocSeen();
 721
 722   if (!InstrLabel)
 723     InstrLabel = BC.Ctx->createTempSymbol();
 724
 725   BC.getDwarfLineTable(FunctionUnitIndex)
 726       .getMCLineSections()
 727       .addLineEntry(MCDwarfLineEntry(InstrLabel, DwarfLoc),
 728                     Streamer.getCurrentSectionOnly());
 729
 730   return NewLoc;
 731 }
 732
 733 void BinaryEmitter::emitLineInfoEnd(const BinaryFunction &BF,
 734                                     MCSymbol *FunctionEndLabel) {
 735   DWARFUnit *FunctionCU = BF.getDWARFUnit();
 736   assert(FunctionCU && "DWARF unit expected");
 737   BC.Ctx->setCurrentDwarfLoc(0, 0, 0, DWARF2_FLAG_END_SEQUENCE, 0, 0);
 738   const MCDwarfLoc &DwarfLoc = BC.Ctx->getCurrentDwarfLoc();
 739   BC.Ctx->clearDwarfLocSeen();
 740   BC.getDwarfLineTable(FunctionCU->getOffset())
 741       .getMCLineSections()
 742       .addLineEntry(MCDwarfLineEntry(FunctionEndLabel, DwarfLoc),
 743                     Streamer.getCurrentSectionOnly());
 744 }
 745
 746 void BinaryEmitter::emitJumpTables(const BinaryFunction &BF) {
 747   MCSection *ReadOnlySection = BC.MOFI->getReadOnlySection();
 748   MCSection *ReadOnlyColdSection = BC.MOFI->getContext().getELFSection(
 749       ".rodata.cold", ELF::SHT_PROGBITS, ELF::SHF_ALLOC);
 750
 751   if (!BF.hasJumpTables())
 752     return;
 753
 754   if (opts::PrintJumpTables)
 755     BC.outs() << "BOLT-INFO: jump tables for function " << BF << ":\n";
 756
 757   for (auto &JTI : BF.jumpTables()) {
 758     JumpTable &JT = *JTI.second;
 759     // Only emit shared jump tables once, when processing the first parent
 760     if (JT.Parents.size() > 1 && JT.Parents[0] != &BF)
 761       continue;
 762     if (opts::PrintJumpTables)
 763       JT.print(BC.outs());
 764     if (opts::JumpTables == JTS_BASIC) {
 765       JT.updateOriginal();
 766     } else {
 767       MCSection *HotSection, *ColdSection;
 768       if (BF.isSimple()) {
 769         HotSection = ReadOnlySection;
 770         ColdSection = ReadOnlyColdSection;
 771       } else {
 772         HotSection = BF.hasProfile() ? ReadOnlySection : ReadOnlyColdSection;
 773         ColdSection = HotSection;
 774       }
 775       emitJumpTable(JT, HotSection, ColdSection);
 776     }
 777   }
 778 }
 779
 780 void BinaryEmitter::emitJumpTable(const JumpTable &JT, MCSection *HotSection,
 781                                   MCSection *ColdSection) {
 782   // Pre-process entries for aggressive splitting.
 783   // Each label represents a separate switch table and gets its own count
 784   // determining its destination.
 785   std::map<MCSymbol *, uint64_t> LabelCounts;
 786   if (opts::JumpTables > JTS_SPLIT && !JT.Counts.empty()) {
 787     auto It = JT.Labels.find(0);
 788     assert(It != JT.Labels.end());
 789     MCSymbol *CurrentLabel = It->second;
 790     uint64_t CurrentLabelCount = 0;
 791     for (unsigned Index = 0; Index < JT.Entries.size(); ++Index) {
 792       auto LI = JT.Labels.find(Index * JT.EntrySize);
 793       if (LI != JT.Labels.end()) {
 794         LabelCounts[CurrentLabel] = CurrentLabelCount;
 795         CurrentLabel = LI->second;
 796         CurrentLabelCount = 0;
 797       }
 798       CurrentLabelCount += JT.Counts[Index].Count;
 799     }
 800     LabelCounts[CurrentLabel] = CurrentLabelCount;
 801   } else {
 802     Streamer.switchSection(JT.Count > 0 ? HotSection : ColdSection);
 803     Streamer.emitValueToAlignment(Align(JT.EntrySize));
 804   }
 805   MCSymbol *LastLabel = nullptr;
 806   uint64_t Offset = 0;
 807   for (MCSymbol *Entry : JT.Entries) {
 808     auto LI = JT.Labels.find(Offset);
 809     if (LI != JT.Labels.end()) {
 810       LLVM_DEBUG({
 811         dbgs() << "BOLT-DEBUG: emitting jump table " << LI->second->getName()
 812                << " (originally was at address 0x"
 813                << Twine::utohexstr(JT.getAddress() + Offset)
 814                << (Offset ? ") as part of larger jump table\n" : ")\n");
 815       });
 816       if (!LabelCounts.empty()) {
 817         LLVM_DEBUG(dbgs() << "BOLT-DEBUG: jump table count: "
 818                           << LabelCounts[LI->second] << '\n');
 819         if (LabelCounts[LI->second] > 0)
 820           Streamer.switchSection(HotSection);
 821         else
 822           Streamer.switchSection(ColdSection);
 823         Streamer.emitValueToAlignment(Align(JT.EntrySize));
 824       }
 825       // Emit all labels registered at the address of this jump table
 826       // to sync with our global symbol table.  We may have two labels
 827       // registered at this address if one label was created via
 828       // getOrCreateGlobalSymbol() (e.g. LEA instructions referencing
 829       // this location) and another via getOrCreateJumpTable().  This
 830       // creates a race where the symbols created by these two
 831       // functions may or may not be the same, but they are both
 832       // registered in our symbol table at the same address. By
 833       // emitting them all here we make sure there is no ambiguity
 834       // that depends on the order that these symbols were created, so
 835       // whenever this address is referenced in the binary, it is
 836       // certain to point to the jump table identified at this
 837       // address.
 838       if (BinaryData *BD = BC.getBinaryDataByName(LI->second->getName())) {
 839         for (MCSymbol *S : BD->getSymbols())
 840           Streamer.emitLabel(S);
 841       } else {
 842         Streamer.emitLabel(LI->second);
 843       }
 844       LastLabel = LI->second;
 845     }
 846     if (JT.Type == JumpTable::JTT_NORMAL) {
 847       Streamer.emitSymbolValue(Entry, JT.OutputEntrySize);
 848     } else { // JTT_PIC
 849       const MCSymbolRefExpr *JTExpr =
 850           MCSymbolRefExpr::create(LastLabel, Streamer.getContext());
 851       const MCSymbolRefExpr *E =
 852           MCSymbolRefExpr::create(Entry, Streamer.getContext());
 853       const MCBinaryExpr *Value =
 854           MCBinaryExpr::createSub(E, JTExpr, Streamer.getContext());
 855       Streamer.emitValue(Value, JT.EntrySize);
 856     }
 857     Offset += JT.EntrySize;
 858   }
 859 }
 860
 861 void BinaryEmitter::emitCFIInstruction(const MCCFIInstruction &Inst) const {
 862   switch (Inst.getOperation()) {
 863   default:
 864     llvm_unreachable("Unexpected instruction");
 865   case MCCFIInstruction::OpDefCfaOffset:
 866     Streamer.emitCFIDefCfaOffset(Inst.getOffset());
 867     break;
 868   case MCCFIInstruction::OpAdjustCfaOffset:
 869     Streamer.emitCFIAdjustCfaOffset(Inst.getOffset());
 870     break;
 871   case MCCFIInstruction::OpDefCfa:
 872     Streamer.emitCFIDefCfa(Inst.getRegister(), Inst.getOffset());
 873     break;
 874   case MCCFIInstruction::OpDefCfaRegister:
 875     Streamer.emitCFIDefCfaRegister(Inst.getRegister());
 876     break;
 877   case MCCFIInstruction::OpOffset:
 878     Streamer.emitCFIOffset(Inst.getRegister(), Inst.getOffset());
 879     break;
 880   case MCCFIInstruction::OpRegister:
 881     Streamer.emitCFIRegister(Inst.getRegister(), Inst.getRegister2());
 882     break;
 883   case MCCFIInstruction::OpWindowSave:
 884     Streamer.emitCFIWindowSave();
 885     break;
 886   case MCCFIInstruction::OpNegateRAState:
 887     Streamer.emitCFINegateRAState();
 888     break;
 889   case MCCFIInstruction::OpSameValue:
 890     Streamer.emitCFISameValue(Inst.getRegister());
 891     break;
 892   case MCCFIInstruction::OpGnuArgsSize:
 893     Streamer.emitCFIGnuArgsSize(Inst.getOffset());
 894     break;
 895   case MCCFIInstruction::OpEscape:
 896     Streamer.AddComment(Inst.getComment());
 897     Streamer.emitCFIEscape(Inst.getValues());
 898     break;
 899   case MCCFIInstruction::OpRestore:
 900     Streamer.emitCFIRestore(Inst.getRegister());
 901     break;
 902   case MCCFIInstruction::OpUndefined:
 903     Streamer.emitCFIUndefined(Inst.getRegister());
 904     break;
 905   }
 906 }
 907
 908 // The code is based on EHStreamer::emitExceptionTable().
 909 void BinaryEmitter::emitLSDA(BinaryFunction &BF, const FunctionFragment &FF) {
 910   const BinaryFunction::CallSitesRange Sites =
 911       BF.getCallSites(FF.getFragmentNum());
 912   if (Sites.empty())
 913     return;
 914
 915   Streamer.switchSection(BC.MOFI->getLSDASection());
 916
 917   const unsigned TTypeEncoding = BF.getLSDATypeEncoding();
 918   const unsigned TTypeEncodingSize = BC.getDWARFEncodingSize(TTypeEncoding);
 919   const uint16_t TTypeAlignment = 4;
 920
 921   // Type tables have to be aligned at 4 bytes.
 922   Streamer.emitValueToAlignment(Align(TTypeAlignment));
 923
 924   // Emit the LSDA label.
 925   MCSymbol *LSDASymbol = BF.getLSDASymbol(FF.getFragmentNum());
 926   assert(LSDASymbol && "no LSDA symbol set");
 927   Streamer.emitLabel(LSDASymbol);
 928
 929   // Corresponding FDE start.
 930   const MCSymbol *StartSymbol = BF.getSymbol(FF.getFragmentNum());
 931
 932   // Emit the LSDA header.
 933
 934   // If LPStart is omitted, then the start of the FDE is used as a base for
 935   // landing pad displacements. Then, if a cold fragment starts with
 936   // a landing pad, this means that the first landing pad offset will be 0.
 937   // However, C++ runtime will treat 0 as if there is no landing pad, thus we
 938   // cannot emit LP offset as 0.
 939   //
 940   // As a solution, for fixed-address binaries we set LPStart to 0, and for
 941   // position-independent binaries we offset LP start by one byte.
 942   bool NeedsLPAdjustment = false;
 943   std::function<void(const MCSymbol *)> emitLandingPad;
 944
 945   // Check if there's a symbol associated with a landing pad fragment.
 946   const MCSymbol *LPStartSymbol = BF.getLPStartSymbol(FF.getFragmentNum());
 947   if (!LPStartSymbol) {
 948     // Since landing pads are not in the same fragment, we fall back to emitting
 949     // absolute addresses for this FDE.
 950     if (opts::Verbosity >= 2) {
 951       BC.outs() << "BOLT-INFO: falling back to generating absolute-address "
 952                 << "exception ranges for " << BF << '\n';
 953     }
 954
 955     assert(BC.HasFixedLoadAddress &&
 956            "Cannot emit absolute-address landing pads for PIE/DSO");
 957
 958     Streamer.emitIntValue(dwarf::DW_EH_PE_udata4, 1); // LPStart format
 959     Streamer.emitIntValue(0, 4);                      // LPStart
 960     emitLandingPad = [&](const MCSymbol *LPSymbol) {
 961       if (LPSymbol)
 962         Streamer.emitSymbolValue(LPSymbol, 4);
 963       else
 964         Streamer.emitIntValue(0, 4);
 965     };
 966   } else {
 967     std::optional<FragmentNum> LPFN = BF.getLPFragment(FF.getFragmentNum());
 968     const FunctionFragment &LPFragment = BF.getLayout().getFragment(*LPFN);
 969     NeedsLPAdjustment =
 970         (!LPFragment.empty() && LPFragment.front()->isLandingPad());
 971
 972     // Emit LPStart encoding and optionally LPStart.
 973     if (NeedsLPAdjustment || LPStartSymbol != StartSymbol) {
 974       Streamer.emitIntValue(dwarf::DW_EH_PE_pcrel | dwarf::DW_EH_PE_sdata4, 1);
 975       MCSymbol *DotSymbol = BC.Ctx->createTempSymbol("LPBase");
 976       Streamer.emitLabel(DotSymbol);
 977
 978       const MCExpr *LPStartExpr = MCBinaryExpr::createSub(
 979           MCSymbolRefExpr::create(LPStartSymbol, *BC.Ctx),
 980           MCSymbolRefExpr::create(DotSymbol, *BC.Ctx), *BC.Ctx);
 981       if (NeedsLPAdjustment)
 982         LPStartExpr = MCBinaryExpr::createSub(
 983             LPStartExpr, MCConstantExpr::create(1, *BC.Ctx), *BC.Ctx);
 984       Streamer.emitValue(LPStartExpr, 4);
 985     } else {
 986       // DW_EH_PE_omit means FDE start (StartSymbol) will be used as LPStart.
 987       Streamer.emitIntValue(dwarf::DW_EH_PE_omit, 1);
 988     }
 989     emitLandingPad = [&](const MCSymbol *LPSymbol) {
 990       if (LPSymbol) {
 991         const MCExpr *LPOffsetExpr = MCBinaryExpr::createSub(
 992             MCSymbolRefExpr::create(LPSymbol, *BC.Ctx),
 993             MCSymbolRefExpr::create(LPStartSymbol, *BC.Ctx), *BC.Ctx);
 994         if (NeedsLPAdjustment)
 995           LPOffsetExpr = MCBinaryExpr::createAdd(
 996               LPOffsetExpr, MCConstantExpr::create(1, *BC.Ctx), *BC.Ctx);
 997         Streamer.emitULEB128Value(LPOffsetExpr);
 998       } else {
 999         Streamer.emitULEB128IntValue(0);
1000       }
1001     };
1002   }
1003
1004   Streamer.emitIntValue(TTypeEncoding, 1); // TType format
1005
1006   MCSymbol *TTBaseLabel = nullptr;
1007   if (TTypeEncoding != dwarf::DW_EH_PE_omit) {
1008     TTBaseLabel = BC.Ctx->createTempSymbol("TTBase");
1009     MCSymbol *TTBaseRefLabel = BC.Ctx->createTempSymbol("TTBaseRef");
1010     Streamer.emitAbsoluteSymbolDiffAsULEB128(TTBaseLabel, TTBaseRefLabel);
1011     Streamer.emitLabel(TTBaseRefLabel);
1012   }
1013
1014   // Emit encoding of entries in the call site table. The format is used for the
1015   // call site start, length, and corresponding landing pad.
1016   if (!LPStartSymbol)
1017     Streamer.emitIntValue(dwarf::DW_EH_PE_sdata4, 1);
1018   else
1019     Streamer.emitIntValue(dwarf::DW_EH_PE_uleb128, 1);
1020
1021   MCSymbol *CSTStartLabel = BC.Ctx->createTempSymbol("CSTStart");
1022   MCSymbol *CSTEndLabel = BC.Ctx->createTempSymbol("CSTEnd");
1023   Streamer.emitAbsoluteSymbolDiffAsULEB128(CSTEndLabel, CSTStartLabel);
1024
1025   Streamer.emitLabel(CSTStartLabel);
1026   for (const auto &FragmentCallSite : Sites) {
1027     const BinaryFunction::CallSite &CallSite = FragmentCallSite.second;
1028     const MCSymbol *BeginLabel = CallSite.Start;
1029     const MCSymbol *EndLabel = CallSite.End;
1030
1031     assert(BeginLabel && "start EH label expected");
1032     assert(EndLabel && "end EH label expected");
1033
1034     // Start of the range is emitted relative to the start of current
1035     // function split part.
1036     if (!LPStartSymbol) {
1037       Streamer.emitAbsoluteSymbolDiff(BeginLabel, StartSymbol, 4);
1038       Streamer.emitAbsoluteSymbolDiff(EndLabel, BeginLabel, 4);
1039     } else {
1040       Streamer.emitAbsoluteSymbolDiffAsULEB128(BeginLabel, StartSymbol);
1041       Streamer.emitAbsoluteSymbolDiffAsULEB128(EndLabel, BeginLabel);
1042     }
1043     emitLandingPad(CallSite.LP);
1044     Streamer.emitULEB128IntValue(CallSite.Action);
1045   }
1046   Streamer.emitLabel(CSTEndLabel);
1047
1048   // Write out action, type, and type index tables at the end.
1049   //
1050   // For action and type index tables there's no need to change the original
1051   // table format unless we are doing function splitting, in which case we can
1052   // split and optimize the tables.
1053   //
1054   // For type table we (re-)encode the table using TTypeEncoding matching
1055   // the current assembler mode.
1056   for (uint8_t const &Byte : BF.getLSDAActionTable())
1057     Streamer.emitIntValue(Byte, 1);
1058
1059   const BinaryFunction::LSDATypeTableTy &TypeTable =
1060       (TTypeEncoding & dwarf::DW_EH_PE_indirect) ? BF.getLSDATypeAddressTable()
1061                                                  : BF.getLSDATypeTable();
1062   assert(TypeTable.size() == BF.getLSDATypeTable().size() &&
1063          "indirect type table size mismatch");
1064
1065   Streamer.emitValueToAlignment(Align(TTypeAlignment));
1066
1067   for (int Index = TypeTable.size() - 1; Index >= 0; --Index) {
1068     const uint64_t TypeAddress = TypeTable[Index];
1069     switch (TTypeEncoding & 0x70) {
1070     default:
1071       llvm_unreachable("unsupported TTypeEncoding");
1072     case dwarf::DW_EH_PE_absptr:
1073       Streamer.emitIntValue(TypeAddress, TTypeEncodingSize);
1074       break;
1075     case dwarf::DW_EH_PE_pcrel: {
1076       if (TypeAddress) {
1077         const MCSymbol *TypeSymbol =
1078             BC.getOrCreateGlobalSymbol(TypeAddress, "TI", 0, TTypeAlignment);
1079         MCSymbol *DotSymbol = BC.Ctx->createNamedTempSymbol();
1080         Streamer.emitLabel(DotSymbol);
1081         const MCBinaryExpr *SubDotExpr = MCBinaryExpr::createSub(
1082             MCSymbolRefExpr::create(TypeSymbol, *BC.Ctx),
1083             MCSymbolRefExpr::create(DotSymbol, *BC.Ctx), *BC.Ctx);
1084         Streamer.emitValue(SubDotExpr, TTypeEncodingSize);
1085       } else {
1086         Streamer.emitIntValue(0, TTypeEncodingSize);
1087       }
1088       break;
1089     }
1090     }
1091   }
1092
1093   if (TTypeEncoding != dwarf::DW_EH_PE_omit)
1094     Streamer.emitLabel(TTBaseLabel);
1095
1096   for (uint8_t const &Byte : BF.getLSDATypeIndexTable())
1097     Streamer.emitIntValue(Byte, 1);
1098 }
1099
1100 void BinaryEmitter::emitDebugLineInfoForOriginalFunctions() {
1101   // If a function is in a CU containing at least one processed function, we
1102   // have to rewrite the whole line table for that CU. For unprocessed functions
1103   // we use data from the input line table.
1104   for (auto &It : BC.getBinaryFunctions()) {
1105     const BinaryFunction &Function = It.second;
1106
1107     // If the function was emitted, its line info was emitted with it.
1108     if (Function.isEmitted())
1109       continue;
1110
1111     const DWARFDebugLine::LineTable *LineTable = Function.getDWARFLineTable();
1112     if (!LineTable)
1113       continue; // nothing to update for this function
1114
1115     const uint64_t Address = Function.getAddress();
1116     std::vector<uint32_t> Results;
1117     if (!LineTable->lookupAddressRange(
1118             {Address, object::SectionedAddress::UndefSection},
1119             Function.getSize(), Results))
1120       continue;
1121
1122     if (Results.empty())
1123       continue;
1124
1125     // The first row returned could be the last row matching the start address.
1126     // Find the first row with the same address that is not the end of the
1127     // sequence.
1128     uint64_t FirstRow = Results.front();
1129     while (FirstRow > 0) {
1130       const DWARFDebugLine::Row &PrevRow = LineTable->Rows[FirstRow - 1];
1131       if (PrevRow.Address.Address != Address || PrevRow.EndSequence)
1132         break;
1133       --FirstRow;
1134     }
1135
1136     const uint64_t EndOfSequenceAddress =
1137         Function.getAddress() + Function.getMaxSize();
1138     BC.getDwarfLineTable(Function.getDWARFUnit()->getOffset())
1139         .addLineTableSequence(LineTable, FirstRow, Results.back(),
1140                               EndOfSequenceAddress);
1141   }
1142
1143   // For units that are completely unprocessed, use original debug line contents
1144   // eliminating the need to regenerate line info program.
1145   emitDebugLineInfoForUnprocessedCUs();
1146 }
1147
1148 void BinaryEmitter::emitDebugLineInfoForUnprocessedCUs() {
1149   // Sorted list of section offsets provides boundaries for section fragments,
1150   // where each fragment is the unit's contribution to debug line section.
1151   std::vector<uint64_t> StmtListOffsets;
1152   StmtListOffsets.reserve(BC.DwCtx->getNumCompileUnits());
1153   for (const std::unique_ptr<DWARFUnit> &CU : BC.DwCtx->compile_units()) {
1154     DWARFDie CUDie = CU->getUnitDIE();
1155     auto StmtList = dwarf::toSectionOffset(CUDie.find(dwarf::DW_AT_stmt_list));
1156     if (!StmtList)
1157       continue;
1158
1159     StmtListOffsets.push_back(*StmtList);
1160   }
1161   llvm::sort(StmtListOffsets);
1162
1163   // For each CU that was not processed, emit its line info as a binary blob.
1164   for (const std::unique_ptr<DWARFUnit> &CU : BC.DwCtx->compile_units()) {
1165     if (BC.ProcessedCUs.count(CU.get()))
1166       continue;
1167
1168     DWARFDie CUDie = CU->getUnitDIE();
1169     auto StmtList = dwarf::toSectionOffset(CUDie.find(dwarf::DW_AT_stmt_list));
1170     if (!StmtList)
1171       continue;
1172
1173     StringRef DebugLineContents = CU->getLineSection().Data;
1174
1175     const uint64_t Begin = *StmtList;
1176
1177     // Statement list ends where the next unit contribution begins, or at the
1178     // end of the section.
1179     auto It = llvm::upper_bound(StmtListOffsets, Begin);
1180     const uint64_t End =
1181         It == StmtListOffsets.end() ? DebugLineContents.size() : *It;
1182
1183     BC.getDwarfLineTable(CU->getOffset())
1184         .addRawContents(DebugLineContents.slice(Begin, End));
1185   }
1186 }
1187
1188 void BinaryEmitter::emitDataSections(StringRef OrgSecPrefix) {
1189   for (BinarySection &Section : BC.sections()) {
1190     if (!Section.hasRelocations())
1191       continue;
1192
1193     StringRef Prefix = Section.hasSectionRef() ? OrgSecPrefix : "";
1194     Section.emitAsData(Streamer, Prefix + Section.getName());
1195     Section.clearRelocations();
1196   }
1197 }
1198
1199 namespace llvm {
1200 namespace bolt {
1201
1202 void emitBinaryContext(MCStreamer &Streamer, BinaryContext &BC,
1203                        StringRef OrgSecPrefix) {
1204   BinaryEmitter(Streamer, BC).emitAll(OrgSecPrefix);
1205 }
1206
1207 void emitFunctionBody(MCStreamer &Streamer, BinaryFunction &BF,
1208                       FunctionFragment &FF, bool EmitCodeOnly) {
1209   BinaryEmitter(Streamer, BF.getBinaryContext())
1210       .emitFunctionBody(BF, FF, EmitCodeOnly);
1211 }
1212
1213 } // namespace bolt
1214 } // namespace llvm