bolt/lib/Core/BinaryEmitter.cpp

   1 //===- bolt/Core/BinaryEmitter.cpp - Emit code and data -------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file implements the collection of functions and classes used for
  10 // emission of code and data into object/binary file.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "bolt/Core/BinaryEmitter.h"
  15 #include "bolt/Core/BinaryContext.h"
  16 #include "bolt/Core/BinaryFunction.h"
  17 #include "bolt/Core/DebugData.h"
  18 #include "bolt/Core/FunctionLayout.h"
  19 #include "bolt/Utils/CommandLineOpts.h"
  20 #include "bolt/Utils/Utils.h"
  21 #include "llvm/DebugInfo/DWARF/DWARFCompileUnit.h"
  22 #include "llvm/MC/MCSection.h"
  23 #include "llvm/MC/MCStreamer.h"
  24 #include "llvm/Support/CommandLine.h"
  25 #include "llvm/Support/LEB128.h"
  26 #include "llvm/Support/SMLoc.h"
  27
  28 #define DEBUG_TYPE "bolt"
  29
  30 using namespace llvm;
  31 using namespace bolt;
  32
  33 namespace opts {
  34
  35 extern cl::opt<JumpTableSupportLevel> JumpTables;
  36 extern cl::opt<bool> PreserveBlocksAlignment;
  37
  38 cl::opt<bool> AlignBlocks("align-blocks", cl::desc("align basic blocks"),
  39                           cl::cat(BoltOptCategory));
  40
  41 cl::opt<MacroFusionType>
  42 AlignMacroOpFusion("align-macro-fusion",
  43   cl::desc("fix instruction alignment for macro-fusion (x86 relocation mode)"),
  44   cl::init(MFT_HOT),
  45   cl::values(clEnumValN(MFT_NONE, "none",
  46                "do not insert alignment no-ops for macro-fusion"),
  47              clEnumValN(MFT_HOT, "hot",
  48                "only insert alignment no-ops on hot execution paths (default)"),
  49              clEnumValN(MFT_ALL, "all",
  50                "always align instructions to allow macro-fusion")),
  51   cl::ZeroOrMore,
  52   cl::cat(BoltRelocCategory));
  53
  54 static cl::list<std::string>
  55 BreakFunctionNames("break-funcs",
  56   cl::CommaSeparated,
  57   cl::desc("list of functions to core dump on (debugging)"),
  58   cl::value_desc("func1,func2,func3,..."),
  59   cl::Hidden,
  60   cl::cat(BoltCategory));
  61
  62 static cl::list<std::string>
  63 FunctionPadSpec("pad-funcs",
  64   cl::CommaSeparated,
  65   cl::desc("list of functions to pad with amount of bytes"),
  66   cl::value_desc("func1:pad1,func2:pad2,func3:pad3,..."),
  67   cl::Hidden,
  68   cl::cat(BoltCategory));
  69
  70 static cl::opt<bool> MarkFuncs(
  71     "mark-funcs",
  72     cl::desc("mark function boundaries with break instruction to make "
  73              "sure we accidentally don't cross them"),
  74     cl::ReallyHidden, cl::cat(BoltCategory));
  75
  76 static cl::opt<bool> PrintJumpTables("print-jump-tables",
  77                                      cl::desc("print jump tables"), cl::Hidden,
  78                                      cl::cat(BoltCategory));
  79
  80 static cl::opt<bool>
  81 X86AlignBranchBoundaryHotOnly("x86-align-branch-boundary-hot-only",
  82   cl::desc("only apply branch boundary alignment in hot code"),
  83   cl::init(true),
  84   cl::cat(BoltOptCategory));
  85
  86 size_t padFunction(const BinaryFunction &Function) {
  87   static std::map<std::string, size_t> FunctionPadding;
  88
  89   if (FunctionPadding.empty() && !FunctionPadSpec.empty()) {
  90     for (std::string &Spec : FunctionPadSpec) {
  91       size_t N = Spec.find(':');
  92       if (N == std::string::npos)
  93         continue;
  94       std::string Name = Spec.substr(0, N);
  95       size_t Padding = std::stoull(Spec.substr(N + 1));
  96       FunctionPadding[Name] = Padding;
  97     }
  98   }
  99
 100   for (auto &FPI : FunctionPadding) {
 101     std::string Name = FPI.first;
 102     size_t Padding = FPI.second;
 103     if (Function.hasNameRegex(Name))
 104       return Padding;
 105   }
 106
 107   return 0;
 108 }
 109
 110 } // namespace opts
 111
 112 namespace {
 113 using JumpTable = bolt::JumpTable;
 114
 115 class BinaryEmitter {
 116 private:
 117   BinaryEmitter(const BinaryEmitter &) = delete;
 118   BinaryEmitter &operator=(const BinaryEmitter &) = delete;
 119
 120   MCStreamer &Streamer;
 121   BinaryContext &BC;
 122
 123 public:
 124   BinaryEmitter(MCStreamer &Streamer, BinaryContext &BC)
 125       : Streamer(Streamer), BC(BC) {}
 126
 127   /// Emit all code and data.
 128   void emitAll(StringRef OrgSecPrefix);
 129
 130   /// Emit function code. The caller is responsible for emitting function
 131   /// symbol(s) and setting the section to emit the code to.
 132   void emitFunctionBody(BinaryFunction &BF, FunctionFragment &FF,
 133                         bool EmitCodeOnly = false);
 134
 135 private:
 136   /// Emit function code.
 137   void emitFunctions();
 138
 139   /// Emit a single function.
 140   bool emitFunction(BinaryFunction &BF, FunctionFragment &FF);
 141
 142   /// Helper for emitFunctionBody to write data inside a function
 143   /// (used for AArch64)
 144   void emitConstantIslands(BinaryFunction &BF, bool EmitColdPart,
 145                            BinaryFunction *OnBehalfOf = nullptr);
 146
 147   /// Emit jump tables for the function.
 148   void emitJumpTables(const BinaryFunction &BF);
 149
 150   /// Emit jump table data. Callee supplies sections for the data.
 151   void emitJumpTable(const JumpTable &JT, MCSection *HotSection,
 152                      MCSection *ColdSection);
 153
 154   void emitCFIInstruction(const MCCFIInstruction &Inst) const;
 155
 156   /// Emit exception handling ranges for the function.
 157   void emitLSDA(BinaryFunction &BF, const FunctionFragment &FF);
 158
 159   /// Emit line number information corresponding to \p NewLoc. \p PrevLoc
 160   /// provides a context for de-duplication of line number info.
 161   /// \p FirstInstr indicates if \p NewLoc represents the first instruction
 162   /// in a sequence, such as a function fragment.
 163   ///
 164   /// Return new current location which is either \p NewLoc or \p PrevLoc.
 165   SMLoc emitLineInfo(const BinaryFunction &BF, SMLoc NewLoc, SMLoc PrevLoc,
 166                      bool FirstInstr);
 167
 168   /// Use \p FunctionEndSymbol to mark the end of the line info sequence.
 169   /// Note that it does not automatically result in the insertion of the EOS
 170   /// marker in the line table program, but provides one to the DWARF generator
 171   /// when it needs it.
 172   void emitLineInfoEnd(const BinaryFunction &BF, MCSymbol *FunctionEndSymbol);
 173
 174   /// Emit debug line info for unprocessed functions from CUs that include
 175   /// emitted functions.
 176   void emitDebugLineInfoForOriginalFunctions();
 177
 178   /// Emit debug line for CUs that were not modified.
 179   void emitDebugLineInfoForUnprocessedCUs();
 180
 181   /// Emit data sections that have code references in them.
 182   void emitDataSections(StringRef OrgSecPrefix);
 183 };
 184
 185 } // anonymous namespace
 186
 187 void BinaryEmitter::emitAll(StringRef OrgSecPrefix) {
 188   Streamer.initSections(false, *BC.STI);
 189
 190   if (opts::UpdateDebugSections && BC.isELF()) {
 191     // Force the emission of debug line info into allocatable section to ensure
 192     // JITLink will process it.
 193     //
 194     // NB: on MachO all sections are required for execution, hence no need
 195     //     to change flags/attributes.
 196     MCSectionELF *ELFDwarfLineSection =
 197         static_cast<MCSectionELF *>(BC.MOFI->getDwarfLineSection());
 198     ELFDwarfLineSection->setFlags(ELF::SHF_ALLOC);
 199     MCSectionELF *ELFDwarfLineStrSection =
 200         static_cast<MCSectionELF *>(BC.MOFI->getDwarfLineStrSection());
 201     ELFDwarfLineStrSection->setFlags(ELF::SHF_ALLOC);
 202   }
 203
 204   if (RuntimeLibrary *RtLibrary = BC.getRuntimeLibrary())
 205     RtLibrary->emitBinary(BC, Streamer);
 206
 207   BC.getTextSection()->setAlignment(Align(opts::AlignText));
 208
 209   emitFunctions();
 210
 211   if (opts::UpdateDebugSections) {
 212     emitDebugLineInfoForOriginalFunctions();
 213     DwarfLineTable::emit(BC, Streamer);
 214   }
 215
 216   emitDataSections(OrgSecPrefix);
 217
 218   // TODO Enable for Mach-O once BinaryContext::getDataSection supports it.
 219   if (BC.isELF())
 220     AddressMap::emit(Streamer, BC);
 221 }
 222
 223 void BinaryEmitter::emitFunctions() {
 224   auto emit = [&](const std::vector<BinaryFunction *> &Functions) {
 225     const bool HasProfile = BC.NumProfiledFuncs > 0;
 226     const bool OriginalAllowAutoPadding = Streamer.getAllowAutoPadding();
 227     for (BinaryFunction *Function : Functions) {
 228       if (!BC.shouldEmit(*Function))
 229         continue;
 230
 231       LLVM_DEBUG(dbgs() << "BOLT: generating code for function \"" << *Function
 232                         << "\" : " << Function->getFunctionNumber() << '\n');
 233
 234       // Was any part of the function emitted.
 235       bool Emitted = false;
 236
 237       // Turn off Intel JCC Erratum mitigation for cold code if requested
 238       if (HasProfile && opts::X86AlignBranchBoundaryHotOnly &&
 239           !Function->hasValidProfile())
 240         Streamer.setAllowAutoPadding(false);
 241
 242       FunctionLayout &Layout = Function->getLayout();
 243       Emitted |= emitFunction(*Function, Layout.getMainFragment());
 244
 245       if (Function->isSplit()) {
 246         if (opts::X86AlignBranchBoundaryHotOnly)
 247           Streamer.setAllowAutoPadding(false);
 248
 249         assert((Layout.fragment_size() == 1 || Function->isSimple()) &&
 250                "Only simple functions can have fragments");
 251         for (FunctionFragment &FF : Layout.getSplitFragments()) {
 252           // Skip empty fragments so no symbols and sections for empty fragments
 253           // are generated
 254           if (FF.empty() && !Function->hasConstantIsland())
 255             continue;
 256           Emitted |= emitFunction(*Function, FF);
 257         }
 258       }
 259
 260       Streamer.setAllowAutoPadding(OriginalAllowAutoPadding);
 261
 262       if (Emitted)
 263         Function->setEmitted(/*KeepCFG=*/opts::PrintCacheMetrics);
 264     }
 265   };
 266
 267   // Mark the start of hot text.
 268   if (opts::HotText) {
 269     Streamer.switchSection(BC.getTextSection());
 270     Streamer.emitLabel(BC.getHotTextStartSymbol());
 271   }
 272
 273   // Emit functions in sorted order.
 274   std::vector<BinaryFunction *> SortedFunctions = BC.getSortedFunctions();
 275   emit(SortedFunctions);
 276
 277   // Emit functions added by BOLT.
 278   emit(BC.getInjectedBinaryFunctions());
 279
 280   // Mark the end of hot text.
 281   if (opts::HotText) {
 282     Streamer.switchSection(BC.getTextSection());
 283     Streamer.emitLabel(BC.getHotTextEndSymbol());
 284   }
 285 }
 286
 287 bool BinaryEmitter::emitFunction(BinaryFunction &Function,
 288                                  FunctionFragment &FF) {
 289   if (Function.size() == 0 && !Function.hasIslandsInfo())
 290     return false;
 291
 292   if (Function.getState() == BinaryFunction::State::Empty)
 293     return false;
 294
 295   // Avoid emitting function without instructions when overwriting the original
 296   // function in-place. Otherwise, emit the empty function to define the symbol.
 297   if (!BC.HasRelocations && !Function.hasNonPseudoInstructions())
 298     return false;
 299
 300   MCSection *Section =
 301       BC.getCodeSection(Function.getCodeSectionName(FF.getFragmentNum()));
 302   Streamer.switchSection(Section);
 303   Section->setHasInstructions(true);
 304   BC.Ctx->addGenDwarfSection(Section);
 305
 306   if (BC.HasRelocations) {
 307     // Set section alignment to at least maximum possible object alignment.
 308     // We need this to support LongJmp and other passes that calculates
 309     // tentative layout.
 310     Section->ensureMinAlignment(Align(opts::AlignFunctions));
 311
 312     Streamer.emitCodeAlignment(Function.getMinAlign(), &*BC.STI);
 313     uint16_t MaxAlignBytes = FF.isSplitFragment()
 314                                  ? Function.getMaxColdAlignmentBytes()
 315                                  : Function.getMaxAlignmentBytes();
 316     if (MaxAlignBytes > 0)
 317       Streamer.emitCodeAlignment(Function.getAlign(), &*BC.STI, MaxAlignBytes);
 318   } else {
 319     Streamer.emitCodeAlignment(Function.getAlign(), &*BC.STI);
 320   }
 321
 322   MCContext &Context = Streamer.getContext();
 323   const MCAsmInfo *MAI = Context.getAsmInfo();
 324
 325   MCSymbol *const StartSymbol = Function.getSymbol(FF.getFragmentNum());
 326
 327   // Emit all symbols associated with the main function entry.
 328   if (FF.isMainFragment()) {
 329     for (MCSymbol *Symbol : Function.getSymbols()) {
 330       Streamer.emitSymbolAttribute(Symbol, MCSA_ELF_TypeFunction);
 331       Streamer.emitLabel(Symbol);
 332     }
 333   } else {
 334     Streamer.emitSymbolAttribute(StartSymbol, MCSA_ELF_TypeFunction);
 335     Streamer.emitLabel(StartSymbol);
 336   }
 337
 338   // Emit CFI start
 339   if (Function.hasCFI()) {
 340     Streamer.emitCFIStartProc(/*IsSimple=*/false);
 341     if (Function.getPersonalityFunction() != nullptr)
 342       Streamer.emitCFIPersonality(Function.getPersonalityFunction(),
 343                                   Function.getPersonalityEncoding());
 344     MCSymbol *LSDASymbol = Function.getLSDASymbol(FF.getFragmentNum());
 345     if (LSDASymbol)
 346       Streamer.emitCFILsda(LSDASymbol, BC.LSDAEncoding);
 347     else
 348       Streamer.emitCFILsda(0, dwarf::DW_EH_PE_omit);
 349     // Emit CFI instructions relative to the CIE
 350     for (const MCCFIInstruction &CFIInstr : Function.cie()) {
 351       // Only write CIE CFI insns that LLVM will not already emit
 352       const std::vector<MCCFIInstruction> &FrameInstrs =
 353           MAI->getInitialFrameState();
 354       if (!llvm::is_contained(FrameInstrs, CFIInstr))
 355         emitCFIInstruction(CFIInstr);
 356     }
 357   }
 358
 359   assert((Function.empty() || !(*Function.begin()).isCold()) &&
 360          "first basic block should never be cold");
 361
 362   // Emit UD2 at the beginning if requested by user.
 363   if (!opts::BreakFunctionNames.empty()) {
 364     for (std::string &Name : opts::BreakFunctionNames) {
 365       if (Function.hasNameRegex(Name)) {
 366         Streamer.emitIntValue(0x0B0F, 2); // UD2: 0F 0B
 367         break;
 368       }
 369     }
 370   }
 371
 372   // Emit code.
 373   emitFunctionBody(Function, FF, /*EmitCodeOnly=*/false);
 374
 375   // Emit padding if requested.
 376   if (size_t Padding = opts::padFunction(Function)) {
 377     LLVM_DEBUG(dbgs() << "BOLT-DEBUG: padding function " << Function << " with "
 378                       << Padding << " bytes\n");
 379     Streamer.emitFill(Padding, MAI->getTextAlignFillValue());
 380   }
 381
 382   if (opts::MarkFuncs)
 383     Streamer.emitBytes(BC.MIB->getTrapFillValue());
 384
 385   // Emit CFI end
 386   if (Function.hasCFI())
 387     Streamer.emitCFIEndProc();
 388
 389   MCSymbol *EndSymbol = Function.getFunctionEndLabel(FF.getFragmentNum());
 390   Streamer.emitLabel(EndSymbol);
 391
 392   if (MAI->hasDotTypeDotSizeDirective()) {
 393     const MCExpr *SizeExpr = MCBinaryExpr::createSub(
 394         MCSymbolRefExpr::create(EndSymbol, Context),
 395         MCSymbolRefExpr::create(StartSymbol, Context), Context);
 396     Streamer.emitELFSize(StartSymbol, SizeExpr);
 397   }
 398
 399   if (opts::UpdateDebugSections && Function.getDWARFUnit())
 400     emitLineInfoEnd(Function, EndSymbol);
 401
 402   // Exception handling info for the function.
 403   emitLSDA(Function, FF);
 404
 405   if (FF.isMainFragment() && opts::JumpTables > JTS_NONE)
 406     emitJumpTables(Function);
 407
 408   return true;
 409 }
 410
 411 void BinaryEmitter::emitFunctionBody(BinaryFunction &BF, FunctionFragment &FF,
 412                                      bool EmitCodeOnly) {
 413   if (!EmitCodeOnly && FF.isSplitFragment() && BF.hasConstantIsland()) {
 414     assert(BF.getLayout().isHotColdSplit() &&
 415            "Constant island support only with hot/cold split");
 416     BF.duplicateConstantIslands();
 417   }
 418
 419   if (!FF.empty() && FF.front()->isLandingPad()) {
 420     assert(!FF.front()->isEntryPoint() &&
 421            "Landing pad cannot be entry point of function");
 422     // If the first block of the fragment is a landing pad, it's offset from the
 423     // start of the area that the corresponding LSDA describes is zero. In this
 424     // case, the call site entries in that LSDA have 0 as offset to the landing
 425     // pad, which the runtime interprets as "no handler". To prevent this,
 426     // insert some padding.
 427     Streamer.emitBytes(BC.MIB->getTrapFillValue());
 428   }
 429
 430   // Track the first emitted instruction with debug info.
 431   bool FirstInstr = true;
 432   for (BinaryBasicBlock *const BB : FF) {
 433     if ((opts::AlignBlocks || opts::PreserveBlocksAlignment) &&
 434         BB->getAlignment() > 1)
 435       Streamer.emitCodeAlignment(BB->getAlign(), &*BC.STI,
 436                                  BB->getAlignmentMaxBytes());
 437     Streamer.emitLabel(BB->getLabel());
 438     if (!EmitCodeOnly) {
 439       if (MCSymbol *EntrySymbol = BF.getSecondaryEntryPointSymbol(*BB))
 440         Streamer.emitLabel(EntrySymbol);
 441     }
 442
 443     // Check if special alignment for macro-fusion is needed.
 444     bool MayNeedMacroFusionAlignment =
 445         (opts::AlignMacroOpFusion == MFT_ALL) ||
 446         (opts::AlignMacroOpFusion == MFT_HOT && BB->getKnownExecutionCount());
 447     BinaryBasicBlock::const_iterator MacroFusionPair;
 448     if (MayNeedMacroFusionAlignment) {
 449       MacroFusionPair = BB->getMacroOpFusionPair();
 450       if (MacroFusionPair == BB->end())
 451         MayNeedMacroFusionAlignment = false;
 452     }
 453
 454     SMLoc LastLocSeen;
 455     // Remember if the last instruction emitted was a prefix.
 456     bool LastIsPrefix = false;
 457     for (auto I = BB->begin(), E = BB->end(); I != E; ++I) {
 458       MCInst &Instr = *I;
 459
 460       if (EmitCodeOnly && BC.MIB->isPseudo(Instr))
 461         continue;
 462
 463       // Handle pseudo instructions.
 464       if (BC.MIB->isEHLabel(Instr)) {
 465         const MCSymbol *Label = BC.MIB->getTargetSymbol(Instr);
 466         assert(Instr.getNumOperands() >= 1 && Label &&
 467                "bad EH_LABEL instruction");
 468         Streamer.emitLabel(const_cast<MCSymbol *>(Label));
 469         continue;
 470       }
 471       if (BC.MIB->isCFI(Instr)) {
 472         emitCFIInstruction(*BF.getCFIFor(Instr));
 473         continue;
 474       }
 475
 476       // Handle macro-fusion alignment. If we emitted a prefix as
 477       // the last instruction, we should've already emitted the associated
 478       // alignment hint, so don't emit it twice.
 479       if (MayNeedMacroFusionAlignment && !LastIsPrefix &&
 480           I == MacroFusionPair) {
 481         // This assumes the second instruction in the macro-op pair will get
 482         // assigned to its own MCRelaxableFragment. Since all JCC instructions
 483         // are relaxable, we should be safe.
 484       }
 485
 486       if (!EmitCodeOnly && opts::UpdateDebugSections && BF.getDWARFUnit()) {
 487         LastLocSeen = emitLineInfo(BF, Instr.getLoc(), LastLocSeen, FirstInstr);
 488         FirstInstr = false;
 489       }
 490
 491       // Prepare to tag this location with a label if we need to keep track of
 492       // the location of calls/returns for BOLT address translation maps
 493       if (!EmitCodeOnly && BF.requiresAddressTranslation() &&
 494           BC.MIB->getOffset(Instr)) {
 495         const uint32_t Offset = *BC.MIB->getOffset(Instr);
 496         MCSymbol *LocSym = BC.Ctx->createTempSymbol();
 497         Streamer.emitLabel(LocSym);
 498         BB->getLocSyms().emplace_back(Offset, LocSym);
 499       }
 500
 501       if (auto Label = BC.MIB->getLabel(Instr))
 502         Streamer.emitLabel(*Label);
 503
 504       Streamer.emitInstruction(Instr, *BC.STI);
 505       LastIsPrefix = BC.MIB->isPrefix(Instr);
 506     }
 507   }
 508
 509   if (!EmitCodeOnly)
 510     emitConstantIslands(BF, FF.isSplitFragment());
 511 }
 512
 513 void BinaryEmitter::emitConstantIslands(BinaryFunction &BF, bool EmitColdPart,
 514                                         BinaryFunction *OnBehalfOf) {
 515   if (!BF.hasIslandsInfo())
 516     return;
 517
 518   BinaryFunction::IslandInfo &Islands = BF.getIslandInfo();
 519   if (Islands.DataOffsets.empty() && Islands.Dependency.empty())
 520     return;
 521
 522   // AArch64 requires CI to be aligned to 8 bytes due to access instructions
 523   // restrictions. E.g. the ldr with imm, where imm must be aligned to 8 bytes.
 524   const uint16_t Alignment = OnBehalfOf
 525                                  ? OnBehalfOf->getConstantIslandAlignment()
 526                                  : BF.getConstantIslandAlignment();
 527   Streamer.emitCodeAlignment(Align(Alignment), &*BC.STI);
 528
 529   if (!OnBehalfOf) {
 530     if (!EmitColdPart)
 531       Streamer.emitLabel(BF.getFunctionConstantIslandLabel());
 532     else
 533       Streamer.emitLabel(BF.getFunctionColdConstantIslandLabel());
 534   }
 535
 536   assert((!OnBehalfOf || Islands.Proxies[OnBehalfOf].size() > 0) &&
 537          "spurious OnBehalfOf constant island emission");
 538
 539   assert(!BF.isInjected() &&
 540          "injected functions should not have constant islands");
 541   // Raw contents of the function.
 542   StringRef SectionContents = BF.getOriginSection()->getContents();
 543
 544   // Raw contents of the function.
 545   StringRef FunctionContents = SectionContents.substr(
 546       BF.getAddress() - BF.getOriginSection()->getAddress(), BF.getMaxSize());
 547
 548   if (opts::Verbosity && !OnBehalfOf)
 549     outs() << "BOLT-INFO: emitting constant island for function " << BF << "\n";
 550
 551   // We split the island into smaller blocks and output labels between them.
 552   auto IS = Islands.Offsets.begin();
 553   for (auto DataIter = Islands.DataOffsets.begin();
 554        DataIter != Islands.DataOffsets.end(); ++DataIter) {
 555     uint64_t FunctionOffset = *DataIter;
 556     uint64_t EndOffset = 0ULL;
 557
 558     // Determine size of this data chunk
 559     auto NextData = std::next(DataIter);
 560     auto CodeIter = Islands.CodeOffsets.lower_bound(*DataIter);
 561     if (CodeIter == Islands.CodeOffsets.end() &&
 562         NextData == Islands.DataOffsets.end())
 563       EndOffset = BF.getMaxSize();
 564     else if (CodeIter == Islands.CodeOffsets.end())
 565       EndOffset = *NextData;
 566     else if (NextData == Islands.DataOffsets.end())
 567       EndOffset = *CodeIter;
 568     else
 569       EndOffset = (*CodeIter > *NextData) ? *NextData : *CodeIter;
 570
 571     if (FunctionOffset == EndOffset)
 572       continue; // Size is zero, nothing to emit
 573
 574     auto emitCI = [&](uint64_t &FunctionOffset, uint64_t EndOffset) {
 575       if (FunctionOffset >= EndOffset)
 576         return;
 577
 578       for (auto It = Islands.Relocations.lower_bound(FunctionOffset);
 579            It != Islands.Relocations.end(); ++It) {
 580         if (It->first >= EndOffset)
 581           break;
 582
 583         const Relocation &Relocation = It->second;
 584         if (FunctionOffset < Relocation.Offset) {
 585           Streamer.emitBytes(
 586               FunctionContents.slice(FunctionOffset, Relocation.Offset));
 587           FunctionOffset = Relocation.Offset;
 588         }
 589
 590         LLVM_DEBUG(
 591             dbgs() << "BOLT-DEBUG: emitting constant island relocation"
 592                    << " for " << BF << " at offset 0x"
 593                    << Twine::utohexstr(Relocation.Offset) << " with size "
 594                    << Relocation::getSizeForType(Relocation.Type) << '\n');
 595
 596         FunctionOffset += Relocation.emit(&Streamer);
 597       }
 598
 599       assert(FunctionOffset <= EndOffset && "overflow error");
 600       if (FunctionOffset < EndOffset) {
 601         Streamer.emitBytes(FunctionContents.slice(FunctionOffset, EndOffset));
 602         FunctionOffset = EndOffset;
 603       }
 604     };
 605
 606     // Emit labels, relocs and data
 607     while (IS != Islands.Offsets.end() && IS->first < EndOffset) {
 608       auto NextLabelOffset =
 609           IS == Islands.Offsets.end() ? EndOffset : IS->first;
 610       auto NextStop = std::min(NextLabelOffset, EndOffset);
 611       assert(NextStop <= EndOffset && "internal overflow error");
 612       emitCI(FunctionOffset, NextStop);
 613       if (IS != Islands.Offsets.end() && FunctionOffset == IS->first) {
 614         // This is a slightly complex code to decide which label to emit. We
 615         // have 4 cases to handle: regular symbol, cold symbol, regular or cold
 616         // symbol being emitted on behalf of an external function.
 617         if (!OnBehalfOf) {
 618           if (!EmitColdPart) {
 619             LLVM_DEBUG(dbgs() << "BOLT-DEBUG: emitted label "
 620                               << IS->second->getName() << " at offset 0x"
 621                               << Twine::utohexstr(IS->first) << '\n');
 622             if (IS->second->isUndefined())
 623               Streamer.emitLabel(IS->second);
 624             else
 625               assert(BF.hasName(std::string(IS->second->getName())));
 626           } else if (Islands.ColdSymbols.count(IS->second) != 0) {
 627             LLVM_DEBUG(dbgs()
 628                        << "BOLT-DEBUG: emitted label "
 629                        << Islands.ColdSymbols[IS->second]->getName() << '\n');
 630             if (Islands.ColdSymbols[IS->second]->isUndefined())
 631               Streamer.emitLabel(Islands.ColdSymbols[IS->second]);
 632           }
 633         } else {
 634           if (!EmitColdPart) {
 635             if (MCSymbol *Sym = Islands.Proxies[OnBehalfOf][IS->second]) {
 636               LLVM_DEBUG(dbgs() << "BOLT-DEBUG: emitted label "
 637                                 << Sym->getName() << '\n');
 638               Streamer.emitLabel(Sym);
 639             }
 640           } else if (MCSymbol *Sym =
 641                          Islands.ColdProxies[OnBehalfOf][IS->second]) {
 642             LLVM_DEBUG(dbgs() << "BOLT-DEBUG: emitted label " << Sym->getName()
 643                               << '\n');
 644             Streamer.emitLabel(Sym);
 645           }
 646         }
 647         ++IS;
 648       }
 649     }
 650     assert(FunctionOffset <= EndOffset && "overflow error");
 651     emitCI(FunctionOffset, EndOffset);
 652   }
 653   assert(IS == Islands.Offsets.end() && "some symbols were not emitted!");
 654
 655   if (OnBehalfOf)
 656     return;
 657   // Now emit constant islands from other functions that we may have used in
 658   // this function.
 659   for (BinaryFunction *ExternalFunc : Islands.Dependency)
 660     emitConstantIslands(*ExternalFunc, EmitColdPart, &BF);
 661 }
 662
 663 SMLoc BinaryEmitter::emitLineInfo(const BinaryFunction &BF, SMLoc NewLoc,
 664                                   SMLoc PrevLoc, bool FirstInstr) {
 665   DWARFUnit *FunctionCU = BF.getDWARFUnit();
 666   const DWARFDebugLine::LineTable *FunctionLineTable = BF.getDWARFLineTable();
 667   assert(FunctionCU && "cannot emit line info for function without CU");
 668
 669   DebugLineTableRowRef RowReference = DebugLineTableRowRef::fromSMLoc(NewLoc);
 670
 671   // Check if no new line info needs to be emitted.
 672   if (RowReference == DebugLineTableRowRef::NULL_ROW ||
 673       NewLoc.getPointer() == PrevLoc.getPointer())
 674     return PrevLoc;
 675
 676   unsigned CurrentFilenum = 0;
 677   const DWARFDebugLine::LineTable *CurrentLineTable = FunctionLineTable;
 678
 679   // If the CU id from the current instruction location does not
 680   // match the CU id from the current function, it means that we
 681   // have come across some inlined code.  We must look up the CU
 682   // for the instruction's original function and get the line table
 683   // from that.
 684   const uint64_t FunctionUnitIndex = FunctionCU->getOffset();
 685   const uint32_t CurrentUnitIndex = RowReference.DwCompileUnitIndex;
 686   if (CurrentUnitIndex != FunctionUnitIndex) {
 687     CurrentLineTable = BC.DwCtx->getLineTableForUnit(
 688         BC.DwCtx->getCompileUnitForOffset(CurrentUnitIndex));
 689     // Add filename from the inlined function to the current CU.
 690     CurrentFilenum = BC.addDebugFilenameToUnit(
 691         FunctionUnitIndex, CurrentUnitIndex,
 692         CurrentLineTable->Rows[RowReference.RowIndex - 1].File);
 693   }
 694
 695   const DWARFDebugLine::Row &CurrentRow =
 696       CurrentLineTable->Rows[RowReference.RowIndex - 1];
 697   if (!CurrentFilenum)
 698     CurrentFilenum = CurrentRow.File;
 699
 700   unsigned Flags = (DWARF2_FLAG_IS_STMT * CurrentRow.IsStmt) |
 701                    (DWARF2_FLAG_BASIC_BLOCK * CurrentRow.BasicBlock) |
 702                    (DWARF2_FLAG_PROLOGUE_END * CurrentRow.PrologueEnd) |
 703                    (DWARF2_FLAG_EPILOGUE_BEGIN * CurrentRow.EpilogueBegin);
 704
 705   // Always emit is_stmt at the beginning of function fragment.
 706   if (FirstInstr)
 707     Flags |= DWARF2_FLAG_IS_STMT;
 708
 709   BC.Ctx->setCurrentDwarfLoc(CurrentFilenum, CurrentRow.Line, CurrentRow.Column,
 710                              Flags, CurrentRow.Isa, CurrentRow.Discriminator);
 711   const MCDwarfLoc &DwarfLoc = BC.Ctx->getCurrentDwarfLoc();
 712   BC.Ctx->clearDwarfLocSeen();
 713
 714   MCSymbol *LineSym = BC.Ctx->createTempSymbol();
 715   Streamer.emitLabel(LineSym);
 716
 717   BC.getDwarfLineTable(FunctionUnitIndex)
 718       .getMCLineSections()
 719       .addLineEntry(MCDwarfLineEntry(LineSym, DwarfLoc),
 720                     Streamer.getCurrentSectionOnly());
 721
 722   return NewLoc;
 723 }
 724
 725 void BinaryEmitter::emitLineInfoEnd(const BinaryFunction &BF,
 726                                     MCSymbol *FunctionEndLabel) {
 727   DWARFUnit *FunctionCU = BF.getDWARFUnit();
 728   assert(FunctionCU && "DWARF unit expected");
 729   BC.Ctx->setCurrentDwarfLoc(0, 0, 0, DWARF2_FLAG_END_SEQUENCE, 0, 0);
 730   const MCDwarfLoc &DwarfLoc = BC.Ctx->getCurrentDwarfLoc();
 731   BC.Ctx->clearDwarfLocSeen();
 732   BC.getDwarfLineTable(FunctionCU->getOffset())
 733       .getMCLineSections()
 734       .addLineEntry(MCDwarfLineEntry(FunctionEndLabel, DwarfLoc),
 735                     Streamer.getCurrentSectionOnly());
 736 }
 737
 738 void BinaryEmitter::emitJumpTables(const BinaryFunction &BF) {
 739   MCSection *ReadOnlySection = BC.MOFI->getReadOnlySection();
 740   MCSection *ReadOnlyColdSection = BC.MOFI->getContext().getELFSection(
 741       ".rodata.cold", ELF::SHT_PROGBITS, ELF::SHF_ALLOC);
 742
 743   if (!BF.hasJumpTables())
 744     return;
 745
 746   if (opts::PrintJumpTables)
 747     outs() << "BOLT-INFO: jump tables for function " << BF << ":\n";
 748
 749   for (auto &JTI : BF.jumpTables()) {
 750     JumpTable &JT = *JTI.second;
 751     // Only emit shared jump tables once, when processing the first parent
 752     if (JT.Parents.size() > 1 && JT.Parents[0] != &BF)
 753       continue;
 754     if (opts::PrintJumpTables)
 755       JT.print(outs());
 756     if (opts::JumpTables == JTS_BASIC && BC.HasRelocations) {
 757       JT.updateOriginal();
 758     } else {
 759       MCSection *HotSection, *ColdSection;
 760       if (opts::JumpTables == JTS_BASIC) {
 761         // In non-relocation mode we have to emit jump tables in local sections.
 762         // This way we only overwrite them when the corresponding function is
 763         // overwritten.
 764         std::string Name = ".local." + JT.Labels[0]->getName().str();
 765         std::replace(Name.begin(), Name.end(), '/', '.');
 766         BinarySection &Section =
 767             BC.registerOrUpdateSection(Name, ELF::SHT_PROGBITS, ELF::SHF_ALLOC);
 768         Section.setAnonymous(true);
 769         JT.setOutputSection(Section);
 770         HotSection = BC.getDataSection(Name);
 771         ColdSection = HotSection;
 772       } else {
 773         if (BF.isSimple()) {
 774           HotSection = ReadOnlySection;
 775           ColdSection = ReadOnlyColdSection;
 776         } else {
 777           HotSection = BF.hasProfile() ? ReadOnlySection : ReadOnlyColdSection;
 778           ColdSection = HotSection;
 779         }
 780       }
 781       emitJumpTable(JT, HotSection, ColdSection);
 782     }
 783   }
 784 }
 785
 786 void BinaryEmitter::emitJumpTable(const JumpTable &JT, MCSection *HotSection,
 787                                   MCSection *ColdSection) {
 788   // Pre-process entries for aggressive splitting.
 789   // Each label represents a separate switch table and gets its own count
 790   // determining its destination.
 791   std::map<MCSymbol *, uint64_t> LabelCounts;
 792   if (opts::JumpTables > JTS_SPLIT && !JT.Counts.empty()) {
 793     MCSymbol *CurrentLabel = JT.Labels.at(0);
 794     uint64_t CurrentLabelCount = 0;
 795     for (unsigned Index = 0; Index < JT.Entries.size(); ++Index) {
 796       auto LI = JT.Labels.find(Index * JT.EntrySize);
 797       if (LI != JT.Labels.end()) {
 798         LabelCounts[CurrentLabel] = CurrentLabelCount;
 799         CurrentLabel = LI->second;
 800         CurrentLabelCount = 0;
 801       }
 802       CurrentLabelCount += JT.Counts[Index].Count;
 803     }
 804     LabelCounts[CurrentLabel] = CurrentLabelCount;
 805   } else {
 806     Streamer.switchSection(JT.Count > 0 ? HotSection : ColdSection);
 807     Streamer.emitValueToAlignment(Align(JT.EntrySize));
 808   }
 809   MCSymbol *LastLabel = nullptr;
 810   uint64_t Offset = 0;
 811   for (MCSymbol *Entry : JT.Entries) {
 812     auto LI = JT.Labels.find(Offset);
 813     if (LI != JT.Labels.end()) {
 814       LLVM_DEBUG({
 815         dbgs() << "BOLT-DEBUG: emitting jump table " << LI->second->getName()
 816                << " (originally was at address 0x"
 817                << Twine::utohexstr(JT.getAddress() + Offset)
 818                << (Offset ? ") as part of larger jump table\n" : ")\n");
 819       });
 820       if (!LabelCounts.empty()) {
 821         LLVM_DEBUG(dbgs() << "BOLT-DEBUG: jump table count: "
 822                           << LabelCounts[LI->second] << '\n');
 823         if (LabelCounts[LI->second] > 0)
 824           Streamer.switchSection(HotSection);
 825         else
 826           Streamer.switchSection(ColdSection);
 827         Streamer.emitValueToAlignment(Align(JT.EntrySize));
 828       }
 829       // Emit all labels registered at the address of this jump table
 830       // to sync with our global symbol table.  We may have two labels
 831       // registered at this address if one label was created via
 832       // getOrCreateGlobalSymbol() (e.g. LEA instructions referencing
 833       // this location) and another via getOrCreateJumpTable().  This
 834       // creates a race where the symbols created by these two
 835       // functions may or may not be the same, but they are both
 836       // registered in our symbol table at the same address. By
 837       // emitting them all here we make sure there is no ambiguity
 838       // that depends on the order that these symbols were created, so
 839       // whenever this address is referenced in the binary, it is
 840       // certain to point to the jump table identified at this
 841       // address.
 842       if (BinaryData *BD = BC.getBinaryDataByName(LI->second->getName())) {
 843         for (MCSymbol *S : BD->getSymbols())
 844           Streamer.emitLabel(S);
 845       } else {
 846         Streamer.emitLabel(LI->second);
 847       }
 848       LastLabel = LI->second;
 849     }
 850     if (JT.Type == JumpTable::JTT_NORMAL) {
 851       Streamer.emitSymbolValue(Entry, JT.OutputEntrySize);
 852     } else { // JTT_PIC
 853       const MCSymbolRefExpr *JTExpr =
 854           MCSymbolRefExpr::create(LastLabel, Streamer.getContext());
 855       const MCSymbolRefExpr *E =
 856           MCSymbolRefExpr::create(Entry, Streamer.getContext());
 857       const MCBinaryExpr *Value =
 858           MCBinaryExpr::createSub(E, JTExpr, Streamer.getContext());
 859       Streamer.emitValue(Value, JT.EntrySize);
 860     }
 861     Offset += JT.EntrySize;
 862   }
 863 }
 864
 865 void BinaryEmitter::emitCFIInstruction(const MCCFIInstruction &Inst) const {
 866   switch (Inst.getOperation()) {
 867   default:
 868     llvm_unreachable("Unexpected instruction");
 869   case MCCFIInstruction::OpDefCfaOffset:
 870     Streamer.emitCFIDefCfaOffset(Inst.getOffset());
 871     break;
 872   case MCCFIInstruction::OpAdjustCfaOffset:
 873     Streamer.emitCFIAdjustCfaOffset(Inst.getOffset());
 874     break;
 875   case MCCFIInstruction::OpDefCfa:
 876     Streamer.emitCFIDefCfa(Inst.getRegister(), Inst.getOffset());
 877     break;
 878   case MCCFIInstruction::OpDefCfaRegister:
 879     Streamer.emitCFIDefCfaRegister(Inst.getRegister());
 880     break;
 881   case MCCFIInstruction::OpOffset:
 882     Streamer.emitCFIOffset(Inst.getRegister(), Inst.getOffset());
 883     break;
 884   case MCCFIInstruction::OpRegister:
 885     Streamer.emitCFIRegister(Inst.getRegister(), Inst.getRegister2());
 886     break;
 887   case MCCFIInstruction::OpWindowSave:
 888     Streamer.emitCFIWindowSave();
 889     break;
 890   case MCCFIInstruction::OpNegateRAState:
 891     Streamer.emitCFINegateRAState();
 892     break;
 893   case MCCFIInstruction::OpSameValue:
 894     Streamer.emitCFISameValue(Inst.getRegister());
 895     break;
 896   case MCCFIInstruction::OpGnuArgsSize:
 897     Streamer.emitCFIGnuArgsSize(Inst.getOffset());
 898     break;
 899   case MCCFIInstruction::OpEscape:
 900     Streamer.AddComment(Inst.getComment());
 901     Streamer.emitCFIEscape(Inst.getValues());
 902     break;
 903   case MCCFIInstruction::OpRestore:
 904     Streamer.emitCFIRestore(Inst.getRegister());
 905     break;
 906   case MCCFIInstruction::OpUndefined:
 907     Streamer.emitCFIUndefined(Inst.getRegister());
 908     break;
 909   }
 910 }
 911
 912 // The code is based on EHStreamer::emitExceptionTable().
 913 void BinaryEmitter::emitLSDA(BinaryFunction &BF, const FunctionFragment &FF) {
 914   const BinaryFunction::CallSitesRange Sites =
 915       BF.getCallSites(FF.getFragmentNum());
 916   if (Sites.empty())
 917     return;
 918
 919   // Calculate callsite table size. Size of each callsite entry is:
 920   //
 921   //  sizeof(start) + sizeof(length) + sizeof(LP) + sizeof(uleb128(action))
 922   //
 923   // or
 924   //
 925   //  sizeof(dwarf::DW_EH_PE_data4) * 3 + sizeof(uleb128(action))
 926   uint64_t CallSiteTableLength = llvm::size(Sites) * 4 * 3;
 927   for (const auto &FragmentCallSite : Sites)
 928     CallSiteTableLength += getULEB128Size(FragmentCallSite.second.Action);
 929
 930   Streamer.switchSection(BC.MOFI->getLSDASection());
 931
 932   const unsigned TTypeEncoding = BF.getLSDATypeEncoding();
 933   const unsigned TTypeEncodingSize = BC.getDWARFEncodingSize(TTypeEncoding);
 934   const uint16_t TTypeAlignment = 4;
 935
 936   // Type tables have to be aligned at 4 bytes.
 937   Streamer.emitValueToAlignment(Align(TTypeAlignment));
 938
 939   // Emit the LSDA label.
 940   MCSymbol *LSDASymbol = BF.getLSDASymbol(FF.getFragmentNum());
 941   assert(LSDASymbol && "no LSDA symbol set");
 942   Streamer.emitLabel(LSDASymbol);
 943
 944   // Corresponding FDE start.
 945   const MCSymbol *StartSymbol = BF.getSymbol(FF.getFragmentNum());
 946
 947   // Emit the LSDA header.
 948
 949   // If LPStart is omitted, then the start of the FDE is used as a base for
 950   // landing pad displacements. Then if a cold fragment starts with
 951   // a landing pad, this means that the first landing pad offset will be 0.
 952   // As a result, the exception handling runtime will ignore this landing pad
 953   // because zero offset denotes the absence of a landing pad.
 954   // For this reason, when the binary has fixed starting address we emit LPStart
 955   // as 0 and output the absolute value of the landing pad in the table.
 956   //
 957   // If the base address can change, we cannot use absolute addresses for
 958   // landing pads (at least not without runtime relocations). Hence, we fall
 959   // back to emitting landing pads relative to the FDE start.
 960   // As we are emitting label differences, we have to guarantee both labels are
 961   // defined in the same section and hence cannot place the landing pad into a
 962   // cold fragment when the corresponding call site is in the hot fragment.
 963   // Because of this issue and the previously described issue of possible
 964   // zero-offset landing pad we have to place landing pads in the same section
 965   // as the corresponding invokes for shared objects.
 966   std::function<void(const MCSymbol *)> emitLandingPad;
 967   if (BC.HasFixedLoadAddress) {
 968     Streamer.emitIntValue(dwarf::DW_EH_PE_udata4, 1); // LPStart format
 969     Streamer.emitIntValue(0, 4);                      // LPStart
 970     emitLandingPad = [&](const MCSymbol *LPSymbol) {
 971       if (!LPSymbol)
 972         Streamer.emitIntValue(0, 4);
 973       else
 974         Streamer.emitSymbolValue(LPSymbol, 4);
 975     };
 976   } else {
 977     Streamer.emitIntValue(dwarf::DW_EH_PE_omit, 1); // LPStart format
 978     emitLandingPad = [&](const MCSymbol *LPSymbol) {
 979       if (!LPSymbol)
 980         Streamer.emitIntValue(0, 4);
 981       else
 982         Streamer.emitAbsoluteSymbolDiff(LPSymbol, StartSymbol, 4);
 983     };
 984   }
 985
 986   Streamer.emitIntValue(TTypeEncoding, 1); // TType format
 987
 988   // See the comment in EHStreamer::emitExceptionTable() on to use
 989   // uleb128 encoding (which can use variable number of bytes to encode the same
 990   // value) to ensure type info table is properly aligned at 4 bytes without
 991   // iteratively fixing sizes of the tables.
 992   unsigned CallSiteTableLengthSize = getULEB128Size(CallSiteTableLength);
 993   unsigned TTypeBaseOffset =
 994       sizeof(int8_t) +                 // Call site format
 995       CallSiteTableLengthSize +        // Call site table length size
 996       CallSiteTableLength +            // Call site table length
 997       BF.getLSDAActionTable().size() + // Actions table size
 998       BF.getLSDATypeTable().size() * TTypeEncodingSize; // Types table size
 999   unsigned TTypeBaseOffsetSize = getULEB128Size(TTypeBaseOffset);
1000   unsigned TotalSize = sizeof(int8_t) +      // LPStart format
1001                        sizeof(int8_t) +      // TType format
1002                        TTypeBaseOffsetSize + // TType base offset size
1003                        TTypeBaseOffset;      // TType base offset
1004   unsigned SizeAlign = (4 - TotalSize) & 3;
1005
1006   if (TTypeEncoding != dwarf::DW_EH_PE_omit)
1007     // Account for any extra padding that will be added to the call site table
1008     // length.
1009     Streamer.emitULEB128IntValue(TTypeBaseOffset,
1010                                  /*PadTo=*/TTypeBaseOffsetSize + SizeAlign);
1011
1012   // Emit the landing pad call site table. We use signed data4 since we can emit
1013   // a landing pad in a different part of the split function that could appear
1014   // earlier in the address space than LPStart.
1015   Streamer.emitIntValue(dwarf::DW_EH_PE_sdata4, 1);
1016   Streamer.emitULEB128IntValue(CallSiteTableLength);
1017
1018   for (const auto &FragmentCallSite : Sites) {
1019     const BinaryFunction::CallSite &CallSite = FragmentCallSite.second;
1020     const MCSymbol *BeginLabel = CallSite.Start;
1021     const MCSymbol *EndLabel = CallSite.End;
1022
1023     assert(BeginLabel && "start EH label expected");
1024     assert(EndLabel && "end EH label expected");
1025
1026     // Start of the range is emitted relative to the start of current
1027     // function split part.
1028     Streamer.emitAbsoluteSymbolDiff(BeginLabel, StartSymbol, 4);
1029     Streamer.emitAbsoluteSymbolDiff(EndLabel, BeginLabel, 4);
1030     emitLandingPad(CallSite.LP);
1031     Streamer.emitULEB128IntValue(CallSite.Action);
1032   }
1033
1034   // Write out action, type, and type index tables at the end.
1035   //
1036   // For action and type index tables there's no need to change the original
1037   // table format unless we are doing function splitting, in which case we can
1038   // split and optimize the tables.
1039   //
1040   // For type table we (re-)encode the table using TTypeEncoding matching
1041   // the current assembler mode.
1042   for (uint8_t const &Byte : BF.getLSDAActionTable())
1043     Streamer.emitIntValue(Byte, 1);
1044
1045   const BinaryFunction::LSDATypeTableTy &TypeTable =
1046       (TTypeEncoding & dwarf::DW_EH_PE_indirect) ? BF.getLSDATypeAddressTable()
1047                                                  : BF.getLSDATypeTable();
1048   assert(TypeTable.size() == BF.getLSDATypeTable().size() &&
1049          "indirect type table size mismatch");
1050
1051   for (int Index = TypeTable.size() - 1; Index >= 0; --Index) {
1052     const uint64_t TypeAddress = TypeTable[Index];
1053     switch (TTypeEncoding & 0x70) {
1054     default:
1055       llvm_unreachable("unsupported TTypeEncoding");
1056     case dwarf::DW_EH_PE_absptr:
1057       Streamer.emitIntValue(TypeAddress, TTypeEncodingSize);
1058       break;
1059     case dwarf::DW_EH_PE_pcrel: {
1060       if (TypeAddress) {
1061         const MCSymbol *TypeSymbol =
1062             BC.getOrCreateGlobalSymbol(TypeAddress, "TI", 0, TTypeAlignment);
1063         MCSymbol *DotSymbol = BC.Ctx->createNamedTempSymbol();
1064         Streamer.emitLabel(DotSymbol);
1065         const MCBinaryExpr *SubDotExpr = MCBinaryExpr::createSub(
1066             MCSymbolRefExpr::create(TypeSymbol, *BC.Ctx),
1067             MCSymbolRefExpr::create(DotSymbol, *BC.Ctx), *BC.Ctx);
1068         Streamer.emitValue(SubDotExpr, TTypeEncodingSize);
1069       } else {
1070         Streamer.emitIntValue(0, TTypeEncodingSize);
1071       }
1072       break;
1073     }
1074     }
1075   }
1076   for (uint8_t const &Byte : BF.getLSDATypeIndexTable())
1077     Streamer.emitIntValue(Byte, 1);
1078 }
1079
1080 void BinaryEmitter::emitDebugLineInfoForOriginalFunctions() {
1081   // If a function is in a CU containing at least one processed function, we
1082   // have to rewrite the whole line table for that CU. For unprocessed functions
1083   // we use data from the input line table.
1084   for (auto &It : BC.getBinaryFunctions()) {
1085     const BinaryFunction &Function = It.second;
1086
1087     // If the function was emitted, its line info was emitted with it.
1088     if (Function.isEmitted())
1089       continue;
1090
1091     const DWARFDebugLine::LineTable *LineTable = Function.getDWARFLineTable();
1092     if (!LineTable)
1093       continue; // nothing to update for this function
1094
1095     const uint64_t Address = Function.getAddress();
1096     std::vector<uint32_t> Results;
1097     if (!LineTable->lookupAddressRange(
1098             {Address, object::SectionedAddress::UndefSection},
1099             Function.getSize(), Results))
1100       continue;
1101
1102     if (Results.empty())
1103       continue;
1104
1105     // The first row returned could be the last row matching the start address.
1106     // Find the first row with the same address that is not the end of the
1107     // sequence.
1108     uint64_t FirstRow = Results.front();
1109     while (FirstRow > 0) {
1110       const DWARFDebugLine::Row &PrevRow = LineTable->Rows[FirstRow - 1];
1111       if (PrevRow.Address.Address != Address || PrevRow.EndSequence)
1112         break;
1113       --FirstRow;
1114     }
1115
1116     const uint64_t EndOfSequenceAddress =
1117         Function.getAddress() + Function.getMaxSize();
1118     BC.getDwarfLineTable(Function.getDWARFUnit()->getOffset())
1119         .addLineTableSequence(LineTable, FirstRow, Results.back(),
1120                               EndOfSequenceAddress);
1121   }
1122
1123   // For units that are completely unprocessed, use original debug line contents
1124   // eliminating the need to regenerate line info program.
1125   emitDebugLineInfoForUnprocessedCUs();
1126 }
1127
1128 void BinaryEmitter::emitDebugLineInfoForUnprocessedCUs() {
1129   // Sorted list of section offsets provides boundaries for section fragments,
1130   // where each fragment is the unit's contribution to debug line section.
1131   std::vector<uint64_t> StmtListOffsets;
1132   StmtListOffsets.reserve(BC.DwCtx->getNumCompileUnits());
1133   for (const std::unique_ptr<DWARFUnit> &CU : BC.DwCtx->compile_units()) {
1134     DWARFDie CUDie = CU->getUnitDIE();
1135     auto StmtList = dwarf::toSectionOffset(CUDie.find(dwarf::DW_AT_stmt_list));
1136     if (!StmtList)
1137       continue;
1138
1139     StmtListOffsets.push_back(*StmtList);
1140   }
1141   llvm::sort(StmtListOffsets);
1142
1143   // For each CU that was not processed, emit its line info as a binary blob.
1144   for (const std::unique_ptr<DWARFUnit> &CU : BC.DwCtx->compile_units()) {
1145     if (BC.ProcessedCUs.count(CU.get()))
1146       continue;
1147
1148     DWARFDie CUDie = CU->getUnitDIE();
1149     auto StmtList = dwarf::toSectionOffset(CUDie.find(dwarf::DW_AT_stmt_list));
1150     if (!StmtList)
1151       continue;
1152
1153     StringRef DebugLineContents = CU->getLineSection().Data;
1154
1155     const uint64_t Begin = *StmtList;
1156
1157     // Statement list ends where the next unit contribution begins, or at the
1158     // end of the section.
1159     auto It = llvm::upper_bound(StmtListOffsets, Begin);
1160     const uint64_t End =
1161         It == StmtListOffsets.end() ? DebugLineContents.size() : *It;
1162
1163     BC.getDwarfLineTable(CU->getOffset())
1164         .addRawContents(DebugLineContents.slice(Begin, End));
1165   }
1166 }
1167
1168 void BinaryEmitter::emitDataSections(StringRef OrgSecPrefix) {
1169   for (BinarySection &Section : BC.sections()) {
1170     if (!Section.hasRelocations())
1171       continue;
1172
1173     StringRef Prefix = Section.hasSectionRef() ? OrgSecPrefix : "";
1174     Section.emitAsData(Streamer, Prefix + Section.getName());
1175     Section.clearRelocations();
1176   }
1177 }
1178
1179 namespace llvm {
1180 namespace bolt {
1181
1182 void emitBinaryContext(MCStreamer &Streamer, BinaryContext &BC,
1183                        StringRef OrgSecPrefix) {
1184   BinaryEmitter(Streamer, BC).emitAll(OrgSecPrefix);
1185 }
1186
1187 void emitFunctionBody(MCStreamer &Streamer, BinaryFunction &BF,
1188                       FunctionFragment &FF, bool EmitCodeOnly) {
1189   BinaryEmitter(Streamer, BF.getBinaryContext())
1190       .emitFunctionBody(BF, FF, EmitCodeOnly);
1191 }
1192
1193 } // namespace bolt
1194 } // namespace llvm