bolt/lib/Core/BinaryEmitter.cpp

   1 //===- bolt/Core/BinaryEmitter.cpp - Emit code and data -------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file implements the collection of functions and classes used for
  10 // emission of code and data into object/binary file.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "bolt/Core/BinaryEmitter.h"
  15 #include "bolt/Core/BinaryContext.h"
  16 #include "bolt/Core/BinaryFunction.h"
  17 #include "bolt/Core/DebugData.h"
  18 #include "bolt/Core/FunctionLayout.h"
  19 #include "bolt/Utils/CommandLineOpts.h"
  20 #include "bolt/Utils/Utils.h"
  21 #include "llvm/DebugInfo/DWARF/DWARFCompileUnit.h"
  22 #include "llvm/MC/MCSection.h"
  23 #include "llvm/MC/MCStreamer.h"
  24 #include "llvm/Support/CommandLine.h"
  25 #include "llvm/Support/LEB128.h"
  26 #include "llvm/Support/SMLoc.h"
  27
  28 #define DEBUG_TYPE "bolt"
  29
  30 using namespace llvm;
  31 using namespace bolt;
  32
  33 namespace opts {
  34
  35 extern cl::opt<JumpTableSupportLevel> JumpTables;
  36 extern cl::opt<bool> PreserveBlocksAlignment;
  37
  38 cl::opt<bool> AlignBlocks("align-blocks", cl::desc("align basic blocks"),
  39                           cl::cat(BoltOptCategory));
  40
  41 static cl::list<std::string>
  42 BreakFunctionNames("break-funcs",
  43   cl::CommaSeparated,
  44   cl::desc("list of functions to core dump on (debugging)"),
  45   cl::value_desc("func1,func2,func3,..."),
  46   cl::Hidden,
  47   cl::cat(BoltCategory));
  48
  49 static cl::list<std::string>
  50 FunctionPadSpec("pad-funcs",
  51   cl::CommaSeparated,
  52   cl::desc("list of functions to pad with amount of bytes"),
  53   cl::value_desc("func1:pad1,func2:pad2,func3:pad3,..."),
  54   cl::Hidden,
  55   cl::cat(BoltCategory));
  56
  57 static cl::opt<bool> MarkFuncs(
  58     "mark-funcs",
  59     cl::desc("mark function boundaries with break instruction to make "
  60              "sure we accidentally don't cross them"),
  61     cl::ReallyHidden, cl::cat(BoltCategory));
  62
  63 static cl::opt<bool> PrintJumpTables("print-jump-tables",
  64                                      cl::desc("print jump tables"), cl::Hidden,
  65                                      cl::cat(BoltCategory));
  66
  67 static cl::opt<bool>
  68 X86AlignBranchBoundaryHotOnly("x86-align-branch-boundary-hot-only",
  69   cl::desc("only apply branch boundary alignment in hot code"),
  70   cl::init(true),
  71   cl::cat(BoltOptCategory));
  72
  73 size_t padFunction(const BinaryFunction &Function) {
  74   static std::map<std::string, size_t> FunctionPadding;
  75
  76   if (FunctionPadding.empty() && !FunctionPadSpec.empty()) {
  77     for (std::string &Spec : FunctionPadSpec) {
  78       size_t N = Spec.find(':');
  79       if (N == std::string::npos)
  80         continue;
  81       std::string Name = Spec.substr(0, N);
  82       size_t Padding = std::stoull(Spec.substr(N + 1));
  83       FunctionPadding[Name] = Padding;
  84     }
  85   }
  86
  87   for (auto &FPI : FunctionPadding) {
  88     std::string Name = FPI.first;
  89     size_t Padding = FPI.second;
  90     if (Function.hasNameRegex(Name))
  91       return Padding;
  92   }
  93
  94   return 0;
  95 }
  96
  97 } // namespace opts
  98
  99 namespace {
 100 using JumpTable = bolt::JumpTable;
 101
 102 class BinaryEmitter {
 103 private:
 104   BinaryEmitter(const BinaryEmitter &) = delete;
 105   BinaryEmitter &operator=(const BinaryEmitter &) = delete;
 106
 107   MCStreamer &Streamer;
 108   BinaryContext &BC;
 109
 110 public:
 111   BinaryEmitter(MCStreamer &Streamer, BinaryContext &BC)
 112       : Streamer(Streamer), BC(BC) {}
 113
 114   /// Emit all code and data.
 115   void emitAll(StringRef OrgSecPrefix);
 116
 117   /// Emit function code. The caller is responsible for emitting function
 118   /// symbol(s) and setting the section to emit the code to.
 119   void emitFunctionBody(BinaryFunction &BF, FunctionFragment &FF,
 120                         bool EmitCodeOnly = false);
 121
 122 private:
 123   /// Emit function code.
 124   void emitFunctions();
 125
 126   /// Emit a single function.
 127   bool emitFunction(BinaryFunction &BF, FunctionFragment &FF);
 128
 129   /// Helper for emitFunctionBody to write data inside a function
 130   /// (used for AArch64)
 131   void emitConstantIslands(BinaryFunction &BF, bool EmitColdPart,
 132                            BinaryFunction *OnBehalfOf = nullptr);
 133
 134   /// Emit jump tables for the function.
 135   void emitJumpTables(const BinaryFunction &BF);
 136
 137   /// Emit jump table data. Callee supplies sections for the data.
 138   void emitJumpTable(const JumpTable &JT, MCSection *HotSection,
 139                      MCSection *ColdSection);
 140
 141   void emitCFIInstruction(const MCCFIInstruction &Inst) const;
 142
 143   /// Emit exception handling ranges for the function.
 144   void emitLSDA(BinaryFunction &BF, const FunctionFragment &FF);
 145
 146   /// Emit line number information corresponding to \p NewLoc. \p PrevLoc
 147   /// provides a context for de-duplication of line number info.
 148   /// \p FirstInstr indicates if \p NewLoc represents the first instruction
 149   /// in a sequence, such as a function fragment.
 150   ///
 151   /// If \p NewLoc location matches \p PrevLoc, no new line number entry will be
 152   /// created and the function will return \p PrevLoc while \p InstrLabel will
 153   /// be ignored. Otherwise, the caller should use \p InstrLabel to mark the
 154   /// corresponding instruction by emitting \p InstrLabel before it.
 155   /// If \p InstrLabel is set by the caller, its value will be used with \p
 156   /// \p NewLoc. If it was nullptr on entry, it will be populated with a pointer
 157   /// to a new temp symbol used with \p NewLoc.
 158   ///
 159   /// Return new current location which is either \p NewLoc or \p PrevLoc.
 160   SMLoc emitLineInfo(const BinaryFunction &BF, SMLoc NewLoc, SMLoc PrevLoc,
 161                      bool FirstInstr, MCSymbol *&InstrLabel);
 162
 163   /// Use \p FunctionEndSymbol to mark the end of the line info sequence.
 164   /// Note that it does not automatically result in the insertion of the EOS
 165   /// marker in the line table program, but provides one to the DWARF generator
 166   /// when it needs it.
 167   void emitLineInfoEnd(const BinaryFunction &BF, MCSymbol *FunctionEndSymbol);
 168
 169   /// Emit debug line info for unprocessed functions from CUs that include
 170   /// emitted functions.
 171   void emitDebugLineInfoForOriginalFunctions();
 172
 173   /// Emit debug line for CUs that were not modified.
 174   void emitDebugLineInfoForUnprocessedCUs();
 175
 176   /// Emit data sections that have code references in them.
 177   void emitDataSections(StringRef OrgSecPrefix);
 178 };
 179
 180 } // anonymous namespace
 181
 182 void BinaryEmitter::emitAll(StringRef OrgSecPrefix) {
 183   Streamer.initSections(false, *BC.STI);
 184   Streamer.setUseAssemblerInfoForParsing(false);
 185
 186   if (opts::UpdateDebugSections && BC.isELF()) {
 187     // Force the emission of debug line info into allocatable section to ensure
 188     // JITLink will process it.
 189     //
 190     // NB: on MachO all sections are required for execution, hence no need
 191     //     to change flags/attributes.
 192     MCSectionELF *ELFDwarfLineSection =
 193         static_cast<MCSectionELF *>(BC.MOFI->getDwarfLineSection());
 194     ELFDwarfLineSection->setFlags(ELF::SHF_ALLOC);
 195     MCSectionELF *ELFDwarfLineStrSection =
 196         static_cast<MCSectionELF *>(BC.MOFI->getDwarfLineStrSection());
 197     ELFDwarfLineStrSection->setFlags(ELF::SHF_ALLOC);
 198   }
 199
 200   if (RuntimeLibrary *RtLibrary = BC.getRuntimeLibrary())
 201     RtLibrary->emitBinary(BC, Streamer);
 202
 203   BC.getTextSection()->setAlignment(Align(opts::AlignText));
 204
 205   emitFunctions();
 206
 207   if (opts::UpdateDebugSections) {
 208     emitDebugLineInfoForOriginalFunctions();
 209     DwarfLineTable::emit(BC, Streamer);
 210   }
 211
 212   emitDataSections(OrgSecPrefix);
 213
 214   // TODO Enable for Mach-O once BinaryContext::getDataSection supports it.
 215   if (BC.isELF())
 216     AddressMap::emit(Streamer, BC);
 217   Streamer.setUseAssemblerInfoForParsing(true);
 218 }
 219
 220 void BinaryEmitter::emitFunctions() {
 221   auto emit = [&](const std::vector<BinaryFunction *> &Functions) {
 222     const bool HasProfile = BC.NumProfiledFuncs > 0;
 223     const bool OriginalAllowAutoPadding = Streamer.getAllowAutoPadding();
 224     for (BinaryFunction *Function : Functions) {
 225       if (!BC.shouldEmit(*Function))
 226         continue;
 227
 228       LLVM_DEBUG(dbgs() << "BOLT: generating code for function \"" << *Function
 229                         << "\" : " << Function->getFunctionNumber() << '\n');
 230
 231       // Was any part of the function emitted.
 232       bool Emitted = false;
 233
 234       // Turn off Intel JCC Erratum mitigation for cold code if requested
 235       if (HasProfile && opts::X86AlignBranchBoundaryHotOnly &&
 236           !Function->hasValidProfile())
 237         Streamer.setAllowAutoPadding(false);
 238
 239       FunctionLayout &Layout = Function->getLayout();
 240       Emitted |= emitFunction(*Function, Layout.getMainFragment());
 241
 242       if (Function->isSplit()) {
 243         if (opts::X86AlignBranchBoundaryHotOnly)
 244           Streamer.setAllowAutoPadding(false);
 245
 246         assert((Layout.fragment_size() == 1 || Function->isSimple()) &&
 247                "Only simple functions can have fragments");
 248         for (FunctionFragment &FF : Layout.getSplitFragments()) {
 249           // Skip empty fragments so no symbols and sections for empty fragments
 250           // are generated
 251           if (FF.empty() && !Function->hasConstantIsland())
 252             continue;
 253           Emitted |= emitFunction(*Function, FF);
 254         }
 255       }
 256
 257       Streamer.setAllowAutoPadding(OriginalAllowAutoPadding);
 258
 259       if (Emitted)
 260         Function->setEmitted(/*KeepCFG=*/opts::PrintCacheMetrics);
 261     }
 262   };
 263
 264   // Mark the start of hot text.
 265   if (opts::HotText) {
 266     Streamer.switchSection(BC.getTextSection());
 267     Streamer.emitLabel(BC.getHotTextStartSymbol());
 268   }
 269
 270   // Emit functions in sorted order.
 271   std::vector<BinaryFunction *> SortedFunctions = BC.getSortedFunctions();
 272   emit(SortedFunctions);
 273
 274   // Emit functions added by BOLT.
 275   emit(BC.getInjectedBinaryFunctions());
 276
 277   // Mark the end of hot text.
 278   if (opts::HotText) {
 279     if (BC.HasWarmSection)
 280       Streamer.switchSection(BC.getCodeSection(BC.getWarmCodeSectionName()));
 281     else
 282       Streamer.switchSection(BC.getTextSection());
 283     Streamer.emitLabel(BC.getHotTextEndSymbol());
 284   }
 285 }
 286
 287 bool BinaryEmitter::emitFunction(BinaryFunction &Function,
 288                                  FunctionFragment &FF) {
 289   if (Function.size() == 0 && !Function.hasIslandsInfo())
 290     return false;
 291
 292   if (Function.getState() == BinaryFunction::State::Empty)
 293     return false;
 294
 295   // Avoid emitting function without instructions when overwriting the original
 296   // function in-place. Otherwise, emit the empty function to define the symbol.
 297   if (!BC.HasRelocations && !Function.hasNonPseudoInstructions())
 298     return false;
 299
 300   MCSection *Section =
 301       BC.getCodeSection(Function.getCodeSectionName(FF.getFragmentNum()));
 302   Streamer.switchSection(Section);
 303   Section->setHasInstructions(true);
 304   BC.Ctx->addGenDwarfSection(Section);
 305
 306   if (BC.HasRelocations) {
 307     // Set section alignment to at least maximum possible object alignment.
 308     // We need this to support LongJmp and other passes that calculates
 309     // tentative layout.
 310     Section->ensureMinAlignment(Align(opts::AlignFunctions));
 311
 312     Streamer.emitCodeAlignment(Function.getMinAlign(), &*BC.STI);
 313     uint16_t MaxAlignBytes = FF.isSplitFragment()
 314                                  ? Function.getMaxColdAlignmentBytes()
 315                                  : Function.getMaxAlignmentBytes();
 316     if (MaxAlignBytes > 0)
 317       Streamer.emitCodeAlignment(Function.getAlign(), &*BC.STI, MaxAlignBytes);
 318   } else {
 319     Streamer.emitCodeAlignment(Function.getAlign(), &*BC.STI);
 320   }
 321
 322   MCContext &Context = Streamer.getContext();
 323   const MCAsmInfo *MAI = Context.getAsmInfo();
 324
 325   MCSymbol *const StartSymbol = Function.getSymbol(FF.getFragmentNum());
 326
 327   // Emit all symbols associated with the main function entry.
 328   if (FF.isMainFragment()) {
 329     for (MCSymbol *Symbol : Function.getSymbols()) {
 330       Streamer.emitSymbolAttribute(Symbol, MCSA_ELF_TypeFunction);
 331       Streamer.emitLabel(Symbol);
 332     }
 333   } else {
 334     Streamer.emitSymbolAttribute(StartSymbol, MCSA_ELF_TypeFunction);
 335     Streamer.emitLabel(StartSymbol);
 336   }
 337
 338   // Emit CFI start
 339   if (Function.hasCFI()) {
 340     Streamer.emitCFIStartProc(/*IsSimple=*/false);
 341     if (Function.getPersonalityFunction() != nullptr)
 342       Streamer.emitCFIPersonality(Function.getPersonalityFunction(),
 343                                   Function.getPersonalityEncoding());
 344     MCSymbol *LSDASymbol = Function.getLSDASymbol(FF.getFragmentNum());
 345     if (LSDASymbol)
 346       Streamer.emitCFILsda(LSDASymbol, BC.LSDAEncoding);
 347     else
 348       Streamer.emitCFILsda(0, dwarf::DW_EH_PE_omit);
 349     // Emit CFI instructions relative to the CIE
 350     for (const MCCFIInstruction &CFIInstr : Function.cie()) {
 351       // Only write CIE CFI insns that LLVM will not already emit
 352       const std::vector<MCCFIInstruction> &FrameInstrs =
 353           MAI->getInitialFrameState();
 354       if (!llvm::is_contained(FrameInstrs, CFIInstr))
 355         emitCFIInstruction(CFIInstr);
 356     }
 357   }
 358
 359   assert((Function.empty() || !(*Function.begin()).isCold()) &&
 360          "first basic block should never be cold");
 361
 362   // Emit UD2 at the beginning if requested by user.
 363   if (!opts::BreakFunctionNames.empty()) {
 364     for (std::string &Name : opts::BreakFunctionNames) {
 365       if (Function.hasNameRegex(Name)) {
 366         Streamer.emitIntValue(0x0B0F, 2); // UD2: 0F 0B
 367         break;
 368       }
 369     }
 370   }
 371
 372   // Emit code.
 373   emitFunctionBody(Function, FF, /*EmitCodeOnly=*/false);
 374
 375   // Emit padding if requested.
 376   if (size_t Padding = opts::padFunction(Function)) {
 377     LLVM_DEBUG(dbgs() << "BOLT-DEBUG: padding function " << Function << " with "
 378                       << Padding << " bytes\n");
 379     Streamer.emitFill(Padding, MAI->getTextAlignFillValue());
 380   }
 381
 382   if (opts::MarkFuncs)
 383     Streamer.emitBytes(BC.MIB->getTrapFillValue());
 384
 385   // Emit CFI end
 386   if (Function.hasCFI())
 387     Streamer.emitCFIEndProc();
 388
 389   MCSymbol *EndSymbol = Function.getFunctionEndLabel(FF.getFragmentNum());
 390   Streamer.emitLabel(EndSymbol);
 391
 392   if (MAI->hasDotTypeDotSizeDirective()) {
 393     const MCExpr *SizeExpr = MCBinaryExpr::createSub(
 394         MCSymbolRefExpr::create(EndSymbol, Context),
 395         MCSymbolRefExpr::create(StartSymbol, Context), Context);
 396     Streamer.emitELFSize(StartSymbol, SizeExpr);
 397   }
 398
 399   if (opts::UpdateDebugSections && Function.getDWARFUnit())
 400     emitLineInfoEnd(Function, EndSymbol);
 401
 402   // Exception handling info for the function.
 403   emitLSDA(Function, FF);
 404
 405   if (FF.isMainFragment() && opts::JumpTables > JTS_NONE)
 406     emitJumpTables(Function);
 407
 408   return true;
 409 }
 410
 411 void BinaryEmitter::emitFunctionBody(BinaryFunction &BF, FunctionFragment &FF,
 412                                      bool EmitCodeOnly) {
 413   if (!EmitCodeOnly && FF.isSplitFragment() && BF.hasConstantIsland()) {
 414     assert(BF.getLayout().isHotColdSplit() &&
 415            "Constant island support only with hot/cold split");
 416     BF.duplicateConstantIslands();
 417   }
 418
 419   if (!FF.empty() && FF.front()->isLandingPad()) {
 420     assert(!FF.front()->isEntryPoint() &&
 421            "Landing pad cannot be entry point of function");
 422     // If the first block of the fragment is a landing pad, it's offset from the
 423     // start of the area that the corresponding LSDA describes is zero. In this
 424     // case, the call site entries in that LSDA have 0 as offset to the landing
 425     // pad, which the runtime interprets as "no handler". To prevent this,
 426     // insert some padding.
 427     Streamer.emitBytes(BC.MIB->getTrapFillValue());
 428   }
 429
 430   // Track the first emitted instruction with debug info.
 431   bool FirstInstr = true;
 432   for (BinaryBasicBlock *const BB : FF) {
 433     if ((opts::AlignBlocks || opts::PreserveBlocksAlignment) &&
 434         BB->getAlignment() > 1)
 435       Streamer.emitCodeAlignment(BB->getAlign(), &*BC.STI,
 436                                  BB->getAlignmentMaxBytes());
 437     Streamer.emitLabel(BB->getLabel());
 438     if (!EmitCodeOnly) {
 439       if (MCSymbol *EntrySymbol = BF.getSecondaryEntryPointSymbol(*BB))
 440         Streamer.emitLabel(EntrySymbol);
 441     }
 442
 443     SMLoc LastLocSeen;
 444     for (auto I = BB->begin(), E = BB->end(); I != E; ++I) {
 445       MCInst &Instr = *I;
 446
 447       if (EmitCodeOnly && BC.MIB->isPseudo(Instr))
 448         continue;
 449
 450       // Handle pseudo instructions.
 451       if (BC.MIB->isCFI(Instr)) {
 452         emitCFIInstruction(*BF.getCFIFor(Instr));
 453         continue;
 454       }
 455
 456       if (!EmitCodeOnly) {
 457         // A symbol to be emitted before the instruction to mark its location.
 458         MCSymbol *InstrLabel = BC.MIB->getInstLabel(Instr);
 459
 460         if (opts::UpdateDebugSections && BF.getDWARFUnit()) {
 461           LastLocSeen = emitLineInfo(BF, Instr.getLoc(), LastLocSeen,
 462                                      FirstInstr, InstrLabel);
 463           FirstInstr = false;
 464         }
 465
 466         // Prepare to tag this location with a label if we need to keep track of
 467         // the location of calls/returns for BOLT address translation maps
 468         if (BF.requiresAddressTranslation() && BC.MIB->getOffset(Instr)) {
 469           const uint32_t Offset = *BC.MIB->getOffset(Instr);
 470           if (!InstrLabel)
 471             InstrLabel = BC.Ctx->createTempSymbol();
 472           BB->getLocSyms().emplace_back(Offset, InstrLabel);
 473         }
 474
 475         if (InstrLabel)
 476           Streamer.emitLabel(InstrLabel);
 477       }
 478
 479       // Emit sized NOPs via MCAsmBackend::writeNopData() interface on x86.
 480       // This is a workaround for invalid NOPs handling by asm/disasm layer.
 481       if (BC.isX86() && BC.MIB->isNoop(Instr)) {
 482         if (std::optional<uint32_t> Size = BC.MIB->getSize(Instr)) {
 483           SmallString<15> Code;
 484           raw_svector_ostream VecOS(Code);
 485           BC.MAB->writeNopData(VecOS, *Size, BC.STI.get());
 486           Streamer.emitBytes(Code);
 487           continue;
 488         }
 489       }
 490
 491       Streamer.emitInstruction(Instr, *BC.STI);
 492     }
 493   }
 494
 495   if (!EmitCodeOnly)
 496     emitConstantIslands(BF, FF.isSplitFragment());
 497 }
 498
 499 void BinaryEmitter::emitConstantIslands(BinaryFunction &BF, bool EmitColdPart,
 500                                         BinaryFunction *OnBehalfOf) {
 501   if (!BF.hasIslandsInfo())
 502     return;
 503
 504   BinaryFunction::IslandInfo &Islands = BF.getIslandInfo();
 505   if (Islands.DataOffsets.empty() && Islands.Dependency.empty())
 506     return;
 507
 508   // AArch64 requires CI to be aligned to 8 bytes due to access instructions
 509   // restrictions. E.g. the ldr with imm, where imm must be aligned to 8 bytes.
 510   const uint16_t Alignment = OnBehalfOf
 511                                  ? OnBehalfOf->getConstantIslandAlignment()
 512                                  : BF.getConstantIslandAlignment();
 513   Streamer.emitCodeAlignment(Align(Alignment), &*BC.STI);
 514
 515   if (!OnBehalfOf) {
 516     if (!EmitColdPart)
 517       Streamer.emitLabel(BF.getFunctionConstantIslandLabel());
 518     else
 519       Streamer.emitLabel(BF.getFunctionColdConstantIslandLabel());
 520   }
 521
 522   assert((!OnBehalfOf || Islands.Proxies[OnBehalfOf].size() > 0) &&
 523          "spurious OnBehalfOf constant island emission");
 524
 525   assert(!BF.isInjected() &&
 526          "injected functions should not have constant islands");
 527   // Raw contents of the function.
 528   StringRef SectionContents = BF.getOriginSection()->getContents();
 529
 530   // Raw contents of the function.
 531   StringRef FunctionContents = SectionContents.substr(
 532       BF.getAddress() - BF.getOriginSection()->getAddress(), BF.getMaxSize());
 533
 534   if (opts::Verbosity && !OnBehalfOf)
 535     BC.outs() << "BOLT-INFO: emitting constant island for function " << BF
 536               << "\n";
 537
 538   // We split the island into smaller blocks and output labels between them.
 539   auto IS = Islands.Offsets.begin();
 540   for (auto DataIter = Islands.DataOffsets.begin();
 541        DataIter != Islands.DataOffsets.end(); ++DataIter) {
 542     uint64_t FunctionOffset = *DataIter;
 543     uint64_t EndOffset = 0ULL;
 544
 545     // Determine size of this data chunk
 546     auto NextData = std::next(DataIter);
 547     auto CodeIter = Islands.CodeOffsets.lower_bound(*DataIter);
 548     if (CodeIter == Islands.CodeOffsets.end() &&
 549         NextData == Islands.DataOffsets.end())
 550       EndOffset = BF.getMaxSize();
 551     else if (CodeIter == Islands.CodeOffsets.end())
 552       EndOffset = *NextData;
 553     else if (NextData == Islands.DataOffsets.end())
 554       EndOffset = *CodeIter;
 555     else
 556       EndOffset = (*CodeIter > *NextData) ? *NextData : *CodeIter;
 557
 558     if (FunctionOffset == EndOffset)
 559       continue; // Size is zero, nothing to emit
 560
 561     auto emitCI = [&](uint64_t &FunctionOffset, uint64_t EndOffset) {
 562       if (FunctionOffset >= EndOffset)
 563         return;
 564
 565       for (auto It = Islands.Relocations.lower_bound(FunctionOffset);
 566            It != Islands.Relocations.end(); ++It) {
 567         if (It->first >= EndOffset)
 568           break;
 569
 570         const Relocation &Relocation = It->second;
 571         if (FunctionOffset < Relocation.Offset) {
 572           Streamer.emitBytes(
 573               FunctionContents.slice(FunctionOffset, Relocation.Offset));
 574           FunctionOffset = Relocation.Offset;
 575         }
 576
 577         LLVM_DEBUG(
 578             dbgs() << "BOLT-DEBUG: emitting constant island relocation"
 579                    << " for " << BF << " at offset 0x"
 580                    << Twine::utohexstr(Relocation.Offset) << " with size "
 581                    << Relocation::getSizeForType(Relocation.Type) << '\n');
 582
 583         FunctionOffset += Relocation.emit(&Streamer);
 584       }
 585
 586       assert(FunctionOffset <= EndOffset && "overflow error");
 587       if (FunctionOffset < EndOffset) {
 588         Streamer.emitBytes(FunctionContents.slice(FunctionOffset, EndOffset));
 589         FunctionOffset = EndOffset;
 590       }
 591     };
 592
 593     // Emit labels, relocs and data
 594     while (IS != Islands.Offsets.end() && IS->first < EndOffset) {
 595       auto NextLabelOffset =
 596           IS == Islands.Offsets.end() ? EndOffset : IS->first;
 597       auto NextStop = std::min(NextLabelOffset, EndOffset);
 598       assert(NextStop <= EndOffset && "internal overflow error");
 599       emitCI(FunctionOffset, NextStop);
 600       if (IS != Islands.Offsets.end() && FunctionOffset == IS->first) {
 601         // This is a slightly complex code to decide which label to emit. We
 602         // have 4 cases to handle: regular symbol, cold symbol, regular or cold
 603         // symbol being emitted on behalf of an external function.
 604         if (!OnBehalfOf) {
 605           if (!EmitColdPart) {
 606             LLVM_DEBUG(dbgs() << "BOLT-DEBUG: emitted label "
 607                               << IS->second->getName() << " at offset 0x"
 608                               << Twine::utohexstr(IS->first) << '\n');
 609             if (IS->second->isUndefined())
 610               Streamer.emitLabel(IS->second);
 611             else
 612               assert(BF.hasName(std::string(IS->second->getName())));
 613           } else if (Islands.ColdSymbols.count(IS->second) != 0) {
 614             LLVM_DEBUG(dbgs()
 615                        << "BOLT-DEBUG: emitted label "
 616                        << Islands.ColdSymbols[IS->second]->getName() << '\n');
 617             if (Islands.ColdSymbols[IS->second]->isUndefined())
 618               Streamer.emitLabel(Islands.ColdSymbols[IS->second]);
 619           }
 620         } else {
 621           if (!EmitColdPart) {
 622             if (MCSymbol *Sym = Islands.Proxies[OnBehalfOf][IS->second]) {
 623               LLVM_DEBUG(dbgs() << "BOLT-DEBUG: emitted label "
 624                                 << Sym->getName() << '\n');
 625               Streamer.emitLabel(Sym);
 626             }
 627           } else if (MCSymbol *Sym =
 628                          Islands.ColdProxies[OnBehalfOf][IS->second]) {
 629             LLVM_DEBUG(dbgs() << "BOLT-DEBUG: emitted label " << Sym->getName()
 630                               << '\n');
 631             Streamer.emitLabel(Sym);
 632           }
 633         }
 634         ++IS;
 635       }
 636     }
 637     assert(FunctionOffset <= EndOffset && "overflow error");
 638     emitCI(FunctionOffset, EndOffset);
 639   }
 640   assert(IS == Islands.Offsets.end() && "some symbols were not emitted!");
 641
 642   if (OnBehalfOf)
 643     return;
 644   // Now emit constant islands from other functions that we may have used in
 645   // this function.
 646   for (BinaryFunction *ExternalFunc : Islands.Dependency)
 647     emitConstantIslands(*ExternalFunc, EmitColdPart, &BF);
 648 }
 649
 650 SMLoc BinaryEmitter::emitLineInfo(const BinaryFunction &BF, SMLoc NewLoc,
 651                                   SMLoc PrevLoc, bool FirstInstr,
 652                                   MCSymbol *&InstrLabel) {
 653   DWARFUnit *FunctionCU = BF.getDWARFUnit();
 654   const DWARFDebugLine::LineTable *FunctionLineTable = BF.getDWARFLineTable();
 655   assert(FunctionCU && "cannot emit line info for function without CU");
 656
 657   DebugLineTableRowRef RowReference = DebugLineTableRowRef::fromSMLoc(NewLoc);
 658
 659   // Check if no new line info needs to be emitted.
 660   if (RowReference == DebugLineTableRowRef::NULL_ROW ||
 661       NewLoc.getPointer() == PrevLoc.getPointer())
 662     return PrevLoc;
 663
 664   unsigned CurrentFilenum = 0;
 665   const DWARFDebugLine::LineTable *CurrentLineTable = FunctionLineTable;
 666
 667   // If the CU id from the current instruction location does not
 668   // match the CU id from the current function, it means that we
 669   // have come across some inlined code.  We must look up the CU
 670   // for the instruction's original function and get the line table
 671   // from that.
 672   const uint64_t FunctionUnitIndex = FunctionCU->getOffset();
 673   const uint32_t CurrentUnitIndex = RowReference.DwCompileUnitIndex;
 674   if (CurrentUnitIndex != FunctionUnitIndex) {
 675     CurrentLineTable = BC.DwCtx->getLineTableForUnit(
 676         BC.DwCtx->getCompileUnitForOffset(CurrentUnitIndex));
 677     // Add filename from the inlined function to the current CU.
 678     CurrentFilenum = BC.addDebugFilenameToUnit(
 679         FunctionUnitIndex, CurrentUnitIndex,
 680         CurrentLineTable->Rows[RowReference.RowIndex - 1].File);
 681   }
 682
 683   const DWARFDebugLine::Row &CurrentRow =
 684       CurrentLineTable->Rows[RowReference.RowIndex - 1];
 685   if (!CurrentFilenum)
 686     CurrentFilenum = CurrentRow.File;
 687
 688   unsigned Flags = (DWARF2_FLAG_IS_STMT * CurrentRow.IsStmt) |
 689                    (DWARF2_FLAG_BASIC_BLOCK * CurrentRow.BasicBlock) |
 690                    (DWARF2_FLAG_PROLOGUE_END * CurrentRow.PrologueEnd) |
 691                    (DWARF2_FLAG_EPILOGUE_BEGIN * CurrentRow.EpilogueBegin);
 692
 693   // Always emit is_stmt at the beginning of function fragment.
 694   if (FirstInstr)
 695     Flags |= DWARF2_FLAG_IS_STMT;
 696
 697   BC.Ctx->setCurrentDwarfLoc(CurrentFilenum, CurrentRow.Line, CurrentRow.Column,
 698                              Flags, CurrentRow.Isa, CurrentRow.Discriminator);
 699   const MCDwarfLoc &DwarfLoc = BC.Ctx->getCurrentDwarfLoc();
 700   BC.Ctx->clearDwarfLocSeen();
 701
 702   if (!InstrLabel)
 703     InstrLabel = BC.Ctx->createTempSymbol();
 704
 705   BC.getDwarfLineTable(FunctionUnitIndex)
 706       .getMCLineSections()
 707       .addLineEntry(MCDwarfLineEntry(InstrLabel, DwarfLoc),
 708                     Streamer.getCurrentSectionOnly());
 709
 710   return NewLoc;
 711 }
 712
 713 void BinaryEmitter::emitLineInfoEnd(const BinaryFunction &BF,
 714                                     MCSymbol *FunctionEndLabel) {
 715   DWARFUnit *FunctionCU = BF.getDWARFUnit();
 716   assert(FunctionCU && "DWARF unit expected");
 717   BC.Ctx->setCurrentDwarfLoc(0, 0, 0, DWARF2_FLAG_END_SEQUENCE, 0, 0);
 718   const MCDwarfLoc &DwarfLoc = BC.Ctx->getCurrentDwarfLoc();
 719   BC.Ctx->clearDwarfLocSeen();
 720   BC.getDwarfLineTable(FunctionCU->getOffset())
 721       .getMCLineSections()
 722       .addLineEntry(MCDwarfLineEntry(FunctionEndLabel, DwarfLoc),
 723                     Streamer.getCurrentSectionOnly());
 724 }
 725
 726 void BinaryEmitter::emitJumpTables(const BinaryFunction &BF) {
 727   MCSection *ReadOnlySection = BC.MOFI->getReadOnlySection();
 728   MCSection *ReadOnlyColdSection = BC.MOFI->getContext().getELFSection(
 729       ".rodata.cold", ELF::SHT_PROGBITS, ELF::SHF_ALLOC);
 730
 731   if (!BF.hasJumpTables())
 732     return;
 733
 734   if (opts::PrintJumpTables)
 735     BC.outs() << "BOLT-INFO: jump tables for function " << BF << ":\n";
 736
 737   for (auto &JTI : BF.jumpTables()) {
 738     JumpTable &JT = *JTI.second;
 739     // Only emit shared jump tables once, when processing the first parent
 740     if (JT.Parents.size() > 1 && JT.Parents[0] != &BF)
 741       continue;
 742     if (opts::PrintJumpTables)
 743       JT.print(BC.outs());
 744     if (opts::JumpTables == JTS_BASIC && BC.HasRelocations) {
 745       JT.updateOriginal();
 746     } else {
 747       MCSection *HotSection, *ColdSection;
 748       if (opts::JumpTables == JTS_BASIC) {
 749         // In non-relocation mode we have to emit jump tables in local sections.
 750         // This way we only overwrite them when the corresponding function is
 751         // overwritten.
 752         std::string Name = ".local." + JT.Labels[0]->getName().str();
 753         std::replace(Name.begin(), Name.end(), '/', '.');
 754         BinarySection &Section =
 755             BC.registerOrUpdateSection(Name, ELF::SHT_PROGBITS, ELF::SHF_ALLOC);
 756         Section.setAnonymous(true);
 757         JT.setOutputSection(Section);
 758         HotSection = BC.getDataSection(Name);
 759         ColdSection = HotSection;
 760       } else {
 761         if (BF.isSimple()) {
 762           HotSection = ReadOnlySection;
 763           ColdSection = ReadOnlyColdSection;
 764         } else {
 765           HotSection = BF.hasProfile() ? ReadOnlySection : ReadOnlyColdSection;
 766           ColdSection = HotSection;
 767         }
 768       }
 769       emitJumpTable(JT, HotSection, ColdSection);
 770     }
 771   }
 772 }
 773
 774 void BinaryEmitter::emitJumpTable(const JumpTable &JT, MCSection *HotSection,
 775                                   MCSection *ColdSection) {
 776   // Pre-process entries for aggressive splitting.
 777   // Each label represents a separate switch table and gets its own count
 778   // determining its destination.
 779   std::map<MCSymbol *, uint64_t> LabelCounts;
 780   if (opts::JumpTables > JTS_SPLIT && !JT.Counts.empty()) {
 781     auto It = JT.Labels.find(0);
 782     assert(It != JT.Labels.end());
 783     MCSymbol *CurrentLabel = It->second;
 784     uint64_t CurrentLabelCount = 0;
 785     for (unsigned Index = 0; Index < JT.Entries.size(); ++Index) {
 786       auto LI = JT.Labels.find(Index * JT.EntrySize);
 787       if (LI != JT.Labels.end()) {
 788         LabelCounts[CurrentLabel] = CurrentLabelCount;
 789         CurrentLabel = LI->second;
 790         CurrentLabelCount = 0;
 791       }
 792       CurrentLabelCount += JT.Counts[Index].Count;
 793     }
 794     LabelCounts[CurrentLabel] = CurrentLabelCount;
 795   } else {
 796     Streamer.switchSection(JT.Count > 0 ? HotSection : ColdSection);
 797     Streamer.emitValueToAlignment(Align(JT.EntrySize));
 798   }
 799   MCSymbol *LastLabel = nullptr;
 800   uint64_t Offset = 0;
 801   for (MCSymbol *Entry : JT.Entries) {
 802     auto LI = JT.Labels.find(Offset);
 803     if (LI != JT.Labels.end()) {
 804       LLVM_DEBUG({
 805         dbgs() << "BOLT-DEBUG: emitting jump table " << LI->second->getName()
 806                << " (originally was at address 0x"
 807                << Twine::utohexstr(JT.getAddress() + Offset)
 808                << (Offset ? ") as part of larger jump table\n" : ")\n");
 809       });
 810       if (!LabelCounts.empty()) {
 811         LLVM_DEBUG(dbgs() << "BOLT-DEBUG: jump table count: "
 812                           << LabelCounts[LI->second] << '\n');
 813         if (LabelCounts[LI->second] > 0)
 814           Streamer.switchSection(HotSection);
 815         else
 816           Streamer.switchSection(ColdSection);
 817         Streamer.emitValueToAlignment(Align(JT.EntrySize));
 818       }
 819       // Emit all labels registered at the address of this jump table
 820       // to sync with our global symbol table.  We may have two labels
 821       // registered at this address if one label was created via
 822       // getOrCreateGlobalSymbol() (e.g. LEA instructions referencing
 823       // this location) and another via getOrCreateJumpTable().  This
 824       // creates a race where the symbols created by these two
 825       // functions may or may not be the same, but they are both
 826       // registered in our symbol table at the same address. By
 827       // emitting them all here we make sure there is no ambiguity
 828       // that depends on the order that these symbols were created, so
 829       // whenever this address is referenced in the binary, it is
 830       // certain to point to the jump table identified at this
 831       // address.
 832       if (BinaryData *BD = BC.getBinaryDataByName(LI->second->getName())) {
 833         for (MCSymbol *S : BD->getSymbols())
 834           Streamer.emitLabel(S);
 835       } else {
 836         Streamer.emitLabel(LI->second);
 837       }
 838       LastLabel = LI->second;
 839     }
 840     if (JT.Type == JumpTable::JTT_NORMAL) {
 841       Streamer.emitSymbolValue(Entry, JT.OutputEntrySize);
 842     } else { // JTT_PIC
 843       const MCSymbolRefExpr *JTExpr =
 844           MCSymbolRefExpr::create(LastLabel, Streamer.getContext());
 845       const MCSymbolRefExpr *E =
 846           MCSymbolRefExpr::create(Entry, Streamer.getContext());
 847       const MCBinaryExpr *Value =
 848           MCBinaryExpr::createSub(E, JTExpr, Streamer.getContext());
 849       Streamer.emitValue(Value, JT.EntrySize);
 850     }
 851     Offset += JT.EntrySize;
 852   }
 853 }
 854
 855 void BinaryEmitter::emitCFIInstruction(const MCCFIInstruction &Inst) const {
 856   switch (Inst.getOperation()) {
 857   default:
 858     llvm_unreachable("Unexpected instruction");
 859   case MCCFIInstruction::OpDefCfaOffset:
 860     Streamer.emitCFIDefCfaOffset(Inst.getOffset());
 861     break;
 862   case MCCFIInstruction::OpAdjustCfaOffset:
 863     Streamer.emitCFIAdjustCfaOffset(Inst.getOffset());
 864     break;
 865   case MCCFIInstruction::OpDefCfa:
 866     Streamer.emitCFIDefCfa(Inst.getRegister(), Inst.getOffset());
 867     break;
 868   case MCCFIInstruction::OpDefCfaRegister:
 869     Streamer.emitCFIDefCfaRegister(Inst.getRegister());
 870     break;
 871   case MCCFIInstruction::OpOffset:
 872     Streamer.emitCFIOffset(Inst.getRegister(), Inst.getOffset());
 873     break;
 874   case MCCFIInstruction::OpRegister:
 875     Streamer.emitCFIRegister(Inst.getRegister(), Inst.getRegister2());
 876     break;
 877   case MCCFIInstruction::OpWindowSave:
 878     Streamer.emitCFIWindowSave();
 879     break;
 880   case MCCFIInstruction::OpNegateRAState:
 881     Streamer.emitCFINegateRAState();
 882     break;
 883   case MCCFIInstruction::OpSameValue:
 884     Streamer.emitCFISameValue(Inst.getRegister());
 885     break;
 886   case MCCFIInstruction::OpGnuArgsSize:
 887     Streamer.emitCFIGnuArgsSize(Inst.getOffset());
 888     break;
 889   case MCCFIInstruction::OpEscape:
 890     Streamer.AddComment(Inst.getComment());
 891     Streamer.emitCFIEscape(Inst.getValues());
 892     break;
 893   case MCCFIInstruction::OpRestore:
 894     Streamer.emitCFIRestore(Inst.getRegister());
 895     break;
 896   case MCCFIInstruction::OpUndefined:
 897     Streamer.emitCFIUndefined(Inst.getRegister());
 898     break;
 899   }
 900 }
 901
 902 // The code is based on EHStreamer::emitExceptionTable().
 903 void BinaryEmitter::emitLSDA(BinaryFunction &BF, const FunctionFragment &FF) {
 904   const BinaryFunction::CallSitesRange Sites =
 905       BF.getCallSites(FF.getFragmentNum());
 906   if (Sites.empty())
 907     return;
 908
 909   // Calculate callsite table size. Size of each callsite entry is:
 910   //
 911   //  sizeof(start) + sizeof(length) + sizeof(LP) + sizeof(uleb128(action))
 912   //
 913   // or
 914   //
 915   //  sizeof(dwarf::DW_EH_PE_data4) * 3 + sizeof(uleb128(action))
 916   uint64_t CallSiteTableLength = llvm::size(Sites) * 4 * 3;
 917   for (const auto &FragmentCallSite : Sites)
 918     CallSiteTableLength += getULEB128Size(FragmentCallSite.second.Action);
 919
 920   Streamer.switchSection(BC.MOFI->getLSDASection());
 921
 922   const unsigned TTypeEncoding = BF.getLSDATypeEncoding();
 923   const unsigned TTypeEncodingSize = BC.getDWARFEncodingSize(TTypeEncoding);
 924   const uint16_t TTypeAlignment = 4;
 925
 926   // Type tables have to be aligned at 4 bytes.
 927   Streamer.emitValueToAlignment(Align(TTypeAlignment));
 928
 929   // Emit the LSDA label.
 930   MCSymbol *LSDASymbol = BF.getLSDASymbol(FF.getFragmentNum());
 931   assert(LSDASymbol && "no LSDA symbol set");
 932   Streamer.emitLabel(LSDASymbol);
 933
 934   // Corresponding FDE start.
 935   const MCSymbol *StartSymbol = BF.getSymbol(FF.getFragmentNum());
 936
 937   // Emit the LSDA header.
 938
 939   // If LPStart is omitted, then the start of the FDE is used as a base for
 940   // landing pad displacements. Then if a cold fragment starts with
 941   // a landing pad, this means that the first landing pad offset will be 0.
 942   // As a result, the exception handling runtime will ignore this landing pad
 943   // because zero offset denotes the absence of a landing pad.
 944   // For this reason, when the binary has fixed starting address we emit LPStart
 945   // as 0 and output the absolute value of the landing pad in the table.
 946   //
 947   // If the base address can change, we cannot use absolute addresses for
 948   // landing pads (at least not without runtime relocations). Hence, we fall
 949   // back to emitting landing pads relative to the FDE start.
 950   // As we are emitting label differences, we have to guarantee both labels are
 951   // defined in the same section and hence cannot place the landing pad into a
 952   // cold fragment when the corresponding call site is in the hot fragment.
 953   // Because of this issue and the previously described issue of possible
 954   // zero-offset landing pad we have to place landing pads in the same section
 955   // as the corresponding invokes for shared objects.
 956   std::function<void(const MCSymbol *)> emitLandingPad;
 957   if (BC.HasFixedLoadAddress) {
 958     Streamer.emitIntValue(dwarf::DW_EH_PE_udata4, 1); // LPStart format
 959     Streamer.emitIntValue(0, 4);                      // LPStart
 960     emitLandingPad = [&](const MCSymbol *LPSymbol) {
 961       if (!LPSymbol)
 962         Streamer.emitIntValue(0, 4);
 963       else
 964         Streamer.emitSymbolValue(LPSymbol, 4);
 965     };
 966   } else {
 967     Streamer.emitIntValue(dwarf::DW_EH_PE_omit, 1); // LPStart format
 968     emitLandingPad = [&](const MCSymbol *LPSymbol) {
 969       if (!LPSymbol)
 970         Streamer.emitIntValue(0, 4);
 971       else
 972         Streamer.emitAbsoluteSymbolDiff(LPSymbol, StartSymbol, 4);
 973     };
 974   }
 975
 976   Streamer.emitIntValue(TTypeEncoding, 1); // TType format
 977
 978   // See the comment in EHStreamer::emitExceptionTable() on to use
 979   // uleb128 encoding (which can use variable number of bytes to encode the same
 980   // value) to ensure type info table is properly aligned at 4 bytes without
 981   // iteratively fixing sizes of the tables.
 982   unsigned CallSiteTableLengthSize = getULEB128Size(CallSiteTableLength);
 983   unsigned TTypeBaseOffset =
 984       sizeof(int8_t) +                 // Call site format
 985       CallSiteTableLengthSize +        // Call site table length size
 986       CallSiteTableLength +            // Call site table length
 987       BF.getLSDAActionTable().size() + // Actions table size
 988       BF.getLSDATypeTable().size() * TTypeEncodingSize; // Types table size
 989   unsigned TTypeBaseOffsetSize = getULEB128Size(TTypeBaseOffset);
 990   unsigned TotalSize = sizeof(int8_t) +      // LPStart format
 991                        sizeof(int8_t) +      // TType format
 992                        TTypeBaseOffsetSize + // TType base offset size
 993                        TTypeBaseOffset;      // TType base offset
 994   unsigned SizeAlign = (4 - TotalSize) & 3;
 995
 996   if (TTypeEncoding != dwarf::DW_EH_PE_omit)
 997     // Account for any extra padding that will be added to the call site table
 998     // length.
 999     Streamer.emitULEB128IntValue(TTypeBaseOffset,
1000                                  /*PadTo=*/TTypeBaseOffsetSize + SizeAlign);
1001
1002   // Emit the landing pad call site table. We use signed data4 since we can emit
1003   // a landing pad in a different part of the split function that could appear
1004   // earlier in the address space than LPStart.
1005   Streamer.emitIntValue(dwarf::DW_EH_PE_sdata4, 1);
1006   Streamer.emitULEB128IntValue(CallSiteTableLength);
1007
1008   for (const auto &FragmentCallSite : Sites) {
1009     const BinaryFunction::CallSite &CallSite = FragmentCallSite.second;
1010     const MCSymbol *BeginLabel = CallSite.Start;
1011     const MCSymbol *EndLabel = CallSite.End;
1012
1013     assert(BeginLabel && "start EH label expected");
1014     assert(EndLabel && "end EH label expected");
1015
1016     // Start of the range is emitted relative to the start of current
1017     // function split part.
1018     Streamer.emitAbsoluteSymbolDiff(BeginLabel, StartSymbol, 4);
1019     Streamer.emitAbsoluteSymbolDiff(EndLabel, BeginLabel, 4);
1020     emitLandingPad(CallSite.LP);
1021     Streamer.emitULEB128IntValue(CallSite.Action);
1022   }
1023
1024   // Write out action, type, and type index tables at the end.
1025   //
1026   // For action and type index tables there's no need to change the original
1027   // table format unless we are doing function splitting, in which case we can
1028   // split and optimize the tables.
1029   //
1030   // For type table we (re-)encode the table using TTypeEncoding matching
1031   // the current assembler mode.
1032   for (uint8_t const &Byte : BF.getLSDAActionTable())
1033     Streamer.emitIntValue(Byte, 1);
1034
1035   const BinaryFunction::LSDATypeTableTy &TypeTable =
1036       (TTypeEncoding & dwarf::DW_EH_PE_indirect) ? BF.getLSDATypeAddressTable()
1037                                                  : BF.getLSDATypeTable();
1038   assert(TypeTable.size() == BF.getLSDATypeTable().size() &&
1039          "indirect type table size mismatch");
1040
1041   for (int Index = TypeTable.size() - 1; Index >= 0; --Index) {
1042     const uint64_t TypeAddress = TypeTable[Index];
1043     switch (TTypeEncoding & 0x70) {
1044     default:
1045       llvm_unreachable("unsupported TTypeEncoding");
1046     case dwarf::DW_EH_PE_absptr:
1047       Streamer.emitIntValue(TypeAddress, TTypeEncodingSize);
1048       break;
1049     case dwarf::DW_EH_PE_pcrel: {
1050       if (TypeAddress) {
1051         const MCSymbol *TypeSymbol =
1052             BC.getOrCreateGlobalSymbol(TypeAddress, "TI", 0, TTypeAlignment);
1053         MCSymbol *DotSymbol = BC.Ctx->createNamedTempSymbol();
1054         Streamer.emitLabel(DotSymbol);
1055         const MCBinaryExpr *SubDotExpr = MCBinaryExpr::createSub(
1056             MCSymbolRefExpr::create(TypeSymbol, *BC.Ctx),
1057             MCSymbolRefExpr::create(DotSymbol, *BC.Ctx), *BC.Ctx);
1058         Streamer.emitValue(SubDotExpr, TTypeEncodingSize);
1059       } else {
1060         Streamer.emitIntValue(0, TTypeEncodingSize);
1061       }
1062       break;
1063     }
1064     }
1065   }
1066   for (uint8_t const &Byte : BF.getLSDATypeIndexTable())
1067     Streamer.emitIntValue(Byte, 1);
1068 }
1069
1070 void BinaryEmitter::emitDebugLineInfoForOriginalFunctions() {
1071   // If a function is in a CU containing at least one processed function, we
1072   // have to rewrite the whole line table for that CU. For unprocessed functions
1073   // we use data from the input line table.
1074   for (auto &It : BC.getBinaryFunctions()) {
1075     const BinaryFunction &Function = It.second;
1076
1077     // If the function was emitted, its line info was emitted with it.
1078     if (Function.isEmitted())
1079       continue;
1080
1081     const DWARFDebugLine::LineTable *LineTable = Function.getDWARFLineTable();
1082     if (!LineTable)
1083       continue; // nothing to update for this function
1084
1085     const uint64_t Address = Function.getAddress();
1086     std::vector<uint32_t> Results;
1087     if (!LineTable->lookupAddressRange(
1088             {Address, object::SectionedAddress::UndefSection},
1089             Function.getSize(), Results))
1090       continue;
1091
1092     if (Results.empty())
1093       continue;
1094
1095     // The first row returned could be the last row matching the start address.
1096     // Find the first row with the same address that is not the end of the
1097     // sequence.
1098     uint64_t FirstRow = Results.front();
1099     while (FirstRow > 0) {
1100       const DWARFDebugLine::Row &PrevRow = LineTable->Rows[FirstRow - 1];
1101       if (PrevRow.Address.Address != Address || PrevRow.EndSequence)
1102         break;
1103       --FirstRow;
1104     }
1105
1106     const uint64_t EndOfSequenceAddress =
1107         Function.getAddress() + Function.getMaxSize();
1108     BC.getDwarfLineTable(Function.getDWARFUnit()->getOffset())
1109         .addLineTableSequence(LineTable, FirstRow, Results.back(),
1110                               EndOfSequenceAddress);
1111   }
1112
1113   // For units that are completely unprocessed, use original debug line contents
1114   // eliminating the need to regenerate line info program.
1115   emitDebugLineInfoForUnprocessedCUs();
1116 }
1117
1118 void BinaryEmitter::emitDebugLineInfoForUnprocessedCUs() {
1119   // Sorted list of section offsets provides boundaries for section fragments,
1120   // where each fragment is the unit's contribution to debug line section.
1121   std::vector<uint64_t> StmtListOffsets;
1122   StmtListOffsets.reserve(BC.DwCtx->getNumCompileUnits());
1123   for (const std::unique_ptr<DWARFUnit> &CU : BC.DwCtx->compile_units()) {
1124     DWARFDie CUDie = CU->getUnitDIE();
1125     auto StmtList = dwarf::toSectionOffset(CUDie.find(dwarf::DW_AT_stmt_list));
1126     if (!StmtList)
1127       continue;
1128
1129     StmtListOffsets.push_back(*StmtList);
1130   }
1131   llvm::sort(StmtListOffsets);
1132
1133   // For each CU that was not processed, emit its line info as a binary blob.
1134   for (const std::unique_ptr<DWARFUnit> &CU : BC.DwCtx->compile_units()) {
1135     if (BC.ProcessedCUs.count(CU.get()))
1136       continue;
1137
1138     DWARFDie CUDie = CU->getUnitDIE();
1139     auto StmtList = dwarf::toSectionOffset(CUDie.find(dwarf::DW_AT_stmt_list));
1140     if (!StmtList)
1141       continue;
1142
1143     StringRef DebugLineContents = CU->getLineSection().Data;
1144
1145     const uint64_t Begin = *StmtList;
1146
1147     // Statement list ends where the next unit contribution begins, or at the
1148     // end of the section.
1149     auto It = llvm::upper_bound(StmtListOffsets, Begin);
1150     const uint64_t End =
1151         It == StmtListOffsets.end() ? DebugLineContents.size() : *It;
1152
1153     BC.getDwarfLineTable(CU->getOffset())
1154         .addRawContents(DebugLineContents.slice(Begin, End));
1155   }
1156 }
1157
1158 void BinaryEmitter::emitDataSections(StringRef OrgSecPrefix) {
1159   for (BinarySection &Section : BC.sections()) {
1160     if (!Section.hasRelocations())
1161       continue;
1162
1163     StringRef Prefix = Section.hasSectionRef() ? OrgSecPrefix : "";
1164     Section.emitAsData(Streamer, Prefix + Section.getName());
1165     Section.clearRelocations();
1166   }
1167 }
1168
1169 namespace llvm {
1170 namespace bolt {
1171
1172 void emitBinaryContext(MCStreamer &Streamer, BinaryContext &BC,
1173                        StringRef OrgSecPrefix) {
1174   BinaryEmitter(Streamer, BC).emitAll(OrgSecPrefix);
1175 }
1176
1177 void emitFunctionBody(MCStreamer &Streamer, BinaryFunction &BF,
1178                       FunctionFragment &FF, bool EmitCodeOnly) {
1179   BinaryEmitter(Streamer, BF.getBinaryContext())
1180       .emitFunctionBody(BF, FF, EmitCodeOnly);
1181 }
1182
1183 } // namespace bolt
1184 } // namespace llvm