bolt/lib/Core/BinaryEmitter.cpp

   1 //===- bolt/Core/BinaryEmitter.cpp - Emit code and data -------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file implements the collection of functions and classes used for
  10 // emission of code and data into object/binary file.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "bolt/Core/BinaryEmitter.h"
  15 #include "bolt/Core/BinaryContext.h"
  16 #include "bolt/Core/BinaryFunction.h"
  17 #include "bolt/Core/DebugData.h"
  18 #include "bolt/Core/FunctionLayout.h"
  19 #include "bolt/Utils/CommandLineOpts.h"
  20 #include "bolt/Utils/Utils.h"
  21 #include "llvm/DebugInfo/DWARF/DWARFCompileUnit.h"
  22 #include "llvm/MC/MCSection.h"
  23 #include "llvm/MC/MCStreamer.h"
  24 #include "llvm/Support/CommandLine.h"
  25 #include "llvm/Support/LEB128.h"
  26 #include "llvm/Support/SMLoc.h"
  27
  28 #define DEBUG_TYPE "bolt"
  29
  30 using namespace llvm;
  31 using namespace bolt;
  32
  33 namespace opts {
  34
  35 extern cl::opt<JumpTableSupportLevel> JumpTables;
  36 extern cl::opt<bool> PreserveBlocksAlignment;
  37
  38 cl::opt<bool> AlignBlocks("align-blocks", cl::desc("align basic blocks"),
  39                           cl::cat(BoltOptCategory));
  40
  41 static cl::list<std::string>
  42 BreakFunctionNames("break-funcs",
  43   cl::CommaSeparated,
  44   cl::desc("list of functions to core dump on (debugging)"),
  45   cl::value_desc("func1,func2,func3,..."),
  46   cl::Hidden,
  47   cl::cat(BoltCategory));
  48
  49 static cl::list<std::string>
  50 FunctionPadSpec("pad-funcs",
  51   cl::CommaSeparated,
  52   cl::desc("list of functions to pad with amount of bytes"),
  53   cl::value_desc("func1:pad1,func2:pad2,func3:pad3,..."),
  54   cl::Hidden,
  55   cl::cat(BoltCategory));
  56
  57 static cl::opt<bool> MarkFuncs(
  58     "mark-funcs",
  59     cl::desc("mark function boundaries with break instruction to make "
  60              "sure we accidentally don't cross them"),
  61     cl::ReallyHidden, cl::cat(BoltCategory));
  62
  63 static cl::opt<bool> PrintJumpTables("print-jump-tables",
  64                                      cl::desc("print jump tables"), cl::Hidden,
  65                                      cl::cat(BoltCategory));
  66
  67 static cl::opt<bool>
  68 X86AlignBranchBoundaryHotOnly("x86-align-branch-boundary-hot-only",
  69   cl::desc("only apply branch boundary alignment in hot code"),
  70   cl::init(true),
  71   cl::cat(BoltOptCategory));
  72
  73 size_t padFunction(const BinaryFunction &Function) {
  74   static std::map<std::string, size_t> FunctionPadding;
  75
  76   if (FunctionPadding.empty() && !FunctionPadSpec.empty()) {
  77     for (std::string &Spec : FunctionPadSpec) {
  78       size_t N = Spec.find(':');
  79       if (N == std::string::npos)
  80         continue;
  81       std::string Name = Spec.substr(0, N);
  82       size_t Padding = std::stoull(Spec.substr(N + 1));
  83       FunctionPadding[Name] = Padding;
  84     }
  85   }
  86
  87   for (auto &FPI : FunctionPadding) {
  88     std::string Name = FPI.first;
  89     size_t Padding = FPI.second;
  90     if (Function.hasNameRegex(Name))
  91       return Padding;
  92   }
  93
  94   return 0;
  95 }
  96
  97 } // namespace opts
  98
  99 namespace {
 100 using JumpTable = bolt::JumpTable;
 101
 102 class BinaryEmitter {
 103 private:
 104   BinaryEmitter(const BinaryEmitter &) = delete;
 105   BinaryEmitter &operator=(const BinaryEmitter &) = delete;
 106
 107   MCStreamer &Streamer;
 108   BinaryContext &BC;
 109
 110 public:
 111   BinaryEmitter(MCStreamer &Streamer, BinaryContext &BC)
 112       : Streamer(Streamer), BC(BC) {}
 113
 114   /// Emit all code and data.
 115   void emitAll(StringRef OrgSecPrefix);
 116
 117   /// Emit function code. The caller is responsible for emitting function
 118   /// symbol(s) and setting the section to emit the code to.
 119   void emitFunctionBody(BinaryFunction &BF, FunctionFragment &FF,
 120                         bool EmitCodeOnly = false);
 121
 122 private:
 123   /// Emit function code.
 124   void emitFunctions();
 125
 126   /// Emit a single function.
 127   bool emitFunction(BinaryFunction &BF, FunctionFragment &FF);
 128
 129   /// Helper for emitFunctionBody to write data inside a function
 130   /// (used for AArch64)
 131   void emitConstantIslands(BinaryFunction &BF, bool EmitColdPart,
 132                            BinaryFunction *OnBehalfOf = nullptr);
 133
 134   /// Emit jump tables for the function.
 135   void emitJumpTables(const BinaryFunction &BF);
 136
 137   /// Emit jump table data. Callee supplies sections for the data.
 138   void emitJumpTable(const JumpTable &JT, MCSection *HotSection,
 139                      MCSection *ColdSection);
 140
 141   void emitCFIInstruction(const MCCFIInstruction &Inst) const;
 142
 143   /// Emit exception handling ranges for the function.
 144   void emitLSDA(BinaryFunction &BF, const FunctionFragment &FF);
 145
 146   /// Emit line number information corresponding to \p NewLoc. \p PrevLoc
 147   /// provides a context for de-duplication of line number info.
 148   /// \p FirstInstr indicates if \p NewLoc represents the first instruction
 149   /// in a sequence, such as a function fragment.
 150   ///
 151   /// If \p NewLoc location matches \p PrevLoc, no new line number entry will be
 152   /// created and the function will return \p PrevLoc while \p InstrLabel will
 153   /// be ignored. Otherwise, the caller should use \p InstrLabel to mark the
 154   /// corresponding instruction by emitting \p InstrLabel before it.
 155   /// If \p InstrLabel is set by the caller, its value will be used with \p
 156   /// \p NewLoc. If it was nullptr on entry, it will be populated with a pointer
 157   /// to a new temp symbol used with \p NewLoc.
 158   ///
 159   /// Return new current location which is either \p NewLoc or \p PrevLoc.
 160   SMLoc emitLineInfo(const BinaryFunction &BF, SMLoc NewLoc, SMLoc PrevLoc,
 161                      bool FirstInstr, MCSymbol *&InstrLabel);
 162
 163   /// Use \p FunctionEndSymbol to mark the end of the line info sequence.
 164   /// Note that it does not automatically result in the insertion of the EOS
 165   /// marker in the line table program, but provides one to the DWARF generator
 166   /// when it needs it.
 167   void emitLineInfoEnd(const BinaryFunction &BF, MCSymbol *FunctionEndSymbol);
 168
 169   /// Emit debug line info for unprocessed functions from CUs that include
 170   /// emitted functions.
 171   void emitDebugLineInfoForOriginalFunctions();
 172
 173   /// Emit debug line for CUs that were not modified.
 174   void emitDebugLineInfoForUnprocessedCUs();
 175
 176   /// Emit data sections that have code references in them.
 177   void emitDataSections(StringRef OrgSecPrefix);
 178 };
 179
 180 } // anonymous namespace
 181
 182 void BinaryEmitter::emitAll(StringRef OrgSecPrefix) {
 183   Streamer.initSections(false, *BC.STI);
 184   Streamer.setUseAssemblerInfoForParsing(false);
 185
 186   if (opts::UpdateDebugSections && BC.isELF()) {
 187     // Force the emission of debug line info into allocatable section to ensure
 188     // JITLink will process it.
 189     //
 190     // NB: on MachO all sections are required for execution, hence no need
 191     //     to change flags/attributes.
 192     MCSectionELF *ELFDwarfLineSection =
 193         static_cast<MCSectionELF *>(BC.MOFI->getDwarfLineSection());
 194     ELFDwarfLineSection->setFlags(ELF::SHF_ALLOC);
 195     MCSectionELF *ELFDwarfLineStrSection =
 196         static_cast<MCSectionELF *>(BC.MOFI->getDwarfLineStrSection());
 197     ELFDwarfLineStrSection->setFlags(ELF::SHF_ALLOC);
 198   }
 199
 200   if (RuntimeLibrary *RtLibrary = BC.getRuntimeLibrary())
 201     RtLibrary->emitBinary(BC, Streamer);
 202
 203   BC.getTextSection()->setAlignment(Align(opts::AlignText));
 204
 205   emitFunctions();
 206
 207   if (opts::UpdateDebugSections) {
 208     emitDebugLineInfoForOriginalFunctions();
 209     DwarfLineTable::emit(BC, Streamer);
 210   }
 211
 212   emitDataSections(OrgSecPrefix);
 213
 214   // TODO Enable for Mach-O once BinaryContext::getDataSection supports it.
 215   if (BC.isELF())
 216     AddressMap::emit(Streamer, BC);
 217   Streamer.setUseAssemblerInfoForParsing(true);
 218 }
 219
 220 void BinaryEmitter::emitFunctions() {
 221   auto emit = [&](const std::vector<BinaryFunction *> &Functions) {
 222     const bool HasProfile = BC.NumProfiledFuncs > 0;
 223     const bool OriginalAllowAutoPadding = Streamer.getAllowAutoPadding();
 224     for (BinaryFunction *Function : Functions) {
 225       if (!BC.shouldEmit(*Function))
 226         continue;
 227
 228       LLVM_DEBUG(dbgs() << "BOLT: generating code for function \"" << *Function
 229                         << "\" : " << Function->getFunctionNumber() << '\n');
 230
 231       // Was any part of the function emitted.
 232       bool Emitted = false;
 233
 234       // Turn off Intel JCC Erratum mitigation for cold code if requested
 235       if (HasProfile && opts::X86AlignBranchBoundaryHotOnly &&
 236           !Function->hasValidProfile())
 237         Streamer.setAllowAutoPadding(false);
 238
 239       FunctionLayout &Layout = Function->getLayout();
 240       Emitted |= emitFunction(*Function, Layout.getMainFragment());
 241
 242       if (Function->isSplit()) {
 243         if (opts::X86AlignBranchBoundaryHotOnly)
 244           Streamer.setAllowAutoPadding(false);
 245
 246         assert((Layout.fragment_size() == 1 || Function->isSimple()) &&
 247                "Only simple functions can have fragments");
 248         for (FunctionFragment &FF : Layout.getSplitFragments()) {
 249           // Skip empty fragments so no symbols and sections for empty fragments
 250           // are generated
 251           if (FF.empty() && !Function->hasConstantIsland())
 252             continue;
 253           Emitted |= emitFunction(*Function, FF);
 254         }
 255       }
 256
 257       Streamer.setAllowAutoPadding(OriginalAllowAutoPadding);
 258
 259       if (Emitted)
 260         Function->setEmitted(/*KeepCFG=*/opts::PrintCacheMetrics);
 261     }
 262   };
 263
 264   // Mark the start of hot text.
 265   if (opts::HotText) {
 266     Streamer.switchSection(BC.getTextSection());
 267     Streamer.emitLabel(BC.getHotTextStartSymbol());
 268   }
 269
 270   // Emit functions in sorted order.
 271   std::vector<BinaryFunction *> SortedFunctions = BC.getSortedFunctions();
 272   emit(SortedFunctions);
 273
 274   // Emit functions added by BOLT.
 275   emit(BC.getInjectedBinaryFunctions());
 276
 277   // Mark the end of hot text.
 278   if (opts::HotText) {
 279     if (BC.HasWarmSection)
 280       Streamer.switchSection(BC.getCodeSection(BC.getWarmCodeSectionName()));
 281     else
 282       Streamer.switchSection(BC.getTextSection());
 283     Streamer.emitLabel(BC.getHotTextEndSymbol());
 284   }
 285 }
 286
 287 bool BinaryEmitter::emitFunction(BinaryFunction &Function,
 288                                  FunctionFragment &FF) {
 289   if (Function.size() == 0 && !Function.hasIslandsInfo())
 290     return false;
 291
 292   if (Function.getState() == BinaryFunction::State::Empty)
 293     return false;
 294
 295   // Avoid emitting function without instructions when overwriting the original
 296   // function in-place. Otherwise, emit the empty function to define the symbol.
 297   if (!BC.HasRelocations && !Function.hasNonPseudoInstructions())
 298     return false;
 299
 300   MCSection *Section =
 301       BC.getCodeSection(Function.getCodeSectionName(FF.getFragmentNum()));
 302   Streamer.switchSection(Section);
 303   Section->setHasInstructions(true);
 304   BC.Ctx->addGenDwarfSection(Section);
 305
 306   if (BC.HasRelocations) {
 307     // Set section alignment to at least maximum possible object alignment.
 308     // We need this to support LongJmp and other passes that calculates
 309     // tentative layout.
 310     Section->ensureMinAlignment(Align(opts::AlignFunctions));
 311
 312     Streamer.emitCodeAlignment(Function.getMinAlign(), &*BC.STI);
 313     uint16_t MaxAlignBytes = FF.isSplitFragment()
 314                                  ? Function.getMaxColdAlignmentBytes()
 315                                  : Function.getMaxAlignmentBytes();
 316     if (MaxAlignBytes > 0)
 317       Streamer.emitCodeAlignment(Function.getAlign(), &*BC.STI, MaxAlignBytes);
 318   } else {
 319     Streamer.emitCodeAlignment(Function.getAlign(), &*BC.STI);
 320   }
 321
 322   MCContext &Context = Streamer.getContext();
 323   const MCAsmInfo *MAI = Context.getAsmInfo();
 324
 325   MCSymbol *const StartSymbol = Function.getSymbol(FF.getFragmentNum());
 326
 327   // Emit all symbols associated with the main function entry.
 328   if (FF.isMainFragment()) {
 329     for (MCSymbol *Symbol : Function.getSymbols()) {
 330       Streamer.emitSymbolAttribute(Symbol, MCSA_ELF_TypeFunction);
 331       Streamer.emitLabel(Symbol);
 332     }
 333   } else {
 334     Streamer.emitSymbolAttribute(StartSymbol, MCSA_ELF_TypeFunction);
 335     Streamer.emitLabel(StartSymbol);
 336   }
 337
 338   // Emit CFI start
 339   if (Function.hasCFI()) {
 340     Streamer.emitCFIStartProc(/*IsSimple=*/false);
 341     if (Function.getPersonalityFunction() != nullptr)
 342       Streamer.emitCFIPersonality(Function.getPersonalityFunction(),
 343                                   Function.getPersonalityEncoding());
 344     MCSymbol *LSDASymbol = Function.getLSDASymbol(FF.getFragmentNum());
 345     if (LSDASymbol)
 346       Streamer.emitCFILsda(LSDASymbol, BC.LSDAEncoding);
 347     else
 348       Streamer.emitCFILsda(0, dwarf::DW_EH_PE_omit);
 349     // Emit CFI instructions relative to the CIE
 350     for (const MCCFIInstruction &CFIInstr : Function.cie()) {
 351       // Only write CIE CFI insns that LLVM will not already emit
 352       const std::vector<MCCFIInstruction> &FrameInstrs =
 353           MAI->getInitialFrameState();
 354       if (!llvm::is_contained(FrameInstrs, CFIInstr))
 355         emitCFIInstruction(CFIInstr);
 356     }
 357   }
 358
 359   assert((Function.empty() || !(*Function.begin()).isCold()) &&
 360          "first basic block should never be cold");
 361
 362   // Emit UD2 at the beginning if requested by user.
 363   if (!opts::BreakFunctionNames.empty()) {
 364     for (std::string &Name : opts::BreakFunctionNames) {
 365       if (Function.hasNameRegex(Name)) {
 366         Streamer.emitIntValue(0x0B0F, 2); // UD2: 0F 0B
 367         break;
 368       }
 369     }
 370   }
 371
 372   // Emit code.
 373   emitFunctionBody(Function, FF, /*EmitCodeOnly=*/false);
 374
 375   // Emit padding if requested.
 376   if (size_t Padding = opts::padFunction(Function)) {
 377     LLVM_DEBUG(dbgs() << "BOLT-DEBUG: padding function " << Function << " with "
 378                       << Padding << " bytes\n");
 379     Streamer.emitFill(Padding, MAI->getTextAlignFillValue());
 380   }
 381
 382   if (opts::MarkFuncs)
 383     Streamer.emitBytes(BC.MIB->getTrapFillValue());
 384
 385   // Emit CFI end
 386   if (Function.hasCFI())
 387     Streamer.emitCFIEndProc();
 388
 389   MCSymbol *EndSymbol = Function.getFunctionEndLabel(FF.getFragmentNum());
 390   Streamer.emitLabel(EndSymbol);
 391
 392   if (MAI->hasDotTypeDotSizeDirective()) {
 393     const MCExpr *SizeExpr = MCBinaryExpr::createSub(
 394         MCSymbolRefExpr::create(EndSymbol, Context),
 395         MCSymbolRefExpr::create(StartSymbol, Context), Context);
 396     Streamer.emitELFSize(StartSymbol, SizeExpr);
 397   }
 398
 399   if (opts::UpdateDebugSections && Function.getDWARFUnit())
 400     emitLineInfoEnd(Function, EndSymbol);
 401
 402   // Exception handling info for the function.
 403   emitLSDA(Function, FF);
 404
 405   if (FF.isMainFragment() && opts::JumpTables > JTS_NONE)
 406     emitJumpTables(Function);
 407
 408   return true;
 409 }
 410
 411 void BinaryEmitter::emitFunctionBody(BinaryFunction &BF, FunctionFragment &FF,
 412                                      bool EmitCodeOnly) {
 413   if (!EmitCodeOnly && FF.isSplitFragment() && BF.hasConstantIsland()) {
 414     assert(BF.getLayout().isHotColdSplit() &&
 415            "Constant island support only with hot/cold split");
 416     BF.duplicateConstantIslands();
 417   }
 418
 419   // Track the first emitted instruction with debug info.
 420   bool FirstInstr = true;
 421   for (BinaryBasicBlock *const BB : FF) {
 422     if ((opts::AlignBlocks || opts::PreserveBlocksAlignment) &&
 423         BB->getAlignment() > 1)
 424       Streamer.emitCodeAlignment(BB->getAlign(), &*BC.STI,
 425                                  BB->getAlignmentMaxBytes());
 426     Streamer.emitLabel(BB->getLabel());
 427     if (!EmitCodeOnly) {
 428       if (MCSymbol *EntrySymbol = BF.getSecondaryEntryPointSymbol(*BB))
 429         Streamer.emitLabel(EntrySymbol);
 430     }
 431
 432     SMLoc LastLocSeen;
 433     for (auto I = BB->begin(), E = BB->end(); I != E; ++I) {
 434       MCInst &Instr = *I;
 435
 436       if (EmitCodeOnly && BC.MIB->isPseudo(Instr))
 437         continue;
 438
 439       // Handle pseudo instructions.
 440       if (BC.MIB->isCFI(Instr)) {
 441         emitCFIInstruction(*BF.getCFIFor(Instr));
 442         continue;
 443       }
 444
 445       if (!EmitCodeOnly) {
 446         // A symbol to be emitted before the instruction to mark its location.
 447         MCSymbol *InstrLabel = BC.MIB->getInstLabel(Instr);
 448
 449         if (opts::UpdateDebugSections && BF.getDWARFUnit()) {
 450           LastLocSeen = emitLineInfo(BF, Instr.getLoc(), LastLocSeen,
 451                                      FirstInstr, InstrLabel);
 452           FirstInstr = false;
 453         }
 454
 455         // Prepare to tag this location with a label if we need to keep track of
 456         // the location of calls/returns for BOLT address translation maps
 457         if (BF.requiresAddressTranslation() && BC.MIB->getOffset(Instr)) {
 458           const uint32_t Offset = *BC.MIB->getOffset(Instr);
 459           if (!InstrLabel)
 460             InstrLabel = BC.Ctx->createTempSymbol();
 461           BB->getLocSyms().emplace_back(Offset, InstrLabel);
 462         }
 463
 464         if (InstrLabel)
 465           Streamer.emitLabel(InstrLabel);
 466       }
 467
 468       // Emit sized NOPs via MCAsmBackend::writeNopData() interface on x86.
 469       // This is a workaround for invalid NOPs handling by asm/disasm layer.
 470       if (BC.isX86() && BC.MIB->isNoop(Instr)) {
 471         if (std::optional<uint32_t> Size = BC.MIB->getSize(Instr)) {
 472           SmallString<15> Code;
 473           raw_svector_ostream VecOS(Code);
 474           BC.MAB->writeNopData(VecOS, *Size, BC.STI.get());
 475           Streamer.emitBytes(Code);
 476           continue;
 477         }
 478       }
 479
 480       Streamer.emitInstruction(Instr, *BC.STI);
 481     }
 482   }
 483
 484   if (!EmitCodeOnly)
 485     emitConstantIslands(BF, FF.isSplitFragment());
 486 }
 487
 488 void BinaryEmitter::emitConstantIslands(BinaryFunction &BF, bool EmitColdPart,
 489                                         BinaryFunction *OnBehalfOf) {
 490   if (!BF.hasIslandsInfo())
 491     return;
 492
 493   BinaryFunction::IslandInfo &Islands = BF.getIslandInfo();
 494   if (Islands.DataOffsets.empty() && Islands.Dependency.empty())
 495     return;
 496
 497   // AArch64 requires CI to be aligned to 8 bytes due to access instructions
 498   // restrictions. E.g. the ldr with imm, where imm must be aligned to 8 bytes.
 499   const uint16_t Alignment = OnBehalfOf
 500                                  ? OnBehalfOf->getConstantIslandAlignment()
 501                                  : BF.getConstantIslandAlignment();
 502   Streamer.emitCodeAlignment(Align(Alignment), &*BC.STI);
 503
 504   if (!OnBehalfOf) {
 505     if (!EmitColdPart)
 506       Streamer.emitLabel(BF.getFunctionConstantIslandLabel());
 507     else
 508       Streamer.emitLabel(BF.getFunctionColdConstantIslandLabel());
 509   }
 510
 511   assert((!OnBehalfOf || Islands.Proxies[OnBehalfOf].size() > 0) &&
 512          "spurious OnBehalfOf constant island emission");
 513
 514   assert(!BF.isInjected() &&
 515          "injected functions should not have constant islands");
 516   // Raw contents of the function.
 517   StringRef SectionContents = BF.getOriginSection()->getContents();
 518
 519   // Raw contents of the function.
 520   StringRef FunctionContents = SectionContents.substr(
 521       BF.getAddress() - BF.getOriginSection()->getAddress(), BF.getMaxSize());
 522
 523   if (opts::Verbosity && !OnBehalfOf)
 524     BC.outs() << "BOLT-INFO: emitting constant island for function " << BF
 525               << "\n";
 526
 527   // We split the island into smaller blocks and output labels between them.
 528   auto IS = Islands.Offsets.begin();
 529   for (auto DataIter = Islands.DataOffsets.begin();
 530        DataIter != Islands.DataOffsets.end(); ++DataIter) {
 531     uint64_t FunctionOffset = *DataIter;
 532     uint64_t EndOffset = 0ULL;
 533
 534     // Determine size of this data chunk
 535     auto NextData = std::next(DataIter);
 536     auto CodeIter = Islands.CodeOffsets.lower_bound(*DataIter);
 537     if (CodeIter == Islands.CodeOffsets.end() &&
 538         NextData == Islands.DataOffsets.end())
 539       EndOffset = BF.getMaxSize();
 540     else if (CodeIter == Islands.CodeOffsets.end())
 541       EndOffset = *NextData;
 542     else if (NextData == Islands.DataOffsets.end())
 543       EndOffset = *CodeIter;
 544     else
 545       EndOffset = (*CodeIter > *NextData) ? *NextData : *CodeIter;
 546
 547     if (FunctionOffset == EndOffset)
 548       continue; // Size is zero, nothing to emit
 549
 550     auto emitCI = [&](uint64_t &FunctionOffset, uint64_t EndOffset) {
 551       if (FunctionOffset >= EndOffset)
 552         return;
 553
 554       for (auto It = Islands.Relocations.lower_bound(FunctionOffset);
 555            It != Islands.Relocations.end(); ++It) {
 556         if (It->first >= EndOffset)
 557           break;
 558
 559         const Relocation &Relocation = It->second;
 560         if (FunctionOffset < Relocation.Offset) {
 561           Streamer.emitBytes(
 562               FunctionContents.slice(FunctionOffset, Relocation.Offset));
 563           FunctionOffset = Relocation.Offset;
 564         }
 565
 566         LLVM_DEBUG(
 567             dbgs() << "BOLT-DEBUG: emitting constant island relocation"
 568                    << " for " << BF << " at offset 0x"
 569                    << Twine::utohexstr(Relocation.Offset) << " with size "
 570                    << Relocation::getSizeForType(Relocation.Type) << '\n');
 571
 572         FunctionOffset += Relocation.emit(&Streamer);
 573       }
 574
 575       assert(FunctionOffset <= EndOffset && "overflow error");
 576       if (FunctionOffset < EndOffset) {
 577         Streamer.emitBytes(FunctionContents.slice(FunctionOffset, EndOffset));
 578         FunctionOffset = EndOffset;
 579       }
 580     };
 581
 582     // Emit labels, relocs and data
 583     while (IS != Islands.Offsets.end() && IS->first < EndOffset) {
 584       auto NextLabelOffset =
 585           IS == Islands.Offsets.end() ? EndOffset : IS->first;
 586       auto NextStop = std::min(NextLabelOffset, EndOffset);
 587       assert(NextStop <= EndOffset && "internal overflow error");
 588       emitCI(FunctionOffset, NextStop);
 589       if (IS != Islands.Offsets.end() && FunctionOffset == IS->first) {
 590         // This is a slightly complex code to decide which label to emit. We
 591         // have 4 cases to handle: regular symbol, cold symbol, regular or cold
 592         // symbol being emitted on behalf of an external function.
 593         if (!OnBehalfOf) {
 594           if (!EmitColdPart) {
 595             LLVM_DEBUG(dbgs() << "BOLT-DEBUG: emitted label "
 596                               << IS->second->getName() << " at offset 0x"
 597                               << Twine::utohexstr(IS->first) << '\n');
 598             if (IS->second->isUndefined())
 599               Streamer.emitLabel(IS->second);
 600             else
 601               assert(BF.hasName(std::string(IS->second->getName())));
 602           } else if (Islands.ColdSymbols.count(IS->second) != 0) {
 603             LLVM_DEBUG(dbgs()
 604                        << "BOLT-DEBUG: emitted label "
 605                        << Islands.ColdSymbols[IS->second]->getName() << '\n');
 606             if (Islands.ColdSymbols[IS->second]->isUndefined())
 607               Streamer.emitLabel(Islands.ColdSymbols[IS->second]);
 608           }
 609         } else {
 610           if (!EmitColdPart) {
 611             if (MCSymbol *Sym = Islands.Proxies[OnBehalfOf][IS->second]) {
 612               LLVM_DEBUG(dbgs() << "BOLT-DEBUG: emitted label "
 613                                 << Sym->getName() << '\n');
 614               Streamer.emitLabel(Sym);
 615             }
 616           } else if (MCSymbol *Sym =
 617                          Islands.ColdProxies[OnBehalfOf][IS->second]) {
 618             LLVM_DEBUG(dbgs() << "BOLT-DEBUG: emitted label " << Sym->getName()
 619                               << '\n');
 620             Streamer.emitLabel(Sym);
 621           }
 622         }
 623         ++IS;
 624       }
 625     }
 626     assert(FunctionOffset <= EndOffset && "overflow error");
 627     emitCI(FunctionOffset, EndOffset);
 628   }
 629   assert(IS == Islands.Offsets.end() && "some symbols were not emitted!");
 630
 631   if (OnBehalfOf)
 632     return;
 633   // Now emit constant islands from other functions that we may have used in
 634   // this function.
 635   for (BinaryFunction *ExternalFunc : Islands.Dependency)
 636     emitConstantIslands(*ExternalFunc, EmitColdPart, &BF);
 637 }
 638
 639 SMLoc BinaryEmitter::emitLineInfo(const BinaryFunction &BF, SMLoc NewLoc,
 640                                   SMLoc PrevLoc, bool FirstInstr,
 641                                   MCSymbol *&InstrLabel) {
 642   DWARFUnit *FunctionCU = BF.getDWARFUnit();
 643   const DWARFDebugLine::LineTable *FunctionLineTable = BF.getDWARFLineTable();
 644   assert(FunctionCU && "cannot emit line info for function without CU");
 645
 646   DebugLineTableRowRef RowReference = DebugLineTableRowRef::fromSMLoc(NewLoc);
 647
 648   // Check if no new line info needs to be emitted.
 649   if (RowReference == DebugLineTableRowRef::NULL_ROW ||
 650       NewLoc.getPointer() == PrevLoc.getPointer())
 651     return PrevLoc;
 652
 653   unsigned CurrentFilenum = 0;
 654   const DWARFDebugLine::LineTable *CurrentLineTable = FunctionLineTable;
 655
 656   // If the CU id from the current instruction location does not
 657   // match the CU id from the current function, it means that we
 658   // have come across some inlined code.  We must look up the CU
 659   // for the instruction's original function and get the line table
 660   // from that.
 661   const uint64_t FunctionUnitIndex = FunctionCU->getOffset();
 662   const uint32_t CurrentUnitIndex = RowReference.DwCompileUnitIndex;
 663   if (CurrentUnitIndex != FunctionUnitIndex) {
 664     CurrentLineTable = BC.DwCtx->getLineTableForUnit(
 665         BC.DwCtx->getCompileUnitForOffset(CurrentUnitIndex));
 666     // Add filename from the inlined function to the current CU.
 667     CurrentFilenum = BC.addDebugFilenameToUnit(
 668         FunctionUnitIndex, CurrentUnitIndex,
 669         CurrentLineTable->Rows[RowReference.RowIndex - 1].File);
 670   }
 671
 672   const DWARFDebugLine::Row &CurrentRow =
 673       CurrentLineTable->Rows[RowReference.RowIndex - 1];
 674   if (!CurrentFilenum)
 675     CurrentFilenum = CurrentRow.File;
 676
 677   unsigned Flags = (DWARF2_FLAG_IS_STMT * CurrentRow.IsStmt) |
 678                    (DWARF2_FLAG_BASIC_BLOCK * CurrentRow.BasicBlock) |
 679                    (DWARF2_FLAG_PROLOGUE_END * CurrentRow.PrologueEnd) |
 680                    (DWARF2_FLAG_EPILOGUE_BEGIN * CurrentRow.EpilogueBegin);
 681
 682   // Always emit is_stmt at the beginning of function fragment.
 683   if (FirstInstr)
 684     Flags |= DWARF2_FLAG_IS_STMT;
 685
 686   BC.Ctx->setCurrentDwarfLoc(CurrentFilenum, CurrentRow.Line, CurrentRow.Column,
 687                              Flags, CurrentRow.Isa, CurrentRow.Discriminator);
 688   const MCDwarfLoc &DwarfLoc = BC.Ctx->getCurrentDwarfLoc();
 689   BC.Ctx->clearDwarfLocSeen();
 690
 691   if (!InstrLabel)
 692     InstrLabel = BC.Ctx->createTempSymbol();
 693
 694   BC.getDwarfLineTable(FunctionUnitIndex)
 695       .getMCLineSections()
 696       .addLineEntry(MCDwarfLineEntry(InstrLabel, DwarfLoc),
 697                     Streamer.getCurrentSectionOnly());
 698
 699   return NewLoc;
 700 }
 701
 702 void BinaryEmitter::emitLineInfoEnd(const BinaryFunction &BF,
 703                                     MCSymbol *FunctionEndLabel) {
 704   DWARFUnit *FunctionCU = BF.getDWARFUnit();
 705   assert(FunctionCU && "DWARF unit expected");
 706   BC.Ctx->setCurrentDwarfLoc(0, 0, 0, DWARF2_FLAG_END_SEQUENCE, 0, 0);
 707   const MCDwarfLoc &DwarfLoc = BC.Ctx->getCurrentDwarfLoc();
 708   BC.Ctx->clearDwarfLocSeen();
 709   BC.getDwarfLineTable(FunctionCU->getOffset())
 710       .getMCLineSections()
 711       .addLineEntry(MCDwarfLineEntry(FunctionEndLabel, DwarfLoc),
 712                     Streamer.getCurrentSectionOnly());
 713 }
 714
 715 void BinaryEmitter::emitJumpTables(const BinaryFunction &BF) {
 716   MCSection *ReadOnlySection = BC.MOFI->getReadOnlySection();
 717   MCSection *ReadOnlyColdSection = BC.MOFI->getContext().getELFSection(
 718       ".rodata.cold", ELF::SHT_PROGBITS, ELF::SHF_ALLOC);
 719
 720   if (!BF.hasJumpTables())
 721     return;
 722
 723   if (opts::PrintJumpTables)
 724     BC.outs() << "BOLT-INFO: jump tables for function " << BF << ":\n";
 725
 726   for (auto &JTI : BF.jumpTables()) {
 727     JumpTable &JT = *JTI.second;
 728     // Only emit shared jump tables once, when processing the first parent
 729     if (JT.Parents.size() > 1 && JT.Parents[0] != &BF)
 730       continue;
 731     if (opts::PrintJumpTables)
 732       JT.print(BC.outs());
 733     if (opts::JumpTables == JTS_BASIC && BC.HasRelocations) {
 734       JT.updateOriginal();
 735     } else {
 736       MCSection *HotSection, *ColdSection;
 737       if (opts::JumpTables == JTS_BASIC) {
 738         // In non-relocation mode we have to emit jump tables in local sections.
 739         // This way we only overwrite them when the corresponding function is
 740         // overwritten.
 741         std::string Name = ".local." + JT.Labels[0]->getName().str();
 742         std::replace(Name.begin(), Name.end(), '/', '.');
 743         BinarySection &Section =
 744             BC.registerOrUpdateSection(Name, ELF::SHT_PROGBITS, ELF::SHF_ALLOC);
 745         Section.setAnonymous(true);
 746         JT.setOutputSection(Section);
 747         HotSection = BC.getDataSection(Name);
 748         ColdSection = HotSection;
 749       } else {
 750         if (BF.isSimple()) {
 751           HotSection = ReadOnlySection;
 752           ColdSection = ReadOnlyColdSection;
 753         } else {
 754           HotSection = BF.hasProfile() ? ReadOnlySection : ReadOnlyColdSection;
 755           ColdSection = HotSection;
 756         }
 757       }
 758       emitJumpTable(JT, HotSection, ColdSection);
 759     }
 760   }
 761 }
 762
 763 void BinaryEmitter::emitJumpTable(const JumpTable &JT, MCSection *HotSection,
 764                                   MCSection *ColdSection) {
 765   // Pre-process entries for aggressive splitting.
 766   // Each label represents a separate switch table and gets its own count
 767   // determining its destination.
 768   std::map<MCSymbol *, uint64_t> LabelCounts;
 769   if (opts::JumpTables > JTS_SPLIT && !JT.Counts.empty()) {
 770     auto It = JT.Labels.find(0);
 771     assert(It != JT.Labels.end());
 772     MCSymbol *CurrentLabel = It->second;
 773     uint64_t CurrentLabelCount = 0;
 774     for (unsigned Index = 0; Index < JT.Entries.size(); ++Index) {
 775       auto LI = JT.Labels.find(Index * JT.EntrySize);
 776       if (LI != JT.Labels.end()) {
 777         LabelCounts[CurrentLabel] = CurrentLabelCount;
 778         CurrentLabel = LI->second;
 779         CurrentLabelCount = 0;
 780       }
 781       CurrentLabelCount += JT.Counts[Index].Count;
 782     }
 783     LabelCounts[CurrentLabel] = CurrentLabelCount;
 784   } else {
 785     Streamer.switchSection(JT.Count > 0 ? HotSection : ColdSection);
 786     Streamer.emitValueToAlignment(Align(JT.EntrySize));
 787   }
 788   MCSymbol *LastLabel = nullptr;
 789   uint64_t Offset = 0;
 790   for (MCSymbol *Entry : JT.Entries) {
 791     auto LI = JT.Labels.find(Offset);
 792     if (LI != JT.Labels.end()) {
 793       LLVM_DEBUG({
 794         dbgs() << "BOLT-DEBUG: emitting jump table " << LI->second->getName()
 795                << " (originally was at address 0x"
 796                << Twine::utohexstr(JT.getAddress() + Offset)
 797                << (Offset ? ") as part of larger jump table\n" : ")\n");
 798       });
 799       if (!LabelCounts.empty()) {
 800         LLVM_DEBUG(dbgs() << "BOLT-DEBUG: jump table count: "
 801                           << LabelCounts[LI->second] << '\n');
 802         if (LabelCounts[LI->second] > 0)
 803           Streamer.switchSection(HotSection);
 804         else
 805           Streamer.switchSection(ColdSection);
 806         Streamer.emitValueToAlignment(Align(JT.EntrySize));
 807       }
 808       // Emit all labels registered at the address of this jump table
 809       // to sync with our global symbol table.  We may have two labels
 810       // registered at this address if one label was created via
 811       // getOrCreateGlobalSymbol() (e.g. LEA instructions referencing
 812       // this location) and another via getOrCreateJumpTable().  This
 813       // creates a race where the symbols created by these two
 814       // functions may or may not be the same, but they are both
 815       // registered in our symbol table at the same address. By
 816       // emitting them all here we make sure there is no ambiguity
 817       // that depends on the order that these symbols were created, so
 818       // whenever this address is referenced in the binary, it is
 819       // certain to point to the jump table identified at this
 820       // address.
 821       if (BinaryData *BD = BC.getBinaryDataByName(LI->second->getName())) {
 822         for (MCSymbol *S : BD->getSymbols())
 823           Streamer.emitLabel(S);
 824       } else {
 825         Streamer.emitLabel(LI->second);
 826       }
 827       LastLabel = LI->second;
 828     }
 829     if (JT.Type == JumpTable::JTT_NORMAL) {
 830       Streamer.emitSymbolValue(Entry, JT.OutputEntrySize);
 831     } else { // JTT_PIC
 832       const MCSymbolRefExpr *JTExpr =
 833           MCSymbolRefExpr::create(LastLabel, Streamer.getContext());
 834       const MCSymbolRefExpr *E =
 835           MCSymbolRefExpr::create(Entry, Streamer.getContext());
 836       const MCBinaryExpr *Value =
 837           MCBinaryExpr::createSub(E, JTExpr, Streamer.getContext());
 838       Streamer.emitValue(Value, JT.EntrySize);
 839     }
 840     Offset += JT.EntrySize;
 841   }
 842 }
 843
 844 void BinaryEmitter::emitCFIInstruction(const MCCFIInstruction &Inst) const {
 845   switch (Inst.getOperation()) {
 846   default:
 847     llvm_unreachable("Unexpected instruction");
 848   case MCCFIInstruction::OpDefCfaOffset:
 849     Streamer.emitCFIDefCfaOffset(Inst.getOffset());
 850     break;
 851   case MCCFIInstruction::OpAdjustCfaOffset:
 852     Streamer.emitCFIAdjustCfaOffset(Inst.getOffset());
 853     break;
 854   case MCCFIInstruction::OpDefCfa:
 855     Streamer.emitCFIDefCfa(Inst.getRegister(), Inst.getOffset());
 856     break;
 857   case MCCFIInstruction::OpDefCfaRegister:
 858     Streamer.emitCFIDefCfaRegister(Inst.getRegister());
 859     break;
 860   case MCCFIInstruction::OpOffset:
 861     Streamer.emitCFIOffset(Inst.getRegister(), Inst.getOffset());
 862     break;
 863   case MCCFIInstruction::OpRegister:
 864     Streamer.emitCFIRegister(Inst.getRegister(), Inst.getRegister2());
 865     break;
 866   case MCCFIInstruction::OpWindowSave:
 867     Streamer.emitCFIWindowSave();
 868     break;
 869   case MCCFIInstruction::OpNegateRAState:
 870     Streamer.emitCFINegateRAState();
 871     break;
 872   case MCCFIInstruction::OpSameValue:
 873     Streamer.emitCFISameValue(Inst.getRegister());
 874     break;
 875   case MCCFIInstruction::OpGnuArgsSize:
 876     Streamer.emitCFIGnuArgsSize(Inst.getOffset());
 877     break;
 878   case MCCFIInstruction::OpEscape:
 879     Streamer.AddComment(Inst.getComment());
 880     Streamer.emitCFIEscape(Inst.getValues());
 881     break;
 882   case MCCFIInstruction::OpRestore:
 883     Streamer.emitCFIRestore(Inst.getRegister());
 884     break;
 885   case MCCFIInstruction::OpUndefined:
 886     Streamer.emitCFIUndefined(Inst.getRegister());
 887     break;
 888   }
 889 }
 890
 891 // The code is based on EHStreamer::emitExceptionTable().
 892 void BinaryEmitter::emitLSDA(BinaryFunction &BF, const FunctionFragment &FF) {
 893   const BinaryFunction::CallSitesRange Sites =
 894       BF.getCallSites(FF.getFragmentNum());
 895   if (Sites.empty())
 896     return;
 897
 898   Streamer.switchSection(BC.MOFI->getLSDASection());
 899
 900   const unsigned TTypeEncoding = BF.getLSDATypeEncoding();
 901   const unsigned TTypeEncodingSize = BC.getDWARFEncodingSize(TTypeEncoding);
 902   const uint16_t TTypeAlignment = 4;
 903
 904   // Type tables have to be aligned at 4 bytes.
 905   Streamer.emitValueToAlignment(Align(TTypeAlignment));
 906
 907   // Emit the LSDA label.
 908   MCSymbol *LSDASymbol = BF.getLSDASymbol(FF.getFragmentNum());
 909   assert(LSDASymbol && "no LSDA symbol set");
 910   Streamer.emitLabel(LSDASymbol);
 911
 912   // Corresponding FDE start.
 913   const MCSymbol *StartSymbol = BF.getSymbol(FF.getFragmentNum());
 914
 915   // Emit the LSDA header.
 916
 917   // If LPStart is omitted, then the start of the FDE is used as a base for
 918   // landing pad displacements. Then, if a cold fragment starts with a landing
 919   // pad, this means that the first landing pad offset will be 0. However, C++
 920   // runtime treats 0 as if there is no landing pad present, thus we *must* emit
 921   // non-zero offsets for all valid LPs.
 922   //
 923   // As a solution, for fixed-address binaries we set LPStart to 0, and for
 924   // position-independent binaries we set LP start to FDE start minus one byte
 925   // for FDEs that start with a landing pad.
 926   const bool NeedsLPAdjustment = !FF.empty() && FF.front()->isLandingPad();
 927   std::function<void(const MCSymbol *)> emitLandingPad;
 928   if (BC.HasFixedLoadAddress) {
 929     Streamer.emitIntValue(dwarf::DW_EH_PE_udata4, 1); // LPStart format
 930     Streamer.emitIntValue(0, 4);                      // LPStart
 931     emitLandingPad = [&](const MCSymbol *LPSymbol) {
 932       if (LPSymbol)
 933         Streamer.emitSymbolValue(LPSymbol, 4);
 934       else
 935         Streamer.emitIntValue(0, 4);
 936     };
 937   } else {
 938     if (NeedsLPAdjustment) {
 939       // Use relative LPStart format and emit LPStart as [SymbolStart - 1].
 940       Streamer.emitIntValue(dwarf::DW_EH_PE_pcrel | dwarf::DW_EH_PE_sdata4, 1);
 941       MCSymbol *DotSymbol = BC.Ctx->createTempSymbol("LPBase");
 942       Streamer.emitLabel(DotSymbol);
 943
 944       const MCExpr *LPStartExpr = MCBinaryExpr::createSub(
 945           MCSymbolRefExpr::create(StartSymbol, *BC.Ctx),
 946           MCSymbolRefExpr::create(DotSymbol, *BC.Ctx), *BC.Ctx);
 947       LPStartExpr = MCBinaryExpr::createSub(
 948           LPStartExpr, MCConstantExpr::create(1, *BC.Ctx), *BC.Ctx);
 949       Streamer.emitValue(LPStartExpr, 4);
 950     } else {
 951       // DW_EH_PE_omit means FDE start (StartSymbol) will be used as LPStart.
 952       Streamer.emitIntValue(dwarf::DW_EH_PE_omit, 1);
 953     }
 954     emitLandingPad = [&](const MCSymbol *LPSymbol) {
 955       if (LPSymbol) {
 956         const MCExpr *LPOffsetExpr = MCBinaryExpr::createSub(
 957             MCSymbolRefExpr::create(LPSymbol, *BC.Ctx),
 958             MCSymbolRefExpr::create(StartSymbol, *BC.Ctx), *BC.Ctx);
 959         if (NeedsLPAdjustment)
 960           LPOffsetExpr = MCBinaryExpr::createAdd(
 961               LPOffsetExpr, MCConstantExpr::create(1, *BC.Ctx), *BC.Ctx);
 962         Streamer.emitULEB128Value(LPOffsetExpr);
 963       } else {
 964         Streamer.emitULEB128IntValue(0);
 965       }
 966     };
 967   }
 968
 969   Streamer.emitIntValue(TTypeEncoding, 1); // TType format
 970
 971   MCSymbol *TTBaseLabel = nullptr;
 972   if (TTypeEncoding != dwarf::DW_EH_PE_omit) {
 973     TTBaseLabel = BC.Ctx->createTempSymbol("TTBase");
 974     MCSymbol *TTBaseRefLabel = BC.Ctx->createTempSymbol("TTBaseRef");
 975     Streamer.emitAbsoluteSymbolDiffAsULEB128(TTBaseLabel, TTBaseRefLabel);
 976     Streamer.emitLabel(TTBaseRefLabel);
 977   }
 978
 979   // Emit encoding of entries in the call site table. The format is used for the
 980   // call site start, length, and corresponding landing pad.
 981   if (BC.HasFixedLoadAddress)
 982     Streamer.emitIntValue(dwarf::DW_EH_PE_sdata4, 1);
 983   else
 984     Streamer.emitIntValue(dwarf::DW_EH_PE_uleb128, 1);
 985
 986   MCSymbol *CSTStartLabel = BC.Ctx->createTempSymbol("CSTStart");
 987   MCSymbol *CSTEndLabel = BC.Ctx->createTempSymbol("CSTEnd");
 988   Streamer.emitAbsoluteSymbolDiffAsULEB128(CSTEndLabel, CSTStartLabel);
 989
 990   Streamer.emitLabel(CSTStartLabel);
 991   for (const auto &FragmentCallSite : Sites) {
 992     const BinaryFunction::CallSite &CallSite = FragmentCallSite.second;
 993     const MCSymbol *BeginLabel = CallSite.Start;
 994     const MCSymbol *EndLabel = CallSite.End;
 995
 996     assert(BeginLabel && "start EH label expected");
 997     assert(EndLabel && "end EH label expected");
 998
 999     // Start of the range is emitted relative to the start of current
1000     // function split part.
1001     if (BC.HasFixedLoadAddress) {
1002       Streamer.emitAbsoluteSymbolDiff(BeginLabel, StartSymbol, 4);
1003       Streamer.emitAbsoluteSymbolDiff(EndLabel, BeginLabel, 4);
1004     } else {
1005       Streamer.emitAbsoluteSymbolDiffAsULEB128(BeginLabel, StartSymbol);
1006       Streamer.emitAbsoluteSymbolDiffAsULEB128(EndLabel, BeginLabel);
1007     }
1008     emitLandingPad(CallSite.LP);
1009     Streamer.emitULEB128IntValue(CallSite.Action);
1010   }
1011   Streamer.emitLabel(CSTEndLabel);
1012
1013   // Write out action, type, and type index tables at the end.
1014   //
1015   // For action and type index tables there's no need to change the original
1016   // table format unless we are doing function splitting, in which case we can
1017   // split and optimize the tables.
1018   //
1019   // For type table we (re-)encode the table using TTypeEncoding matching
1020   // the current assembler mode.
1021   for (uint8_t const &Byte : BF.getLSDAActionTable())
1022     Streamer.emitIntValue(Byte, 1);
1023
1024   const BinaryFunction::LSDATypeTableTy &TypeTable =
1025       (TTypeEncoding & dwarf::DW_EH_PE_indirect) ? BF.getLSDATypeAddressTable()
1026                                                  : BF.getLSDATypeTable();
1027   assert(TypeTable.size() == BF.getLSDATypeTable().size() &&
1028          "indirect type table size mismatch");
1029
1030   Streamer.emitValueToAlignment(Align(TTypeAlignment));
1031
1032   for (int Index = TypeTable.size() - 1; Index >= 0; --Index) {
1033     const uint64_t TypeAddress = TypeTable[Index];
1034     switch (TTypeEncoding & 0x70) {
1035     default:
1036       llvm_unreachable("unsupported TTypeEncoding");
1037     case dwarf::DW_EH_PE_absptr:
1038       Streamer.emitIntValue(TypeAddress, TTypeEncodingSize);
1039       break;
1040     case dwarf::DW_EH_PE_pcrel: {
1041       if (TypeAddress) {
1042         const MCSymbol *TypeSymbol =
1043             BC.getOrCreateGlobalSymbol(TypeAddress, "TI", 0, TTypeAlignment);
1044         MCSymbol *DotSymbol = BC.Ctx->createNamedTempSymbol();
1045         Streamer.emitLabel(DotSymbol);
1046         const MCBinaryExpr *SubDotExpr = MCBinaryExpr::createSub(
1047             MCSymbolRefExpr::create(TypeSymbol, *BC.Ctx),
1048             MCSymbolRefExpr::create(DotSymbol, *BC.Ctx), *BC.Ctx);
1049         Streamer.emitValue(SubDotExpr, TTypeEncodingSize);
1050       } else {
1051         Streamer.emitIntValue(0, TTypeEncodingSize);
1052       }
1053       break;
1054     }
1055     }
1056   }
1057
1058   if (TTypeEncoding != dwarf::DW_EH_PE_omit)
1059     Streamer.emitLabel(TTBaseLabel);
1060
1061   for (uint8_t const &Byte : BF.getLSDATypeIndexTable())
1062     Streamer.emitIntValue(Byte, 1);
1063 }
1064
1065 void BinaryEmitter::emitDebugLineInfoForOriginalFunctions() {
1066   // If a function is in a CU containing at least one processed function, we
1067   // have to rewrite the whole line table for that CU. For unprocessed functions
1068   // we use data from the input line table.
1069   for (auto &It : BC.getBinaryFunctions()) {
1070     const BinaryFunction &Function = It.second;
1071
1072     // If the function was emitted, its line info was emitted with it.
1073     if (Function.isEmitted())
1074       continue;
1075
1076     const DWARFDebugLine::LineTable *LineTable = Function.getDWARFLineTable();
1077     if (!LineTable)
1078       continue; // nothing to update for this function
1079
1080     const uint64_t Address = Function.getAddress();
1081     std::vector<uint32_t> Results;
1082     if (!LineTable->lookupAddressRange(
1083             {Address, object::SectionedAddress::UndefSection},
1084             Function.getSize(), Results))
1085       continue;
1086
1087     if (Results.empty())
1088       continue;
1089
1090     // The first row returned could be the last row matching the start address.
1091     // Find the first row with the same address that is not the end of the
1092     // sequence.
1093     uint64_t FirstRow = Results.front();
1094     while (FirstRow > 0) {
1095       const DWARFDebugLine::Row &PrevRow = LineTable->Rows[FirstRow - 1];
1096       if (PrevRow.Address.Address != Address || PrevRow.EndSequence)
1097         break;
1098       --FirstRow;
1099     }
1100
1101     const uint64_t EndOfSequenceAddress =
1102         Function.getAddress() + Function.getMaxSize();
1103     BC.getDwarfLineTable(Function.getDWARFUnit()->getOffset())
1104         .addLineTableSequence(LineTable, FirstRow, Results.back(),
1105                               EndOfSequenceAddress);
1106   }
1107
1108   // For units that are completely unprocessed, use original debug line contents
1109   // eliminating the need to regenerate line info program.
1110   emitDebugLineInfoForUnprocessedCUs();
1111 }
1112
1113 void BinaryEmitter::emitDebugLineInfoForUnprocessedCUs() {
1114   // Sorted list of section offsets provides boundaries for section fragments,
1115   // where each fragment is the unit's contribution to debug line section.
1116   std::vector<uint64_t> StmtListOffsets;
1117   StmtListOffsets.reserve(BC.DwCtx->getNumCompileUnits());
1118   for (const std::unique_ptr<DWARFUnit> &CU : BC.DwCtx->compile_units()) {
1119     DWARFDie CUDie = CU->getUnitDIE();
1120     auto StmtList = dwarf::toSectionOffset(CUDie.find(dwarf::DW_AT_stmt_list));
1121     if (!StmtList)
1122       continue;
1123
1124     StmtListOffsets.push_back(*StmtList);
1125   }
1126   llvm::sort(StmtListOffsets);
1127
1128   // For each CU that was not processed, emit its line info as a binary blob.
1129   for (const std::unique_ptr<DWARFUnit> &CU : BC.DwCtx->compile_units()) {
1130     if (BC.ProcessedCUs.count(CU.get()))
1131       continue;
1132
1133     DWARFDie CUDie = CU->getUnitDIE();
1134     auto StmtList = dwarf::toSectionOffset(CUDie.find(dwarf::DW_AT_stmt_list));
1135     if (!StmtList)
1136       continue;
1137
1138     StringRef DebugLineContents = CU->getLineSection().Data;
1139
1140     const uint64_t Begin = *StmtList;
1141
1142     // Statement list ends where the next unit contribution begins, or at the
1143     // end of the section.
1144     auto It = llvm::upper_bound(StmtListOffsets, Begin);
1145     const uint64_t End =
1146         It == StmtListOffsets.end() ? DebugLineContents.size() : *It;
1147
1148     BC.getDwarfLineTable(CU->getOffset())
1149         .addRawContents(DebugLineContents.slice(Begin, End));
1150   }
1151 }
1152
1153 void BinaryEmitter::emitDataSections(StringRef OrgSecPrefix) {
1154   for (BinarySection &Section : BC.sections()) {
1155     if (!Section.hasRelocations())
1156       continue;
1157
1158     StringRef Prefix = Section.hasSectionRef() ? OrgSecPrefix : "";
1159     Section.emitAsData(Streamer, Prefix + Section.getName());
1160     Section.clearRelocations();
1161   }
1162 }
1163
1164 namespace llvm {
1165 namespace bolt {
1166
1167 void emitBinaryContext(MCStreamer &Streamer, BinaryContext &BC,
1168                        StringRef OrgSecPrefix) {
1169   BinaryEmitter(Streamer, BC).emitAll(OrgSecPrefix);
1170 }
1171
1172 void emitFunctionBody(MCStreamer &Streamer, BinaryFunction &BF,
1173                       FunctionFragment &FF, bool EmitCodeOnly) {
1174   BinaryEmitter(Streamer, BF.getBinaryContext())
1175       .emitFunctionBody(BF, FF, EmitCodeOnly);
1176 }
1177
1178 } // namespace bolt
1179 } // namespace llvm