bolt/lib/Core/BinaryEmitter.cpp

   1 //===- bolt/Core/BinaryEmitter.cpp - Emit code and data -------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file implements the collection of functions and classes used for
  10 // emission of code and data into object/binary file.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "bolt/Core/BinaryEmitter.h"
  15 #include "bolt/Core/BinaryContext.h"
  16 #include "bolt/Core/BinaryFunction.h"
  17 #include "bolt/Core/DebugData.h"
  18 #include "bolt/Core/FunctionLayout.h"
  19 #include "bolt/Utils/CommandLineOpts.h"
  20 #include "bolt/Utils/Utils.h"
  21 #include "llvm/DebugInfo/DWARF/DWARFCompileUnit.h"
  22 #include "llvm/MC/MCSection.h"
  23 #include "llvm/MC/MCStreamer.h"
  24 #include "llvm/Support/CommandLine.h"
  25 #include "llvm/Support/LEB128.h"
  26 #include "llvm/Support/SMLoc.h"
  27
  28 #define DEBUG_TYPE "bolt"
  29
  30 using namespace llvm;
  31 using namespace bolt;
  32
  33 namespace opts {
  34
  35 extern cl::opt<JumpTableSupportLevel> JumpTables;
  36 extern cl::opt<bool> PreserveBlocksAlignment;
  37
  38 cl::opt<bool> AlignBlocks("align-blocks", cl::desc("align basic blocks"),
  39                           cl::cat(BoltOptCategory));
  40
  41 cl::opt<MacroFusionType>
  42 AlignMacroOpFusion("align-macro-fusion",
  43   cl::desc("fix instruction alignment for macro-fusion (x86 relocation mode)"),
  44   cl::init(MFT_HOT),
  45   cl::values(clEnumValN(MFT_NONE, "none",
  46                "do not insert alignment no-ops for macro-fusion"),
  47              clEnumValN(MFT_HOT, "hot",
  48                "only insert alignment no-ops on hot execution paths (default)"),
  49              clEnumValN(MFT_ALL, "all",
  50                "always align instructions to allow macro-fusion")),
  51   cl::ZeroOrMore,
  52   cl::cat(BoltRelocCategory));
  53
  54 static cl::list<std::string>
  55 BreakFunctionNames("break-funcs",
  56   cl::CommaSeparated,
  57   cl::desc("list of functions to core dump on (debugging)"),
  58   cl::value_desc("func1,func2,func3,..."),
  59   cl::Hidden,
  60   cl::cat(BoltCategory));
  61
  62 static cl::list<std::string>
  63 FunctionPadSpec("pad-funcs",
  64   cl::CommaSeparated,
  65   cl::desc("list of functions to pad with amount of bytes"),
  66   cl::value_desc("func1:pad1,func2:pad2,func3:pad3,..."),
  67   cl::Hidden,
  68   cl::cat(BoltCategory));
  69
  70 static cl::opt<bool> MarkFuncs(
  71     "mark-funcs",
  72     cl::desc("mark function boundaries with break instruction to make "
  73              "sure we accidentally don't cross them"),
  74     cl::ReallyHidden, cl::cat(BoltCategory));
  75
  76 static cl::opt<bool> PrintJumpTables("print-jump-tables",
  77                                      cl::desc("print jump tables"), cl::Hidden,
  78                                      cl::cat(BoltCategory));
  79
  80 static cl::opt<bool>
  81 X86AlignBranchBoundaryHotOnly("x86-align-branch-boundary-hot-only",
  82   cl::desc("only apply branch boundary alignment in hot code"),
  83   cl::init(true),
  84   cl::cat(BoltOptCategory));
  85
  86 size_t padFunction(const BinaryFunction &Function) {
  87   static std::map<std::string, size_t> FunctionPadding;
  88
  89   if (FunctionPadding.empty() && !FunctionPadSpec.empty()) {
  90     for (std::string &Spec : FunctionPadSpec) {
  91       size_t N = Spec.find(':');
  92       if (N == std::string::npos)
  93         continue;
  94       std::string Name = Spec.substr(0, N);
  95       size_t Padding = std::stoull(Spec.substr(N + 1));
  96       FunctionPadding[Name] = Padding;
  97     }
  98   }
  99
 100   for (auto &FPI : FunctionPadding) {
 101     std::string Name = FPI.first;
 102     size_t Padding = FPI.second;
 103     if (Function.hasNameRegex(Name))
 104       return Padding;
 105   }
 106
 107   return 0;
 108 }
 109
 110 } // namespace opts
 111
 112 namespace {
 113 using JumpTable = bolt::JumpTable;
 114
 115 class BinaryEmitter {
 116 private:
 117   BinaryEmitter(const BinaryEmitter &) = delete;
 118   BinaryEmitter &operator=(const BinaryEmitter &) = delete;
 119
 120   MCStreamer &Streamer;
 121   BinaryContext &BC;
 122
 123 public:
 124   BinaryEmitter(MCStreamer &Streamer, BinaryContext &BC)
 125       : Streamer(Streamer), BC(BC) {}
 126
 127   /// Emit all code and data.
 128   void emitAll(StringRef OrgSecPrefix);
 129
 130   /// Emit function code. The caller is responsible for emitting function
 131   /// symbol(s) and setting the section to emit the code to.
 132   void emitFunctionBody(BinaryFunction &BF, FunctionFragment &FF,
 133                         bool EmitCodeOnly = false);
 134
 135 private:
 136   /// Emit function code.
 137   void emitFunctions();
 138
 139   /// Emit a single function.
 140   bool emitFunction(BinaryFunction &BF, FunctionFragment &FF);
 141
 142   /// Helper for emitFunctionBody to write data inside a function
 143   /// (used for AArch64)
 144   void emitConstantIslands(BinaryFunction &BF, bool EmitColdPart,
 145                            BinaryFunction *OnBehalfOf = nullptr);
 146
 147   /// Emit jump tables for the function.
 148   void emitJumpTables(const BinaryFunction &BF);
 149
 150   /// Emit jump table data. Callee supplies sections for the data.
 151   void emitJumpTable(const JumpTable &JT, MCSection *HotSection,
 152                      MCSection *ColdSection);
 153
 154   void emitCFIInstruction(const MCCFIInstruction &Inst) const;
 155
 156   /// Emit exception handling ranges for the function.
 157   void emitLSDA(BinaryFunction &BF, const FunctionFragment &FF);
 158
 159   /// Emit line number information corresponding to \p NewLoc. \p PrevLoc
 160   /// provides a context for de-duplication of line number info.
 161   /// \p FirstInstr indicates if \p NewLoc represents the first instruction
 162   /// in a sequence, such as a function fragment.
 163   ///
 164   /// If \p NewLoc location matches \p PrevLoc, no new line number entry will be
 165   /// created and the function will return \p PrevLoc while \p InstrLabel will
 166   /// be ignored. Otherwise, the caller should use \p InstrLabel to mark the
 167   /// corresponding instruction by emitting \p InstrLabel before it.
 168   /// If \p InstrLabel is set by the caller, its value will be used with \p
 169   /// \p NewLoc. If it was nullptr on entry, it will be populated with a pointer
 170   /// to a new temp symbol used with \p NewLoc.
 171   ///
 172   /// Return new current location which is either \p NewLoc or \p PrevLoc.
 173   SMLoc emitLineInfo(const BinaryFunction &BF, SMLoc NewLoc, SMLoc PrevLoc,
 174                      bool FirstInstr, MCSymbol *&InstrLabel);
 175
 176   /// Use \p FunctionEndSymbol to mark the end of the line info sequence.
 177   /// Note that it does not automatically result in the insertion of the EOS
 178   /// marker in the line table program, but provides one to the DWARF generator
 179   /// when it needs it.
 180   void emitLineInfoEnd(const BinaryFunction &BF, MCSymbol *FunctionEndSymbol);
 181
 182   /// Emit debug line info for unprocessed functions from CUs that include
 183   /// emitted functions.
 184   void emitDebugLineInfoForOriginalFunctions();
 185
 186   /// Emit debug line for CUs that were not modified.
 187   void emitDebugLineInfoForUnprocessedCUs();
 188
 189   /// Emit data sections that have code references in them.
 190   void emitDataSections(StringRef OrgSecPrefix);
 191 };
 192
 193 } // anonymous namespace
 194
 195 void BinaryEmitter::emitAll(StringRef OrgSecPrefix) {
 196   Streamer.initSections(false, *BC.STI);
 197
 198   if (opts::UpdateDebugSections && BC.isELF()) {
 199     // Force the emission of debug line info into allocatable section to ensure
 200     // JITLink will process it.
 201     //
 202     // NB: on MachO all sections are required for execution, hence no need
 203     //     to change flags/attributes.
 204     MCSectionELF *ELFDwarfLineSection =
 205         static_cast<MCSectionELF *>(BC.MOFI->getDwarfLineSection());
 206     ELFDwarfLineSection->setFlags(ELF::SHF_ALLOC);
 207     MCSectionELF *ELFDwarfLineStrSection =
 208         static_cast<MCSectionELF *>(BC.MOFI->getDwarfLineStrSection());
 209     ELFDwarfLineStrSection->setFlags(ELF::SHF_ALLOC);
 210   }
 211
 212   if (RuntimeLibrary *RtLibrary = BC.getRuntimeLibrary())
 213     RtLibrary->emitBinary(BC, Streamer);
 214
 215   BC.getTextSection()->setAlignment(Align(opts::AlignText));
 216
 217   emitFunctions();
 218
 219   if (opts::UpdateDebugSections) {
 220     emitDebugLineInfoForOriginalFunctions();
 221     DwarfLineTable::emit(BC, Streamer);
 222   }
 223
 224   emitDataSections(OrgSecPrefix);
 225
 226   // TODO Enable for Mach-O once BinaryContext::getDataSection supports it.
 227   if (BC.isELF())
 228     AddressMap::emit(Streamer, BC);
 229 }
 230
 231 void BinaryEmitter::emitFunctions() {
 232   auto emit = [&](const std::vector<BinaryFunction *> &Functions) {
 233     const bool HasProfile = BC.NumProfiledFuncs > 0;
 234     const bool OriginalAllowAutoPadding = Streamer.getAllowAutoPadding();
 235     for (BinaryFunction *Function : Functions) {
 236       if (!BC.shouldEmit(*Function))
 237         continue;
 238
 239       LLVM_DEBUG(dbgs() << "BOLT: generating code for function \"" << *Function
 240                         << "\" : " << Function->getFunctionNumber() << '\n');
 241
 242       // Was any part of the function emitted.
 243       bool Emitted = false;
 244
 245       // Turn off Intel JCC Erratum mitigation for cold code if requested
 246       if (HasProfile && opts::X86AlignBranchBoundaryHotOnly &&
 247           !Function->hasValidProfile())
 248         Streamer.setAllowAutoPadding(false);
 249
 250       FunctionLayout &Layout = Function->getLayout();
 251       Emitted |= emitFunction(*Function, Layout.getMainFragment());
 252
 253       if (Function->isSplit()) {
 254         if (opts::X86AlignBranchBoundaryHotOnly)
 255           Streamer.setAllowAutoPadding(false);
 256
 257         assert((Layout.fragment_size() == 1 || Function->isSimple()) &&
 258                "Only simple functions can have fragments");
 259         for (FunctionFragment &FF : Layout.getSplitFragments()) {
 260           // Skip empty fragments so no symbols and sections for empty fragments
 261           // are generated
 262           if (FF.empty() && !Function->hasConstantIsland())
 263             continue;
 264           Emitted |= emitFunction(*Function, FF);
 265         }
 266       }
 267
 268       Streamer.setAllowAutoPadding(OriginalAllowAutoPadding);
 269
 270       if (Emitted)
 271         Function->setEmitted(/*KeepCFG=*/opts::PrintCacheMetrics);
 272     }
 273   };
 274
 275   // Mark the start of hot text.
 276   if (opts::HotText) {
 277     Streamer.switchSection(BC.getTextSection());
 278     Streamer.emitLabel(BC.getHotTextStartSymbol());
 279   }
 280
 281   // Emit functions in sorted order.
 282   std::vector<BinaryFunction *> SortedFunctions = BC.getSortedFunctions();
 283   emit(SortedFunctions);
 284
 285   // Emit functions added by BOLT.
 286   emit(BC.getInjectedBinaryFunctions());
 287
 288   // Mark the end of hot text.
 289   if (opts::HotText) {
 290     if (BC.HasWarmSection)
 291       Streamer.switchSection(BC.getCodeSection(BC.getWarmCodeSectionName()));
 292     else
 293       Streamer.switchSection(BC.getTextSection());
 294     Streamer.emitLabel(BC.getHotTextEndSymbol());
 295   }
 296 }
 297
 298 bool BinaryEmitter::emitFunction(BinaryFunction &Function,
 299                                  FunctionFragment &FF) {
 300   if (Function.size() == 0 && !Function.hasIslandsInfo())
 301     return false;
 302
 303   if (Function.getState() == BinaryFunction::State::Empty)
 304     return false;
 305
 306   // Avoid emitting function without instructions when overwriting the original
 307   // function in-place. Otherwise, emit the empty function to define the symbol.
 308   if (!BC.HasRelocations && !Function.hasNonPseudoInstructions())
 309     return false;
 310
 311   MCSection *Section =
 312       BC.getCodeSection(Function.getCodeSectionName(FF.getFragmentNum()));
 313   Streamer.switchSection(Section);
 314   Section->setHasInstructions(true);
 315   BC.Ctx->addGenDwarfSection(Section);
 316
 317   if (BC.HasRelocations) {
 318     // Set section alignment to at least maximum possible object alignment.
 319     // We need this to support LongJmp and other passes that calculates
 320     // tentative layout.
 321     Section->ensureMinAlignment(Align(opts::AlignFunctions));
 322
 323     Streamer.emitCodeAlignment(Function.getMinAlign(), &*BC.STI);
 324     uint16_t MaxAlignBytes = FF.isSplitFragment()
 325                                  ? Function.getMaxColdAlignmentBytes()
 326                                  : Function.getMaxAlignmentBytes();
 327     if (MaxAlignBytes > 0)
 328       Streamer.emitCodeAlignment(Function.getAlign(), &*BC.STI, MaxAlignBytes);
 329   } else {
 330     Streamer.emitCodeAlignment(Function.getAlign(), &*BC.STI);
 331   }
 332
 333   MCContext &Context = Streamer.getContext();
 334   const MCAsmInfo *MAI = Context.getAsmInfo();
 335
 336   MCSymbol *const StartSymbol = Function.getSymbol(FF.getFragmentNum());
 337
 338   // Emit all symbols associated with the main function entry.
 339   if (FF.isMainFragment()) {
 340     for (MCSymbol *Symbol : Function.getSymbols()) {
 341       Streamer.emitSymbolAttribute(Symbol, MCSA_ELF_TypeFunction);
 342       Streamer.emitLabel(Symbol);
 343     }
 344   } else {
 345     Streamer.emitSymbolAttribute(StartSymbol, MCSA_ELF_TypeFunction);
 346     Streamer.emitLabel(StartSymbol);
 347   }
 348
 349   // Emit CFI start
 350   if (Function.hasCFI()) {
 351     Streamer.emitCFIStartProc(/*IsSimple=*/false);
 352     if (Function.getPersonalityFunction() != nullptr)
 353       Streamer.emitCFIPersonality(Function.getPersonalityFunction(),
 354                                   Function.getPersonalityEncoding());
 355     MCSymbol *LSDASymbol = Function.getLSDASymbol(FF.getFragmentNum());
 356     if (LSDASymbol)
 357       Streamer.emitCFILsda(LSDASymbol, BC.LSDAEncoding);
 358     else
 359       Streamer.emitCFILsda(0, dwarf::DW_EH_PE_omit);
 360     // Emit CFI instructions relative to the CIE
 361     for (const MCCFIInstruction &CFIInstr : Function.cie()) {
 362       // Only write CIE CFI insns that LLVM will not already emit
 363       const std::vector<MCCFIInstruction> &FrameInstrs =
 364           MAI->getInitialFrameState();
 365       if (!llvm::is_contained(FrameInstrs, CFIInstr))
 366         emitCFIInstruction(CFIInstr);
 367     }
 368   }
 369
 370   assert((Function.empty() || !(*Function.begin()).isCold()) &&
 371          "first basic block should never be cold");
 372
 373   // Emit UD2 at the beginning if requested by user.
 374   if (!opts::BreakFunctionNames.empty()) {
 375     for (std::string &Name : opts::BreakFunctionNames) {
 376       if (Function.hasNameRegex(Name)) {
 377         Streamer.emitIntValue(0x0B0F, 2); // UD2: 0F 0B
 378         break;
 379       }
 380     }
 381   }
 382
 383   // Emit code.
 384   emitFunctionBody(Function, FF, /*EmitCodeOnly=*/false);
 385
 386   // Emit padding if requested.
 387   if (size_t Padding = opts::padFunction(Function)) {
 388     LLVM_DEBUG(dbgs() << "BOLT-DEBUG: padding function " << Function << " with "
 389                       << Padding << " bytes\n");
 390     Streamer.emitFill(Padding, MAI->getTextAlignFillValue());
 391   }
 392
 393   if (opts::MarkFuncs)
 394     Streamer.emitBytes(BC.MIB->getTrapFillValue());
 395
 396   // Emit CFI end
 397   if (Function.hasCFI())
 398     Streamer.emitCFIEndProc();
 399
 400   MCSymbol *EndSymbol = Function.getFunctionEndLabel(FF.getFragmentNum());
 401   Streamer.emitLabel(EndSymbol);
 402
 403   if (MAI->hasDotTypeDotSizeDirective()) {
 404     const MCExpr *SizeExpr = MCBinaryExpr::createSub(
 405         MCSymbolRefExpr::create(EndSymbol, Context),
 406         MCSymbolRefExpr::create(StartSymbol, Context), Context);
 407     Streamer.emitELFSize(StartSymbol, SizeExpr);
 408   }
 409
 410   if (opts::UpdateDebugSections && Function.getDWARFUnit())
 411     emitLineInfoEnd(Function, EndSymbol);
 412
 413   // Exception handling info for the function.
 414   emitLSDA(Function, FF);
 415
 416   if (FF.isMainFragment() && opts::JumpTables > JTS_NONE)
 417     emitJumpTables(Function);
 418
 419   return true;
 420 }
 421
 422 void BinaryEmitter::emitFunctionBody(BinaryFunction &BF, FunctionFragment &FF,
 423                                      bool EmitCodeOnly) {
 424   if (!EmitCodeOnly && FF.isSplitFragment() && BF.hasConstantIsland()) {
 425     assert(BF.getLayout().isHotColdSplit() &&
 426            "Constant island support only with hot/cold split");
 427     BF.duplicateConstantIslands();
 428   }
 429
 430   if (!FF.empty() && FF.front()->isLandingPad()) {
 431     assert(!FF.front()->isEntryPoint() &&
 432            "Landing pad cannot be entry point of function");
 433     // If the first block of the fragment is a landing pad, it's offset from the
 434     // start of the area that the corresponding LSDA describes is zero. In this
 435     // case, the call site entries in that LSDA have 0 as offset to the landing
 436     // pad, which the runtime interprets as "no handler". To prevent this,
 437     // insert some padding.
 438     Streamer.emitBytes(BC.MIB->getTrapFillValue());
 439   }
 440
 441   // Track the first emitted instruction with debug info.
 442   bool FirstInstr = true;
 443   for (BinaryBasicBlock *const BB : FF) {
 444     if ((opts::AlignBlocks || opts::PreserveBlocksAlignment) &&
 445         BB->getAlignment() > 1)
 446       Streamer.emitCodeAlignment(BB->getAlign(), &*BC.STI,
 447                                  BB->getAlignmentMaxBytes());
 448     Streamer.emitLabel(BB->getLabel());
 449     if (!EmitCodeOnly) {
 450       if (MCSymbol *EntrySymbol = BF.getSecondaryEntryPointSymbol(*BB))
 451         Streamer.emitLabel(EntrySymbol);
 452     }
 453
 454     // Check if special alignment for macro-fusion is needed.
 455     bool MayNeedMacroFusionAlignment =
 456         (opts::AlignMacroOpFusion == MFT_ALL) ||
 457         (opts::AlignMacroOpFusion == MFT_HOT && BB->getKnownExecutionCount());
 458     BinaryBasicBlock::const_iterator MacroFusionPair;
 459     if (MayNeedMacroFusionAlignment) {
 460       MacroFusionPair = BB->getMacroOpFusionPair();
 461       if (MacroFusionPair == BB->end())
 462         MayNeedMacroFusionAlignment = false;
 463     }
 464
 465     SMLoc LastLocSeen;
 466     // Remember if the last instruction emitted was a prefix.
 467     bool LastIsPrefix = false;
 468     for (auto I = BB->begin(), E = BB->end(); I != E; ++I) {
 469       MCInst &Instr = *I;
 470
 471       if (EmitCodeOnly && BC.MIB->isPseudo(Instr))
 472         continue;
 473
 474       // Handle pseudo instructions.
 475       if (BC.MIB->isCFI(Instr)) {
 476         emitCFIInstruction(*BF.getCFIFor(Instr));
 477         continue;
 478       }
 479
 480       // Handle macro-fusion alignment. If we emitted a prefix as
 481       // the last instruction, we should've already emitted the associated
 482       // alignment hint, so don't emit it twice.
 483       if (MayNeedMacroFusionAlignment && !LastIsPrefix &&
 484           I == MacroFusionPair) {
 485         // This assumes the second instruction in the macro-op pair will get
 486         // assigned to its own MCRelaxableFragment. Since all JCC instructions
 487         // are relaxable, we should be safe.
 488       }
 489
 490       if (!EmitCodeOnly) {
 491         // A symbol to be emitted before the instruction to mark its location.
 492         MCSymbol *InstrLabel = BC.MIB->getLabel(Instr);
 493
 494         if (opts::UpdateDebugSections && BF.getDWARFUnit()) {
 495           LastLocSeen = emitLineInfo(BF, Instr.getLoc(), LastLocSeen,
 496                                      FirstInstr, InstrLabel);
 497           FirstInstr = false;
 498         }
 499
 500         // Prepare to tag this location with a label if we need to keep track of
 501         // the location of calls/returns for BOLT address translation maps
 502         if (BF.requiresAddressTranslation() && BC.MIB->getOffset(Instr)) {
 503           const uint32_t Offset = *BC.MIB->getOffset(Instr);
 504           if (!InstrLabel)
 505             InstrLabel = BC.Ctx->createTempSymbol();
 506           BB->getLocSyms().emplace_back(Offset, InstrLabel);
 507         }
 508
 509         if (InstrLabel)
 510           Streamer.emitLabel(InstrLabel);
 511       }
 512
 513       // Emit sized NOPs via MCAsmBackend::writeNopData() interface on x86.
 514       // This is a workaround for invalid NOPs handling by asm/disasm layer.
 515       if (BC.MIB->isNoop(Instr) && BC.isX86()) {
 516         if (std::optional<uint32_t> Size = BC.MIB->getSize(Instr)) {
 517           SmallString<15> Code;
 518           raw_svector_ostream VecOS(Code);
 519           BC.MAB->writeNopData(VecOS, *Size, BC.STI.get());
 520           Streamer.emitBytes(Code);
 521           continue;
 522         }
 523       }
 524
 525       Streamer.emitInstruction(Instr, *BC.STI);
 526       LastIsPrefix = BC.MIB->isPrefix(Instr);
 527     }
 528   }
 529
 530   if (!EmitCodeOnly)
 531     emitConstantIslands(BF, FF.isSplitFragment());
 532 }
 533
 534 void BinaryEmitter::emitConstantIslands(BinaryFunction &BF, bool EmitColdPart,
 535                                         BinaryFunction *OnBehalfOf) {
 536   if (!BF.hasIslandsInfo())
 537     return;
 538
 539   BinaryFunction::IslandInfo &Islands = BF.getIslandInfo();
 540   if (Islands.DataOffsets.empty() && Islands.Dependency.empty())
 541     return;
 542
 543   // AArch64 requires CI to be aligned to 8 bytes due to access instructions
 544   // restrictions. E.g. the ldr with imm, where imm must be aligned to 8 bytes.
 545   const uint16_t Alignment = OnBehalfOf
 546                                  ? OnBehalfOf->getConstantIslandAlignment()
 547                                  : BF.getConstantIslandAlignment();
 548   Streamer.emitCodeAlignment(Align(Alignment), &*BC.STI);
 549
 550   if (!OnBehalfOf) {
 551     if (!EmitColdPart)
 552       Streamer.emitLabel(BF.getFunctionConstantIslandLabel());
 553     else
 554       Streamer.emitLabel(BF.getFunctionColdConstantIslandLabel());
 555   }
 556
 557   assert((!OnBehalfOf || Islands.Proxies[OnBehalfOf].size() > 0) &&
 558          "spurious OnBehalfOf constant island emission");
 559
 560   assert(!BF.isInjected() &&
 561          "injected functions should not have constant islands");
 562   // Raw contents of the function.
 563   StringRef SectionContents = BF.getOriginSection()->getContents();
 564
 565   // Raw contents of the function.
 566   StringRef FunctionContents = SectionContents.substr(
 567       BF.getAddress() - BF.getOriginSection()->getAddress(), BF.getMaxSize());
 568
 569   if (opts::Verbosity && !OnBehalfOf)
 570     outs() << "BOLT-INFO: emitting constant island for function " << BF << "\n";
 571
 572   // We split the island into smaller blocks and output labels between them.
 573   auto IS = Islands.Offsets.begin();
 574   for (auto DataIter = Islands.DataOffsets.begin();
 575        DataIter != Islands.DataOffsets.end(); ++DataIter) {
 576     uint64_t FunctionOffset = *DataIter;
 577     uint64_t EndOffset = 0ULL;
 578
 579     // Determine size of this data chunk
 580     auto NextData = std::next(DataIter);
 581     auto CodeIter = Islands.CodeOffsets.lower_bound(*DataIter);
 582     if (CodeIter == Islands.CodeOffsets.end() &&
 583         NextData == Islands.DataOffsets.end())
 584       EndOffset = BF.getMaxSize();
 585     else if (CodeIter == Islands.CodeOffsets.end())
 586       EndOffset = *NextData;
 587     else if (NextData == Islands.DataOffsets.end())
 588       EndOffset = *CodeIter;
 589     else
 590       EndOffset = (*CodeIter > *NextData) ? *NextData : *CodeIter;
 591
 592     if (FunctionOffset == EndOffset)
 593       continue; // Size is zero, nothing to emit
 594
 595     auto emitCI = [&](uint64_t &FunctionOffset, uint64_t EndOffset) {
 596       if (FunctionOffset >= EndOffset)
 597         return;
 598
 599       for (auto It = Islands.Relocations.lower_bound(FunctionOffset);
 600            It != Islands.Relocations.end(); ++It) {
 601         if (It->first >= EndOffset)
 602           break;
 603
 604         const Relocation &Relocation = It->second;
 605         if (FunctionOffset < Relocation.Offset) {
 606           Streamer.emitBytes(
 607               FunctionContents.slice(FunctionOffset, Relocation.Offset));
 608           FunctionOffset = Relocation.Offset;
 609         }
 610
 611         LLVM_DEBUG(
 612             dbgs() << "BOLT-DEBUG: emitting constant island relocation"
 613                    << " for " << BF << " at offset 0x"
 614                    << Twine::utohexstr(Relocation.Offset) << " with size "
 615                    << Relocation::getSizeForType(Relocation.Type) << '\n');
 616
 617         FunctionOffset += Relocation.emit(&Streamer);
 618       }
 619
 620       assert(FunctionOffset <= EndOffset && "overflow error");
 621       if (FunctionOffset < EndOffset) {
 622         Streamer.emitBytes(FunctionContents.slice(FunctionOffset, EndOffset));
 623         FunctionOffset = EndOffset;
 624       }
 625     };
 626
 627     // Emit labels, relocs and data
 628     while (IS != Islands.Offsets.end() && IS->first < EndOffset) {
 629       auto NextLabelOffset =
 630           IS == Islands.Offsets.end() ? EndOffset : IS->first;
 631       auto NextStop = std::min(NextLabelOffset, EndOffset);
 632       assert(NextStop <= EndOffset && "internal overflow error");
 633       emitCI(FunctionOffset, NextStop);
 634       if (IS != Islands.Offsets.end() && FunctionOffset == IS->first) {
 635         // This is a slightly complex code to decide which label to emit. We
 636         // have 4 cases to handle: regular symbol, cold symbol, regular or cold
 637         // symbol being emitted on behalf of an external function.
 638         if (!OnBehalfOf) {
 639           if (!EmitColdPart) {
 640             LLVM_DEBUG(dbgs() << "BOLT-DEBUG: emitted label "
 641                               << IS->second->getName() << " at offset 0x"
 642                               << Twine::utohexstr(IS->first) << '\n');
 643             if (IS->second->isUndefined())
 644               Streamer.emitLabel(IS->second);
 645             else
 646               assert(BF.hasName(std::string(IS->second->getName())));
 647           } else if (Islands.ColdSymbols.count(IS->second) != 0) {
 648             LLVM_DEBUG(dbgs()
 649                        << "BOLT-DEBUG: emitted label "
 650                        << Islands.ColdSymbols[IS->second]->getName() << '\n');
 651             if (Islands.ColdSymbols[IS->second]->isUndefined())
 652               Streamer.emitLabel(Islands.ColdSymbols[IS->second]);
 653           }
 654         } else {
 655           if (!EmitColdPart) {
 656             if (MCSymbol *Sym = Islands.Proxies[OnBehalfOf][IS->second]) {
 657               LLVM_DEBUG(dbgs() << "BOLT-DEBUG: emitted label "
 658                                 << Sym->getName() << '\n');
 659               Streamer.emitLabel(Sym);
 660             }
 661           } else if (MCSymbol *Sym =
 662                          Islands.ColdProxies[OnBehalfOf][IS->second]) {
 663             LLVM_DEBUG(dbgs() << "BOLT-DEBUG: emitted label " << Sym->getName()
 664                               << '\n');
 665             Streamer.emitLabel(Sym);
 666           }
 667         }
 668         ++IS;
 669       }
 670     }
 671     assert(FunctionOffset <= EndOffset && "overflow error");
 672     emitCI(FunctionOffset, EndOffset);
 673   }
 674   assert(IS == Islands.Offsets.end() && "some symbols were not emitted!");
 675
 676   if (OnBehalfOf)
 677     return;
 678   // Now emit constant islands from other functions that we may have used in
 679   // this function.
 680   for (BinaryFunction *ExternalFunc : Islands.Dependency)
 681     emitConstantIslands(*ExternalFunc, EmitColdPart, &BF);
 682 }
 683
 684 SMLoc BinaryEmitter::emitLineInfo(const BinaryFunction &BF, SMLoc NewLoc,
 685                                   SMLoc PrevLoc, bool FirstInstr,
 686                                   MCSymbol *&InstrLabel) {
 687   DWARFUnit *FunctionCU = BF.getDWARFUnit();
 688   const DWARFDebugLine::LineTable *FunctionLineTable = BF.getDWARFLineTable();
 689   assert(FunctionCU && "cannot emit line info for function without CU");
 690
 691   DebugLineTableRowRef RowReference = DebugLineTableRowRef::fromSMLoc(NewLoc);
 692
 693   // Check if no new line info needs to be emitted.
 694   if (RowReference == DebugLineTableRowRef::NULL_ROW ||
 695       NewLoc.getPointer() == PrevLoc.getPointer())
 696     return PrevLoc;
 697
 698   unsigned CurrentFilenum = 0;
 699   const DWARFDebugLine::LineTable *CurrentLineTable = FunctionLineTable;
 700
 701   // If the CU id from the current instruction location does not
 702   // match the CU id from the current function, it means that we
 703   // have come across some inlined code.  We must look up the CU
 704   // for the instruction's original function and get the line table
 705   // from that.
 706   const uint64_t FunctionUnitIndex = FunctionCU->getOffset();
 707   const uint32_t CurrentUnitIndex = RowReference.DwCompileUnitIndex;
 708   if (CurrentUnitIndex != FunctionUnitIndex) {
 709     CurrentLineTable = BC.DwCtx->getLineTableForUnit(
 710         BC.DwCtx->getCompileUnitForOffset(CurrentUnitIndex));
 711     // Add filename from the inlined function to the current CU.
 712     CurrentFilenum = BC.addDebugFilenameToUnit(
 713         FunctionUnitIndex, CurrentUnitIndex,
 714         CurrentLineTable->Rows[RowReference.RowIndex - 1].File);
 715   }
 716
 717   const DWARFDebugLine::Row &CurrentRow =
 718       CurrentLineTable->Rows[RowReference.RowIndex - 1];
 719   if (!CurrentFilenum)
 720     CurrentFilenum = CurrentRow.File;
 721
 722   unsigned Flags = (DWARF2_FLAG_IS_STMT * CurrentRow.IsStmt) |
 723                    (DWARF2_FLAG_BASIC_BLOCK * CurrentRow.BasicBlock) |
 724                    (DWARF2_FLAG_PROLOGUE_END * CurrentRow.PrologueEnd) |
 725                    (DWARF2_FLAG_EPILOGUE_BEGIN * CurrentRow.EpilogueBegin);
 726
 727   // Always emit is_stmt at the beginning of function fragment.
 728   if (FirstInstr)
 729     Flags |= DWARF2_FLAG_IS_STMT;
 730
 731   BC.Ctx->setCurrentDwarfLoc(CurrentFilenum, CurrentRow.Line, CurrentRow.Column,
 732                              Flags, CurrentRow.Isa, CurrentRow.Discriminator);
 733   const MCDwarfLoc &DwarfLoc = BC.Ctx->getCurrentDwarfLoc();
 734   BC.Ctx->clearDwarfLocSeen();
 735
 736   if (!InstrLabel)
 737     InstrLabel = BC.Ctx->createTempSymbol();
 738
 739   BC.getDwarfLineTable(FunctionUnitIndex)
 740       .getMCLineSections()
 741       .addLineEntry(MCDwarfLineEntry(InstrLabel, DwarfLoc),
 742                     Streamer.getCurrentSectionOnly());
 743
 744   return NewLoc;
 745 }
 746
 747 void BinaryEmitter::emitLineInfoEnd(const BinaryFunction &BF,
 748                                     MCSymbol *FunctionEndLabel) {
 749   DWARFUnit *FunctionCU = BF.getDWARFUnit();
 750   assert(FunctionCU && "DWARF unit expected");
 751   BC.Ctx->setCurrentDwarfLoc(0, 0, 0, DWARF2_FLAG_END_SEQUENCE, 0, 0);
 752   const MCDwarfLoc &DwarfLoc = BC.Ctx->getCurrentDwarfLoc();
 753   BC.Ctx->clearDwarfLocSeen();
 754   BC.getDwarfLineTable(FunctionCU->getOffset())
 755       .getMCLineSections()
 756       .addLineEntry(MCDwarfLineEntry(FunctionEndLabel, DwarfLoc),
 757                     Streamer.getCurrentSectionOnly());
 758 }
 759
 760 void BinaryEmitter::emitJumpTables(const BinaryFunction &BF) {
 761   MCSection *ReadOnlySection = BC.MOFI->getReadOnlySection();
 762   MCSection *ReadOnlyColdSection = BC.MOFI->getContext().getELFSection(
 763       ".rodata.cold", ELF::SHT_PROGBITS, ELF::SHF_ALLOC);
 764
 765   if (!BF.hasJumpTables())
 766     return;
 767
 768   if (opts::PrintJumpTables)
 769     outs() << "BOLT-INFO: jump tables for function " << BF << ":\n";
 770
 771   for (auto &JTI : BF.jumpTables()) {
 772     JumpTable &JT = *JTI.second;
 773     // Only emit shared jump tables once, when processing the first parent
 774     if (JT.Parents.size() > 1 && JT.Parents[0] != &BF)
 775       continue;
 776     if (opts::PrintJumpTables)
 777       JT.print(outs());
 778     if (opts::JumpTables == JTS_BASIC && BC.HasRelocations) {
 779       JT.updateOriginal();
 780     } else {
 781       MCSection *HotSection, *ColdSection;
 782       if (opts::JumpTables == JTS_BASIC) {
 783         // In non-relocation mode we have to emit jump tables in local sections.
 784         // This way we only overwrite them when the corresponding function is
 785         // overwritten.
 786         std::string Name = ".local." + JT.Labels[0]->getName().str();
 787         std::replace(Name.begin(), Name.end(), '/', '.');
 788         BinarySection &Section =
 789             BC.registerOrUpdateSection(Name, ELF::SHT_PROGBITS, ELF::SHF_ALLOC);
 790         Section.setAnonymous(true);
 791         JT.setOutputSection(Section);
 792         HotSection = BC.getDataSection(Name);
 793         ColdSection = HotSection;
 794       } else {
 795         if (BF.isSimple()) {
 796           HotSection = ReadOnlySection;
 797           ColdSection = ReadOnlyColdSection;
 798         } else {
 799           HotSection = BF.hasProfile() ? ReadOnlySection : ReadOnlyColdSection;
 800           ColdSection = HotSection;
 801         }
 802       }
 803       emitJumpTable(JT, HotSection, ColdSection);
 804     }
 805   }
 806 }
 807
 808 void BinaryEmitter::emitJumpTable(const JumpTable &JT, MCSection *HotSection,
 809                                   MCSection *ColdSection) {
 810   // Pre-process entries for aggressive splitting.
 811   // Each label represents a separate switch table and gets its own count
 812   // determining its destination.
 813   std::map<MCSymbol *, uint64_t> LabelCounts;
 814   if (opts::JumpTables > JTS_SPLIT && !JT.Counts.empty()) {
 815     MCSymbol *CurrentLabel = JT.Labels.at(0);
 816     uint64_t CurrentLabelCount = 0;
 817     for (unsigned Index = 0; Index < JT.Entries.size(); ++Index) {
 818       auto LI = JT.Labels.find(Index * JT.EntrySize);
 819       if (LI != JT.Labels.end()) {
 820         LabelCounts[CurrentLabel] = CurrentLabelCount;
 821         CurrentLabel = LI->second;
 822         CurrentLabelCount = 0;
 823       }
 824       CurrentLabelCount += JT.Counts[Index].Count;
 825     }
 826     LabelCounts[CurrentLabel] = CurrentLabelCount;
 827   } else {
 828     Streamer.switchSection(JT.Count > 0 ? HotSection : ColdSection);
 829     Streamer.emitValueToAlignment(Align(JT.EntrySize));
 830   }
 831   MCSymbol *LastLabel = nullptr;
 832   uint64_t Offset = 0;
 833   for (MCSymbol *Entry : JT.Entries) {
 834     auto LI = JT.Labels.find(Offset);
 835     if (LI != JT.Labels.end()) {
 836       LLVM_DEBUG({
 837         dbgs() << "BOLT-DEBUG: emitting jump table " << LI->second->getName()
 838                << " (originally was at address 0x"
 839                << Twine::utohexstr(JT.getAddress() + Offset)
 840                << (Offset ? ") as part of larger jump table\n" : ")\n");
 841       });
 842       if (!LabelCounts.empty()) {
 843         LLVM_DEBUG(dbgs() << "BOLT-DEBUG: jump table count: "
 844                           << LabelCounts[LI->second] << '\n');
 845         if (LabelCounts[LI->second] > 0)
 846           Streamer.switchSection(HotSection);
 847         else
 848           Streamer.switchSection(ColdSection);
 849         Streamer.emitValueToAlignment(Align(JT.EntrySize));
 850       }
 851       // Emit all labels registered at the address of this jump table
 852       // to sync with our global symbol table.  We may have two labels
 853       // registered at this address if one label was created via
 854       // getOrCreateGlobalSymbol() (e.g. LEA instructions referencing
 855       // this location) and another via getOrCreateJumpTable().  This
 856       // creates a race where the symbols created by these two
 857       // functions may or may not be the same, but they are both
 858       // registered in our symbol table at the same address. By
 859       // emitting them all here we make sure there is no ambiguity
 860       // that depends on the order that these symbols were created, so
 861       // whenever this address is referenced in the binary, it is
 862       // certain to point to the jump table identified at this
 863       // address.
 864       if (BinaryData *BD = BC.getBinaryDataByName(LI->second->getName())) {
 865         for (MCSymbol *S : BD->getSymbols())
 866           Streamer.emitLabel(S);
 867       } else {
 868         Streamer.emitLabel(LI->second);
 869       }
 870       LastLabel = LI->second;
 871     }
 872     if (JT.Type == JumpTable::JTT_NORMAL) {
 873       Streamer.emitSymbolValue(Entry, JT.OutputEntrySize);
 874     } else { // JTT_PIC
 875       const MCSymbolRefExpr *JTExpr =
 876           MCSymbolRefExpr::create(LastLabel, Streamer.getContext());
 877       const MCSymbolRefExpr *E =
 878           MCSymbolRefExpr::create(Entry, Streamer.getContext());
 879       const MCBinaryExpr *Value =
 880           MCBinaryExpr::createSub(E, JTExpr, Streamer.getContext());
 881       Streamer.emitValue(Value, JT.EntrySize);
 882     }
 883     Offset += JT.EntrySize;
 884   }
 885 }
 886
 887 void BinaryEmitter::emitCFIInstruction(const MCCFIInstruction &Inst) const {
 888   switch (Inst.getOperation()) {
 889   default:
 890     llvm_unreachable("Unexpected instruction");
 891   case MCCFIInstruction::OpDefCfaOffset:
 892     Streamer.emitCFIDefCfaOffset(Inst.getOffset());
 893     break;
 894   case MCCFIInstruction::OpAdjustCfaOffset:
 895     Streamer.emitCFIAdjustCfaOffset(Inst.getOffset());
 896     break;
 897   case MCCFIInstruction::OpDefCfa:
 898     Streamer.emitCFIDefCfa(Inst.getRegister(), Inst.getOffset());
 899     break;
 900   case MCCFIInstruction::OpDefCfaRegister:
 901     Streamer.emitCFIDefCfaRegister(Inst.getRegister());
 902     break;
 903   case MCCFIInstruction::OpOffset:
 904     Streamer.emitCFIOffset(Inst.getRegister(), Inst.getOffset());
 905     break;
 906   case MCCFIInstruction::OpRegister:
 907     Streamer.emitCFIRegister(Inst.getRegister(), Inst.getRegister2());
 908     break;
 909   case MCCFIInstruction::OpWindowSave:
 910     Streamer.emitCFIWindowSave();
 911     break;
 912   case MCCFIInstruction::OpNegateRAState:
 913     Streamer.emitCFINegateRAState();
 914     break;
 915   case MCCFIInstruction::OpSameValue:
 916     Streamer.emitCFISameValue(Inst.getRegister());
 917     break;
 918   case MCCFIInstruction::OpGnuArgsSize:
 919     Streamer.emitCFIGnuArgsSize(Inst.getOffset());
 920     break;
 921   case MCCFIInstruction::OpEscape:
 922     Streamer.AddComment(Inst.getComment());
 923     Streamer.emitCFIEscape(Inst.getValues());
 924     break;
 925   case MCCFIInstruction::OpRestore:
 926     Streamer.emitCFIRestore(Inst.getRegister());
 927     break;
 928   case MCCFIInstruction::OpUndefined:
 929     Streamer.emitCFIUndefined(Inst.getRegister());
 930     break;
 931   }
 932 }
 933
 934 // The code is based on EHStreamer::emitExceptionTable().
 935 void BinaryEmitter::emitLSDA(BinaryFunction &BF, const FunctionFragment &FF) {
 936   const BinaryFunction::CallSitesRange Sites =
 937       BF.getCallSites(FF.getFragmentNum());
 938   if (Sites.empty())
 939     return;
 940
 941   // Calculate callsite table size. Size of each callsite entry is:
 942   //
 943   //  sizeof(start) + sizeof(length) + sizeof(LP) + sizeof(uleb128(action))
 944   //
 945   // or
 946   //
 947   //  sizeof(dwarf::DW_EH_PE_data4) * 3 + sizeof(uleb128(action))
 948   uint64_t CallSiteTableLength = llvm::size(Sites) * 4 * 3;
 949   for (const auto &FragmentCallSite : Sites)
 950     CallSiteTableLength += getULEB128Size(FragmentCallSite.second.Action);
 951
 952   Streamer.switchSection(BC.MOFI->getLSDASection());
 953
 954   const unsigned TTypeEncoding = BF.getLSDATypeEncoding();
 955   const unsigned TTypeEncodingSize = BC.getDWARFEncodingSize(TTypeEncoding);
 956   const uint16_t TTypeAlignment = 4;
 957
 958   // Type tables have to be aligned at 4 bytes.
 959   Streamer.emitValueToAlignment(Align(TTypeAlignment));
 960
 961   // Emit the LSDA label.
 962   MCSymbol *LSDASymbol = BF.getLSDASymbol(FF.getFragmentNum());
 963   assert(LSDASymbol && "no LSDA symbol set");
 964   Streamer.emitLabel(LSDASymbol);
 965
 966   // Corresponding FDE start.
 967   const MCSymbol *StartSymbol = BF.getSymbol(FF.getFragmentNum());
 968
 969   // Emit the LSDA header.
 970
 971   // If LPStart is omitted, then the start of the FDE is used as a base for
 972   // landing pad displacements. Then if a cold fragment starts with
 973   // a landing pad, this means that the first landing pad offset will be 0.
 974   // As a result, the exception handling runtime will ignore this landing pad
 975   // because zero offset denotes the absence of a landing pad.
 976   // For this reason, when the binary has fixed starting address we emit LPStart
 977   // as 0 and output the absolute value of the landing pad in the table.
 978   //
 979   // If the base address can change, we cannot use absolute addresses for
 980   // landing pads (at least not without runtime relocations). Hence, we fall
 981   // back to emitting landing pads relative to the FDE start.
 982   // As we are emitting label differences, we have to guarantee both labels are
 983   // defined in the same section and hence cannot place the landing pad into a
 984   // cold fragment when the corresponding call site is in the hot fragment.
 985   // Because of this issue and the previously described issue of possible
 986   // zero-offset landing pad we have to place landing pads in the same section
 987   // as the corresponding invokes for shared objects.
 988   std::function<void(const MCSymbol *)> emitLandingPad;
 989   if (BC.HasFixedLoadAddress) {
 990     Streamer.emitIntValue(dwarf::DW_EH_PE_udata4, 1); // LPStart format
 991     Streamer.emitIntValue(0, 4);                      // LPStart
 992     emitLandingPad = [&](const MCSymbol *LPSymbol) {
 993       if (!LPSymbol)
 994         Streamer.emitIntValue(0, 4);
 995       else
 996         Streamer.emitSymbolValue(LPSymbol, 4);
 997     };
 998   } else {
 999     Streamer.emitIntValue(dwarf::DW_EH_PE_omit, 1); // LPStart format
1000     emitLandingPad = [&](const MCSymbol *LPSymbol) {
1001       if (!LPSymbol)
1002         Streamer.emitIntValue(0, 4);
1003       else
1004         Streamer.emitAbsoluteSymbolDiff(LPSymbol, StartSymbol, 4);
1005     };
1006   }
1007
1008   Streamer.emitIntValue(TTypeEncoding, 1); // TType format
1009
1010   // See the comment in EHStreamer::emitExceptionTable() on to use
1011   // uleb128 encoding (which can use variable number of bytes to encode the same
1012   // value) to ensure type info table is properly aligned at 4 bytes without
1013   // iteratively fixing sizes of the tables.
1014   unsigned CallSiteTableLengthSize = getULEB128Size(CallSiteTableLength);
1015   unsigned TTypeBaseOffset =
1016       sizeof(int8_t) +                 // Call site format
1017       CallSiteTableLengthSize +        // Call site table length size
1018       CallSiteTableLength +            // Call site table length
1019       BF.getLSDAActionTable().size() + // Actions table size
1020       BF.getLSDATypeTable().size() * TTypeEncodingSize; // Types table size
1021   unsigned TTypeBaseOffsetSize = getULEB128Size(TTypeBaseOffset);
1022   unsigned TotalSize = sizeof(int8_t) +      // LPStart format
1023                        sizeof(int8_t) +      // TType format
1024                        TTypeBaseOffsetSize + // TType base offset size
1025                        TTypeBaseOffset;      // TType base offset
1026   unsigned SizeAlign = (4 - TotalSize) & 3;
1027
1028   if (TTypeEncoding != dwarf::DW_EH_PE_omit)
1029     // Account for any extra padding that will be added to the call site table
1030     // length.
1031     Streamer.emitULEB128IntValue(TTypeBaseOffset,
1032                                  /*PadTo=*/TTypeBaseOffsetSize + SizeAlign);
1033
1034   // Emit the landing pad call site table. We use signed data4 since we can emit
1035   // a landing pad in a different part of the split function that could appear
1036   // earlier in the address space than LPStart.
1037   Streamer.emitIntValue(dwarf::DW_EH_PE_sdata4, 1);
1038   Streamer.emitULEB128IntValue(CallSiteTableLength);
1039
1040   for (const auto &FragmentCallSite : Sites) {
1041     const BinaryFunction::CallSite &CallSite = FragmentCallSite.second;
1042     const MCSymbol *BeginLabel = CallSite.Start;
1043     const MCSymbol *EndLabel = CallSite.End;
1044
1045     assert(BeginLabel && "start EH label expected");
1046     assert(EndLabel && "end EH label expected");
1047
1048     // Start of the range is emitted relative to the start of current
1049     // function split part.
1050     Streamer.emitAbsoluteSymbolDiff(BeginLabel, StartSymbol, 4);
1051     Streamer.emitAbsoluteSymbolDiff(EndLabel, BeginLabel, 4);
1052     emitLandingPad(CallSite.LP);
1053     Streamer.emitULEB128IntValue(CallSite.Action);
1054   }
1055
1056   // Write out action, type, and type index tables at the end.
1057   //
1058   // For action and type index tables there's no need to change the original
1059   // table format unless we are doing function splitting, in which case we can
1060   // split and optimize the tables.
1061   //
1062   // For type table we (re-)encode the table using TTypeEncoding matching
1063   // the current assembler mode.
1064   for (uint8_t const &Byte : BF.getLSDAActionTable())
1065     Streamer.emitIntValue(Byte, 1);
1066
1067   const BinaryFunction::LSDATypeTableTy &TypeTable =
1068       (TTypeEncoding & dwarf::DW_EH_PE_indirect) ? BF.getLSDATypeAddressTable()
1069                                                  : BF.getLSDATypeTable();
1070   assert(TypeTable.size() == BF.getLSDATypeTable().size() &&
1071          "indirect type table size mismatch");
1072
1073   for (int Index = TypeTable.size() - 1; Index >= 0; --Index) {
1074     const uint64_t TypeAddress = TypeTable[Index];
1075     switch (TTypeEncoding & 0x70) {
1076     default:
1077       llvm_unreachable("unsupported TTypeEncoding");
1078     case dwarf::DW_EH_PE_absptr:
1079       Streamer.emitIntValue(TypeAddress, TTypeEncodingSize);
1080       break;
1081     case dwarf::DW_EH_PE_pcrel: {
1082       if (TypeAddress) {
1083         const MCSymbol *TypeSymbol =
1084             BC.getOrCreateGlobalSymbol(TypeAddress, "TI", 0, TTypeAlignment);
1085         MCSymbol *DotSymbol = BC.Ctx->createNamedTempSymbol();
1086         Streamer.emitLabel(DotSymbol);
1087         const MCBinaryExpr *SubDotExpr = MCBinaryExpr::createSub(
1088             MCSymbolRefExpr::create(TypeSymbol, *BC.Ctx),
1089             MCSymbolRefExpr::create(DotSymbol, *BC.Ctx), *BC.Ctx);
1090         Streamer.emitValue(SubDotExpr, TTypeEncodingSize);
1091       } else {
1092         Streamer.emitIntValue(0, TTypeEncodingSize);
1093       }
1094       break;
1095     }
1096     }
1097   }
1098   for (uint8_t const &Byte : BF.getLSDATypeIndexTable())
1099     Streamer.emitIntValue(Byte, 1);
1100 }
1101
1102 void BinaryEmitter::emitDebugLineInfoForOriginalFunctions() {
1103   // If a function is in a CU containing at least one processed function, we
1104   // have to rewrite the whole line table for that CU. For unprocessed functions
1105   // we use data from the input line table.
1106   for (auto &It : BC.getBinaryFunctions()) {
1107     const BinaryFunction &Function = It.second;
1108
1109     // If the function was emitted, its line info was emitted with it.
1110     if (Function.isEmitted())
1111       continue;
1112
1113     const DWARFDebugLine::LineTable *LineTable = Function.getDWARFLineTable();
1114     if (!LineTable)
1115       continue; // nothing to update for this function
1116
1117     const uint64_t Address = Function.getAddress();
1118     std::vector<uint32_t> Results;
1119     if (!LineTable->lookupAddressRange(
1120             {Address, object::SectionedAddress::UndefSection},
1121             Function.getSize(), Results))
1122       continue;
1123
1124     if (Results.empty())
1125       continue;
1126
1127     // The first row returned could be the last row matching the start address.
1128     // Find the first row with the same address that is not the end of the
1129     // sequence.
1130     uint64_t FirstRow = Results.front();
1131     while (FirstRow > 0) {
1132       const DWARFDebugLine::Row &PrevRow = LineTable->Rows[FirstRow - 1];
1133       if (PrevRow.Address.Address != Address || PrevRow.EndSequence)
1134         break;
1135       --FirstRow;
1136     }
1137
1138     const uint64_t EndOfSequenceAddress =
1139         Function.getAddress() + Function.getMaxSize();
1140     BC.getDwarfLineTable(Function.getDWARFUnit()->getOffset())
1141         .addLineTableSequence(LineTable, FirstRow, Results.back(),
1142                               EndOfSequenceAddress);
1143   }
1144
1145   // For units that are completely unprocessed, use original debug line contents
1146   // eliminating the need to regenerate line info program.
1147   emitDebugLineInfoForUnprocessedCUs();
1148 }
1149
1150 void BinaryEmitter::emitDebugLineInfoForUnprocessedCUs() {
1151   // Sorted list of section offsets provides boundaries for section fragments,
1152   // where each fragment is the unit's contribution to debug line section.
1153   std::vector<uint64_t> StmtListOffsets;
1154   StmtListOffsets.reserve(BC.DwCtx->getNumCompileUnits());
1155   for (const std::unique_ptr<DWARFUnit> &CU : BC.DwCtx->compile_units()) {
1156     DWARFDie CUDie = CU->getUnitDIE();
1157     auto StmtList = dwarf::toSectionOffset(CUDie.find(dwarf::DW_AT_stmt_list));
1158     if (!StmtList)
1159       continue;
1160
1161     StmtListOffsets.push_back(*StmtList);
1162   }
1163   llvm::sort(StmtListOffsets);
1164
1165   // For each CU that was not processed, emit its line info as a binary blob.
1166   for (const std::unique_ptr<DWARFUnit> &CU : BC.DwCtx->compile_units()) {
1167     if (BC.ProcessedCUs.count(CU.get()))
1168       continue;
1169
1170     DWARFDie CUDie = CU->getUnitDIE();
1171     auto StmtList = dwarf::toSectionOffset(CUDie.find(dwarf::DW_AT_stmt_list));
1172     if (!StmtList)
1173       continue;
1174
1175     StringRef DebugLineContents = CU->getLineSection().Data;
1176
1177     const uint64_t Begin = *StmtList;
1178
1179     // Statement list ends where the next unit contribution begins, or at the
1180     // end of the section.
1181     auto It = llvm::upper_bound(StmtListOffsets, Begin);
1182     const uint64_t End =
1183         It == StmtListOffsets.end() ? DebugLineContents.size() : *It;
1184
1185     BC.getDwarfLineTable(CU->getOffset())
1186         .addRawContents(DebugLineContents.slice(Begin, End));
1187   }
1188 }
1189
1190 void BinaryEmitter::emitDataSections(StringRef OrgSecPrefix) {
1191   for (BinarySection &Section : BC.sections()) {
1192     if (!Section.hasRelocations())
1193       continue;
1194
1195     StringRef Prefix = Section.hasSectionRef() ? OrgSecPrefix : "";
1196     Section.emitAsData(Streamer, Prefix + Section.getName());
1197     Section.clearRelocations();
1198   }
1199 }
1200
1201 namespace llvm {
1202 namespace bolt {
1203
1204 void emitBinaryContext(MCStreamer &Streamer, BinaryContext &BC,
1205                        StringRef OrgSecPrefix) {
1206   BinaryEmitter(Streamer, BC).emitAll(OrgSecPrefix);
1207 }
1208
1209 void emitFunctionBody(MCStreamer &Streamer, BinaryFunction &BF,
1210                       FunctionFragment &FF, bool EmitCodeOnly) {
1211   BinaryEmitter(Streamer, BF.getBinaryContext())
1212       .emitFunctionBody(BF, FF, EmitCodeOnly);
1213 }
1214
1215 } // namespace bolt
1216 } // namespace llvm