llvm/lib/LTO/ThinLTOCodeGenerator.cpp

   1 //===-ThinLTOCodeGenerator.cpp - LLVM Link Time Optimizer -----------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file implements the Thin Link Time Optimization library. This library is
  10 // intended to be used by linker to optimize code at link time.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "llvm/LTO/legacy/ThinLTOCodeGenerator.h"
  15 #include "llvm/Support/CommandLine.h"
  16
  17 #include "llvm/ADT/ScopeExit.h"
  18 #include "llvm/ADT/Statistic.h"
  19 #include "llvm/ADT/StringExtras.h"
  20 #include "llvm/Analysis/AliasAnalysis.h"
  21 #include "llvm/Analysis/ModuleSummaryAnalysis.h"
  22 #include "llvm/Analysis/ProfileSummaryInfo.h"
  23 #include "llvm/Analysis/TargetLibraryInfo.h"
  24 #include "llvm/Bitcode/BitcodeReader.h"
  25 #include "llvm/Bitcode/BitcodeWriter.h"
  26 #include "llvm/Bitcode/BitcodeWriterPass.h"
  27 #include "llvm/Config/llvm-config.h"
  28 #include "llvm/IR/DebugInfo.h"
  29 #include "llvm/IR/DiagnosticPrinter.h"
  30 #include "llvm/IR/LegacyPassManager.h"
  31 #include "llvm/IR/LLVMContext.h"
  32 #include "llvm/IR/LLVMRemarkStreamer.h"
  33 #include "llvm/IR/Mangler.h"
  34 #include "llvm/IR/PassTimingInfo.h"
  35 #include "llvm/IR/Verifier.h"
  36 #include "llvm/IRReader/IRReader.h"
  37 #include "llvm/LTO/LTO.h"
  38 #include "llvm/LTO/SummaryBasedOptimizations.h"
  39 #include "llvm/MC/TargetRegistry.h"
  40 #include "llvm/Object/IRObjectFile.h"
  41 #include "llvm/Passes/PassBuilder.h"
  42 #include "llvm/Passes/StandardInstrumentations.h"
  43 #include "llvm/Remarks/HotnessThresholdParser.h"
  44 #include "llvm/Support/CachePruning.h"
  45 #include "llvm/Support/Debug.h"
  46 #include "llvm/Support/Error.h"
  47 #include "llvm/Support/FileSystem.h"
  48 #include "llvm/Support/Path.h"
  49 #include "llvm/Support/SHA1.h"
  50 #include "llvm/Support/SmallVectorMemoryBuffer.h"
  51 #include "llvm/Support/ThreadPool.h"
  52 #include "llvm/Support/Threading.h"
  53 #include "llvm/Support/ToolOutputFile.h"
  54 #include "llvm/Support/raw_ostream.h"
  55 #include "llvm/Target/TargetMachine.h"
  56 #include "llvm/TargetParser/SubtargetFeature.h"
  57 #include "llvm/Transforms/IPO/FunctionAttrs.h"
  58 #include "llvm/Transforms/IPO/FunctionImport.h"
  59 #include "llvm/Transforms/IPO/Internalize.h"
  60 #include "llvm/Transforms/IPO/WholeProgramDevirt.h"
  61 #include "llvm/Transforms/ObjCARC.h"
  62 #include "llvm/Transforms/Utils/FunctionImportUtils.h"
  63
  64 #include <numeric>
  65
  66 #if !defined(_MSC_VER) && !defined(__MINGW32__)
  67 #include <unistd.h>
  68 #else
  69 #include <io.h>
  70 #endif
  71
  72 using namespace llvm;
  73
  74 #define DEBUG_TYPE "thinlto"
  75
  76 namespace llvm {
  77 // Flags -discard-value-names, defined in LTOCodeGenerator.cpp
  78 extern cl::opt<bool> LTODiscardValueNames;
  79 extern cl::opt<std::string> RemarksFilename;
  80 extern cl::opt<std::string> RemarksPasses;
  81 extern cl::opt<bool> RemarksWithHotness;
  82 extern cl::opt<std::optional<uint64_t>, false, remarks::HotnessThresholdParser>
  83     RemarksHotnessThreshold;
  84 extern cl::opt<std::string> RemarksFormat;
  85 }
  86
  87 namespace {
  88
  89 // Default to using all available threads in the system, but using only one
  90 // thred per core, as indicated by the usage of
  91 // heavyweight_hardware_concurrency() below.
  92 static cl::opt<int> ThreadCount("threads", cl::init(0));
  93
  94 // Simple helper to save temporary files for debug.
  95 static void saveTempBitcode(const Module &TheModule, StringRef TempDir,
  96                             unsigned count, StringRef Suffix) {
  97   if (TempDir.empty())
  98     return;
  99   // User asked to save temps, let dump the bitcode file after import.
 100   std::string SaveTempPath = (TempDir + llvm::Twine(count) + Suffix).str();
 101   std::error_code EC;
 102   raw_fd_ostream OS(SaveTempPath, EC, sys::fs::OF_None);
 103   if (EC)
 104     report_fatal_error(Twine("Failed to open ") + SaveTempPath +
 105                        " to save optimized bitcode\n");
 106   WriteBitcodeToFile(TheModule, OS, /* ShouldPreserveUseListOrder */ true);
 107 }
 108
 109 static const GlobalValueSummary *
 110 getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) {
 111   // If there is any strong definition anywhere, get it.
 112   auto StrongDefForLinker = llvm::find_if(
 113       GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
 114         auto Linkage = Summary->linkage();
 115         return !GlobalValue::isAvailableExternallyLinkage(Linkage) &&
 116                !GlobalValue::isWeakForLinker(Linkage);
 117       });
 118   if (StrongDefForLinker != GVSummaryList.end())
 119     return StrongDefForLinker->get();
 120   // Get the first *linker visible* definition for this global in the summary
 121   // list.
 122   auto FirstDefForLinker = llvm::find_if(
 123       GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
 124         auto Linkage = Summary->linkage();
 125         return !GlobalValue::isAvailableExternallyLinkage(Linkage);
 126       });
 127   // Extern templates can be emitted as available_externally.
 128   if (FirstDefForLinker == GVSummaryList.end())
 129     return nullptr;
 130   return FirstDefForLinker->get();
 131 }
 132
 133 // Populate map of GUID to the prevailing copy for any multiply defined
 134 // symbols. Currently assume first copy is prevailing, or any strong
 135 // definition. Can be refined with Linker information in the future.
 136 static void computePrevailingCopies(
 137     const ModuleSummaryIndex &Index,
 138     DenseMap<GlobalValue::GUID, const GlobalValueSummary *> &PrevailingCopy) {
 139   auto HasMultipleCopies = [&](const GlobalValueSummaryList &GVSummaryList) {
 140     return GVSummaryList.size() > 1;
 141   };
 142
 143   for (auto &I : Index) {
 144     if (HasMultipleCopies(I.second.SummaryList))
 145       PrevailingCopy[I.first] =
 146           getFirstDefinitionForLinker(I.second.SummaryList);
 147   }
 148 }
 149
 150 static StringMap<lto::InputFile *>
 151 generateModuleMap(std::vector<std::unique_ptr<lto::InputFile>> &Modules) {
 152   StringMap<lto::InputFile *> ModuleMap;
 153   for (auto &M : Modules) {
 154     LLVM_DEBUG(dbgs() << "Adding module " << M->getName() << " to ModuleMap\n");
 155     assert(!ModuleMap.contains(M->getName()) &&
 156            "Expect unique Buffer Identifier");
 157     ModuleMap[M->getName()] = M.get();
 158   }
 159   return ModuleMap;
 160 }
 161
 162 static void promoteModule(Module &TheModule, const ModuleSummaryIndex &Index,
 163                           bool ClearDSOLocalOnDeclarations) {
 164   if (renameModuleForThinLTO(TheModule, Index, ClearDSOLocalOnDeclarations))
 165     report_fatal_error("renameModuleForThinLTO failed");
 166 }
 167
 168 namespace {
 169 class ThinLTODiagnosticInfo : public DiagnosticInfo {
 170   const Twine &Msg;
 171 public:
 172   ThinLTODiagnosticInfo(const Twine &DiagMsg,
 173                         DiagnosticSeverity Severity = DS_Error)
 174       : DiagnosticInfo(DK_Linker, Severity), Msg(DiagMsg) {}
 175   void print(DiagnosticPrinter &DP) const override { DP << Msg; }
 176 };
 177 }
 178
 179 /// Verify the module and strip broken debug info.
 180 static void verifyLoadedModule(Module &TheModule) {
 181   bool BrokenDebugInfo = false;
 182   if (verifyModule(TheModule, &dbgs(), &BrokenDebugInfo))
 183     report_fatal_error("Broken module found, compilation aborted!");
 184   if (BrokenDebugInfo) {
 185     TheModule.getContext().diagnose(ThinLTODiagnosticInfo(
 186         "Invalid debug info found, debug info will be stripped", DS_Warning));
 187     StripDebugInfo(TheModule);
 188   }
 189 }
 190
 191 static std::unique_ptr<Module> loadModuleFromInput(lto::InputFile *Input,
 192                                                    LLVMContext &Context,
 193                                                    bool Lazy,
 194                                                    bool IsImporting) {
 195   auto &Mod = Input->getSingleBitcodeModule();
 196   SMDiagnostic Err;
 197   Expected<std::unique_ptr<Module>> ModuleOrErr =
 198       Lazy ? Mod.getLazyModule(Context,
 199                                /* ShouldLazyLoadMetadata */ true, IsImporting)
 200            : Mod.parseModule(Context);
 201   if (!ModuleOrErr) {
 202     handleAllErrors(ModuleOrErr.takeError(), [&](ErrorInfoBase &EIB) {
 203       SMDiagnostic Err = SMDiagnostic(Mod.getModuleIdentifier(),
 204                                       SourceMgr::DK_Error, EIB.message());
 205       Err.print("ThinLTO", errs());
 206     });
 207     report_fatal_error("Can't load module, abort.");
 208   }
 209   if (!Lazy)
 210     verifyLoadedModule(*ModuleOrErr.get());
 211   return std::move(*ModuleOrErr);
 212 }
 213
 214 static void
 215 crossImportIntoModule(Module &TheModule, const ModuleSummaryIndex &Index,
 216                       StringMap<lto::InputFile *> &ModuleMap,
 217                       const FunctionImporter::ImportMapTy &ImportList,
 218                       bool ClearDSOLocalOnDeclarations) {
 219   auto Loader = [&](StringRef Identifier) {
 220     auto &Input = ModuleMap[Identifier];
 221     return loadModuleFromInput(Input, TheModule.getContext(),
 222                                /*Lazy=*/true, /*IsImporting*/ true);
 223   };
 224
 225   FunctionImporter Importer(Index, Loader, ClearDSOLocalOnDeclarations);
 226   Expected<bool> Result = Importer.importFunctions(TheModule, ImportList);
 227   if (!Result) {
 228     handleAllErrors(Result.takeError(), [&](ErrorInfoBase &EIB) {
 229       SMDiagnostic Err = SMDiagnostic(TheModule.getModuleIdentifier(),
 230                                       SourceMgr::DK_Error, EIB.message());
 231       Err.print("ThinLTO", errs());
 232     });
 233     report_fatal_error("importFunctions failed");
 234   }
 235   // Verify again after cross-importing.
 236   verifyLoadedModule(TheModule);
 237 }
 238
 239 static void optimizeModule(Module &TheModule, TargetMachine &TM,
 240                            unsigned OptLevel, bool Freestanding,
 241                            bool DebugPassManager, ModuleSummaryIndex *Index) {
 242   std::optional<PGOOptions> PGOOpt;
 243   LoopAnalysisManager LAM;
 244   FunctionAnalysisManager FAM;
 245   CGSCCAnalysisManager CGAM;
 246   ModuleAnalysisManager MAM;
 247
 248   PassInstrumentationCallbacks PIC;
 249   StandardInstrumentations SI(TheModule.getContext(), DebugPassManager);
 250   SI.registerCallbacks(PIC, &MAM);
 251   PipelineTuningOptions PTO;
 252   PTO.LoopVectorization = true;
 253   PTO.SLPVectorization = true;
 254   PassBuilder PB(&TM, PTO, PGOOpt, &PIC);
 255
 256   std::unique_ptr<TargetLibraryInfoImpl> TLII(
 257       new TargetLibraryInfoImpl(Triple(TM.getTargetTriple())));
 258   if (Freestanding)
 259     TLII->disableAllFunctions();
 260   FAM.registerPass([&] { return TargetLibraryAnalysis(*TLII); });
 261
 262   // Register all the basic analyses with the managers.
 263   PB.registerModuleAnalyses(MAM);
 264   PB.registerCGSCCAnalyses(CGAM);
 265   PB.registerFunctionAnalyses(FAM);
 266   PB.registerLoopAnalyses(LAM);
 267   PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
 268
 269   ModulePassManager MPM;
 270
 271   OptimizationLevel OL;
 272
 273   switch (OptLevel) {
 274   default:
 275     llvm_unreachable("Invalid optimization level");
 276   case 0:
 277     OL = OptimizationLevel::O0;
 278     break;
 279   case 1:
 280     OL = OptimizationLevel::O1;
 281     break;
 282   case 2:
 283     OL = OptimizationLevel::O2;
 284     break;
 285   case 3:
 286     OL = OptimizationLevel::O3;
 287     break;
 288   }
 289
 290   MPM.addPass(PB.buildThinLTODefaultPipeline(OL, Index));
 291
 292   MPM.run(TheModule, MAM);
 293 }
 294
 295 static void
 296 addUsedSymbolToPreservedGUID(const lto::InputFile &File,
 297                              DenseSet<GlobalValue::GUID> &PreservedGUID) {
 298   for (const auto &Sym : File.symbols()) {
 299     if (Sym.isUsed())
 300       PreservedGUID.insert(GlobalValue::getGUID(Sym.getIRName()));
 301   }
 302 }
 303
 304 // Convert the PreservedSymbols map from "Name" based to "GUID" based.
 305 static void computeGUIDPreservedSymbols(const lto::InputFile &File,
 306                                         const StringSet<> &PreservedSymbols,
 307                                         const Triple &TheTriple,
 308                                         DenseSet<GlobalValue::GUID> &GUIDs) {
 309   // Iterate the symbols in the input file and if the input has preserved symbol
 310   // compute the GUID for the symbol.
 311   for (const auto &Sym : File.symbols()) {
 312     if (PreservedSymbols.count(Sym.getName()) && !Sym.getIRName().empty())
 313       GUIDs.insert(GlobalValue::getGUID(GlobalValue::getGlobalIdentifier(
 314           Sym.getIRName(), GlobalValue::ExternalLinkage, "")));
 315   }
 316 }
 317
 318 static DenseSet<GlobalValue::GUID>
 319 computeGUIDPreservedSymbols(const lto::InputFile &File,
 320                             const StringSet<> &PreservedSymbols,
 321                             const Triple &TheTriple) {
 322   DenseSet<GlobalValue::GUID> GUIDPreservedSymbols(PreservedSymbols.size());
 323   computeGUIDPreservedSymbols(File, PreservedSymbols, TheTriple,
 324                               GUIDPreservedSymbols);
 325   return GUIDPreservedSymbols;
 326 }
 327
 328 std::unique_ptr<MemoryBuffer> codegenModule(Module &TheModule,
 329                                             TargetMachine &TM) {
 330   SmallVector<char, 128> OutputBuffer;
 331
 332   // CodeGen
 333   {
 334     raw_svector_ostream OS(OutputBuffer);
 335     legacy::PassManager PM;
 336
 337     // If the bitcode files contain ARC code and were compiled with optimization,
 338     // the ObjCARCContractPass must be run, so do it unconditionally here.
 339     PM.add(createObjCARCContractPass());
 340
 341     // Setup the codegen now.
 342     if (TM.addPassesToEmitFile(PM, OS, nullptr, CodeGenFileType::ObjectFile,
 343                                /* DisableVerify */ true))
 344       report_fatal_error("Failed to setup codegen");
 345
 346     // Run codegen now. resulting binary is in OutputBuffer.
 347     PM.run(TheModule);
 348   }
 349   return std::make_unique<SmallVectorMemoryBuffer>(
 350       std::move(OutputBuffer), /*RequiresNullTerminator=*/false);
 351 }
 352
 353 /// Manage caching for a single Module.
 354 class ModuleCacheEntry {
 355   SmallString<128> EntryPath;
 356
 357 public:
 358   // Create a cache entry. This compute a unique hash for the Module considering
 359   // the current list of export/import, and offer an interface to query to
 360   // access the content in the cache.
 361   ModuleCacheEntry(
 362       StringRef CachePath, const ModuleSummaryIndex &Index, StringRef ModuleID,
 363       const FunctionImporter::ImportMapTy &ImportList,
 364       const FunctionImporter::ExportSetTy &ExportList,
 365       const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
 366       const GVSummaryMapTy &DefinedGVSummaries, unsigned OptLevel,
 367       bool Freestanding, const TargetMachineBuilder &TMBuilder) {
 368     if (CachePath.empty())
 369       return;
 370
 371     if (!Index.modulePaths().count(ModuleID))
 372       // The module does not have an entry, it can't have a hash at all
 373       return;
 374
 375     if (all_of(Index.getModuleHash(ModuleID),
 376                [](uint32_t V) { return V == 0; }))
 377       // No hash entry, no caching!
 378       return;
 379
 380     llvm::lto::Config Conf;
 381     Conf.OptLevel = OptLevel;
 382     Conf.Options = TMBuilder.Options;
 383     Conf.CPU = TMBuilder.MCpu;
 384     Conf.MAttrs.push_back(TMBuilder.MAttr);
 385     Conf.RelocModel = TMBuilder.RelocModel;
 386     Conf.CGOptLevel = TMBuilder.CGOptLevel;
 387     Conf.Freestanding = Freestanding;
 388     SmallString<40> Key;
 389     computeLTOCacheKey(Key, Conf, Index, ModuleID, ImportList, ExportList,
 390                        ResolvedODR, DefinedGVSummaries);
 391
 392     // This choice of file name allows the cache to be pruned (see pruneCache()
 393     // in include/llvm/Support/CachePruning.h).
 394     sys::path::append(EntryPath, CachePath, "llvmcache-" + Key);
 395   }
 396
 397   // Access the path to this entry in the cache.
 398   StringRef getEntryPath() { return EntryPath; }
 399
 400   // Try loading the buffer for this cache entry.
 401   ErrorOr<std::unique_ptr<MemoryBuffer>> tryLoadingBuffer() {
 402     if (EntryPath.empty())
 403       return std::error_code();
 404     SmallString<64> ResultPath;
 405     Expected<sys::fs::file_t> FDOrErr = sys::fs::openNativeFileForRead(
 406         Twine(EntryPath), sys::fs::OF_UpdateAtime, &ResultPath);
 407     if (!FDOrErr)
 408       return errorToErrorCode(FDOrErr.takeError());
 409     ErrorOr<std::unique_ptr<MemoryBuffer>> MBOrErr = MemoryBuffer::getOpenFile(
 410         *FDOrErr, EntryPath, /*FileSize=*/-1, /*RequiresNullTerminator=*/false);
 411     sys::fs::closeFile(*FDOrErr);
 412     return MBOrErr;
 413   }
 414
 415   // Cache the Produced object file
 416   void write(const MemoryBuffer &OutputBuffer) {
 417     if (EntryPath.empty())
 418       return;
 419
 420     if (auto Err = llvm::writeToOutput(
 421             EntryPath, [&OutputBuffer](llvm::raw_ostream &OS) -> llvm::Error {
 422               OS << OutputBuffer.getBuffer();
 423               return llvm::Error::success();
 424             }))
 425       report_fatal_error(llvm::formatv("ThinLTO: Can't write file {0}: {1}",
 426                                        EntryPath,
 427                                        toString(std::move(Err)).c_str()));
 428   }
 429 };
 430
 431 static std::unique_ptr<MemoryBuffer>
 432 ProcessThinLTOModule(Module &TheModule, ModuleSummaryIndex &Index,
 433                      StringMap<lto::InputFile *> &ModuleMap, TargetMachine &TM,
 434                      const FunctionImporter::ImportMapTy &ImportList,
 435                      const FunctionImporter::ExportSetTy &ExportList,
 436                      const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
 437                      const GVSummaryMapTy &DefinedGlobals,
 438                      const ThinLTOCodeGenerator::CachingOptions &CacheOptions,
 439                      bool DisableCodeGen, StringRef SaveTempsDir,
 440                      bool Freestanding, unsigned OptLevel, unsigned count,
 441                      bool DebugPassManager) {
 442   // "Benchmark"-like optimization: single-source case
 443   bool SingleModule = (ModuleMap.size() == 1);
 444
 445   // When linking an ELF shared object, dso_local should be dropped. We
 446   // conservatively do this for -fpic.
 447   bool ClearDSOLocalOnDeclarations =
 448       TM.getTargetTriple().isOSBinFormatELF() &&
 449       TM.getRelocationModel() != Reloc::Static &&
 450       TheModule.getPIELevel() == PIELevel::Default;
 451
 452   if (!SingleModule) {
 453     promoteModule(TheModule, Index, ClearDSOLocalOnDeclarations);
 454
 455     // Apply summary-based prevailing-symbol resolution decisions.
 456     thinLTOFinalizeInModule(TheModule, DefinedGlobals, /*PropagateAttrs=*/true);
 457
 458     // Save temps: after promotion.
 459     saveTempBitcode(TheModule, SaveTempsDir, count, ".1.promoted.bc");
 460   }
 461
 462   // Be friendly and don't nuke totally the module when the client didn't
 463   // supply anything to preserve.
 464   if (!ExportList.empty() || !GUIDPreservedSymbols.empty()) {
 465     // Apply summary-based internalization decisions.
 466     thinLTOInternalizeModule(TheModule, DefinedGlobals);
 467   }
 468
 469   // Save internalized bitcode
 470   saveTempBitcode(TheModule, SaveTempsDir, count, ".2.internalized.bc");
 471
 472   if (!SingleModule)
 473     crossImportIntoModule(TheModule, Index, ModuleMap, ImportList,
 474                           ClearDSOLocalOnDeclarations);
 475
 476   // Do this after any importing so that imported code is updated.
 477   // See comment at call to updateVCallVisibilityInIndex() for why
 478   // WholeProgramVisibilityEnabledInLTO is false.
 479   updatePublicTypeTestCalls(TheModule,
 480                             /* WholeProgramVisibilityEnabledInLTO */ false);
 481
 482   // Save temps: after cross-module import.
 483   saveTempBitcode(TheModule, SaveTempsDir, count, ".3.imported.bc");
 484
 485   optimizeModule(TheModule, TM, OptLevel, Freestanding, DebugPassManager,
 486                  &Index);
 487
 488   saveTempBitcode(TheModule, SaveTempsDir, count, ".4.opt.bc");
 489
 490   if (DisableCodeGen) {
 491     // Configured to stop before CodeGen, serialize the bitcode and return.
 492     SmallVector<char, 128> OutputBuffer;
 493     {
 494       raw_svector_ostream OS(OutputBuffer);
 495       ProfileSummaryInfo PSI(TheModule);
 496       auto Index = buildModuleSummaryIndex(TheModule, nullptr, &PSI);
 497       WriteBitcodeToFile(TheModule, OS, true, &Index);
 498     }
 499     return std::make_unique<SmallVectorMemoryBuffer>(
 500         std::move(OutputBuffer), /*RequiresNullTerminator=*/false);
 501   }
 502
 503   return codegenModule(TheModule, TM);
 504 }
 505
 506 /// Resolve prevailing symbols. Record resolutions in the \p ResolvedODR map
 507 /// for caching, and in the \p Index for application during the ThinLTO
 508 /// backends. This is needed for correctness for exported symbols (ensure
 509 /// at least one copy kept) and a compile-time optimization (to drop duplicate
 510 /// copies when possible).
 511 static void resolvePrevailingInIndex(
 512     ModuleSummaryIndex &Index,
 513     StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>>
 514         &ResolvedODR,
 515     const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
 516     const DenseMap<GlobalValue::GUID, const GlobalValueSummary *>
 517         &PrevailingCopy) {
 518
 519   auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) {
 520     const auto &Prevailing = PrevailingCopy.find(GUID);
 521     // Not in map means that there was only one copy, which must be prevailing.
 522     if (Prevailing == PrevailingCopy.end())
 523       return true;
 524     return Prevailing->second == S;
 525   };
 526
 527   auto recordNewLinkage = [&](StringRef ModuleIdentifier,
 528                               GlobalValue::GUID GUID,
 529                               GlobalValue::LinkageTypes NewLinkage) {
 530     ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;
 531   };
 532
 533   // TODO Conf.VisibilityScheme can be lto::Config::ELF for ELF.
 534   lto::Config Conf;
 535   thinLTOResolvePrevailingInIndex(Conf, Index, isPrevailing, recordNewLinkage,
 536                                   GUIDPreservedSymbols);
 537 }
 538
 539 // Initialize the TargetMachine builder for a given Triple
 540 static void initTMBuilder(TargetMachineBuilder &TMBuilder,
 541                           const Triple &TheTriple) {
 542   // Set a default CPU for Darwin triples (copied from LTOCodeGenerator).
 543   // FIXME this looks pretty terrible...
 544   if (TMBuilder.MCpu.empty() && TheTriple.isOSDarwin()) {
 545     if (TheTriple.getArch() == llvm::Triple::x86_64)
 546       TMBuilder.MCpu = "core2";
 547     else if (TheTriple.getArch() == llvm::Triple::x86)
 548       TMBuilder.MCpu = "yonah";
 549     else if (TheTriple.getArch() == llvm::Triple::aarch64 ||
 550              TheTriple.getArch() == llvm::Triple::aarch64_32)
 551       TMBuilder.MCpu = "cyclone";
 552   }
 553   TMBuilder.TheTriple = std::move(TheTriple);
 554 }
 555
 556 } // end anonymous namespace
 557
 558 void ThinLTOCodeGenerator::addModule(StringRef Identifier, StringRef Data) {
 559   MemoryBufferRef Buffer(Data, Identifier);
 560
 561   auto InputOrError = lto::InputFile::create(Buffer);
 562   if (!InputOrError)
 563     report_fatal_error(Twine("ThinLTO cannot create input file: ") +
 564                        toString(InputOrError.takeError()));
 565
 566   auto TripleStr = (*InputOrError)->getTargetTriple();
 567   Triple TheTriple(TripleStr);
 568
 569   if (Modules.empty())
 570     initTMBuilder(TMBuilder, Triple(TheTriple));
 571   else if (TMBuilder.TheTriple != TheTriple) {
 572     if (!TMBuilder.TheTriple.isCompatibleWith(TheTriple))
 573       report_fatal_error("ThinLTO modules with incompatible triples not "
 574                          "supported");
 575     initTMBuilder(TMBuilder, Triple(TMBuilder.TheTriple.merge(TheTriple)));
 576   }
 577
 578   Modules.emplace_back(std::move(*InputOrError));
 579 }
 580
 581 void ThinLTOCodeGenerator::preserveSymbol(StringRef Name) {
 582   PreservedSymbols.insert(Name);
 583 }
 584
 585 void ThinLTOCodeGenerator::crossReferenceSymbol(StringRef Name) {
 586   // FIXME: At the moment, we don't take advantage of this extra information,
 587   // we're conservatively considering cross-references as preserved.
 588   //  CrossReferencedSymbols.insert(Name);
 589   PreservedSymbols.insert(Name);
 590 }
 591
 592 // TargetMachine factory
 593 std::unique_ptr<TargetMachine> TargetMachineBuilder::create() const {
 594   std::string ErrMsg;
 595   const Target *TheTarget =
 596       TargetRegistry::lookupTarget(TheTriple.str(), ErrMsg);
 597   if (!TheTarget) {
 598     report_fatal_error(Twine("Can't load target for this Triple: ") + ErrMsg);
 599   }
 600
 601   // Use MAttr as the default set of features.
 602   SubtargetFeatures Features(MAttr);
 603   Features.getDefaultSubtargetFeatures(TheTriple);
 604   std::string FeatureStr = Features.getString();
 605
 606   std::unique_ptr<TargetMachine> TM(
 607       TheTarget->createTargetMachine(TheTriple.str(), MCpu, FeatureStr, Options,
 608                                      RelocModel, std::nullopt, CGOptLevel));
 609   assert(TM && "Cannot create target machine");
 610
 611   return TM;
 612 }
 613
 614 /**
 615  * Produce the combined summary index from all the bitcode files:
 616  * "thin-link".
 617  */
 618 std::unique_ptr<ModuleSummaryIndex> ThinLTOCodeGenerator::linkCombinedIndex() {
 619   std::unique_ptr<ModuleSummaryIndex> CombinedIndex =
 620       std::make_unique<ModuleSummaryIndex>(/*HaveGVs=*/false);
 621   for (auto &Mod : Modules) {
 622     auto &M = Mod->getSingleBitcodeModule();
 623     if (Error Err = M.readSummary(*CombinedIndex, Mod->getName())) {
 624       // FIXME diagnose
 625       logAllUnhandledErrors(
 626           std::move(Err), errs(),
 627           "error: can't create module summary index for buffer: ");
 628       return nullptr;
 629     }
 630   }
 631   return CombinedIndex;
 632 }
 633
 634 namespace {
 635 struct IsExported {
 636   const DenseMap<StringRef, FunctionImporter::ExportSetTy> &ExportLists;
 637   const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols;
 638
 639   IsExported(
 640       const DenseMap<StringRef, FunctionImporter::ExportSetTy> &ExportLists,
 641       const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols)
 642       : ExportLists(ExportLists), GUIDPreservedSymbols(GUIDPreservedSymbols) {}
 643
 644   bool operator()(StringRef ModuleIdentifier, ValueInfo VI) const {
 645     const auto &ExportList = ExportLists.find(ModuleIdentifier);
 646     return (ExportList != ExportLists.end() && ExportList->second.count(VI)) ||
 647            GUIDPreservedSymbols.count(VI.getGUID());
 648   }
 649 };
 650
 651 struct IsPrevailing {
 652   const DenseMap<GlobalValue::GUID, const GlobalValueSummary *> &PrevailingCopy;
 653   IsPrevailing(const DenseMap<GlobalValue::GUID, const GlobalValueSummary *>
 654                    &PrevailingCopy)
 655       : PrevailingCopy(PrevailingCopy) {}
 656
 657   bool operator()(GlobalValue::GUID GUID, const GlobalValueSummary *S) const {
 658     const auto &Prevailing = PrevailingCopy.find(GUID);
 659     // Not in map means that there was only one copy, which must be prevailing.
 660     if (Prevailing == PrevailingCopy.end())
 661       return true;
 662     return Prevailing->second == S;
 663   };
 664 };
 665 } // namespace
 666
 667 static void computeDeadSymbolsInIndex(
 668     ModuleSummaryIndex &Index,
 669     const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) {
 670   // We have no symbols resolution available. And can't do any better now in the
 671   // case where the prevailing symbol is in a native object. It can be refined
 672   // with linker information in the future.
 673   auto isPrevailing = [&](GlobalValue::GUID G) {
 674     return PrevailingType::Unknown;
 675   };
 676   computeDeadSymbolsWithConstProp(Index, GUIDPreservedSymbols, isPrevailing,
 677                                   /* ImportEnabled = */ true);
 678 }
 679
 680 /**
 681  * Perform promotion and renaming of exported internal functions.
 682  * Index is updated to reflect linkage changes from weak resolution.
 683  */
 684 void ThinLTOCodeGenerator::promote(Module &TheModule, ModuleSummaryIndex &Index,
 685                                    const lto::InputFile &File) {
 686   auto ModuleCount = Index.modulePaths().size();
 687   auto ModuleIdentifier = TheModule.getModuleIdentifier();
 688
 689   // Collect for each module the list of function it defines (GUID -> Summary).
 690   DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries;
 691   Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
 692
 693   // Convert the preserved symbols set from string to GUID
 694   auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
 695       File, PreservedSymbols, Triple(TheModule.getTargetTriple()));
 696
 697   // Add used symbol to the preserved symbols.
 698   addUsedSymbolToPreservedGUID(File, GUIDPreservedSymbols);
 699
 700   // Compute "dead" symbols, we don't want to import/export these!
 701   computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
 702
 703   // Compute prevailing symbols
 704   DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
 705   computePrevailingCopies(Index, PrevailingCopy);
 706
 707   // Generate import/export list
 708   DenseMap<StringRef, FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
 709   DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
 710   ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries,
 711                            IsPrevailing(PrevailingCopy), ImportLists,
 712                            ExportLists);
 713
 714   // Resolve prevailing symbols
 715   StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
 716   resolvePrevailingInIndex(Index, ResolvedODR, GUIDPreservedSymbols,
 717                            PrevailingCopy);
 718
 719   thinLTOFinalizeInModule(TheModule,
 720                           ModuleToDefinedGVSummaries[ModuleIdentifier],
 721                           /*PropagateAttrs=*/false);
 722
 723   // Promote the exported values in the index, so that they are promoted
 724   // in the module.
 725   thinLTOInternalizeAndPromoteInIndex(
 726       Index, IsExported(ExportLists, GUIDPreservedSymbols),
 727       IsPrevailing(PrevailingCopy));
 728
 729   // FIXME Set ClearDSOLocalOnDeclarations.
 730   promoteModule(TheModule, Index, /*ClearDSOLocalOnDeclarations=*/false);
 731 }
 732
 733 /**
 734  * Perform cross-module importing for the module identified by ModuleIdentifier.
 735  */
 736 void ThinLTOCodeGenerator::crossModuleImport(Module &TheModule,
 737                                              ModuleSummaryIndex &Index,
 738                                              const lto::InputFile &File) {
 739   auto ModuleMap = generateModuleMap(Modules);
 740   auto ModuleCount = Index.modulePaths().size();
 741
 742   // Collect for each module the list of function it defines (GUID -> Summary).
 743   DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
 744   Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
 745
 746   // Convert the preserved symbols set from string to GUID
 747   auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
 748       File, PreservedSymbols, Triple(TheModule.getTargetTriple()));
 749
 750   addUsedSymbolToPreservedGUID(File, GUIDPreservedSymbols);
 751
 752   // Compute "dead" symbols, we don't want to import/export these!
 753   computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
 754
 755   // Compute prevailing symbols
 756   DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
 757   computePrevailingCopies(Index, PrevailingCopy);
 758
 759   // Generate import/export list
 760   DenseMap<StringRef, FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
 761   DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
 762   ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries,
 763                            IsPrevailing(PrevailingCopy), ImportLists,
 764                            ExportLists);
 765   auto &ImportList = ImportLists[TheModule.getModuleIdentifier()];
 766
 767   // FIXME Set ClearDSOLocalOnDeclarations.
 768   crossImportIntoModule(TheModule, Index, ModuleMap, ImportList,
 769                         /*ClearDSOLocalOnDeclarations=*/false);
 770 }
 771
 772 /**
 773  * Compute the list of summaries needed for importing into module.
 774  */
 775 void ThinLTOCodeGenerator::gatherImportedSummariesForModule(
 776     Module &TheModule, ModuleSummaryIndex &Index,
 777     std::map<std::string, GVSummaryMapTy> &ModuleToSummariesForIndex,
 778     const lto::InputFile &File) {
 779   auto ModuleCount = Index.modulePaths().size();
 780   auto ModuleIdentifier = TheModule.getModuleIdentifier();
 781
 782   // Collect for each module the list of function it defines (GUID -> Summary).
 783   DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
 784   Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
 785
 786   // Convert the preserved symbols set from string to GUID
 787   auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
 788       File, PreservedSymbols, Triple(TheModule.getTargetTriple()));
 789
 790   addUsedSymbolToPreservedGUID(File, GUIDPreservedSymbols);
 791
 792   // Compute "dead" symbols, we don't want to import/export these!
 793   computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
 794
 795   // Compute prevailing symbols
 796   DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
 797   computePrevailingCopies(Index, PrevailingCopy);
 798
 799   // Generate import/export list
 800   DenseMap<StringRef, FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
 801   DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
 802   ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries,
 803                            IsPrevailing(PrevailingCopy), ImportLists,
 804                            ExportLists);
 805
 806   llvm::gatherImportedSummariesForModule(
 807       ModuleIdentifier, ModuleToDefinedGVSummaries,
 808       ImportLists[ModuleIdentifier], ModuleToSummariesForIndex);
 809 }
 810
 811 /**
 812  * Emit the list of files needed for importing into module.
 813  */
 814 void ThinLTOCodeGenerator::emitImports(Module &TheModule, StringRef OutputName,
 815                                        ModuleSummaryIndex &Index,
 816                                        const lto::InputFile &File) {
 817   auto ModuleCount = Index.modulePaths().size();
 818   auto ModuleIdentifier = TheModule.getModuleIdentifier();
 819
 820   // Collect for each module the list of function it defines (GUID -> Summary).
 821   DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
 822   Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
 823
 824   // Convert the preserved symbols set from string to GUID
 825   auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
 826       File, PreservedSymbols, Triple(TheModule.getTargetTriple()));
 827
 828   addUsedSymbolToPreservedGUID(File, GUIDPreservedSymbols);
 829
 830   // Compute "dead" symbols, we don't want to import/export these!
 831   computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
 832
 833   // Compute prevailing symbols
 834   DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
 835   computePrevailingCopies(Index, PrevailingCopy);
 836
 837   // Generate import/export list
 838   DenseMap<StringRef, FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
 839   DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
 840   ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries,
 841                            IsPrevailing(PrevailingCopy), ImportLists,
 842                            ExportLists);
 843
 844   std::map<std::string, GVSummaryMapTy> ModuleToSummariesForIndex;
 845   llvm::gatherImportedSummariesForModule(
 846       ModuleIdentifier, ModuleToDefinedGVSummaries,
 847       ImportLists[ModuleIdentifier], ModuleToSummariesForIndex);
 848
 849   std::error_code EC;
 850   if ((EC = EmitImportsFiles(ModuleIdentifier, OutputName,
 851                              ModuleToSummariesForIndex)))
 852     report_fatal_error(Twine("Failed to open ") + OutputName +
 853                        " to save imports lists\n");
 854 }
 855
 856 /**
 857  * Perform internalization. Runs promote and internalization together.
 858  * Index is updated to reflect linkage changes.
 859  */
 860 void ThinLTOCodeGenerator::internalize(Module &TheModule,
 861                                        ModuleSummaryIndex &Index,
 862                                        const lto::InputFile &File) {
 863   initTMBuilder(TMBuilder, Triple(TheModule.getTargetTriple()));
 864   auto ModuleCount = Index.modulePaths().size();
 865   auto ModuleIdentifier = TheModule.getModuleIdentifier();
 866
 867   // Convert the preserved symbols set from string to GUID
 868   auto GUIDPreservedSymbols =
 869       computeGUIDPreservedSymbols(File, PreservedSymbols, TMBuilder.TheTriple);
 870
 871   addUsedSymbolToPreservedGUID(File, GUIDPreservedSymbols);
 872
 873   // Collect for each module the list of function it defines (GUID -> Summary).
 874   DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
 875   Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
 876
 877   // Compute "dead" symbols, we don't want to import/export these!
 878   computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
 879
 880   // Compute prevailing symbols
 881   DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
 882   computePrevailingCopies(Index, PrevailingCopy);
 883
 884   // Generate import/export list
 885   DenseMap<StringRef, FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
 886   DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
 887   ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries,
 888                            IsPrevailing(PrevailingCopy), ImportLists,
 889                            ExportLists);
 890   auto &ExportList = ExportLists[ModuleIdentifier];
 891
 892   // Be friendly and don't nuke totally the module when the client didn't
 893   // supply anything to preserve.
 894   if (ExportList.empty() && GUIDPreservedSymbols.empty())
 895     return;
 896
 897   // Resolve prevailing symbols
 898   StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
 899   resolvePrevailingInIndex(Index, ResolvedODR, GUIDPreservedSymbols,
 900                            PrevailingCopy);
 901
 902   // Promote the exported values in the index, so that they are promoted
 903   // in the module.
 904   thinLTOInternalizeAndPromoteInIndex(
 905       Index, IsExported(ExportLists, GUIDPreservedSymbols),
 906       IsPrevailing(PrevailingCopy));
 907
 908   // FIXME Set ClearDSOLocalOnDeclarations.
 909   promoteModule(TheModule, Index, /*ClearDSOLocalOnDeclarations=*/false);
 910
 911   // Internalization
 912   thinLTOFinalizeInModule(TheModule,
 913                           ModuleToDefinedGVSummaries[ModuleIdentifier],
 914                           /*PropagateAttrs=*/false);
 915
 916   thinLTOInternalizeModule(TheModule,
 917                            ModuleToDefinedGVSummaries[ModuleIdentifier]);
 918 }
 919
 920 /**
 921  * Perform post-importing ThinLTO optimizations.
 922  */
 923 void ThinLTOCodeGenerator::optimize(Module &TheModule) {
 924   initTMBuilder(TMBuilder, Triple(TheModule.getTargetTriple()));
 925
 926   // Optimize now
 927   optimizeModule(TheModule, *TMBuilder.create(), OptLevel, Freestanding,
 928                  DebugPassManager, nullptr);
 929 }
 930
 931 /// Write out the generated object file, either from CacheEntryPath or from
 932 /// OutputBuffer, preferring hard-link when possible.
 933 /// Returns the path to the generated file in SavedObjectsDirectoryPath.
 934 std::string
 935 ThinLTOCodeGenerator::writeGeneratedObject(int count, StringRef CacheEntryPath,
 936                                            const MemoryBuffer &OutputBuffer) {
 937   auto ArchName = TMBuilder.TheTriple.getArchName();
 938   SmallString<128> OutputPath(SavedObjectsDirectoryPath);
 939   llvm::sys::path::append(OutputPath,
 940                           Twine(count) + "." + ArchName + ".thinlto.o");
 941   OutputPath.c_str(); // Ensure the string is null terminated.
 942   if (sys::fs::exists(OutputPath))
 943     sys::fs::remove(OutputPath);
 944
 945   // We don't return a memory buffer to the linker, just a list of files.
 946   if (!CacheEntryPath.empty()) {
 947     // Cache is enabled, hard-link the entry (or copy if hard-link fails).
 948     auto Err = sys::fs::create_hard_link(CacheEntryPath, OutputPath);
 949     if (!Err)
 950       return std::string(OutputPath.str());
 951     // Hard linking failed, try to copy.
 952     Err = sys::fs::copy_file(CacheEntryPath, OutputPath);
 953     if (!Err)
 954       return std::string(OutputPath.str());
 955     // Copy failed (could be because the CacheEntry was removed from the cache
 956     // in the meantime by another process), fall back and try to write down the
 957     // buffer to the output.
 958     errs() << "remark: can't link or copy from cached entry '" << CacheEntryPath
 959            << "' to '" << OutputPath << "'\n";
 960   }
 961   // No cache entry, just write out the buffer.
 962   std::error_code Err;
 963   raw_fd_ostream OS(OutputPath, Err, sys::fs::OF_None);
 964   if (Err)
 965     report_fatal_error(Twine("Can't open output '") + OutputPath + "'\n");
 966   OS << OutputBuffer.getBuffer();
 967   return std::string(OutputPath.str());
 968 }
 969
 970 // Main entry point for the ThinLTO processing
 971 void ThinLTOCodeGenerator::run() {
 972   timeTraceProfilerBegin("ThinLink", StringRef(""));
 973   auto TimeTraceScopeExit = llvm::make_scope_exit([]() {
 974     if (llvm::timeTraceProfilerEnabled())
 975       llvm::timeTraceProfilerEnd();
 976   });
 977   // Prepare the resulting object vector
 978   assert(ProducedBinaries.empty() && "The generator should not be reused");
 979   if (SavedObjectsDirectoryPath.empty())
 980     ProducedBinaries.resize(Modules.size());
 981   else {
 982     sys::fs::create_directories(SavedObjectsDirectoryPath);
 983     bool IsDir;
 984     sys::fs::is_directory(SavedObjectsDirectoryPath, IsDir);
 985     if (!IsDir)
 986       report_fatal_error(Twine("Unexistent dir: '") + SavedObjectsDirectoryPath + "'");
 987     ProducedBinaryFiles.resize(Modules.size());
 988   }
 989
 990   if (CodeGenOnly) {
 991     // Perform only parallel codegen and return.
 992     ThreadPool Pool;
 993     int count = 0;
 994     for (auto &Mod : Modules) {
 995       Pool.async([&](int count) {
 996         LLVMContext Context;
 997         Context.setDiscardValueNames(LTODiscardValueNames);
 998
 999         // Parse module now
1000         auto TheModule = loadModuleFromInput(Mod.get(), Context, false,
1001                                              /*IsImporting*/ false);
1002
1003         // CodeGen
1004         auto OutputBuffer = codegenModule(*TheModule, *TMBuilder.create());
1005         if (SavedObjectsDirectoryPath.empty())
1006           ProducedBinaries[count] = std::move(OutputBuffer);
1007         else
1008           ProducedBinaryFiles[count] =
1009               writeGeneratedObject(count, "", *OutputBuffer);
1010       }, count++);
1011     }
1012
1013     return;
1014   }
1015
1016   // Sequential linking phase
1017   auto Index = linkCombinedIndex();
1018
1019   // Save temps: index.
1020   if (!SaveTempsDir.empty()) {
1021     auto SaveTempPath = SaveTempsDir + "index.bc";
1022     std::error_code EC;
1023     raw_fd_ostream OS(SaveTempPath, EC, sys::fs::OF_None);
1024     if (EC)
1025       report_fatal_error(Twine("Failed to open ") + SaveTempPath +
1026                          " to save optimized bitcode\n");
1027     writeIndexToFile(*Index, OS);
1028   }
1029
1030
1031   // Prepare the module map.
1032   auto ModuleMap = generateModuleMap(Modules);
1033   auto ModuleCount = Modules.size();
1034
1035   // Collect for each module the list of function it defines (GUID -> Summary).
1036   DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
1037   Index->collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
1038
1039   // Convert the preserved symbols set from string to GUID, this is needed for
1040   // computing the caching hash and the internalization.
1041   DenseSet<GlobalValue::GUID> GUIDPreservedSymbols;
1042   for (const auto &M : Modules)
1043     computeGUIDPreservedSymbols(*M, PreservedSymbols, TMBuilder.TheTriple,
1044                                 GUIDPreservedSymbols);
1045
1046   // Add used symbol from inputs to the preserved symbols.
1047   for (const auto &M : Modules)
1048     addUsedSymbolToPreservedGUID(*M, GUIDPreservedSymbols);
1049
1050   // Compute "dead" symbols, we don't want to import/export these!
1051   computeDeadSymbolsInIndex(*Index, GUIDPreservedSymbols);
1052
1053   // Synthesize entry counts for functions in the combined index.
1054   computeSyntheticCounts(*Index);
1055
1056   // Currently there is no support for enabling whole program visibility via a
1057   // linker option in the old LTO API, but this call allows it to be specified
1058   // via the internal option. Must be done before WPD below.
1059   if (hasWholeProgramVisibility(/* WholeProgramVisibilityEnabledInLTO */ false))
1060     Index->setWithWholeProgramVisibility();
1061
1062   // FIXME: This needs linker information via a TBD new interface
1063   updateVCallVisibilityInIndex(*Index,
1064                                /*WholeProgramVisibilityEnabledInLTO=*/false,
1065                                // FIXME: These need linker information via a
1066                                // TBD new interface.
1067                                /*DynamicExportSymbols=*/{},
1068                                /*VisibleToRegularObjSymbols=*/{});
1069
1070   // Perform index-based WPD. This will return immediately if there are
1071   // no index entries in the typeIdMetadata map (e.g. if we are instead
1072   // performing IR-based WPD in hybrid regular/thin LTO mode).
1073   std::map<ValueInfo, std::vector<VTableSlotSummary>> LocalWPDTargetsMap;
1074   std::set<GlobalValue::GUID> ExportedGUIDs;
1075   runWholeProgramDevirtOnIndex(*Index, ExportedGUIDs, LocalWPDTargetsMap);
1076   for (auto GUID : ExportedGUIDs)
1077     GUIDPreservedSymbols.insert(GUID);
1078
1079   // Compute prevailing symbols
1080   DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
1081   computePrevailingCopies(*Index, PrevailingCopy);
1082
1083   // Collect the import/export lists for all modules from the call-graph in the
1084   // combined index.
1085   DenseMap<StringRef, FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
1086   DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
1087   ComputeCrossModuleImport(*Index, ModuleToDefinedGVSummaries,
1088                            IsPrevailing(PrevailingCopy), ImportLists,
1089                            ExportLists);
1090
1091   // We use a std::map here to be able to have a defined ordering when
1092   // producing a hash for the cache entry.
1093   // FIXME: we should be able to compute the caching hash for the entry based
1094   // on the index, and nuke this map.
1095   StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
1096
1097   // Resolve prevailing symbols, this has to be computed early because it
1098   // impacts the caching.
1099   resolvePrevailingInIndex(*Index, ResolvedODR, GUIDPreservedSymbols,
1100                            PrevailingCopy);
1101
1102   // Use global summary-based analysis to identify symbols that can be
1103   // internalized (because they aren't exported or preserved as per callback).
1104   // Changes are made in the index, consumed in the ThinLTO backends.
1105   updateIndexWPDForExports(*Index,
1106                            IsExported(ExportLists, GUIDPreservedSymbols),
1107                            LocalWPDTargetsMap);
1108   thinLTOInternalizeAndPromoteInIndex(
1109       *Index, IsExported(ExportLists, GUIDPreservedSymbols),
1110       IsPrevailing(PrevailingCopy));
1111
1112   thinLTOPropagateFunctionAttrs(*Index, IsPrevailing(PrevailingCopy));
1113
1114   // Make sure that every module has an entry in the ExportLists, ImportList,
1115   // GVSummary and ResolvedODR maps to enable threaded access to these maps
1116   // below.
1117   for (auto &Module : Modules) {
1118     auto ModuleIdentifier = Module->getName();
1119     ExportLists[ModuleIdentifier];
1120     ImportLists[ModuleIdentifier];
1121     ResolvedODR[ModuleIdentifier];
1122     ModuleToDefinedGVSummaries[ModuleIdentifier];
1123   }
1124
1125   std::vector<BitcodeModule *> ModulesVec;
1126   ModulesVec.reserve(Modules.size());
1127   for (auto &Mod : Modules)
1128     ModulesVec.push_back(&Mod->getSingleBitcodeModule());
1129   std::vector<int> ModulesOrdering = lto::generateModulesOrdering(ModulesVec);
1130
1131   if (llvm::timeTraceProfilerEnabled())
1132     llvm::timeTraceProfilerEnd();
1133
1134   TimeTraceScopeExit.release();
1135
1136   // Parallel optimizer + codegen
1137   {
1138     ThreadPool Pool(heavyweight_hardware_concurrency(ThreadCount));
1139     for (auto IndexCount : ModulesOrdering) {
1140       auto &Mod = Modules[IndexCount];
1141       Pool.async([&](int count) {
1142         auto ModuleIdentifier = Mod->getName();
1143         auto &ExportList = ExportLists[ModuleIdentifier];
1144
1145         auto &DefinedGVSummaries = ModuleToDefinedGVSummaries[ModuleIdentifier];
1146
1147         // The module may be cached, this helps handling it.
1148         ModuleCacheEntry CacheEntry(CacheOptions.Path, *Index, ModuleIdentifier,
1149                                     ImportLists[ModuleIdentifier], ExportList,
1150                                     ResolvedODR[ModuleIdentifier],
1151                                     DefinedGVSummaries, OptLevel, Freestanding,
1152                                     TMBuilder);
1153         auto CacheEntryPath = CacheEntry.getEntryPath();
1154
1155         {
1156           auto ErrOrBuffer = CacheEntry.tryLoadingBuffer();
1157           LLVM_DEBUG(dbgs() << "Cache " << (ErrOrBuffer ? "hit" : "miss")
1158                             << " '" << CacheEntryPath << "' for buffer "
1159                             << count << " " << ModuleIdentifier << "\n");
1160
1161           if (ErrOrBuffer) {
1162             // Cache Hit!
1163             if (SavedObjectsDirectoryPath.empty())
1164               ProducedBinaries[count] = std::move(ErrOrBuffer.get());
1165             else
1166               ProducedBinaryFiles[count] = writeGeneratedObject(
1167                   count, CacheEntryPath, *ErrOrBuffer.get());
1168             return;
1169           }
1170         }
1171
1172         LLVMContext Context;
1173         Context.setDiscardValueNames(LTODiscardValueNames);
1174         Context.enableDebugTypeODRUniquing();
1175         auto DiagFileOrErr = lto::setupLLVMOptimizationRemarks(
1176             Context, RemarksFilename, RemarksPasses, RemarksFormat,
1177             RemarksWithHotness, RemarksHotnessThreshold, count);
1178         if (!DiagFileOrErr) {
1179           errs() << "Error: " << toString(DiagFileOrErr.takeError()) << "\n";
1180           report_fatal_error("ThinLTO: Can't get an output file for the "
1181                              "remarks");
1182         }
1183
1184         // Parse module now
1185         auto TheModule = loadModuleFromInput(Mod.get(), Context, false,
1186                                              /*IsImporting*/ false);
1187
1188         // Save temps: original file.
1189         saveTempBitcode(*TheModule, SaveTempsDir, count, ".0.original.bc");
1190
1191         auto &ImportList = ImportLists[ModuleIdentifier];
1192         // Run the main process now, and generates a binary
1193         auto OutputBuffer = ProcessThinLTOModule(
1194             *TheModule, *Index, ModuleMap, *TMBuilder.create(), ImportList,
1195             ExportList, GUIDPreservedSymbols,
1196             ModuleToDefinedGVSummaries[ModuleIdentifier], CacheOptions,
1197             DisableCodeGen, SaveTempsDir, Freestanding, OptLevel, count,
1198             DebugPassManager);
1199
1200         // Commit to the cache (if enabled)
1201         CacheEntry.write(*OutputBuffer);
1202
1203         if (SavedObjectsDirectoryPath.empty()) {
1204           // We need to generated a memory buffer for the linker.
1205           if (!CacheEntryPath.empty()) {
1206             // When cache is enabled, reload from the cache if possible.
1207             // Releasing the buffer from the heap and reloading it from the
1208             // cache file with mmap helps us to lower memory pressure.
1209             // The freed memory can be used for the next input file.
1210             // The final binary link will read from the VFS cache (hopefully!)
1211             // or from disk (if the memory pressure was too high).
1212             auto ReloadedBufferOrErr = CacheEntry.tryLoadingBuffer();
1213             if (auto EC = ReloadedBufferOrErr.getError()) {
1214               // On error, keep the preexisting buffer and print a diagnostic.
1215               errs() << "remark: can't reload cached file '" << CacheEntryPath
1216                      << "': " << EC.message() << "\n";
1217             } else {
1218               OutputBuffer = std::move(*ReloadedBufferOrErr);
1219             }
1220           }
1221           ProducedBinaries[count] = std::move(OutputBuffer);
1222           return;
1223         }
1224         ProducedBinaryFiles[count] = writeGeneratedObject(
1225             count, CacheEntryPath, *OutputBuffer);
1226       }, IndexCount);
1227     }
1228   }
1229
1230   pruneCache(CacheOptions.Path, CacheOptions.Policy, ProducedBinaries);
1231
1232   // If statistics were requested, print them out now.
1233   if (llvm::AreStatisticsEnabled())
1234     llvm::PrintStatistics();
1235   reportAndResetTimings();
1236 }