llvm/lib/LTO/ThinLTOCodeGenerator.cpp

   1 //===-ThinLTOCodeGenerator.cpp - LLVM Link Time Optimizer -----------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file implements the Thin Link Time Optimization library. This library is
  10 // intended to be used by linker to optimize code at link time.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "llvm/LTO/legacy/ThinLTOCodeGenerator.h"
  15 #include "llvm/Support/CommandLine.h"
  16
  17 #include "llvm/ADT/ScopeExit.h"
  18 #include "llvm/ADT/Statistic.h"
  19 #include "llvm/ADT/StringExtras.h"
  20 #include "llvm/Analysis/AliasAnalysis.h"
  21 #include "llvm/Analysis/ModuleSummaryAnalysis.h"
  22 #include "llvm/Analysis/ProfileSummaryInfo.h"
  23 #include "llvm/Analysis/TargetLibraryInfo.h"
  24 #include "llvm/Analysis/TargetTransformInfo.h"
  25 #include "llvm/Bitcode/BitcodeReader.h"
  26 #include "llvm/Bitcode/BitcodeWriter.h"
  27 #include "llvm/Bitcode/BitcodeWriterPass.h"
  28 #include "llvm/Config/llvm-config.h"
  29 #include "llvm/IR/DebugInfo.h"
  30 #include "llvm/IR/DiagnosticPrinter.h"
  31 #include "llvm/IR/LLVMContext.h"
  32 #include "llvm/IR/LLVMRemarkStreamer.h"
  33 #include "llvm/IR/LegacyPassManager.h"
  34 #include "llvm/IR/Mangler.h"
  35 #include "llvm/IR/PassTimingInfo.h"
  36 #include "llvm/IR/Verifier.h"
  37 #include "llvm/IRReader/IRReader.h"
  38 #include "llvm/LTO/LTO.h"
  39 #include "llvm/LTO/SummaryBasedOptimizations.h"
  40 #include "llvm/MC/SubtargetFeature.h"
  41 #include "llvm/Object/IRObjectFile.h"
  42 #include "llvm/Passes/PassBuilder.h"
  43 #include "llvm/Passes/StandardInstrumentations.h"
  44 #include "llvm/Remarks/HotnessThresholdParser.h"
  45 #include "llvm/Support/CachePruning.h"
  46 #include "llvm/Support/Debug.h"
  47 #include "llvm/Support/Error.h"
  48 #include "llvm/Support/FileUtilities.h"
  49 #include "llvm/Support/Path.h"
  50 #include "llvm/Support/SHA1.h"
  51 #include "llvm/Support/SmallVectorMemoryBuffer.h"
  52 #include "llvm/Support/TargetRegistry.h"
  53 #include "llvm/Support/ThreadPool.h"
  54 #include "llvm/Support/Threading.h"
  55 #include "llvm/Support/ToolOutputFile.h"
  56 #include "llvm/Target/TargetMachine.h"
  57 #include "llvm/Transforms/IPO.h"
  58 #include "llvm/Transforms/IPO/FunctionImport.h"
  59 #include "llvm/Transforms/IPO/Internalize.h"
  60 #include "llvm/Transforms/IPO/PassManagerBuilder.h"
  61 #include "llvm/Transforms/IPO/WholeProgramDevirt.h"
  62 #include "llvm/Transforms/ObjCARC.h"
  63 #include "llvm/Transforms/Utils/FunctionImportUtils.h"
  64
  65 #include <numeric>
  66
  67 #if !defined(_MSC_VER) && !defined(__MINGW32__)
  68 #include <unistd.h>
  69 #else
  70 #include <io.h>
  71 #endif
  72
  73 using namespace llvm;
  74
  75 #define DEBUG_TYPE "thinlto"
  76
  77 namespace llvm {
  78 // Flags -discard-value-names, defined in LTOCodeGenerator.cpp
  79 extern cl::opt<bool> LTODiscardValueNames;
  80 extern cl::opt<std::string> RemarksFilename;
  81 extern cl::opt<std::string> RemarksPasses;
  82 extern cl::opt<bool> RemarksWithHotness;
  83 extern cl::opt<Optional<uint64_t>, false, remarks::HotnessThresholdParser>
  84     RemarksHotnessThreshold;
  85 extern cl::opt<std::string> RemarksFormat;
  86 }
  87
  88 namespace {
  89
  90 // Default to using all available threads in the system, but using only one
  91 // thred per core, as indicated by the usage of
  92 // heavyweight_hardware_concurrency() below.
  93 static cl::opt<int> ThreadCount("threads", cl::init(0));
  94
  95 // Simple helper to save temporary files for debug.
  96 static void saveTempBitcode(const Module &TheModule, StringRef TempDir,
  97                             unsigned count, StringRef Suffix) {
  98   if (TempDir.empty())
  99     return;
 100   // User asked to save temps, let dump the bitcode file after import.
 101   std::string SaveTempPath = (TempDir + llvm::Twine(count) + Suffix).str();
 102   std::error_code EC;
 103   raw_fd_ostream OS(SaveTempPath, EC, sys::fs::OF_None);
 104   if (EC)
 105     report_fatal_error(Twine("Failed to open ") + SaveTempPath +
 106                        " to save optimized bitcode\n");
 107   WriteBitcodeToFile(TheModule, OS, /* ShouldPreserveUseListOrder */ true);
 108 }
 109
 110 static const GlobalValueSummary *
 111 getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) {
 112   // If there is any strong definition anywhere, get it.
 113   auto StrongDefForLinker = llvm::find_if(
 114       GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
 115         auto Linkage = Summary->linkage();
 116         return !GlobalValue::isAvailableExternallyLinkage(Linkage) &&
 117                !GlobalValue::isWeakForLinker(Linkage);
 118       });
 119   if (StrongDefForLinker != GVSummaryList.end())
 120     return StrongDefForLinker->get();
 121   // Get the first *linker visible* definition for this global in the summary
 122   // list.
 123   auto FirstDefForLinker = llvm::find_if(
 124       GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
 125         auto Linkage = Summary->linkage();
 126         return !GlobalValue::isAvailableExternallyLinkage(Linkage);
 127       });
 128   // Extern templates can be emitted as available_externally.
 129   if (FirstDefForLinker == GVSummaryList.end())
 130     return nullptr;
 131   return FirstDefForLinker->get();
 132 }
 133
 134 // Populate map of GUID to the prevailing copy for any multiply defined
 135 // symbols. Currently assume first copy is prevailing, or any strong
 136 // definition. Can be refined with Linker information in the future.
 137 static void computePrevailingCopies(
 138     const ModuleSummaryIndex &Index,
 139     DenseMap<GlobalValue::GUID, const GlobalValueSummary *> &PrevailingCopy) {
 140   auto HasMultipleCopies = [&](const GlobalValueSummaryList &GVSummaryList) {
 141     return GVSummaryList.size() > 1;
 142   };
 143
 144   for (auto &I : Index) {
 145     if (HasMultipleCopies(I.second.SummaryList))
 146       PrevailingCopy[I.first] =
 147           getFirstDefinitionForLinker(I.second.SummaryList);
 148   }
 149 }
 150
 151 static StringMap<lto::InputFile *>
 152 generateModuleMap(std::vector<std::unique_ptr<lto::InputFile>> &Modules) {
 153   StringMap<lto::InputFile *> ModuleMap;
 154   for (auto &M : Modules) {
 155     assert(ModuleMap.find(M->getName()) == ModuleMap.end() &&
 156            "Expect unique Buffer Identifier");
 157     ModuleMap[M->getName()] = M.get();
 158   }
 159   return ModuleMap;
 160 }
 161
 162 static void promoteModule(Module &TheModule, const ModuleSummaryIndex &Index,
 163                           bool ClearDSOLocalOnDeclarations) {
 164   if (renameModuleForThinLTO(TheModule, Index, ClearDSOLocalOnDeclarations))
 165     report_fatal_error("renameModuleForThinLTO failed");
 166 }
 167
 168 namespace {
 169 class ThinLTODiagnosticInfo : public DiagnosticInfo {
 170   const Twine &Msg;
 171 public:
 172   ThinLTODiagnosticInfo(const Twine &DiagMsg,
 173                         DiagnosticSeverity Severity = DS_Error)
 174       : DiagnosticInfo(DK_Linker, Severity), Msg(DiagMsg) {}
 175   void print(DiagnosticPrinter &DP) const override { DP << Msg; }
 176 };
 177 }
 178
 179 /// Verify the module and strip broken debug info.
 180 static void verifyLoadedModule(Module &TheModule) {
 181   bool BrokenDebugInfo = false;
 182   if (verifyModule(TheModule, &dbgs(), &BrokenDebugInfo))
 183     report_fatal_error("Broken module found, compilation aborted!");
 184   if (BrokenDebugInfo) {
 185     TheModule.getContext().diagnose(ThinLTODiagnosticInfo(
 186         "Invalid debug info found, debug info will be stripped", DS_Warning));
 187     StripDebugInfo(TheModule);
 188   }
 189 }
 190
 191 static std::unique_ptr<Module> loadModuleFromInput(lto::InputFile *Input,
 192                                                    LLVMContext &Context,
 193                                                    bool Lazy,
 194                                                    bool IsImporting) {
 195   auto &Mod = Input->getSingleBitcodeModule();
 196   SMDiagnostic Err;
 197   Expected<std::unique_ptr<Module>> ModuleOrErr =
 198       Lazy ? Mod.getLazyModule(Context,
 199                                /* ShouldLazyLoadMetadata */ true, IsImporting)
 200            : Mod.parseModule(Context);
 201   if (!ModuleOrErr) {
 202     handleAllErrors(ModuleOrErr.takeError(), [&](ErrorInfoBase &EIB) {
 203       SMDiagnostic Err = SMDiagnostic(Mod.getModuleIdentifier(),
 204                                       SourceMgr::DK_Error, EIB.message());
 205       Err.print("ThinLTO", errs());
 206     });
 207     report_fatal_error("Can't load module, abort.");
 208   }
 209   if (!Lazy)
 210     verifyLoadedModule(*ModuleOrErr.get());
 211   return std::move(*ModuleOrErr);
 212 }
 213
 214 static void
 215 crossImportIntoModule(Module &TheModule, const ModuleSummaryIndex &Index,
 216                       StringMap<lto::InputFile *> &ModuleMap,
 217                       const FunctionImporter::ImportMapTy &ImportList,
 218                       bool ClearDSOLocalOnDeclarations) {
 219   auto Loader = [&](StringRef Identifier) {
 220     auto &Input = ModuleMap[Identifier];
 221     return loadModuleFromInput(Input, TheModule.getContext(),
 222                                /*Lazy=*/true, /*IsImporting*/ true);
 223   };
 224
 225   FunctionImporter Importer(Index, Loader, ClearDSOLocalOnDeclarations);
 226   Expected<bool> Result = Importer.importFunctions(TheModule, ImportList);
 227   if (!Result) {
 228     handleAllErrors(Result.takeError(), [&](ErrorInfoBase &EIB) {
 229       SMDiagnostic Err = SMDiagnostic(TheModule.getModuleIdentifier(),
 230                                       SourceMgr::DK_Error, EIB.message());
 231       Err.print("ThinLTO", errs());
 232     });
 233     report_fatal_error("importFunctions failed");
 234   }
 235   // Verify again after cross-importing.
 236   verifyLoadedModule(TheModule);
 237 }
 238
 239 static void optimizeModule(Module &TheModule, TargetMachine &TM,
 240                            unsigned OptLevel, bool Freestanding,
 241                            ModuleSummaryIndex *Index) {
 242   // Populate the PassManager
 243   PassManagerBuilder PMB;
 244   PMB.LibraryInfo = new TargetLibraryInfoImpl(TM.getTargetTriple());
 245   if (Freestanding)
 246     PMB.LibraryInfo->disableAllFunctions();
 247   PMB.Inliner = createFunctionInliningPass();
 248   // FIXME: should get it from the bitcode?
 249   PMB.OptLevel = OptLevel;
 250   PMB.LoopVectorize = true;
 251   PMB.SLPVectorize = true;
 252   // Already did this in verifyLoadedModule().
 253   PMB.VerifyInput = false;
 254   PMB.VerifyOutput = false;
 255   PMB.ImportSummary = Index;
 256
 257   legacy::PassManager PM;
 258
 259   // Add the TTI (required to inform the vectorizer about register size for
 260   // instance)
 261   PM.add(createTargetTransformInfoWrapperPass(TM.getTargetIRAnalysis()));
 262
 263   // Add optimizations
 264   PMB.populateThinLTOPassManager(PM);
 265
 266   PM.run(TheModule);
 267 }
 268
 269 static void optimizeModuleNewPM(Module &TheModule, TargetMachine &TM,
 270                                 unsigned OptLevel, bool Freestanding,
 271                                 bool DebugPassManager,
 272                                 ModuleSummaryIndex *Index) {
 273   Optional<PGOOptions> PGOOpt;
 274   LoopAnalysisManager LAM;
 275   FunctionAnalysisManager FAM;
 276   CGSCCAnalysisManager CGAM;
 277   ModuleAnalysisManager MAM;
 278
 279   PassInstrumentationCallbacks PIC;
 280   StandardInstrumentations SI(DebugPassManager);
 281   SI.registerCallbacks(PIC, &FAM);
 282   PipelineTuningOptions PTO;
 283   PTO.LoopVectorization = true;
 284   PTO.SLPVectorization = true;
 285   PassBuilder PB(&TM, PTO, PGOOpt, &PIC);
 286
 287   std::unique_ptr<TargetLibraryInfoImpl> TLII(
 288       new TargetLibraryInfoImpl(Triple(TM.getTargetTriple())));
 289   if (Freestanding)
 290     TLII->disableAllFunctions();
 291   FAM.registerPass([&] { return TargetLibraryAnalysis(*TLII); });
 292
 293   AAManager AA = PB.buildDefaultAAPipeline();
 294
 295   // Register the AA manager first so that our version is the one used.
 296   FAM.registerPass([&] { return std::move(AA); });
 297
 298   // Register all the basic analyses with the managers.
 299   PB.registerModuleAnalyses(MAM);
 300   PB.registerCGSCCAnalyses(CGAM);
 301   PB.registerFunctionAnalyses(FAM);
 302   PB.registerLoopAnalyses(LAM);
 303   PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
 304
 305   ModulePassManager MPM;
 306
 307   OptimizationLevel OL;
 308
 309   switch (OptLevel) {
 310   default:
 311     llvm_unreachable("Invalid optimization level");
 312   case 0:
 313     OL = OptimizationLevel::O0;
 314     break;
 315   case 1:
 316     OL = OptimizationLevel::O1;
 317     break;
 318   case 2:
 319     OL = OptimizationLevel::O2;
 320     break;
 321   case 3:
 322     OL = OptimizationLevel::O3;
 323     break;
 324   }
 325
 326   MPM.addPass(PB.buildThinLTODefaultPipeline(OL, Index));
 327
 328   MPM.run(TheModule, MAM);
 329 }
 330
 331 static void
 332 addUsedSymbolToPreservedGUID(const lto::InputFile &File,
 333                              DenseSet<GlobalValue::GUID> &PreservedGUID) {
 334   for (const auto &Sym : File.symbols()) {
 335     if (Sym.isUsed())
 336       PreservedGUID.insert(GlobalValue::getGUID(Sym.getIRName()));
 337   }
 338 }
 339
 340 // Convert the PreservedSymbols map from "Name" based to "GUID" based.
 341 static void computeGUIDPreservedSymbols(const lto::InputFile &File,
 342                                         const StringSet<> &PreservedSymbols,
 343                                         const Triple &TheTriple,
 344                                         DenseSet<GlobalValue::GUID> &GUIDs) {
 345   // Iterate the symbols in the input file and if the input has preserved symbol
 346   // compute the GUID for the symbol.
 347   for (const auto &Sym : File.symbols()) {
 348     if (PreservedSymbols.count(Sym.getName()) && !Sym.getIRName().empty())
 349       GUIDs.insert(GlobalValue::getGUID(GlobalValue::getGlobalIdentifier(
 350           Sym.getIRName(), GlobalValue::ExternalLinkage, "")));
 351   }
 352 }
 353
 354 static DenseSet<GlobalValue::GUID>
 355 computeGUIDPreservedSymbols(const lto::InputFile &File,
 356                             const StringSet<> &PreservedSymbols,
 357                             const Triple &TheTriple) {
 358   DenseSet<GlobalValue::GUID> GUIDPreservedSymbols(PreservedSymbols.size());
 359   computeGUIDPreservedSymbols(File, PreservedSymbols, TheTriple,
 360                               GUIDPreservedSymbols);
 361   return GUIDPreservedSymbols;
 362 }
 363
 364 std::unique_ptr<MemoryBuffer> codegenModule(Module &TheModule,
 365                                             TargetMachine &TM) {
 366   SmallVector<char, 128> OutputBuffer;
 367
 368   // CodeGen
 369   {
 370     raw_svector_ostream OS(OutputBuffer);
 371     legacy::PassManager PM;
 372
 373     // If the bitcode files contain ARC code and were compiled with optimization,
 374     // the ObjCARCContractPass must be run, so do it unconditionally here.
 375     PM.add(createObjCARCContractPass());
 376
 377     // Setup the codegen now.
 378     if (TM.addPassesToEmitFile(PM, OS, nullptr, CGFT_ObjectFile,
 379                                /* DisableVerify */ true))
 380       report_fatal_error("Failed to setup codegen");
 381
 382     // Run codegen now. resulting binary is in OutputBuffer.
 383     PM.run(TheModule);
 384   }
 385   return std::make_unique<SmallVectorMemoryBuffer>(std::move(OutputBuffer));
 386 }
 387
 388 /// Manage caching for a single Module.
 389 class ModuleCacheEntry {
 390   SmallString<128> EntryPath;
 391
 392 public:
 393   // Create a cache entry. This compute a unique hash for the Module considering
 394   // the current list of export/import, and offer an interface to query to
 395   // access the content in the cache.
 396   ModuleCacheEntry(
 397       StringRef CachePath, const ModuleSummaryIndex &Index, StringRef ModuleID,
 398       const FunctionImporter::ImportMapTy &ImportList,
 399       const FunctionImporter::ExportSetTy &ExportList,
 400       const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
 401       const GVSummaryMapTy &DefinedGVSummaries, unsigned OptLevel,
 402       bool Freestanding, const TargetMachineBuilder &TMBuilder) {
 403     if (CachePath.empty())
 404       return;
 405
 406     if (!Index.modulePaths().count(ModuleID))
 407       // The module does not have an entry, it can't have a hash at all
 408       return;
 409
 410     if (all_of(Index.getModuleHash(ModuleID),
 411                [](uint32_t V) { return V == 0; }))
 412       // No hash entry, no caching!
 413       return;
 414
 415     llvm::lto::Config Conf;
 416     Conf.OptLevel = OptLevel;
 417     Conf.Options = TMBuilder.Options;
 418     Conf.CPU = TMBuilder.MCpu;
 419     Conf.MAttrs.push_back(TMBuilder.MAttr);
 420     Conf.RelocModel = TMBuilder.RelocModel;
 421     Conf.CGOptLevel = TMBuilder.CGOptLevel;
 422     Conf.Freestanding = Freestanding;
 423     SmallString<40> Key;
 424     computeLTOCacheKey(Key, Conf, Index, ModuleID, ImportList, ExportList,
 425                        ResolvedODR, DefinedGVSummaries);
 426
 427     // This choice of file name allows the cache to be pruned (see pruneCache()
 428     // in include/llvm/Support/CachePruning.h).
 429     sys::path::append(EntryPath, CachePath, "llvmcache-" + Key);
 430   }
 431
 432   // Access the path to this entry in the cache.
 433   StringRef getEntryPath() { return EntryPath; }
 434
 435   // Try loading the buffer for this cache entry.
 436   ErrorOr<std::unique_ptr<MemoryBuffer>> tryLoadingBuffer() {
 437     if (EntryPath.empty())
 438       return std::error_code();
 439     SmallString<64> ResultPath;
 440     Expected<sys::fs::file_t> FDOrErr = sys::fs::openNativeFileForRead(
 441         Twine(EntryPath), sys::fs::OF_UpdateAtime, &ResultPath);
 442     if (!FDOrErr)
 443       return errorToErrorCode(FDOrErr.takeError());
 444     ErrorOr<std::unique_ptr<MemoryBuffer>> MBOrErr = MemoryBuffer::getOpenFile(
 445         *FDOrErr, EntryPath, /*FileSize=*/-1, /*RequiresNullTerminator=*/false);
 446     sys::fs::closeFile(*FDOrErr);
 447     return MBOrErr;
 448   }
 449
 450   // Cache the Produced object file
 451   void write(const MemoryBuffer &OutputBuffer) {
 452     if (EntryPath.empty())
 453       return;
 454
 455     // Write to a temporary to avoid race condition
 456     SmallString<128> TempFilename;
 457     SmallString<128> CachePath(EntryPath);
 458     llvm::sys::path::remove_filename(CachePath);
 459     sys::path::append(TempFilename, CachePath, "Thin-%%%%%%.tmp.o");
 460
 461     if (auto Err = handleErrors(
 462             llvm::writeFileAtomically(TempFilename, EntryPath,
 463                                       OutputBuffer.getBuffer()),
 464             [](const llvm::AtomicFileWriteError &E) {
 465               std::string ErrorMsgBuffer;
 466               llvm::raw_string_ostream S(ErrorMsgBuffer);
 467               E.log(S);
 468
 469               if (E.Error ==
 470                   llvm::atomic_write_error::failed_to_create_uniq_file) {
 471                 errs() << "Error: " << ErrorMsgBuffer << "\n";
 472                 report_fatal_error("ThinLTO: Can't get a temporary file");
 473               }
 474             })) {
 475       // FIXME
 476       consumeError(std::move(Err));
 477     }
 478   }
 479 };
 480
 481 static std::unique_ptr<MemoryBuffer>
 482 ProcessThinLTOModule(Module &TheModule, ModuleSummaryIndex &Index,
 483                      StringMap<lto::InputFile *> &ModuleMap, TargetMachine &TM,
 484                      const FunctionImporter::ImportMapTy &ImportList,
 485                      const FunctionImporter::ExportSetTy &ExportList,
 486                      const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
 487                      const GVSummaryMapTy &DefinedGlobals,
 488                      const ThinLTOCodeGenerator::CachingOptions &CacheOptions,
 489                      bool DisableCodeGen, StringRef SaveTempsDir,
 490                      bool Freestanding, unsigned OptLevel, unsigned count,
 491                      bool UseNewPM, bool DebugPassManager) {
 492
 493   // "Benchmark"-like optimization: single-source case
 494   bool SingleModule = (ModuleMap.size() == 1);
 495
 496   // When linking an ELF shared object, dso_local should be dropped. We
 497   // conservatively do this for -fpic.
 498   bool ClearDSOLocalOnDeclarations =
 499       TM.getTargetTriple().isOSBinFormatELF() &&
 500       TM.getRelocationModel() != Reloc::Static &&
 501       TheModule.getPIELevel() == PIELevel::Default;
 502
 503   if (!SingleModule) {
 504     promoteModule(TheModule, Index, ClearDSOLocalOnDeclarations);
 505
 506     // Apply summary-based prevailing-symbol resolution decisions.
 507     thinLTOResolvePrevailingInModule(TheModule, DefinedGlobals);
 508
 509     // Save temps: after promotion.
 510     saveTempBitcode(TheModule, SaveTempsDir, count, ".1.promoted.bc");
 511   }
 512
 513   // Be friendly and don't nuke totally the module when the client didn't
 514   // supply anything to preserve.
 515   if (!ExportList.empty() || !GUIDPreservedSymbols.empty()) {
 516     // Apply summary-based internalization decisions.
 517     thinLTOInternalizeModule(TheModule, DefinedGlobals);
 518   }
 519
 520   // Save internalized bitcode
 521   saveTempBitcode(TheModule, SaveTempsDir, count, ".2.internalized.bc");
 522
 523   if (!SingleModule) {
 524     crossImportIntoModule(TheModule, Index, ModuleMap, ImportList,
 525                           ClearDSOLocalOnDeclarations);
 526
 527     // Save temps: after cross-module import.
 528     saveTempBitcode(TheModule, SaveTempsDir, count, ".3.imported.bc");
 529   }
 530
 531   if (UseNewPM)
 532     optimizeModuleNewPM(TheModule, TM, OptLevel, Freestanding, DebugPassManager,
 533                         &Index);
 534   else
 535     optimizeModule(TheModule, TM, OptLevel, Freestanding, &Index);
 536
 537   saveTempBitcode(TheModule, SaveTempsDir, count, ".4.opt.bc");
 538
 539   if (DisableCodeGen) {
 540     // Configured to stop before CodeGen, serialize the bitcode and return.
 541     SmallVector<char, 128> OutputBuffer;
 542     {
 543       raw_svector_ostream OS(OutputBuffer);
 544       ProfileSummaryInfo PSI(TheModule);
 545       auto Index = buildModuleSummaryIndex(TheModule, nullptr, &PSI);
 546       WriteBitcodeToFile(TheModule, OS, true, &Index);
 547     }
 548     return std::make_unique<SmallVectorMemoryBuffer>(std::move(OutputBuffer));
 549   }
 550
 551   return codegenModule(TheModule, TM);
 552 }
 553
 554 /// Resolve prevailing symbols. Record resolutions in the \p ResolvedODR map
 555 /// for caching, and in the \p Index for application during the ThinLTO
 556 /// backends. This is needed for correctness for exported symbols (ensure
 557 /// at least one copy kept) and a compile-time optimization (to drop duplicate
 558 /// copies when possible).
 559 static void resolvePrevailingInIndex(
 560     ModuleSummaryIndex &Index,
 561     StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>>
 562         &ResolvedODR,
 563     const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
 564     const DenseMap<GlobalValue::GUID, const GlobalValueSummary *>
 565         &PrevailingCopy) {
 566
 567   auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) {
 568     const auto &Prevailing = PrevailingCopy.find(GUID);
 569     // Not in map means that there was only one copy, which must be prevailing.
 570     if (Prevailing == PrevailingCopy.end())
 571       return true;
 572     return Prevailing->second == S;
 573   };
 574
 575   auto recordNewLinkage = [&](StringRef ModuleIdentifier,
 576                               GlobalValue::GUID GUID,
 577                               GlobalValue::LinkageTypes NewLinkage) {
 578     ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;
 579   };
 580
 581   // TODO Conf.VisibilityScheme can be lto::Config::ELF for ELF.
 582   lto::Config Conf;
 583   thinLTOResolvePrevailingInIndex(Conf, Index, isPrevailing, recordNewLinkage,
 584                                   GUIDPreservedSymbols);
 585 }
 586
 587 // Initialize the TargetMachine builder for a given Triple
 588 static void initTMBuilder(TargetMachineBuilder &TMBuilder,
 589                           const Triple &TheTriple) {
 590   // Set a default CPU for Darwin triples (copied from LTOCodeGenerator).
 591   // FIXME this looks pretty terrible...
 592   if (TMBuilder.MCpu.empty() && TheTriple.isOSDarwin()) {
 593     if (TheTriple.getArch() == llvm::Triple::x86_64)
 594       TMBuilder.MCpu = "core2";
 595     else if (TheTriple.getArch() == llvm::Triple::x86)
 596       TMBuilder.MCpu = "yonah";
 597     else if (TheTriple.getArch() == llvm::Triple::aarch64 ||
 598              TheTriple.getArch() == llvm::Triple::aarch64_32)
 599       TMBuilder.MCpu = "cyclone";
 600   }
 601   TMBuilder.TheTriple = std::move(TheTriple);
 602 }
 603
 604 } // end anonymous namespace
 605
 606 void ThinLTOCodeGenerator::addModule(StringRef Identifier, StringRef Data) {
 607   MemoryBufferRef Buffer(Data, Identifier);
 608
 609   auto InputOrError = lto::InputFile::create(Buffer);
 610   if (!InputOrError)
 611     report_fatal_error("ThinLTO cannot create input file: " +
 612                        toString(InputOrError.takeError()));
 613
 614   auto TripleStr = (*InputOrError)->getTargetTriple();
 615   Triple TheTriple(TripleStr);
 616
 617   if (Modules.empty())
 618     initTMBuilder(TMBuilder, Triple(TheTriple));
 619   else if (TMBuilder.TheTriple != TheTriple) {
 620     if (!TMBuilder.TheTriple.isCompatibleWith(TheTriple))
 621       report_fatal_error("ThinLTO modules with incompatible triples not "
 622                          "supported");
 623     initTMBuilder(TMBuilder, Triple(TMBuilder.TheTriple.merge(TheTriple)));
 624   }
 625
 626   Modules.emplace_back(std::move(*InputOrError));
 627 }
 628
 629 void ThinLTOCodeGenerator::preserveSymbol(StringRef Name) {
 630   PreservedSymbols.insert(Name);
 631 }
 632
 633 void ThinLTOCodeGenerator::crossReferenceSymbol(StringRef Name) {
 634   // FIXME: At the moment, we don't take advantage of this extra information,
 635   // we're conservatively considering cross-references as preserved.
 636   //  CrossReferencedSymbols.insert(Name);
 637   PreservedSymbols.insert(Name);
 638 }
 639
 640 // TargetMachine factory
 641 std::unique_ptr<TargetMachine> TargetMachineBuilder::create() const {
 642   std::string ErrMsg;
 643   const Target *TheTarget =
 644       TargetRegistry::lookupTarget(TheTriple.str(), ErrMsg);
 645   if (!TheTarget) {
 646     report_fatal_error("Can't load target for this Triple: " + ErrMsg);
 647   }
 648
 649   // Use MAttr as the default set of features.
 650   SubtargetFeatures Features(MAttr);
 651   Features.getDefaultSubtargetFeatures(TheTriple);
 652   std::string FeatureStr = Features.getString();
 653
 654   std::unique_ptr<TargetMachine> TM(
 655       TheTarget->createTargetMachine(TheTriple.str(), MCpu, FeatureStr, Options,
 656                                      RelocModel, None, CGOptLevel));
 657   assert(TM && "Cannot create target machine");
 658
 659   return TM;
 660 }
 661
 662 /**
 663  * Produce the combined summary index from all the bitcode files:
 664  * "thin-link".
 665  */
 666 std::unique_ptr<ModuleSummaryIndex> ThinLTOCodeGenerator::linkCombinedIndex() {
 667   std::unique_ptr<ModuleSummaryIndex> CombinedIndex =
 668       std::make_unique<ModuleSummaryIndex>(/*HaveGVs=*/false);
 669   uint64_t NextModuleId = 0;
 670   for (auto &Mod : Modules) {
 671     auto &M = Mod->getSingleBitcodeModule();
 672     if (Error Err =
 673             M.readSummary(*CombinedIndex, Mod->getName(), NextModuleId++)) {
 674       // FIXME diagnose
 675       logAllUnhandledErrors(
 676           std::move(Err), errs(),
 677           "error: can't create module summary index for buffer: ");
 678       return nullptr;
 679     }
 680   }
 681   return CombinedIndex;
 682 }
 683
 684 namespace {
 685 struct IsExported {
 686   const StringMap<FunctionImporter::ExportSetTy> &ExportLists;
 687   const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols;
 688
 689   IsExported(const StringMap<FunctionImporter::ExportSetTy> &ExportLists,
 690              const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols)
 691       : ExportLists(ExportLists), GUIDPreservedSymbols(GUIDPreservedSymbols) {}
 692
 693   bool operator()(StringRef ModuleIdentifier, ValueInfo VI) const {
 694     const auto &ExportList = ExportLists.find(ModuleIdentifier);
 695     return (ExportList != ExportLists.end() && ExportList->second.count(VI)) ||
 696            GUIDPreservedSymbols.count(VI.getGUID());
 697   }
 698 };
 699
 700 struct IsPrevailing {
 701   const DenseMap<GlobalValue::GUID, const GlobalValueSummary *> &PrevailingCopy;
 702   IsPrevailing(const DenseMap<GlobalValue::GUID, const GlobalValueSummary *>
 703                    &PrevailingCopy)
 704       : PrevailingCopy(PrevailingCopy) {}
 705
 706   bool operator()(GlobalValue::GUID GUID, const GlobalValueSummary *S) const {
 707     const auto &Prevailing = PrevailingCopy.find(GUID);
 708     // Not in map means that there was only one copy, which must be prevailing.
 709     if (Prevailing == PrevailingCopy.end())
 710       return true;
 711     return Prevailing->second == S;
 712   };
 713 };
 714 } // namespace
 715
 716 static void computeDeadSymbolsInIndex(
 717     ModuleSummaryIndex &Index,
 718     const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) {
 719   // We have no symbols resolution available. And can't do any better now in the
 720   // case where the prevailing symbol is in a native object. It can be refined
 721   // with linker information in the future.
 722   auto isPrevailing = [&](GlobalValue::GUID G) {
 723     return PrevailingType::Unknown;
 724   };
 725   computeDeadSymbolsWithConstProp(Index, GUIDPreservedSymbols, isPrevailing,
 726                                   /* ImportEnabled = */ true);
 727 }
 728
 729 /**
 730  * Perform promotion and renaming of exported internal functions.
 731  * Index is updated to reflect linkage changes from weak resolution.
 732  */
 733 void ThinLTOCodeGenerator::promote(Module &TheModule, ModuleSummaryIndex &Index,
 734                                    const lto::InputFile &File) {
 735   auto ModuleCount = Index.modulePaths().size();
 736   auto ModuleIdentifier = TheModule.getModuleIdentifier();
 737
 738   // Collect for each module the list of function it defines (GUID -> Summary).
 739   StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries;
 740   Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
 741
 742   // Convert the preserved symbols set from string to GUID
 743   auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
 744       File, PreservedSymbols, Triple(TheModule.getTargetTriple()));
 745
 746   // Add used symbol to the preserved symbols.
 747   addUsedSymbolToPreservedGUID(File, GUIDPreservedSymbols);
 748
 749   // Compute "dead" symbols, we don't want to import/export these!
 750   computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
 751
 752   // Generate import/export list
 753   StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
 754   StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
 755   ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
 756                            ExportLists);
 757
 758   DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
 759   computePrevailingCopies(Index, PrevailingCopy);
 760
 761   // Resolve prevailing symbols
 762   StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
 763   resolvePrevailingInIndex(Index, ResolvedODR, GUIDPreservedSymbols,
 764                            PrevailingCopy);
 765
 766   thinLTOResolvePrevailingInModule(
 767       TheModule, ModuleToDefinedGVSummaries[ModuleIdentifier]);
 768
 769   // Promote the exported values in the index, so that they are promoted
 770   // in the module.
 771   thinLTOInternalizeAndPromoteInIndex(
 772       Index, IsExported(ExportLists, GUIDPreservedSymbols),
 773       IsPrevailing(PrevailingCopy));
 774
 775   // FIXME Set ClearDSOLocalOnDeclarations.
 776   promoteModule(TheModule, Index, /*ClearDSOLocalOnDeclarations=*/false);
 777 }
 778
 779 /**
 780  * Perform cross-module importing for the module identified by ModuleIdentifier.
 781  */
 782 void ThinLTOCodeGenerator::crossModuleImport(Module &TheModule,
 783                                              ModuleSummaryIndex &Index,
 784                                              const lto::InputFile &File) {
 785   auto ModuleMap = generateModuleMap(Modules);
 786   auto ModuleCount = Index.modulePaths().size();
 787
 788   // Collect for each module the list of function it defines (GUID -> Summary).
 789   StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
 790   Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
 791
 792   // Convert the preserved symbols set from string to GUID
 793   auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
 794       File, PreservedSymbols, Triple(TheModule.getTargetTriple()));
 795
 796   addUsedSymbolToPreservedGUID(File, GUIDPreservedSymbols);
 797
 798   // Compute "dead" symbols, we don't want to import/export these!
 799   computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
 800
 801   // Generate import/export list
 802   StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
 803   StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
 804   ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
 805                            ExportLists);
 806   auto &ImportList = ImportLists[TheModule.getModuleIdentifier()];
 807
 808   // FIXME Set ClearDSOLocalOnDeclarations.
 809   crossImportIntoModule(TheModule, Index, ModuleMap, ImportList,
 810                         /*ClearDSOLocalOnDeclarations=*/false);
 811 }
 812
 813 /**
 814  * Compute the list of summaries needed for importing into module.
 815  */
 816 void ThinLTOCodeGenerator::gatherImportedSummariesForModule(
 817     Module &TheModule, ModuleSummaryIndex &Index,
 818     std::map<std::string, GVSummaryMapTy> &ModuleToSummariesForIndex,
 819     const lto::InputFile &File) {
 820   auto ModuleCount = Index.modulePaths().size();
 821   auto ModuleIdentifier = TheModule.getModuleIdentifier();
 822
 823   // Collect for each module the list of function it defines (GUID -> Summary).
 824   StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
 825   Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
 826
 827   // Convert the preserved symbols set from string to GUID
 828   auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
 829       File, PreservedSymbols, Triple(TheModule.getTargetTriple()));
 830
 831   addUsedSymbolToPreservedGUID(File, GUIDPreservedSymbols);
 832
 833   // Compute "dead" symbols, we don't want to import/export these!
 834   computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
 835
 836   // Generate import/export list
 837   StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
 838   StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
 839   ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
 840                            ExportLists);
 841
 842   llvm::gatherImportedSummariesForModule(
 843       ModuleIdentifier, ModuleToDefinedGVSummaries,
 844       ImportLists[ModuleIdentifier], ModuleToSummariesForIndex);
 845 }
 846
 847 /**
 848  * Emit the list of files needed for importing into module.
 849  */
 850 void ThinLTOCodeGenerator::emitImports(Module &TheModule, StringRef OutputName,
 851                                        ModuleSummaryIndex &Index,
 852                                        const lto::InputFile &File) {
 853   auto ModuleCount = Index.modulePaths().size();
 854   auto ModuleIdentifier = TheModule.getModuleIdentifier();
 855
 856   // Collect for each module the list of function it defines (GUID -> Summary).
 857   StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
 858   Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
 859
 860   // Convert the preserved symbols set from string to GUID
 861   auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
 862       File, PreservedSymbols, Triple(TheModule.getTargetTriple()));
 863
 864   addUsedSymbolToPreservedGUID(File, GUIDPreservedSymbols);
 865
 866   // Compute "dead" symbols, we don't want to import/export these!
 867   computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
 868
 869   // Generate import/export list
 870   StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
 871   StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
 872   ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
 873                            ExportLists);
 874
 875   std::map<std::string, GVSummaryMapTy> ModuleToSummariesForIndex;
 876   llvm::gatherImportedSummariesForModule(
 877       ModuleIdentifier, ModuleToDefinedGVSummaries,
 878       ImportLists[ModuleIdentifier], ModuleToSummariesForIndex);
 879
 880   std::error_code EC;
 881   if ((EC = EmitImportsFiles(ModuleIdentifier, OutputName,
 882                              ModuleToSummariesForIndex)))
 883     report_fatal_error(Twine("Failed to open ") + OutputName +
 884                        " to save imports lists\n");
 885 }
 886
 887 /**
 888  * Perform internalization. Runs promote and internalization together.
 889  * Index is updated to reflect linkage changes.
 890  */
 891 void ThinLTOCodeGenerator::internalize(Module &TheModule,
 892                                        ModuleSummaryIndex &Index,
 893                                        const lto::InputFile &File) {
 894   initTMBuilder(TMBuilder, Triple(TheModule.getTargetTriple()));
 895   auto ModuleCount = Index.modulePaths().size();
 896   auto ModuleIdentifier = TheModule.getModuleIdentifier();
 897
 898   // Convert the preserved symbols set from string to GUID
 899   auto GUIDPreservedSymbols =
 900       computeGUIDPreservedSymbols(File, PreservedSymbols, TMBuilder.TheTriple);
 901
 902   addUsedSymbolToPreservedGUID(File, GUIDPreservedSymbols);
 903
 904   // Collect for each module the list of function it defines (GUID -> Summary).
 905   StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
 906   Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
 907
 908   // Compute "dead" symbols, we don't want to import/export these!
 909   computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
 910
 911   // Generate import/export list
 912   StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
 913   StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
 914   ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
 915                            ExportLists);
 916   auto &ExportList = ExportLists[ModuleIdentifier];
 917
 918   // Be friendly and don't nuke totally the module when the client didn't
 919   // supply anything to preserve.
 920   if (ExportList.empty() && GUIDPreservedSymbols.empty())
 921     return;
 922
 923   DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
 924   computePrevailingCopies(Index, PrevailingCopy);
 925
 926   // Resolve prevailing symbols
 927   StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
 928   resolvePrevailingInIndex(Index, ResolvedODR, GUIDPreservedSymbols,
 929                            PrevailingCopy);
 930
 931   // Promote the exported values in the index, so that they are promoted
 932   // in the module.
 933   thinLTOInternalizeAndPromoteInIndex(
 934       Index, IsExported(ExportLists, GUIDPreservedSymbols),
 935       IsPrevailing(PrevailingCopy));
 936
 937   // FIXME Set ClearDSOLocalOnDeclarations.
 938   promoteModule(TheModule, Index, /*ClearDSOLocalOnDeclarations=*/false);
 939
 940   // Internalization
 941   thinLTOResolvePrevailingInModule(
 942       TheModule, ModuleToDefinedGVSummaries[ModuleIdentifier]);
 943
 944   thinLTOInternalizeModule(TheModule,
 945                            ModuleToDefinedGVSummaries[ModuleIdentifier]);
 946 }
 947
 948 /**
 949  * Perform post-importing ThinLTO optimizations.
 950  */
 951 void ThinLTOCodeGenerator::optimize(Module &TheModule) {
 952   initTMBuilder(TMBuilder, Triple(TheModule.getTargetTriple()));
 953
 954   // Optimize now
 955   optimizeModule(TheModule, *TMBuilder.create(), OptLevel, Freestanding,
 956                  nullptr);
 957 }
 958
 959 /// Write out the generated object file, either from CacheEntryPath or from
 960 /// OutputBuffer, preferring hard-link when possible.
 961 /// Returns the path to the generated file in SavedObjectsDirectoryPath.
 962 std::string
 963 ThinLTOCodeGenerator::writeGeneratedObject(int count, StringRef CacheEntryPath,
 964                                            const MemoryBuffer &OutputBuffer) {
 965   auto ArchName = TMBuilder.TheTriple.getArchName();
 966   SmallString<128> OutputPath(SavedObjectsDirectoryPath);
 967   llvm::sys::path::append(OutputPath,
 968                           Twine(count) + "." + ArchName + ".thinlto.o");
 969   OutputPath.c_str(); // Ensure the string is null terminated.
 970   if (sys::fs::exists(OutputPath))
 971     sys::fs::remove(OutputPath);
 972
 973   // We don't return a memory buffer to the linker, just a list of files.
 974   if (!CacheEntryPath.empty()) {
 975     // Cache is enabled, hard-link the entry (or copy if hard-link fails).
 976     auto Err = sys::fs::create_hard_link(CacheEntryPath, OutputPath);
 977     if (!Err)
 978       return std::string(OutputPath.str());
 979     // Hard linking failed, try to copy.
 980     Err = sys::fs::copy_file(CacheEntryPath, OutputPath);
 981     if (!Err)
 982       return std::string(OutputPath.str());
 983     // Copy failed (could be because the CacheEntry was removed from the cache
 984     // in the meantime by another process), fall back and try to write down the
 985     // buffer to the output.
 986     errs() << "remark: can't link or copy from cached entry '" << CacheEntryPath
 987            << "' to '" << OutputPath << "'\n";
 988   }
 989   // No cache entry, just write out the buffer.
 990   std::error_code Err;
 991   raw_fd_ostream OS(OutputPath, Err, sys::fs::OF_None);
 992   if (Err)
 993     report_fatal_error("Can't open output '" + OutputPath + "'\n");
 994   OS << OutputBuffer.getBuffer();
 995   return std::string(OutputPath.str());
 996 }
 997
 998 // Main entry point for the ThinLTO processing
 999 void ThinLTOCodeGenerator::run() {
1000   timeTraceProfilerBegin("ThinLink", StringRef(""));
1001   auto TimeTraceScopeExit = llvm::make_scope_exit([]() {
1002     if (llvm::timeTraceProfilerEnabled())
1003       llvm::timeTraceProfilerEnd();
1004   });
1005   // Prepare the resulting object vector
1006   assert(ProducedBinaries.empty() && "The generator should not be reused");
1007   if (SavedObjectsDirectoryPath.empty())
1008     ProducedBinaries.resize(Modules.size());
1009   else {
1010     sys::fs::create_directories(SavedObjectsDirectoryPath);
1011     bool IsDir;
1012     sys::fs::is_directory(SavedObjectsDirectoryPath, IsDir);
1013     if (!IsDir)
1014       report_fatal_error("Unexistent dir: '" + SavedObjectsDirectoryPath + "'");
1015     ProducedBinaryFiles.resize(Modules.size());
1016   }
1017
1018   if (CodeGenOnly) {
1019     // Perform only parallel codegen and return.
1020     ThreadPool Pool;
1021     int count = 0;
1022     for (auto &Mod : Modules) {
1023       Pool.async([&](int count) {
1024         LLVMContext Context;
1025         Context.setDiscardValueNames(LTODiscardValueNames);
1026
1027         // Parse module now
1028         auto TheModule = loadModuleFromInput(Mod.get(), Context, false,
1029                                              /*IsImporting*/ false);
1030
1031         // CodeGen
1032         auto OutputBuffer = codegenModule(*TheModule, *TMBuilder.create());
1033         if (SavedObjectsDirectoryPath.empty())
1034           ProducedBinaries[count] = std::move(OutputBuffer);
1035         else
1036           ProducedBinaryFiles[count] =
1037               writeGeneratedObject(count, "", *OutputBuffer);
1038       }, count++);
1039     }
1040
1041     return;
1042   }
1043
1044   // Sequential linking phase
1045   auto Index = linkCombinedIndex();
1046
1047   // Save temps: index.
1048   if (!SaveTempsDir.empty()) {
1049     auto SaveTempPath = SaveTempsDir + "index.bc";
1050     std::error_code EC;
1051     raw_fd_ostream OS(SaveTempPath, EC, sys::fs::OF_None);
1052     if (EC)
1053       report_fatal_error(Twine("Failed to open ") + SaveTempPath +
1054                          " to save optimized bitcode\n");
1055     WriteIndexToFile(*Index, OS);
1056   }
1057
1058
1059   // Prepare the module map.
1060   auto ModuleMap = generateModuleMap(Modules);
1061   auto ModuleCount = Modules.size();
1062
1063   // Collect for each module the list of function it defines (GUID -> Summary).
1064   StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
1065   Index->collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
1066
1067   // Convert the preserved symbols set from string to GUID, this is needed for
1068   // computing the caching hash and the internalization.
1069   DenseSet<GlobalValue::GUID> GUIDPreservedSymbols;
1070   for (const auto &M : Modules)
1071     computeGUIDPreservedSymbols(*M, PreservedSymbols, TMBuilder.TheTriple,
1072                                 GUIDPreservedSymbols);
1073
1074   // Add used symbol from inputs to the preserved symbols.
1075   for (const auto &M : Modules)
1076     addUsedSymbolToPreservedGUID(*M, GUIDPreservedSymbols);
1077
1078   // Compute "dead" symbols, we don't want to import/export these!
1079   computeDeadSymbolsInIndex(*Index, GUIDPreservedSymbols);
1080
1081   // Synthesize entry counts for functions in the combined index.
1082   computeSyntheticCounts(*Index);
1083
1084   // Currently there is no support for enabling whole program visibility via a
1085   // linker option in the old LTO API, but this call allows it to be specified
1086   // via the internal option. Must be done before WPD below.
1087   updateVCallVisibilityInIndex(*Index,
1088                                /* WholeProgramVisibilityEnabledInLTO */ false,
1089                                // FIXME: This needs linker information via a
1090                                // TBD new interface.
1091                                /* DynamicExportSymbols */ {});
1092
1093   // Perform index-based WPD. This will return immediately if there are
1094   // no index entries in the typeIdMetadata map (e.g. if we are instead
1095   // performing IR-based WPD in hybrid regular/thin LTO mode).
1096   std::map<ValueInfo, std::vector<VTableSlotSummary>> LocalWPDTargetsMap;
1097   std::set<GlobalValue::GUID> ExportedGUIDs;
1098   runWholeProgramDevirtOnIndex(*Index, ExportedGUIDs, LocalWPDTargetsMap);
1099   for (auto GUID : ExportedGUIDs)
1100     GUIDPreservedSymbols.insert(GUID);
1101
1102   // Collect the import/export lists for all modules from the call-graph in the
1103   // combined index.
1104   StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
1105   StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
1106   ComputeCrossModuleImport(*Index, ModuleToDefinedGVSummaries, ImportLists,
1107                            ExportLists);
1108
1109   // We use a std::map here to be able to have a defined ordering when
1110   // producing a hash for the cache entry.
1111   // FIXME: we should be able to compute the caching hash for the entry based
1112   // on the index, and nuke this map.
1113   StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
1114
1115   DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
1116   computePrevailingCopies(*Index, PrevailingCopy);
1117
1118   // Resolve prevailing symbols, this has to be computed early because it
1119   // impacts the caching.
1120   resolvePrevailingInIndex(*Index, ResolvedODR, GUIDPreservedSymbols,
1121                            PrevailingCopy);
1122
1123   // Use global summary-based analysis to identify symbols that can be
1124   // internalized (because they aren't exported or preserved as per callback).
1125   // Changes are made in the index, consumed in the ThinLTO backends.
1126   updateIndexWPDForExports(*Index,
1127                            IsExported(ExportLists, GUIDPreservedSymbols),
1128                            LocalWPDTargetsMap);
1129   thinLTOInternalizeAndPromoteInIndex(
1130       *Index, IsExported(ExportLists, GUIDPreservedSymbols),
1131       IsPrevailing(PrevailingCopy));
1132
1133   // Make sure that every module has an entry in the ExportLists, ImportList,
1134   // GVSummary and ResolvedODR maps to enable threaded access to these maps
1135   // below.
1136   for (auto &Module : Modules) {
1137     auto ModuleIdentifier = Module->getName();
1138     ExportLists[ModuleIdentifier];
1139     ImportLists[ModuleIdentifier];
1140     ResolvedODR[ModuleIdentifier];
1141     ModuleToDefinedGVSummaries[ModuleIdentifier];
1142   }
1143
1144   std::vector<BitcodeModule *> ModulesVec;
1145   ModulesVec.reserve(Modules.size());
1146   for (auto &Mod : Modules)
1147     ModulesVec.push_back(&Mod->getSingleBitcodeModule());
1148   std::vector<int> ModulesOrdering = lto::generateModulesOrdering(ModulesVec);
1149
1150   if (llvm::timeTraceProfilerEnabled())
1151     llvm::timeTraceProfilerEnd();
1152
1153   TimeTraceScopeExit.release();
1154
1155   // Parallel optimizer + codegen
1156   {
1157     ThreadPool Pool(heavyweight_hardware_concurrency(ThreadCount));
1158     for (auto IndexCount : ModulesOrdering) {
1159       auto &Mod = Modules[IndexCount];
1160       Pool.async([&](int count) {
1161         auto ModuleIdentifier = Mod->getName();
1162         auto &ExportList = ExportLists[ModuleIdentifier];
1163
1164         auto &DefinedGVSummaries = ModuleToDefinedGVSummaries[ModuleIdentifier];
1165
1166         // The module may be cached, this helps handling it.
1167         ModuleCacheEntry CacheEntry(CacheOptions.Path, *Index, ModuleIdentifier,
1168                                     ImportLists[ModuleIdentifier], ExportList,
1169                                     ResolvedODR[ModuleIdentifier],
1170                                     DefinedGVSummaries, OptLevel, Freestanding,
1171                                     TMBuilder);
1172         auto CacheEntryPath = CacheEntry.getEntryPath();
1173
1174         {
1175           auto ErrOrBuffer = CacheEntry.tryLoadingBuffer();
1176           LLVM_DEBUG(dbgs() << "Cache " << (ErrOrBuffer ? "hit" : "miss")
1177                             << " '" << CacheEntryPath << "' for buffer "
1178                             << count << " " << ModuleIdentifier << "\n");
1179
1180           if (ErrOrBuffer) {
1181             // Cache Hit!
1182             if (SavedObjectsDirectoryPath.empty())
1183               ProducedBinaries[count] = std::move(ErrOrBuffer.get());
1184             else
1185               ProducedBinaryFiles[count] = writeGeneratedObject(
1186                   count, CacheEntryPath, *ErrOrBuffer.get());
1187             return;
1188           }
1189         }
1190
1191         LLVMContext Context;
1192         Context.setDiscardValueNames(LTODiscardValueNames);
1193         Context.enableDebugTypeODRUniquing();
1194         auto DiagFileOrErr = lto::setupLLVMOptimizationRemarks(
1195             Context, RemarksFilename, RemarksPasses, RemarksFormat,
1196             RemarksWithHotness, RemarksHotnessThreshold, count);
1197         if (!DiagFileOrErr) {
1198           errs() << "Error: " << toString(DiagFileOrErr.takeError()) << "\n";
1199           report_fatal_error("ThinLTO: Can't get an output file for the "
1200                              "remarks");
1201         }
1202
1203         // Parse module now
1204         auto TheModule = loadModuleFromInput(Mod.get(), Context, false,
1205                                              /*IsImporting*/ false);
1206
1207         // Save temps: original file.
1208         saveTempBitcode(*TheModule, SaveTempsDir, count, ".0.original.bc");
1209
1210         auto &ImportList = ImportLists[ModuleIdentifier];
1211         // Run the main process now, and generates a binary
1212         auto OutputBuffer = ProcessThinLTOModule(
1213             *TheModule, *Index, ModuleMap, *TMBuilder.create(), ImportList,
1214             ExportList, GUIDPreservedSymbols,
1215             ModuleToDefinedGVSummaries[ModuleIdentifier], CacheOptions,
1216             DisableCodeGen, SaveTempsDir, Freestanding, OptLevel, count,
1217             UseNewPM, DebugPassManager);
1218
1219         // Commit to the cache (if enabled)
1220         CacheEntry.write(*OutputBuffer);
1221
1222         if (SavedObjectsDirectoryPath.empty()) {
1223           // We need to generated a memory buffer for the linker.
1224           if (!CacheEntryPath.empty()) {
1225             // When cache is enabled, reload from the cache if possible.
1226             // Releasing the buffer from the heap and reloading it from the
1227             // cache file with mmap helps us to lower memory pressure.
1228             // The freed memory can be used for the next input file.
1229             // The final binary link will read from the VFS cache (hopefully!)
1230             // or from disk (if the memory pressure was too high).
1231             auto ReloadedBufferOrErr = CacheEntry.tryLoadingBuffer();
1232             if (auto EC = ReloadedBufferOrErr.getError()) {
1233               // On error, keep the preexisting buffer and print a diagnostic.
1234               errs() << "remark: can't reload cached file '" << CacheEntryPath
1235                      << "': " << EC.message() << "\n";
1236             } else {
1237               OutputBuffer = std::move(*ReloadedBufferOrErr);
1238             }
1239           }
1240           ProducedBinaries[count] = std::move(OutputBuffer);
1241           return;
1242         }
1243         ProducedBinaryFiles[count] = writeGeneratedObject(
1244             count, CacheEntryPath, *OutputBuffer);
1245       }, IndexCount);
1246     }
1247   }
1248
1249   pruneCache(CacheOptions.Path, CacheOptions.Policy);
1250
1251   // If statistics were requested, print them out now.
1252   if (llvm::AreStatisticsEnabled())
1253     llvm::PrintStatistics();
1254   reportAndResetTimings();
1255 }