lib/LTO/ThinLTOCodeGenerator.cpp

   1 //===-ThinLTOCodeGenerator.cpp - LLVM Link Time Optimizer -----------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file implements the Thin Link Time Optimization library. This library is
  10 // intended to be used by linker to optimize code at link time.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "llvm/LTO/legacy/ThinLTOCodeGenerator.h"
  15
  16 #include "llvm/ADT/Statistic.h"
  17 #include "llvm/ADT/StringExtras.h"
  18 #include "llvm/Analysis/ModuleSummaryAnalysis.h"
  19 #include "llvm/Analysis/ProfileSummaryInfo.h"
  20 #include "llvm/Analysis/TargetLibraryInfo.h"
  21 #include "llvm/Analysis/TargetTransformInfo.h"
  22 #include "llvm/Bitcode/BitcodeReader.h"
  23 #include "llvm/Bitcode/BitcodeWriter.h"
  24 #include "llvm/Bitcode/BitcodeWriterPass.h"
  25 #include "llvm/Config/llvm-config.h"
  26 #include "llvm/IR/DebugInfo.h"
  27 #include "llvm/IR/DiagnosticPrinter.h"
  28 #include "llvm/IR/LLVMContext.h"
  29 #include "llvm/IR/LegacyPassManager.h"
  30 #include "llvm/IR/Mangler.h"
  31 #include "llvm/IR/PassTimingInfo.h"
  32 #include "llvm/IR/RemarkStreamer.h"
  33 #include "llvm/IR/Verifier.h"
  34 #include "llvm/IRReader/IRReader.h"
  35 #include "llvm/LTO/LTO.h"
  36 #include "llvm/LTO/SummaryBasedOptimizations.h"
  37 #include "llvm/MC/SubtargetFeature.h"
  38 #include "llvm/Object/IRObjectFile.h"
  39 #include "llvm/Support/CachePruning.h"
  40 #include "llvm/Support/Debug.h"
  41 #include "llvm/Support/Error.h"
  42 #include "llvm/Support/Path.h"
  43 #include "llvm/Support/SHA1.h"
  44 #include "llvm/Support/SmallVectorMemoryBuffer.h"
  45 #include "llvm/Support/TargetRegistry.h"
  46 #include "llvm/Support/ThreadPool.h"
  47 #include "llvm/Support/Threading.h"
  48 #include "llvm/Support/ToolOutputFile.h"
  49 #include "llvm/Support/VCSRevision.h"
  50 #include "llvm/Target/TargetMachine.h"
  51 #include "llvm/Transforms/IPO.h"
  52 #include "llvm/Transforms/IPO/FunctionImport.h"
  53 #include "llvm/Transforms/IPO/Internalize.h"
  54 #include "llvm/Transforms/IPO/PassManagerBuilder.h"
  55 #include "llvm/Transforms/ObjCARC.h"
  56 #include "llvm/Transforms/Utils/FunctionImportUtils.h"
  57
  58 #include <numeric>
  59
  60 #if !defined(_MSC_VER) && !defined(__MINGW32__)
  61 #include <unistd.h>
  62 #else
  63 #include <io.h>
  64 #endif
  65
  66 using namespace llvm;
  67
  68 #define DEBUG_TYPE "thinlto"
  69
  70 namespace llvm {
  71 // Flags -discard-value-names, defined in LTOCodeGenerator.cpp
  72 extern cl::opt<bool> LTODiscardValueNames;
  73 extern cl::opt<std::string> RemarksFilename;
  74 extern cl::opt<std::string> RemarksPasses;
  75 extern cl::opt<bool> RemarksWithHotness;
  76 extern cl::opt<std::string> RemarksFormat;
  77 }
  78
  79 namespace {
  80
  81 static cl::opt<int>
  82     ThreadCount("threads", cl::init(llvm::heavyweight_hardware_concurrency()));
  83
  84 // Simple helper to save temporary files for debug.
  85 static void saveTempBitcode(const Module &TheModule, StringRef TempDir,
  86                             unsigned count, StringRef Suffix) {
  87   if (TempDir.empty())
  88     return;
  89   // User asked to save temps, let dump the bitcode file after import.
  90   std::string SaveTempPath = (TempDir + llvm::Twine(count) + Suffix).str();
  91   std::error_code EC;
  92   raw_fd_ostream OS(SaveTempPath, EC, sys::fs::OF_None);
  93   if (EC)
  94     report_fatal_error(Twine("Failed to open ") + SaveTempPath +
  95                        " to save optimized bitcode\n");
  96   WriteBitcodeToFile(TheModule, OS, /* ShouldPreserveUseListOrder */ true);
  97 }
  98
  99 static const GlobalValueSummary *
 100 getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) {
 101   // If there is any strong definition anywhere, get it.
 102   auto StrongDefForLinker = llvm::find_if(
 103       GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
 104         auto Linkage = Summary->linkage();
 105         return !GlobalValue::isAvailableExternallyLinkage(Linkage) &&
 106                !GlobalValue::isWeakForLinker(Linkage);
 107       });
 108   if (StrongDefForLinker != GVSummaryList.end())
 109     return StrongDefForLinker->get();
 110   // Get the first *linker visible* definition for this global in the summary
 111   // list.
 112   auto FirstDefForLinker = llvm::find_if(
 113       GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
 114         auto Linkage = Summary->linkage();
 115         return !GlobalValue::isAvailableExternallyLinkage(Linkage);
 116       });
 117   // Extern templates can be emitted as available_externally.
 118   if (FirstDefForLinker == GVSummaryList.end())
 119     return nullptr;
 120   return FirstDefForLinker->get();
 121 }
 122
 123 // Populate map of GUID to the prevailing copy for any multiply defined
 124 // symbols. Currently assume first copy is prevailing, or any strong
 125 // definition. Can be refined with Linker information in the future.
 126 static void computePrevailingCopies(
 127     const ModuleSummaryIndex &Index,
 128     DenseMap<GlobalValue::GUID, const GlobalValueSummary *> &PrevailingCopy) {
 129   auto HasMultipleCopies = [&](const GlobalValueSummaryList &GVSummaryList) {
 130     return GVSummaryList.size() > 1;
 131   };
 132
 133   for (auto &I : Index) {
 134     if (HasMultipleCopies(I.second.SummaryList))
 135       PrevailingCopy[I.first] =
 136           getFirstDefinitionForLinker(I.second.SummaryList);
 137   }
 138 }
 139
 140 static StringMap<lto::InputFile *>
 141 generateModuleMap(std::vector<std::unique_ptr<lto::InputFile>> &Modules) {
 142   StringMap<lto::InputFile *> ModuleMap;
 143   for (auto &M : Modules) {
 144     assert(ModuleMap.find(M->getName()) == ModuleMap.end() &&
 145            "Expect unique Buffer Identifier");
 146     ModuleMap[M->getName()] = M.get();
 147   }
 148   return ModuleMap;
 149 }
 150
 151 static void promoteModule(Module &TheModule, const ModuleSummaryIndex &Index) {
 152   if (renameModuleForThinLTO(TheModule, Index))
 153     report_fatal_error("renameModuleForThinLTO failed");
 154 }
 155
 156 namespace {
 157 class ThinLTODiagnosticInfo : public DiagnosticInfo {
 158   const Twine &Msg;
 159 public:
 160   ThinLTODiagnosticInfo(const Twine &DiagMsg,
 161                         DiagnosticSeverity Severity = DS_Error)
 162       : DiagnosticInfo(DK_Linker, Severity), Msg(DiagMsg) {}
 163   void print(DiagnosticPrinter &DP) const override { DP << Msg; }
 164 };
 165 }
 166
 167 /// Verify the module and strip broken debug info.
 168 static void verifyLoadedModule(Module &TheModule) {
 169   bool BrokenDebugInfo = false;
 170   if (verifyModule(TheModule, &dbgs(), &BrokenDebugInfo))
 171     report_fatal_error("Broken module found, compilation aborted!");
 172   if (BrokenDebugInfo) {
 173     TheModule.getContext().diagnose(ThinLTODiagnosticInfo(
 174         "Invalid debug info found, debug info will be stripped", DS_Warning));
 175     StripDebugInfo(TheModule);
 176   }
 177 }
 178
 179 static std::unique_ptr<Module> loadModuleFromInput(lto::InputFile *Input,
 180                                                    LLVMContext &Context,
 181                                                    bool Lazy,
 182                                                    bool IsImporting) {
 183   auto &Mod = Input->getSingleBitcodeModule();
 184   SMDiagnostic Err;
 185   Expected<std::unique_ptr<Module>> ModuleOrErr =
 186       Lazy ? Mod.getLazyModule(Context,
 187                                /* ShouldLazyLoadMetadata */ true, IsImporting)
 188            : Mod.parseModule(Context);
 189   if (!ModuleOrErr) {
 190     handleAllErrors(ModuleOrErr.takeError(), [&](ErrorInfoBase &EIB) {
 191       SMDiagnostic Err = SMDiagnostic(Mod.getModuleIdentifier(),
 192                                       SourceMgr::DK_Error, EIB.message());
 193       Err.print("ThinLTO", errs());
 194     });
 195     report_fatal_error("Can't load module, abort.");
 196   }
 197   if (!Lazy)
 198     verifyLoadedModule(*ModuleOrErr.get());
 199   return std::move(*ModuleOrErr);
 200 }
 201
 202 static void
 203 crossImportIntoModule(Module &TheModule, const ModuleSummaryIndex &Index,
 204                       StringMap<lto::InputFile*> &ModuleMap,
 205                       const FunctionImporter::ImportMapTy &ImportList) {
 206   auto Loader = [&](StringRef Identifier) {
 207     auto &Input = ModuleMap[Identifier];
 208     return loadModuleFromInput(Input, TheModule.getContext(),
 209                                /*Lazy=*/true, /*IsImporting*/ true);
 210   };
 211
 212   FunctionImporter Importer(Index, Loader);
 213   Expected<bool> Result = Importer.importFunctions(TheModule, ImportList);
 214   if (!Result) {
 215     handleAllErrors(Result.takeError(), [&](ErrorInfoBase &EIB) {
 216       SMDiagnostic Err = SMDiagnostic(TheModule.getModuleIdentifier(),
 217                                       SourceMgr::DK_Error, EIB.message());
 218       Err.print("ThinLTO", errs());
 219     });
 220     report_fatal_error("importFunctions failed");
 221   }
 222   // Verify again after cross-importing.
 223   verifyLoadedModule(TheModule);
 224 }
 225
 226 static void optimizeModule(Module &TheModule, TargetMachine &TM,
 227                            unsigned OptLevel, bool Freestanding) {
 228   // Populate the PassManager
 229   PassManagerBuilder PMB;
 230   PMB.LibraryInfo = new TargetLibraryInfoImpl(TM.getTargetTriple());
 231   if (Freestanding)
 232     PMB.LibraryInfo->disableAllFunctions();
 233   PMB.Inliner = createFunctionInliningPass();
 234   // FIXME: should get it from the bitcode?
 235   PMB.OptLevel = OptLevel;
 236   PMB.LoopVectorize = true;
 237   PMB.SLPVectorize = true;
 238   // Already did this in verifyLoadedModule().
 239   PMB.VerifyInput = false;
 240   PMB.VerifyOutput = false;
 241
 242   legacy::PassManager PM;
 243
 244   // Add the TTI (required to inform the vectorizer about register size for
 245   // instance)
 246   PM.add(createTargetTransformInfoWrapperPass(TM.getTargetIRAnalysis()));
 247
 248   // Add optimizations
 249   PMB.populateThinLTOPassManager(PM);
 250
 251   PM.run(TheModule);
 252 }
 253
 254 static void
 255 addUsedSymbolToPreservedGUID(const lto::InputFile &File,
 256                              DenseSet<GlobalValue::GUID> &PreservedGUID) {
 257   for (const auto &Sym : File.symbols()) {
 258     if (Sym.isUsed())
 259       PreservedGUID.insert(GlobalValue::getGUID(Sym.getIRName()));
 260   }
 261 }
 262
 263 // Convert the PreservedSymbols map from "Name" based to "GUID" based.
 264 static DenseSet<GlobalValue::GUID>
 265 computeGUIDPreservedSymbols(const StringSet<> &PreservedSymbols,
 266                             const Triple &TheTriple) {
 267   DenseSet<GlobalValue::GUID> GUIDPreservedSymbols(PreservedSymbols.size());
 268   for (auto &Entry : PreservedSymbols) {
 269     StringRef Name = Entry.first();
 270     if (TheTriple.isOSBinFormatMachO() && Name.size() > 0 && Name[0] == '_')
 271       Name = Name.drop_front();
 272     GUIDPreservedSymbols.insert(GlobalValue::getGUID(Name));
 273   }
 274   return GUIDPreservedSymbols;
 275 }
 276
 277 std::unique_ptr<MemoryBuffer> codegenModule(Module &TheModule,
 278                                             TargetMachine &TM) {
 279   SmallVector<char, 128> OutputBuffer;
 280
 281   // CodeGen
 282   {
 283     raw_svector_ostream OS(OutputBuffer);
 284     legacy::PassManager PM;
 285
 286     // If the bitcode files contain ARC code and were compiled with optimization,
 287     // the ObjCARCContractPass must be run, so do it unconditionally here.
 288     PM.add(createObjCARCContractPass());
 289
 290     // Setup the codegen now.
 291     if (TM.addPassesToEmitFile(PM, OS, nullptr, TargetMachine::CGFT_ObjectFile,
 292                                /* DisableVerify */ true))
 293       report_fatal_error("Failed to setup codegen");
 294
 295     // Run codegen now. resulting binary is in OutputBuffer.
 296     PM.run(TheModule);
 297   }
 298   return std::make_unique<SmallVectorMemoryBuffer>(std::move(OutputBuffer));
 299 }
 300
 301 /// Manage caching for a single Module.
 302 class ModuleCacheEntry {
 303   SmallString<128> EntryPath;
 304
 305 public:
 306   // Create a cache entry. This compute a unique hash for the Module considering
 307   // the current list of export/import, and offer an interface to query to
 308   // access the content in the cache.
 309   ModuleCacheEntry(
 310       StringRef CachePath, const ModuleSummaryIndex &Index, StringRef ModuleID,
 311       const FunctionImporter::ImportMapTy &ImportList,
 312       const FunctionImporter::ExportSetTy &ExportList,
 313       const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
 314       const GVSummaryMapTy &DefinedGVSummaries, unsigned OptLevel,
 315       bool Freestanding, const TargetMachineBuilder &TMBuilder) {
 316     if (CachePath.empty())
 317       return;
 318
 319     if (!Index.modulePaths().count(ModuleID))
 320       // The module does not have an entry, it can't have a hash at all
 321       return;
 322
 323     if (all_of(Index.getModuleHash(ModuleID),
 324                [](uint32_t V) { return V == 0; }))
 325       // No hash entry, no caching!
 326       return;
 327
 328     llvm::lto::Config Conf;
 329     Conf.OptLevel = OptLevel;
 330     Conf.Options = TMBuilder.Options;
 331     Conf.CPU = TMBuilder.MCpu;
 332     Conf.MAttrs.push_back(TMBuilder.MAttr);
 333     Conf.RelocModel = TMBuilder.RelocModel;
 334     Conf.CGOptLevel = TMBuilder.CGOptLevel;
 335     Conf.Freestanding = Freestanding;
 336     SmallString<40> Key;
 337     computeLTOCacheKey(Key, Conf, Index, ModuleID, ImportList, ExportList,
 338                        ResolvedODR, DefinedGVSummaries);
 339
 340     // This choice of file name allows the cache to be pruned (see pruneCache()
 341     // in include/llvm/Support/CachePruning.h).
 342     sys::path::append(EntryPath, CachePath, "llvmcache-" + Key);
 343   }
 344
 345   // Access the path to this entry in the cache.
 346   StringRef getEntryPath() { return EntryPath; }
 347
 348   // Try loading the buffer for this cache entry.
 349   ErrorOr<std::unique_ptr<MemoryBuffer>> tryLoadingBuffer() {
 350     if (EntryPath.empty())
 351       return std::error_code();
 352     SmallString<64> ResultPath;
 353     Expected<sys::fs::file_t> FDOrErr = sys::fs::openNativeFileForRead(
 354         Twine(EntryPath), sys::fs::OF_UpdateAtime, &ResultPath);
 355     if (!FDOrErr)
 356       return errorToErrorCode(FDOrErr.takeError());
 357     ErrorOr<std::unique_ptr<MemoryBuffer>> MBOrErr = MemoryBuffer::getOpenFile(
 358         *FDOrErr, EntryPath, /*FileSize=*/-1, /*RequiresNullTerminator=*/false);
 359     sys::fs::closeFile(*FDOrErr);
 360     return MBOrErr;
 361   }
 362
 363   // Cache the Produced object file
 364   void write(const MemoryBuffer &OutputBuffer) {
 365     if (EntryPath.empty())
 366       return;
 367
 368     // Write to a temporary to avoid race condition
 369     SmallString<128> TempFilename;
 370     SmallString<128> CachePath(EntryPath);
 371     int TempFD;
 372     llvm::sys::path::remove_filename(CachePath);
 373     sys::path::append(TempFilename, CachePath, "Thin-%%%%%%.tmp.o");
 374     std::error_code EC =
 375       sys::fs::createUniqueFile(TempFilename, TempFD, TempFilename);
 376     if (EC) {
 377       errs() << "Error: " << EC.message() << "\n";
 378       report_fatal_error("ThinLTO: Can't get a temporary file");
 379     }
 380     {
 381       raw_fd_ostream OS(TempFD, /* ShouldClose */ true);
 382       OS << OutputBuffer.getBuffer();
 383     }
 384     // Rename temp file to final destination; rename is atomic
 385     EC = sys::fs::rename(TempFilename, EntryPath);
 386     if (EC)
 387       sys::fs::remove(TempFilename);
 388   }
 389 };
 390
 391 static std::unique_ptr<MemoryBuffer>
 392 ProcessThinLTOModule(Module &TheModule, ModuleSummaryIndex &Index,
 393                      StringMap<lto::InputFile *> &ModuleMap, TargetMachine &TM,
 394                      const FunctionImporter::ImportMapTy &ImportList,
 395                      const FunctionImporter::ExportSetTy &ExportList,
 396                      const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
 397                      const GVSummaryMapTy &DefinedGlobals,
 398                      const ThinLTOCodeGenerator::CachingOptions &CacheOptions,
 399                      bool DisableCodeGen, StringRef SaveTempsDir,
 400                      bool Freestanding, unsigned OptLevel, unsigned count) {
 401
 402   // "Benchmark"-like optimization: single-source case
 403   bool SingleModule = (ModuleMap.size() == 1);
 404
 405   if (!SingleModule) {
 406     promoteModule(TheModule, Index);
 407
 408     // Apply summary-based prevailing-symbol resolution decisions.
 409     thinLTOResolvePrevailingInModule(TheModule, DefinedGlobals);
 410
 411     // Save temps: after promotion.
 412     saveTempBitcode(TheModule, SaveTempsDir, count, ".1.promoted.bc");
 413   }
 414
 415   // Be friendly and don't nuke totally the module when the client didn't
 416   // supply anything to preserve.
 417   if (!ExportList.empty() || !GUIDPreservedSymbols.empty()) {
 418     // Apply summary-based internalization decisions.
 419     thinLTOInternalizeModule(TheModule, DefinedGlobals);
 420   }
 421
 422   // Save internalized bitcode
 423   saveTempBitcode(TheModule, SaveTempsDir, count, ".2.internalized.bc");
 424
 425   if (!SingleModule) {
 426     crossImportIntoModule(TheModule, Index, ModuleMap, ImportList);
 427
 428     // Save temps: after cross-module import.
 429     saveTempBitcode(TheModule, SaveTempsDir, count, ".3.imported.bc");
 430   }
 431
 432   optimizeModule(TheModule, TM, OptLevel, Freestanding);
 433
 434   saveTempBitcode(TheModule, SaveTempsDir, count, ".4.opt.bc");
 435
 436   if (DisableCodeGen) {
 437     // Configured to stop before CodeGen, serialize the bitcode and return.
 438     SmallVector<char, 128> OutputBuffer;
 439     {
 440       raw_svector_ostream OS(OutputBuffer);
 441       ProfileSummaryInfo PSI(TheModule);
 442       auto Index = buildModuleSummaryIndex(TheModule, nullptr, &PSI);
 443       WriteBitcodeToFile(TheModule, OS, true, &Index);
 444     }
 445     return std::make_unique<SmallVectorMemoryBuffer>(std::move(OutputBuffer));
 446   }
 447
 448   return codegenModule(TheModule, TM);
 449 }
 450
 451 /// Resolve prevailing symbols. Record resolutions in the \p ResolvedODR map
 452 /// for caching, and in the \p Index for application during the ThinLTO
 453 /// backends. This is needed for correctness for exported symbols (ensure
 454 /// at least one copy kept) and a compile-time optimization (to drop duplicate
 455 /// copies when possible).
 456 static void resolvePrevailingInIndex(
 457     ModuleSummaryIndex &Index,
 458     StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>>
 459         &ResolvedODR,
 460     const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
 461     const DenseMap<GlobalValue::GUID, const GlobalValueSummary *>
 462         &PrevailingCopy) {
 463
 464   auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) {
 465     const auto &Prevailing = PrevailingCopy.find(GUID);
 466     // Not in map means that there was only one copy, which must be prevailing.
 467     if (Prevailing == PrevailingCopy.end())
 468       return true;
 469     return Prevailing->second == S;
 470   };
 471
 472   auto recordNewLinkage = [&](StringRef ModuleIdentifier,
 473                               GlobalValue::GUID GUID,
 474                               GlobalValue::LinkageTypes NewLinkage) {
 475     ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;
 476   };
 477
 478   thinLTOResolvePrevailingInIndex(Index, isPrevailing, recordNewLinkage,
 479                                   GUIDPreservedSymbols);
 480 }
 481
 482 // Initialize the TargetMachine builder for a given Triple
 483 static void initTMBuilder(TargetMachineBuilder &TMBuilder,
 484                           const Triple &TheTriple) {
 485   // Set a default CPU for Darwin triples (copied from LTOCodeGenerator).
 486   // FIXME this looks pretty terrible...
 487   if (TMBuilder.MCpu.empty() && TheTriple.isOSDarwin()) {
 488     if (TheTriple.getArch() == llvm::Triple::x86_64)
 489       TMBuilder.MCpu = "core2";
 490     else if (TheTriple.getArch() == llvm::Triple::x86)
 491       TMBuilder.MCpu = "yonah";
 492     else if (TheTriple.getArch() == llvm::Triple::aarch64)
 493       TMBuilder.MCpu = "cyclone";
 494   }
 495   TMBuilder.TheTriple = std::move(TheTriple);
 496 }
 497
 498 } // end anonymous namespace
 499
 500 void ThinLTOCodeGenerator::addModule(StringRef Identifier, StringRef Data) {
 501   MemoryBufferRef Buffer(Data, Identifier);
 502
 503   auto InputOrError = lto::InputFile::create(Buffer);
 504   if (!InputOrError)
 505     report_fatal_error("ThinLTO cannot create input file: " +
 506                        toString(InputOrError.takeError()));
 507
 508   auto TripleStr = (*InputOrError)->getTargetTriple();
 509   Triple TheTriple(TripleStr);
 510
 511   if (Modules.empty())
 512     initTMBuilder(TMBuilder, Triple(TheTriple));
 513   else if (TMBuilder.TheTriple != TheTriple) {
 514     if (!TMBuilder.TheTriple.isCompatibleWith(TheTriple))
 515       report_fatal_error("ThinLTO modules with incompatible triples not "
 516                          "supported");
 517     initTMBuilder(TMBuilder, Triple(TMBuilder.TheTriple.merge(TheTriple)));
 518   }
 519
 520   Modules.emplace_back(std::move(*InputOrError));
 521 }
 522
 523 void ThinLTOCodeGenerator::preserveSymbol(StringRef Name) {
 524   PreservedSymbols.insert(Name);
 525 }
 526
 527 void ThinLTOCodeGenerator::crossReferenceSymbol(StringRef Name) {
 528   // FIXME: At the moment, we don't take advantage of this extra information,
 529   // we're conservatively considering cross-references as preserved.
 530   //  CrossReferencedSymbols.insert(Name);
 531   PreservedSymbols.insert(Name);
 532 }
 533
 534 // TargetMachine factory
 535 std::unique_ptr<TargetMachine> TargetMachineBuilder::create() const {
 536   std::string ErrMsg;
 537   const Target *TheTarget =
 538       TargetRegistry::lookupTarget(TheTriple.str(), ErrMsg);
 539   if (!TheTarget) {
 540     report_fatal_error("Can't load target for this Triple: " + ErrMsg);
 541   }
 542
 543   // Use MAttr as the default set of features.
 544   SubtargetFeatures Features(MAttr);
 545   Features.getDefaultSubtargetFeatures(TheTriple);
 546   std::string FeatureStr = Features.getString();
 547
 548   return std::unique_ptr<TargetMachine>(
 549       TheTarget->createTargetMachine(TheTriple.str(), MCpu, FeatureStr, Options,
 550                                      RelocModel, None, CGOptLevel));
 551 }
 552
 553 /**
 554  * Produce the combined summary index from all the bitcode files:
 555  * "thin-link".
 556  */
 557 std::unique_ptr<ModuleSummaryIndex> ThinLTOCodeGenerator::linkCombinedIndex() {
 558   std::unique_ptr<ModuleSummaryIndex> CombinedIndex =
 559       std::make_unique<ModuleSummaryIndex>(/*HaveGVs=*/false);
 560   uint64_t NextModuleId = 0;
 561   for (auto &Mod : Modules) {
 562     auto &M = Mod->getSingleBitcodeModule();
 563     if (Error Err =
 564             M.readSummary(*CombinedIndex, Mod->getName(), NextModuleId++)) {
 565       // FIXME diagnose
 566       logAllUnhandledErrors(
 567           std::move(Err), errs(),
 568           "error: can't create module summary index for buffer: ");
 569       return nullptr;
 570     }
 571   }
 572   return CombinedIndex;
 573 }
 574
 575 static void internalizeAndPromoteInIndex(
 576     const StringMap<FunctionImporter::ExportSetTy> &ExportLists,
 577     const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
 578     const DenseMap<GlobalValue::GUID, const GlobalValueSummary *>
 579         &PrevailingCopy,
 580     ModuleSummaryIndex &Index) {
 581   auto isExported = [&](StringRef ModuleIdentifier, GlobalValue::GUID GUID) {
 582     const auto &ExportList = ExportLists.find(ModuleIdentifier);
 583     return (ExportList != ExportLists.end() &&
 584             ExportList->second.count(GUID)) ||
 585            GUIDPreservedSymbols.count(GUID);
 586   };
 587
 588   auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) {
 589     const auto &Prevailing = PrevailingCopy.find(GUID);
 590     // Not in map means that there was only one copy, which must be prevailing.
 591     if (Prevailing == PrevailingCopy.end())
 592       return true;
 593     return Prevailing->second == S;
 594   };
 595
 596   thinLTOInternalizeAndPromoteInIndex(Index, isExported, isPrevailing);
 597 }
 598
 599 static void computeDeadSymbolsInIndex(
 600     ModuleSummaryIndex &Index,
 601     const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) {
 602   // We have no symbols resolution available. And can't do any better now in the
 603   // case where the prevailing symbol is in a native object. It can be refined
 604   // with linker information in the future.
 605   auto isPrevailing = [&](GlobalValue::GUID G) {
 606     return PrevailingType::Unknown;
 607   };
 608   computeDeadSymbolsWithConstProp(Index, GUIDPreservedSymbols, isPrevailing,
 609                                   /* ImportEnabled = */ true);
 610 }
 611
 612 /**
 613  * Perform promotion and renaming of exported internal functions.
 614  * Index is updated to reflect linkage changes from weak resolution.
 615  */
 616 void ThinLTOCodeGenerator::promote(Module &TheModule, ModuleSummaryIndex &Index,
 617                                    const lto::InputFile &File) {
 618   auto ModuleCount = Index.modulePaths().size();
 619   auto ModuleIdentifier = TheModule.getModuleIdentifier();
 620
 621   // Collect for each module the list of function it defines (GUID -> Summary).
 622   StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries;
 623   Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
 624
 625   // Convert the preserved symbols set from string to GUID
 626   auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
 627       PreservedSymbols, Triple(TheModule.getTargetTriple()));
 628
 629   // Add used symbol to the preserved symbols.
 630   addUsedSymbolToPreservedGUID(File, GUIDPreservedSymbols);
 631
 632   // Compute "dead" symbols, we don't want to import/export these!
 633   computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
 634
 635   // Generate import/export list
 636   StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
 637   StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
 638   ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
 639                            ExportLists);
 640
 641   DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
 642   computePrevailingCopies(Index, PrevailingCopy);
 643
 644   // Resolve prevailing symbols
 645   StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
 646   resolvePrevailingInIndex(Index, ResolvedODR, GUIDPreservedSymbols,
 647                            PrevailingCopy);
 648
 649   thinLTOResolvePrevailingInModule(
 650       TheModule, ModuleToDefinedGVSummaries[ModuleIdentifier]);
 651
 652   // Promote the exported values in the index, so that they are promoted
 653   // in the module.
 654   internalizeAndPromoteInIndex(ExportLists, GUIDPreservedSymbols,
 655                                PrevailingCopy, Index);
 656
 657   promoteModule(TheModule, Index);
 658 }
 659
 660 /**
 661  * Perform cross-module importing for the module identified by ModuleIdentifier.
 662  */
 663 void ThinLTOCodeGenerator::crossModuleImport(Module &TheModule,
 664                                              ModuleSummaryIndex &Index,
 665                                              const lto::InputFile &File) {
 666   auto ModuleMap = generateModuleMap(Modules);
 667   auto ModuleCount = Index.modulePaths().size();
 668
 669   // Collect for each module the list of function it defines (GUID -> Summary).
 670   StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
 671   Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
 672
 673   // Convert the preserved symbols set from string to GUID
 674   auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
 675       PreservedSymbols, Triple(TheModule.getTargetTriple()));
 676
 677   addUsedSymbolToPreservedGUID(File, GUIDPreservedSymbols);
 678
 679   // Compute "dead" symbols, we don't want to import/export these!
 680   computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
 681
 682   // Generate import/export list
 683   StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
 684   StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
 685   ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
 686                            ExportLists);
 687   auto &ImportList = ImportLists[TheModule.getModuleIdentifier()];
 688
 689   crossImportIntoModule(TheModule, Index, ModuleMap, ImportList);
 690 }
 691
 692 /**
 693  * Compute the list of summaries needed for importing into module.
 694  */
 695 void ThinLTOCodeGenerator::gatherImportedSummariesForModule(
 696     Module &TheModule, ModuleSummaryIndex &Index,
 697     std::map<std::string, GVSummaryMapTy> &ModuleToSummariesForIndex,
 698     const lto::InputFile &File) {
 699   auto ModuleCount = Index.modulePaths().size();
 700   auto ModuleIdentifier = TheModule.getModuleIdentifier();
 701
 702   // Collect for each module the list of function it defines (GUID -> Summary).
 703   StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
 704   Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
 705
 706   // Convert the preserved symbols set from string to GUID
 707   auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
 708       PreservedSymbols, Triple(TheModule.getTargetTriple()));
 709
 710   addUsedSymbolToPreservedGUID(File, GUIDPreservedSymbols);
 711
 712   // Compute "dead" symbols, we don't want to import/export these!
 713   computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
 714
 715   // Generate import/export list
 716   StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
 717   StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
 718   ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
 719                            ExportLists);
 720
 721   llvm::gatherImportedSummariesForModule(
 722       ModuleIdentifier, ModuleToDefinedGVSummaries,
 723       ImportLists[ModuleIdentifier], ModuleToSummariesForIndex);
 724 }
 725
 726 /**
 727  * Emit the list of files needed for importing into module.
 728  */
 729 void ThinLTOCodeGenerator::emitImports(Module &TheModule, StringRef OutputName,
 730                                        ModuleSummaryIndex &Index,
 731                                        const lto::InputFile &File) {
 732   auto ModuleCount = Index.modulePaths().size();
 733   auto ModuleIdentifier = TheModule.getModuleIdentifier();
 734
 735   // Collect for each module the list of function it defines (GUID -> Summary).
 736   StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
 737   Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
 738
 739   // Convert the preserved symbols set from string to GUID
 740   auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
 741       PreservedSymbols, Triple(TheModule.getTargetTriple()));
 742
 743   addUsedSymbolToPreservedGUID(File, GUIDPreservedSymbols);
 744
 745   // Compute "dead" symbols, we don't want to import/export these!
 746   computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
 747
 748   // Generate import/export list
 749   StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
 750   StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
 751   ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
 752                            ExportLists);
 753
 754   std::map<std::string, GVSummaryMapTy> ModuleToSummariesForIndex;
 755   llvm::gatherImportedSummariesForModule(
 756       ModuleIdentifier, ModuleToDefinedGVSummaries,
 757       ImportLists[ModuleIdentifier], ModuleToSummariesForIndex);
 758
 759   std::error_code EC;
 760   if ((EC = EmitImportsFiles(ModuleIdentifier, OutputName,
 761                              ModuleToSummariesForIndex)))
 762     report_fatal_error(Twine("Failed to open ") + OutputName +
 763                        " to save imports lists\n");
 764 }
 765
 766 /**
 767  * Perform internalization. Runs promote and internalization together.
 768  * Index is updated to reflect linkage changes.
 769  */
 770 void ThinLTOCodeGenerator::internalize(Module &TheModule,
 771                                        ModuleSummaryIndex &Index,
 772                                        const lto::InputFile &File) {
 773   initTMBuilder(TMBuilder, Triple(TheModule.getTargetTriple()));
 774   auto ModuleCount = Index.modulePaths().size();
 775   auto ModuleIdentifier = TheModule.getModuleIdentifier();
 776
 777   // Convert the preserved symbols set from string to GUID
 778   auto GUIDPreservedSymbols =
 779       computeGUIDPreservedSymbols(PreservedSymbols, TMBuilder.TheTriple);
 780
 781   addUsedSymbolToPreservedGUID(File, GUIDPreservedSymbols);
 782
 783   // Collect for each module the list of function it defines (GUID -> Summary).
 784   StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
 785   Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
 786
 787   // Compute "dead" symbols, we don't want to import/export these!
 788   computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
 789
 790   // Generate import/export list
 791   StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
 792   StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
 793   ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
 794                            ExportLists);
 795   auto &ExportList = ExportLists[ModuleIdentifier];
 796
 797   // Be friendly and don't nuke totally the module when the client didn't
 798   // supply anything to preserve.
 799   if (ExportList.empty() && GUIDPreservedSymbols.empty())
 800     return;
 801
 802   DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
 803   computePrevailingCopies(Index, PrevailingCopy);
 804
 805   // Resolve prevailing symbols
 806   StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
 807   resolvePrevailingInIndex(Index, ResolvedODR, GUIDPreservedSymbols,
 808                            PrevailingCopy);
 809
 810   // Promote the exported values in the index, so that they are promoted
 811   // in the module.
 812   internalizeAndPromoteInIndex(ExportLists, GUIDPreservedSymbols,
 813                                PrevailingCopy, Index);
 814
 815   promoteModule(TheModule, Index);
 816
 817   // Internalization
 818   thinLTOResolvePrevailingInModule(
 819       TheModule, ModuleToDefinedGVSummaries[ModuleIdentifier]);
 820
 821   thinLTOInternalizeModule(TheModule,
 822                            ModuleToDefinedGVSummaries[ModuleIdentifier]);
 823 }
 824
 825 /**
 826  * Perform post-importing ThinLTO optimizations.
 827  */
 828 void ThinLTOCodeGenerator::optimize(Module &TheModule) {
 829   initTMBuilder(TMBuilder, Triple(TheModule.getTargetTriple()));
 830
 831   // Optimize now
 832   optimizeModule(TheModule, *TMBuilder.create(), OptLevel, Freestanding);
 833 }
 834
 835 /// Write out the generated object file, either from CacheEntryPath or from
 836 /// OutputBuffer, preferring hard-link when possible.
 837 /// Returns the path to the generated file in SavedObjectsDirectoryPath.
 838 std::string
 839 ThinLTOCodeGenerator::writeGeneratedObject(int count, StringRef CacheEntryPath,
 840                                            const MemoryBuffer &OutputBuffer) {
 841   auto ArchName = TMBuilder.TheTriple.getArchName();
 842   SmallString<128> OutputPath(SavedObjectsDirectoryPath);
 843   llvm::sys::path::append(OutputPath,
 844                           Twine(count) + "." + ArchName + ".thinlto.o");
 845   OutputPath.c_str(); // Ensure the string is null terminated.
 846   if (sys::fs::exists(OutputPath))
 847     sys::fs::remove(OutputPath);
 848
 849   // We don't return a memory buffer to the linker, just a list of files.
 850   if (!CacheEntryPath.empty()) {
 851     // Cache is enabled, hard-link the entry (or copy if hard-link fails).
 852     auto Err = sys::fs::create_hard_link(CacheEntryPath, OutputPath);
 853     if (!Err)
 854       return OutputPath.str();
 855     // Hard linking failed, try to copy.
 856     Err = sys::fs::copy_file(CacheEntryPath, OutputPath);
 857     if (!Err)
 858       return OutputPath.str();
 859     // Copy failed (could be because the CacheEntry was removed from the cache
 860     // in the meantime by another process), fall back and try to write down the
 861     // buffer to the output.
 862     errs() << "error: can't link or copy from cached entry '" << CacheEntryPath
 863            << "' to '" << OutputPath << "'\n";
 864   }
 865   // No cache entry, just write out the buffer.
 866   std::error_code Err;
 867   raw_fd_ostream OS(OutputPath, Err, sys::fs::OF_None);
 868   if (Err)
 869     report_fatal_error("Can't open output '" + OutputPath + "'\n");
 870   OS << OutputBuffer.getBuffer();
 871   return OutputPath.str();
 872 }
 873
 874 // Main entry point for the ThinLTO processing
 875 void ThinLTOCodeGenerator::run() {
 876   // Prepare the resulting object vector
 877   assert(ProducedBinaries.empty() && "The generator should not be reused");
 878   if (SavedObjectsDirectoryPath.empty())
 879     ProducedBinaries.resize(Modules.size());
 880   else {
 881     sys::fs::create_directories(SavedObjectsDirectoryPath);
 882     bool IsDir;
 883     sys::fs::is_directory(SavedObjectsDirectoryPath, IsDir);
 884     if (!IsDir)
 885       report_fatal_error("Unexistent dir: '" + SavedObjectsDirectoryPath + "'");
 886     ProducedBinaryFiles.resize(Modules.size());
 887   }
 888
 889   if (CodeGenOnly) {
 890     // Perform only parallel codegen and return.
 891     ThreadPool Pool;
 892     int count = 0;
 893     for (auto &Mod : Modules) {
 894       Pool.async([&](int count) {
 895         LLVMContext Context;
 896         Context.setDiscardValueNames(LTODiscardValueNames);
 897
 898         // Parse module now
 899         auto TheModule = loadModuleFromInput(Mod.get(), Context, false,
 900                                              /*IsImporting*/ false);
 901
 902         // CodeGen
 903         auto OutputBuffer = codegenModule(*TheModule, *TMBuilder.create());
 904         if (SavedObjectsDirectoryPath.empty())
 905           ProducedBinaries[count] = std::move(OutputBuffer);
 906         else
 907           ProducedBinaryFiles[count] =
 908               writeGeneratedObject(count, "", *OutputBuffer);
 909       }, count++);
 910     }
 911
 912     return;
 913   }
 914
 915   // Sequential linking phase
 916   auto Index = linkCombinedIndex();
 917
 918   // Save temps: index.
 919   if (!SaveTempsDir.empty()) {
 920     auto SaveTempPath = SaveTempsDir + "index.bc";
 921     std::error_code EC;
 922     raw_fd_ostream OS(SaveTempPath, EC, sys::fs::OF_None);
 923     if (EC)
 924       report_fatal_error(Twine("Failed to open ") + SaveTempPath +
 925                          " to save optimized bitcode\n");
 926     WriteIndexToFile(*Index, OS);
 927   }
 928
 929
 930   // Prepare the module map.
 931   auto ModuleMap = generateModuleMap(Modules);
 932   auto ModuleCount = Modules.size();
 933
 934   // Collect for each module the list of function it defines (GUID -> Summary).
 935   StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
 936   Index->collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
 937
 938   // Convert the preserved symbols set from string to GUID, this is needed for
 939   // computing the caching hash and the internalization.
 940   auto GUIDPreservedSymbols =
 941       computeGUIDPreservedSymbols(PreservedSymbols, TMBuilder.TheTriple);
 942
 943   // Add used symbol from inputs to the preserved symbols.
 944   for (const auto &M : Modules)
 945     addUsedSymbolToPreservedGUID(*M, GUIDPreservedSymbols);
 946
 947   // Compute "dead" symbols, we don't want to import/export these!
 948   computeDeadSymbolsInIndex(*Index, GUIDPreservedSymbols);
 949
 950   // Synthesize entry counts for functions in the combined index.
 951   computeSyntheticCounts(*Index);
 952
 953   // Collect the import/export lists for all modules from the call-graph in the
 954   // combined index.
 955   StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
 956   StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
 957   ComputeCrossModuleImport(*Index, ModuleToDefinedGVSummaries, ImportLists,
 958                            ExportLists);
 959
 960   // We use a std::map here to be able to have a defined ordering when
 961   // producing a hash for the cache entry.
 962   // FIXME: we should be able to compute the caching hash for the entry based
 963   // on the index, and nuke this map.
 964   StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
 965
 966   DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
 967   computePrevailingCopies(*Index, PrevailingCopy);
 968
 969   // Resolve prevailing symbols, this has to be computed early because it
 970   // impacts the caching.
 971   resolvePrevailingInIndex(*Index, ResolvedODR, GUIDPreservedSymbols,
 972                            PrevailingCopy);
 973
 974   // Use global summary-based analysis to identify symbols that can be
 975   // internalized (because they aren't exported or preserved as per callback).
 976   // Changes are made in the index, consumed in the ThinLTO backends.
 977   internalizeAndPromoteInIndex(ExportLists, GUIDPreservedSymbols,
 978                                PrevailingCopy, *Index);
 979
 980   // Make sure that every module has an entry in the ExportLists, ImportList,
 981   // GVSummary and ResolvedODR maps to enable threaded access to these maps
 982   // below.
 983   for (auto &Module : Modules) {
 984     auto ModuleIdentifier = Module->getName();
 985     ExportLists[ModuleIdentifier];
 986     ImportLists[ModuleIdentifier];
 987     ResolvedODR[ModuleIdentifier];
 988     ModuleToDefinedGVSummaries[ModuleIdentifier];
 989   }
 990
 991   // Compute the ordering we will process the inputs: the rough heuristic here
 992   // is to sort them per size so that the largest module get schedule as soon as
 993   // possible. This is purely a compile-time optimization.
 994   std::vector<int> ModulesOrdering;
 995   ModulesOrdering.resize(Modules.size());
 996   std::iota(ModulesOrdering.begin(), ModulesOrdering.end(), 0);
 997   llvm::sort(ModulesOrdering, [&](int LeftIndex, int RightIndex) {
 998     auto LSize =
 999         Modules[LeftIndex]->getSingleBitcodeModule().getBuffer().size();
1000     auto RSize =
1001         Modules[RightIndex]->getSingleBitcodeModule().getBuffer().size();
1002     return LSize > RSize;
1003   });
1004
1005   // Parallel optimizer + codegen
1006   {
1007     ThreadPool Pool(ThreadCount);
1008     for (auto IndexCount : ModulesOrdering) {
1009       auto &Mod = Modules[IndexCount];
1010       Pool.async([&](int count) {
1011         auto ModuleIdentifier = Mod->getName();
1012         auto &ExportList = ExportLists[ModuleIdentifier];
1013
1014         auto &DefinedGVSummaries = ModuleToDefinedGVSummaries[ModuleIdentifier];
1015
1016         // The module may be cached, this helps handling it.
1017         ModuleCacheEntry CacheEntry(CacheOptions.Path, *Index, ModuleIdentifier,
1018                                     ImportLists[ModuleIdentifier], ExportList,
1019                                     ResolvedODR[ModuleIdentifier],
1020                                     DefinedGVSummaries, OptLevel, Freestanding,
1021                                     TMBuilder);
1022         auto CacheEntryPath = CacheEntry.getEntryPath();
1023
1024         {
1025           auto ErrOrBuffer = CacheEntry.tryLoadingBuffer();
1026           LLVM_DEBUG(dbgs() << "Cache " << (ErrOrBuffer ? "hit" : "miss")
1027                             << " '" << CacheEntryPath << "' for buffer "
1028                             << count << " " << ModuleIdentifier << "\n");
1029
1030           if (ErrOrBuffer) {
1031             // Cache Hit!
1032             if (SavedObjectsDirectoryPath.empty())
1033               ProducedBinaries[count] = std::move(ErrOrBuffer.get());
1034             else
1035               ProducedBinaryFiles[count] = writeGeneratedObject(
1036                   count, CacheEntryPath, *ErrOrBuffer.get());
1037             return;
1038           }
1039         }
1040
1041         LLVMContext Context;
1042         Context.setDiscardValueNames(LTODiscardValueNames);
1043         Context.enableDebugTypeODRUniquing();
1044         auto DiagFileOrErr = lto::setupOptimizationRemarks(
1045             Context, RemarksFilename, RemarksPasses, RemarksFormat,
1046             RemarksWithHotness, count);
1047         if (!DiagFileOrErr) {
1048           errs() << "Error: " << toString(DiagFileOrErr.takeError()) << "\n";
1049           report_fatal_error("ThinLTO: Can't get an output file for the "
1050                              "remarks");
1051         }
1052
1053         // Parse module now
1054         auto TheModule = loadModuleFromInput(Mod.get(), Context, false,
1055                                              /*IsImporting*/ false);
1056
1057         // Save temps: original file.
1058         saveTempBitcode(*TheModule, SaveTempsDir, count, ".0.original.bc");
1059
1060         auto &ImportList = ImportLists[ModuleIdentifier];
1061         // Run the main process now, and generates a binary
1062         auto OutputBuffer = ProcessThinLTOModule(
1063             *TheModule, *Index, ModuleMap, *TMBuilder.create(), ImportList,
1064             ExportList, GUIDPreservedSymbols,
1065             ModuleToDefinedGVSummaries[ModuleIdentifier], CacheOptions,
1066             DisableCodeGen, SaveTempsDir, Freestanding, OptLevel, count);
1067
1068         // Commit to the cache (if enabled)
1069         CacheEntry.write(*OutputBuffer);
1070
1071         if (SavedObjectsDirectoryPath.empty()) {
1072           // We need to generated a memory buffer for the linker.
1073           if (!CacheEntryPath.empty()) {
1074             // When cache is enabled, reload from the cache if possible.
1075             // Releasing the buffer from the heap and reloading it from the
1076             // cache file with mmap helps us to lower memory pressure.
1077             // The freed memory can be used for the next input file.
1078             // The final binary link will read from the VFS cache (hopefully!)
1079             // or from disk (if the memory pressure was too high).
1080             auto ReloadedBufferOrErr = CacheEntry.tryLoadingBuffer();
1081             if (auto EC = ReloadedBufferOrErr.getError()) {
1082               // On error, keep the preexisting buffer and print a diagnostic.
1083               errs() << "error: can't reload cached file '" << CacheEntryPath
1084                      << "': " << EC.message() << "\n";
1085             } else {
1086               OutputBuffer = std::move(*ReloadedBufferOrErr);
1087             }
1088           }
1089           ProducedBinaries[count] = std::move(OutputBuffer);
1090           return;
1091         }
1092         ProducedBinaryFiles[count] = writeGeneratedObject(
1093             count, CacheEntryPath, *OutputBuffer);
1094       }, IndexCount);
1095     }
1096   }
1097
1098   pruneCache(CacheOptions.Path, CacheOptions.Policy);
1099
1100   // If statistics were requested, print them out now.
1101   if (llvm::AreStatisticsEnabled())
1102     llvm::PrintStatistics();
1103   reportAndResetTimings();
1104 }