lib/LTO/ThinLTOCodeGenerator.cpp

   1 //===-ThinLTOCodeGenerator.cpp - LLVM Link Time Optimizer -----------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file implements the Thin Link Time Optimization library. This library is
  10 // intended to be used by linker to optimize code at link time.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "llvm/LTO/legacy/ThinLTOCodeGenerator.h"
  15
  16 #include "llvm/ADT/Statistic.h"
  17 #include "llvm/ADT/StringExtras.h"
  18 #include "llvm/Analysis/ModuleSummaryAnalysis.h"
  19 #include "llvm/Analysis/ProfileSummaryInfo.h"
  20 #include "llvm/Analysis/TargetLibraryInfo.h"
  21 #include "llvm/Analysis/TargetTransformInfo.h"
  22 #include "llvm/Bitcode/BitcodeReader.h"
  23 #include "llvm/Bitcode/BitcodeWriter.h"
  24 #include "llvm/Bitcode/BitcodeWriterPass.h"
  25 #include "llvm/Config/llvm-config.h"
  26 #include "llvm/IR/DebugInfo.h"
  27 #include "llvm/IR/DiagnosticPrinter.h"
  28 #include "llvm/IR/LLVMContext.h"
  29 #include "llvm/IR/LegacyPassManager.h"
  30 #include "llvm/IR/Mangler.h"
  31 #include "llvm/IR/PassTimingInfo.h"
  32 #include "llvm/IR/RemarkStreamer.h"
  33 #include "llvm/IR/Verifier.h"
  34 #include "llvm/IRReader/IRReader.h"
  35 #include "llvm/LTO/LTO.h"
  36 #include "llvm/LTO/SummaryBasedOptimizations.h"
  37 #include "llvm/MC/SubtargetFeature.h"
  38 #include "llvm/Object/IRObjectFile.h"
  39 #include "llvm/Support/CachePruning.h"
  40 #include "llvm/Support/Debug.h"
  41 #include "llvm/Support/Error.h"
  42 #include "llvm/Support/FileUtilities.h"
  43 #include "llvm/Support/Path.h"
  44 #include "llvm/Support/SHA1.h"
  45 #include "llvm/Support/SmallVectorMemoryBuffer.h"
  46 #include "llvm/Support/TargetRegistry.h"
  47 #include "llvm/Support/ThreadPool.h"
  48 #include "llvm/Support/Threading.h"
  49 #include "llvm/Support/ToolOutputFile.h"
  50 #include "llvm/Support/VCSRevision.h"
  51 #include "llvm/Target/TargetMachine.h"
  52 #include "llvm/Transforms/IPO.h"
  53 #include "llvm/Transforms/IPO/FunctionImport.h"
  54 #include "llvm/Transforms/IPO/Internalize.h"
  55 #include "llvm/Transforms/IPO/PassManagerBuilder.h"
  56 #include "llvm/Transforms/IPO/WholeProgramDevirt.h"
  57 #include "llvm/Transforms/ObjCARC.h"
  58 #include "llvm/Transforms/Utils/FunctionImportUtils.h"
  59
  60 #include <numeric>
  61
  62 #if !defined(_MSC_VER) && !defined(__MINGW32__)
  63 #include <unistd.h>
  64 #else
  65 #include <io.h>
  66 #endif
  67
  68 using namespace llvm;
  69
  70 #define DEBUG_TYPE "thinlto"
  71
  72 namespace llvm {
  73 // Flags -discard-value-names, defined in LTOCodeGenerator.cpp
  74 extern cl::opt<bool> LTODiscardValueNames;
  75 extern cl::opt<std::string> RemarksFilename;
  76 extern cl::opt<std::string> RemarksPasses;
  77 extern cl::opt<bool> RemarksWithHotness;
  78 extern cl::opt<std::string> RemarksFormat;
  79 }
  80
  81 namespace {
  82
  83 static cl::opt<int>
  84     ThreadCount("threads", cl::init(llvm::heavyweight_hardware_concurrency()));
  85
  86 // Simple helper to save temporary files for debug.
  87 static void saveTempBitcode(const Module &TheModule, StringRef TempDir,
  88                             unsigned count, StringRef Suffix) {
  89   if (TempDir.empty())
  90     return;
  91   // User asked to save temps, let dump the bitcode file after import.
  92   std::string SaveTempPath = (TempDir + llvm::Twine(count) + Suffix).str();
  93   std::error_code EC;
  94   raw_fd_ostream OS(SaveTempPath, EC, sys::fs::OF_None);
  95   if (EC)
  96     report_fatal_error(Twine("Failed to open ") + SaveTempPath +
  97                        " to save optimized bitcode\n");
  98   WriteBitcodeToFile(TheModule, OS, /* ShouldPreserveUseListOrder */ true);
  99 }
 100
 101 static const GlobalValueSummary *
 102 getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) {
 103   // If there is any strong definition anywhere, get it.
 104   auto StrongDefForLinker = llvm::find_if(
 105       GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
 106         auto Linkage = Summary->linkage();
 107         return !GlobalValue::isAvailableExternallyLinkage(Linkage) &&
 108                !GlobalValue::isWeakForLinker(Linkage);
 109       });
 110   if (StrongDefForLinker != GVSummaryList.end())
 111     return StrongDefForLinker->get();
 112   // Get the first *linker visible* definition for this global in the summary
 113   // list.
 114   auto FirstDefForLinker = llvm::find_if(
 115       GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
 116         auto Linkage = Summary->linkage();
 117         return !GlobalValue::isAvailableExternallyLinkage(Linkage);
 118       });
 119   // Extern templates can be emitted as available_externally.
 120   if (FirstDefForLinker == GVSummaryList.end())
 121     return nullptr;
 122   return FirstDefForLinker->get();
 123 }
 124
 125 // Populate map of GUID to the prevailing copy for any multiply defined
 126 // symbols. Currently assume first copy is prevailing, or any strong
 127 // definition. Can be refined with Linker information in the future.
 128 static void computePrevailingCopies(
 129     const ModuleSummaryIndex &Index,
 130     DenseMap<GlobalValue::GUID, const GlobalValueSummary *> &PrevailingCopy) {
 131   auto HasMultipleCopies = [&](const GlobalValueSummaryList &GVSummaryList) {
 132     return GVSummaryList.size() > 1;
 133   };
 134
 135   for (auto &I : Index) {
 136     if (HasMultipleCopies(I.second.SummaryList))
 137       PrevailingCopy[I.first] =
 138           getFirstDefinitionForLinker(I.second.SummaryList);
 139   }
 140 }
 141
 142 static StringMap<lto::InputFile *>
 143 generateModuleMap(std::vector<std::unique_ptr<lto::InputFile>> &Modules) {
 144   StringMap<lto::InputFile *> ModuleMap;
 145   for (auto &M : Modules) {
 146     assert(ModuleMap.find(M->getName()) == ModuleMap.end() &&
 147            "Expect unique Buffer Identifier");
 148     ModuleMap[M->getName()] = M.get();
 149   }
 150   return ModuleMap;
 151 }
 152
 153 static void promoteModule(Module &TheModule, const ModuleSummaryIndex &Index) {
 154   if (renameModuleForThinLTO(TheModule, Index))
 155     report_fatal_error("renameModuleForThinLTO failed");
 156 }
 157
 158 namespace {
 159 class ThinLTODiagnosticInfo : public DiagnosticInfo {
 160   const Twine &Msg;
 161 public:
 162   ThinLTODiagnosticInfo(const Twine &DiagMsg,
 163                         DiagnosticSeverity Severity = DS_Error)
 164       : DiagnosticInfo(DK_Linker, Severity), Msg(DiagMsg) {}
 165   void print(DiagnosticPrinter &DP) const override { DP << Msg; }
 166 };
 167 }
 168
 169 /// Verify the module and strip broken debug info.
 170 static void verifyLoadedModule(Module &TheModule) {
 171   bool BrokenDebugInfo = false;
 172   if (verifyModule(TheModule, &dbgs(), &BrokenDebugInfo))
 173     report_fatal_error("Broken module found, compilation aborted!");
 174   if (BrokenDebugInfo) {
 175     TheModule.getContext().diagnose(ThinLTODiagnosticInfo(
 176         "Invalid debug info found, debug info will be stripped", DS_Warning));
 177     StripDebugInfo(TheModule);
 178   }
 179 }
 180
 181 static std::unique_ptr<Module> loadModuleFromInput(lto::InputFile *Input,
 182                                                    LLVMContext &Context,
 183                                                    bool Lazy,
 184                                                    bool IsImporting) {
 185   auto &Mod = Input->getSingleBitcodeModule();
 186   SMDiagnostic Err;
 187   Expected<std::unique_ptr<Module>> ModuleOrErr =
 188       Lazy ? Mod.getLazyModule(Context,
 189                                /* ShouldLazyLoadMetadata */ true, IsImporting)
 190            : Mod.parseModule(Context);
 191   if (!ModuleOrErr) {
 192     handleAllErrors(ModuleOrErr.takeError(), [&](ErrorInfoBase &EIB) {
 193       SMDiagnostic Err = SMDiagnostic(Mod.getModuleIdentifier(),
 194                                       SourceMgr::DK_Error, EIB.message());
 195       Err.print("ThinLTO", errs());
 196     });
 197     report_fatal_error("Can't load module, abort.");
 198   }
 199   if (!Lazy)
 200     verifyLoadedModule(*ModuleOrErr.get());
 201   return std::move(*ModuleOrErr);
 202 }
 203
 204 static void
 205 crossImportIntoModule(Module &TheModule, const ModuleSummaryIndex &Index,
 206                       StringMap<lto::InputFile*> &ModuleMap,
 207                       const FunctionImporter::ImportMapTy &ImportList) {
 208   auto Loader = [&](StringRef Identifier) {
 209     auto &Input = ModuleMap[Identifier];
 210     return loadModuleFromInput(Input, TheModule.getContext(),
 211                                /*Lazy=*/true, /*IsImporting*/ true);
 212   };
 213
 214   FunctionImporter Importer(Index, Loader);
 215   Expected<bool> Result = Importer.importFunctions(TheModule, ImportList);
 216   if (!Result) {
 217     handleAllErrors(Result.takeError(), [&](ErrorInfoBase &EIB) {
 218       SMDiagnostic Err = SMDiagnostic(TheModule.getModuleIdentifier(),
 219                                       SourceMgr::DK_Error, EIB.message());
 220       Err.print("ThinLTO", errs());
 221     });
 222     report_fatal_error("importFunctions failed");
 223   }
 224   // Verify again after cross-importing.
 225   verifyLoadedModule(TheModule);
 226 }
 227
 228 static void optimizeModule(Module &TheModule, TargetMachine &TM,
 229                            unsigned OptLevel, bool Freestanding,
 230                            ModuleSummaryIndex *Index) {
 231   // Populate the PassManager
 232   PassManagerBuilder PMB;
 233   PMB.LibraryInfo = new TargetLibraryInfoImpl(TM.getTargetTriple());
 234   if (Freestanding)
 235     PMB.LibraryInfo->disableAllFunctions();
 236   PMB.Inliner = createFunctionInliningPass();
 237   // FIXME: should get it from the bitcode?
 238   PMB.OptLevel = OptLevel;
 239   PMB.LoopVectorize = true;
 240   PMB.SLPVectorize = true;
 241   // Already did this in verifyLoadedModule().
 242   PMB.VerifyInput = false;
 243   PMB.VerifyOutput = false;
 244   PMB.ImportSummary = Index;
 245
 246   legacy::PassManager PM;
 247
 248   // Add the TTI (required to inform the vectorizer about register size for
 249   // instance)
 250   PM.add(createTargetTransformInfoWrapperPass(TM.getTargetIRAnalysis()));
 251
 252   // Add optimizations
 253   PMB.populateThinLTOPassManager(PM);
 254
 255   PM.run(TheModule);
 256 }
 257
 258 static void
 259 addUsedSymbolToPreservedGUID(const lto::InputFile &File,
 260                              DenseSet<GlobalValue::GUID> &PreservedGUID) {
 261   for (const auto &Sym : File.symbols()) {
 262     if (Sym.isUsed())
 263       PreservedGUID.insert(GlobalValue::getGUID(Sym.getIRName()));
 264   }
 265 }
 266
 267 // Convert the PreservedSymbols map from "Name" based to "GUID" based.
 268 static DenseSet<GlobalValue::GUID>
 269 computeGUIDPreservedSymbols(const StringSet<> &PreservedSymbols,
 270                             const Triple &TheTriple) {
 271   DenseSet<GlobalValue::GUID> GUIDPreservedSymbols(PreservedSymbols.size());
 272   for (auto &Entry : PreservedSymbols) {
 273     StringRef Name = Entry.first();
 274     if (TheTriple.isOSBinFormatMachO() && Name.size() > 0 && Name[0] == '_')
 275       Name = Name.drop_front();
 276     GUIDPreservedSymbols.insert(GlobalValue::getGUID(Name));
 277   }
 278   return GUIDPreservedSymbols;
 279 }
 280
 281 std::unique_ptr<MemoryBuffer> codegenModule(Module &TheModule,
 282                                             TargetMachine &TM) {
 283   SmallVector<char, 128> OutputBuffer;
 284
 285   // CodeGen
 286   {
 287     raw_svector_ostream OS(OutputBuffer);
 288     legacy::PassManager PM;
 289
 290     // If the bitcode files contain ARC code and were compiled with optimization,
 291     // the ObjCARCContractPass must be run, so do it unconditionally here.
 292     PM.add(createObjCARCContractPass());
 293
 294     // Setup the codegen now.
 295     if (TM.addPassesToEmitFile(PM, OS, nullptr, TargetMachine::CGFT_ObjectFile,
 296                                /* DisableVerify */ true))
 297       report_fatal_error("Failed to setup codegen");
 298
 299     // Run codegen now. resulting binary is in OutputBuffer.
 300     PM.run(TheModule);
 301   }
 302   return std::make_unique<SmallVectorMemoryBuffer>(std::move(OutputBuffer));
 303 }
 304
 305 /// Manage caching for a single Module.
 306 class ModuleCacheEntry {
 307   SmallString<128> EntryPath;
 308
 309 public:
 310   // Create a cache entry. This compute a unique hash for the Module considering
 311   // the current list of export/import, and offer an interface to query to
 312   // access the content in the cache.
 313   ModuleCacheEntry(
 314       StringRef CachePath, const ModuleSummaryIndex &Index, StringRef ModuleID,
 315       const FunctionImporter::ImportMapTy &ImportList,
 316       const FunctionImporter::ExportSetTy &ExportList,
 317       const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
 318       const GVSummaryMapTy &DefinedGVSummaries, unsigned OptLevel,
 319       bool Freestanding, const TargetMachineBuilder &TMBuilder) {
 320     if (CachePath.empty())
 321       return;
 322
 323     if (!Index.modulePaths().count(ModuleID))
 324       // The module does not have an entry, it can't have a hash at all
 325       return;
 326
 327     if (all_of(Index.getModuleHash(ModuleID),
 328                [](uint32_t V) { return V == 0; }))
 329       // No hash entry, no caching!
 330       return;
 331
 332     llvm::lto::Config Conf;
 333     Conf.OptLevel = OptLevel;
 334     Conf.Options = TMBuilder.Options;
 335     Conf.CPU = TMBuilder.MCpu;
 336     Conf.MAttrs.push_back(TMBuilder.MAttr);
 337     Conf.RelocModel = TMBuilder.RelocModel;
 338     Conf.CGOptLevel = TMBuilder.CGOptLevel;
 339     Conf.Freestanding = Freestanding;
 340     SmallString<40> Key;
 341     computeLTOCacheKey(Key, Conf, Index, ModuleID, ImportList, ExportList,
 342                        ResolvedODR, DefinedGVSummaries);
 343
 344     // This choice of file name allows the cache to be pruned (see pruneCache()
 345     // in include/llvm/Support/CachePruning.h).
 346     sys::path::append(EntryPath, CachePath, "llvmcache-" + Key);
 347   }
 348
 349   // Access the path to this entry in the cache.
 350   StringRef getEntryPath() { return EntryPath; }
 351
 352   // Try loading the buffer for this cache entry.
 353   ErrorOr<std::unique_ptr<MemoryBuffer>> tryLoadingBuffer() {
 354     if (EntryPath.empty())
 355       return std::error_code();
 356     SmallString<64> ResultPath;
 357     Expected<sys::fs::file_t> FDOrErr = sys::fs::openNativeFileForRead(
 358         Twine(EntryPath), sys::fs::OF_UpdateAtime, &ResultPath);
 359     if (!FDOrErr)
 360       return errorToErrorCode(FDOrErr.takeError());
 361     ErrorOr<std::unique_ptr<MemoryBuffer>> MBOrErr = MemoryBuffer::getOpenFile(
 362         *FDOrErr, EntryPath, /*FileSize=*/-1, /*RequiresNullTerminator=*/false);
 363     sys::fs::closeFile(*FDOrErr);
 364     return MBOrErr;
 365   }
 366
 367   // Cache the Produced object file
 368   void write(const MemoryBuffer &OutputBuffer) {
 369     if (EntryPath.empty())
 370       return;
 371
 372     // Write to a temporary to avoid race condition
 373     SmallString<128> TempFilename;
 374     SmallString<128> CachePath(EntryPath);
 375     llvm::sys::path::remove_filename(CachePath);
 376     sys::path::append(TempFilename, CachePath, "Thin-%%%%%%.tmp.o");
 377
 378     if (auto Err = handleErrors(
 379             llvm::writeFileAtomically(TempFilename, EntryPath,
 380                                       OutputBuffer.getBuffer()),
 381             [](const llvm::AtomicFileWriteError &E) {
 382               std::string ErrorMsgBuffer;
 383               llvm::raw_string_ostream S(ErrorMsgBuffer);
 384               E.log(S);
 385
 386               if (E.Error ==
 387                   llvm::atomic_write_error::failed_to_create_uniq_file) {
 388                 errs() << "Error: " << ErrorMsgBuffer << "\n";
 389                 report_fatal_error("ThinLTO: Can't get a temporary file");
 390               }
 391             })) {
 392       // FIXME
 393       consumeError(std::move(Err));
 394     }
 395   }
 396 };
 397
 398 static std::unique_ptr<MemoryBuffer>
 399 ProcessThinLTOModule(Module &TheModule, ModuleSummaryIndex &Index,
 400                      StringMap<lto::InputFile *> &ModuleMap, TargetMachine &TM,
 401                      const FunctionImporter::ImportMapTy &ImportList,
 402                      const FunctionImporter::ExportSetTy &ExportList,
 403                      const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
 404                      const GVSummaryMapTy &DefinedGlobals,
 405                      const ThinLTOCodeGenerator::CachingOptions &CacheOptions,
 406                      bool DisableCodeGen, StringRef SaveTempsDir,
 407                      bool Freestanding, unsigned OptLevel, unsigned count) {
 408
 409   // "Benchmark"-like optimization: single-source case
 410   bool SingleModule = (ModuleMap.size() == 1);
 411
 412   if (!SingleModule) {
 413     promoteModule(TheModule, Index);
 414
 415     // Apply summary-based prevailing-symbol resolution decisions.
 416     thinLTOResolvePrevailingInModule(TheModule, DefinedGlobals);
 417
 418     // Save temps: after promotion.
 419     saveTempBitcode(TheModule, SaveTempsDir, count, ".1.promoted.bc");
 420   }
 421
 422   // Be friendly and don't nuke totally the module when the client didn't
 423   // supply anything to preserve.
 424   if (!ExportList.empty() || !GUIDPreservedSymbols.empty()) {
 425     // Apply summary-based internalization decisions.
 426     thinLTOInternalizeModule(TheModule, DefinedGlobals);
 427   }
 428
 429   // Save internalized bitcode
 430   saveTempBitcode(TheModule, SaveTempsDir, count, ".2.internalized.bc");
 431
 432   if (!SingleModule) {
 433     crossImportIntoModule(TheModule, Index, ModuleMap, ImportList);
 434
 435     // Save temps: after cross-module import.
 436     saveTempBitcode(TheModule, SaveTempsDir, count, ".3.imported.bc");
 437   }
 438
 439   optimizeModule(TheModule, TM, OptLevel, Freestanding, &Index);
 440
 441   saveTempBitcode(TheModule, SaveTempsDir, count, ".4.opt.bc");
 442
 443   if (DisableCodeGen) {
 444     // Configured to stop before CodeGen, serialize the bitcode and return.
 445     SmallVector<char, 128> OutputBuffer;
 446     {
 447       raw_svector_ostream OS(OutputBuffer);
 448       ProfileSummaryInfo PSI(TheModule);
 449       auto Index = buildModuleSummaryIndex(TheModule, nullptr, &PSI);
 450       WriteBitcodeToFile(TheModule, OS, true, &Index);
 451     }
 452     return std::make_unique<SmallVectorMemoryBuffer>(std::move(OutputBuffer));
 453   }
 454
 455   return codegenModule(TheModule, TM);
 456 }
 457
 458 /// Resolve prevailing symbols. Record resolutions in the \p ResolvedODR map
 459 /// for caching, and in the \p Index for application during the ThinLTO
 460 /// backends. This is needed for correctness for exported symbols (ensure
 461 /// at least one copy kept) and a compile-time optimization (to drop duplicate
 462 /// copies when possible).
 463 static void resolvePrevailingInIndex(
 464     ModuleSummaryIndex &Index,
 465     StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>>
 466         &ResolvedODR,
 467     const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
 468     const DenseMap<GlobalValue::GUID, const GlobalValueSummary *>
 469         &PrevailingCopy) {
 470
 471   auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) {
 472     const auto &Prevailing = PrevailingCopy.find(GUID);
 473     // Not in map means that there was only one copy, which must be prevailing.
 474     if (Prevailing == PrevailingCopy.end())
 475       return true;
 476     return Prevailing->second == S;
 477   };
 478
 479   auto recordNewLinkage = [&](StringRef ModuleIdentifier,
 480                               GlobalValue::GUID GUID,
 481                               GlobalValue::LinkageTypes NewLinkage) {
 482     ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;
 483   };
 484
 485   thinLTOResolvePrevailingInIndex(Index, isPrevailing, recordNewLinkage,
 486                                   GUIDPreservedSymbols);
 487 }
 488
 489 // Initialize the TargetMachine builder for a given Triple
 490 static void initTMBuilder(TargetMachineBuilder &TMBuilder,
 491                           const Triple &TheTriple) {
 492   // Set a default CPU for Darwin triples (copied from LTOCodeGenerator).
 493   // FIXME this looks pretty terrible...
 494   if (TMBuilder.MCpu.empty() && TheTriple.isOSDarwin()) {
 495     if (TheTriple.getArch() == llvm::Triple::x86_64)
 496       TMBuilder.MCpu = "core2";
 497     else if (TheTriple.getArch() == llvm::Triple::x86)
 498       TMBuilder.MCpu = "yonah";
 499     else if (TheTriple.getArch() == llvm::Triple::aarch64 ||
 500              TheTriple.getArch() == llvm::Triple::aarch64_32)
 501       TMBuilder.MCpu = "cyclone";
 502   }
 503   TMBuilder.TheTriple = std::move(TheTriple);
 504 }
 505
 506 } // end anonymous namespace
 507
 508 void ThinLTOCodeGenerator::addModule(StringRef Identifier, StringRef Data) {
 509   MemoryBufferRef Buffer(Data, Identifier);
 510
 511   auto InputOrError = lto::InputFile::create(Buffer);
 512   if (!InputOrError)
 513     report_fatal_error("ThinLTO cannot create input file: " +
 514                        toString(InputOrError.takeError()));
 515
 516   auto TripleStr = (*InputOrError)->getTargetTriple();
 517   Triple TheTriple(TripleStr);
 518
 519   if (Modules.empty())
 520     initTMBuilder(TMBuilder, Triple(TheTriple));
 521   else if (TMBuilder.TheTriple != TheTriple) {
 522     if (!TMBuilder.TheTriple.isCompatibleWith(TheTriple))
 523       report_fatal_error("ThinLTO modules with incompatible triples not "
 524                          "supported");
 525     initTMBuilder(TMBuilder, Triple(TMBuilder.TheTriple.merge(TheTriple)));
 526   }
 527
 528   Modules.emplace_back(std::move(*InputOrError));
 529 }
 530
 531 void ThinLTOCodeGenerator::preserveSymbol(StringRef Name) {
 532   PreservedSymbols.insert(Name);
 533 }
 534
 535 void ThinLTOCodeGenerator::crossReferenceSymbol(StringRef Name) {
 536   // FIXME: At the moment, we don't take advantage of this extra information,
 537   // we're conservatively considering cross-references as preserved.
 538   //  CrossReferencedSymbols.insert(Name);
 539   PreservedSymbols.insert(Name);
 540 }
 541
 542 // TargetMachine factory
 543 std::unique_ptr<TargetMachine> TargetMachineBuilder::create() const {
 544   std::string ErrMsg;
 545   const Target *TheTarget =
 546       TargetRegistry::lookupTarget(TheTriple.str(), ErrMsg);
 547   if (!TheTarget) {
 548     report_fatal_error("Can't load target for this Triple: " + ErrMsg);
 549   }
 550
 551   // Use MAttr as the default set of features.
 552   SubtargetFeatures Features(MAttr);
 553   Features.getDefaultSubtargetFeatures(TheTriple);
 554   std::string FeatureStr = Features.getString();
 555
 556   return std::unique_ptr<TargetMachine>(
 557       TheTarget->createTargetMachine(TheTriple.str(), MCpu, FeatureStr, Options,
 558                                      RelocModel, None, CGOptLevel));
 559 }
 560
 561 /**
 562  * Produce the combined summary index from all the bitcode files:
 563  * "thin-link".
 564  */
 565 std::unique_ptr<ModuleSummaryIndex> ThinLTOCodeGenerator::linkCombinedIndex() {
 566   std::unique_ptr<ModuleSummaryIndex> CombinedIndex =
 567       std::make_unique<ModuleSummaryIndex>(/*HaveGVs=*/false);
 568   uint64_t NextModuleId = 0;
 569   for (auto &Mod : Modules) {
 570     auto &M = Mod->getSingleBitcodeModule();
 571     if (Error Err =
 572             M.readSummary(*CombinedIndex, Mod->getName(), NextModuleId++)) {
 573       // FIXME diagnose
 574       logAllUnhandledErrors(
 575           std::move(Err), errs(),
 576           "error: can't create module summary index for buffer: ");
 577       return nullptr;
 578     }
 579   }
 580   return CombinedIndex;
 581 }
 582
 583 struct IsExported {
 584   const StringMap<FunctionImporter::ExportSetTy> &ExportLists;
 585   const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols;
 586
 587   IsExported(const StringMap<FunctionImporter::ExportSetTy> &ExportLists,
 588              const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols)
 589       : ExportLists(ExportLists), GUIDPreservedSymbols(GUIDPreservedSymbols) {}
 590
 591   bool operator()(StringRef ModuleIdentifier, GlobalValue::GUID GUID) const {
 592     const auto &ExportList = ExportLists.find(ModuleIdentifier);
 593     return (ExportList != ExportLists.end() &&
 594             ExportList->second.count(GUID)) ||
 595            GUIDPreservedSymbols.count(GUID);
 596   }
 597 };
 598
 599 struct IsPrevailing {
 600   const DenseMap<GlobalValue::GUID, const GlobalValueSummary *> &PrevailingCopy;
 601   IsPrevailing(const DenseMap<GlobalValue::GUID, const GlobalValueSummary *>
 602                    &PrevailingCopy)
 603       : PrevailingCopy(PrevailingCopy) {}
 604
 605   bool operator()(GlobalValue::GUID GUID, const GlobalValueSummary *S) const {
 606     const auto &Prevailing = PrevailingCopy.find(GUID);
 607     // Not in map means that there was only one copy, which must be prevailing.
 608     if (Prevailing == PrevailingCopy.end())
 609       return true;
 610     return Prevailing->second == S;
 611   };
 612 };
 613
 614 static void computeDeadSymbolsInIndex(
 615     ModuleSummaryIndex &Index,
 616     const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) {
 617   // We have no symbols resolution available. And can't do any better now in the
 618   // case where the prevailing symbol is in a native object. It can be refined
 619   // with linker information in the future.
 620   auto isPrevailing = [&](GlobalValue::GUID G) {
 621     return PrevailingType::Unknown;
 622   };
 623   computeDeadSymbolsWithConstProp(Index, GUIDPreservedSymbols, isPrevailing,
 624                                   /* ImportEnabled = */ true);
 625 }
 626
 627 /**
 628  * Perform promotion and renaming of exported internal functions.
 629  * Index is updated to reflect linkage changes from weak resolution.
 630  */
 631 void ThinLTOCodeGenerator::promote(Module &TheModule, ModuleSummaryIndex &Index,
 632                                    const lto::InputFile &File) {
 633   auto ModuleCount = Index.modulePaths().size();
 634   auto ModuleIdentifier = TheModule.getModuleIdentifier();
 635
 636   // Collect for each module the list of function it defines (GUID -> Summary).
 637   StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries;
 638   Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
 639
 640   // Convert the preserved symbols set from string to GUID
 641   auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
 642       PreservedSymbols, Triple(TheModule.getTargetTriple()));
 643
 644   // Add used symbol to the preserved symbols.
 645   addUsedSymbolToPreservedGUID(File, GUIDPreservedSymbols);
 646
 647   // Compute "dead" symbols, we don't want to import/export these!
 648   computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
 649
 650   // Generate import/export list
 651   StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
 652   StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
 653   ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
 654                            ExportLists);
 655
 656   DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
 657   computePrevailingCopies(Index, PrevailingCopy);
 658
 659   // Resolve prevailing symbols
 660   StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
 661   resolvePrevailingInIndex(Index, ResolvedODR, GUIDPreservedSymbols,
 662                            PrevailingCopy);
 663
 664   thinLTOResolvePrevailingInModule(
 665       TheModule, ModuleToDefinedGVSummaries[ModuleIdentifier]);
 666
 667   // Promote the exported values in the index, so that they are promoted
 668   // in the module.
 669   thinLTOInternalizeAndPromoteInIndex(
 670       Index, IsExported(ExportLists, GUIDPreservedSymbols),
 671       IsPrevailing(PrevailingCopy));
 672
 673   promoteModule(TheModule, Index);
 674 }
 675
 676 /**
 677  * Perform cross-module importing for the module identified by ModuleIdentifier.
 678  */
 679 void ThinLTOCodeGenerator::crossModuleImport(Module &TheModule,
 680                                              ModuleSummaryIndex &Index,
 681                                              const lto::InputFile &File) {
 682   auto ModuleMap = generateModuleMap(Modules);
 683   auto ModuleCount = Index.modulePaths().size();
 684
 685   // Collect for each module the list of function it defines (GUID -> Summary).
 686   StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
 687   Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
 688
 689   // Convert the preserved symbols set from string to GUID
 690   auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
 691       PreservedSymbols, Triple(TheModule.getTargetTriple()));
 692
 693   addUsedSymbolToPreservedGUID(File, GUIDPreservedSymbols);
 694
 695   // Compute "dead" symbols, we don't want to import/export these!
 696   computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
 697
 698   // Generate import/export list
 699   StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
 700   StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
 701   ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
 702                            ExportLists);
 703   auto &ImportList = ImportLists[TheModule.getModuleIdentifier()];
 704
 705   crossImportIntoModule(TheModule, Index, ModuleMap, ImportList);
 706 }
 707
 708 /**
 709  * Compute the list of summaries needed for importing into module.
 710  */
 711 void ThinLTOCodeGenerator::gatherImportedSummariesForModule(
 712     Module &TheModule, ModuleSummaryIndex &Index,
 713     std::map<std::string, GVSummaryMapTy> &ModuleToSummariesForIndex,
 714     const lto::InputFile &File) {
 715   auto ModuleCount = Index.modulePaths().size();
 716   auto ModuleIdentifier = TheModule.getModuleIdentifier();
 717
 718   // Collect for each module the list of function it defines (GUID -> Summary).
 719   StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
 720   Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
 721
 722   // Convert the preserved symbols set from string to GUID
 723   auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
 724       PreservedSymbols, Triple(TheModule.getTargetTriple()));
 725
 726   addUsedSymbolToPreservedGUID(File, GUIDPreservedSymbols);
 727
 728   // Compute "dead" symbols, we don't want to import/export these!
 729   computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
 730
 731   // Generate import/export list
 732   StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
 733   StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
 734   ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
 735                            ExportLists);
 736
 737   llvm::gatherImportedSummariesForModule(
 738       ModuleIdentifier, ModuleToDefinedGVSummaries,
 739       ImportLists[ModuleIdentifier], ModuleToSummariesForIndex);
 740 }
 741
 742 /**
 743  * Emit the list of files needed for importing into module.
 744  */
 745 void ThinLTOCodeGenerator::emitImports(Module &TheModule, StringRef OutputName,
 746                                        ModuleSummaryIndex &Index,
 747                                        const lto::InputFile &File) {
 748   auto ModuleCount = Index.modulePaths().size();
 749   auto ModuleIdentifier = TheModule.getModuleIdentifier();
 750
 751   // Collect for each module the list of function it defines (GUID -> Summary).
 752   StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
 753   Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
 754
 755   // Convert the preserved symbols set from string to GUID
 756   auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
 757       PreservedSymbols, Triple(TheModule.getTargetTriple()));
 758
 759   addUsedSymbolToPreservedGUID(File, GUIDPreservedSymbols);
 760
 761   // Compute "dead" symbols, we don't want to import/export these!
 762   computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
 763
 764   // Generate import/export list
 765   StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
 766   StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
 767   ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
 768                            ExportLists);
 769
 770   std::map<std::string, GVSummaryMapTy> ModuleToSummariesForIndex;
 771   llvm::gatherImportedSummariesForModule(
 772       ModuleIdentifier, ModuleToDefinedGVSummaries,
 773       ImportLists[ModuleIdentifier], ModuleToSummariesForIndex);
 774
 775   std::error_code EC;
 776   if ((EC = EmitImportsFiles(ModuleIdentifier, OutputName,
 777                              ModuleToSummariesForIndex)))
 778     report_fatal_error(Twine("Failed to open ") + OutputName +
 779                        " to save imports lists\n");
 780 }
 781
 782 /**
 783  * Perform internalization. Runs promote and internalization together.
 784  * Index is updated to reflect linkage changes.
 785  */
 786 void ThinLTOCodeGenerator::internalize(Module &TheModule,
 787                                        ModuleSummaryIndex &Index,
 788                                        const lto::InputFile &File) {
 789   initTMBuilder(TMBuilder, Triple(TheModule.getTargetTriple()));
 790   auto ModuleCount = Index.modulePaths().size();
 791   auto ModuleIdentifier = TheModule.getModuleIdentifier();
 792
 793   // Convert the preserved symbols set from string to GUID
 794   auto GUIDPreservedSymbols =
 795       computeGUIDPreservedSymbols(PreservedSymbols, TMBuilder.TheTriple);
 796
 797   addUsedSymbolToPreservedGUID(File, GUIDPreservedSymbols);
 798
 799   // Collect for each module the list of function it defines (GUID -> Summary).
 800   StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
 801   Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
 802
 803   // Compute "dead" symbols, we don't want to import/export these!
 804   computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
 805
 806   // Generate import/export list
 807   StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
 808   StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
 809   ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
 810                            ExportLists);
 811   auto &ExportList = ExportLists[ModuleIdentifier];
 812
 813   // Be friendly and don't nuke totally the module when the client didn't
 814   // supply anything to preserve.
 815   if (ExportList.empty() && GUIDPreservedSymbols.empty())
 816     return;
 817
 818   DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
 819   computePrevailingCopies(Index, PrevailingCopy);
 820
 821   // Resolve prevailing symbols
 822   StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
 823   resolvePrevailingInIndex(Index, ResolvedODR, GUIDPreservedSymbols,
 824                            PrevailingCopy);
 825
 826   // Promote the exported values in the index, so that they are promoted
 827   // in the module.
 828   thinLTOInternalizeAndPromoteInIndex(
 829       Index, IsExported(ExportLists, GUIDPreservedSymbols),
 830       IsPrevailing(PrevailingCopy));
 831
 832   promoteModule(TheModule, Index);
 833
 834   // Internalization
 835   thinLTOResolvePrevailingInModule(
 836       TheModule, ModuleToDefinedGVSummaries[ModuleIdentifier]);
 837
 838   thinLTOInternalizeModule(TheModule,
 839                            ModuleToDefinedGVSummaries[ModuleIdentifier]);
 840 }
 841
 842 /**
 843  * Perform post-importing ThinLTO optimizations.
 844  */
 845 void ThinLTOCodeGenerator::optimize(Module &TheModule) {
 846   initTMBuilder(TMBuilder, Triple(TheModule.getTargetTriple()));
 847
 848   // Optimize now
 849   optimizeModule(TheModule, *TMBuilder.create(), OptLevel, Freestanding,
 850                  nullptr);
 851 }
 852
 853 /// Write out the generated object file, either from CacheEntryPath or from
 854 /// OutputBuffer, preferring hard-link when possible.
 855 /// Returns the path to the generated file in SavedObjectsDirectoryPath.
 856 std::string
 857 ThinLTOCodeGenerator::writeGeneratedObject(int count, StringRef CacheEntryPath,
 858                                            const MemoryBuffer &OutputBuffer) {
 859   auto ArchName = TMBuilder.TheTriple.getArchName();
 860   SmallString<128> OutputPath(SavedObjectsDirectoryPath);
 861   llvm::sys::path::append(OutputPath,
 862                           Twine(count) + "." + ArchName + ".thinlto.o");
 863   OutputPath.c_str(); // Ensure the string is null terminated.
 864   if (sys::fs::exists(OutputPath))
 865     sys::fs::remove(OutputPath);
 866
 867   // We don't return a memory buffer to the linker, just a list of files.
 868   if (!CacheEntryPath.empty()) {
 869     // Cache is enabled, hard-link the entry (or copy if hard-link fails).
 870     auto Err = sys::fs::create_hard_link(CacheEntryPath, OutputPath);
 871     if (!Err)
 872       return OutputPath.str();
 873     // Hard linking failed, try to copy.
 874     Err = sys::fs::copy_file(CacheEntryPath, OutputPath);
 875     if (!Err)
 876       return OutputPath.str();
 877     // Copy failed (could be because the CacheEntry was removed from the cache
 878     // in the meantime by another process), fall back and try to write down the
 879     // buffer to the output.
 880     errs() << "error: can't link or copy from cached entry '" << CacheEntryPath
 881            << "' to '" << OutputPath << "'\n";
 882   }
 883   // No cache entry, just write out the buffer.
 884   std::error_code Err;
 885   raw_fd_ostream OS(OutputPath, Err, sys::fs::OF_None);
 886   if (Err)
 887     report_fatal_error("Can't open output '" + OutputPath + "'\n");
 888   OS << OutputBuffer.getBuffer();
 889   return OutputPath.str();
 890 }
 891
 892 // Main entry point for the ThinLTO processing
 893 void ThinLTOCodeGenerator::run() {
 894   // Prepare the resulting object vector
 895   assert(ProducedBinaries.empty() && "The generator should not be reused");
 896   if (SavedObjectsDirectoryPath.empty())
 897     ProducedBinaries.resize(Modules.size());
 898   else {
 899     sys::fs::create_directories(SavedObjectsDirectoryPath);
 900     bool IsDir;
 901     sys::fs::is_directory(SavedObjectsDirectoryPath, IsDir);
 902     if (!IsDir)
 903       report_fatal_error("Unexistent dir: '" + SavedObjectsDirectoryPath + "'");
 904     ProducedBinaryFiles.resize(Modules.size());
 905   }
 906
 907   if (CodeGenOnly) {
 908     // Perform only parallel codegen and return.
 909     ThreadPool Pool;
 910     int count = 0;
 911     for (auto &Mod : Modules) {
 912       Pool.async([&](int count) {
 913         LLVMContext Context;
 914         Context.setDiscardValueNames(LTODiscardValueNames);
 915
 916         // Parse module now
 917         auto TheModule = loadModuleFromInput(Mod.get(), Context, false,
 918                                              /*IsImporting*/ false);
 919
 920         // CodeGen
 921         auto OutputBuffer = codegenModule(*TheModule, *TMBuilder.create());
 922         if (SavedObjectsDirectoryPath.empty())
 923           ProducedBinaries[count] = std::move(OutputBuffer);
 924         else
 925           ProducedBinaryFiles[count] =
 926               writeGeneratedObject(count, "", *OutputBuffer);
 927       }, count++);
 928     }
 929
 930     return;
 931   }
 932
 933   // Sequential linking phase
 934   auto Index = linkCombinedIndex();
 935
 936   // Save temps: index.
 937   if (!SaveTempsDir.empty()) {
 938     auto SaveTempPath = SaveTempsDir + "index.bc";
 939     std::error_code EC;
 940     raw_fd_ostream OS(SaveTempPath, EC, sys::fs::OF_None);
 941     if (EC)
 942       report_fatal_error(Twine("Failed to open ") + SaveTempPath +
 943                          " to save optimized bitcode\n");
 944     WriteIndexToFile(*Index, OS);
 945   }
 946
 947
 948   // Prepare the module map.
 949   auto ModuleMap = generateModuleMap(Modules);
 950   auto ModuleCount = Modules.size();
 951
 952   // Collect for each module the list of function it defines (GUID -> Summary).
 953   StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
 954   Index->collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
 955
 956   // Convert the preserved symbols set from string to GUID, this is needed for
 957   // computing the caching hash and the internalization.
 958   auto GUIDPreservedSymbols =
 959       computeGUIDPreservedSymbols(PreservedSymbols, TMBuilder.TheTriple);
 960
 961   // Add used symbol from inputs to the preserved symbols.
 962   for (const auto &M : Modules)
 963     addUsedSymbolToPreservedGUID(*M, GUIDPreservedSymbols);
 964
 965   // Compute "dead" symbols, we don't want to import/export these!
 966   computeDeadSymbolsInIndex(*Index, GUIDPreservedSymbols);
 967
 968   // Synthesize entry counts for functions in the combined index.
 969   computeSyntheticCounts(*Index);
 970
 971   // Perform index-based WPD. This will return immediately if there are
 972   // no index entries in the typeIdMetadata map (e.g. if we are instead
 973   // performing IR-based WPD in hybrid regular/thin LTO mode).
 974   std::map<ValueInfo, std::vector<VTableSlotSummary>> LocalWPDTargetsMap;
 975   std::set<GlobalValue::GUID> ExportedGUIDs;
 976   runWholeProgramDevirtOnIndex(*Index, ExportedGUIDs, LocalWPDTargetsMap);
 977   for (auto GUID : ExportedGUIDs)
 978     GUIDPreservedSymbols.insert(GUID);
 979
 980   // Collect the import/export lists for all modules from the call-graph in the
 981   // combined index.
 982   StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
 983   StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
 984   ComputeCrossModuleImport(*Index, ModuleToDefinedGVSummaries, ImportLists,
 985                            ExportLists);
 986
 987   // We use a std::map here to be able to have a defined ordering when
 988   // producing a hash for the cache entry.
 989   // FIXME: we should be able to compute the caching hash for the entry based
 990   // on the index, and nuke this map.
 991   StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
 992
 993   DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
 994   computePrevailingCopies(*Index, PrevailingCopy);
 995
 996   // Resolve prevailing symbols, this has to be computed early because it
 997   // impacts the caching.
 998   resolvePrevailingInIndex(*Index, ResolvedODR, GUIDPreservedSymbols,
 999                            PrevailingCopy);
1000
1001   // Use global summary-based analysis to identify symbols that can be
1002   // internalized (because they aren't exported or preserved as per callback).
1003   // Changes are made in the index, consumed in the ThinLTO backends.
1004   updateIndexWPDForExports(*Index,
1005                            IsExported(ExportLists, GUIDPreservedSymbols),
1006                            LocalWPDTargetsMap);
1007   thinLTOInternalizeAndPromoteInIndex(
1008       *Index, IsExported(ExportLists, GUIDPreservedSymbols),
1009       IsPrevailing(PrevailingCopy));
1010
1011   // Make sure that every module has an entry in the ExportLists, ImportList,
1012   // GVSummary and ResolvedODR maps to enable threaded access to these maps
1013   // below.
1014   for (auto &Module : Modules) {
1015     auto ModuleIdentifier = Module->getName();
1016     ExportLists[ModuleIdentifier];
1017     ImportLists[ModuleIdentifier];
1018     ResolvedODR[ModuleIdentifier];
1019     ModuleToDefinedGVSummaries[ModuleIdentifier];
1020   }
1021
1022   // Compute the ordering we will process the inputs: the rough heuristic here
1023   // is to sort them per size so that the largest module get schedule as soon as
1024   // possible. This is purely a compile-time optimization.
1025   std::vector<int> ModulesOrdering;
1026   ModulesOrdering.resize(Modules.size());
1027   std::iota(ModulesOrdering.begin(), ModulesOrdering.end(), 0);
1028   llvm::sort(ModulesOrdering, [&](int LeftIndex, int RightIndex) {
1029     auto LSize =
1030         Modules[LeftIndex]->getSingleBitcodeModule().getBuffer().size();
1031     auto RSize =
1032         Modules[RightIndex]->getSingleBitcodeModule().getBuffer().size();
1033     return LSize > RSize;
1034   });
1035
1036   // Parallel optimizer + codegen
1037   {
1038     ThreadPool Pool(ThreadCount);
1039     for (auto IndexCount : ModulesOrdering) {
1040       auto &Mod = Modules[IndexCount];
1041       Pool.async([&](int count) {
1042         auto ModuleIdentifier = Mod->getName();
1043         auto &ExportList = ExportLists[ModuleIdentifier];
1044
1045         auto &DefinedGVSummaries = ModuleToDefinedGVSummaries[ModuleIdentifier];
1046
1047         // The module may be cached, this helps handling it.
1048         ModuleCacheEntry CacheEntry(CacheOptions.Path, *Index, ModuleIdentifier,
1049                                     ImportLists[ModuleIdentifier], ExportList,
1050                                     ResolvedODR[ModuleIdentifier],
1051                                     DefinedGVSummaries, OptLevel, Freestanding,
1052                                     TMBuilder);
1053         auto CacheEntryPath = CacheEntry.getEntryPath();
1054
1055         {
1056           auto ErrOrBuffer = CacheEntry.tryLoadingBuffer();
1057           LLVM_DEBUG(dbgs() << "Cache " << (ErrOrBuffer ? "hit" : "miss")
1058                             << " '" << CacheEntryPath << "' for buffer "
1059                             << count << " " << ModuleIdentifier << "\n");
1060
1061           if (ErrOrBuffer) {
1062             // Cache Hit!
1063             if (SavedObjectsDirectoryPath.empty())
1064               ProducedBinaries[count] = std::move(ErrOrBuffer.get());
1065             else
1066               ProducedBinaryFiles[count] = writeGeneratedObject(
1067                   count, CacheEntryPath, *ErrOrBuffer.get());
1068             return;
1069           }
1070         }
1071
1072         LLVMContext Context;
1073         Context.setDiscardValueNames(LTODiscardValueNames);
1074         Context.enableDebugTypeODRUniquing();
1075         auto DiagFileOrErr = lto::setupOptimizationRemarks(
1076             Context, RemarksFilename, RemarksPasses, RemarksFormat,
1077             RemarksWithHotness, count);
1078         if (!DiagFileOrErr) {
1079           errs() << "Error: " << toString(DiagFileOrErr.takeError()) << "\n";
1080           report_fatal_error("ThinLTO: Can't get an output file for the "
1081                              "remarks");
1082         }
1083
1084         // Parse module now
1085         auto TheModule = loadModuleFromInput(Mod.get(), Context, false,
1086                                              /*IsImporting*/ false);
1087
1088         // Save temps: original file.
1089         saveTempBitcode(*TheModule, SaveTempsDir, count, ".0.original.bc");
1090
1091         auto &ImportList = ImportLists[ModuleIdentifier];
1092         // Run the main process now, and generates a binary
1093         auto OutputBuffer = ProcessThinLTOModule(
1094             *TheModule, *Index, ModuleMap, *TMBuilder.create(), ImportList,
1095             ExportList, GUIDPreservedSymbols,
1096             ModuleToDefinedGVSummaries[ModuleIdentifier], CacheOptions,
1097             DisableCodeGen, SaveTempsDir, Freestanding, OptLevel, count);
1098
1099         // Commit to the cache (if enabled)
1100         CacheEntry.write(*OutputBuffer);
1101
1102         if (SavedObjectsDirectoryPath.empty()) {
1103           // We need to generated a memory buffer for the linker.
1104           if (!CacheEntryPath.empty()) {
1105             // When cache is enabled, reload from the cache if possible.
1106             // Releasing the buffer from the heap and reloading it from the
1107             // cache file with mmap helps us to lower memory pressure.
1108             // The freed memory can be used for the next input file.
1109             // The final binary link will read from the VFS cache (hopefully!)
1110             // or from disk (if the memory pressure was too high).
1111             auto ReloadedBufferOrErr = CacheEntry.tryLoadingBuffer();
1112             if (auto EC = ReloadedBufferOrErr.getError()) {
1113               // On error, keep the preexisting buffer and print a diagnostic.
1114               errs() << "error: can't reload cached file '" << CacheEntryPath
1115                      << "': " << EC.message() << "\n";
1116             } else {
1117               OutputBuffer = std::move(*ReloadedBufferOrErr);
1118             }
1119           }
1120           ProducedBinaries[count] = std::move(OutputBuffer);
1121           return;
1122         }
1123         ProducedBinaryFiles[count] = writeGeneratedObject(
1124             count, CacheEntryPath, *OutputBuffer);
1125       }, IndexCount);
1126     }
1127   }
1128
1129   pruneCache(CacheOptions.Path, CacheOptions.Policy);
1130
1131   // If statistics were requested, print them out now.
1132   if (llvm::AreStatisticsEnabled())
1133     llvm::PrintStatistics();
1134   reportAndResetTimings();
1135 }