llvm/lib/LTO/ThinLTOCodeGenerator.cpp

   1 //===-ThinLTOCodeGenerator.cpp - LLVM Link Time Optimizer -----------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file implements the Thin Link Time Optimization library. This library is
  10 // intended to be used by linker to optimize code at link time.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "llvm/LTO/legacy/ThinLTOCodeGenerator.h"
  15 #include "llvm/Support/CommandLine.h"
  16
  17 #include "llvm/ADT/ScopeExit.h"
  18 #include "llvm/ADT/Statistic.h"
  19 #include "llvm/ADT/StringExtras.h"
  20 #include "llvm/Analysis/AliasAnalysis.h"
  21 #include "llvm/Analysis/ModuleSummaryAnalysis.h"
  22 #include "llvm/Analysis/ProfileSummaryInfo.h"
  23 #include "llvm/Analysis/TargetLibraryInfo.h"
  24 #include "llvm/Bitcode/BitcodeReader.h"
  25 #include "llvm/Bitcode/BitcodeWriter.h"
  26 #include "llvm/Bitcode/BitcodeWriterPass.h"
  27 #include "llvm/Config/llvm-config.h"
  28 #include "llvm/IR/DebugInfo.h"
  29 #include "llvm/IR/DiagnosticPrinter.h"
  30 #include "llvm/IR/LegacyPassManager.h"
  31 #include "llvm/IR/LLVMContext.h"
  32 #include "llvm/IR/LLVMRemarkStreamer.h"
  33 #include "llvm/IR/Mangler.h"
  34 #include "llvm/IR/PassTimingInfo.h"
  35 #include "llvm/IR/Verifier.h"
  36 #include "llvm/IRReader/IRReader.h"
  37 #include "llvm/LTO/LTO.h"
  38 #include "llvm/LTO/SummaryBasedOptimizations.h"
  39 #include "llvm/MC/SubtargetFeature.h"
  40 #include "llvm/MC/TargetRegistry.h"
  41 #include "llvm/Object/IRObjectFile.h"
  42 #include "llvm/Passes/PassBuilder.h"
  43 #include "llvm/Passes/StandardInstrumentations.h"
  44 #include "llvm/Remarks/HotnessThresholdParser.h"
  45 #include "llvm/Support/CachePruning.h"
  46 #include "llvm/Support/Debug.h"
  47 #include "llvm/Support/Error.h"
  48 #include "llvm/Support/FileUtilities.h"
  49 #include "llvm/Support/Path.h"
  50 #include "llvm/Support/SHA1.h"
  51 #include "llvm/Support/SmallVectorMemoryBuffer.h"
  52 #include "llvm/Support/ThreadPool.h"
  53 #include "llvm/Support/Threading.h"
  54 #include "llvm/Support/ToolOutputFile.h"
  55 #include "llvm/Target/TargetMachine.h"
  56 #include "llvm/Transforms/IPO/FunctionAttrs.h"
  57 #include "llvm/Transforms/IPO/FunctionImport.h"
  58 #include "llvm/Transforms/IPO/Internalize.h"
  59 #include "llvm/Transforms/IPO/WholeProgramDevirt.h"
  60 #include "llvm/Transforms/ObjCARC.h"
  61 #include "llvm/Transforms/Utils/FunctionImportUtils.h"
  62
  63 #include <numeric>
  64
  65 #if !defined(_MSC_VER) && !defined(__MINGW32__)
  66 #include <unistd.h>
  67 #else
  68 #include <io.h>
  69 #endif
  70
  71 using namespace llvm;
  72
  73 #define DEBUG_TYPE "thinlto"
  74
  75 namespace llvm {
  76 // Flags -discard-value-names, defined in LTOCodeGenerator.cpp
  77 extern cl::opt<bool> LTODiscardValueNames;
  78 extern cl::opt<std::string> RemarksFilename;
  79 extern cl::opt<std::string> RemarksPasses;
  80 extern cl::opt<bool> RemarksWithHotness;
  81 extern cl::opt<std::optional<uint64_t>, false, remarks::HotnessThresholdParser>
  82     RemarksHotnessThreshold;
  83 extern cl::opt<std::string> RemarksFormat;
  84 }
  85
  86 namespace {
  87
  88 // Default to using all available threads in the system, but using only one
  89 // thred per core, as indicated by the usage of
  90 // heavyweight_hardware_concurrency() below.
  91 static cl::opt<int> ThreadCount("threads", cl::init(0));
  92
  93 // Simple helper to save temporary files for debug.
  94 static void saveTempBitcode(const Module &TheModule, StringRef TempDir,
  95                             unsigned count, StringRef Suffix) {
  96   if (TempDir.empty())
  97     return;
  98   // User asked to save temps, let dump the bitcode file after import.
  99   std::string SaveTempPath = (TempDir + llvm::Twine(count) + Suffix).str();
 100   std::error_code EC;
 101   raw_fd_ostream OS(SaveTempPath, EC, sys::fs::OF_None);
 102   if (EC)
 103     report_fatal_error(Twine("Failed to open ") + SaveTempPath +
 104                        " to save optimized bitcode\n");
 105   WriteBitcodeToFile(TheModule, OS, /* ShouldPreserveUseListOrder */ true);
 106 }
 107
 108 static const GlobalValueSummary *
 109 getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) {
 110   // If there is any strong definition anywhere, get it.
 111   auto StrongDefForLinker = llvm::find_if(
 112       GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
 113         auto Linkage = Summary->linkage();
 114         return !GlobalValue::isAvailableExternallyLinkage(Linkage) &&
 115                !GlobalValue::isWeakForLinker(Linkage);
 116       });
 117   if (StrongDefForLinker != GVSummaryList.end())
 118     return StrongDefForLinker->get();
 119   // Get the first *linker visible* definition for this global in the summary
 120   // list.
 121   auto FirstDefForLinker = llvm::find_if(
 122       GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
 123         auto Linkage = Summary->linkage();
 124         return !GlobalValue::isAvailableExternallyLinkage(Linkage);
 125       });
 126   // Extern templates can be emitted as available_externally.
 127   if (FirstDefForLinker == GVSummaryList.end())
 128     return nullptr;
 129   return FirstDefForLinker->get();
 130 }
 131
 132 // Populate map of GUID to the prevailing copy for any multiply defined
 133 // symbols. Currently assume first copy is prevailing, or any strong
 134 // definition. Can be refined with Linker information in the future.
 135 static void computePrevailingCopies(
 136     const ModuleSummaryIndex &Index,
 137     DenseMap<GlobalValue::GUID, const GlobalValueSummary *> &PrevailingCopy) {
 138   auto HasMultipleCopies = [&](const GlobalValueSummaryList &GVSummaryList) {
 139     return GVSummaryList.size() > 1;
 140   };
 141
 142   for (auto &I : Index) {
 143     if (HasMultipleCopies(I.second.SummaryList))
 144       PrevailingCopy[I.first] =
 145           getFirstDefinitionForLinker(I.second.SummaryList);
 146   }
 147 }
 148
 149 static StringMap<lto::InputFile *>
 150 generateModuleMap(std::vector<std::unique_ptr<lto::InputFile>> &Modules) {
 151   StringMap<lto::InputFile *> ModuleMap;
 152   for (auto &M : Modules) {
 153     assert(!ModuleMap.contains(M->getName()) &&
 154            "Expect unique Buffer Identifier");
 155     ModuleMap[M->getName()] = M.get();
 156   }
 157   return ModuleMap;
 158 }
 159
 160 static void promoteModule(Module &TheModule, const ModuleSummaryIndex &Index,
 161                           bool ClearDSOLocalOnDeclarations) {
 162   if (renameModuleForThinLTO(TheModule, Index, ClearDSOLocalOnDeclarations))
 163     report_fatal_error("renameModuleForThinLTO failed");
 164 }
 165
 166 namespace {
 167 class ThinLTODiagnosticInfo : public DiagnosticInfo {
 168   const Twine &Msg;
 169 public:
 170   ThinLTODiagnosticInfo(const Twine &DiagMsg,
 171                         DiagnosticSeverity Severity = DS_Error)
 172       : DiagnosticInfo(DK_Linker, Severity), Msg(DiagMsg) {}
 173   void print(DiagnosticPrinter &DP) const override { DP << Msg; }
 174 };
 175 }
 176
 177 /// Verify the module and strip broken debug info.
 178 static void verifyLoadedModule(Module &TheModule) {
 179   bool BrokenDebugInfo = false;
 180   if (verifyModule(TheModule, &dbgs(), &BrokenDebugInfo))
 181     report_fatal_error("Broken module found, compilation aborted!");
 182   if (BrokenDebugInfo) {
 183     TheModule.getContext().diagnose(ThinLTODiagnosticInfo(
 184         "Invalid debug info found, debug info will be stripped", DS_Warning));
 185     StripDebugInfo(TheModule);
 186   }
 187 }
 188
 189 static std::unique_ptr<Module> loadModuleFromInput(lto::InputFile *Input,
 190                                                    LLVMContext &Context,
 191                                                    bool Lazy,
 192                                                    bool IsImporting) {
 193   auto &Mod = Input->getSingleBitcodeModule();
 194   SMDiagnostic Err;
 195   Expected<std::unique_ptr<Module>> ModuleOrErr =
 196       Lazy ? Mod.getLazyModule(Context,
 197                                /* ShouldLazyLoadMetadata */ true, IsImporting)
 198            : Mod.parseModule(Context);
 199   if (!ModuleOrErr) {
 200     handleAllErrors(ModuleOrErr.takeError(), [&](ErrorInfoBase &EIB) {
 201       SMDiagnostic Err = SMDiagnostic(Mod.getModuleIdentifier(),
 202                                       SourceMgr::DK_Error, EIB.message());
 203       Err.print("ThinLTO", errs());
 204     });
 205     report_fatal_error("Can't load module, abort.");
 206   }
 207   if (!Lazy)
 208     verifyLoadedModule(*ModuleOrErr.get());
 209   return std::move(*ModuleOrErr);
 210 }
 211
 212 static void
 213 crossImportIntoModule(Module &TheModule, const ModuleSummaryIndex &Index,
 214                       StringMap<lto::InputFile *> &ModuleMap,
 215                       const FunctionImporter::ImportMapTy &ImportList,
 216                       bool ClearDSOLocalOnDeclarations) {
 217   auto Loader = [&](StringRef Identifier) {
 218     auto &Input = ModuleMap[Identifier];
 219     return loadModuleFromInput(Input, TheModule.getContext(),
 220                                /*Lazy=*/true, /*IsImporting*/ true);
 221   };
 222
 223   FunctionImporter Importer(Index, Loader, ClearDSOLocalOnDeclarations);
 224   Expected<bool> Result = Importer.importFunctions(TheModule, ImportList);
 225   if (!Result) {
 226     handleAllErrors(Result.takeError(), [&](ErrorInfoBase &EIB) {
 227       SMDiagnostic Err = SMDiagnostic(TheModule.getModuleIdentifier(),
 228                                       SourceMgr::DK_Error, EIB.message());
 229       Err.print("ThinLTO", errs());
 230     });
 231     report_fatal_error("importFunctions failed");
 232   }
 233   // Verify again after cross-importing.
 234   verifyLoadedModule(TheModule);
 235 }
 236
 237 static void optimizeModule(Module &TheModule, TargetMachine &TM,
 238                            unsigned OptLevel, bool Freestanding,
 239                            bool DebugPassManager, ModuleSummaryIndex *Index) {
 240   std::optional<PGOOptions> PGOOpt;
 241   LoopAnalysisManager LAM;
 242   FunctionAnalysisManager FAM;
 243   CGSCCAnalysisManager CGAM;
 244   ModuleAnalysisManager MAM;
 245
 246   PassInstrumentationCallbacks PIC;
 247   StandardInstrumentations SI(TheModule.getContext(), DebugPassManager);
 248   SI.registerCallbacks(PIC, &MAM);
 249   PipelineTuningOptions PTO;
 250   PTO.LoopVectorization = true;
 251   PTO.SLPVectorization = true;
 252   PassBuilder PB(&TM, PTO, PGOOpt, &PIC);
 253
 254   std::unique_ptr<TargetLibraryInfoImpl> TLII(
 255       new TargetLibraryInfoImpl(Triple(TM.getTargetTriple())));
 256   if (Freestanding)
 257     TLII->disableAllFunctions();
 258   FAM.registerPass([&] { return TargetLibraryAnalysis(*TLII); });
 259
 260   // Register all the basic analyses with the managers.
 261   PB.registerModuleAnalyses(MAM);
 262   PB.registerCGSCCAnalyses(CGAM);
 263   PB.registerFunctionAnalyses(FAM);
 264   PB.registerLoopAnalyses(LAM);
 265   PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
 266
 267   ModulePassManager MPM;
 268
 269   OptimizationLevel OL;
 270
 271   switch (OptLevel) {
 272   default:
 273     llvm_unreachable("Invalid optimization level");
 274   case 0:
 275     OL = OptimizationLevel::O0;
 276     break;
 277   case 1:
 278     OL = OptimizationLevel::O1;
 279     break;
 280   case 2:
 281     OL = OptimizationLevel::O2;
 282     break;
 283   case 3:
 284     OL = OptimizationLevel::O3;
 285     break;
 286   }
 287
 288   MPM.addPass(PB.buildThinLTODefaultPipeline(OL, Index));
 289
 290   MPM.run(TheModule, MAM);
 291 }
 292
 293 static void
 294 addUsedSymbolToPreservedGUID(const lto::InputFile &File,
 295                              DenseSet<GlobalValue::GUID> &PreservedGUID) {
 296   for (const auto &Sym : File.symbols()) {
 297     if (Sym.isUsed())
 298       PreservedGUID.insert(GlobalValue::getGUID(Sym.getIRName()));
 299   }
 300 }
 301
 302 // Convert the PreservedSymbols map from "Name" based to "GUID" based.
 303 static void computeGUIDPreservedSymbols(const lto::InputFile &File,
 304                                         const StringSet<> &PreservedSymbols,
 305                                         const Triple &TheTriple,
 306                                         DenseSet<GlobalValue::GUID> &GUIDs) {
 307   // Iterate the symbols in the input file and if the input has preserved symbol
 308   // compute the GUID for the symbol.
 309   for (const auto &Sym : File.symbols()) {
 310     if (PreservedSymbols.count(Sym.getName()) && !Sym.getIRName().empty())
 311       GUIDs.insert(GlobalValue::getGUID(GlobalValue::getGlobalIdentifier(
 312           Sym.getIRName(), GlobalValue::ExternalLinkage, "")));
 313   }
 314 }
 315
 316 static DenseSet<GlobalValue::GUID>
 317 computeGUIDPreservedSymbols(const lto::InputFile &File,
 318                             const StringSet<> &PreservedSymbols,
 319                             const Triple &TheTriple) {
 320   DenseSet<GlobalValue::GUID> GUIDPreservedSymbols(PreservedSymbols.size());
 321   computeGUIDPreservedSymbols(File, PreservedSymbols, TheTriple,
 322                               GUIDPreservedSymbols);
 323   return GUIDPreservedSymbols;
 324 }
 325
 326 std::unique_ptr<MemoryBuffer> codegenModule(Module &TheModule,
 327                                             TargetMachine &TM) {
 328   SmallVector<char, 128> OutputBuffer;
 329
 330   // CodeGen
 331   {
 332     raw_svector_ostream OS(OutputBuffer);
 333     legacy::PassManager PM;
 334
 335     // If the bitcode files contain ARC code and were compiled with optimization,
 336     // the ObjCARCContractPass must be run, so do it unconditionally here.
 337     PM.add(createObjCARCContractPass());
 338
 339     // Setup the codegen now.
 340     if (TM.addPassesToEmitFile(PM, OS, nullptr, CGFT_ObjectFile,
 341                                /* DisableVerify */ true))
 342       report_fatal_error("Failed to setup codegen");
 343
 344     // Run codegen now. resulting binary is in OutputBuffer.
 345     PM.run(TheModule);
 346   }
 347   return std::make_unique<SmallVectorMemoryBuffer>(
 348       std::move(OutputBuffer), /*RequiresNullTerminator=*/false);
 349 }
 350
 351 /// Manage caching for a single Module.
 352 class ModuleCacheEntry {
 353   SmallString<128> EntryPath;
 354
 355 public:
 356   // Create a cache entry. This compute a unique hash for the Module considering
 357   // the current list of export/import, and offer an interface to query to
 358   // access the content in the cache.
 359   ModuleCacheEntry(
 360       StringRef CachePath, const ModuleSummaryIndex &Index, StringRef ModuleID,
 361       const FunctionImporter::ImportMapTy &ImportList,
 362       const FunctionImporter::ExportSetTy &ExportList,
 363       const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
 364       const GVSummaryMapTy &DefinedGVSummaries, unsigned OptLevel,
 365       bool Freestanding, const TargetMachineBuilder &TMBuilder) {
 366     if (CachePath.empty())
 367       return;
 368
 369     if (!Index.modulePaths().count(ModuleID))
 370       // The module does not have an entry, it can't have a hash at all
 371       return;
 372
 373     if (all_of(Index.getModuleHash(ModuleID),
 374                [](uint32_t V) { return V == 0; }))
 375       // No hash entry, no caching!
 376       return;
 377
 378     llvm::lto::Config Conf;
 379     Conf.OptLevel = OptLevel;
 380     Conf.Options = TMBuilder.Options;
 381     Conf.CPU = TMBuilder.MCpu;
 382     Conf.MAttrs.push_back(TMBuilder.MAttr);
 383     Conf.RelocModel = TMBuilder.RelocModel;
 384     Conf.CGOptLevel = TMBuilder.CGOptLevel;
 385     Conf.Freestanding = Freestanding;
 386     SmallString<40> Key;
 387     computeLTOCacheKey(Key, Conf, Index, ModuleID, ImportList, ExportList,
 388                        ResolvedODR, DefinedGVSummaries);
 389
 390     // This choice of file name allows the cache to be pruned (see pruneCache()
 391     // in include/llvm/Support/CachePruning.h).
 392     sys::path::append(EntryPath, CachePath, "llvmcache-" + Key);
 393   }
 394
 395   // Access the path to this entry in the cache.
 396   StringRef getEntryPath() { return EntryPath; }
 397
 398   // Try loading the buffer for this cache entry.
 399   ErrorOr<std::unique_ptr<MemoryBuffer>> tryLoadingBuffer() {
 400     if (EntryPath.empty())
 401       return std::error_code();
 402     SmallString<64> ResultPath;
 403     Expected<sys::fs::file_t> FDOrErr = sys::fs::openNativeFileForRead(
 404         Twine(EntryPath), sys::fs::OF_UpdateAtime, &ResultPath);
 405     if (!FDOrErr)
 406       return errorToErrorCode(FDOrErr.takeError());
 407     ErrorOr<std::unique_ptr<MemoryBuffer>> MBOrErr = MemoryBuffer::getOpenFile(
 408         *FDOrErr, EntryPath, /*FileSize=*/-1, /*RequiresNullTerminator=*/false);
 409     sys::fs::closeFile(*FDOrErr);
 410     return MBOrErr;
 411   }
 412
 413   // Cache the Produced object file
 414   void write(const MemoryBuffer &OutputBuffer) {
 415     if (EntryPath.empty())
 416       return;
 417
 418     // Write to a temporary to avoid race condition
 419     SmallString<128> TempFilename;
 420     SmallString<128> CachePath(EntryPath);
 421     llvm::sys::path::remove_filename(CachePath);
 422     sys::path::append(TempFilename, CachePath, "Thin-%%%%%%.tmp.o");
 423
 424     if (auto Err = handleErrors(
 425             llvm::writeFileAtomically(TempFilename, EntryPath,
 426                                       OutputBuffer.getBuffer()),
 427             [](const llvm::AtomicFileWriteError &E) {
 428               std::string ErrorMsgBuffer;
 429               llvm::raw_string_ostream S(ErrorMsgBuffer);
 430               E.log(S);
 431
 432               if (E.Error ==
 433                   llvm::atomic_write_error::failed_to_create_uniq_file) {
 434                 errs() << "Error: " << ErrorMsgBuffer << "\n";
 435                 report_fatal_error("ThinLTO: Can't get a temporary file");
 436               }
 437             })) {
 438       // FIXME
 439       consumeError(std::move(Err));
 440     }
 441   }
 442 };
 443
 444 static std::unique_ptr<MemoryBuffer>
 445 ProcessThinLTOModule(Module &TheModule, ModuleSummaryIndex &Index,
 446                      StringMap<lto::InputFile *> &ModuleMap, TargetMachine &TM,
 447                      const FunctionImporter::ImportMapTy &ImportList,
 448                      const FunctionImporter::ExportSetTy &ExportList,
 449                      const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
 450                      const GVSummaryMapTy &DefinedGlobals,
 451                      const ThinLTOCodeGenerator::CachingOptions &CacheOptions,
 452                      bool DisableCodeGen, StringRef SaveTempsDir,
 453                      bool Freestanding, unsigned OptLevel, unsigned count,
 454                      bool DebugPassManager) {
 455   // "Benchmark"-like optimization: single-source case
 456   bool SingleModule = (ModuleMap.size() == 1);
 457
 458   // When linking an ELF shared object, dso_local should be dropped. We
 459   // conservatively do this for -fpic.
 460   bool ClearDSOLocalOnDeclarations =
 461       TM.getTargetTriple().isOSBinFormatELF() &&
 462       TM.getRelocationModel() != Reloc::Static &&
 463       TheModule.getPIELevel() == PIELevel::Default;
 464
 465   if (!SingleModule) {
 466     promoteModule(TheModule, Index, ClearDSOLocalOnDeclarations);
 467
 468     // Apply summary-based prevailing-symbol resolution decisions.
 469     thinLTOFinalizeInModule(TheModule, DefinedGlobals, /*PropagateAttrs=*/true);
 470
 471     // Save temps: after promotion.
 472     saveTempBitcode(TheModule, SaveTempsDir, count, ".1.promoted.bc");
 473   }
 474
 475   // Be friendly and don't nuke totally the module when the client didn't
 476   // supply anything to preserve.
 477   if (!ExportList.empty() || !GUIDPreservedSymbols.empty()) {
 478     // Apply summary-based internalization decisions.
 479     thinLTOInternalizeModule(TheModule, DefinedGlobals);
 480   }
 481
 482   // Save internalized bitcode
 483   saveTempBitcode(TheModule, SaveTempsDir, count, ".2.internalized.bc");
 484
 485   if (!SingleModule)
 486     crossImportIntoModule(TheModule, Index, ModuleMap, ImportList,
 487                           ClearDSOLocalOnDeclarations);
 488
 489   // Do this after any importing so that imported code is updated.
 490   // See comment at call to updateVCallVisibilityInIndex() for why
 491   // WholeProgramVisibilityEnabledInLTO is false.
 492   updatePublicTypeTestCalls(TheModule,
 493                             /* WholeProgramVisibilityEnabledInLTO */ false);
 494
 495   // Save temps: after cross-module import.
 496   saveTempBitcode(TheModule, SaveTempsDir, count, ".3.imported.bc");
 497
 498   optimizeModule(TheModule, TM, OptLevel, Freestanding, DebugPassManager,
 499                  &Index);
 500
 501   saveTempBitcode(TheModule, SaveTempsDir, count, ".4.opt.bc");
 502
 503   if (DisableCodeGen) {
 504     // Configured to stop before CodeGen, serialize the bitcode and return.
 505     SmallVector<char, 128> OutputBuffer;
 506     {
 507       raw_svector_ostream OS(OutputBuffer);
 508       ProfileSummaryInfo PSI(TheModule);
 509       auto Index = buildModuleSummaryIndex(TheModule, nullptr, &PSI);
 510       WriteBitcodeToFile(TheModule, OS, true, &Index);
 511     }
 512     return std::make_unique<SmallVectorMemoryBuffer>(
 513         std::move(OutputBuffer), /*RequiresNullTerminator=*/false);
 514   }
 515
 516   return codegenModule(TheModule, TM);
 517 }
 518
 519 /// Resolve prevailing symbols. Record resolutions in the \p ResolvedODR map
 520 /// for caching, and in the \p Index for application during the ThinLTO
 521 /// backends. This is needed for correctness for exported symbols (ensure
 522 /// at least one copy kept) and a compile-time optimization (to drop duplicate
 523 /// copies when possible).
 524 static void resolvePrevailingInIndex(
 525     ModuleSummaryIndex &Index,
 526     StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>>
 527         &ResolvedODR,
 528     const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
 529     const DenseMap<GlobalValue::GUID, const GlobalValueSummary *>
 530         &PrevailingCopy) {
 531
 532   auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) {
 533     const auto &Prevailing = PrevailingCopy.find(GUID);
 534     // Not in map means that there was only one copy, which must be prevailing.
 535     if (Prevailing == PrevailingCopy.end())
 536       return true;
 537     return Prevailing->second == S;
 538   };
 539
 540   auto recordNewLinkage = [&](StringRef ModuleIdentifier,
 541                               GlobalValue::GUID GUID,
 542                               GlobalValue::LinkageTypes NewLinkage) {
 543     ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;
 544   };
 545
 546   // TODO Conf.VisibilityScheme can be lto::Config::ELF for ELF.
 547   lto::Config Conf;
 548   thinLTOResolvePrevailingInIndex(Conf, Index, isPrevailing, recordNewLinkage,
 549                                   GUIDPreservedSymbols);
 550 }
 551
 552 // Initialize the TargetMachine builder for a given Triple
 553 static void initTMBuilder(TargetMachineBuilder &TMBuilder,
 554                           const Triple &TheTriple) {
 555   // Set a default CPU for Darwin triples (copied from LTOCodeGenerator).
 556   // FIXME this looks pretty terrible...
 557   if (TMBuilder.MCpu.empty() && TheTriple.isOSDarwin()) {
 558     if (TheTriple.getArch() == llvm::Triple::x86_64)
 559       TMBuilder.MCpu = "core2";
 560     else if (TheTriple.getArch() == llvm::Triple::x86)
 561       TMBuilder.MCpu = "yonah";
 562     else if (TheTriple.getArch() == llvm::Triple::aarch64 ||
 563              TheTriple.getArch() == llvm::Triple::aarch64_32)
 564       TMBuilder.MCpu = "cyclone";
 565   }
 566   TMBuilder.TheTriple = std::move(TheTriple);
 567 }
 568
 569 } // end anonymous namespace
 570
 571 void ThinLTOCodeGenerator::addModule(StringRef Identifier, StringRef Data) {
 572   MemoryBufferRef Buffer(Data, Identifier);
 573
 574   auto InputOrError = lto::InputFile::create(Buffer);
 575   if (!InputOrError)
 576     report_fatal_error(Twine("ThinLTO cannot create input file: ") +
 577                        toString(InputOrError.takeError()));
 578
 579   auto TripleStr = (*InputOrError)->getTargetTriple();
 580   Triple TheTriple(TripleStr);
 581
 582   if (Modules.empty())
 583     initTMBuilder(TMBuilder, Triple(TheTriple));
 584   else if (TMBuilder.TheTriple != TheTriple) {
 585     if (!TMBuilder.TheTriple.isCompatibleWith(TheTriple))
 586       report_fatal_error("ThinLTO modules with incompatible triples not "
 587                          "supported");
 588     initTMBuilder(TMBuilder, Triple(TMBuilder.TheTriple.merge(TheTriple)));
 589   }
 590
 591   Modules.emplace_back(std::move(*InputOrError));
 592 }
 593
 594 void ThinLTOCodeGenerator::preserveSymbol(StringRef Name) {
 595   PreservedSymbols.insert(Name);
 596 }
 597
 598 void ThinLTOCodeGenerator::crossReferenceSymbol(StringRef Name) {
 599   // FIXME: At the moment, we don't take advantage of this extra information,
 600   // we're conservatively considering cross-references as preserved.
 601   //  CrossReferencedSymbols.insert(Name);
 602   PreservedSymbols.insert(Name);
 603 }
 604
 605 // TargetMachine factory
 606 std::unique_ptr<TargetMachine> TargetMachineBuilder::create() const {
 607   std::string ErrMsg;
 608   const Target *TheTarget =
 609       TargetRegistry::lookupTarget(TheTriple.str(), ErrMsg);
 610   if (!TheTarget) {
 611     report_fatal_error(Twine("Can't load target for this Triple: ") + ErrMsg);
 612   }
 613
 614   // Use MAttr as the default set of features.
 615   SubtargetFeatures Features(MAttr);
 616   Features.getDefaultSubtargetFeatures(TheTriple);
 617   std::string FeatureStr = Features.getString();
 618
 619   std::unique_ptr<TargetMachine> TM(
 620       TheTarget->createTargetMachine(TheTriple.str(), MCpu, FeatureStr, Options,
 621                                      RelocModel, std::nullopt, CGOptLevel));
 622   assert(TM && "Cannot create target machine");
 623
 624   return TM;
 625 }
 626
 627 /**
 628  * Produce the combined summary index from all the bitcode files:
 629  * "thin-link".
 630  */
 631 std::unique_ptr<ModuleSummaryIndex> ThinLTOCodeGenerator::linkCombinedIndex() {
 632   std::unique_ptr<ModuleSummaryIndex> CombinedIndex =
 633       std::make_unique<ModuleSummaryIndex>(/*HaveGVs=*/false);
 634   uint64_t NextModuleId = 0;
 635   for (auto &Mod : Modules) {
 636     auto &M = Mod->getSingleBitcodeModule();
 637     if (Error Err =
 638             M.readSummary(*CombinedIndex, Mod->getName(), NextModuleId++)) {
 639       // FIXME diagnose
 640       logAllUnhandledErrors(
 641           std::move(Err), errs(),
 642           "error: can't create module summary index for buffer: ");
 643       return nullptr;
 644     }
 645   }
 646   return CombinedIndex;
 647 }
 648
 649 namespace {
 650 struct IsExported {
 651   const StringMap<FunctionImporter::ExportSetTy> &ExportLists;
 652   const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols;
 653
 654   IsExported(const StringMap<FunctionImporter::ExportSetTy> &ExportLists,
 655              const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols)
 656       : ExportLists(ExportLists), GUIDPreservedSymbols(GUIDPreservedSymbols) {}
 657
 658   bool operator()(StringRef ModuleIdentifier, ValueInfo VI) const {
 659     const auto &ExportList = ExportLists.find(ModuleIdentifier);
 660     return (ExportList != ExportLists.end() && ExportList->second.count(VI)) ||
 661            GUIDPreservedSymbols.count(VI.getGUID());
 662   }
 663 };
 664
 665 struct IsPrevailing {
 666   const DenseMap<GlobalValue::GUID, const GlobalValueSummary *> &PrevailingCopy;
 667   IsPrevailing(const DenseMap<GlobalValue::GUID, const GlobalValueSummary *>
 668                    &PrevailingCopy)
 669       : PrevailingCopy(PrevailingCopy) {}
 670
 671   bool operator()(GlobalValue::GUID GUID, const GlobalValueSummary *S) const {
 672     const auto &Prevailing = PrevailingCopy.find(GUID);
 673     // Not in map means that there was only one copy, which must be prevailing.
 674     if (Prevailing == PrevailingCopy.end())
 675       return true;
 676     return Prevailing->second == S;
 677   };
 678 };
 679 } // namespace
 680
 681 static void computeDeadSymbolsInIndex(
 682     ModuleSummaryIndex &Index,
 683     const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) {
 684   // We have no symbols resolution available. And can't do any better now in the
 685   // case where the prevailing symbol is in a native object. It can be refined
 686   // with linker information in the future.
 687   auto isPrevailing = [&](GlobalValue::GUID G) {
 688     return PrevailingType::Unknown;
 689   };
 690   computeDeadSymbolsWithConstProp(Index, GUIDPreservedSymbols, isPrevailing,
 691                                   /* ImportEnabled = */ true);
 692 }
 693
 694 /**
 695  * Perform promotion and renaming of exported internal functions.
 696  * Index is updated to reflect linkage changes from weak resolution.
 697  */
 698 void ThinLTOCodeGenerator::promote(Module &TheModule, ModuleSummaryIndex &Index,
 699                                    const lto::InputFile &File) {
 700   auto ModuleCount = Index.modulePaths().size();
 701   auto ModuleIdentifier = TheModule.getModuleIdentifier();
 702
 703   // Collect for each module the list of function it defines (GUID -> Summary).
 704   StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries;
 705   Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
 706
 707   // Convert the preserved symbols set from string to GUID
 708   auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
 709       File, PreservedSymbols, Triple(TheModule.getTargetTriple()));
 710
 711   // Add used symbol to the preserved symbols.
 712   addUsedSymbolToPreservedGUID(File, GUIDPreservedSymbols);
 713
 714   // Compute "dead" symbols, we don't want to import/export these!
 715   computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
 716
 717   // Compute prevailing symbols
 718   DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
 719   computePrevailingCopies(Index, PrevailingCopy);
 720
 721   // Generate import/export list
 722   StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
 723   StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
 724   ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries,
 725                            IsPrevailing(PrevailingCopy), ImportLists,
 726                            ExportLists);
 727
 728   // Resolve prevailing symbols
 729   StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
 730   resolvePrevailingInIndex(Index, ResolvedODR, GUIDPreservedSymbols,
 731                            PrevailingCopy);
 732
 733   thinLTOFinalizeInModule(TheModule,
 734                           ModuleToDefinedGVSummaries[ModuleIdentifier],
 735                           /*PropagateAttrs=*/false);
 736
 737   // Promote the exported values in the index, so that they are promoted
 738   // in the module.
 739   thinLTOInternalizeAndPromoteInIndex(
 740       Index, IsExported(ExportLists, GUIDPreservedSymbols),
 741       IsPrevailing(PrevailingCopy));
 742
 743   // FIXME Set ClearDSOLocalOnDeclarations.
 744   promoteModule(TheModule, Index, /*ClearDSOLocalOnDeclarations=*/false);
 745 }
 746
 747 /**
 748  * Perform cross-module importing for the module identified by ModuleIdentifier.
 749  */
 750 void ThinLTOCodeGenerator::crossModuleImport(Module &TheModule,
 751                                              ModuleSummaryIndex &Index,
 752                                              const lto::InputFile &File) {
 753   auto ModuleMap = generateModuleMap(Modules);
 754   auto ModuleCount = Index.modulePaths().size();
 755
 756   // Collect for each module the list of function it defines (GUID -> Summary).
 757   StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
 758   Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
 759
 760   // Convert the preserved symbols set from string to GUID
 761   auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
 762       File, PreservedSymbols, Triple(TheModule.getTargetTriple()));
 763
 764   addUsedSymbolToPreservedGUID(File, GUIDPreservedSymbols);
 765
 766   // Compute "dead" symbols, we don't want to import/export these!
 767   computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
 768
 769   // Compute prevailing symbols
 770   DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
 771   computePrevailingCopies(Index, PrevailingCopy);
 772
 773   // Generate import/export list
 774   StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
 775   StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
 776   ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries,
 777                            IsPrevailing(PrevailingCopy), ImportLists,
 778                            ExportLists);
 779   auto &ImportList = ImportLists[TheModule.getModuleIdentifier()];
 780
 781   // FIXME Set ClearDSOLocalOnDeclarations.
 782   crossImportIntoModule(TheModule, Index, ModuleMap, ImportList,
 783                         /*ClearDSOLocalOnDeclarations=*/false);
 784 }
 785
 786 /**
 787  * Compute the list of summaries needed for importing into module.
 788  */
 789 void ThinLTOCodeGenerator::gatherImportedSummariesForModule(
 790     Module &TheModule, ModuleSummaryIndex &Index,
 791     std::map<std::string, GVSummaryMapTy> &ModuleToSummariesForIndex,
 792     const lto::InputFile &File) {
 793   auto ModuleCount = Index.modulePaths().size();
 794   auto ModuleIdentifier = TheModule.getModuleIdentifier();
 795
 796   // Collect for each module the list of function it defines (GUID -> Summary).
 797   StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
 798   Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
 799
 800   // Convert the preserved symbols set from string to GUID
 801   auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
 802       File, PreservedSymbols, Triple(TheModule.getTargetTriple()));
 803
 804   addUsedSymbolToPreservedGUID(File, GUIDPreservedSymbols);
 805
 806   // Compute "dead" symbols, we don't want to import/export these!
 807   computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
 808
 809   // Compute prevailing symbols
 810   DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
 811   computePrevailingCopies(Index, PrevailingCopy);
 812
 813   // Generate import/export list
 814   StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
 815   StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
 816   ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries,
 817                            IsPrevailing(PrevailingCopy), ImportLists,
 818                            ExportLists);
 819
 820   llvm::gatherImportedSummariesForModule(
 821       ModuleIdentifier, ModuleToDefinedGVSummaries,
 822       ImportLists[ModuleIdentifier], ModuleToSummariesForIndex);
 823 }
 824
 825 /**
 826  * Emit the list of files needed for importing into module.
 827  */
 828 void ThinLTOCodeGenerator::emitImports(Module &TheModule, StringRef OutputName,
 829                                        ModuleSummaryIndex &Index,
 830                                        const lto::InputFile &File) {
 831   auto ModuleCount = Index.modulePaths().size();
 832   auto ModuleIdentifier = TheModule.getModuleIdentifier();
 833
 834   // Collect for each module the list of function it defines (GUID -> Summary).
 835   StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
 836   Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
 837
 838   // Convert the preserved symbols set from string to GUID
 839   auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
 840       File, PreservedSymbols, Triple(TheModule.getTargetTriple()));
 841
 842   addUsedSymbolToPreservedGUID(File, GUIDPreservedSymbols);
 843
 844   // Compute "dead" symbols, we don't want to import/export these!
 845   computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
 846
 847   // Compute prevailing symbols
 848   DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
 849   computePrevailingCopies(Index, PrevailingCopy);
 850
 851   // Generate import/export list
 852   StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
 853   StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
 854   ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries,
 855                            IsPrevailing(PrevailingCopy), ImportLists,
 856                            ExportLists);
 857
 858   std::map<std::string, GVSummaryMapTy> ModuleToSummariesForIndex;
 859   llvm::gatherImportedSummariesForModule(
 860       ModuleIdentifier, ModuleToDefinedGVSummaries,
 861       ImportLists[ModuleIdentifier], ModuleToSummariesForIndex);
 862
 863   std::error_code EC;
 864   if ((EC = EmitImportsFiles(ModuleIdentifier, OutputName,
 865                              ModuleToSummariesForIndex)))
 866     report_fatal_error(Twine("Failed to open ") + OutputName +
 867                        " to save imports lists\n");
 868 }
 869
 870 /**
 871  * Perform internalization. Runs promote and internalization together.
 872  * Index is updated to reflect linkage changes.
 873  */
 874 void ThinLTOCodeGenerator::internalize(Module &TheModule,
 875                                        ModuleSummaryIndex &Index,
 876                                        const lto::InputFile &File) {
 877   initTMBuilder(TMBuilder, Triple(TheModule.getTargetTriple()));
 878   auto ModuleCount = Index.modulePaths().size();
 879   auto ModuleIdentifier = TheModule.getModuleIdentifier();
 880
 881   // Convert the preserved symbols set from string to GUID
 882   auto GUIDPreservedSymbols =
 883       computeGUIDPreservedSymbols(File, PreservedSymbols, TMBuilder.TheTriple);
 884
 885   addUsedSymbolToPreservedGUID(File, GUIDPreservedSymbols);
 886
 887   // Collect for each module the list of function it defines (GUID -> Summary).
 888   StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
 889   Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
 890
 891   // Compute "dead" symbols, we don't want to import/export these!
 892   computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
 893
 894   // Compute prevailing symbols
 895   DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
 896   computePrevailingCopies(Index, PrevailingCopy);
 897
 898   // Generate import/export list
 899   StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
 900   StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
 901   ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries,
 902                            IsPrevailing(PrevailingCopy), ImportLists,
 903                            ExportLists);
 904   auto &ExportList = ExportLists[ModuleIdentifier];
 905
 906   // Be friendly and don't nuke totally the module when the client didn't
 907   // supply anything to preserve.
 908   if (ExportList.empty() && GUIDPreservedSymbols.empty())
 909     return;
 910
 911   // Resolve prevailing symbols
 912   StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
 913   resolvePrevailingInIndex(Index, ResolvedODR, GUIDPreservedSymbols,
 914                            PrevailingCopy);
 915
 916   // Promote the exported values in the index, so that they are promoted
 917   // in the module.
 918   thinLTOInternalizeAndPromoteInIndex(
 919       Index, IsExported(ExportLists, GUIDPreservedSymbols),
 920       IsPrevailing(PrevailingCopy));
 921
 922   // FIXME Set ClearDSOLocalOnDeclarations.
 923   promoteModule(TheModule, Index, /*ClearDSOLocalOnDeclarations=*/false);
 924
 925   // Internalization
 926   thinLTOFinalizeInModule(TheModule,
 927                           ModuleToDefinedGVSummaries[ModuleIdentifier],
 928                           /*PropagateAttrs=*/false);
 929
 930   thinLTOInternalizeModule(TheModule,
 931                            ModuleToDefinedGVSummaries[ModuleIdentifier]);
 932 }
 933
 934 /**
 935  * Perform post-importing ThinLTO optimizations.
 936  */
 937 void ThinLTOCodeGenerator::optimize(Module &TheModule) {
 938   initTMBuilder(TMBuilder, Triple(TheModule.getTargetTriple()));
 939
 940   // Optimize now
 941   optimizeModule(TheModule, *TMBuilder.create(), OptLevel, Freestanding,
 942                  DebugPassManager, nullptr);
 943 }
 944
 945 /// Write out the generated object file, either from CacheEntryPath or from
 946 /// OutputBuffer, preferring hard-link when possible.
 947 /// Returns the path to the generated file in SavedObjectsDirectoryPath.
 948 std::string
 949 ThinLTOCodeGenerator::writeGeneratedObject(int count, StringRef CacheEntryPath,
 950                                            const MemoryBuffer &OutputBuffer) {
 951   auto ArchName = TMBuilder.TheTriple.getArchName();
 952   SmallString<128> OutputPath(SavedObjectsDirectoryPath);
 953   llvm::sys::path::append(OutputPath,
 954                           Twine(count) + "." + ArchName + ".thinlto.o");
 955   OutputPath.c_str(); // Ensure the string is null terminated.
 956   if (sys::fs::exists(OutputPath))
 957     sys::fs::remove(OutputPath);
 958
 959   // We don't return a memory buffer to the linker, just a list of files.
 960   if (!CacheEntryPath.empty()) {
 961     // Cache is enabled, hard-link the entry (or copy if hard-link fails).
 962     auto Err = sys::fs::create_hard_link(CacheEntryPath, OutputPath);
 963     if (!Err)
 964       return std::string(OutputPath.str());
 965     // Hard linking failed, try to copy.
 966     Err = sys::fs::copy_file(CacheEntryPath, OutputPath);
 967     if (!Err)
 968       return std::string(OutputPath.str());
 969     // Copy failed (could be because the CacheEntry was removed from the cache
 970     // in the meantime by another process), fall back and try to write down the
 971     // buffer to the output.
 972     errs() << "remark: can't link or copy from cached entry '" << CacheEntryPath
 973            << "' to '" << OutputPath << "'\n";
 974   }
 975   // No cache entry, just write out the buffer.
 976   std::error_code Err;
 977   raw_fd_ostream OS(OutputPath, Err, sys::fs::OF_None);
 978   if (Err)
 979     report_fatal_error(Twine("Can't open output '") + OutputPath + "'\n");
 980   OS << OutputBuffer.getBuffer();
 981   return std::string(OutputPath.str());
 982 }
 983
 984 // Main entry point for the ThinLTO processing
 985 void ThinLTOCodeGenerator::run() {
 986   timeTraceProfilerBegin("ThinLink", StringRef(""));
 987   auto TimeTraceScopeExit = llvm::make_scope_exit([]() {
 988     if (llvm::timeTraceProfilerEnabled())
 989       llvm::timeTraceProfilerEnd();
 990   });
 991   // Prepare the resulting object vector
 992   assert(ProducedBinaries.empty() && "The generator should not be reused");
 993   if (SavedObjectsDirectoryPath.empty())
 994     ProducedBinaries.resize(Modules.size());
 995   else {
 996     sys::fs::create_directories(SavedObjectsDirectoryPath);
 997     bool IsDir;
 998     sys::fs::is_directory(SavedObjectsDirectoryPath, IsDir);
 999     if (!IsDir)
1000       report_fatal_error(Twine("Unexistent dir: '") + SavedObjectsDirectoryPath + "'");
1001     ProducedBinaryFiles.resize(Modules.size());
1002   }
1003
1004   if (CodeGenOnly) {
1005     // Perform only parallel codegen and return.
1006     ThreadPool Pool;
1007     int count = 0;
1008     for (auto &Mod : Modules) {
1009       Pool.async([&](int count) {
1010         LLVMContext Context;
1011         Context.setDiscardValueNames(LTODiscardValueNames);
1012
1013         // Parse module now
1014         auto TheModule = loadModuleFromInput(Mod.get(), Context, false,
1015                                              /*IsImporting*/ false);
1016
1017         // CodeGen
1018         auto OutputBuffer = codegenModule(*TheModule, *TMBuilder.create());
1019         if (SavedObjectsDirectoryPath.empty())
1020           ProducedBinaries[count] = std::move(OutputBuffer);
1021         else
1022           ProducedBinaryFiles[count] =
1023               writeGeneratedObject(count, "", *OutputBuffer);
1024       }, count++);
1025     }
1026
1027     return;
1028   }
1029
1030   // Sequential linking phase
1031   auto Index = linkCombinedIndex();
1032
1033   // Save temps: index.
1034   if (!SaveTempsDir.empty()) {
1035     auto SaveTempPath = SaveTempsDir + "index.bc";
1036     std::error_code EC;
1037     raw_fd_ostream OS(SaveTempPath, EC, sys::fs::OF_None);
1038     if (EC)
1039       report_fatal_error(Twine("Failed to open ") + SaveTempPath +
1040                          " to save optimized bitcode\n");
1041     writeIndexToFile(*Index, OS);
1042   }
1043
1044
1045   // Prepare the module map.
1046   auto ModuleMap = generateModuleMap(Modules);
1047   auto ModuleCount = Modules.size();
1048
1049   // Collect for each module the list of function it defines (GUID -> Summary).
1050   StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
1051   Index->collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
1052
1053   // Convert the preserved symbols set from string to GUID, this is needed for
1054   // computing the caching hash and the internalization.
1055   DenseSet<GlobalValue::GUID> GUIDPreservedSymbols;
1056   for (const auto &M : Modules)
1057     computeGUIDPreservedSymbols(*M, PreservedSymbols, TMBuilder.TheTriple,
1058                                 GUIDPreservedSymbols);
1059
1060   // Add used symbol from inputs to the preserved symbols.
1061   for (const auto &M : Modules)
1062     addUsedSymbolToPreservedGUID(*M, GUIDPreservedSymbols);
1063
1064   // Compute "dead" symbols, we don't want to import/export these!
1065   computeDeadSymbolsInIndex(*Index, GUIDPreservedSymbols);
1066
1067   // Synthesize entry counts for functions in the combined index.
1068   computeSyntheticCounts(*Index);
1069
1070   // Currently there is no support for enabling whole program visibility via a
1071   // linker option in the old LTO API, but this call allows it to be specified
1072   // via the internal option. Must be done before WPD below.
1073   if (hasWholeProgramVisibility(/* WholeProgramVisibilityEnabledInLTO */ false))
1074     Index->setWithWholeProgramVisibility();
1075   updateVCallVisibilityInIndex(*Index,
1076                                /* WholeProgramVisibilityEnabledInLTO */ false,
1077                                // FIXME: This needs linker information via a
1078                                // TBD new interface.
1079                                /* DynamicExportSymbols */ {});
1080
1081   // Perform index-based WPD. This will return immediately if there are
1082   // no index entries in the typeIdMetadata map (e.g. if we are instead
1083   // performing IR-based WPD in hybrid regular/thin LTO mode).
1084   std::map<ValueInfo, std::vector<VTableSlotSummary>> LocalWPDTargetsMap;
1085   std::set<GlobalValue::GUID> ExportedGUIDs;
1086   runWholeProgramDevirtOnIndex(*Index, ExportedGUIDs, LocalWPDTargetsMap);
1087   for (auto GUID : ExportedGUIDs)
1088     GUIDPreservedSymbols.insert(GUID);
1089
1090   // Compute prevailing symbols
1091   DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
1092   computePrevailingCopies(*Index, PrevailingCopy);
1093
1094   // Collect the import/export lists for all modules from the call-graph in the
1095   // combined index.
1096   StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
1097   StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
1098   ComputeCrossModuleImport(*Index, ModuleToDefinedGVSummaries,
1099                            IsPrevailing(PrevailingCopy), ImportLists,
1100                            ExportLists);
1101
1102   // We use a std::map here to be able to have a defined ordering when
1103   // producing a hash for the cache entry.
1104   // FIXME: we should be able to compute the caching hash for the entry based
1105   // on the index, and nuke this map.
1106   StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
1107
1108   // Resolve prevailing symbols, this has to be computed early because it
1109   // impacts the caching.
1110   resolvePrevailingInIndex(*Index, ResolvedODR, GUIDPreservedSymbols,
1111                            PrevailingCopy);
1112
1113   // Use global summary-based analysis to identify symbols that can be
1114   // internalized (because they aren't exported or preserved as per callback).
1115   // Changes are made in the index, consumed in the ThinLTO backends.
1116   updateIndexWPDForExports(*Index,
1117                            IsExported(ExportLists, GUIDPreservedSymbols),
1118                            LocalWPDTargetsMap);
1119   thinLTOInternalizeAndPromoteInIndex(
1120       *Index, IsExported(ExportLists, GUIDPreservedSymbols),
1121       IsPrevailing(PrevailingCopy));
1122
1123   thinLTOPropagateFunctionAttrs(*Index, IsPrevailing(PrevailingCopy));
1124
1125   // Make sure that every module has an entry in the ExportLists, ImportList,
1126   // GVSummary and ResolvedODR maps to enable threaded access to these maps
1127   // below.
1128   for (auto &Module : Modules) {
1129     auto ModuleIdentifier = Module->getName();
1130     ExportLists[ModuleIdentifier];
1131     ImportLists[ModuleIdentifier];
1132     ResolvedODR[ModuleIdentifier];
1133     ModuleToDefinedGVSummaries[ModuleIdentifier];
1134   }
1135
1136   std::vector<BitcodeModule *> ModulesVec;
1137   ModulesVec.reserve(Modules.size());
1138   for (auto &Mod : Modules)
1139     ModulesVec.push_back(&Mod->getSingleBitcodeModule());
1140   std::vector<int> ModulesOrdering = lto::generateModulesOrdering(ModulesVec);
1141
1142   if (llvm::timeTraceProfilerEnabled())
1143     llvm::timeTraceProfilerEnd();
1144
1145   TimeTraceScopeExit.release();
1146
1147   // Parallel optimizer + codegen
1148   {
1149     ThreadPool Pool(heavyweight_hardware_concurrency(ThreadCount));
1150     for (auto IndexCount : ModulesOrdering) {
1151       auto &Mod = Modules[IndexCount];
1152       Pool.async([&](int count) {
1153         auto ModuleIdentifier = Mod->getName();
1154         auto &ExportList = ExportLists[ModuleIdentifier];
1155
1156         auto &DefinedGVSummaries = ModuleToDefinedGVSummaries[ModuleIdentifier];
1157
1158         // The module may be cached, this helps handling it.
1159         ModuleCacheEntry CacheEntry(CacheOptions.Path, *Index, ModuleIdentifier,
1160                                     ImportLists[ModuleIdentifier], ExportList,
1161                                     ResolvedODR[ModuleIdentifier],
1162                                     DefinedGVSummaries, OptLevel, Freestanding,
1163                                     TMBuilder);
1164         auto CacheEntryPath = CacheEntry.getEntryPath();
1165
1166         {
1167           auto ErrOrBuffer = CacheEntry.tryLoadingBuffer();
1168           LLVM_DEBUG(dbgs() << "Cache " << (ErrOrBuffer ? "hit" : "miss")
1169                             << " '" << CacheEntryPath << "' for buffer "
1170                             << count << " " << ModuleIdentifier << "\n");
1171
1172           if (ErrOrBuffer) {
1173             // Cache Hit!
1174             if (SavedObjectsDirectoryPath.empty())
1175               ProducedBinaries[count] = std::move(ErrOrBuffer.get());
1176             else
1177               ProducedBinaryFiles[count] = writeGeneratedObject(
1178                   count, CacheEntryPath, *ErrOrBuffer.get());
1179             return;
1180           }
1181         }
1182
1183         LLVMContext Context;
1184         Context.setDiscardValueNames(LTODiscardValueNames);
1185         Context.enableDebugTypeODRUniquing();
1186         auto DiagFileOrErr = lto::setupLLVMOptimizationRemarks(
1187             Context, RemarksFilename, RemarksPasses, RemarksFormat,
1188             RemarksWithHotness, RemarksHotnessThreshold, count);
1189         if (!DiagFileOrErr) {
1190           errs() << "Error: " << toString(DiagFileOrErr.takeError()) << "\n";
1191           report_fatal_error("ThinLTO: Can't get an output file for the "
1192                              "remarks");
1193         }
1194
1195         // Parse module now
1196         auto TheModule = loadModuleFromInput(Mod.get(), Context, false,
1197                                              /*IsImporting*/ false);
1198
1199         // Save temps: original file.
1200         saveTempBitcode(*TheModule, SaveTempsDir, count, ".0.original.bc");
1201
1202         auto &ImportList = ImportLists[ModuleIdentifier];
1203         // Run the main process now, and generates a binary
1204         auto OutputBuffer = ProcessThinLTOModule(
1205             *TheModule, *Index, ModuleMap, *TMBuilder.create(), ImportList,
1206             ExportList, GUIDPreservedSymbols,
1207             ModuleToDefinedGVSummaries[ModuleIdentifier], CacheOptions,
1208             DisableCodeGen, SaveTempsDir, Freestanding, OptLevel, count,
1209             DebugPassManager);
1210
1211         // Commit to the cache (if enabled)
1212         CacheEntry.write(*OutputBuffer);
1213
1214         if (SavedObjectsDirectoryPath.empty()) {
1215           // We need to generated a memory buffer for the linker.
1216           if (!CacheEntryPath.empty()) {
1217             // When cache is enabled, reload from the cache if possible.
1218             // Releasing the buffer from the heap and reloading it from the
1219             // cache file with mmap helps us to lower memory pressure.
1220             // The freed memory can be used for the next input file.
1221             // The final binary link will read from the VFS cache (hopefully!)
1222             // or from disk (if the memory pressure was too high).
1223             auto ReloadedBufferOrErr = CacheEntry.tryLoadingBuffer();
1224             if (auto EC = ReloadedBufferOrErr.getError()) {
1225               // On error, keep the preexisting buffer and print a diagnostic.
1226               errs() << "remark: can't reload cached file '" << CacheEntryPath
1227                      << "': " << EC.message() << "\n";
1228             } else {
1229               OutputBuffer = std::move(*ReloadedBufferOrErr);
1230             }
1231           }
1232           ProducedBinaries[count] = std::move(OutputBuffer);
1233           return;
1234         }
1235         ProducedBinaryFiles[count] = writeGeneratedObject(
1236             count, CacheEntryPath, *OutputBuffer);
1237       }, IndexCount);
1238     }
1239   }
1240
1241   pruneCache(CacheOptions.Path, CacheOptions.Policy, ProducedBinaries);
1242
1243   // If statistics were requested, print them out now.
1244   if (llvm::AreStatisticsEnabled())
1245     llvm::PrintStatistics();
1246   reportAndResetTimings();
1247 }