llvm/lib/LTO/LTOBackend.cpp

   1 //===-LTOBackend.cpp - LLVM Link Time Optimizer Backend -------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file implements the "backend" phase of LTO, i.e. it performs
  10 // optimization and code generation on a loaded module. It is generally used
  11 // internally by the LTO class but can also be used independently, for example
  12 // to implement a standalone ThinLTO backend.
  13 //
  14 //===----------------------------------------------------------------------===//
  15
  16 #include "llvm/LTO/LTOBackend.h"
  17 #include "llvm/Analysis/AliasAnalysis.h"
  18 #include "llvm/Analysis/CGSCCPassManager.h"
  19 #include "llvm/Analysis/ModuleSummaryAnalysis.h"
  20 #include "llvm/Analysis/TargetLibraryInfo.h"
  21 #include "llvm/Bitcode/BitcodeReader.h"
  22 #include "llvm/Bitcode/BitcodeWriter.h"
  23 #include "llvm/CGData/CodeGenData.h"
  24 #include "llvm/IR/LLVMRemarkStreamer.h"
  25 #include "llvm/IR/LegacyPassManager.h"
  26 #include "llvm/IR/PassManager.h"
  27 #include "llvm/IR/Verifier.h"
  28 #include "llvm/LTO/LTO.h"
  29 #include "llvm/MC/TargetRegistry.h"
  30 #include "llvm/Object/ModuleSymbolTable.h"
  31 #include "llvm/Passes/PassBuilder.h"
  32 #include "llvm/Passes/PassPlugin.h"
  33 #include "llvm/Passes/StandardInstrumentations.h"
  34 #include "llvm/Support/Error.h"
  35 #include "llvm/Support/FileSystem.h"
  36 #include "llvm/Support/MemoryBuffer.h"
  37 #include "llvm/Support/Path.h"
  38 #include "llvm/Support/Program.h"
  39 #include "llvm/Support/ThreadPool.h"
  40 #include "llvm/Support/ToolOutputFile.h"
  41 #include "llvm/Support/VirtualFileSystem.h"
  42 #include "llvm/Support/raw_ostream.h"
  43 #include "llvm/Target/TargetMachine.h"
  44 #include "llvm/TargetParser/SubtargetFeature.h"
  45 #include "llvm/Transforms/IPO/WholeProgramDevirt.h"
  46 #include "llvm/Transforms/Scalar/LoopPassManager.h"
  47 #include "llvm/Transforms/Utils/FunctionImportUtils.h"
  48 #include "llvm/Transforms/Utils/SplitModule.h"
  49 #include <optional>
  50
  51 using namespace llvm;
  52 using namespace lto;
  53
  54 #define DEBUG_TYPE "lto-backend"
  55
  56 enum class LTOBitcodeEmbedding {
  57   DoNotEmbed = 0,
  58   EmbedOptimized = 1,
  59   EmbedPostMergePreOptimized = 2
  60 };
  61
  62 static cl::opt<LTOBitcodeEmbedding> EmbedBitcode(
  63     "lto-embed-bitcode", cl::init(LTOBitcodeEmbedding::DoNotEmbed),
  64     cl::values(clEnumValN(LTOBitcodeEmbedding::DoNotEmbed, "none",
  65                           "Do not embed"),
  66                clEnumValN(LTOBitcodeEmbedding::EmbedOptimized, "optimized",
  67                           "Embed after all optimization passes"),
  68                clEnumValN(LTOBitcodeEmbedding::EmbedPostMergePreOptimized,
  69                           "post-merge-pre-opt",
  70                           "Embed post merge, but before optimizations")),
  71     cl::desc("Embed LLVM bitcode in object files produced by LTO"));
  72
  73 static cl::opt<bool> ThinLTOAssumeMerged(
  74     "thinlto-assume-merged", cl::init(false),
  75     cl::desc("Assume the input has already undergone ThinLTO function "
  76              "importing and the other pre-optimization pipeline changes."));
  77
  78 namespace llvm {
  79 extern cl::opt<bool> NoPGOWarnMismatch;
  80 }
  81
  82 [[noreturn]] static void reportOpenError(StringRef Path, Twine Msg) {
  83   errs() << "failed to open " << Path << ": " << Msg << '\n';
  84   errs().flush();
  85   exit(1);
  86 }
  87
  88 Error Config::addSaveTemps(std::string OutputFileName, bool UseInputModulePath,
  89                            const DenseSet<StringRef> &SaveTempsArgs) {
  90   ShouldDiscardValueNames = false;
  91
  92   std::error_code EC;
  93   if (SaveTempsArgs.empty() || SaveTempsArgs.contains("resolution")) {
  94     ResolutionFile =
  95         std::make_unique<raw_fd_ostream>(OutputFileName + "resolution.txt", EC,
  96                                          sys::fs::OpenFlags::OF_TextWithCRLF);
  97     if (EC) {
  98       ResolutionFile.reset();
  99       return errorCodeToError(EC);
 100     }
 101   }
 102
 103   auto setHook = [&](std::string PathSuffix, ModuleHookFn &Hook) {
 104     // Keep track of the hook provided by the linker, which also needs to run.
 105     ModuleHookFn LinkerHook = Hook;
 106     Hook = [=](unsigned Task, const Module &M) {
 107       // If the linker's hook returned false, we need to pass that result
 108       // through.
 109       if (LinkerHook && !LinkerHook(Task, M))
 110         return false;
 111
 112       std::string PathPrefix;
 113       // If this is the combined module (not a ThinLTO backend compile) or the
 114       // user hasn't requested using the input module's path, emit to a file
 115       // named from the provided OutputFileName with the Task ID appended.
 116       if (M.getModuleIdentifier() == "ld-temp.o" || !UseInputModulePath) {
 117         PathPrefix = OutputFileName;
 118         if (Task != (unsigned)-1)
 119           PathPrefix += utostr(Task) + ".";
 120       } else
 121         PathPrefix = M.getModuleIdentifier() + ".";
 122       std::string Path = PathPrefix + PathSuffix + ".bc";
 123       std::error_code EC;
 124       raw_fd_ostream OS(Path, EC, sys::fs::OpenFlags::OF_None);
 125       // Because -save-temps is a debugging feature, we report the error
 126       // directly and exit.
 127       if (EC)
 128         reportOpenError(Path, EC.message());
 129       WriteBitcodeToFile(M, OS, /*ShouldPreserveUseListOrder=*/false);
 130       return true;
 131     };
 132   };
 133
 134   auto SaveCombinedIndex =
 135       [=](const ModuleSummaryIndex &Index,
 136           const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) {
 137         std::string Path = OutputFileName + "index.bc";
 138         std::error_code EC;
 139         raw_fd_ostream OS(Path, EC, sys::fs::OpenFlags::OF_None);
 140         // Because -save-temps is a debugging feature, we report the error
 141         // directly and exit.
 142         if (EC)
 143           reportOpenError(Path, EC.message());
 144         writeIndexToFile(Index, OS);
 145
 146         Path = OutputFileName + "index.dot";
 147         raw_fd_ostream OSDot(Path, EC, sys::fs::OpenFlags::OF_Text);
 148         if (EC)
 149           reportOpenError(Path, EC.message());
 150         Index.exportToDot(OSDot, GUIDPreservedSymbols);
 151         return true;
 152       };
 153
 154   if (SaveTempsArgs.empty()) {
 155     setHook("0.preopt", PreOptModuleHook);
 156     setHook("1.promote", PostPromoteModuleHook);
 157     setHook("2.internalize", PostInternalizeModuleHook);
 158     setHook("3.import", PostImportModuleHook);
 159     setHook("4.opt", PostOptModuleHook);
 160     setHook("5.precodegen", PreCodeGenModuleHook);
 161     CombinedIndexHook = SaveCombinedIndex;
 162   } else {
 163     if (SaveTempsArgs.contains("preopt"))
 164       setHook("0.preopt", PreOptModuleHook);
 165     if (SaveTempsArgs.contains("promote"))
 166       setHook("1.promote", PostPromoteModuleHook);
 167     if (SaveTempsArgs.contains("internalize"))
 168       setHook("2.internalize", PostInternalizeModuleHook);
 169     if (SaveTempsArgs.contains("import"))
 170       setHook("3.import", PostImportModuleHook);
 171     if (SaveTempsArgs.contains("opt"))
 172       setHook("4.opt", PostOptModuleHook);
 173     if (SaveTempsArgs.contains("precodegen"))
 174       setHook("5.precodegen", PreCodeGenModuleHook);
 175     if (SaveTempsArgs.contains("combinedindex"))
 176       CombinedIndexHook = SaveCombinedIndex;
 177   }
 178
 179   return Error::success();
 180 }
 181
 182 #define HANDLE_EXTENSION(Ext)                                                  \
 183   llvm::PassPluginLibraryInfo get##Ext##PluginInfo();
 184 #include "llvm/Support/Extension.def"
 185 #undef HANDLE_EXTENSION
 186
 187 static void RegisterPassPlugins(ArrayRef<std::string> PassPlugins,
 188                                 PassBuilder &PB) {
 189 #define HANDLE_EXTENSION(Ext)                                                  \
 190   get##Ext##PluginInfo().RegisterPassBuilderCallbacks(PB);
 191 #include "llvm/Support/Extension.def"
 192 #undef HANDLE_EXTENSION
 193
 194   // Load requested pass plugins and let them register pass builder callbacks
 195   for (auto &PluginFN : PassPlugins) {
 196     auto PassPlugin = PassPlugin::Load(PluginFN);
 197     if (!PassPlugin)
 198       report_fatal_error(PassPlugin.takeError(), /*gen_crash_diag=*/false);
 199     PassPlugin->registerPassBuilderCallbacks(PB);
 200   }
 201 }
 202
 203 static std::unique_ptr<TargetMachine>
 204 createTargetMachine(const Config &Conf, const Target *TheTarget, Module &M) {
 205   StringRef TheTriple = M.getTargetTriple();
 206   SubtargetFeatures Features;
 207   Features.getDefaultSubtargetFeatures(Triple(TheTriple));
 208   for (const std::string &A : Conf.MAttrs)
 209     Features.AddFeature(A);
 210
 211   std::optional<Reloc::Model> RelocModel;
 212   if (Conf.RelocModel)
 213     RelocModel = *Conf.RelocModel;
 214   else if (M.getModuleFlag("PIC Level"))
 215     RelocModel =
 216         M.getPICLevel() == PICLevel::NotPIC ? Reloc::Static : Reloc::PIC_;
 217
 218   std::optional<CodeModel::Model> CodeModel;
 219   if (Conf.CodeModel)
 220     CodeModel = *Conf.CodeModel;
 221   else
 222     CodeModel = M.getCodeModel();
 223
 224   std::unique_ptr<TargetMachine> TM(TheTarget->createTargetMachine(
 225       TheTriple, Conf.CPU, Features.getString(), Conf.Options, RelocModel,
 226       CodeModel, Conf.CGOptLevel));
 227
 228   assert(TM && "Failed to create target machine");
 229
 230   if (std::optional<uint64_t> LargeDataThreshold = M.getLargeDataThreshold())
 231     TM->setLargeDataThreshold(*LargeDataThreshold);
 232
 233   return TM;
 234 }
 235
 236 static void runNewPMPasses(const Config &Conf, Module &Mod, TargetMachine *TM,
 237                            unsigned OptLevel, bool IsThinLTO,
 238                            ModuleSummaryIndex *ExportSummary,
 239                            const ModuleSummaryIndex *ImportSummary) {
 240   auto FS = vfs::getRealFileSystem();
 241   std::optional<PGOOptions> PGOOpt;
 242   if (!Conf.SampleProfile.empty())
 243     PGOOpt = PGOOptions(Conf.SampleProfile, "", Conf.ProfileRemapping,
 244                         /*MemoryProfile=*/"", FS, PGOOptions::SampleUse,
 245                         PGOOptions::NoCSAction,
 246                         PGOOptions::ColdFuncOpt::Default, true);
 247   else if (Conf.RunCSIRInstr) {
 248     PGOOpt = PGOOptions("", Conf.CSIRProfile, Conf.ProfileRemapping,
 249                         /*MemoryProfile=*/"", FS, PGOOptions::IRUse,
 250                         PGOOptions::CSIRInstr, PGOOptions::ColdFuncOpt::Default,
 251                         Conf.AddFSDiscriminator);
 252   } else if (!Conf.CSIRProfile.empty()) {
 253     PGOOpt = PGOOptions(Conf.CSIRProfile, "", Conf.ProfileRemapping,
 254                         /*MemoryProfile=*/"", FS, PGOOptions::IRUse,
 255                         PGOOptions::CSIRUse, PGOOptions::ColdFuncOpt::Default,
 256                         Conf.AddFSDiscriminator);
 257     NoPGOWarnMismatch = !Conf.PGOWarnMismatch;
 258   } else if (Conf.AddFSDiscriminator) {
 259     PGOOpt = PGOOptions("", "", "", /*MemoryProfile=*/"", nullptr,
 260                         PGOOptions::NoAction, PGOOptions::NoCSAction,
 261                         PGOOptions::ColdFuncOpt::Default, true);
 262   }
 263   TM->setPGOOption(PGOOpt);
 264
 265   LoopAnalysisManager LAM;
 266   FunctionAnalysisManager FAM;
 267   CGSCCAnalysisManager CGAM;
 268   ModuleAnalysisManager MAM;
 269
 270   PassInstrumentationCallbacks PIC;
 271   StandardInstrumentations SI(Mod.getContext(), Conf.DebugPassManager,
 272                               Conf.VerifyEach);
 273   SI.registerCallbacks(PIC, &MAM);
 274   PassBuilder PB(TM, Conf.PTO, PGOOpt, &PIC);
 275
 276   RegisterPassPlugins(Conf.PassPlugins, PB);
 277
 278   std::unique_ptr<TargetLibraryInfoImpl> TLII(
 279       new TargetLibraryInfoImpl(Triple(TM->getTargetTriple())));
 280   if (Conf.Freestanding)
 281     TLII->disableAllFunctions();
 282   FAM.registerPass([&] { return TargetLibraryAnalysis(*TLII); });
 283
 284   // Parse a custom AA pipeline if asked to.
 285   if (!Conf.AAPipeline.empty()) {
 286     AAManager AA;
 287     if (auto Err = PB.parseAAPipeline(AA, Conf.AAPipeline)) {
 288       report_fatal_error(Twine("unable to parse AA pipeline description '") +
 289                          Conf.AAPipeline + "': " + toString(std::move(Err)));
 290     }
 291     // Register the AA manager first so that our version is the one used.
 292     FAM.registerPass([&] { return std::move(AA); });
 293   }
 294
 295   // Register all the basic analyses with the managers.
 296   PB.registerModuleAnalyses(MAM);
 297   PB.registerCGSCCAnalyses(CGAM);
 298   PB.registerFunctionAnalyses(FAM);
 299   PB.registerLoopAnalyses(LAM);
 300   PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
 301
 302   ModulePassManager MPM;
 303
 304   if (!Conf.DisableVerify)
 305     MPM.addPass(VerifierPass());
 306
 307   OptimizationLevel OL;
 308
 309   switch (OptLevel) {
 310   default:
 311     llvm_unreachable("Invalid optimization level");
 312   case 0:
 313     OL = OptimizationLevel::O0;
 314     break;
 315   case 1:
 316     OL = OptimizationLevel::O1;
 317     break;
 318   case 2:
 319     OL = OptimizationLevel::O2;
 320     break;
 321   case 3:
 322     OL = OptimizationLevel::O3;
 323     break;
 324   }
 325
 326   // Parse a custom pipeline if asked to.
 327   if (!Conf.OptPipeline.empty()) {
 328     if (auto Err = PB.parsePassPipeline(MPM, Conf.OptPipeline)) {
 329       report_fatal_error(Twine("unable to parse pass pipeline description '") +
 330                          Conf.OptPipeline + "': " + toString(std::move(Err)));
 331     }
 332   } else if (IsThinLTO) {
 333     MPM.addPass(PB.buildThinLTODefaultPipeline(OL, ImportSummary));
 334   } else {
 335     MPM.addPass(PB.buildLTODefaultPipeline(OL, ExportSummary));
 336   }
 337
 338   if (!Conf.DisableVerify)
 339     MPM.addPass(VerifierPass());
 340
 341   if (PrintPipelinePasses) {
 342     std::string PipelineStr;
 343     raw_string_ostream OS(PipelineStr);
 344     MPM.printPipeline(OS, [&PIC](StringRef ClassName) {
 345       auto PassName = PIC.getPassNameForClassName(ClassName);
 346       return PassName.empty() ? ClassName : PassName;
 347     });
 348     outs() << "pipeline-passes: " << PipelineStr << '\n';
 349   }
 350
 351   MPM.run(Mod, MAM);
 352 }
 353
 354 bool lto::opt(const Config &Conf, TargetMachine *TM, unsigned Task, Module &Mod,
 355               bool IsThinLTO, ModuleSummaryIndex *ExportSummary,
 356               const ModuleSummaryIndex *ImportSummary,
 357               const std::vector<uint8_t> &CmdArgs) {
 358   if (EmbedBitcode == LTOBitcodeEmbedding::EmbedPostMergePreOptimized) {
 359     // FIXME: the motivation for capturing post-merge bitcode and command line
 360     // is replicating the compilation environment from bitcode, without needing
 361     // to understand the dependencies (the functions to be imported). This
 362     // assumes a clang - based invocation, case in which we have the command
 363     // line.
 364     // It's not very clear how the above motivation would map in the
 365     // linker-based case, so we currently don't plumb the command line args in
 366     // that case.
 367     if (CmdArgs.empty())
 368       LLVM_DEBUG(
 369           dbgs() << "Post-(Thin)LTO merge bitcode embedding was requested, but "
 370                     "command line arguments are not available");
 371     llvm::embedBitcodeInModule(Mod, llvm::MemoryBufferRef(),
 372                                /*EmbedBitcode*/ true, /*EmbedCmdline*/ true,
 373                                /*Cmdline*/ CmdArgs);
 374   }
 375   // FIXME: Plumb the combined index into the new pass manager.
 376   runNewPMPasses(Conf, Mod, TM, Conf.OptLevel, IsThinLTO, ExportSummary,
 377                  ImportSummary);
 378   return !Conf.PostOptModuleHook || Conf.PostOptModuleHook(Task, Mod);
 379 }
 380
 381 static void codegen(const Config &Conf, TargetMachine *TM,
 382                     AddStreamFn AddStream, unsigned Task, Module &Mod,
 383                     const ModuleSummaryIndex &CombinedIndex) {
 384   if (Conf.PreCodeGenModuleHook && !Conf.PreCodeGenModuleHook(Task, Mod))
 385     return;
 386
 387   if (EmbedBitcode == LTOBitcodeEmbedding::EmbedOptimized)
 388     llvm::embedBitcodeInModule(Mod, llvm::MemoryBufferRef(),
 389                                /*EmbedBitcode*/ true,
 390                                /*EmbedCmdline*/ false,
 391                                /*CmdArgs*/ std::vector<uint8_t>());
 392
 393   std::unique_ptr<ToolOutputFile> DwoOut;
 394   SmallString<1024> DwoFile(Conf.SplitDwarfOutput);
 395   if (!Conf.DwoDir.empty()) {
 396     std::error_code EC;
 397     if (auto EC = llvm::sys::fs::create_directories(Conf.DwoDir))
 398       report_fatal_error(Twine("Failed to create directory ") + Conf.DwoDir +
 399                          ": " + EC.message());
 400
 401     DwoFile = Conf.DwoDir;
 402     sys::path::append(DwoFile, std::to_string(Task) + ".dwo");
 403     TM->Options.MCOptions.SplitDwarfFile = std::string(DwoFile);
 404   } else
 405     TM->Options.MCOptions.SplitDwarfFile = Conf.SplitDwarfFile;
 406
 407   if (!DwoFile.empty()) {
 408     std::error_code EC;
 409     DwoOut = std::make_unique<ToolOutputFile>(DwoFile, EC, sys::fs::OF_None);
 410     if (EC)
 411       report_fatal_error(Twine("Failed to open ") + DwoFile + ": " +
 412                          EC.message());
 413   }
 414
 415   Expected<std::unique_ptr<CachedFileStream>> StreamOrErr =
 416       AddStream(Task, Mod.getModuleIdentifier());
 417   if (Error Err = StreamOrErr.takeError())
 418     report_fatal_error(std::move(Err));
 419   std::unique_ptr<CachedFileStream> &Stream = *StreamOrErr;
 420   TM->Options.ObjectFilenameForDebug = Stream->ObjectPathName;
 421
 422   legacy::PassManager CodeGenPasses;
 423   TargetLibraryInfoImpl TLII(Triple(Mod.getTargetTriple()));
 424   CodeGenPasses.add(new TargetLibraryInfoWrapperPass(TLII));
 425   CodeGenPasses.add(
 426       createImmutableModuleSummaryIndexWrapperPass(&CombinedIndex));
 427   if (Conf.PreCodeGenPassesHook)
 428     Conf.PreCodeGenPassesHook(CodeGenPasses);
 429   if (TM->addPassesToEmitFile(CodeGenPasses, *Stream->OS,
 430                               DwoOut ? &DwoOut->os() : nullptr,
 431                               Conf.CGFileType))
 432     report_fatal_error("Failed to setup codegen");
 433   CodeGenPasses.run(Mod);
 434
 435   if (DwoOut)
 436     DwoOut->keep();
 437 }
 438
 439 static void splitCodeGen(const Config &C, TargetMachine *TM,
 440                          AddStreamFn AddStream,
 441                          unsigned ParallelCodeGenParallelismLevel, Module &Mod,
 442                          const ModuleSummaryIndex &CombinedIndex) {
 443   DefaultThreadPool CodegenThreadPool(
 444       heavyweight_hardware_concurrency(ParallelCodeGenParallelismLevel));
 445   unsigned ThreadCount = 0;
 446   const Target *T = &TM->getTarget();
 447
 448   const auto HandleModulePartition =
 449       [&](std::unique_ptr<Module> MPart) {
 450         // We want to clone the module in a new context to multi-thread the
 451         // codegen. We do it by serializing partition modules to bitcode
 452         // (while still on the main thread, in order to avoid data races) and
 453         // spinning up new threads which deserialize the partitions into
 454         // separate contexts.
 455         // FIXME: Provide a more direct way to do this in LLVM.
 456         SmallString<0> BC;
 457         raw_svector_ostream BCOS(BC);
 458         WriteBitcodeToFile(*MPart, BCOS);
 459
 460         // Enqueue the task
 461         CodegenThreadPool.async(
 462             [&](const SmallString<0> &BC, unsigned ThreadId) {
 463               LTOLLVMContext Ctx(C);
 464               Expected<std::unique_ptr<Module>> MOrErr =
 465                   parseBitcodeFile(MemoryBufferRef(BC.str(), "ld-temp.o"), Ctx);
 466               if (!MOrErr)
 467                 report_fatal_error("Failed to read bitcode");
 468               std::unique_ptr<Module> MPartInCtx = std::move(MOrErr.get());
 469
 470               std::unique_ptr<TargetMachine> TM =
 471                   createTargetMachine(C, T, *MPartInCtx);
 472
 473               codegen(C, TM.get(), AddStream, ThreadId, *MPartInCtx,
 474                       CombinedIndex);
 475             },
 476             // Pass BC using std::move to ensure that it get moved rather than
 477             // copied into the thread's context.
 478             std::move(BC), ThreadCount++);
 479       };
 480
 481   // Try target-specific module splitting first, then fallback to the default.
 482   if (!TM->splitModule(Mod, ParallelCodeGenParallelismLevel,
 483                        HandleModulePartition)) {
 484     SplitModule(Mod, ParallelCodeGenParallelismLevel, HandleModulePartition,
 485                 false);
 486   }
 487
 488   // Because the inner lambda (which runs in a worker thread) captures our local
 489   // variables, we need to wait for the worker threads to terminate before we
 490   // can leave the function scope.
 491   CodegenThreadPool.wait();
 492 }
 493
 494 static Expected<const Target *> initAndLookupTarget(const Config &C,
 495                                                     Module &Mod) {
 496   if (!C.OverrideTriple.empty())
 497     Mod.setTargetTriple(C.OverrideTriple);
 498   else if (Mod.getTargetTriple().empty())
 499     Mod.setTargetTriple(C.DefaultTriple);
 500
 501   std::string Msg;
 502   const Target *T = TargetRegistry::lookupTarget(Mod.getTargetTriple(), Msg);
 503   if (!T)
 504     return make_error<StringError>(Msg, inconvertibleErrorCode());
 505   return T;
 506 }
 507
 508 Error lto::finalizeOptimizationRemarks(
 509     std::unique_ptr<ToolOutputFile> DiagOutputFile) {
 510   // Make sure we flush the diagnostic remarks file in case the linker doesn't
 511   // call the global destructors before exiting.
 512   if (!DiagOutputFile)
 513     return Error::success();
 514   DiagOutputFile->keep();
 515   DiagOutputFile->os().flush();
 516   return Error::success();
 517 }
 518
 519 Error lto::backend(const Config &C, AddStreamFn AddStream,
 520                    unsigned ParallelCodeGenParallelismLevel, Module &Mod,
 521                    ModuleSummaryIndex &CombinedIndex) {
 522   Expected<const Target *> TOrErr = initAndLookupTarget(C, Mod);
 523   if (!TOrErr)
 524     return TOrErr.takeError();
 525
 526   std::unique_ptr<TargetMachine> TM = createTargetMachine(C, *TOrErr, Mod);
 527
 528   LLVM_DEBUG(dbgs() << "Running regular LTO\n");
 529   if (!C.CodeGenOnly) {
 530     if (!opt(C, TM.get(), 0, Mod, /*IsThinLTO=*/false,
 531              /*ExportSummary=*/&CombinedIndex, /*ImportSummary=*/nullptr,
 532              /*CmdArgs*/ std::vector<uint8_t>()))
 533       return Error::success();
 534   }
 535
 536   if (ParallelCodeGenParallelismLevel == 1) {
 537     codegen(C, TM.get(), AddStream, 0, Mod, CombinedIndex);
 538   } else {
 539     splitCodeGen(C, TM.get(), AddStream, ParallelCodeGenParallelismLevel, Mod,
 540                  CombinedIndex);
 541   }
 542   return Error::success();
 543 }
 544
 545 static void dropDeadSymbols(Module &Mod, const GVSummaryMapTy &DefinedGlobals,
 546                             const ModuleSummaryIndex &Index) {
 547   std::vector<GlobalValue*> DeadGVs;
 548   for (auto &GV : Mod.global_values())
 549     if (GlobalValueSummary *GVS = DefinedGlobals.lookup(GV.getGUID()))
 550       if (!Index.isGlobalValueLive(GVS)) {
 551         DeadGVs.push_back(&GV);
 552         convertToDeclaration(GV);
 553       }
 554
 555   // Now that all dead bodies have been dropped, delete the actual objects
 556   // themselves when possible.
 557   for (GlobalValue *GV : DeadGVs) {
 558     GV->removeDeadConstantUsers();
 559     // Might reference something defined in native object (i.e. dropped a
 560     // non-prevailing IR def, but we need to keep the declaration).
 561     if (GV->use_empty())
 562       GV->eraseFromParent();
 563   }
 564 }
 565
 566 Error lto::thinBackend(const Config &Conf, unsigned Task, AddStreamFn AddStream,
 567                        Module &Mod, const ModuleSummaryIndex &CombinedIndex,
 568                        const FunctionImporter::ImportMapTy &ImportList,
 569                        const GVSummaryMapTy &DefinedGlobals,
 570                        MapVector<StringRef, BitcodeModule> *ModuleMap,
 571                        bool CodeGenOnly, AddStreamFn IRAddStream,
 572                        const std::vector<uint8_t> &CmdArgs) {
 573   Expected<const Target *> TOrErr = initAndLookupTarget(Conf, Mod);
 574   if (!TOrErr)
 575     return TOrErr.takeError();
 576
 577   std::unique_ptr<TargetMachine> TM = createTargetMachine(Conf, *TOrErr, Mod);
 578
 579   // Setup optimization remarks.
 580   auto DiagFileOrErr = lto::setupLLVMOptimizationRemarks(
 581       Mod.getContext(), Conf.RemarksFilename, Conf.RemarksPasses,
 582       Conf.RemarksFormat, Conf.RemarksWithHotness, Conf.RemarksHotnessThreshold,
 583       Task);
 584   if (!DiagFileOrErr)
 585     return DiagFileOrErr.takeError();
 586   auto DiagnosticOutputFile = std::move(*DiagFileOrErr);
 587
 588   // Set the partial sample profile ratio in the profile summary module flag of
 589   // the module, if applicable.
 590   Mod.setPartialSampleProfileRatio(CombinedIndex);
 591
 592   LLVM_DEBUG(dbgs() << "Running ThinLTO\n");
 593   if (CodeGenOnly) {
 594     // If CodeGenOnly is set, we only perform code generation and skip
 595     // optimization. This value may differ from Conf.CodeGenOnly.
 596     codegen(Conf, TM.get(), AddStream, Task, Mod, CombinedIndex);
 597     return finalizeOptimizationRemarks(std::move(DiagnosticOutputFile));
 598   }
 599
 600   if (Conf.PreOptModuleHook && !Conf.PreOptModuleHook(Task, Mod))
 601     return finalizeOptimizationRemarks(std::move(DiagnosticOutputFile));
 602
 603   auto OptimizeAndCodegen =
 604       [&](Module &Mod, TargetMachine *TM,
 605           std::unique_ptr<ToolOutputFile> DiagnosticOutputFile) {
 606         // Perform optimization and code generation for ThinLTO.
 607         if (!opt(Conf, TM, Task, Mod, /*IsThinLTO=*/true,
 608                  /*ExportSummary=*/nullptr, /*ImportSummary=*/&CombinedIndex,
 609                  CmdArgs))
 610           return finalizeOptimizationRemarks(std::move(DiagnosticOutputFile));
 611
 612         // Save the current module before the first codegen round.
 613         // Note that the second codegen round runs only `codegen()` without
 614         // running `opt()`. We're not reaching here as it's bailed out earlier
 615         // with `CodeGenOnly` which has been set in `SecondRoundThinBackend`.
 616         if (IRAddStream)
 617           cgdata::saveModuleForTwoRounds(Mod, Task, IRAddStream);
 618
 619         codegen(Conf, TM, AddStream, Task, Mod, CombinedIndex);
 620         return finalizeOptimizationRemarks(std::move(DiagnosticOutputFile));
 621       };
 622
 623   if (ThinLTOAssumeMerged)
 624     return OptimizeAndCodegen(Mod, TM.get(), std::move(DiagnosticOutputFile));
 625
 626   // When linking an ELF shared object, dso_local should be dropped. We
 627   // conservatively do this for -fpic.
 628   bool ClearDSOLocalOnDeclarations =
 629       TM->getTargetTriple().isOSBinFormatELF() &&
 630       TM->getRelocationModel() != Reloc::Static &&
 631       Mod.getPIELevel() == PIELevel::Default;
 632   renameModuleForThinLTO(Mod, CombinedIndex, ClearDSOLocalOnDeclarations);
 633
 634   dropDeadSymbols(Mod, DefinedGlobals, CombinedIndex);
 635
 636   thinLTOFinalizeInModule(Mod, DefinedGlobals, /*PropagateAttrs=*/true);
 637
 638   if (Conf.PostPromoteModuleHook && !Conf.PostPromoteModuleHook(Task, Mod))
 639     return finalizeOptimizationRemarks(std::move(DiagnosticOutputFile));
 640
 641   if (!DefinedGlobals.empty())
 642     thinLTOInternalizeModule(Mod, DefinedGlobals);
 643
 644   if (Conf.PostInternalizeModuleHook &&
 645       !Conf.PostInternalizeModuleHook(Task, Mod))
 646     return finalizeOptimizationRemarks(std::move(DiagnosticOutputFile));
 647
 648   auto ModuleLoader = [&](StringRef Identifier) {
 649     assert(Mod.getContext().isODRUniquingDebugTypes() &&
 650            "ODR Type uniquing should be enabled on the context");
 651     if (ModuleMap) {
 652       auto I = ModuleMap->find(Identifier);
 653       assert(I != ModuleMap->end());
 654       return I->second.getLazyModule(Mod.getContext(),
 655                                      /*ShouldLazyLoadMetadata=*/true,
 656                                      /*IsImporting*/ true);
 657     }
 658
 659     ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> MBOrErr =
 660         llvm::MemoryBuffer::getFile(Identifier);
 661     if (!MBOrErr)
 662       return Expected<std::unique_ptr<llvm::Module>>(make_error<StringError>(
 663           Twine("Error loading imported file ") + Identifier + " : ",
 664           MBOrErr.getError()));
 665
 666     Expected<BitcodeModule> BMOrErr = findThinLTOModule(**MBOrErr);
 667     if (!BMOrErr)
 668       return Expected<std::unique_ptr<llvm::Module>>(make_error<StringError>(
 669           Twine("Error loading imported file ") + Identifier + " : " +
 670               toString(BMOrErr.takeError()),
 671           inconvertibleErrorCode()));
 672
 673     Expected<std::unique_ptr<Module>> MOrErr =
 674         BMOrErr->getLazyModule(Mod.getContext(),
 675                                /*ShouldLazyLoadMetadata=*/true,
 676                                /*IsImporting*/ true);
 677     if (MOrErr)
 678       (*MOrErr)->setOwnedMemoryBuffer(std::move(*MBOrErr));
 679     return MOrErr;
 680   };
 681
 682   FunctionImporter Importer(CombinedIndex, ModuleLoader,
 683                             ClearDSOLocalOnDeclarations);
 684   if (Error Err = Importer.importFunctions(Mod, ImportList).takeError())
 685     return Err;
 686
 687   // Do this after any importing so that imported code is updated.
 688   updateMemProfAttributes(Mod, CombinedIndex);
 689   updatePublicTypeTestCalls(Mod, CombinedIndex.withWholeProgramVisibility());
 690
 691   if (Conf.PostImportModuleHook && !Conf.PostImportModuleHook(Task, Mod))
 692     return finalizeOptimizationRemarks(std::move(DiagnosticOutputFile));
 693
 694   return OptimizeAndCodegen(Mod, TM.get(), std::move(DiagnosticOutputFile));
 695 }
 696
 697 BitcodeModule *lto::findThinLTOModule(MutableArrayRef<BitcodeModule> BMs) {
 698   if (ThinLTOAssumeMerged && BMs.size() == 1)
 699     return BMs.begin();
 700
 701   for (BitcodeModule &BM : BMs) {
 702     Expected<BitcodeLTOInfo> LTOInfo = BM.getLTOInfo();
 703     if (LTOInfo && LTOInfo->IsThinLTO)
 704       return &BM;
 705   }
 706   return nullptr;
 707 }
 708
 709 Expected<BitcodeModule> lto::findThinLTOModule(MemoryBufferRef MBRef) {
 710   Expected<std::vector<BitcodeModule>> BMsOrErr = getBitcodeModuleList(MBRef);
 711   if (!BMsOrErr)
 712     return BMsOrErr.takeError();
 713
 714   // The bitcode file may contain multiple modules, we want the one that is
 715   // marked as being the ThinLTO module.
 716   if (const BitcodeModule *Bm = lto::findThinLTOModule(*BMsOrErr))
 717     return *Bm;
 718
 719   return make_error<StringError>("Could not find module summary",
 720                                  inconvertibleErrorCode());
 721 }
 722
 723 bool lto::initImportList(const Module &M,
 724                          const ModuleSummaryIndex &CombinedIndex,
 725                          FunctionImporter::ImportMapTy &ImportList) {
 726   if (ThinLTOAssumeMerged)
 727     return true;
 728   // We can simply import the values mentioned in the combined index, since
 729   // we should only invoke this using the individual indexes written out
 730   // via a WriteIndexesThinBackend.
 731   for (const auto &GlobalList : CombinedIndex) {
 732     // Ignore entries for undefined references.
 733     if (GlobalList.second.SummaryList.empty())
 734       continue;
 735
 736     auto GUID = GlobalList.first;
 737     for (const auto &Summary : GlobalList.second.SummaryList) {
 738       // Skip the summaries for the importing module. These are included to
 739       // e.g. record required linkage changes.
 740       if (Summary->modulePath() == M.getModuleIdentifier())
 741         continue;
 742       // Add an entry to provoke importing by thinBackend.
 743       ImportList.addGUID(Summary->modulePath(), GUID, Summary->importType());
 744     }
 745   }
 746   return true;
 747 }