tools/llvm-lto/llvm-lto.cpp

   1 //===- llvm-lto: a simple command-line program to link modules with LTO ---===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This program takes in a list of bitcode files, links them, performs link-time
  10 // optimization, and outputs an object file.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "llvm-c/lto.h"
  15 #include "llvm/ADT/ArrayRef.h"
  16 #include "llvm/ADT/STLExtras.h"
  17 #include "llvm/ADT/SmallString.h"
  18 #include "llvm/ADT/StringExtras.h"
  19 #include "llvm/ADT/StringRef.h"
  20 #include "llvm/ADT/StringSet.h"
  21 #include "llvm/ADT/Twine.h"
  22 #include "llvm/Bitcode/BitcodeReader.h"
  23 #include "llvm/Bitcode/BitcodeWriter.h"
  24 #include "llvm/CodeGen/CommandFlags.inc"
  25 #include "llvm/IR/DiagnosticInfo.h"
  26 #include "llvm/IR/DiagnosticPrinter.h"
  27 #include "llvm/IR/LLVMContext.h"
  28 #include "llvm/IR/Module.h"
  29 #include "llvm/IR/ModuleSummaryIndex.h"
  30 #include "llvm/IR/Verifier.h"
  31 #include "llvm/IRReader/IRReader.h"
  32 #include "llvm/LTO/legacy/LTOCodeGenerator.h"
  33 #include "llvm/LTO/legacy/LTOModule.h"
  34 #include "llvm/LTO/legacy/ThinLTOCodeGenerator.h"
  35 #include "llvm/Support/Allocator.h"
  36 #include "llvm/Support/Casting.h"
  37 #include "llvm/Support/CommandLine.h"
  38 #include "llvm/Support/Error.h"
  39 #include "llvm/Support/ErrorHandling.h"
  40 #include "llvm/Support/ErrorOr.h"
  41 #include "llvm/Support/FileSystem.h"
  42 #include "llvm/Support/InitLLVM.h"
  43 #include "llvm/Support/MemoryBuffer.h"
  44 #include "llvm/Support/Path.h"
  45 #include "llvm/Support/SourceMgr.h"
  46 #include "llvm/Support/TargetSelect.h"
  47 #include "llvm/Support/ToolOutputFile.h"
  48 #include "llvm/Support/raw_ostream.h"
  49 #include "llvm/Target/TargetOptions.h"
  50 #include <algorithm>
  51 #include <cassert>
  52 #include <cstdint>
  53 #include <cstdlib>
  54 #include <list>
  55 #include <map>
  56 #include <memory>
  57 #include <string>
  58 #include <system_error>
  59 #include <tuple>
  60 #include <utility>
  61 #include <vector>
  62
  63 using namespace llvm;
  64
  65 static cl::opt<char>
  66     OptLevel("O", cl::desc("Optimization level. [-O0, -O1, -O2, or -O3] "
  67                            "(default = '-O2')"),
  68              cl::Prefix, cl::ZeroOrMore, cl::init('2'));
  69
  70 static cl::opt<bool>
  71     IndexStats("thinlto-index-stats",
  72                cl::desc("Print statistic for the index in every input files"),
  73                cl::init(false));
  74
  75 static cl::opt<bool> DisableVerify(
  76     "disable-verify", cl::init(false),
  77     cl::desc("Do not run the verifier during the optimization pipeline"));
  78
  79 static cl::opt<bool> DisableInline("disable-inlining", cl::init(false),
  80                                    cl::desc("Do not run the inliner pass"));
  81
  82 static cl::opt<bool>
  83     DisableGVNLoadPRE("disable-gvn-loadpre", cl::init(false),
  84                       cl::desc("Do not run the GVN load PRE pass"));
  85
  86 static cl::opt<bool> DisableLTOVectorization(
  87     "disable-lto-vectorization", cl::init(false),
  88     cl::desc("Do not run loop or slp vectorization during LTO"));
  89
  90 static cl::opt<bool> EnableFreestanding(
  91     "lto-freestanding", cl::init(false),
  92     cl::desc("Enable Freestanding (disable builtins / TLI) during LTO"));
  93
  94 static cl::opt<bool> UseDiagnosticHandler(
  95     "use-diagnostic-handler", cl::init(false),
  96     cl::desc("Use a diagnostic handler to test the handler interface"));
  97
  98 static cl::opt<bool>
  99     ThinLTO("thinlto", cl::init(false),
 100             cl::desc("Only write combined global index for ThinLTO backends"));
 101
 102 enum ThinLTOModes {
 103   THINLINK,
 104   THINDISTRIBUTE,
 105   THINEMITIMPORTS,
 106   THINPROMOTE,
 107   THINIMPORT,
 108   THININTERNALIZE,
 109   THINOPT,
 110   THINCODEGEN,
 111   THINALL
 112 };
 113
 114 cl::opt<ThinLTOModes> ThinLTOMode(
 115     "thinlto-action", cl::desc("Perform a single ThinLTO stage:"),
 116     cl::values(
 117         clEnumValN(
 118             THINLINK, "thinlink",
 119             "ThinLink: produces the index by linking only the summaries."),
 120         clEnumValN(THINDISTRIBUTE, "distributedindexes",
 121                    "Produces individual indexes for distributed backends."),
 122         clEnumValN(THINEMITIMPORTS, "emitimports",
 123                    "Emit imports files for distributed backends."),
 124         clEnumValN(THINPROMOTE, "promote",
 125                    "Perform pre-import promotion (requires -thinlto-index)."),
 126         clEnumValN(THINIMPORT, "import", "Perform both promotion and "
 127                                          "cross-module importing (requires "
 128                                          "-thinlto-index)."),
 129         clEnumValN(THININTERNALIZE, "internalize",
 130                    "Perform internalization driven by -exported-symbol "
 131                    "(requires -thinlto-index)."),
 132         clEnumValN(THINOPT, "optimize", "Perform ThinLTO optimizations."),
 133         clEnumValN(THINCODEGEN, "codegen", "CodeGen (expected to match llc)"),
 134         clEnumValN(THINALL, "run", "Perform ThinLTO end-to-end")));
 135
 136 static cl::opt<std::string>
 137     ThinLTOIndex("thinlto-index",
 138                  cl::desc("Provide the index produced by a ThinLink, required "
 139                           "to perform the promotion and/or importing."));
 140
 141 static cl::opt<std::string> ThinLTOPrefixReplace(
 142     "thinlto-prefix-replace",
 143     cl::desc("Control where files for distributed backends are "
 144              "created. Expects 'oldprefix;newprefix' and if path "
 145              "prefix of output file is oldprefix it will be "
 146              "replaced with newprefix."));
 147
 148 static cl::opt<std::string> ThinLTOModuleId(
 149     "thinlto-module-id",
 150     cl::desc("For the module ID for the file to process, useful to "
 151              "match what is in the index."));
 152
 153 static cl::opt<std::string>
 154     ThinLTOCacheDir("thinlto-cache-dir", cl::desc("Enable ThinLTO caching."));
 155
 156 static cl::opt<int>
 157     ThinLTOCachePruningInterval("thinlto-cache-pruning-interval",
 158     cl::init(1200), cl::desc("Set ThinLTO cache pruning interval."));
 159
 160 static cl::opt<unsigned long long>
 161     ThinLTOCacheMaxSizeBytes("thinlto-cache-max-size-bytes",
 162     cl::desc("Set ThinLTO cache pruning directory maximum size in bytes."));
 163
 164 static cl::opt<int>
 165     ThinLTOCacheMaxSizeFiles("thinlto-cache-max-size-files", cl::init(1000000),
 166     cl::desc("Set ThinLTO cache pruning directory maximum number of files."));
 167
 168 static cl::opt<unsigned>
 169     ThinLTOCacheEntryExpiration("thinlto-cache-entry-expiration", cl::init(604800) /* 1w */,
 170     cl::desc("Set ThinLTO cache entry expiration time."));
 171
 172 static cl::opt<std::string> ThinLTOSaveTempsPrefix(
 173     "thinlto-save-temps",
 174     cl::desc("Save ThinLTO temp files using filenames created by adding "
 175              "suffixes to the given file path prefix."));
 176
 177 static cl::opt<std::string> ThinLTOGeneratedObjectsDir(
 178     "thinlto-save-objects",
 179     cl::desc("Save ThinLTO generated object files using filenames created in "
 180              "the given directory."));
 181
 182 static cl::opt<bool>
 183     SaveModuleFile("save-merged-module", cl::init(false),
 184                    cl::desc("Write merged LTO module to file before CodeGen"));
 185
 186 static cl::list<std::string> InputFilenames(cl::Positional, cl::OneOrMore,
 187                                             cl::desc("<input bitcode files>"));
 188
 189 static cl::opt<std::string> OutputFilename("o", cl::init(""),
 190                                            cl::desc("Override output filename"),
 191                                            cl::value_desc("filename"));
 192
 193 static cl::list<std::string> ExportedSymbols(
 194     "exported-symbol",
 195     cl::desc("List of symbols to export from the resulting object file"),
 196     cl::ZeroOrMore);
 197
 198 static cl::list<std::string>
 199     DSOSymbols("dso-symbol",
 200                cl::desc("Symbol to put in the symtab in the resulting dso"),
 201                cl::ZeroOrMore);
 202
 203 static cl::opt<bool> ListSymbolsOnly(
 204     "list-symbols-only", cl::init(false),
 205     cl::desc("Instead of running LTO, list the symbols in each IR file"));
 206
 207 static cl::opt<bool> SetMergedModule(
 208     "set-merged-module", cl::init(false),
 209     cl::desc("Use the first input module as the merged module"));
 210
 211 static cl::opt<unsigned> Parallelism("j", cl::Prefix, cl::init(1),
 212                                      cl::desc("Number of backend threads"));
 213
 214 static cl::opt<bool> RestoreGlobalsLinkage(
 215     "restore-linkage", cl::init(false),
 216     cl::desc("Restore original linkage of globals prior to CodeGen"));
 217
 218 static cl::opt<bool> CheckHasObjC(
 219     "check-for-objc", cl::init(false),
 220     cl::desc("Only check if the module has objective-C defined in it"));
 221
 222 namespace {
 223
 224 struct ModuleInfo {
 225   std::vector<bool> CanBeHidden;
 226 };
 227
 228 } // end anonymous namespace
 229
 230 static void handleDiagnostics(lto_codegen_diagnostic_severity_t Severity,
 231                               const char *Msg, void *) {
 232   errs() << "llvm-lto: ";
 233   switch (Severity) {
 234   case LTO_DS_NOTE:
 235     errs() << "note: ";
 236     break;
 237   case LTO_DS_REMARK:
 238     errs() << "remark: ";
 239     break;
 240   case LTO_DS_ERROR:
 241     errs() << "error: ";
 242     break;
 243   case LTO_DS_WARNING:
 244     errs() << "warning: ";
 245     break;
 246   }
 247   errs() << Msg << "\n";
 248 }
 249
 250 static std::string CurrentActivity;
 251
 252 namespace {
 253   struct LLVMLTODiagnosticHandler : public DiagnosticHandler {
 254     bool handleDiagnostics(const DiagnosticInfo &DI) override {
 255       raw_ostream &OS = errs();
 256       OS << "llvm-lto: ";
 257       switch (DI.getSeverity()) {
 258       case DS_Error:
 259         OS << "error";
 260         break;
 261       case DS_Warning:
 262         OS << "warning";
 263         break;
 264       case DS_Remark:
 265         OS << "remark";
 266         break;
 267       case DS_Note:
 268         OS << "note";
 269         break;
 270       }
 271       if (!CurrentActivity.empty())
 272         OS << ' ' << CurrentActivity;
 273       OS << ": ";
 274
 275       DiagnosticPrinterRawOStream DP(OS);
 276       DI.print(DP);
 277       OS << '\n';
 278
 279       if (DI.getSeverity() == DS_Error)
 280         exit(1);
 281       return true;
 282     }
 283   };
 284   }
 285
 286 static void error(const Twine &Msg) {
 287   errs() << "llvm-lto: " << Msg << '\n';
 288   exit(1);
 289 }
 290
 291 static void error(std::error_code EC, const Twine &Prefix) {
 292   if (EC)
 293     error(Prefix + ": " + EC.message());
 294 }
 295
 296 template <typename T>
 297 static void error(const ErrorOr<T> &V, const Twine &Prefix) {
 298   error(V.getError(), Prefix);
 299 }
 300
 301 static void maybeVerifyModule(const Module &Mod) {
 302   if (!DisableVerify && verifyModule(Mod, &errs()))
 303     error("Broken Module");
 304 }
 305
 306 static std::unique_ptr<LTOModule>
 307 getLocalLTOModule(StringRef Path, std::unique_ptr<MemoryBuffer> &Buffer,
 308                   const TargetOptions &Options) {
 309   ErrorOr<std::unique_ptr<MemoryBuffer>> BufferOrErr =
 310       MemoryBuffer::getFile(Path);
 311   error(BufferOrErr, "error loading file '" + Path + "'");
 312   Buffer = std::move(BufferOrErr.get());
 313   CurrentActivity = ("loading file '" + Path + "'").str();
 314   std::unique_ptr<LLVMContext> Context = llvm::make_unique<LLVMContext>();
 315   Context->setDiagnosticHandler(llvm::make_unique<LLVMLTODiagnosticHandler>(),
 316                                 true);
 317   ErrorOr<std::unique_ptr<LTOModule>> Ret = LTOModule::createInLocalContext(
 318       std::move(Context), Buffer->getBufferStart(), Buffer->getBufferSize(),
 319       Options, Path);
 320   CurrentActivity = "";
 321   maybeVerifyModule((*Ret)->getModule());
 322   return std::move(*Ret);
 323 }
 324
 325 /// Print some statistics on the index for each input files.
 326 void printIndexStats() {
 327   for (auto &Filename : InputFilenames) {
 328     ExitOnError ExitOnErr("llvm-lto: error loading file '" + Filename + "': ");
 329     std::unique_ptr<ModuleSummaryIndex> Index =
 330         ExitOnErr(getModuleSummaryIndexForFile(Filename));
 331     // Skip files without a module summary.
 332     if (!Index)
 333       report_fatal_error(Filename + " does not contain an index");
 334
 335     unsigned Calls = 0, Refs = 0, Functions = 0, Alias = 0, Globals = 0;
 336     for (auto &Summaries : *Index) {
 337       for (auto &Summary : Summaries.second.SummaryList) {
 338         Refs += Summary->refs().size();
 339         if (auto *FuncSummary = dyn_cast<FunctionSummary>(Summary.get())) {
 340           Functions++;
 341           Calls += FuncSummary->calls().size();
 342         } else if (isa<AliasSummary>(Summary.get()))
 343           Alias++;
 344         else
 345           Globals++;
 346       }
 347     }
 348     outs() << "Index " << Filename << " contains "
 349            << (Alias + Globals + Functions) << " nodes (" << Functions
 350            << " functions, " << Alias << " alias, " << Globals
 351            << " globals) and " << (Calls + Refs) << " edges (" << Refs
 352            << " refs and " << Calls << " calls)\n";
 353   }
 354 }
 355
 356 /// List symbols in each IR file.
 357 ///
 358 /// The main point here is to provide lit-testable coverage for the LTOModule
 359 /// functionality that's exposed by the C API to list symbols.  Moreover, this
 360 /// provides testing coverage for modules that have been created in their own
 361 /// contexts.
 362 static void listSymbols(const TargetOptions &Options) {
 363   for (auto &Filename : InputFilenames) {
 364     std::unique_ptr<MemoryBuffer> Buffer;
 365     std::unique_ptr<LTOModule> Module =
 366         getLocalLTOModule(Filename, Buffer, Options);
 367
 368     // List the symbols.
 369     outs() << Filename << ":\n";
 370     for (int I = 0, E = Module->getSymbolCount(); I != E; ++I)
 371       outs() << Module->getSymbolName(I) << "\n";
 372   }
 373 }
 374
 375 /// Create a combined index file from the input IR files and write it.
 376 ///
 377 /// This is meant to enable testing of ThinLTO combined index generation,
 378 /// currently available via the gold plugin via -thinlto.
 379 static void createCombinedModuleSummaryIndex() {
 380   ModuleSummaryIndex CombinedIndex(/*HaveGVs=*/false);
 381   uint64_t NextModuleId = 0;
 382   for (auto &Filename : InputFilenames) {
 383     ExitOnError ExitOnErr("llvm-lto: error loading file '" + Filename + "': ");
 384     std::unique_ptr<MemoryBuffer> MB =
 385         ExitOnErr(errorOrToExpected(MemoryBuffer::getFileOrSTDIN(Filename)));
 386     ExitOnErr(readModuleSummaryIndex(*MB, CombinedIndex, NextModuleId++));
 387   }
 388   std::error_code EC;
 389   assert(!OutputFilename.empty());
 390   raw_fd_ostream OS(OutputFilename + ".thinlto.bc", EC,
 391                     sys::fs::OpenFlags::F_None);
 392   error(EC, "error opening the file '" + OutputFilename + ".thinlto.bc'");
 393   WriteIndexToFile(CombinedIndex, OS);
 394   OS.close();
 395 }
 396
 397 /// Parse the thinlto_prefix_replace option into the \p OldPrefix and
 398 /// \p NewPrefix strings, if it was specified.
 399 static void getThinLTOOldAndNewPrefix(std::string &OldPrefix,
 400                                       std::string &NewPrefix) {
 401   assert(ThinLTOPrefixReplace.empty() ||
 402          ThinLTOPrefixReplace.find(";") != StringRef::npos);
 403   StringRef PrefixReplace = ThinLTOPrefixReplace;
 404   std::pair<StringRef, StringRef> Split = PrefixReplace.split(";");
 405   OldPrefix = Split.first.str();
 406   NewPrefix = Split.second.str();
 407 }
 408
 409 /// Given the original \p Path to an output file, replace any path
 410 /// prefix matching \p OldPrefix with \p NewPrefix. Also, create the
 411 /// resulting directory if it does not yet exist.
 412 static std::string getThinLTOOutputFile(const std::string &Path,
 413                                         const std::string &OldPrefix,
 414                                         const std::string &NewPrefix) {
 415   if (OldPrefix.empty() && NewPrefix.empty())
 416     return Path;
 417   SmallString<128> NewPath(Path);
 418   llvm::sys::path::replace_path_prefix(NewPath, OldPrefix, NewPrefix);
 419   StringRef ParentPath = llvm::sys::path::parent_path(NewPath.str());
 420   if (!ParentPath.empty()) {
 421     // Make sure the new directory exists, creating it if necessary.
 422     if (std::error_code EC = llvm::sys::fs::create_directories(ParentPath))
 423       error(EC, "error creating the directory '" + ParentPath + "'");
 424   }
 425   return NewPath.str();
 426 }
 427
 428 namespace thinlto {
 429
 430 std::vector<std::unique_ptr<MemoryBuffer>>
 431 loadAllFilesForIndex(const ModuleSummaryIndex &Index) {
 432   std::vector<std::unique_ptr<MemoryBuffer>> InputBuffers;
 433
 434   for (auto &ModPath : Index.modulePaths()) {
 435     const auto &Filename = ModPath.first();
 436     std::string CurrentActivity = ("loading file '" + Filename + "'").str();
 437     auto InputOrErr = MemoryBuffer::getFile(Filename);
 438     error(InputOrErr, "error " + CurrentActivity);
 439     InputBuffers.push_back(std::move(*InputOrErr));
 440   }
 441   return InputBuffers;
 442 }
 443
 444 std::unique_ptr<ModuleSummaryIndex> loadCombinedIndex() {
 445   if (ThinLTOIndex.empty())
 446     report_fatal_error("Missing -thinlto-index for ThinLTO promotion stage");
 447   ExitOnError ExitOnErr("llvm-lto: error loading file '" + ThinLTOIndex +
 448                         "': ");
 449   return ExitOnErr(getModuleSummaryIndexForFile(ThinLTOIndex));
 450 }
 451
 452 static std::unique_ptr<Module> loadModule(StringRef Filename,
 453                                           LLVMContext &Ctx) {
 454   SMDiagnostic Err;
 455   std::unique_ptr<Module> M(parseIRFile(Filename, Err, Ctx));
 456   if (!M) {
 457     Err.print("llvm-lto", errs());
 458     report_fatal_error("Can't load module for file " + Filename);
 459   }
 460   maybeVerifyModule(*M);
 461
 462   if (ThinLTOModuleId.getNumOccurrences()) {
 463     if (InputFilenames.size() != 1)
 464       report_fatal_error("Can't override the module id for multiple files");
 465     M->setModuleIdentifier(ThinLTOModuleId);
 466   }
 467   return M;
 468 }
 469
 470 static void writeModuleToFile(Module &TheModule, StringRef Filename) {
 471   std::error_code EC;
 472   raw_fd_ostream OS(Filename, EC, sys::fs::OpenFlags::F_None);
 473   error(EC, "error opening the file '" + Filename + "'");
 474   maybeVerifyModule(TheModule);
 475   WriteBitcodeToFile(TheModule, OS, /* ShouldPreserveUseListOrder */ true);
 476 }
 477
 478 class ThinLTOProcessing {
 479 public:
 480   ThinLTOCodeGenerator ThinGenerator;
 481
 482   ThinLTOProcessing(const TargetOptions &Options) {
 483     ThinGenerator.setCodePICModel(getRelocModel());
 484     ThinGenerator.setTargetOptions(Options);
 485     ThinGenerator.setCacheDir(ThinLTOCacheDir);
 486     ThinGenerator.setCachePruningInterval(ThinLTOCachePruningInterval);
 487     ThinGenerator.setCacheEntryExpiration(ThinLTOCacheEntryExpiration);
 488     ThinGenerator.setCacheMaxSizeFiles(ThinLTOCacheMaxSizeFiles);
 489     ThinGenerator.setCacheMaxSizeBytes(ThinLTOCacheMaxSizeBytes);
 490     ThinGenerator.setFreestanding(EnableFreestanding);
 491
 492     // Add all the exported symbols to the table of symbols to preserve.
 493     for (unsigned i = 0; i < ExportedSymbols.size(); ++i)
 494       ThinGenerator.preserveSymbol(ExportedSymbols[i]);
 495   }
 496
 497   void run() {
 498     switch (ThinLTOMode) {
 499     case THINLINK:
 500       return thinLink();
 501     case THINDISTRIBUTE:
 502       return distributedIndexes();
 503     case THINEMITIMPORTS:
 504       return emitImports();
 505     case THINPROMOTE:
 506       return promote();
 507     case THINIMPORT:
 508       return import();
 509     case THININTERNALIZE:
 510       return internalize();
 511     case THINOPT:
 512       return optimize();
 513     case THINCODEGEN:
 514       return codegen();
 515     case THINALL:
 516       return runAll();
 517     }
 518   }
 519
 520 private:
 521   /// Load the input files, create the combined index, and write it out.
 522   void thinLink() {
 523     // Perform "ThinLink": just produce the index
 524     if (OutputFilename.empty())
 525       report_fatal_error(
 526           "OutputFilename is necessary to store the combined index.\n");
 527
 528     LLVMContext Ctx;
 529     std::vector<std::unique_ptr<MemoryBuffer>> InputBuffers;
 530     for (unsigned i = 0; i < InputFilenames.size(); ++i) {
 531       auto &Filename = InputFilenames[i];
 532       std::string CurrentActivity = "loading file '" + Filename + "'";
 533       auto InputOrErr = MemoryBuffer::getFile(Filename);
 534       error(InputOrErr, "error " + CurrentActivity);
 535       InputBuffers.push_back(std::move(*InputOrErr));
 536       ThinGenerator.addModule(Filename, InputBuffers.back()->getBuffer());
 537     }
 538
 539     auto CombinedIndex = ThinGenerator.linkCombinedIndex();
 540     if (!CombinedIndex)
 541       report_fatal_error("ThinLink didn't create an index");
 542     std::error_code EC;
 543     raw_fd_ostream OS(OutputFilename, EC, sys::fs::OpenFlags::F_None);
 544     error(EC, "error opening the file '" + OutputFilename + "'");
 545     WriteIndexToFile(*CombinedIndex, OS);
 546   }
 547
 548   /// Load the combined index from disk, then compute and generate
 549   /// individual index files suitable for ThinLTO distributed backend builds
 550   /// on the files mentioned on the command line (these must match the index
 551   /// content).
 552   void distributedIndexes() {
 553     if (InputFilenames.size() != 1 && !OutputFilename.empty())
 554       report_fatal_error("Can't handle a single output filename and multiple "
 555                          "input files, do not provide an output filename and "
 556                          "the output files will be suffixed from the input "
 557                          "ones.");
 558
 559     std::string OldPrefix, NewPrefix;
 560     getThinLTOOldAndNewPrefix(OldPrefix, NewPrefix);
 561
 562     auto Index = loadCombinedIndex();
 563     for (auto &Filename : InputFilenames) {
 564       LLVMContext Ctx;
 565       auto TheModule = loadModule(Filename, Ctx);
 566
 567       // Build a map of module to the GUIDs and summary objects that should
 568       // be written to its index.
 569       std::map<std::string, GVSummaryMapTy> ModuleToSummariesForIndex;
 570       ThinGenerator.gatherImportedSummariesForModule(*TheModule, *Index,
 571                                                      ModuleToSummariesForIndex);
 572
 573       std::string OutputName = OutputFilename;
 574       if (OutputName.empty()) {
 575         OutputName = Filename + ".thinlto.bc";
 576       }
 577       OutputName = getThinLTOOutputFile(OutputName, OldPrefix, NewPrefix);
 578       std::error_code EC;
 579       raw_fd_ostream OS(OutputName, EC, sys::fs::OpenFlags::F_None);
 580       error(EC, "error opening the file '" + OutputName + "'");
 581       WriteIndexToFile(*Index, OS, &ModuleToSummariesForIndex);
 582     }
 583   }
 584
 585   /// Load the combined index from disk, compute the imports, and emit
 586   /// the import file lists for each module to disk.
 587   void emitImports() {
 588     if (InputFilenames.size() != 1 && !OutputFilename.empty())
 589       report_fatal_error("Can't handle a single output filename and multiple "
 590                          "input files, do not provide an output filename and "
 591                          "the output files will be suffixed from the input "
 592                          "ones.");
 593
 594     std::string OldPrefix, NewPrefix;
 595     getThinLTOOldAndNewPrefix(OldPrefix, NewPrefix);
 596
 597     auto Index = loadCombinedIndex();
 598     for (auto &Filename : InputFilenames) {
 599       LLVMContext Ctx;
 600       auto TheModule = loadModule(Filename, Ctx);
 601       std::string OutputName = OutputFilename;
 602       if (OutputName.empty()) {
 603         OutputName = Filename + ".imports";
 604       }
 605       OutputName = getThinLTOOutputFile(OutputName, OldPrefix, NewPrefix);
 606       ThinGenerator.emitImports(*TheModule, OutputName, *Index);
 607     }
 608   }
 609
 610   /// Load the combined index from disk, then load every file referenced by
 611   /// the index and add them to the generator, finally perform the promotion
 612   /// on the files mentioned on the command line (these must match the index
 613   /// content).
 614   void promote() {
 615     if (InputFilenames.size() != 1 && !OutputFilename.empty())
 616       report_fatal_error("Can't handle a single output filename and multiple "
 617                          "input files, do not provide an output filename and "
 618                          "the output files will be suffixed from the input "
 619                          "ones.");
 620
 621     auto Index = loadCombinedIndex();
 622     for (auto &Filename : InputFilenames) {
 623       LLVMContext Ctx;
 624       auto TheModule = loadModule(Filename, Ctx);
 625
 626       ThinGenerator.promote(*TheModule, *Index);
 627
 628       std::string OutputName = OutputFilename;
 629       if (OutputName.empty()) {
 630         OutputName = Filename + ".thinlto.promoted.bc";
 631       }
 632       writeModuleToFile(*TheModule, OutputName);
 633     }
 634   }
 635
 636   /// Load the combined index from disk, then load every file referenced by
 637   /// the index and add them to the generator, then performs the promotion and
 638   /// cross module importing on the files mentioned on the command line
 639   /// (these must match the index content).
 640   void import() {
 641     if (InputFilenames.size() != 1 && !OutputFilename.empty())
 642       report_fatal_error("Can't handle a single output filename and multiple "
 643                          "input files, do not provide an output filename and "
 644                          "the output files will be suffixed from the input "
 645                          "ones.");
 646
 647     auto Index = loadCombinedIndex();
 648     auto InputBuffers = loadAllFilesForIndex(*Index);
 649     for (auto &MemBuffer : InputBuffers)
 650       ThinGenerator.addModule(MemBuffer->getBufferIdentifier(),
 651                               MemBuffer->getBuffer());
 652
 653     for (auto &Filename : InputFilenames) {
 654       LLVMContext Ctx;
 655       auto TheModule = loadModule(Filename, Ctx);
 656
 657       ThinGenerator.crossModuleImport(*TheModule, *Index);
 658
 659       std::string OutputName = OutputFilename;
 660       if (OutputName.empty()) {
 661         OutputName = Filename + ".thinlto.imported.bc";
 662       }
 663       writeModuleToFile(*TheModule, OutputName);
 664     }
 665   }
 666
 667   void internalize() {
 668     if (InputFilenames.size() != 1 && !OutputFilename.empty())
 669       report_fatal_error("Can't handle a single output filename and multiple "
 670                          "input files, do not provide an output filename and "
 671                          "the output files will be suffixed from the input "
 672                          "ones.");
 673
 674     if (ExportedSymbols.empty())
 675       errs() << "Warning: -internalize will not perform without "
 676                 "-exported-symbol\n";
 677
 678     auto Index = loadCombinedIndex();
 679     auto InputBuffers = loadAllFilesForIndex(*Index);
 680     for (auto &MemBuffer : InputBuffers)
 681       ThinGenerator.addModule(MemBuffer->getBufferIdentifier(),
 682                               MemBuffer->getBuffer());
 683
 684     for (auto &Filename : InputFilenames) {
 685       LLVMContext Ctx;
 686       auto TheModule = loadModule(Filename, Ctx);
 687
 688       ThinGenerator.internalize(*TheModule, *Index);
 689
 690       std::string OutputName = OutputFilename;
 691       if (OutputName.empty()) {
 692         OutputName = Filename + ".thinlto.internalized.bc";
 693       }
 694       writeModuleToFile(*TheModule, OutputName);
 695     }
 696   }
 697
 698   void optimize() {
 699     if (InputFilenames.size() != 1 && !OutputFilename.empty())
 700       report_fatal_error("Can't handle a single output filename and multiple "
 701                          "input files, do not provide an output filename and "
 702                          "the output files will be suffixed from the input "
 703                          "ones.");
 704     if (!ThinLTOIndex.empty())
 705       errs() << "Warning: -thinlto-index ignored for optimize stage";
 706
 707     for (auto &Filename : InputFilenames) {
 708       LLVMContext Ctx;
 709       auto TheModule = loadModule(Filename, Ctx);
 710
 711       ThinGenerator.optimize(*TheModule);
 712
 713       std::string OutputName = OutputFilename;
 714       if (OutputName.empty()) {
 715         OutputName = Filename + ".thinlto.imported.bc";
 716       }
 717       writeModuleToFile(*TheModule, OutputName);
 718     }
 719   }
 720
 721   void codegen() {
 722     if (InputFilenames.size() != 1 && !OutputFilename.empty())
 723       report_fatal_error("Can't handle a single output filename and multiple "
 724                          "input files, do not provide an output filename and "
 725                          "the output files will be suffixed from the input "
 726                          "ones.");
 727     if (!ThinLTOIndex.empty())
 728       errs() << "Warning: -thinlto-index ignored for codegen stage";
 729
 730     std::vector<std::unique_ptr<MemoryBuffer>> InputBuffers;
 731     for (auto &Filename : InputFilenames) {
 732       LLVMContext Ctx;
 733       auto InputOrErr = MemoryBuffer::getFile(Filename);
 734       error(InputOrErr, "error " + CurrentActivity);
 735       InputBuffers.push_back(std::move(*InputOrErr));
 736       ThinGenerator.addModule(Filename, InputBuffers.back()->getBuffer());
 737     }
 738     ThinGenerator.setCodeGenOnly(true);
 739     ThinGenerator.run();
 740     for (auto BinName :
 741          zip(ThinGenerator.getProducedBinaries(), InputFilenames)) {
 742       std::string OutputName = OutputFilename;
 743       if (OutputName.empty())
 744         OutputName = std::get<1>(BinName) + ".thinlto.o";
 745       else if (OutputName == "-") {
 746         outs() << std::get<0>(BinName)->getBuffer();
 747         return;
 748       }
 749
 750       std::error_code EC;
 751       raw_fd_ostream OS(OutputName, EC, sys::fs::OpenFlags::F_None);
 752       error(EC, "error opening the file '" + OutputName + "'");
 753       OS << std::get<0>(BinName)->getBuffer();
 754     }
 755   }
 756
 757   /// Full ThinLTO process
 758   void runAll() {
 759     if (!OutputFilename.empty())
 760       report_fatal_error("Do not provide an output filename for ThinLTO "
 761                          " processing, the output files will be suffixed from "
 762                          "the input ones.");
 763
 764     if (!ThinLTOIndex.empty())
 765       errs() << "Warning: -thinlto-index ignored for full ThinLTO process";
 766
 767     LLVMContext Ctx;
 768     std::vector<std::unique_ptr<MemoryBuffer>> InputBuffers;
 769     for (unsigned i = 0; i < InputFilenames.size(); ++i) {
 770       auto &Filename = InputFilenames[i];
 771       std::string CurrentActivity = "loading file '" + Filename + "'";
 772       auto InputOrErr = MemoryBuffer::getFile(Filename);
 773       error(InputOrErr, "error " + CurrentActivity);
 774       InputBuffers.push_back(std::move(*InputOrErr));
 775       ThinGenerator.addModule(Filename, InputBuffers.back()->getBuffer());
 776     }
 777
 778     if (!ThinLTOSaveTempsPrefix.empty())
 779       ThinGenerator.setSaveTempsDir(ThinLTOSaveTempsPrefix);
 780
 781     if (!ThinLTOGeneratedObjectsDir.empty()) {
 782       ThinGenerator.setGeneratedObjectsDirectory(ThinLTOGeneratedObjectsDir);
 783       ThinGenerator.run();
 784       return;
 785     }
 786
 787     ThinGenerator.run();
 788
 789     auto &Binaries = ThinGenerator.getProducedBinaries();
 790     if (Binaries.size() != InputFilenames.size())
 791       report_fatal_error("Number of output objects does not match the number "
 792                          "of inputs");
 793
 794     for (unsigned BufID = 0; BufID < Binaries.size(); ++BufID) {
 795       auto OutputName = InputFilenames[BufID] + ".thinlto.o";
 796       std::error_code EC;
 797       raw_fd_ostream OS(OutputName, EC, sys::fs::OpenFlags::F_None);
 798       error(EC, "error opening the file '" + OutputName + "'");
 799       OS << Binaries[BufID]->getBuffer();
 800     }
 801   }
 802
 803   /// Load the combined index from disk, then load every file referenced by
 804 };
 805
 806 } // end namespace thinlto
 807
 808 int main(int argc, char **argv) {
 809   InitLLVM X(argc, argv);
 810   cl::ParseCommandLineOptions(argc, argv, "llvm LTO linker\n");
 811
 812   if (OptLevel < '0' || OptLevel > '3')
 813     error("optimization level must be between 0 and 3");
 814
 815   // Initialize the configured targets.
 816   InitializeAllTargets();
 817   InitializeAllTargetMCs();
 818   InitializeAllAsmPrinters();
 819   InitializeAllAsmParsers();
 820
 821   // set up the TargetOptions for the machine
 822   TargetOptions Options = InitTargetOptionsFromCodeGenFlags();
 823
 824   if (ListSymbolsOnly) {
 825     listSymbols(Options);
 826     return 0;
 827   }
 828
 829   if (IndexStats) {
 830     printIndexStats();
 831     return 0;
 832   }
 833
 834   if (CheckHasObjC) {
 835     for (auto &Filename : InputFilenames) {
 836       ExitOnError ExitOnErr(std::string(*argv) + ": error loading file '" +
 837                             Filename + "': ");
 838       std::unique_ptr<MemoryBuffer> BufferOrErr =
 839           ExitOnErr(errorOrToExpected(MemoryBuffer::getFile(Filename)));
 840       auto Buffer = std::move(BufferOrErr.get());
 841       if (ExitOnErr(isBitcodeContainingObjCCategory(*Buffer)))
 842         outs() << "Bitcode " << Filename << " contains ObjC\n";
 843       else
 844         outs() << "Bitcode " << Filename << " does not contain ObjC\n";
 845     }
 846     return 0;
 847   }
 848
 849   if (ThinLTOMode.getNumOccurrences()) {
 850     if (ThinLTOMode.getNumOccurrences() > 1)
 851       report_fatal_error("You can't specify more than one -thinlto-action");
 852     thinlto::ThinLTOProcessing ThinLTOProcessor(Options);
 853     ThinLTOProcessor.run();
 854     return 0;
 855   }
 856
 857   if (ThinLTO) {
 858     createCombinedModuleSummaryIndex();
 859     return 0;
 860   }
 861
 862   unsigned BaseArg = 0;
 863
 864   LLVMContext Context;
 865   Context.setDiagnosticHandler(llvm::make_unique<LLVMLTODiagnosticHandler>(),
 866                                true);
 867
 868   LTOCodeGenerator CodeGen(Context);
 869
 870   if (UseDiagnosticHandler)
 871     CodeGen.setDiagnosticHandler(handleDiagnostics, nullptr);
 872
 873   CodeGen.setCodePICModel(getRelocModel());
 874   CodeGen.setFreestanding(EnableFreestanding);
 875
 876   CodeGen.setDebugInfo(LTO_DEBUG_MODEL_DWARF);
 877   CodeGen.setTargetOptions(Options);
 878   CodeGen.setShouldRestoreGlobalsLinkage(RestoreGlobalsLinkage);
 879
 880   StringSet<MallocAllocator> DSOSymbolsSet;
 881   for (unsigned i = 0; i < DSOSymbols.size(); ++i)
 882     DSOSymbolsSet.insert(DSOSymbols[i]);
 883
 884   std::vector<std::string> KeptDSOSyms;
 885
 886   for (unsigned i = BaseArg; i < InputFilenames.size(); ++i) {
 887     CurrentActivity = "loading file '" + InputFilenames[i] + "'";
 888     ErrorOr<std::unique_ptr<LTOModule>> ModuleOrErr =
 889         LTOModule::createFromFile(Context, InputFilenames[i], Options);
 890     std::unique_ptr<LTOModule> &Module = *ModuleOrErr;
 891     CurrentActivity = "";
 892
 893     unsigned NumSyms = Module->getSymbolCount();
 894     for (unsigned I = 0; I < NumSyms; ++I) {
 895       StringRef Name = Module->getSymbolName(I);
 896       if (!DSOSymbolsSet.count(Name))
 897         continue;
 898       lto_symbol_attributes Attrs = Module->getSymbolAttributes(I);
 899       unsigned Scope = Attrs & LTO_SYMBOL_SCOPE_MASK;
 900       if (Scope != LTO_SYMBOL_SCOPE_DEFAULT_CAN_BE_HIDDEN)
 901         KeptDSOSyms.push_back(Name);
 902     }
 903
 904     // We use the first input module as the destination module when
 905     // SetMergedModule is true.
 906     if (SetMergedModule && i == BaseArg) {
 907       // Transfer ownership to the code generator.
 908       CodeGen.setModule(std::move(Module));
 909     } else if (!CodeGen.addModule(Module.get())) {
 910       // Print a message here so that we know addModule() did not abort.
 911       error("error adding file '" + InputFilenames[i] + "'");
 912     }
 913   }
 914
 915   // Add all the exported symbols to the table of symbols to preserve.
 916   for (unsigned i = 0; i < ExportedSymbols.size(); ++i)
 917     CodeGen.addMustPreserveSymbol(ExportedSymbols[i]);
 918
 919   // Add all the dso symbols to the table of symbols to expose.
 920   for (unsigned i = 0; i < KeptDSOSyms.size(); ++i)
 921     CodeGen.addMustPreserveSymbol(KeptDSOSyms[i]);
 922
 923   // Set cpu and attrs strings for the default target/subtarget.
 924   CodeGen.setCpu(MCPU.c_str());
 925
 926   CodeGen.setOptLevel(OptLevel - '0');
 927
 928   std::string attrs;
 929   for (unsigned i = 0; i < MAttrs.size(); ++i) {
 930     if (i > 0)
 931       attrs.append(",");
 932     attrs.append(MAttrs[i]);
 933   }
 934
 935   if (!attrs.empty())
 936     CodeGen.setAttr(attrs);
 937
 938   if (FileType.getNumOccurrences())
 939     CodeGen.setFileType(FileType);
 940
 941   if (!OutputFilename.empty()) {
 942     if (!CodeGen.optimize(DisableVerify, DisableInline, DisableGVNLoadPRE,
 943                           DisableLTOVectorization)) {
 944       // Diagnostic messages should have been printed by the handler.
 945       error("error optimizing the code");
 946     }
 947
 948     if (SaveModuleFile) {
 949       std::string ModuleFilename = OutputFilename;
 950       ModuleFilename += ".merged.bc";
 951       std::string ErrMsg;
 952
 953       if (!CodeGen.writeMergedModules(ModuleFilename))
 954         error("writing merged module failed.");
 955     }
 956
 957     std::list<ToolOutputFile> OSs;
 958     std::vector<raw_pwrite_stream *> OSPtrs;
 959     for (unsigned I = 0; I != Parallelism; ++I) {
 960       std::string PartFilename = OutputFilename;
 961       if (Parallelism != 1)
 962         PartFilename += "." + utostr(I);
 963       std::error_code EC;
 964       OSs.emplace_back(PartFilename, EC, sys::fs::F_None);
 965       if (EC)
 966         error("error opening the file '" + PartFilename + "': " + EC.message());
 967       OSPtrs.push_back(&OSs.back().os());
 968     }
 969
 970     if (!CodeGen.compileOptimized(OSPtrs))
 971       // Diagnostic messages should have been printed by the handler.
 972       error("error compiling the code");
 973
 974     for (ToolOutputFile &OS : OSs)
 975       OS.keep();
 976   } else {
 977     if (Parallelism != 1)
 978       error("-j must be specified together with -o");
 979
 980     if (SaveModuleFile)
 981       error(": -save-merged-module must be specified with -o");
 982
 983     const char *OutputName = nullptr;
 984     if (!CodeGen.compile_to_file(&OutputName, DisableVerify, DisableInline,
 985                                  DisableGVNLoadPRE, DisableLTOVectorization))
 986       error("error compiling the code");
 987       // Diagnostic messages should have been printed by the handler.
 988
 989     outs() << "Wrote native object file '" << OutputName << "'\n";
 990   }
 991
 992   return 0;
 993 }