1 //===- llvm-lto: a simple command-line program to link modules with LTO ---===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // This program takes in a list of bitcode files, links them, performs link-time
10 // optimization, and outputs an object file.
12 //===----------------------------------------------------------------------===//
14 #include "llvm-c/lto.h"
15 #include "llvm/ADT/ArrayRef.h"
16 #include "llvm/ADT/STLExtras.h"
17 #include "llvm/ADT/SmallString.h"
18 #include "llvm/ADT/StringExtras.h"
19 #include "llvm/ADT/StringRef.h"
20 #include "llvm/ADT/StringSet.h"
21 #include "llvm/ADT/Twine.h"
22 #include "llvm/Bitcode/BitcodeReader.h"
23 #include "llvm/Bitcode/BitcodeWriter.h"
24 #include "llvm/CodeGen/CommandFlags.h"
25 #include "llvm/IR/DiagnosticInfo.h"
26 #include "llvm/IR/DiagnosticPrinter.h"
27 #include "llvm/IR/LLVMContext.h"
28 #include "llvm/IR/Module.h"
29 #include "llvm/IR/ModuleSummaryIndex.h"
30 #include "llvm/IR/Verifier.h"
31 #include "llvm/IRReader/IRReader.h"
32 #include "llvm/LTO/legacy/LTOCodeGenerator.h"
33 #include "llvm/LTO/legacy/LTOModule.h"
34 #include "llvm/LTO/legacy/ThinLTOCodeGenerator.h"
35 #include "llvm/Support/Allocator.h"
36 #include "llvm/Support/Casting.h"
37 #include "llvm/Support/CommandLine.h"
38 #include "llvm/Support/Error.h"
39 #include "llvm/Support/ErrorHandling.h"
40 #include "llvm/Support/ErrorOr.h"
41 #include "llvm/Support/FileSystem.h"
42 #include "llvm/Support/InitLLVM.h"
43 #include "llvm/Support/MemoryBuffer.h"
44 #include "llvm/Support/Path.h"
45 #include "llvm/Support/SourceMgr.h"
46 #include "llvm/Support/TargetSelect.h"
47 #include "llvm/Support/ToolOutputFile.h"
48 #include "llvm/Support/raw_ostream.h"
49 #include "llvm/Target/TargetOptions.h"
58 #include <system_error>
65 static codegen::RegisterCodeGenFlags CGF
;
67 static cl::OptionCategory
LTOCategory("LTO Options");
71 cl::desc("Optimization level. [-O0, -O1, -O2, or -O3] "
73 cl::Prefix
, cl::ZeroOrMore
, cl::init('2'), cl::cat(LTOCategory
));
76 IndexStats("thinlto-index-stats",
77 cl::desc("Print statistic for the index in every input files"),
78 cl::init(false), cl::cat(LTOCategory
));
80 static cl::opt
<bool> DisableVerify(
81 "disable-verify", cl::init(false),
82 cl::desc("Do not run the verifier during the optimization pipeline"),
83 cl::cat(LTOCategory
));
85 static cl::opt
<bool> EnableFreestanding(
86 "lto-freestanding", cl::init(false),
87 cl::desc("Enable Freestanding (disable builtins / TLI) during LTO"),
88 cl::cat(LTOCategory
));
90 static cl::opt
<bool> UseDiagnosticHandler(
91 "use-diagnostic-handler", cl::init(false),
92 cl::desc("Use a diagnostic handler to test the handler interface"),
93 cl::cat(LTOCategory
));
96 ThinLTO("thinlto", cl::init(false),
97 cl::desc("Only write combined global index for ThinLTO backends"),
98 cl::cat(LTOCategory
));
112 cl::opt
<ThinLTOModes
> ThinLTOMode(
113 "thinlto-action", cl::desc("Perform a single ThinLTO stage:"),
116 THINLINK
, "thinlink",
117 "ThinLink: produces the index by linking only the summaries."),
118 clEnumValN(THINDISTRIBUTE
, "distributedindexes",
119 "Produces individual indexes for distributed backends."),
120 clEnumValN(THINEMITIMPORTS
, "emitimports",
121 "Emit imports files for distributed backends."),
122 clEnumValN(THINPROMOTE
, "promote",
123 "Perform pre-import promotion (requires -thinlto-index)."),
124 clEnumValN(THINIMPORT
, "import",
125 "Perform both promotion and "
126 "cross-module importing (requires "
128 clEnumValN(THININTERNALIZE
, "internalize",
129 "Perform internalization driven by -exported-symbol "
130 "(requires -thinlto-index)."),
131 clEnumValN(THINOPT
, "optimize", "Perform ThinLTO optimizations."),
132 clEnumValN(THINCODEGEN
, "codegen", "CodeGen (expected to match llc)"),
133 clEnumValN(THINALL
, "run", "Perform ThinLTO end-to-end")),
134 cl::cat(LTOCategory
));
136 static cl::opt
<std::string
>
137 ThinLTOIndex("thinlto-index",
138 cl::desc("Provide the index produced by a ThinLink, required "
139 "to perform the promotion and/or importing."),
140 cl::cat(LTOCategory
));
142 static cl::opt
<std::string
> ThinLTOPrefixReplace(
143 "thinlto-prefix-replace",
144 cl::desc("Control where files for distributed backends are "
145 "created. Expects 'oldprefix;newprefix' and if path "
146 "prefix of output file is oldprefix it will be "
147 "replaced with newprefix."),
148 cl::cat(LTOCategory
));
150 static cl::opt
<std::string
> ThinLTOModuleId(
152 cl::desc("For the module ID for the file to process, useful to "
153 "match what is in the index."),
154 cl::cat(LTOCategory
));
156 static cl::opt
<std::string
> ThinLTOCacheDir("thinlto-cache-dir",
157 cl::desc("Enable ThinLTO caching."),
158 cl::cat(LTOCategory
));
160 static cl::opt
<int> ThinLTOCachePruningInterval(
161 "thinlto-cache-pruning-interval", cl::init(1200),
162 cl::desc("Set ThinLTO cache pruning interval."), cl::cat(LTOCategory
));
164 static cl::opt
<uint64_t> ThinLTOCacheMaxSizeBytes(
165 "thinlto-cache-max-size-bytes",
166 cl::desc("Set ThinLTO cache pruning directory maximum size in bytes."),
167 cl::cat(LTOCategory
));
169 static cl::opt
<int> ThinLTOCacheMaxSizeFiles(
170 "thinlto-cache-max-size-files", cl::init(1000000),
171 cl::desc("Set ThinLTO cache pruning directory maximum number of files."),
172 cl::cat(LTOCategory
));
174 static cl::opt
<unsigned> ThinLTOCacheEntryExpiration(
175 "thinlto-cache-entry-expiration", cl::init(604800) /* 1w */,
176 cl::desc("Set ThinLTO cache entry expiration time."), cl::cat(LTOCategory
));
178 static cl::opt
<std::string
> ThinLTOSaveTempsPrefix(
179 "thinlto-save-temps",
180 cl::desc("Save ThinLTO temp files using filenames created by adding "
181 "suffixes to the given file path prefix."),
182 cl::cat(LTOCategory
));
184 static cl::opt
<std::string
> ThinLTOGeneratedObjectsDir(
185 "thinlto-save-objects",
186 cl::desc("Save ThinLTO generated object files using filenames created in "
187 "the given directory."),
188 cl::cat(LTOCategory
));
190 static cl::opt
<bool> SaveLinkedModuleFile(
191 "save-linked-module", cl::init(false),
192 cl::desc("Write linked LTO module to file before optimize"),
193 cl::cat(LTOCategory
));
196 SaveModuleFile("save-merged-module", cl::init(false),
197 cl::desc("Write merged LTO module to file before CodeGen"),
198 cl::cat(LTOCategory
));
200 static cl::list
<std::string
> InputFilenames(cl::Positional
, cl::OneOrMore
,
201 cl::desc("<input bitcode files>"),
202 cl::cat(LTOCategory
));
204 static cl::opt
<std::string
> OutputFilename("o", cl::init(""),
205 cl::desc("Override output filename"),
206 cl::value_desc("filename"),
207 cl::cat(LTOCategory
));
209 static cl::list
<std::string
> ExportedSymbols(
211 cl::desc("List of symbols to export from the resulting object file"),
212 cl::ZeroOrMore
, cl::cat(LTOCategory
));
214 static cl::list
<std::string
>
215 DSOSymbols("dso-symbol",
216 cl::desc("Symbol to put in the symtab in the resulting dso"),
217 cl::ZeroOrMore
, cl::cat(LTOCategory
));
219 static cl::opt
<bool> ListSymbolsOnly(
220 "list-symbols-only", cl::init(false),
221 cl::desc("Instead of running LTO, list the symbols in each IR file"),
222 cl::cat(LTOCategory
));
224 static cl::opt
<bool> ListDependentLibrariesOnly(
225 "list-dependent-libraries-only", cl::init(false),
227 "Instead of running LTO, list the dependent libraries in each IR file"),
228 cl::cat(LTOCategory
));
230 static cl::opt
<bool> QueryHasCtorDtor(
231 "query-hasCtorDtor", cl::init(false),
232 cl::desc("Queries LTOModule::hasCtorDtor() on each IR file"));
235 SetMergedModule("set-merged-module", cl::init(false),
236 cl::desc("Use the first input module as the merged module"),
237 cl::cat(LTOCategory
));
239 static cl::opt
<unsigned> Parallelism("j", cl::Prefix
, cl::init(1),
240 cl::desc("Number of backend threads"),
241 cl::cat(LTOCategory
));
243 static cl::opt
<bool> RestoreGlobalsLinkage(
244 "restore-linkage", cl::init(false),
245 cl::desc("Restore original linkage of globals prior to CodeGen"),
246 cl::cat(LTOCategory
));
248 static cl::opt
<bool> CheckHasObjC(
249 "check-for-objc", cl::init(false),
250 cl::desc("Only check if the module has objective-C defined in it"),
251 cl::cat(LTOCategory
));
253 static cl::opt
<bool> PrintMachOCPUOnly(
254 "print-macho-cpu-only", cl::init(false),
255 cl::desc("Instead of running LTO, print the mach-o cpu in each IR file"),
256 cl::cat(LTOCategory
));
258 static cl::opt
<bool> UseNewPM(
259 "use-new-pm", cl::desc("Run LTO passes using the new pass manager"),
260 cl::init(LLVM_ENABLE_NEW_PASS_MANAGER
), cl::Hidden
, cl::cat(LTOCategory
));
263 DebugPassManager("debug-pass-manager", cl::init(false), cl::Hidden
,
264 cl::desc("Print pass management debugging information"),
265 cl::cat(LTOCategory
));
270 BitVector CanBeHidden
;
273 } // end anonymous namespace
275 static void handleDiagnostics(lto_codegen_diagnostic_severity_t Severity
,
276 const char *Msg
, void *) {
277 errs() << "llvm-lto: ";
283 errs() << "remark: ";
289 errs() << "warning: ";
292 errs() << Msg
<< "\n";
295 static std::string CurrentActivity
;
298 struct LLVMLTODiagnosticHandler
: public DiagnosticHandler
{
299 bool handleDiagnostics(const DiagnosticInfo
&DI
) override
{
300 raw_ostream
&OS
= errs();
302 switch (DI
.getSeverity()) {
316 if (!CurrentActivity
.empty())
317 OS
<< ' ' << CurrentActivity
;
320 DiagnosticPrinterRawOStream
DP(OS
);
324 if (DI
.getSeverity() == DS_Error
)
331 static void error(const Twine
&Msg
) {
332 errs() << "llvm-lto: " << Msg
<< '\n';
336 static void error(std::error_code EC
, const Twine
&Prefix
) {
338 error(Prefix
+ ": " + EC
.message());
341 template <typename T
>
342 static void error(const ErrorOr
<T
> &V
, const Twine
&Prefix
) {
343 error(V
.getError(), Prefix
);
346 static void maybeVerifyModule(const Module
&Mod
) {
347 if (!DisableVerify
&& verifyModule(Mod
, &errs()))
348 error("Broken Module");
351 static std::unique_ptr
<LTOModule
>
352 getLocalLTOModule(StringRef Path
, std::unique_ptr
<MemoryBuffer
> &Buffer
,
353 const TargetOptions
&Options
) {
354 ErrorOr
<std::unique_ptr
<MemoryBuffer
>> BufferOrErr
=
355 MemoryBuffer::getFile(Path
);
356 error(BufferOrErr
, "error loading file '" + Path
+ "'");
357 Buffer
= std::move(BufferOrErr
.get());
358 CurrentActivity
= ("loading file '" + Path
+ "'").str();
359 std::unique_ptr
<LLVMContext
> Context
= std::make_unique
<LLVMContext
>();
360 Context
->setDiagnosticHandler(std::make_unique
<LLVMLTODiagnosticHandler
>(),
362 ErrorOr
<std::unique_ptr
<LTOModule
>> Ret
= LTOModule::createInLocalContext(
363 std::move(Context
), Buffer
->getBufferStart(), Buffer
->getBufferSize(),
365 CurrentActivity
= "";
366 maybeVerifyModule((*Ret
)->getModule());
367 return std::move(*Ret
);
370 /// Print some statistics on the index for each input files.
371 static void printIndexStats() {
372 for (auto &Filename
: InputFilenames
) {
373 ExitOnError
ExitOnErr("llvm-lto: error loading file '" + Filename
+ "': ");
374 std::unique_ptr
<ModuleSummaryIndex
> Index
=
375 ExitOnErr(getModuleSummaryIndexForFile(Filename
));
376 // Skip files without a module summary.
378 report_fatal_error(Twine(Filename
) + " does not contain an index");
380 unsigned Calls
= 0, Refs
= 0, Functions
= 0, Alias
= 0, Globals
= 0;
381 for (auto &Summaries
: *Index
) {
382 for (auto &Summary
: Summaries
.second
.SummaryList
) {
383 Refs
+= Summary
->refs().size();
384 if (auto *FuncSummary
= dyn_cast
<FunctionSummary
>(Summary
.get())) {
386 Calls
+= FuncSummary
->calls().size();
387 } else if (isa
<AliasSummary
>(Summary
.get()))
393 outs() << "Index " << Filename
<< " contains "
394 << (Alias
+ Globals
+ Functions
) << " nodes (" << Functions
395 << " functions, " << Alias
<< " alias, " << Globals
396 << " globals) and " << (Calls
+ Refs
) << " edges (" << Refs
397 << " refs and " << Calls
<< " calls)\n";
401 /// Load each IR file and dump certain information based on active flags.
403 /// The main point here is to provide lit-testable coverage for the LTOModule
404 /// functionality that's exposed by the C API. Moreover, this provides testing
405 /// coverage for modules that have been created in their own contexts.
406 static void testLTOModule(const TargetOptions
&Options
) {
407 for (auto &Filename
: InputFilenames
) {
408 std::unique_ptr
<MemoryBuffer
> Buffer
;
409 std::unique_ptr
<LTOModule
> Module
=
410 getLocalLTOModule(Filename
, Buffer
, Options
);
412 if (ListSymbolsOnly
) {
414 outs() << Filename
<< ":\n";
415 for (int I
= 0, E
= Module
->getSymbolCount(); I
!= E
; ++I
)
416 outs() << Module
->getSymbolName(I
) << "\n";
418 if (QueryHasCtorDtor
)
420 << ": hasCtorDtor = " << (Module
->hasCtorDtor() ? "true" : "false")
425 static std::unique_ptr
<MemoryBuffer
> loadFile(StringRef Filename
) {
426 ExitOnError
ExitOnErr("llvm-lto: error loading file '" + Filename
.str() +
428 return ExitOnErr(errorOrToExpected(MemoryBuffer::getFileOrSTDIN(Filename
)));
431 static void listDependentLibraries() {
432 for (auto &Filename
: InputFilenames
) {
433 auto Buffer
= loadFile(Filename
);
435 std::unique_ptr
<lto::InputFile
> Input(LTOModule::createInputFile(
436 Buffer
->getBufferStart(), Buffer
->getBufferSize(), Filename
.c_str(),
441 // List the dependent libraries.
442 outs() << Filename
<< ":\n";
443 for (size_t I
= 0, C
= LTOModule::getDependentLibraryCount(Input
.get());
446 const char *S
= LTOModule::getDependentLibrary(Input
.get(), I
, &L
);
448 outs() << StringRef(S
, L
) << "\n";
453 static void printMachOCPUOnly() {
455 Context
.setDiagnosticHandler(std::make_unique
<LLVMLTODiagnosticHandler
>(),
457 TargetOptions Options
= codegen::InitTargetOptionsFromCodeGenFlags(Triple());
458 for (auto &Filename
: InputFilenames
) {
459 ErrorOr
<std::unique_ptr
<LTOModule
>> ModuleOrErr
=
460 LTOModule::createFromFile(Context
, Filename
, Options
);
462 error(ModuleOrErr
, "llvm-lto: ");
464 Expected
<uint32_t> CPUType
= (*ModuleOrErr
)->getMachOCPUType();
465 Expected
<uint32_t> CPUSubType
= (*ModuleOrErr
)->getMachOCPUSubType();
467 error("Error while printing mach-o cputype: " +
468 toString(CPUType
.takeError()));
470 error("Error while printing mach-o cpusubtype: " +
471 toString(CPUSubType
.takeError()));
472 outs() << llvm::format("%s:\ncputype: %u\ncpusubtype: %u\n",
473 Filename
.c_str(), *CPUType
, *CPUSubType
);
477 /// Create a combined index file from the input IR files and write it.
479 /// This is meant to enable testing of ThinLTO combined index generation,
480 /// currently available via the gold plugin via -thinlto.
481 static void createCombinedModuleSummaryIndex() {
482 ModuleSummaryIndex
CombinedIndex(/*HaveGVs=*/false);
483 uint64_t NextModuleId
= 0;
484 for (auto &Filename
: InputFilenames
) {
485 ExitOnError
ExitOnErr("llvm-lto: error loading file '" + Filename
+ "': ");
486 std::unique_ptr
<MemoryBuffer
> MB
=
487 ExitOnErr(errorOrToExpected(MemoryBuffer::getFileOrSTDIN(Filename
)));
488 ExitOnErr(readModuleSummaryIndex(*MB
, CombinedIndex
, NextModuleId
++));
490 // In order to use this index for testing, specifically import testing, we
491 // need to update any indirect call edges created from SamplePGO, so that they
492 // point to the correct GUIDs.
493 updateIndirectCalls(CombinedIndex
);
495 assert(!OutputFilename
.empty());
496 raw_fd_ostream
OS(OutputFilename
+ ".thinlto.bc", EC
,
497 sys::fs::OpenFlags::OF_None
);
498 error(EC
, "error opening the file '" + OutputFilename
+ ".thinlto.bc'");
499 WriteIndexToFile(CombinedIndex
, OS
);
503 /// Parse the thinlto_prefix_replace option into the \p OldPrefix and
504 /// \p NewPrefix strings, if it was specified.
505 static void getThinLTOOldAndNewPrefix(std::string
&OldPrefix
,
506 std::string
&NewPrefix
) {
507 assert(ThinLTOPrefixReplace
.empty() ||
508 ThinLTOPrefixReplace
.find(';') != StringRef::npos
);
509 StringRef PrefixReplace
= ThinLTOPrefixReplace
;
510 std::pair
<StringRef
, StringRef
> Split
= PrefixReplace
.split(";");
511 OldPrefix
= Split
.first
.str();
512 NewPrefix
= Split
.second
.str();
515 /// Given the original \p Path to an output file, replace any path
516 /// prefix matching \p OldPrefix with \p NewPrefix. Also, create the
517 /// resulting directory if it does not yet exist.
518 static std::string
getThinLTOOutputFile(const std::string
&Path
,
519 const std::string
&OldPrefix
,
520 const std::string
&NewPrefix
) {
521 if (OldPrefix
.empty() && NewPrefix
.empty())
523 SmallString
<128> NewPath(Path
);
524 llvm::sys::path::replace_path_prefix(NewPath
, OldPrefix
, NewPrefix
);
525 StringRef ParentPath
= llvm::sys::path::parent_path(NewPath
.str());
526 if (!ParentPath
.empty()) {
527 // Make sure the new directory exists, creating it if necessary.
528 if (std::error_code EC
= llvm::sys::fs::create_directories(ParentPath
))
529 error(EC
, "error creating the directory '" + ParentPath
+ "'");
531 return std::string(NewPath
.str());
536 std::vector
<std::unique_ptr
<MemoryBuffer
>>
537 loadAllFilesForIndex(const ModuleSummaryIndex
&Index
) {
538 std::vector
<std::unique_ptr
<MemoryBuffer
>> InputBuffers
;
540 for (auto &ModPath
: Index
.modulePaths()) {
541 const auto &Filename
= ModPath
.first();
542 std::string CurrentActivity
= ("loading file '" + Filename
+ "'").str();
543 auto InputOrErr
= MemoryBuffer::getFile(Filename
);
544 error(InputOrErr
, "error " + CurrentActivity
);
545 InputBuffers
.push_back(std::move(*InputOrErr
));
550 std::unique_ptr
<ModuleSummaryIndex
> loadCombinedIndex() {
551 if (ThinLTOIndex
.empty())
552 report_fatal_error("Missing -thinlto-index for ThinLTO promotion stage");
553 ExitOnError
ExitOnErr("llvm-lto: error loading file '" + ThinLTOIndex
+
555 return ExitOnErr(getModuleSummaryIndexForFile(ThinLTOIndex
));
558 static std::unique_ptr
<lto::InputFile
> loadInputFile(MemoryBufferRef Buffer
) {
559 ExitOnError
ExitOnErr("llvm-lto: error loading input '" +
560 Buffer
.getBufferIdentifier().str() + "': ");
561 return ExitOnErr(lto::InputFile::create(Buffer
));
564 static std::unique_ptr
<Module
> loadModuleFromInput(lto::InputFile
&File
,
566 auto &Mod
= File
.getSingleBitcodeModule();
567 auto ModuleOrErr
= Mod
.parseModule(CTX
);
569 handleAllErrors(ModuleOrErr
.takeError(), [&](ErrorInfoBase
&EIB
) {
570 SMDiagnostic Err
= SMDiagnostic(Mod
.getModuleIdentifier(),
571 SourceMgr::DK_Error
, EIB
.message());
572 Err
.print("llvm-lto", errs());
574 report_fatal_error("Can't load module, abort.");
576 maybeVerifyModule(**ModuleOrErr
);
577 if (ThinLTOModuleId
.getNumOccurrences()) {
578 if (InputFilenames
.size() != 1)
579 report_fatal_error("Can't override the module id for multiple files");
580 (*ModuleOrErr
)->setModuleIdentifier(ThinLTOModuleId
);
582 return std::move(*ModuleOrErr
);
585 static void writeModuleToFile(Module
&TheModule
, StringRef Filename
) {
587 raw_fd_ostream
OS(Filename
, EC
, sys::fs::OpenFlags::OF_None
);
588 error(EC
, "error opening the file '" + Filename
+ "'");
589 maybeVerifyModule(TheModule
);
590 WriteBitcodeToFile(TheModule
, OS
, /* ShouldPreserveUseListOrder */ true);
593 class ThinLTOProcessing
{
595 ThinLTOCodeGenerator ThinGenerator
;
597 ThinLTOProcessing(const TargetOptions
&Options
) {
598 ThinGenerator
.setCodePICModel(codegen::getExplicitRelocModel());
599 ThinGenerator
.setTargetOptions(Options
);
600 ThinGenerator
.setCacheDir(ThinLTOCacheDir
);
601 ThinGenerator
.setCachePruningInterval(ThinLTOCachePruningInterval
);
602 ThinGenerator
.setCacheEntryExpiration(ThinLTOCacheEntryExpiration
);
603 ThinGenerator
.setCacheMaxSizeFiles(ThinLTOCacheMaxSizeFiles
);
604 ThinGenerator
.setCacheMaxSizeBytes(ThinLTOCacheMaxSizeBytes
);
605 ThinGenerator
.setFreestanding(EnableFreestanding
);
606 ThinGenerator
.setUseNewPM(UseNewPM
);
607 ThinGenerator
.setDebugPassManager(DebugPassManager
);
609 // Add all the exported symbols to the table of symbols to preserve.
610 for (unsigned i
= 0; i
< ExportedSymbols
.size(); ++i
)
611 ThinGenerator
.preserveSymbol(ExportedSymbols
[i
]);
615 switch (ThinLTOMode
) {
619 return distributedIndexes();
620 case THINEMITIMPORTS
:
621 return emitImports();
626 case THININTERNALIZE
:
627 return internalize();
638 /// Load the input files, create the combined index, and write it out.
640 // Perform "ThinLink": just produce the index
641 if (OutputFilename
.empty())
643 "OutputFilename is necessary to store the combined index.\n");
646 std::vector
<std::unique_ptr
<MemoryBuffer
>> InputBuffers
;
647 for (unsigned i
= 0; i
< InputFilenames
.size(); ++i
) {
648 auto &Filename
= InputFilenames
[i
];
649 std::string CurrentActivity
= "loading file '" + Filename
+ "'";
650 auto InputOrErr
= MemoryBuffer::getFile(Filename
);
651 error(InputOrErr
, "error " + CurrentActivity
);
652 InputBuffers
.push_back(std::move(*InputOrErr
));
653 ThinGenerator
.addModule(Filename
, InputBuffers
.back()->getBuffer());
656 auto CombinedIndex
= ThinGenerator
.linkCombinedIndex();
658 report_fatal_error("ThinLink didn't create an index");
660 raw_fd_ostream
OS(OutputFilename
, EC
, sys::fs::OpenFlags::OF_None
);
661 error(EC
, "error opening the file '" + OutputFilename
+ "'");
662 WriteIndexToFile(*CombinedIndex
, OS
);
665 /// Load the combined index from disk, then compute and generate
666 /// individual index files suitable for ThinLTO distributed backend builds
667 /// on the files mentioned on the command line (these must match the index
669 void distributedIndexes() {
670 if (InputFilenames
.size() != 1 && !OutputFilename
.empty())
671 report_fatal_error("Can't handle a single output filename and multiple "
672 "input files, do not provide an output filename and "
673 "the output files will be suffixed from the input "
676 std::string OldPrefix
, NewPrefix
;
677 getThinLTOOldAndNewPrefix(OldPrefix
, NewPrefix
);
679 auto Index
= loadCombinedIndex();
680 for (auto &Filename
: InputFilenames
) {
682 auto Buffer
= loadFile(Filename
);
683 auto Input
= loadInputFile(Buffer
->getMemBufferRef());
684 auto TheModule
= loadModuleFromInput(*Input
, Ctx
);
686 // Build a map of module to the GUIDs and summary objects that should
687 // be written to its index.
688 std::map
<std::string
, GVSummaryMapTy
> ModuleToSummariesForIndex
;
689 ThinGenerator
.gatherImportedSummariesForModule(
690 *TheModule
, *Index
, ModuleToSummariesForIndex
, *Input
);
692 std::string OutputName
= OutputFilename
;
693 if (OutputName
.empty()) {
694 OutputName
= Filename
+ ".thinlto.bc";
696 OutputName
= getThinLTOOutputFile(OutputName
, OldPrefix
, NewPrefix
);
698 raw_fd_ostream
OS(OutputName
, EC
, sys::fs::OpenFlags::OF_None
);
699 error(EC
, "error opening the file '" + OutputName
+ "'");
700 WriteIndexToFile(*Index
, OS
, &ModuleToSummariesForIndex
);
704 /// Load the combined index from disk, compute the imports, and emit
705 /// the import file lists for each module to disk.
707 if (InputFilenames
.size() != 1 && !OutputFilename
.empty())
708 report_fatal_error("Can't handle a single output filename and multiple "
709 "input files, do not provide an output filename and "
710 "the output files will be suffixed from the input "
713 std::string OldPrefix
, NewPrefix
;
714 getThinLTOOldAndNewPrefix(OldPrefix
, NewPrefix
);
716 auto Index
= loadCombinedIndex();
717 for (auto &Filename
: InputFilenames
) {
719 auto Buffer
= loadFile(Filename
);
720 auto Input
= loadInputFile(Buffer
->getMemBufferRef());
721 auto TheModule
= loadModuleFromInput(*Input
, Ctx
);
722 std::string OutputName
= OutputFilename
;
723 if (OutputName
.empty()) {
724 OutputName
= Filename
+ ".imports";
727 getThinLTOOutputFile(OutputName
, OldPrefix
, NewPrefix
);
728 ThinGenerator
.emitImports(*TheModule
, OutputName
, *Index
, *Input
);
732 /// Load the combined index from disk, then load every file referenced by
733 /// the index and add them to the generator, finally perform the promotion
734 /// on the files mentioned on the command line (these must match the index
737 if (InputFilenames
.size() != 1 && !OutputFilename
.empty())
738 report_fatal_error("Can't handle a single output filename and multiple "
739 "input files, do not provide an output filename and "
740 "the output files will be suffixed from the input "
743 auto Index
= loadCombinedIndex();
744 for (auto &Filename
: InputFilenames
) {
746 auto Buffer
= loadFile(Filename
);
747 auto Input
= loadInputFile(Buffer
->getMemBufferRef());
748 auto TheModule
= loadModuleFromInput(*Input
, Ctx
);
750 ThinGenerator
.promote(*TheModule
, *Index
, *Input
);
752 std::string OutputName
= OutputFilename
;
753 if (OutputName
.empty()) {
754 OutputName
= Filename
+ ".thinlto.promoted.bc";
756 writeModuleToFile(*TheModule
, OutputName
);
760 /// Load the combined index from disk, then load every file referenced by
761 /// the index and add them to the generator, then performs the promotion and
762 /// cross module importing on the files mentioned on the command line
763 /// (these must match the index content).
765 if (InputFilenames
.size() != 1 && !OutputFilename
.empty())
766 report_fatal_error("Can't handle a single output filename and multiple "
767 "input files, do not provide an output filename and "
768 "the output files will be suffixed from the input "
771 auto Index
= loadCombinedIndex();
772 auto InputBuffers
= loadAllFilesForIndex(*Index
);
773 for (auto &MemBuffer
: InputBuffers
)
774 ThinGenerator
.addModule(MemBuffer
->getBufferIdentifier(),
775 MemBuffer
->getBuffer());
777 for (auto &Filename
: InputFilenames
) {
779 auto Buffer
= loadFile(Filename
);
780 auto Input
= loadInputFile(Buffer
->getMemBufferRef());
781 auto TheModule
= loadModuleFromInput(*Input
, Ctx
);
783 ThinGenerator
.crossModuleImport(*TheModule
, *Index
, *Input
);
785 std::string OutputName
= OutputFilename
;
786 if (OutputName
.empty()) {
787 OutputName
= Filename
+ ".thinlto.imported.bc";
789 writeModuleToFile(*TheModule
, OutputName
);
794 if (InputFilenames
.size() != 1 && !OutputFilename
.empty())
795 report_fatal_error("Can't handle a single output filename and multiple "
796 "input files, do not provide an output filename and "
797 "the output files will be suffixed from the input "
800 if (ExportedSymbols
.empty())
801 errs() << "Warning: -internalize will not perform without "
802 "-exported-symbol\n";
804 auto Index
= loadCombinedIndex();
805 auto InputBuffers
= loadAllFilesForIndex(*Index
);
806 for (auto &MemBuffer
: InputBuffers
)
807 ThinGenerator
.addModule(MemBuffer
->getBufferIdentifier(),
808 MemBuffer
->getBuffer());
810 for (auto &Filename
: InputFilenames
) {
812 auto Buffer
= loadFile(Filename
);
813 auto Input
= loadInputFile(Buffer
->getMemBufferRef());
814 auto TheModule
= loadModuleFromInput(*Input
, Ctx
);
816 ThinGenerator
.internalize(*TheModule
, *Index
, *Input
);
818 std::string OutputName
= OutputFilename
;
819 if (OutputName
.empty()) {
820 OutputName
= Filename
+ ".thinlto.internalized.bc";
822 writeModuleToFile(*TheModule
, OutputName
);
827 if (InputFilenames
.size() != 1 && !OutputFilename
.empty())
828 report_fatal_error("Can't handle a single output filename and multiple "
829 "input files, do not provide an output filename and "
830 "the output files will be suffixed from the input "
832 if (!ThinLTOIndex
.empty())
833 errs() << "Warning: -thinlto-index ignored for optimize stage";
835 for (auto &Filename
: InputFilenames
) {
837 auto Buffer
= loadFile(Filename
);
838 auto Input
= loadInputFile(Buffer
->getMemBufferRef());
839 auto TheModule
= loadModuleFromInput(*Input
, Ctx
);
841 ThinGenerator
.optimize(*TheModule
);
843 std::string OutputName
= OutputFilename
;
844 if (OutputName
.empty()) {
845 OutputName
= Filename
+ ".thinlto.imported.bc";
847 writeModuleToFile(*TheModule
, OutputName
);
852 if (InputFilenames
.size() != 1 && !OutputFilename
.empty())
853 report_fatal_error("Can't handle a single output filename and multiple "
854 "input files, do not provide an output filename and "
855 "the output files will be suffixed from the input "
857 if (!ThinLTOIndex
.empty())
858 errs() << "Warning: -thinlto-index ignored for codegen stage";
860 std::vector
<std::unique_ptr
<MemoryBuffer
>> InputBuffers
;
861 for (auto &Filename
: InputFilenames
) {
863 auto InputOrErr
= MemoryBuffer::getFile(Filename
);
864 error(InputOrErr
, "error " + CurrentActivity
);
865 InputBuffers
.push_back(std::move(*InputOrErr
));
866 ThinGenerator
.addModule(Filename
, InputBuffers
.back()->getBuffer());
868 ThinGenerator
.setCodeGenOnly(true);
871 zip(ThinGenerator
.getProducedBinaries(), InputFilenames
)) {
872 std::string OutputName
= OutputFilename
;
873 if (OutputName
.empty())
874 OutputName
= std::get
<1>(BinName
) + ".thinlto.o";
875 else if (OutputName
== "-") {
876 outs() << std::get
<0>(BinName
)->getBuffer();
881 raw_fd_ostream
OS(OutputName
, EC
, sys::fs::OpenFlags::OF_None
);
882 error(EC
, "error opening the file '" + OutputName
+ "'");
883 OS
<< std::get
<0>(BinName
)->getBuffer();
887 /// Full ThinLTO process
889 if (!OutputFilename
.empty())
890 report_fatal_error("Do not provide an output filename for ThinLTO "
891 " processing, the output files will be suffixed from "
894 if (!ThinLTOIndex
.empty())
895 errs() << "Warning: -thinlto-index ignored for full ThinLTO process";
898 std::vector
<std::unique_ptr
<MemoryBuffer
>> InputBuffers
;
899 for (unsigned i
= 0; i
< InputFilenames
.size(); ++i
) {
900 auto &Filename
= InputFilenames
[i
];
901 std::string CurrentActivity
= "loading file '" + Filename
+ "'";
902 auto InputOrErr
= MemoryBuffer::getFile(Filename
);
903 error(InputOrErr
, "error " + CurrentActivity
);
904 InputBuffers
.push_back(std::move(*InputOrErr
));
905 ThinGenerator
.addModule(Filename
, InputBuffers
.back()->getBuffer());
908 if (!ThinLTOSaveTempsPrefix
.empty())
909 ThinGenerator
.setSaveTempsDir(ThinLTOSaveTempsPrefix
);
911 if (!ThinLTOGeneratedObjectsDir
.empty()) {
912 ThinGenerator
.setGeneratedObjectsDirectory(ThinLTOGeneratedObjectsDir
);
919 auto &Binaries
= ThinGenerator
.getProducedBinaries();
920 if (Binaries
.size() != InputFilenames
.size())
921 report_fatal_error("Number of output objects does not match the number "
924 for (unsigned BufID
= 0; BufID
< Binaries
.size(); ++BufID
) {
925 auto OutputName
= InputFilenames
[BufID
] + ".thinlto.o";
927 raw_fd_ostream
OS(OutputName
, EC
, sys::fs::OpenFlags::OF_None
);
928 error(EC
, "error opening the file '" + OutputName
+ "'");
929 OS
<< Binaries
[BufID
]->getBuffer();
933 /// Load the combined index from disk, then load every file referenced by
936 } // end namespace thinlto
938 int main(int argc
, char **argv
) {
939 InitLLVM
X(argc
, argv
);
940 cl::HideUnrelatedOptions({<OCategory
, &getColorCategory()});
941 cl::ParseCommandLineOptions(argc
, argv
, "llvm LTO linker\n");
943 if (OptLevel
< '0' || OptLevel
> '3')
944 error("optimization level must be between 0 and 3");
946 // Initialize the configured targets.
947 InitializeAllTargets();
948 InitializeAllTargetMCs();
949 InitializeAllAsmPrinters();
950 InitializeAllAsmParsers();
952 // set up the TargetOptions for the machine
953 TargetOptions Options
= codegen::InitTargetOptionsFromCodeGenFlags(Triple());
955 if (ListSymbolsOnly
|| QueryHasCtorDtor
) {
956 testLTOModule(Options
);
960 if (ListDependentLibrariesOnly
) {
961 listDependentLibraries();
971 for (auto &Filename
: InputFilenames
) {
972 ExitOnError
ExitOnErr(std::string(*argv
) + ": error loading file '" +
974 std::unique_ptr
<MemoryBuffer
> BufferOrErr
=
975 ExitOnErr(errorOrToExpected(MemoryBuffer::getFile(Filename
)));
976 auto Buffer
= std::move(BufferOrErr
.get());
977 if (ExitOnErr(isBitcodeContainingObjCCategory(*Buffer
)))
978 outs() << "Bitcode " << Filename
<< " contains ObjC\n";
980 outs() << "Bitcode " << Filename
<< " does not contain ObjC\n";
985 if (PrintMachOCPUOnly
) {
990 if (ThinLTOMode
.getNumOccurrences()) {
991 if (ThinLTOMode
.getNumOccurrences() > 1)
992 report_fatal_error("You can't specify more than one -thinlto-action");
993 thinlto::ThinLTOProcessing
ThinLTOProcessor(Options
);
994 ThinLTOProcessor
.run();
999 createCombinedModuleSummaryIndex();
1003 unsigned BaseArg
= 0;
1005 LLVMContext Context
;
1006 Context
.setDiagnosticHandler(std::make_unique
<LLVMLTODiagnosticHandler
>(),
1009 LTOCodeGenerator
CodeGen(Context
);
1010 CodeGen
.setDisableVerify(DisableVerify
);
1012 if (UseDiagnosticHandler
)
1013 CodeGen
.setDiagnosticHandler(handleDiagnostics
, nullptr);
1015 CodeGen
.setCodePICModel(codegen::getExplicitRelocModel());
1016 CodeGen
.setFreestanding(EnableFreestanding
);
1018 CodeGen
.setDebugInfo(LTO_DEBUG_MODEL_DWARF
);
1019 CodeGen
.setTargetOptions(Options
);
1020 CodeGen
.setShouldRestoreGlobalsLinkage(RestoreGlobalsLinkage
);
1022 StringSet
<MallocAllocator
> DSOSymbolsSet
;
1023 for (unsigned i
= 0; i
< DSOSymbols
.size(); ++i
)
1024 DSOSymbolsSet
.insert(DSOSymbols
[i
]);
1026 std::vector
<std::string
> KeptDSOSyms
;
1028 for (unsigned i
= BaseArg
; i
< InputFilenames
.size(); ++i
) {
1029 CurrentActivity
= "loading file '" + InputFilenames
[i
] + "'";
1030 ErrorOr
<std::unique_ptr
<LTOModule
>> ModuleOrErr
=
1031 LTOModule::createFromFile(Context
, InputFilenames
[i
], Options
);
1032 std::unique_ptr
<LTOModule
> &Module
= *ModuleOrErr
;
1033 CurrentActivity
= "";
1035 unsigned NumSyms
= Module
->getSymbolCount();
1036 for (unsigned I
= 0; I
< NumSyms
; ++I
) {
1037 StringRef Name
= Module
->getSymbolName(I
);
1038 if (!DSOSymbolsSet
.count(Name
))
1040 lto_symbol_attributes Attrs
= Module
->getSymbolAttributes(I
);
1041 unsigned Scope
= Attrs
& LTO_SYMBOL_SCOPE_MASK
;
1042 if (Scope
!= LTO_SYMBOL_SCOPE_DEFAULT_CAN_BE_HIDDEN
)
1043 KeptDSOSyms
.push_back(std::string(Name
));
1046 // We use the first input module as the destination module when
1047 // SetMergedModule is true.
1048 if (SetMergedModule
&& i
== BaseArg
) {
1049 // Transfer ownership to the code generator.
1050 CodeGen
.setModule(std::move(Module
));
1051 } else if (!CodeGen
.addModule(Module
.get())) {
1052 // Print a message here so that we know addModule() did not abort.
1053 error("error adding file '" + InputFilenames
[i
] + "'");
1057 // Add all the exported symbols to the table of symbols to preserve.
1058 for (unsigned i
= 0; i
< ExportedSymbols
.size(); ++i
)
1059 CodeGen
.addMustPreserveSymbol(ExportedSymbols
[i
]);
1061 // Add all the dso symbols to the table of symbols to expose.
1062 for (unsigned i
= 0; i
< KeptDSOSyms
.size(); ++i
)
1063 CodeGen
.addMustPreserveSymbol(KeptDSOSyms
[i
]);
1065 // Set cpu and attrs strings for the default target/subtarget.
1066 CodeGen
.setCpu(codegen::getMCPU());
1068 CodeGen
.setOptLevel(OptLevel
- '0');
1069 CodeGen
.setAttrs(codegen::getMAttrs());
1071 CodeGen
.setUseNewPM(UseNewPM
);
1073 if (auto FT
= codegen::getExplicitFileType())
1074 CodeGen
.setFileType(FT
.getValue());
1076 if (!OutputFilename
.empty()) {
1077 if (SaveLinkedModuleFile
) {
1078 std::string ModuleFilename
= OutputFilename
;
1079 ModuleFilename
+= ".linked.bc";
1082 if (!CodeGen
.writeMergedModules(ModuleFilename
))
1083 error("writing linked module failed.");
1086 if (!CodeGen
.optimize()) {
1087 // Diagnostic messages should have been printed by the handler.
1088 error("error optimizing the code");
1091 if (SaveModuleFile
) {
1092 std::string ModuleFilename
= OutputFilename
;
1093 ModuleFilename
+= ".merged.bc";
1096 if (!CodeGen
.writeMergedModules(ModuleFilename
))
1097 error("writing merged module failed.");
1100 auto AddStream
= [&](size_t Task
) -> std::unique_ptr
<CachedFileStream
> {
1101 std::string PartFilename
= OutputFilename
;
1102 if (Parallelism
!= 1)
1103 PartFilename
+= "." + utostr(Task
);
1107 std::make_unique
<raw_fd_ostream
>(PartFilename
, EC
, sys::fs::OF_None
);
1109 error("error opening the file '" + PartFilename
+ "': " + EC
.message());
1110 return std::make_unique
<CachedFileStream
>(std::move(S
));
1113 if (!CodeGen
.compileOptimized(AddStream
, Parallelism
))
1114 // Diagnostic messages should have been printed by the handler.
1115 error("error compiling the code");
1118 if (Parallelism
!= 1)
1119 error("-j must be specified together with -o");
1122 error(": -save-merged-module must be specified with -o");
1124 const char *OutputName
= nullptr;
1125 if (!CodeGen
.compile_to_file(&OutputName
))
1126 error("error compiling the code");
1127 // Diagnostic messages should have been printed by the handler.
1129 outs() << "Wrote native object file '" << OutputName
<< "'\n";