1 //===-- clang-offload-bundler/ClangOffloadBundler.cpp ---------------------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
10 /// This file implements a stand-alone clang-offload-bundler tool using the
11 /// OffloadBundler API.
13 //===----------------------------------------------------------------------===//
15 #include "clang/Basic/Cuda.h"
16 #include "clang/Basic/TargetID.h"
17 #include "clang/Basic/Version.h"
18 #include "clang/Driver/OffloadBundler.h"
19 #include "llvm/ADT/ArrayRef.h"
20 #include "llvm/ADT/SmallString.h"
21 #include "llvm/ADT/SmallVector.h"
22 #include "llvm/ADT/StringRef.h"
23 #include "llvm/Object/Archive.h"
24 #include "llvm/Object/ArchiveWriter.h"
25 #include "llvm/Object/Binary.h"
26 #include "llvm/Object/ObjectFile.h"
27 #include "llvm/Support/Casting.h"
28 #include "llvm/Support/CommandLine.h"
29 #include "llvm/Support/Debug.h"
30 #include "llvm/Support/Errc.h"
31 #include "llvm/Support/Error.h"
32 #include "llvm/Support/ErrorOr.h"
33 #include "llvm/Support/FileSystem.h"
34 #include "llvm/Support/MemoryBuffer.h"
35 #include "llvm/Support/Path.h"
36 #include "llvm/Support/Program.h"
37 #include "llvm/Support/Signals.h"
38 #include "llvm/Support/StringSaver.h"
39 #include "llvm/Support/WithColor.h"
40 #include "llvm/Support/raw_ostream.h"
41 #include "llvm/TargetParser/Host.h"
42 #include "llvm/TargetParser/Triple.h"
47 #include <forward_list>
52 #include <system_error>
56 using namespace llvm::object
;
57 using namespace clang
;
59 static void PrintVersion(raw_ostream
&OS
) {
60 OS
<< clang::getClangToolFullVersion("clang-offload-bundler") << '\n';
63 int main(int argc
, const char **argv
) {
65 cl::opt
<bool> Help("h", cl::desc("Alias for -help"), cl::Hidden
);
67 // Mark all our options with this category, everything else (except for
68 // -version and -help) will be hidden.
70 ClangOffloadBundlerCategory("clang-offload-bundler options");
72 InputFileNames("input",
73 cl::desc("Input file."
74 " Can be specified multiple times "
75 "for multiple input files."),
76 cl::cat(ClangOffloadBundlerCategory
));
78 InputFileNamesDeprecatedOpt("inputs", cl::CommaSeparated
,
79 cl::desc("[<input file>,...] (deprecated)"),
80 cl::cat(ClangOffloadBundlerCategory
));
82 OutputFileNames("output",
83 cl::desc("Output file."
84 " Can be specified multiple times "
85 "for multiple output files."),
86 cl::cat(ClangOffloadBundlerCategory
));
88 OutputFileNamesDeprecatedOpt("outputs", cl::CommaSeparated
,
89 cl::desc("[<output file>,...] (deprecated)"),
90 cl::cat(ClangOffloadBundlerCategory
));
92 TargetNames("targets", cl::CommaSeparated
,
93 cl::desc("[<offload kind>-<target triple>,...]"),
94 cl::cat(ClangOffloadBundlerCategory
));
95 cl::opt
<std::string
> FilesType(
97 cl::desc("Type of the files to be bundled/unbundled.\n"
98 "Current supported types are:\n"
100 " ii - c++-cpp-output\n"
101 " cui - cuda-cpp-output\n"
102 " hipi - hip-cpp-output\n"
108 " a - archive of objects\n"
109 " gch - precompiled-header\n"
110 " ast - clang AST file"),
111 cl::cat(ClangOffloadBundlerCategory
));
114 cl::desc("Unbundle bundled file into several output files.\n"),
115 cl::init(false), cl::cat(ClangOffloadBundlerCategory
));
117 ListBundleIDs("list", cl::desc("List bundle IDs in the bundled file.\n"),
118 cl::init(false), cl::cat(ClangOffloadBundlerCategory
));
119 cl::opt
<bool> PrintExternalCommands(
121 cl::desc("Print any external commands that are to be executed "
122 "instead of actually executing them - for testing purposes.\n"),
123 cl::init(false), cl::cat(ClangOffloadBundlerCategory
));
125 AllowMissingBundles("allow-missing-bundles",
126 cl::desc("Create empty files if bundles are missing "
127 "when unbundling.\n"),
128 cl::init(false), cl::cat(ClangOffloadBundlerCategory
));
130 BundleAlignment("bundle-align",
131 cl::desc("Alignment of bundle for binary files"),
132 cl::init(1), cl::cat(ClangOffloadBundlerCategory
));
133 cl::opt
<bool> CheckInputArchive(
134 "check-input-archive",
135 cl::desc("Check if input heterogeneous archive is "
136 "valid in terms of TargetID rules.\n"),
137 cl::init(false), cl::cat(ClangOffloadBundlerCategory
));
138 cl::opt
<bool> HipOpenmpCompatible(
139 "hip-openmp-compatible",
140 cl::desc("Treat hip and hipv4 offload kinds as "
141 "compatible with openmp kind, and vice versa.\n"),
142 cl::init(false), cl::cat(ClangOffloadBundlerCategory
));
143 cl::opt
<bool> Compress("compress",
144 cl::desc("Compress output file when bundling.\n"),
145 cl::init(false), cl::cat(ClangOffloadBundlerCategory
));
146 cl::opt
<bool> Verbose("verbose", cl::desc("Print debug information.\n"),
147 cl::init(false), cl::cat(ClangOffloadBundlerCategory
));
149 // Process commandline options and report errors
150 sys::PrintStackTraceOnErrorSignal(argv
[0]);
152 cl::HideUnrelatedOptions(ClangOffloadBundlerCategory
);
153 cl::SetVersionPrinter(PrintVersion
);
154 cl::ParseCommandLineOptions(
156 "A tool to bundle several input files of the specified type <type> \n"
157 "referring to the same source file but different targets into a single \n"
158 "one. The resulting file can also be unbundled into different files by \n"
159 "this tool if -unbundle is provided.\n");
162 cl::PrintHelpMessage();
166 /// Class to store bundler options in standard (non-cl::opt) data structures
167 // Avoid using cl::opt variables after these assignments when possible
168 OffloadBundlerConfig BundlerConfig
;
169 BundlerConfig
.AllowMissingBundles
= AllowMissingBundles
;
170 BundlerConfig
.CheckInputArchive
= CheckInputArchive
;
171 BundlerConfig
.PrintExternalCommands
= PrintExternalCommands
;
172 BundlerConfig
.HipOpenmpCompatible
= HipOpenmpCompatible
;
173 BundlerConfig
.BundleAlignment
= BundleAlignment
;
174 BundlerConfig
.FilesType
= FilesType
;
175 BundlerConfig
.ObjcopyPath
= "";
176 // Do not override the default value Compress and Verbose in BundlerConfig.
177 if (Compress
.getNumOccurrences() > 0)
178 BundlerConfig
.Compress
= Compress
;
179 if (Verbose
.getNumOccurrences() > 0)
180 BundlerConfig
.Verbose
= Verbose
;
182 BundlerConfig
.TargetNames
= TargetNames
;
183 BundlerConfig
.InputFileNames
= InputFileNames
;
184 BundlerConfig
.OutputFileNames
= OutputFileNames
;
186 /// The index of the host input in the list of inputs.
187 BundlerConfig
.HostInputIndex
= ~0u;
189 /// Whether not having host target is allowed.
190 BundlerConfig
.AllowNoHost
= false;
192 auto reportError
= [argv
](Error E
) {
193 logAllUnhandledErrors(std::move(E
), WithColor::error(errs(), argv
[0]));
197 auto doWork
= [&](std::function
<llvm::Error()> Work
) {
198 if (llvm::Error Err
= Work()) {
199 reportError(std::move(Err
));
203 auto warningOS
= [argv
]() -> raw_ostream
& {
204 return WithColor::warning(errs(), StringRef(argv
[0]));
207 /// Path to the current binary.
208 std::string BundlerExecutable
= argv
[0];
210 if (!llvm::sys::fs::exists(BundlerExecutable
))
212 sys::fs::getMainExecutable(argv
[0], &BundlerExecutable
);
214 // Find llvm-objcopy in order to create the bundle binary.
215 ErrorOr
<std::string
> Objcopy
= sys::findProgramByName(
217 sys::path::parent_path(BundlerExecutable
));
219 Objcopy
= sys::findProgramByName("llvm-objcopy");
221 reportError(createStringError(Objcopy
.getError(),
222 "unable to find 'llvm-objcopy' in path"));
224 BundlerConfig
.ObjcopyPath
= *Objcopy
;
226 if (InputFileNames
.getNumOccurrences() != 0 &&
227 InputFileNamesDeprecatedOpt
.getNumOccurrences() != 0) {
228 reportError(createStringError(
229 errc::invalid_argument
,
230 "-inputs and -input cannot be used together, use only -input instead"));
233 if (InputFileNamesDeprecatedOpt
.size()) {
234 warningOS() << "-inputs is deprecated, use -input instead\n";
235 // temporary hack to support -inputs
236 std::vector
<std::string
> &s
= InputFileNames
;
237 s
.insert(s
.end(), InputFileNamesDeprecatedOpt
.begin(),
238 InputFileNamesDeprecatedOpt
.end());
240 BundlerConfig
.InputFileNames
= InputFileNames
;
242 if (OutputFileNames
.getNumOccurrences() != 0 &&
243 OutputFileNamesDeprecatedOpt
.getNumOccurrences() != 0) {
244 reportError(createStringError(errc::invalid_argument
,
245 "-outputs and -output cannot be used "
246 "together, use only -output instead"));
249 if (OutputFileNamesDeprecatedOpt
.size()) {
250 warningOS() << "-outputs is deprecated, use -output instead\n";
251 // temporary hack to support -outputs
252 std::vector
<std::string
> &s
= OutputFileNames
;
253 s
.insert(s
.end(), OutputFileNamesDeprecatedOpt
.begin(),
254 OutputFileNamesDeprecatedOpt
.end());
256 BundlerConfig
.OutputFileNames
= OutputFileNames
;
261 createStringError(errc::invalid_argument
,
262 "-unbundle and -list cannot be used together"));
264 if (InputFileNames
.size() != 1) {
265 reportError(createStringError(errc::invalid_argument
,
266 "only one input file supported for -list"));
268 if (OutputFileNames
.size()) {
269 reportError(createStringError(errc::invalid_argument
,
270 "-outputs option is invalid for -list"));
272 if (TargetNames
.size()) {
273 reportError(createStringError(errc::invalid_argument
,
274 "-targets option is invalid for -list"));
277 doWork([&]() { return OffloadBundler::ListBundleIDsInFile(
278 InputFileNames
.front(),
283 if (BundlerConfig
.CheckInputArchive
) {
285 reportError(createStringError(errc::invalid_argument
,
286 "-check-input-archive cannot be used while "
289 if (Unbundle
&& BundlerConfig
.FilesType
!= "a") {
290 reportError(createStringError(errc::invalid_argument
,
291 "-check-input-archive can only be used for "
292 "unbundling archives (-type=a)"));
296 if (OutputFileNames
.size() == 0) {
298 createStringError(errc::invalid_argument
, "no output file specified!"));
301 if (TargetNames
.getNumOccurrences() == 0) {
302 reportError(createStringError(
303 errc::invalid_argument
,
304 "for the --targets option: must be specified at least once!"));
308 if (InputFileNames
.size() != 1) {
309 reportError(createStringError(
310 errc::invalid_argument
,
311 "only one input file supported in unbundling mode"));
313 if (OutputFileNames
.size() != TargetNames
.size()) {
314 reportError(createStringError(errc::invalid_argument
,
315 "number of output files and targets should "
316 "match in unbundling mode"));
319 if (BundlerConfig
.FilesType
== "a") {
320 reportError(createStringError(errc::invalid_argument
,
321 "Archive files are only supported "
324 if (OutputFileNames
.size() != 1) {
325 reportError(createStringError(
326 errc::invalid_argument
,
327 "only one output file supported in bundling mode"));
329 if (InputFileNames
.size() != TargetNames
.size()) {
330 reportError(createStringError(
331 errc::invalid_argument
,
332 "number of input files and targets should match in bundling mode"));
336 // Verify that the offload kinds and triples are known. We also check that we
337 // have exactly one host target.
339 unsigned HostTargetNum
= 0u;
341 llvm::DenseSet
<StringRef
> ParsedTargets
;
342 // Map {offload-kind}-{triple} to target IDs.
343 std::map
<std::string
, std::set
<StringRef
>> TargetIDs
;
344 // Standardize target names to include env field
345 std::vector
<std::string
> StandardizedTargetNames
;
346 for (StringRef Target
: TargetNames
) {
347 if (ParsedTargets
.contains(Target
)) {
348 reportError(createStringError(errc::invalid_argument
,
349 "Duplicate targets are not allowed"));
351 ParsedTargets
.insert(Target
);
353 auto OffloadInfo
= OffloadTargetInfo(Target
, BundlerConfig
);
354 bool KindIsValid
= OffloadInfo
.isOffloadKindValid();
355 bool TripleIsValid
= OffloadInfo
.isTripleValid();
357 StandardizedTargetNames
.push_back(OffloadInfo
.str());
359 if (!KindIsValid
|| !TripleIsValid
) {
360 SmallVector
<char, 128u> Buf
;
361 raw_svector_ostream
Msg(Buf
);
362 Msg
<< "invalid target '" << Target
<< "'";
364 Msg
<< ", unknown offloading kind '" << OffloadInfo
.OffloadKind
<< "'";
366 Msg
<< ", unknown target triple '" << OffloadInfo
.Triple
.str() << "'";
367 reportError(createStringError(errc::invalid_argument
, Msg
.str()));
370 TargetIDs
[OffloadInfo
.OffloadKind
.str() + "-" + OffloadInfo
.Triple
.str()]
371 .insert(OffloadInfo
.TargetID
);
372 if (KindIsValid
&& OffloadInfo
.hasHostKind()) {
374 // Save the index of the input that refers to the host.
375 BundlerConfig
.HostInputIndex
= Index
;
378 if (OffloadInfo
.OffloadKind
!= "hip" && OffloadInfo
.OffloadKind
!= "hipv4")
384 BundlerConfig
.TargetNames
= StandardizedTargetNames
;
386 for (const auto &TargetID
: TargetIDs
) {
387 if (auto ConflictingTID
=
388 clang::getConflictTargetIDCombination(TargetID
.second
)) {
389 SmallVector
<char, 128u> Buf
;
390 raw_svector_ostream
Msg(Buf
);
391 Msg
<< "Cannot bundle inputs with conflicting targets: '"
392 << TargetID
.first
+ "-" + ConflictingTID
->first
<< "' and '"
393 << TargetID
.first
+ "-" + ConflictingTID
->second
<< "'";
394 reportError(createStringError(errc::invalid_argument
, Msg
.str()));
398 // HIP uses clang-offload-bundler to bundle device-only compilation results
399 // for multiple GPU archs, therefore allow no host target if all entries
401 BundlerConfig
.AllowNoHost
= HIPOnly
;
403 // Host triple is not really needed for unbundling operation, so do not
404 // treat missing host triple as error if we do unbundling.
405 if ((Unbundle
&& HostTargetNum
> 1) ||
406 (!Unbundle
&& HostTargetNum
!= 1 && !BundlerConfig
.AllowNoHost
)) {
407 reportError(createStringError(errc::invalid_argument
,
408 "expecting exactly one host target but got " +
409 Twine(HostTargetNum
)));
412 OffloadBundler
Bundler(BundlerConfig
);
416 if (BundlerConfig
.FilesType
== "a")
417 return Bundler
.UnbundleArchive();
419 return Bundler
.UnbundleFiles();
421 return Bundler
.BundleFiles();