clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp

   1 //===-- clang-offload-bundler/ClangOffloadBundler.cpp ---------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 ///
   9 /// \file
  10 /// This file implements a stand-alone clang-offload-bundler tool using the
  11 /// OffloadBundler API.
  12 ///
  13 //===----------------------------------------------------------------------===//
  14
  15 #include "clang/Basic/Cuda.h"
  16 #include "clang/Basic/Version.h"
  17 #include "clang/Driver/OffloadBundler.h"
  18 #include "llvm/ADT/ArrayRef.h"
  19 #include "llvm/ADT/SmallString.h"
  20 #include "llvm/ADT/SmallVector.h"
  21 #include "llvm/ADT/StringMap.h"
  22 #include "llvm/ADT/StringRef.h"
  23 #include "llvm/ADT/StringSwitch.h"
  24 #include "llvm/ADT/Triple.h"
  25 #include "llvm/Object/Archive.h"
  26 #include "llvm/Object/ArchiveWriter.h"
  27 #include "llvm/Object/Binary.h"
  28 #include "llvm/Object/ObjectFile.h"
  29 #include "llvm/Support/Casting.h"
  30 #include "llvm/Support/CommandLine.h"
  31 #include "llvm/Support/Debug.h"
  32 #include "llvm/Support/Errc.h"
  33 #include "llvm/Support/Error.h"
  34 #include "llvm/Support/ErrorOr.h"
  35 #include "llvm/Support/FileSystem.h"
  36 #include "llvm/Support/Host.h"
  37 #include "llvm/Support/MemoryBuffer.h"
  38 #include "llvm/Support/Path.h"
  39 #include "llvm/Support/Program.h"
  40 #include "llvm/Support/Signals.h"
  41 #include "llvm/Support/StringSaver.h"
  42 #include "llvm/Support/WithColor.h"
  43 #include "llvm/Support/raw_ostream.h"
  44 #include <algorithm>
  45 #include <cassert>
  46 #include <cstddef>
  47 #include <cstdint>
  48 #include <forward_list>
  49 #include <memory>
  50 #include <set>
  51 #include <string>
  52 #include <system_error>
  53 #include <utility>
  54
  55 using namespace llvm;
  56 using namespace llvm::object;
  57 using namespace clang;
  58
  59 static void PrintVersion(raw_ostream &OS) {
  60   OS << clang::getClangToolFullVersion("clang-offload-bundler") << '\n';
  61 }
  62
  63 int main(int argc, const char **argv) {
  64
  65   cl::opt<bool> Help("h", cl::desc("Alias for -help"), cl::Hidden);
  66
  67   // Mark all our options with this category, everything else (except for
  68   // -version and -help) will be hidden.
  69   cl::OptionCategory
  70     ClangOffloadBundlerCategory("clang-offload-bundler options");
  71   cl::list<std::string>
  72     InputFileNames("input",
  73                    cl::desc("Input file."
  74                             " Can be specified multiple times "
  75                             "for multiple input files."),
  76                    cl::cat(ClangOffloadBundlerCategory));
  77   cl::list<std::string>
  78     InputFileNamesDeprecatedOpt("inputs", cl::CommaSeparated,
  79                                 cl::desc("[<input file>,...] (deprecated)"),
  80                                 cl::cat(ClangOffloadBundlerCategory));
  81   cl::list<std::string>
  82     OutputFileNames("output",
  83                     cl::desc("Output file."
  84                              " Can be specified multiple times "
  85                              "for multiple output files."),
  86                     cl::cat(ClangOffloadBundlerCategory));
  87   cl::list<std::string>
  88     OutputFileNamesDeprecatedOpt("outputs", cl::CommaSeparated,
  89                                  cl::desc("[<output file>,...] (deprecated)"),
  90                                  cl::cat(ClangOffloadBundlerCategory));
  91   cl::list<std::string>
  92     TargetNames("targets", cl::CommaSeparated,
  93                 cl::desc("[<offload kind>-<target triple>,...]"),
  94                 cl::cat(ClangOffloadBundlerCategory));
  95   cl::opt<std::string>
  96     FilesType("type", cl::Required,
  97               cl::desc("Type of the files to be bundled/unbundled.\n"
  98                        "Current supported types are:\n"
  99                        "  i   - cpp-output\n"
 100                        "  ii  - c++-cpp-output\n"
 101                        "  cui - cuda/hip-output\n"
 102                        "  d   - dependency\n"
 103                        "  ll  - llvm\n"
 104                        "  bc  - llvm-bc\n"
 105                        "  s   - assembler\n"
 106                        "  o   - object\n"
 107                        "  a   - archive of objects\n"
 108                        "  gch - precompiled-header\n"
 109                        "  ast - clang AST file"),
 110               cl::cat(ClangOffloadBundlerCategory));
 111   cl::opt<bool>
 112     Unbundle("unbundle",
 113              cl::desc("Unbundle bundled file into several output files.\n"),
 114              cl::init(false), cl::cat(ClangOffloadBundlerCategory));
 115   cl::opt<bool>
 116     ListBundleIDs("list", cl::desc("List bundle IDs in the bundled file.\n"),
 117                   cl::init(false), cl::cat(ClangOffloadBundlerCategory));
 118   cl::opt<bool> PrintExternalCommands(
 119     "###",
 120     cl::desc("Print any external commands that are to be executed "
 121              "instead of actually executing them - for testing purposes.\n"),
 122     cl::init(false), cl::cat(ClangOffloadBundlerCategory));
 123   cl::opt<bool>
 124     AllowMissingBundles("allow-missing-bundles",
 125                         cl::desc("Create empty files if bundles are missing "
 126                                  "when unbundling.\n"),
 127                         cl::init(false), cl::cat(ClangOffloadBundlerCategory));
 128   cl::opt<unsigned>
 129     BundleAlignment("bundle-align",
 130                     cl::desc("Alignment of bundle for binary files"),
 131                     cl::init(1), cl::cat(ClangOffloadBundlerCategory));
 132   cl::opt<bool> HipOpenmpCompatible(
 133     "hip-openmp-compatible",
 134     cl::desc("Treat hip and hipv4 offload kinds as "
 135              "compatible with openmp kind, and vice versa.\n"),
 136     cl::init(false), cl::cat(ClangOffloadBundlerCategory));
 137
 138   // Process commandline options and report errors
 139   sys::PrintStackTraceOnErrorSignal(argv[0]);
 140
 141   cl::HideUnrelatedOptions(ClangOffloadBundlerCategory);
 142   cl::SetVersionPrinter(PrintVersion);
 143   cl::ParseCommandLineOptions(
 144       argc, argv,
 145       "A tool to bundle several input files of the specified type <type> \n"
 146       "referring to the same source file but different targets into a single \n"
 147       "one. The resulting file can also be unbundled into different files by \n"
 148       "this tool if -unbundle is provided.\n");
 149
 150   if (Help) {
 151     cl::PrintHelpMessage();
 152     return 0;
 153   }
 154
 155   /// Class to store bundler options in standard (non-cl::opt) data structures
 156   // Avoid using cl::opt variables after these assignments when possible
 157   OffloadBundlerConfig BundlerConfig;
 158   BundlerConfig.AllowMissingBundles = AllowMissingBundles;
 159   BundlerConfig.PrintExternalCommands = PrintExternalCommands;
 160   BundlerConfig.HipOpenmpCompatible = HipOpenmpCompatible;
 161   BundlerConfig.BundleAlignment = BundleAlignment;
 162   BundlerConfig.FilesType = FilesType;
 163   BundlerConfig.ObjcopyPath = "";
 164
 165   BundlerConfig.TargetNames = TargetNames;
 166   BundlerConfig.InputFileNames = InputFileNames;
 167   BundlerConfig.OutputFileNames = OutputFileNames;
 168
 169   /// The index of the host input in the list of inputs.
 170   BundlerConfig.HostInputIndex = ~0u;
 171
 172   /// Whether not having host target is allowed.
 173   BundlerConfig.AllowNoHost = false;
 174
 175   auto reportError = [argv](Error E) {
 176     logAllUnhandledErrors(std::move(E), WithColor::error(errs(), argv[0]));
 177     exit(1);
 178   };
 179
 180   auto doWork = [&](std::function<llvm::Error()> Work) {
 181     if (llvm::Error Err = Work()) {
 182       reportError(std::move(Err));
 183     }
 184   };
 185
 186   auto warningOS = [argv]() -> raw_ostream & {
 187     return WithColor::warning(errs(), StringRef(argv[0]));
 188   };
 189
 190   /// Path to the current binary.
 191   std::string BundlerExecutable = argv[0];
 192
 193   if (!llvm::sys::fs::exists(BundlerExecutable))
 194     BundlerExecutable =
 195       sys::fs::getMainExecutable(argv[0], &BundlerExecutable);
 196
 197   // Find llvm-objcopy in order to create the bundle binary.
 198   ErrorOr<std::string> Objcopy = sys::findProgramByName(
 199     "llvm-objcopy",
 200     sys::path::parent_path(BundlerExecutable));
 201   if (!Objcopy)
 202     Objcopy = sys::findProgramByName("llvm-objcopy");
 203   if (!Objcopy)
 204     reportError(createStringError(Objcopy.getError(),
 205                              "unable to find 'llvm-objcopy' in path"));
 206   else
 207     BundlerConfig.ObjcopyPath = *Objcopy;
 208
 209   if (InputFileNames.getNumOccurrences() != 0 &&
 210       InputFileNamesDeprecatedOpt.getNumOccurrences() != 0) {
 211     reportError(createStringError(
 212         errc::invalid_argument,
 213         "-inputs and -input cannot be used together, use only -input instead"));
 214   }
 215
 216   if (InputFileNamesDeprecatedOpt.size()) {
 217     warningOS() << "-inputs is deprecated, use -input instead\n";
 218     // temporary hack to support -inputs
 219     std::vector<std::string> &s = InputFileNames;
 220     s.insert(s.end(), InputFileNamesDeprecatedOpt.begin(),
 221              InputFileNamesDeprecatedOpt.end());
 222   }
 223   BundlerConfig.InputFileNames = InputFileNames;
 224
 225   if (OutputFileNames.getNumOccurrences() != 0 &&
 226       OutputFileNamesDeprecatedOpt.getNumOccurrences() != 0) {
 227     reportError(createStringError(errc::invalid_argument,
 228                                   "-outputs and -output cannot be used "
 229                                   "together, use only -output instead"));
 230   }
 231
 232   if (OutputFileNamesDeprecatedOpt.size()) {
 233     warningOS() << "-outputs is deprecated, use -output instead\n";
 234     // temporary hack to support -outputs
 235     std::vector<std::string> &s = OutputFileNames;
 236     s.insert(s.end(), OutputFileNamesDeprecatedOpt.begin(),
 237              OutputFileNamesDeprecatedOpt.end());
 238   }
 239   BundlerConfig.OutputFileNames = OutputFileNames;
 240
 241   if (ListBundleIDs) {
 242     if (Unbundle) {
 243       reportError(
 244           createStringError(errc::invalid_argument,
 245                             "-unbundle and -list cannot be used together"));
 246     }
 247     if (InputFileNames.size() != 1) {
 248       reportError(createStringError(errc::invalid_argument,
 249                                     "only one input file supported for -list"));
 250     }
 251     if (OutputFileNames.size()) {
 252       reportError(createStringError(errc::invalid_argument,
 253                                     "-outputs option is invalid for -list"));
 254     }
 255     if (TargetNames.size()) {
 256       reportError(createStringError(errc::invalid_argument,
 257                                     "-targets option is invalid for -list"));
 258     }
 259
 260     doWork([&]() { return OffloadBundler::ListBundleIDsInFile(
 261           InputFileNames.front(),
 262           BundlerConfig); });
 263     return 0;
 264   }
 265
 266   if (OutputFileNames.size() == 0) {
 267     reportError(
 268         createStringError(errc::invalid_argument, "no output file specified!"));
 269   }
 270
 271   if (TargetNames.getNumOccurrences() == 0) {
 272     reportError(createStringError(
 273         errc::invalid_argument,
 274         "for the --targets option: must be specified at least once!"));
 275   }
 276
 277   if (Unbundle) {
 278     if (InputFileNames.size() != 1) {
 279       reportError(createStringError(
 280           errc::invalid_argument,
 281           "only one input file supported in unbundling mode"));
 282     }
 283     if (OutputFileNames.size() != TargetNames.size()) {
 284       reportError(createStringError(errc::invalid_argument,
 285                                     "number of output files and targets should "
 286                                     "match in unbundling mode"));
 287     }
 288   } else {
 289     if (BundlerConfig.FilesType == "a") {
 290       reportError(createStringError(errc::invalid_argument,
 291                                     "Archive files are only supported "
 292                                     "for unbundling"));
 293     }
 294     if (OutputFileNames.size() != 1) {
 295       reportError(createStringError(
 296           errc::invalid_argument,
 297           "only one output file supported in bundling mode"));
 298     }
 299     if (InputFileNames.size() != TargetNames.size()) {
 300       reportError(createStringError(
 301           errc::invalid_argument,
 302           "number of input files and targets should match in bundling mode"));
 303     }
 304   }
 305
 306   // Verify that the offload kinds and triples are known. We also check that we
 307   // have exactly one host target.
 308   unsigned Index = 0u;
 309   unsigned HostTargetNum = 0u;
 310   bool HIPOnly = true;
 311   llvm::DenseSet<StringRef> ParsedTargets;
 312   for (StringRef Target : TargetNames) {
 313     if (ParsedTargets.contains(Target)) {
 314       reportError(createStringError(errc::invalid_argument,
 315                                     "Duplicate targets are not allowed"));
 316     }
 317     ParsedTargets.insert(Target);
 318
 319     auto OffloadInfo = OffloadTargetInfo(Target, BundlerConfig);
 320     bool KindIsValid = OffloadInfo.isOffloadKindValid();
 321     bool TripleIsValid = OffloadInfo.isTripleValid();
 322
 323     if (!KindIsValid || !TripleIsValid) {
 324       SmallVector<char, 128u> Buf;
 325       raw_svector_ostream Msg(Buf);
 326       Msg << "invalid target '" << Target << "'";
 327       if (!KindIsValid)
 328         Msg << ", unknown offloading kind '" << OffloadInfo.OffloadKind << "'";
 329       if (!TripleIsValid)
 330         Msg << ", unknown target triple '" << OffloadInfo.Triple.str() << "'";
 331       reportError(createStringError(errc::invalid_argument, Msg.str()));
 332     }
 333
 334     if (KindIsValid && OffloadInfo.hasHostKind()) {
 335       ++HostTargetNum;
 336       // Save the index of the input that refers to the host.
 337       BundlerConfig.HostInputIndex = Index;
 338     }
 339
 340     if (OffloadInfo.OffloadKind != "hip" && OffloadInfo.OffloadKind != "hipv4")
 341       HIPOnly = false;
 342
 343     ++Index;
 344   }
 345
 346   // HIP uses clang-offload-bundler to bundle device-only compilation results
 347   // for multiple GPU archs, therefore allow no host target if all entries
 348   // are for HIP.
 349   BundlerConfig.AllowNoHost = HIPOnly;
 350
 351   // Host triple is not really needed for unbundling operation, so do not
 352   // treat missing host triple as error if we do unbundling.
 353   if ((Unbundle && HostTargetNum > 1) ||
 354       (!Unbundle && HostTargetNum != 1 && !BundlerConfig.AllowNoHost)) {
 355     reportError(createStringError(errc::invalid_argument,
 356                                   "expecting exactly one host target but got " +
 357                                       Twine(HostTargetNum)));
 358   }
 359
 360   OffloadBundler Bundler(BundlerConfig);
 361
 362   doWork([&]() {
 363     if (Unbundle) {
 364       if (BundlerConfig.FilesType == "a")
 365         return Bundler.UnbundleArchive();
 366       else
 367         return Bundler.UnbundleFiles();
 368     } else
 369       return Bundler.BundleFiles();
 370   });
 371   return 0;
 372 }