clang/tools/clang-scan-deps/ClangScanDeps.cpp

   1 //===- ClangScanDeps.cpp - Implementation of clang-scan-deps --------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "clang/Driver/Compilation.h"
  10 #include "clang/Driver/Driver.h"
  11 #include "clang/Frontend/CompilerInstance.h"
  12 #include "clang/Frontend/TextDiagnosticPrinter.h"
  13 #include "clang/Tooling/CommonOptionsParser.h"
  14 #include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
  15 #include "clang/Tooling/DependencyScanning/DependencyScanningTool.h"
  16 #include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h"
  17 #include "clang/Tooling/JSONCompilationDatabase.h"
  18 #include "llvm/ADT/STLExtras.h"
  19 #include "llvm/ADT/Twine.h"
  20 #include "llvm/Support/CommandLine.h"
  21 #include "llvm/Support/FileUtilities.h"
  22 #include "llvm/Support/Format.h"
  23 #include "llvm/Support/InitLLVM.h"
  24 #include "llvm/Support/JSON.h"
  25 #include "llvm/Support/LLVMDriver.h"
  26 #include "llvm/Support/Program.h"
  27 #include "llvm/Support/Signals.h"
  28 #include "llvm/Support/ThreadPool.h"
  29 #include "llvm/Support/Threading.h"
  30 #include "llvm/Support/Timer.h"
  31 #include "llvm/TargetParser/Host.h"
  32 #include <mutex>
  33 #include <optional>
  34 #include <thread>
  35
  36 #include "Opts.inc"
  37
  38 using namespace clang;
  39 using namespace tooling::dependencies;
  40
  41 namespace {
  42
  43 using namespace llvm::opt;
  44 enum ID {
  45   OPT_INVALID = 0, // This is not an option ID.
  46 #define OPTION(...) LLVM_MAKE_OPT_ID(__VA_ARGS__),
  47 #include "Opts.inc"
  48 #undef OPTION
  49 };
  50
  51 #define PREFIX(NAME, VALUE)                                                    \
  52   constexpr llvm::StringLiteral NAME##_init[] = VALUE;                         \
  53   constexpr llvm::ArrayRef<llvm::StringLiteral> NAME(                          \
  54       NAME##_init, std::size(NAME##_init) - 1);
  55 #include "Opts.inc"
  56 #undef PREFIX
  57
  58 const llvm::opt::OptTable::Info InfoTable[] = {
  59 #define OPTION(...) LLVM_CONSTRUCT_OPT_INFO(__VA_ARGS__),
  60 #include "Opts.inc"
  61 #undef OPTION
  62 };
  63
  64 class ScanDepsOptTable : public llvm::opt::GenericOptTable {
  65 public:
  66   ScanDepsOptTable() : GenericOptTable(InfoTable) {
  67     setGroupedShortOptions(true);
  68   }
  69 };
  70
  71 enum ResourceDirRecipeKind {
  72   RDRK_ModifyCompilerPath,
  73   RDRK_InvokeCompiler,
  74 };
  75
  76 static ScanningMode ScanMode = ScanningMode::DependencyDirectivesScan;
  77 static ScanningOutputFormat Format = ScanningOutputFormat::Make;
  78 static std::string ModuleFilesDir;
  79 static bool OptimizeArgs;
  80 static bool EagerLoadModules;
  81 static unsigned NumThreads = 0;
  82 static std::string CompilationDB;
  83 static std::string ModuleName;
  84 static std::vector<std::string> ModuleDepTargets;
  85 static bool DeprecatedDriverCommand;
  86 static ResourceDirRecipeKind ResourceDirRecipe;
  87 static bool Verbose;
  88 static bool PrintTiming;
  89 static std::vector<const char *> CommandLine;
  90
  91 #ifndef NDEBUG
  92 static constexpr bool DoRoundTripDefault = true;
  93 #else
  94 static constexpr bool DoRoundTripDefault = false;
  95 #endif
  96
  97 static bool RoundTripArgs = DoRoundTripDefault;
  98
  99 static void ParseArgs(int argc, char **argv) {
 100   ScanDepsOptTable Tbl;
 101   llvm::StringRef ToolName = argv[0];
 102   llvm::BumpPtrAllocator A;
 103   llvm::StringSaver Saver{A};
 104   llvm::opt::InputArgList Args =
 105       Tbl.parseArgs(argc, argv, OPT_UNKNOWN, Saver, [&](StringRef Msg) {
 106         llvm::errs() << Msg << '\n';
 107         std::exit(1);
 108       });
 109
 110   if (Args.hasArg(OPT_help)) {
 111     Tbl.printHelp(llvm::outs(), "clang-scan-deps [options]", "clang-scan-deps");
 112     std::exit(0);
 113   }
 114   if (Args.hasArg(OPT_version)) {
 115     llvm::outs() << ToolName << '\n';
 116     llvm::cl::PrintVersionMessage();
 117     std::exit(0);
 118   }
 119   if (const llvm::opt::Arg *A = Args.getLastArg(OPT_mode_EQ)) {
 120     auto ModeType =
 121         llvm::StringSwitch<std::optional<ScanningMode>>(A->getValue())
 122             .Case("preprocess-dependency-directives",
 123                   ScanningMode::DependencyDirectivesScan)
 124             .Case("preprocess", ScanningMode::CanonicalPreprocessing)
 125             .Default(std::nullopt);
 126     if (!ModeType) {
 127       llvm::errs() << ToolName
 128                    << ": for the --mode option: Cannot find option named '"
 129                    << A->getValue() << "'\n";
 130       std::exit(1);
 131     }
 132     ScanMode = *ModeType;
 133   }
 134
 135   if (const llvm::opt::Arg *A = Args.getLastArg(OPT_format_EQ)) {
 136     auto FormatType =
 137         llvm::StringSwitch<std::optional<ScanningOutputFormat>>(A->getValue())
 138             .Case("make", ScanningOutputFormat::Make)
 139             .Case("p1689", ScanningOutputFormat::P1689)
 140             .Case("experimental-full", ScanningOutputFormat::Full)
 141             .Default(std::nullopt);
 142     if (!FormatType) {
 143       llvm::errs() << ToolName
 144                    << ": for the --format option: Cannot find option named '"
 145                    << A->getValue() << "'\n";
 146       std::exit(1);
 147     }
 148     Format = *FormatType;
 149   }
 150
 151   if (const llvm::opt::Arg *A = Args.getLastArg(OPT_module_files_dir_EQ))
 152     ModuleFilesDir = A->getValue();
 153
 154   OptimizeArgs = Args.hasArg(OPT_optimize_args);
 155   EagerLoadModules = Args.hasArg(OPT_eager_load_pcm);
 156
 157   if (const llvm::opt::Arg *A = Args.getLastArg(OPT_j)) {
 158     StringRef S{A->getValue()};
 159     if (!llvm::to_integer(S, NumThreads, 0)) {
 160       llvm::errs() << ToolName << ": for the -j option: '" << S
 161                    << "' value invalid for uint argument!\n";
 162       std::exit(1);
 163     }
 164   }
 165
 166   if (const llvm::opt::Arg *A = Args.getLastArg(OPT_compilation_database_EQ)) {
 167     CompilationDB = A->getValue();
 168   } else if (Format != ScanningOutputFormat::P1689) {
 169     llvm::errs() << ToolName
 170                  << ": for the --compiilation-database option: must be "
 171                     "specified at least once!";
 172     std::exit(1);
 173   }
 174
 175   if (const llvm::opt::Arg *A = Args.getLastArg(OPT_module_name_EQ))
 176     ModuleName = A->getValue();
 177
 178   for (const llvm::opt::Arg *A : Args.filtered(OPT_dependency_target_EQ))
 179     ModuleDepTargets.emplace_back(A->getValue());
 180
 181   DeprecatedDriverCommand = Args.hasArg(OPT_deprecated_driver_command);
 182
 183   if (const llvm::opt::Arg *A = Args.getLastArg(OPT_resource_dir_recipe_EQ)) {
 184     auto Kind =
 185         llvm::StringSwitch<std::optional<ResourceDirRecipeKind>>(A->getValue())
 186             .Case("modify-compiler-path", RDRK_ModifyCompilerPath)
 187             .Case("invoke-compiler", RDRK_InvokeCompiler)
 188             .Default(std::nullopt);
 189     if (!Kind) {
 190       llvm::errs() << ToolName
 191                    << ": for the --resource-dir-recipe option: Cannot find "
 192                       "option named '"
 193                    << A->getValue() << "'\n";
 194       std::exit(1);
 195     }
 196     ResourceDirRecipe = *Kind;
 197   }
 198
 199   PrintTiming = Args.hasArg(OPT_print_timing);
 200
 201   Verbose = Args.hasArg(OPT_verbose);
 202
 203   RoundTripArgs = Args.hasArg(OPT_round_trip_args);
 204
 205   if (auto *A = Args.getLastArgNoClaim(OPT_DASH_DASH))
 206     CommandLine.insert(CommandLine.end(), A->getValues().begin(),
 207                        A->getValues().end());
 208 }
 209
 210 class SharedStream {
 211 public:
 212   SharedStream(raw_ostream &OS) : OS(OS) {}
 213   void applyLocked(llvm::function_ref<void(raw_ostream &OS)> Fn) {
 214     std::unique_lock<std::mutex> LockGuard(Lock);
 215     Fn(OS);
 216     OS.flush();
 217   }
 218
 219 private:
 220   std::mutex Lock;
 221   raw_ostream &OS;
 222 };
 223
 224 class ResourceDirectoryCache {
 225 public:
 226   /// findResourceDir finds the resource directory relative to the clang
 227   /// compiler being used in Args, by running it with "-print-resource-dir"
 228   /// option and cache the results for reuse. \returns resource directory path
 229   /// associated with the given invocation command or empty string if the
 230   /// compiler path is NOT an absolute path.
 231   StringRef findResourceDir(const tooling::CommandLineArguments &Args,
 232                             bool ClangCLMode) {
 233     if (Args.size() < 1)
 234       return "";
 235
 236     const std::string &ClangBinaryPath = Args[0];
 237     if (!llvm::sys::path::is_absolute(ClangBinaryPath))
 238       return "";
 239
 240     const std::string &ClangBinaryName =
 241         std::string(llvm::sys::path::filename(ClangBinaryPath));
 242
 243     std::unique_lock<std::mutex> LockGuard(CacheLock);
 244     const auto &CachedResourceDir = Cache.find(ClangBinaryPath);
 245     if (CachedResourceDir != Cache.end())
 246       return CachedResourceDir->second;
 247
 248     std::vector<StringRef> PrintResourceDirArgs{ClangBinaryName};
 249     if (ClangCLMode)
 250       PrintResourceDirArgs.push_back("/clang:-print-resource-dir");
 251     else
 252       PrintResourceDirArgs.push_back("-print-resource-dir");
 253
 254     llvm::SmallString<64> OutputFile, ErrorFile;
 255     llvm::sys::fs::createTemporaryFile("print-resource-dir-output",
 256                                        "" /*no-suffix*/, OutputFile);
 257     llvm::sys::fs::createTemporaryFile("print-resource-dir-error",
 258                                        "" /*no-suffix*/, ErrorFile);
 259     llvm::FileRemover OutputRemover(OutputFile.c_str());
 260     llvm::FileRemover ErrorRemover(ErrorFile.c_str());
 261     std::optional<StringRef> Redirects[] = {
 262         {""}, // Stdin
 263         OutputFile.str(),
 264         ErrorFile.str(),
 265     };
 266     if (llvm::sys::ExecuteAndWait(ClangBinaryPath, PrintResourceDirArgs, {},
 267                                   Redirects)) {
 268       auto ErrorBuf = llvm::MemoryBuffer::getFile(ErrorFile.c_str());
 269       llvm::errs() << ErrorBuf.get()->getBuffer();
 270       return "";
 271     }
 272
 273     auto OutputBuf = llvm::MemoryBuffer::getFile(OutputFile.c_str());
 274     if (!OutputBuf)
 275       return "";
 276     StringRef Output = OutputBuf.get()->getBuffer().rtrim('\n');
 277
 278     Cache[ClangBinaryPath] = Output.str();
 279     return Cache[ClangBinaryPath];
 280   }
 281
 282 private:
 283   std::map<std::string, std::string> Cache;
 284   std::mutex CacheLock;
 285 };
 286
 287 } // end anonymous namespace
 288
 289 /// Takes the result of a dependency scan and prints error / dependency files
 290 /// based on the result.
 291 ///
 292 /// \returns True on error.
 293 static bool
 294 handleMakeDependencyToolResult(const std::string &Input,
 295                                llvm::Expected<std::string> &MaybeFile,
 296                                SharedStream &OS, SharedStream &Errs) {
 297   if (!MaybeFile) {
 298     llvm::handleAllErrors(
 299         MaybeFile.takeError(), [&Input, &Errs](llvm::StringError &Err) {
 300           Errs.applyLocked([&](raw_ostream &OS) {
 301             OS << "Error while scanning dependencies for " << Input << ":\n";
 302             OS << Err.getMessage();
 303           });
 304         });
 305     return true;
 306   }
 307   OS.applyLocked([&](raw_ostream &OS) { OS << *MaybeFile; });
 308   return false;
 309 }
 310
 311 static llvm::json::Array toJSONSorted(const llvm::StringSet<> &Set) {
 312   std::vector<llvm::StringRef> Strings;
 313   for (auto &&I : Set)
 314     Strings.push_back(I.getKey());
 315   llvm::sort(Strings);
 316   return llvm::json::Array(Strings);
 317 }
 318
 319 // Technically, we don't need to sort the dependency list to get determinism.
 320 // Leaving these be will simply preserve the import order.
 321 static llvm::json::Array toJSONSorted(std::vector<ModuleID> V) {
 322   llvm::sort(V);
 323
 324   llvm::json::Array Ret;
 325   for (const ModuleID &MID : V)
 326     Ret.push_back(llvm::json::Object(
 327         {{"module-name", MID.ModuleName}, {"context-hash", MID.ContextHash}}));
 328   return Ret;
 329 }
 330
 331 // Thread safe.
 332 class FullDeps {
 333 public:
 334   FullDeps(size_t NumInputs) : Inputs(NumInputs) {}
 335
 336   void mergeDeps(StringRef Input, TranslationUnitDeps TUDeps,
 337                  size_t InputIndex) {
 338     mergeDeps(std::move(TUDeps.ModuleGraph), InputIndex);
 339
 340     InputDeps ID;
 341     ID.FileName = std::string(Input);
 342     ID.ContextHash = std::move(TUDeps.ID.ContextHash);
 343     ID.FileDeps = std::move(TUDeps.FileDeps);
 344     ID.ModuleDeps = std::move(TUDeps.ClangModuleDeps);
 345     ID.DriverCommandLine = std::move(TUDeps.DriverCommandLine);
 346     ID.Commands = std::move(TUDeps.Commands);
 347
 348     assert(InputIndex < Inputs.size() && "Input index out of bounds");
 349     assert(Inputs[InputIndex].FileName.empty() && "Result already populated");
 350     Inputs[InputIndex] = std::move(ID);
 351   }
 352
 353   void mergeDeps(ModuleDepsGraph Graph, size_t InputIndex) {
 354     std::vector<ModuleDeps *> NewMDs;
 355     {
 356       std::unique_lock<std::mutex> ul(Lock);
 357       for (const ModuleDeps &MD : Graph) {
 358         auto I = Modules.find({MD.ID, 0});
 359         if (I != Modules.end()) {
 360           I->first.InputIndex = std::min(I->first.InputIndex, InputIndex);
 361           continue;
 362         }
 363         auto Res = Modules.insert(I, {{MD.ID, InputIndex}, std::move(MD)});
 364         NewMDs.push_back(&Res->second);
 365       }
 366       // First call to \c getBuildArguments is somewhat expensive. Let's call it
 367       // on the current thread (instead of the main one), and outside the
 368       // critical section.
 369       for (ModuleDeps *MD : NewMDs)
 370         (void)MD->getBuildArguments();
 371     }
 372   }
 373
 374   bool roundTripCommand(ArrayRef<std::string> ArgStrs,
 375                         DiagnosticsEngine &Diags) {
 376     if (ArgStrs.empty() || ArgStrs[0] != "-cc1")
 377       return false;
 378     SmallVector<const char *> Args;
 379     for (const std::string &Arg : ArgStrs)
 380       Args.push_back(Arg.c_str());
 381     return !CompilerInvocation::checkCC1RoundTrip(Args, Diags);
 382   }
 383
 384   // Returns \c true if any command lines fail to round-trip. We expect
 385   // commands already be canonical when output by the scanner.
 386   bool roundTripCommands(raw_ostream &ErrOS) {
 387     IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts = new DiagnosticOptions{};
 388     TextDiagnosticPrinter DiagConsumer(ErrOS, &*DiagOpts);
 389     IntrusiveRefCntPtr<DiagnosticsEngine> Diags =
 390         CompilerInstance::createDiagnostics(&*DiagOpts, &DiagConsumer,
 391                                             /*ShouldOwnClient=*/false);
 392
 393     for (auto &&M : Modules)
 394       if (roundTripCommand(M.second.getBuildArguments(), *Diags))
 395         return true;
 396
 397     for (auto &&I : Inputs)
 398       for (const auto &Cmd : I.Commands)
 399         if (roundTripCommand(Cmd.Arguments, *Diags))
 400           return true;
 401
 402     return false;
 403   }
 404
 405   void printFullOutput(raw_ostream &OS) {
 406     // Sort the modules by name to get a deterministic order.
 407     std::vector<IndexedModuleID> ModuleIDs;
 408     for (auto &&M : Modules)
 409       ModuleIDs.push_back(M.first);
 410     llvm::sort(ModuleIDs);
 411
 412     using namespace llvm::json;
 413
 414     Array OutModules;
 415     for (auto &&ModID : ModuleIDs) {
 416       auto &MD = Modules[ModID];
 417       Object O{
 418           {"name", MD.ID.ModuleName},
 419           {"context-hash", MD.ID.ContextHash},
 420           {"file-deps", toJSONSorted(MD.FileDeps)},
 421           {"clang-module-deps", toJSONSorted(MD.ClangModuleDeps)},
 422           {"clang-modulemap-file", MD.ClangModuleMapFile},
 423           {"command-line", MD.getBuildArguments()},
 424       };
 425       OutModules.push_back(std::move(O));
 426     }
 427
 428     Array TUs;
 429     for (auto &&I : Inputs) {
 430       Array Commands;
 431       if (I.DriverCommandLine.empty()) {
 432         for (const auto &Cmd : I.Commands) {
 433           Object O{
 434               {"input-file", I.FileName},
 435               {"clang-context-hash", I.ContextHash},
 436               {"file-deps", I.FileDeps},
 437               {"clang-module-deps", toJSONSorted(I.ModuleDeps)},
 438               {"executable", Cmd.Executable},
 439               {"command-line", Cmd.Arguments},
 440           };
 441           Commands.push_back(std::move(O));
 442         }
 443       } else {
 444         Object O{
 445             {"input-file", I.FileName},
 446             {"clang-context-hash", I.ContextHash},
 447             {"file-deps", I.FileDeps},
 448             {"clang-module-deps", toJSONSorted(I.ModuleDeps)},
 449             {"executable", "clang"},
 450             {"command-line", I.DriverCommandLine},
 451         };
 452         Commands.push_back(std::move(O));
 453       }
 454       TUs.push_back(Object{
 455           {"commands", std::move(Commands)},
 456       });
 457     }
 458
 459     Object Output{
 460         {"modules", std::move(OutModules)},
 461         {"translation-units", std::move(TUs)},
 462     };
 463
 464     OS << llvm::formatv("{0:2}\n", Value(std::move(Output)));
 465   }
 466
 467 private:
 468   struct IndexedModuleID {
 469     ModuleID ID;
 470
 471     // FIXME: This is mutable so that it can still be updated after insertion
 472     //  into an unordered associative container. This is "fine", since this
 473     //  field doesn't contribute to the hash, but it's a brittle hack.
 474     mutable size_t InputIndex;
 475
 476     bool operator==(const IndexedModuleID &Other) const {
 477       return ID == Other.ID;
 478     }
 479
 480     bool operator<(const IndexedModuleID &Other) const {
 481       /// We need the output of clang-scan-deps to be deterministic. However,
 482       /// the dependency graph may contain two modules with the same name. How
 483       /// do we decide which one to print first? If we made that decision based
 484       /// on the context hash, the ordering would be deterministic, but
 485       /// different across machines. This can happen for example when the inputs
 486       /// or the SDKs (which both contribute to the "context" hash) live in
 487       /// different absolute locations. We solve that by tracking the index of
 488       /// the first input TU that (transitively) imports the dependency, which
 489       /// is always the same for the same input, resulting in deterministic
 490       /// sorting that's also reproducible across machines.
 491       return std::tie(ID.ModuleName, InputIndex) <
 492              std::tie(Other.ID.ModuleName, Other.InputIndex);
 493     }
 494
 495     struct Hasher {
 496       std::size_t operator()(const IndexedModuleID &IMID) const {
 497         return llvm::hash_value(IMID.ID);
 498       }
 499     };
 500   };
 501
 502   struct InputDeps {
 503     std::string FileName;
 504     std::string ContextHash;
 505     std::vector<std::string> FileDeps;
 506     std::vector<ModuleID> ModuleDeps;
 507     std::vector<std::string> DriverCommandLine;
 508     std::vector<Command> Commands;
 509   };
 510
 511   std::mutex Lock;
 512   std::unordered_map<IndexedModuleID, ModuleDeps, IndexedModuleID::Hasher>
 513       Modules;
 514   std::vector<InputDeps> Inputs;
 515 };
 516
 517 static bool handleTranslationUnitResult(
 518     StringRef Input, llvm::Expected<TranslationUnitDeps> &MaybeTUDeps,
 519     FullDeps &FD, size_t InputIndex, SharedStream &OS, SharedStream &Errs) {
 520   if (!MaybeTUDeps) {
 521     llvm::handleAllErrors(
 522         MaybeTUDeps.takeError(), [&Input, &Errs](llvm::StringError &Err) {
 523           Errs.applyLocked([&](raw_ostream &OS) {
 524             OS << "Error while scanning dependencies for " << Input << ":\n";
 525             OS << Err.getMessage();
 526           });
 527         });
 528     return true;
 529   }
 530   FD.mergeDeps(Input, std::move(*MaybeTUDeps), InputIndex);
 531   return false;
 532 }
 533
 534 static bool handleModuleResult(
 535     StringRef ModuleName, llvm::Expected<ModuleDepsGraph> &MaybeModuleGraph,
 536     FullDeps &FD, size_t InputIndex, SharedStream &OS, SharedStream &Errs) {
 537   if (!MaybeModuleGraph) {
 538     llvm::handleAllErrors(MaybeModuleGraph.takeError(),
 539                           [&ModuleName, &Errs](llvm::StringError &Err) {
 540                             Errs.applyLocked([&](raw_ostream &OS) {
 541                               OS << "Error while scanning dependencies for "
 542                                  << ModuleName << ":\n";
 543                               OS << Err.getMessage();
 544                             });
 545                           });
 546     return true;
 547   }
 548   FD.mergeDeps(std::move(*MaybeModuleGraph), InputIndex);
 549   return false;
 550 }
 551
 552 class P1689Deps {
 553 public:
 554   void printDependencies(raw_ostream &OS) {
 555     addSourcePathsToRequires();
 556     // Sort the modules by name to get a deterministic order.
 557     llvm::sort(Rules, [](const P1689Rule &A, const P1689Rule &B) {
 558       return A.PrimaryOutput < B.PrimaryOutput;
 559     });
 560
 561     using namespace llvm::json;
 562     Array OutputRules;
 563     for (const P1689Rule &R : Rules) {
 564       Object O{{"primary-output", R.PrimaryOutput}};
 565
 566       if (R.Provides) {
 567         Array Provides;
 568         Object Provided{{"logical-name", R.Provides->ModuleName},
 569                         {"source-path", R.Provides->SourcePath},
 570                         {"is-interface", R.Provides->IsStdCXXModuleInterface}};
 571         Provides.push_back(std::move(Provided));
 572         O.insert({"provides", std::move(Provides)});
 573       }
 574
 575       Array Requires;
 576       for (const P1689ModuleInfo &Info : R.Requires) {
 577         Object RequiredInfo{{"logical-name", Info.ModuleName}};
 578         if (!Info.SourcePath.empty())
 579           RequiredInfo.insert({"source-path", Info.SourcePath});
 580         Requires.push_back(std::move(RequiredInfo));
 581       }
 582
 583       if (!Requires.empty())
 584         O.insert({"requires", std::move(Requires)});
 585
 586       OutputRules.push_back(std::move(O));
 587     }
 588
 589     Object Output{
 590         {"version", 1}, {"revision", 0}, {"rules", std::move(OutputRules)}};
 591
 592     OS << llvm::formatv("{0:2}\n", Value(std::move(Output)));
 593   }
 594
 595   void addRules(P1689Rule &Rule) {
 596     std::unique_lock<std::mutex> LockGuard(Lock);
 597     Rules.push_back(Rule);
 598   }
 599
 600 private:
 601   void addSourcePathsToRequires() {
 602     llvm::DenseMap<StringRef, StringRef> ModuleSourceMapper;
 603     for (const P1689Rule &R : Rules)
 604       if (R.Provides && !R.Provides->SourcePath.empty())
 605         ModuleSourceMapper[R.Provides->ModuleName] = R.Provides->SourcePath;
 606
 607     for (P1689Rule &R : Rules) {
 608       for (P1689ModuleInfo &Info : R.Requires) {
 609         auto Iter = ModuleSourceMapper.find(Info.ModuleName);
 610         if (Iter != ModuleSourceMapper.end())
 611           Info.SourcePath = Iter->second;
 612       }
 613     }
 614   }
 615
 616   std::mutex Lock;
 617   std::vector<P1689Rule> Rules;
 618 };
 619
 620 static bool
 621 handleP1689DependencyToolResult(const std::string &Input,
 622                                 llvm::Expected<P1689Rule> &MaybeRule,
 623                                 P1689Deps &PD, SharedStream &Errs) {
 624   if (!MaybeRule) {
 625     llvm::handleAllErrors(
 626         MaybeRule.takeError(), [&Input, &Errs](llvm::StringError &Err) {
 627           Errs.applyLocked([&](raw_ostream &OS) {
 628             OS << "Error while scanning dependencies for " << Input << ":\n";
 629             OS << Err.getMessage();
 630           });
 631         });
 632     return true;
 633   }
 634   PD.addRules(*MaybeRule);
 635   return false;
 636 }
 637
 638 /// Construct a path for the explicitly built PCM.
 639 static std::string constructPCMPath(ModuleID MID, StringRef OutputDir) {
 640   SmallString<256> ExplicitPCMPath(OutputDir);
 641   llvm::sys::path::append(ExplicitPCMPath, MID.ContextHash,
 642                           MID.ModuleName + "-" + MID.ContextHash + ".pcm");
 643   return std::string(ExplicitPCMPath);
 644 }
 645
 646 static std::string lookupModuleOutput(const ModuleID &MID, ModuleOutputKind MOK,
 647                                       StringRef OutputDir) {
 648   std::string PCMPath = constructPCMPath(MID, OutputDir);
 649   switch (MOK) {
 650   case ModuleOutputKind::ModuleFile:
 651     return PCMPath;
 652   case ModuleOutputKind::DependencyFile:
 653     return PCMPath + ".d";
 654   case ModuleOutputKind::DependencyTargets:
 655     // Null-separate the list of targets.
 656     return join(ModuleDepTargets, StringRef("\0", 1));
 657   case ModuleOutputKind::DiagnosticSerializationFile:
 658     return PCMPath + ".diag";
 659   }
 660   llvm_unreachable("Fully covered switch above!");
 661 }
 662
 663 static std::string getModuleCachePath(ArrayRef<std::string> Args) {
 664   for (StringRef Arg : llvm::reverse(Args)) {
 665     Arg.consume_front("/clang:");
 666     if (Arg.consume_front("-fmodules-cache-path="))
 667       return std::string(Arg);
 668   }
 669   SmallString<128> Path;
 670   driver::Driver::getDefaultModuleCachePath(Path);
 671   return std::string(Path);
 672 }
 673
 674 // getCompilationDataBase - If -compilation-database is set, load the
 675 // compilation database from the specified file. Otherwise if the we're
 676 // generating P1689 format, trying to generate the compilation database
 677 // form specified command line after the positional parameter "--".
 678 static std::unique_ptr<tooling::CompilationDatabase>
 679 getCompilationDataBase(int argc, char **argv, std::string &ErrorMessage) {
 680   llvm::InitLLVM X(argc, argv);
 681   ParseArgs(argc, argv);
 682
 683   if (!CompilationDB.empty())
 684     return tooling::JSONCompilationDatabase::loadFromFile(
 685         CompilationDB, ErrorMessage,
 686         tooling::JSONCommandLineSyntax::AutoDetect);
 687
 688   if (Format != ScanningOutputFormat::P1689) {
 689     llvm::errs() << "the --compilation-database option: must be specified at "
 690                     "least once!";
 691     return nullptr;
 692   }
 693
 694   // Trying to get the input file, the output file and the command line options
 695   // from the positional parameter "--".
 696   char **DoubleDash = std::find(argv, argv + argc, StringRef("--"));
 697   if (DoubleDash == argv + argc) {
 698     llvm::errs() << "The command line arguments is required after '--' in "
 699                     "P1689 per file mode.";
 700     return nullptr;
 701   }
 702
 703   llvm::IntrusiveRefCntPtr<DiagnosticsEngine> Diags =
 704       CompilerInstance::createDiagnostics(new DiagnosticOptions);
 705   driver::Driver TheDriver(CommandLine[0], llvm::sys::getDefaultTargetTriple(),
 706                            *Diags);
 707   std::unique_ptr<driver::Compilation> C(
 708       TheDriver.BuildCompilation(CommandLine));
 709   if (!C)
 710     return nullptr;
 711
 712   auto Cmd = C->getJobs().begin();
 713   auto CI = std::make_unique<CompilerInvocation>();
 714   CompilerInvocation::CreateFromArgs(*CI, Cmd->getArguments(), *Diags,
 715                                      CommandLine[0]);
 716   if (!CI)
 717     return nullptr;
 718
 719   FrontendOptions &FEOpts = CI->getFrontendOpts();
 720   if (FEOpts.Inputs.size() != 1) {
 721     llvm::errs() << "Only one input file is allowed in P1689 per file mode.";
 722     return nullptr;
 723   }
 724
 725   // There might be multiple jobs for a compilation. Extract the specified
 726   // output filename from the last job.
 727   auto LastCmd = C->getJobs().end();
 728   LastCmd--;
 729   if (LastCmd->getOutputFilenames().size() != 1) {
 730     llvm::errs() << "The command line should provide exactly one output file "
 731                     "in P1689 per file mode.\n";
 732   }
 733   StringRef OutputFile = LastCmd->getOutputFilenames().front();
 734
 735   class InplaceCompilationDatabase : public tooling::CompilationDatabase {
 736   public:
 737     InplaceCompilationDatabase(StringRef InputFile, StringRef OutputFile,
 738                                ArrayRef<const char *> CommandLine)
 739         : Command(".", InputFile, {}, OutputFile) {
 740       for (auto *C : CommandLine)
 741         Command.CommandLine.push_back(C);
 742     }
 743
 744     std::vector<tooling::CompileCommand>
 745     getCompileCommands(StringRef FilePath) const override {
 746       if (FilePath != Command.Filename)
 747         return {};
 748       return {Command};
 749     }
 750
 751     std::vector<std::string> getAllFiles() const override {
 752       return {Command.Filename};
 753     }
 754
 755     std::vector<tooling::CompileCommand>
 756     getAllCompileCommands() const override {
 757       return {Command};
 758     }
 759
 760   private:
 761     tooling::CompileCommand Command;
 762   };
 763
 764   return std::make_unique<InplaceCompilationDatabase>(
 765       FEOpts.Inputs[0].getFile(), OutputFile, CommandLine);
 766 }
 767
 768 int clang_scan_deps_main(int argc, char **argv, const llvm::ToolContext &) {
 769   std::string ErrorMessage;
 770   std::unique_ptr<tooling::CompilationDatabase> Compilations =
 771       getCompilationDataBase(argc, argv, ErrorMessage);
 772   if (!Compilations) {
 773     llvm::errs() << ErrorMessage << "\n";
 774     return 1;
 775   }
 776
 777   llvm::cl::PrintOptionValues();
 778
 779   // The command options are rewritten to run Clang in preprocessor only mode.
 780   auto AdjustingCompilations =
 781       std::make_unique<tooling::ArgumentsAdjustingCompilations>(
 782           std::move(Compilations));
 783   ResourceDirectoryCache ResourceDirCache;
 784
 785   AdjustingCompilations->appendArgumentsAdjuster(
 786       [&ResourceDirCache](const tooling::CommandLineArguments &Args,
 787                           StringRef FileName) {
 788         std::string LastO;
 789         bool HasResourceDir = false;
 790         bool ClangCLMode = false;
 791         auto FlagsEnd = llvm::find(Args, "--");
 792         if (FlagsEnd != Args.begin()) {
 793           ClangCLMode =
 794               llvm::sys::path::stem(Args[0]).contains_insensitive("clang-cl") ||
 795               llvm::is_contained(Args, "--driver-mode=cl");
 796
 797           // Reverse scan, starting at the end or at the element before "--".
 798           auto R = std::make_reverse_iterator(FlagsEnd);
 799           for (auto I = R, E = Args.rend(); I != E; ++I) {
 800             StringRef Arg = *I;
 801             if (ClangCLMode) {
 802               // Ignore arguments that are preceded by "-Xclang".
 803               if ((I + 1) != E && I[1] == "-Xclang")
 804                 continue;
 805               if (LastO.empty()) {
 806                 // With clang-cl, the output obj file can be specified with
 807                 // "/opath", "/o path", "/Fopath", and the dash counterparts.
 808                 // Also, clang-cl adds ".obj" extension if none is found.
 809                 if ((Arg == "-o" || Arg == "/o") && I != R)
 810                   LastO = I[-1]; // Next argument (reverse iterator)
 811                 else if (Arg.startswith("/Fo") || Arg.startswith("-Fo"))
 812                   LastO = Arg.drop_front(3).str();
 813                 else if (Arg.startswith("/o") || Arg.startswith("-o"))
 814                   LastO = Arg.drop_front(2).str();
 815
 816                 if (!LastO.empty() && !llvm::sys::path::has_extension(LastO))
 817                   LastO.append(".obj");
 818               }
 819             }
 820             if (Arg == "-resource-dir")
 821               HasResourceDir = true;
 822           }
 823         }
 824         tooling::CommandLineArguments AdjustedArgs(Args.begin(), FlagsEnd);
 825         // The clang-cl driver passes "-o -" to the frontend. Inject the real
 826         // file here to ensure "-MT" can be deduced if need be.
 827         if (ClangCLMode && !LastO.empty()) {
 828           AdjustedArgs.push_back("/clang:-o");
 829           AdjustedArgs.push_back("/clang:" + LastO);
 830         }
 831
 832         if (!HasResourceDir && ResourceDirRecipe == RDRK_InvokeCompiler) {
 833           StringRef ResourceDir =
 834               ResourceDirCache.findResourceDir(Args, ClangCLMode);
 835           if (!ResourceDir.empty()) {
 836             AdjustedArgs.push_back("-resource-dir");
 837             AdjustedArgs.push_back(std::string(ResourceDir));
 838           }
 839         }
 840         AdjustedArgs.insert(AdjustedArgs.end(), FlagsEnd, Args.end());
 841         return AdjustedArgs;
 842       });
 843
 844   SharedStream Errs(llvm::errs());
 845   // Print out the dependency results to STDOUT by default.
 846   SharedStream DependencyOS(llvm::outs());
 847
 848   DependencyScanningService Service(ScanMode, Format, OptimizeArgs,
 849                                     EagerLoadModules);
 850   llvm::ThreadPool Pool(llvm::hardware_concurrency(NumThreads));
 851   std::vector<std::unique_ptr<DependencyScanningTool>> WorkerTools;
 852   for (unsigned I = 0; I < Pool.getThreadCount(); ++I)
 853     WorkerTools.push_back(std::make_unique<DependencyScanningTool>(Service));
 854
 855   std::vector<tooling::CompileCommand> Inputs =
 856       AdjustingCompilations->getAllCompileCommands();
 857
 858   std::atomic<bool> HadErrors(false);
 859   std::optional<FullDeps> FD;
 860   P1689Deps PD;
 861
 862   std::mutex Lock;
 863   size_t Index = 0;
 864   auto GetNextInputIndex = [&]() -> std::optional<size_t> {
 865     std::unique_lock<std::mutex> LockGuard(Lock);
 866     if (Index < Inputs.size())
 867       return Index++;
 868     return {};
 869   };
 870
 871   if (Format == ScanningOutputFormat::Full)
 872     FD.emplace(ModuleName.empty() ? Inputs.size() : 0);
 873
 874   if (Verbose) {
 875     llvm::outs() << "Running clang-scan-deps on " << Inputs.size()
 876                  << " files using " << Pool.getThreadCount() << " workers\n";
 877   }
 878
 879   llvm::Timer T;
 880   T.startTimer();
 881
 882   for (unsigned I = 0; I < Pool.getThreadCount(); ++I) {
 883     Pool.async([&, I]() {
 884       llvm::DenseSet<ModuleID> AlreadySeenModules;
 885       while (auto MaybeInputIndex = GetNextInputIndex()) {
 886         size_t LocalIndex = *MaybeInputIndex;
 887         const tooling::CompileCommand *Input = &Inputs[LocalIndex];
 888         std::string Filename = std::move(Input->Filename);
 889         std::string CWD = std::move(Input->Directory);
 890
 891         std::optional<StringRef> MaybeModuleName;
 892         if (!ModuleName.empty())
 893           MaybeModuleName = ModuleName;
 894
 895         std::string OutputDir(ModuleFilesDir);
 896         if (OutputDir.empty())
 897           OutputDir = getModuleCachePath(Input->CommandLine);
 898         auto LookupOutput = [&](const ModuleID &MID, ModuleOutputKind MOK) {
 899           return ::lookupModuleOutput(MID, MOK, OutputDir);
 900         };
 901
 902         // Run the tool on it.
 903         if (Format == ScanningOutputFormat::Make) {
 904           auto MaybeFile =
 905               WorkerTools[I]->getDependencyFile(Input->CommandLine, CWD);
 906           if (handleMakeDependencyToolResult(Filename, MaybeFile, DependencyOS,
 907                                              Errs))
 908             HadErrors = true;
 909         } else if (Format == ScanningOutputFormat::P1689) {
 910           // It is useful to generate the make-format dependency output during
 911           // the scanning for P1689. Otherwise the users need to scan again for
 912           // it. We will generate the make-format dependency output if we find
 913           // `-MF` in the command lines.
 914           std::string MakeformatOutputPath;
 915           std::string MakeformatOutput;
 916
 917           auto MaybeRule = WorkerTools[I]->getP1689ModuleDependencyFile(
 918               *Input, CWD, MakeformatOutput, MakeformatOutputPath);
 919
 920           if (handleP1689DependencyToolResult(Filename, MaybeRule, PD, Errs))
 921             HadErrors = true;
 922
 923           if (!MakeformatOutputPath.empty() && !MakeformatOutput.empty() &&
 924               !HadErrors) {
 925             static std::mutex Lock;
 926             // With compilation database, we may open different files
 927             // concurrently or we may write the same file concurrently. So we
 928             // use a map here to allow multiple compile commands to write to the
 929             // same file. Also we need a lock here to avoid data race.
 930             static llvm::StringMap<llvm::raw_fd_ostream> OSs;
 931             std::unique_lock<std::mutex> LockGuard(Lock);
 932
 933             auto OSIter = OSs.find(MakeformatOutputPath);
 934             if (OSIter == OSs.end()) {
 935               std::error_code EC;
 936               OSIter = OSs.try_emplace(MakeformatOutputPath,
 937                                        MakeformatOutputPath, EC)
 938                            .first;
 939               if (EC)
 940                 llvm::errs()
 941                     << "Failed to open P1689 make format output file \""
 942                     << MakeformatOutputPath << "\" for " << EC.message()
 943                     << "\n";
 944             }
 945
 946             SharedStream MakeformatOS(OSIter->second);
 947             llvm::Expected<std::string> MaybeOutput(MakeformatOutput);
 948             if (handleMakeDependencyToolResult(Filename, MaybeOutput,
 949                                                MakeformatOS, Errs))
 950               HadErrors = true;
 951           }
 952         } else if (MaybeModuleName) {
 953           auto MaybeModuleDepsGraph = WorkerTools[I]->getModuleDependencies(
 954               *MaybeModuleName, Input->CommandLine, CWD, AlreadySeenModules,
 955               LookupOutput);
 956           if (handleModuleResult(*MaybeModuleName, MaybeModuleDepsGraph, *FD,
 957                                  LocalIndex, DependencyOS, Errs))
 958             HadErrors = true;
 959         } else {
 960           auto MaybeTUDeps = WorkerTools[I]->getTranslationUnitDependencies(
 961               Input->CommandLine, CWD, AlreadySeenModules, LookupOutput);
 962           if (handleTranslationUnitResult(Filename, MaybeTUDeps, *FD,
 963                                           LocalIndex, DependencyOS, Errs))
 964             HadErrors = true;
 965         }
 966       }
 967     });
 968   }
 969   Pool.wait();
 970
 971   T.stopTimer();
 972   if (PrintTiming)
 973     llvm::errs() << llvm::format(
 974         "clang-scan-deps timing: %0.2fs wall, %0.2fs process\n",
 975         T.getTotalTime().getWallTime(), T.getTotalTime().getProcessTime());
 976
 977   if (RoundTripArgs)
 978     if (FD && FD->roundTripCommands(llvm::errs()))
 979       HadErrors = true;
 980
 981   if (Format == ScanningOutputFormat::Full)
 982     FD->printFullOutput(llvm::outs());
 983   else if (Format == ScanningOutputFormat::P1689)
 984     PD.printDependencies(llvm::outs());
 985
 986   return HadErrors;
 987 }