clang/tools/clang-scan-deps/ClangScanDeps.cpp

   1 //===- ClangScanDeps.cpp - Implementation of clang-scan-deps --------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "clang/Driver/Compilation.h"
  10 #include "clang/Driver/Driver.h"
  11 #include "clang/Frontend/CompilerInstance.h"
  12 #include "clang/Frontend/TextDiagnosticPrinter.h"
  13 #include "clang/Tooling/CommonOptionsParser.h"
  14 #include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
  15 #include "clang/Tooling/DependencyScanning/DependencyScanningTool.h"
  16 #include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h"
  17 #include "clang/Tooling/JSONCompilationDatabase.h"
  18 #include "llvm/ADT/STLExtras.h"
  19 #include "llvm/ADT/Twine.h"
  20 #include "llvm/Support/CommandLine.h"
  21 #include "llvm/Support/FileUtilities.h"
  22 #include "llvm/Support/Format.h"
  23 #include "llvm/Support/InitLLVM.h"
  24 #include "llvm/Support/JSON.h"
  25 #include "llvm/Support/LLVMDriver.h"
  26 #include "llvm/Support/Program.h"
  27 #include "llvm/Support/Signals.h"
  28 #include "llvm/Support/ThreadPool.h"
  29 #include "llvm/Support/Threading.h"
  30 #include "llvm/Support/Timer.h"
  31 #include "llvm/TargetParser/Host.h"
  32 #include <mutex>
  33 #include <optional>
  34 #include <thread>
  35
  36 #include "Opts.inc"
  37
  38 using namespace clang;
  39 using namespace tooling::dependencies;
  40
  41 namespace {
  42
  43 using namespace llvm::opt;
  44 enum ID {
  45   OPT_INVALID = 0, // This is not an option ID.
  46 #define OPTION(...) LLVM_MAKE_OPT_ID(__VA_ARGS__),
  47 #include "Opts.inc"
  48 #undef OPTION
  49 };
  50
  51 #define PREFIX(NAME, VALUE)                                                    \
  52   constexpr llvm::StringLiteral NAME##_init[] = VALUE;                         \
  53   constexpr llvm::ArrayRef<llvm::StringLiteral> NAME(                          \
  54       NAME##_init, std::size(NAME##_init) - 1);
  55 #include "Opts.inc"
  56 #undef PREFIX
  57
  58 const llvm::opt::OptTable::Info InfoTable[] = {
  59 #define OPTION(...) LLVM_CONSTRUCT_OPT_INFO(__VA_ARGS__),
  60 #include "Opts.inc"
  61 #undef OPTION
  62 };
  63
  64 class ScanDepsOptTable : public llvm::opt::GenericOptTable {
  65 public:
  66   ScanDepsOptTable() : GenericOptTable(InfoTable) {
  67     setGroupedShortOptions(true);
  68   }
  69 };
  70
  71 enum ResourceDirRecipeKind {
  72   RDRK_ModifyCompilerPath,
  73   RDRK_InvokeCompiler,
  74 };
  75
  76 static ScanningMode ScanMode = ScanningMode::DependencyDirectivesScan;
  77 static ScanningOutputFormat Format = ScanningOutputFormat::Make;
  78 static ScanningOptimizations OptimizeArgs;
  79 static std::string ModuleFilesDir;
  80 static bool EagerLoadModules;
  81 static unsigned NumThreads = 0;
  82 static std::string CompilationDB;
  83 static std::string ModuleName;
  84 static std::vector<std::string> ModuleDepTargets;
  85 static bool DeprecatedDriverCommand;
  86 static ResourceDirRecipeKind ResourceDirRecipe;
  87 static bool Verbose;
  88 static bool PrintTiming;
  89 static std::vector<const char *> CommandLine;
  90
  91 #ifndef NDEBUG
  92 static constexpr bool DoRoundTripDefault = true;
  93 #else
  94 static constexpr bool DoRoundTripDefault = false;
  95 #endif
  96
  97 static bool RoundTripArgs = DoRoundTripDefault;
  98
  99 static void ParseArgs(int argc, char **argv) {
 100   ScanDepsOptTable Tbl;
 101   llvm::StringRef ToolName = argv[0];
 102   llvm::BumpPtrAllocator A;
 103   llvm::StringSaver Saver{A};
 104   llvm::opt::InputArgList Args =
 105       Tbl.parseArgs(argc, argv, OPT_UNKNOWN, Saver, [&](StringRef Msg) {
 106         llvm::errs() << Msg << '\n';
 107         std::exit(1);
 108       });
 109
 110   if (Args.hasArg(OPT_help)) {
 111     Tbl.printHelp(llvm::outs(), "clang-scan-deps [options]", "clang-scan-deps");
 112     std::exit(0);
 113   }
 114   if (Args.hasArg(OPT_version)) {
 115     llvm::outs() << ToolName << '\n';
 116     llvm::cl::PrintVersionMessage();
 117     std::exit(0);
 118   }
 119   if (const llvm::opt::Arg *A = Args.getLastArg(OPT_mode_EQ)) {
 120     auto ModeType =
 121         llvm::StringSwitch<std::optional<ScanningMode>>(A->getValue())
 122             .Case("preprocess-dependency-directives",
 123                   ScanningMode::DependencyDirectivesScan)
 124             .Case("preprocess", ScanningMode::CanonicalPreprocessing)
 125             .Default(std::nullopt);
 126     if (!ModeType) {
 127       llvm::errs() << ToolName
 128                    << ": for the --mode option: Cannot find option named '"
 129                    << A->getValue() << "'\n";
 130       std::exit(1);
 131     }
 132     ScanMode = *ModeType;
 133   }
 134
 135   if (const llvm::opt::Arg *A = Args.getLastArg(OPT_format_EQ)) {
 136     auto FormatType =
 137         llvm::StringSwitch<std::optional<ScanningOutputFormat>>(A->getValue())
 138             .Case("make", ScanningOutputFormat::Make)
 139             .Case("p1689", ScanningOutputFormat::P1689)
 140             .Case("experimental-full", ScanningOutputFormat::Full)
 141             .Default(std::nullopt);
 142     if (!FormatType) {
 143       llvm::errs() << ToolName
 144                    << ": for the --format option: Cannot find option named '"
 145                    << A->getValue() << "'\n";
 146       std::exit(1);
 147     }
 148     Format = *FormatType;
 149   }
 150
 151   std::vector<std::string> OptimizationFlags =
 152       Args.getAllArgValues(OPT_optimize_args_EQ);
 153   OptimizeArgs = ScanningOptimizations::None;
 154   for (const auto &Arg : OptimizationFlags) {
 155     auto Optimization =
 156         llvm::StringSwitch<std::optional<ScanningOptimizations>>(Arg)
 157             .Case("none", ScanningOptimizations::None)
 158             .Case("header-search", ScanningOptimizations::HeaderSearch)
 159             .Case("system-warnings", ScanningOptimizations::SystemWarnings)
 160             .Case("all", ScanningOptimizations::All)
 161             .Default(std::nullopt);
 162     if (!Optimization) {
 163       llvm::errs()
 164           << ToolName
 165           << ": for the --optimize-args option: Cannot find option named '"
 166           << Arg << "'\n";
 167       std::exit(1);
 168     }
 169     OptimizeArgs |= *Optimization;
 170   }
 171   if (OptimizationFlags.empty())
 172     OptimizeArgs = ScanningOptimizations::Default;
 173
 174   if (const llvm::opt::Arg *A = Args.getLastArg(OPT_module_files_dir_EQ))
 175     ModuleFilesDir = A->getValue();
 176
 177   EagerLoadModules = Args.hasArg(OPT_eager_load_pcm);
 178
 179   if (const llvm::opt::Arg *A = Args.getLastArg(OPT_j)) {
 180     StringRef S{A->getValue()};
 181     if (!llvm::to_integer(S, NumThreads, 0)) {
 182       llvm::errs() << ToolName << ": for the -j option: '" << S
 183                    << "' value invalid for uint argument!\n";
 184       std::exit(1);
 185     }
 186   }
 187
 188   if (const llvm::opt::Arg *A = Args.getLastArg(OPT_compilation_database_EQ)) {
 189     CompilationDB = A->getValue();
 190   } else if (Format != ScanningOutputFormat::P1689) {
 191     llvm::errs() << ToolName
 192                  << ": for the --compiilation-database option: must be "
 193                     "specified at least once!";
 194     std::exit(1);
 195   }
 196
 197   if (const llvm::opt::Arg *A = Args.getLastArg(OPT_module_name_EQ))
 198     ModuleName = A->getValue();
 199
 200   for (const llvm::opt::Arg *A : Args.filtered(OPT_dependency_target_EQ))
 201     ModuleDepTargets.emplace_back(A->getValue());
 202
 203   DeprecatedDriverCommand = Args.hasArg(OPT_deprecated_driver_command);
 204
 205   if (const llvm::opt::Arg *A = Args.getLastArg(OPT_resource_dir_recipe_EQ)) {
 206     auto Kind =
 207         llvm::StringSwitch<std::optional<ResourceDirRecipeKind>>(A->getValue())
 208             .Case("modify-compiler-path", RDRK_ModifyCompilerPath)
 209             .Case("invoke-compiler", RDRK_InvokeCompiler)
 210             .Default(std::nullopt);
 211     if (!Kind) {
 212       llvm::errs() << ToolName
 213                    << ": for the --resource-dir-recipe option: Cannot find "
 214                       "option named '"
 215                    << A->getValue() << "'\n";
 216       std::exit(1);
 217     }
 218     ResourceDirRecipe = *Kind;
 219   }
 220
 221   PrintTiming = Args.hasArg(OPT_print_timing);
 222
 223   Verbose = Args.hasArg(OPT_verbose);
 224
 225   RoundTripArgs = Args.hasArg(OPT_round_trip_args);
 226
 227   if (auto *A = Args.getLastArgNoClaim(OPT_DASH_DASH))
 228     CommandLine.insert(CommandLine.end(), A->getValues().begin(),
 229                        A->getValues().end());
 230 }
 231
 232 class SharedStream {
 233 public:
 234   SharedStream(raw_ostream &OS) : OS(OS) {}
 235   void applyLocked(llvm::function_ref<void(raw_ostream &OS)> Fn) {
 236     std::unique_lock<std::mutex> LockGuard(Lock);
 237     Fn(OS);
 238     OS.flush();
 239   }
 240
 241 private:
 242   std::mutex Lock;
 243   raw_ostream &OS;
 244 };
 245
 246 class ResourceDirectoryCache {
 247 public:
 248   /// findResourceDir finds the resource directory relative to the clang
 249   /// compiler being used in Args, by running it with "-print-resource-dir"
 250   /// option and cache the results for reuse. \returns resource directory path
 251   /// associated with the given invocation command or empty string if the
 252   /// compiler path is NOT an absolute path.
 253   StringRef findResourceDir(const tooling::CommandLineArguments &Args,
 254                             bool ClangCLMode) {
 255     if (Args.size() < 1)
 256       return "";
 257
 258     const std::string &ClangBinaryPath = Args[0];
 259     if (!llvm::sys::path::is_absolute(ClangBinaryPath))
 260       return "";
 261
 262     const std::string &ClangBinaryName =
 263         std::string(llvm::sys::path::filename(ClangBinaryPath));
 264
 265     std::unique_lock<std::mutex> LockGuard(CacheLock);
 266     const auto &CachedResourceDir = Cache.find(ClangBinaryPath);
 267     if (CachedResourceDir != Cache.end())
 268       return CachedResourceDir->second;
 269
 270     std::vector<StringRef> PrintResourceDirArgs{ClangBinaryName};
 271     if (ClangCLMode)
 272       PrintResourceDirArgs.push_back("/clang:-print-resource-dir");
 273     else
 274       PrintResourceDirArgs.push_back("-print-resource-dir");
 275
 276     llvm::SmallString<64> OutputFile, ErrorFile;
 277     llvm::sys::fs::createTemporaryFile("print-resource-dir-output",
 278                                        "" /*no-suffix*/, OutputFile);
 279     llvm::sys::fs::createTemporaryFile("print-resource-dir-error",
 280                                        "" /*no-suffix*/, ErrorFile);
 281     llvm::FileRemover OutputRemover(OutputFile.c_str());
 282     llvm::FileRemover ErrorRemover(ErrorFile.c_str());
 283     std::optional<StringRef> Redirects[] = {
 284         {""}, // Stdin
 285         OutputFile.str(),
 286         ErrorFile.str(),
 287     };
 288     if (llvm::sys::ExecuteAndWait(ClangBinaryPath, PrintResourceDirArgs, {},
 289                                   Redirects)) {
 290       auto ErrorBuf = llvm::MemoryBuffer::getFile(ErrorFile.c_str());
 291       llvm::errs() << ErrorBuf.get()->getBuffer();
 292       return "";
 293     }
 294
 295     auto OutputBuf = llvm::MemoryBuffer::getFile(OutputFile.c_str());
 296     if (!OutputBuf)
 297       return "";
 298     StringRef Output = OutputBuf.get()->getBuffer().rtrim('\n');
 299
 300     Cache[ClangBinaryPath] = Output.str();
 301     return Cache[ClangBinaryPath];
 302   }
 303
 304 private:
 305   std::map<std::string, std::string> Cache;
 306   std::mutex CacheLock;
 307 };
 308
 309 } // end anonymous namespace
 310
 311 /// Takes the result of a dependency scan and prints error / dependency files
 312 /// based on the result.
 313 ///
 314 /// \returns True on error.
 315 static bool
 316 handleMakeDependencyToolResult(const std::string &Input,
 317                                llvm::Expected<std::string> &MaybeFile,
 318                                SharedStream &OS, SharedStream &Errs) {
 319   if (!MaybeFile) {
 320     llvm::handleAllErrors(
 321         MaybeFile.takeError(), [&Input, &Errs](llvm::StringError &Err) {
 322           Errs.applyLocked([&](raw_ostream &OS) {
 323             OS << "Error while scanning dependencies for " << Input << ":\n";
 324             OS << Err.getMessage();
 325           });
 326         });
 327     return true;
 328   }
 329   OS.applyLocked([&](raw_ostream &OS) { OS << *MaybeFile; });
 330   return false;
 331 }
 332
 333 static llvm::json::Array toJSONSorted(const llvm::StringSet<> &Set) {
 334   std::vector<llvm::StringRef> Strings;
 335   for (auto &&I : Set)
 336     Strings.push_back(I.getKey());
 337   llvm::sort(Strings);
 338   return llvm::json::Array(Strings);
 339 }
 340
 341 // Technically, we don't need to sort the dependency list to get determinism.
 342 // Leaving these be will simply preserve the import order.
 343 static llvm::json::Array toJSONSorted(std::vector<ModuleID> V) {
 344   llvm::sort(V);
 345
 346   llvm::json::Array Ret;
 347   for (const ModuleID &MID : V)
 348     Ret.push_back(llvm::json::Object(
 349         {{"module-name", MID.ModuleName}, {"context-hash", MID.ContextHash}}));
 350   return Ret;
 351 }
 352
 353 // Thread safe.
 354 class FullDeps {
 355 public:
 356   FullDeps(size_t NumInputs) : Inputs(NumInputs) {}
 357
 358   void mergeDeps(StringRef Input, TranslationUnitDeps TUDeps,
 359                  size_t InputIndex) {
 360     mergeDeps(std::move(TUDeps.ModuleGraph), InputIndex);
 361
 362     InputDeps ID;
 363     ID.FileName = std::string(Input);
 364     ID.ContextHash = std::move(TUDeps.ID.ContextHash);
 365     ID.FileDeps = std::move(TUDeps.FileDeps);
 366     ID.ModuleDeps = std::move(TUDeps.ClangModuleDeps);
 367     ID.DriverCommandLine = std::move(TUDeps.DriverCommandLine);
 368     ID.Commands = std::move(TUDeps.Commands);
 369
 370     assert(InputIndex < Inputs.size() && "Input index out of bounds");
 371     assert(Inputs[InputIndex].FileName.empty() && "Result already populated");
 372     Inputs[InputIndex] = std::move(ID);
 373   }
 374
 375   void mergeDeps(ModuleDepsGraph Graph, size_t InputIndex) {
 376     std::vector<ModuleDeps *> NewMDs;
 377     {
 378       std::unique_lock<std::mutex> ul(Lock);
 379       for (const ModuleDeps &MD : Graph) {
 380         auto I = Modules.find({MD.ID, 0});
 381         if (I != Modules.end()) {
 382           I->first.InputIndex = std::min(I->first.InputIndex, InputIndex);
 383           continue;
 384         }
 385         auto Res = Modules.insert(I, {{MD.ID, InputIndex}, std::move(MD)});
 386         NewMDs.push_back(&Res->second);
 387       }
 388       // First call to \c getBuildArguments is somewhat expensive. Let's call it
 389       // on the current thread (instead of the main one), and outside the
 390       // critical section.
 391       for (ModuleDeps *MD : NewMDs)
 392         (void)MD->getBuildArguments();
 393     }
 394   }
 395
 396   bool roundTripCommand(ArrayRef<std::string> ArgStrs,
 397                         DiagnosticsEngine &Diags) {
 398     if (ArgStrs.empty() || ArgStrs[0] != "-cc1")
 399       return false;
 400     SmallVector<const char *> Args;
 401     for (const std::string &Arg : ArgStrs)
 402       Args.push_back(Arg.c_str());
 403     return !CompilerInvocation::checkCC1RoundTrip(Args, Diags);
 404   }
 405
 406   // Returns \c true if any command lines fail to round-trip. We expect
 407   // commands already be canonical when output by the scanner.
 408   bool roundTripCommands(raw_ostream &ErrOS) {
 409     IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts = new DiagnosticOptions{};
 410     TextDiagnosticPrinter DiagConsumer(ErrOS, &*DiagOpts);
 411     IntrusiveRefCntPtr<DiagnosticsEngine> Diags =
 412         CompilerInstance::createDiagnostics(&*DiagOpts, &DiagConsumer,
 413                                             /*ShouldOwnClient=*/false);
 414
 415     for (auto &&M : Modules)
 416       if (roundTripCommand(M.second.getBuildArguments(), *Diags))
 417         return true;
 418
 419     for (auto &&I : Inputs)
 420       for (const auto &Cmd : I.Commands)
 421         if (roundTripCommand(Cmd.Arguments, *Diags))
 422           return true;
 423
 424     return false;
 425   }
 426
 427   void printFullOutput(raw_ostream &OS) {
 428     // Sort the modules by name to get a deterministic order.
 429     std::vector<IndexedModuleID> ModuleIDs;
 430     for (auto &&M : Modules)
 431       ModuleIDs.push_back(M.first);
 432     llvm::sort(ModuleIDs);
 433
 434     using namespace llvm::json;
 435
 436     Array OutModules;
 437     for (auto &&ModID : ModuleIDs) {
 438       auto &MD = Modules[ModID];
 439       Object O{
 440           {"name", MD.ID.ModuleName},
 441           {"context-hash", MD.ID.ContextHash},
 442           {"file-deps", toJSONSorted(MD.FileDeps)},
 443           {"clang-module-deps", toJSONSorted(MD.ClangModuleDeps)},
 444           {"clang-modulemap-file", MD.ClangModuleMapFile},
 445           {"command-line", MD.getBuildArguments()},
 446       };
 447       OutModules.push_back(std::move(O));
 448     }
 449
 450     Array TUs;
 451     for (auto &&I : Inputs) {
 452       Array Commands;
 453       if (I.DriverCommandLine.empty()) {
 454         for (const auto &Cmd : I.Commands) {
 455           Object O{
 456               {"input-file", I.FileName},
 457               {"clang-context-hash", I.ContextHash},
 458               {"file-deps", I.FileDeps},
 459               {"clang-module-deps", toJSONSorted(I.ModuleDeps)},
 460               {"executable", Cmd.Executable},
 461               {"command-line", Cmd.Arguments},
 462           };
 463           Commands.push_back(std::move(O));
 464         }
 465       } else {
 466         Object O{
 467             {"input-file", I.FileName},
 468             {"clang-context-hash", I.ContextHash},
 469             {"file-deps", I.FileDeps},
 470             {"clang-module-deps", toJSONSorted(I.ModuleDeps)},
 471             {"executable", "clang"},
 472             {"command-line", I.DriverCommandLine},
 473         };
 474         Commands.push_back(std::move(O));
 475       }
 476       TUs.push_back(Object{
 477           {"commands", std::move(Commands)},
 478       });
 479     }
 480
 481     Object Output{
 482         {"modules", std::move(OutModules)},
 483         {"translation-units", std::move(TUs)},
 484     };
 485
 486     OS << llvm::formatv("{0:2}\n", Value(std::move(Output)));
 487   }
 488
 489 private:
 490   struct IndexedModuleID {
 491     ModuleID ID;
 492
 493     // FIXME: This is mutable so that it can still be updated after insertion
 494     //  into an unordered associative container. This is "fine", since this
 495     //  field doesn't contribute to the hash, but it's a brittle hack.
 496     mutable size_t InputIndex;
 497
 498     bool operator==(const IndexedModuleID &Other) const {
 499       return ID == Other.ID;
 500     }
 501
 502     bool operator<(const IndexedModuleID &Other) const {
 503       /// We need the output of clang-scan-deps to be deterministic. However,
 504       /// the dependency graph may contain two modules with the same name. How
 505       /// do we decide which one to print first? If we made that decision based
 506       /// on the context hash, the ordering would be deterministic, but
 507       /// different across machines. This can happen for example when the inputs
 508       /// or the SDKs (which both contribute to the "context" hash) live in
 509       /// different absolute locations. We solve that by tracking the index of
 510       /// the first input TU that (transitively) imports the dependency, which
 511       /// is always the same for the same input, resulting in deterministic
 512       /// sorting that's also reproducible across machines.
 513       return std::tie(ID.ModuleName, InputIndex) <
 514              std::tie(Other.ID.ModuleName, Other.InputIndex);
 515     }
 516
 517     struct Hasher {
 518       std::size_t operator()(const IndexedModuleID &IMID) const {
 519         return llvm::hash_value(IMID.ID);
 520       }
 521     };
 522   };
 523
 524   struct InputDeps {
 525     std::string FileName;
 526     std::string ContextHash;
 527     std::vector<std::string> FileDeps;
 528     std::vector<ModuleID> ModuleDeps;
 529     std::vector<std::string> DriverCommandLine;
 530     std::vector<Command> Commands;
 531   };
 532
 533   std::mutex Lock;
 534   std::unordered_map<IndexedModuleID, ModuleDeps, IndexedModuleID::Hasher>
 535       Modules;
 536   std::vector<InputDeps> Inputs;
 537 };
 538
 539 static bool handleTranslationUnitResult(
 540     StringRef Input, llvm::Expected<TranslationUnitDeps> &MaybeTUDeps,
 541     FullDeps &FD, size_t InputIndex, SharedStream &OS, SharedStream &Errs) {
 542   if (!MaybeTUDeps) {
 543     llvm::handleAllErrors(
 544         MaybeTUDeps.takeError(), [&Input, &Errs](llvm::StringError &Err) {
 545           Errs.applyLocked([&](raw_ostream &OS) {
 546             OS << "Error while scanning dependencies for " << Input << ":\n";
 547             OS << Err.getMessage();
 548           });
 549         });
 550     return true;
 551   }
 552   FD.mergeDeps(Input, std::move(*MaybeTUDeps), InputIndex);
 553   return false;
 554 }
 555
 556 static bool handleModuleResult(
 557     StringRef ModuleName, llvm::Expected<ModuleDepsGraph> &MaybeModuleGraph,
 558     FullDeps &FD, size_t InputIndex, SharedStream &OS, SharedStream &Errs) {
 559   if (!MaybeModuleGraph) {
 560     llvm::handleAllErrors(MaybeModuleGraph.takeError(),
 561                           [&ModuleName, &Errs](llvm::StringError &Err) {
 562                             Errs.applyLocked([&](raw_ostream &OS) {
 563                               OS << "Error while scanning dependencies for "
 564                                  << ModuleName << ":\n";
 565                               OS << Err.getMessage();
 566                             });
 567                           });
 568     return true;
 569   }
 570   FD.mergeDeps(std::move(*MaybeModuleGraph), InputIndex);
 571   return false;
 572 }
 573
 574 class P1689Deps {
 575 public:
 576   void printDependencies(raw_ostream &OS) {
 577     addSourcePathsToRequires();
 578     // Sort the modules by name to get a deterministic order.
 579     llvm::sort(Rules, [](const P1689Rule &A, const P1689Rule &B) {
 580       return A.PrimaryOutput < B.PrimaryOutput;
 581     });
 582
 583     using namespace llvm::json;
 584     Array OutputRules;
 585     for (const P1689Rule &R : Rules) {
 586       Object O{{"primary-output", R.PrimaryOutput}};
 587
 588       if (R.Provides) {
 589         Array Provides;
 590         Object Provided{{"logical-name", R.Provides->ModuleName},
 591                         {"source-path", R.Provides->SourcePath},
 592                         {"is-interface", R.Provides->IsStdCXXModuleInterface}};
 593         Provides.push_back(std::move(Provided));
 594         O.insert({"provides", std::move(Provides)});
 595       }
 596
 597       Array Requires;
 598       for (const P1689ModuleInfo &Info : R.Requires) {
 599         Object RequiredInfo{{"logical-name", Info.ModuleName}};
 600         if (!Info.SourcePath.empty())
 601           RequiredInfo.insert({"source-path", Info.SourcePath});
 602         Requires.push_back(std::move(RequiredInfo));
 603       }
 604
 605       if (!Requires.empty())
 606         O.insert({"requires", std::move(Requires)});
 607
 608       OutputRules.push_back(std::move(O));
 609     }
 610
 611     Object Output{
 612         {"version", 1}, {"revision", 0}, {"rules", std::move(OutputRules)}};
 613
 614     OS << llvm::formatv("{0:2}\n", Value(std::move(Output)));
 615   }
 616
 617   void addRules(P1689Rule &Rule) {
 618     std::unique_lock<std::mutex> LockGuard(Lock);
 619     Rules.push_back(Rule);
 620   }
 621
 622 private:
 623   void addSourcePathsToRequires() {
 624     llvm::DenseMap<StringRef, StringRef> ModuleSourceMapper;
 625     for (const P1689Rule &R : Rules)
 626       if (R.Provides && !R.Provides->SourcePath.empty())
 627         ModuleSourceMapper[R.Provides->ModuleName] = R.Provides->SourcePath;
 628
 629     for (P1689Rule &R : Rules) {
 630       for (P1689ModuleInfo &Info : R.Requires) {
 631         auto Iter = ModuleSourceMapper.find(Info.ModuleName);
 632         if (Iter != ModuleSourceMapper.end())
 633           Info.SourcePath = Iter->second;
 634       }
 635     }
 636   }
 637
 638   std::mutex Lock;
 639   std::vector<P1689Rule> Rules;
 640 };
 641
 642 static bool
 643 handleP1689DependencyToolResult(const std::string &Input,
 644                                 llvm::Expected<P1689Rule> &MaybeRule,
 645                                 P1689Deps &PD, SharedStream &Errs) {
 646   if (!MaybeRule) {
 647     llvm::handleAllErrors(
 648         MaybeRule.takeError(), [&Input, &Errs](llvm::StringError &Err) {
 649           Errs.applyLocked([&](raw_ostream &OS) {
 650             OS << "Error while scanning dependencies for " << Input << ":\n";
 651             OS << Err.getMessage();
 652           });
 653         });
 654     return true;
 655   }
 656   PD.addRules(*MaybeRule);
 657   return false;
 658 }
 659
 660 /// Construct a path for the explicitly built PCM.
 661 static std::string constructPCMPath(ModuleID MID, StringRef OutputDir) {
 662   SmallString<256> ExplicitPCMPath(OutputDir);
 663   llvm::sys::path::append(ExplicitPCMPath, MID.ContextHash,
 664                           MID.ModuleName + "-" + MID.ContextHash + ".pcm");
 665   return std::string(ExplicitPCMPath);
 666 }
 667
 668 static std::string lookupModuleOutput(const ModuleID &MID, ModuleOutputKind MOK,
 669                                       StringRef OutputDir) {
 670   std::string PCMPath = constructPCMPath(MID, OutputDir);
 671   switch (MOK) {
 672   case ModuleOutputKind::ModuleFile:
 673     return PCMPath;
 674   case ModuleOutputKind::DependencyFile:
 675     return PCMPath + ".d";
 676   case ModuleOutputKind::DependencyTargets:
 677     // Null-separate the list of targets.
 678     return join(ModuleDepTargets, StringRef("\0", 1));
 679   case ModuleOutputKind::DiagnosticSerializationFile:
 680     return PCMPath + ".diag";
 681   }
 682   llvm_unreachable("Fully covered switch above!");
 683 }
 684
 685 static std::string getModuleCachePath(ArrayRef<std::string> Args) {
 686   for (StringRef Arg : llvm::reverse(Args)) {
 687     Arg.consume_front("/clang:");
 688     if (Arg.consume_front("-fmodules-cache-path="))
 689       return std::string(Arg);
 690   }
 691   SmallString<128> Path;
 692   driver::Driver::getDefaultModuleCachePath(Path);
 693   return std::string(Path);
 694 }
 695
 696 // getCompilationDataBase - If -compilation-database is set, load the
 697 // compilation database from the specified file. Otherwise if the we're
 698 // generating P1689 format, trying to generate the compilation database
 699 // form specified command line after the positional parameter "--".
 700 static std::unique_ptr<tooling::CompilationDatabase>
 701 getCompilationDataBase(int argc, char **argv, std::string &ErrorMessage) {
 702   llvm::InitLLVM X(argc, argv);
 703   ParseArgs(argc, argv);
 704
 705   if (!CompilationDB.empty())
 706     return tooling::JSONCompilationDatabase::loadFromFile(
 707         CompilationDB, ErrorMessage,
 708         tooling::JSONCommandLineSyntax::AutoDetect);
 709
 710   if (Format != ScanningOutputFormat::P1689) {
 711     llvm::errs() << "the --compilation-database option: must be specified at "
 712                     "least once!";
 713     return nullptr;
 714   }
 715
 716   // Trying to get the input file, the output file and the command line options
 717   // from the positional parameter "--".
 718   char **DoubleDash = std::find(argv, argv + argc, StringRef("--"));
 719   if (DoubleDash == argv + argc) {
 720     llvm::errs() << "The command line arguments is required after '--' in "
 721                     "P1689 per file mode.";
 722     return nullptr;
 723   }
 724
 725   llvm::IntrusiveRefCntPtr<DiagnosticsEngine> Diags =
 726       CompilerInstance::createDiagnostics(new DiagnosticOptions);
 727   driver::Driver TheDriver(CommandLine[0], llvm::sys::getDefaultTargetTriple(),
 728                            *Diags);
 729   std::unique_ptr<driver::Compilation> C(
 730       TheDriver.BuildCompilation(CommandLine));
 731   if (!C)
 732     return nullptr;
 733
 734   auto Cmd = C->getJobs().begin();
 735   auto CI = std::make_unique<CompilerInvocation>();
 736   CompilerInvocation::CreateFromArgs(*CI, Cmd->getArguments(), *Diags,
 737                                      CommandLine[0]);
 738   if (!CI)
 739     return nullptr;
 740
 741   FrontendOptions &FEOpts = CI->getFrontendOpts();
 742   if (FEOpts.Inputs.size() != 1) {
 743     llvm::errs() << "Only one input file is allowed in P1689 per file mode.";
 744     return nullptr;
 745   }
 746
 747   // There might be multiple jobs for a compilation. Extract the specified
 748   // output filename from the last job.
 749   auto LastCmd = C->getJobs().end();
 750   LastCmd--;
 751   if (LastCmd->getOutputFilenames().size() != 1) {
 752     llvm::errs() << "The command line should provide exactly one output file "
 753                     "in P1689 per file mode.\n";
 754   }
 755   StringRef OutputFile = LastCmd->getOutputFilenames().front();
 756
 757   class InplaceCompilationDatabase : public tooling::CompilationDatabase {
 758   public:
 759     InplaceCompilationDatabase(StringRef InputFile, StringRef OutputFile,
 760                                ArrayRef<const char *> CommandLine)
 761         : Command(".", InputFile, {}, OutputFile) {
 762       for (auto *C : CommandLine)
 763         Command.CommandLine.push_back(C);
 764     }
 765
 766     std::vector<tooling::CompileCommand>
 767     getCompileCommands(StringRef FilePath) const override {
 768       if (FilePath != Command.Filename)
 769         return {};
 770       return {Command};
 771     }
 772
 773     std::vector<std::string> getAllFiles() const override {
 774       return {Command.Filename};
 775     }
 776
 777     std::vector<tooling::CompileCommand>
 778     getAllCompileCommands() const override {
 779       return {Command};
 780     }
 781
 782   private:
 783     tooling::CompileCommand Command;
 784   };
 785
 786   return std::make_unique<InplaceCompilationDatabase>(
 787       FEOpts.Inputs[0].getFile(), OutputFile, CommandLine);
 788 }
 789
 790 int clang_scan_deps_main(int argc, char **argv, const llvm::ToolContext &) {
 791   std::string ErrorMessage;
 792   std::unique_ptr<tooling::CompilationDatabase> Compilations =
 793       getCompilationDataBase(argc, argv, ErrorMessage);
 794   if (!Compilations) {
 795     llvm::errs() << ErrorMessage << "\n";
 796     return 1;
 797   }
 798
 799   llvm::cl::PrintOptionValues();
 800
 801   // The command options are rewritten to run Clang in preprocessor only mode.
 802   auto AdjustingCompilations =
 803       std::make_unique<tooling::ArgumentsAdjustingCompilations>(
 804           std::move(Compilations));
 805   ResourceDirectoryCache ResourceDirCache;
 806
 807   AdjustingCompilations->appendArgumentsAdjuster(
 808       [&ResourceDirCache](const tooling::CommandLineArguments &Args,
 809                           StringRef FileName) {
 810         std::string LastO;
 811         bool HasResourceDir = false;
 812         bool ClangCLMode = false;
 813         auto FlagsEnd = llvm::find(Args, "--");
 814         if (FlagsEnd != Args.begin()) {
 815           ClangCLMode =
 816               llvm::sys::path::stem(Args[0]).contains_insensitive("clang-cl") ||
 817               llvm::is_contained(Args, "--driver-mode=cl");
 818
 819           // Reverse scan, starting at the end or at the element before "--".
 820           auto R = std::make_reverse_iterator(FlagsEnd);
 821           for (auto I = R, E = Args.rend(); I != E; ++I) {
 822             StringRef Arg = *I;
 823             if (ClangCLMode) {
 824               // Ignore arguments that are preceded by "-Xclang".
 825               if ((I + 1) != E && I[1] == "-Xclang")
 826                 continue;
 827               if (LastO.empty()) {
 828                 // With clang-cl, the output obj file can be specified with
 829                 // "/opath", "/o path", "/Fopath", and the dash counterparts.
 830                 // Also, clang-cl adds ".obj" extension if none is found.
 831                 if ((Arg == "-o" || Arg == "/o") && I != R)
 832                   LastO = I[-1]; // Next argument (reverse iterator)
 833                 else if (Arg.starts_with("/Fo") || Arg.starts_with("-Fo"))
 834                   LastO = Arg.drop_front(3).str();
 835                 else if (Arg.starts_with("/o") || Arg.starts_with("-o"))
 836                   LastO = Arg.drop_front(2).str();
 837
 838                 if (!LastO.empty() && !llvm::sys::path::has_extension(LastO))
 839                   LastO.append(".obj");
 840               }
 841             }
 842             if (Arg == "-resource-dir")
 843               HasResourceDir = true;
 844           }
 845         }
 846         tooling::CommandLineArguments AdjustedArgs(Args.begin(), FlagsEnd);
 847         // The clang-cl driver passes "-o -" to the frontend. Inject the real
 848         // file here to ensure "-MT" can be deduced if need be.
 849         if (ClangCLMode && !LastO.empty()) {
 850           AdjustedArgs.push_back("/clang:-o");
 851           AdjustedArgs.push_back("/clang:" + LastO);
 852         }
 853
 854         if (!HasResourceDir && ResourceDirRecipe == RDRK_InvokeCompiler) {
 855           StringRef ResourceDir =
 856               ResourceDirCache.findResourceDir(Args, ClangCLMode);
 857           if (!ResourceDir.empty()) {
 858             AdjustedArgs.push_back("-resource-dir");
 859             AdjustedArgs.push_back(std::string(ResourceDir));
 860           }
 861         }
 862         AdjustedArgs.insert(AdjustedArgs.end(), FlagsEnd, Args.end());
 863         return AdjustedArgs;
 864       });
 865
 866   SharedStream Errs(llvm::errs());
 867   // Print out the dependency results to STDOUT by default.
 868   SharedStream DependencyOS(llvm::outs());
 869
 870   DependencyScanningService Service(ScanMode, Format, OptimizeArgs,
 871                                     EagerLoadModules);
 872   llvm::ThreadPool Pool(llvm::hardware_concurrency(NumThreads));
 873   std::vector<std::unique_ptr<DependencyScanningTool>> WorkerTools;
 874   for (unsigned I = 0; I < Pool.getThreadCount(); ++I)
 875     WorkerTools.push_back(std::make_unique<DependencyScanningTool>(Service));
 876
 877   std::vector<tooling::CompileCommand> Inputs =
 878       AdjustingCompilations->getAllCompileCommands();
 879
 880   std::atomic<bool> HadErrors(false);
 881   std::optional<FullDeps> FD;
 882   P1689Deps PD;
 883
 884   std::mutex Lock;
 885   size_t Index = 0;
 886   auto GetNextInputIndex = [&]() -> std::optional<size_t> {
 887     std::unique_lock<std::mutex> LockGuard(Lock);
 888     if (Index < Inputs.size())
 889       return Index++;
 890     return {};
 891   };
 892
 893   if (Format == ScanningOutputFormat::Full)
 894     FD.emplace(ModuleName.empty() ? Inputs.size() : 0);
 895
 896   if (Verbose) {
 897     llvm::outs() << "Running clang-scan-deps on " << Inputs.size()
 898                  << " files using " << Pool.getThreadCount() << " workers\n";
 899   }
 900
 901   llvm::Timer T;
 902   T.startTimer();
 903
 904   for (unsigned I = 0; I < Pool.getThreadCount(); ++I) {
 905     Pool.async([&, I]() {
 906       llvm::DenseSet<ModuleID> AlreadySeenModules;
 907       while (auto MaybeInputIndex = GetNextInputIndex()) {
 908         size_t LocalIndex = *MaybeInputIndex;
 909         const tooling::CompileCommand *Input = &Inputs[LocalIndex];
 910         std::string Filename = std::move(Input->Filename);
 911         std::string CWD = std::move(Input->Directory);
 912
 913         std::optional<StringRef> MaybeModuleName;
 914         if (!ModuleName.empty())
 915           MaybeModuleName = ModuleName;
 916
 917         std::string OutputDir(ModuleFilesDir);
 918         if (OutputDir.empty())
 919           OutputDir = getModuleCachePath(Input->CommandLine);
 920         auto LookupOutput = [&](const ModuleID &MID, ModuleOutputKind MOK) {
 921           return ::lookupModuleOutput(MID, MOK, OutputDir);
 922         };
 923
 924         // Run the tool on it.
 925         if (Format == ScanningOutputFormat::Make) {
 926           auto MaybeFile =
 927               WorkerTools[I]->getDependencyFile(Input->CommandLine, CWD);
 928           if (handleMakeDependencyToolResult(Filename, MaybeFile, DependencyOS,
 929                                              Errs))
 930             HadErrors = true;
 931         } else if (Format == ScanningOutputFormat::P1689) {
 932           // It is useful to generate the make-format dependency output during
 933           // the scanning for P1689. Otherwise the users need to scan again for
 934           // it. We will generate the make-format dependency output if we find
 935           // `-MF` in the command lines.
 936           std::string MakeformatOutputPath;
 937           std::string MakeformatOutput;
 938
 939           auto MaybeRule = WorkerTools[I]->getP1689ModuleDependencyFile(
 940               *Input, CWD, MakeformatOutput, MakeformatOutputPath);
 941
 942           if (handleP1689DependencyToolResult(Filename, MaybeRule, PD, Errs))
 943             HadErrors = true;
 944
 945           if (!MakeformatOutputPath.empty() && !MakeformatOutput.empty() &&
 946               !HadErrors) {
 947             static std::mutex Lock;
 948             // With compilation database, we may open different files
 949             // concurrently or we may write the same file concurrently. So we
 950             // use a map here to allow multiple compile commands to write to the
 951             // same file. Also we need a lock here to avoid data race.
 952             static llvm::StringMap<llvm::raw_fd_ostream> OSs;
 953             std::unique_lock<std::mutex> LockGuard(Lock);
 954
 955             auto OSIter = OSs.find(MakeformatOutputPath);
 956             if (OSIter == OSs.end()) {
 957               std::error_code EC;
 958               OSIter = OSs.try_emplace(MakeformatOutputPath,
 959                                        MakeformatOutputPath, EC)
 960                            .first;
 961               if (EC)
 962                 llvm::errs()
 963                     << "Failed to open P1689 make format output file \""
 964                     << MakeformatOutputPath << "\" for " << EC.message()
 965                     << "\n";
 966             }
 967
 968             SharedStream MakeformatOS(OSIter->second);
 969             llvm::Expected<std::string> MaybeOutput(MakeformatOutput);
 970             if (handleMakeDependencyToolResult(Filename, MaybeOutput,
 971                                                MakeformatOS, Errs))
 972               HadErrors = true;
 973           }
 974         } else if (MaybeModuleName) {
 975           auto MaybeModuleDepsGraph = WorkerTools[I]->getModuleDependencies(
 976               *MaybeModuleName, Input->CommandLine, CWD, AlreadySeenModules,
 977               LookupOutput);
 978           if (handleModuleResult(*MaybeModuleName, MaybeModuleDepsGraph, *FD,
 979                                  LocalIndex, DependencyOS, Errs))
 980             HadErrors = true;
 981         } else {
 982           auto MaybeTUDeps = WorkerTools[I]->getTranslationUnitDependencies(
 983               Input->CommandLine, CWD, AlreadySeenModules, LookupOutput);
 984           if (handleTranslationUnitResult(Filename, MaybeTUDeps, *FD,
 985                                           LocalIndex, DependencyOS, Errs))
 986             HadErrors = true;
 987         }
 988       }
 989     });
 990   }
 991   Pool.wait();
 992
 993   T.stopTimer();
 994   if (PrintTiming)
 995     llvm::errs() << llvm::format(
 996         "clang-scan-deps timing: %0.2fs wall, %0.2fs process\n",
 997         T.getTotalTime().getWallTime(), T.getTotalTime().getProcessTime());
 998
 999   if (RoundTripArgs)
1000     if (FD && FD->roundTripCommands(llvm::errs()))
1001       HadErrors = true;
1002
1003   if (Format == ScanningOutputFormat::Full)
1004     FD->printFullOutput(llvm::outs());
1005   else if (Format == ScanningOutputFormat::P1689)
1006     PD.printDependencies(llvm::outs());
1007
1008   return HadErrors;
1009 }