clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp

   1 //===- DependencyScanningWorker.cpp - clang-scan-deps worker --------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h"
  10 #include "clang/Basic/DiagnosticDriver.h"
  11 #include "clang/Basic/DiagnosticFrontend.h"
  12 #include "clang/Basic/DiagnosticSerialization.h"
  13 #include "clang/Driver/Compilation.h"
  14 #include "clang/Driver/Driver.h"
  15 #include "clang/Driver/Job.h"
  16 #include "clang/Driver/Tool.h"
  17 #include "clang/Frontend/CompilerInstance.h"
  18 #include "clang/Frontend/CompilerInvocation.h"
  19 #include "clang/Frontend/FrontendActions.h"
  20 #include "clang/Frontend/TextDiagnosticPrinter.h"
  21 #include "clang/Frontend/Utils.h"
  22 #include "clang/Lex/PreprocessorOptions.h"
  23 #include "clang/Serialization/ObjectFilePCHContainerReader.h"
  24 #include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
  25 #include "clang/Tooling/DependencyScanning/ModuleDepCollector.h"
  26 #include "clang/Tooling/Tooling.h"
  27 #include "llvm/ADT/ScopeExit.h"
  28 #include "llvm/Support/Allocator.h"
  29 #include "llvm/Support/Error.h"
  30 #include "llvm/TargetParser/Host.h"
  31 #include <optional>
  32
  33 using namespace clang;
  34 using namespace tooling;
  35 using namespace dependencies;
  36
  37 namespace {
  38
  39 /// Forwards the gatherered dependencies to the consumer.
  40 class DependencyConsumerForwarder : public DependencyFileGenerator {
  41 public:
  42   DependencyConsumerForwarder(std::unique_ptr<DependencyOutputOptions> Opts,
  43                               StringRef WorkingDirectory, DependencyConsumer &C)
  44       : DependencyFileGenerator(*Opts), WorkingDirectory(WorkingDirectory),
  45         Opts(std::move(Opts)), C(C) {}
  46
  47   void finishedMainFile(DiagnosticsEngine &Diags) override {
  48     C.handleDependencyOutputOpts(*Opts);
  49     llvm::SmallString<256> CanonPath;
  50     for (const auto &File : getDependencies()) {
  51       CanonPath = File;
  52       llvm::sys::path::remove_dots(CanonPath, /*remove_dot_dot=*/true);
  53       llvm::sys::fs::make_absolute(WorkingDirectory, CanonPath);
  54       C.handleFileDependency(CanonPath);
  55     }
  56   }
  57
  58 private:
  59   StringRef WorkingDirectory;
  60   std::unique_ptr<DependencyOutputOptions> Opts;
  61   DependencyConsumer &C;
  62 };
  63
  64 static bool checkHeaderSearchPaths(const HeaderSearchOptions &HSOpts,
  65                                    const HeaderSearchOptions &ExistingHSOpts,
  66                                    DiagnosticsEngine *Diags,
  67                                    const LangOptions &LangOpts) {
  68   if (LangOpts.Modules) {
  69     if (HSOpts.VFSOverlayFiles != ExistingHSOpts.VFSOverlayFiles) {
  70       if (Diags) {
  71         Diags->Report(diag::warn_pch_vfsoverlay_mismatch);
  72         auto VFSNote = [&](int Type, ArrayRef<std::string> VFSOverlays) {
  73           if (VFSOverlays.empty()) {
  74             Diags->Report(diag::note_pch_vfsoverlay_empty) << Type;
  75           } else {
  76             std::string Files = llvm::join(VFSOverlays, "\n");
  77             Diags->Report(diag::note_pch_vfsoverlay_files) << Type << Files;
  78           }
  79         };
  80         VFSNote(0, HSOpts.VFSOverlayFiles);
  81         VFSNote(1, ExistingHSOpts.VFSOverlayFiles);
  82       }
  83     }
  84   }
  85   return false;
  86 }
  87
  88 using PrebuiltModuleFilesT = decltype(HeaderSearchOptions::PrebuiltModuleFiles);
  89
  90 /// A listener that collects the imported modules and optionally the input
  91 /// files.
  92 class PrebuiltModuleListener : public ASTReaderListener {
  93 public:
  94   PrebuiltModuleListener(PrebuiltModuleFilesT &PrebuiltModuleFiles,
  95                          llvm::SmallVector<std::string> &NewModuleFiles,
  96                          PrebuiltModuleVFSMapT &PrebuiltModuleVFSMap,
  97                          const HeaderSearchOptions &HSOpts,
  98                          const LangOptions &LangOpts, DiagnosticsEngine &Diags)
  99       : PrebuiltModuleFiles(PrebuiltModuleFiles),
 100         NewModuleFiles(NewModuleFiles),
 101         PrebuiltModuleVFSMap(PrebuiltModuleVFSMap), ExistingHSOpts(HSOpts),
 102         ExistingLangOpts(LangOpts), Diags(Diags) {}
 103
 104   bool needsImportVisitation() const override { return true; }
 105
 106   void visitImport(StringRef ModuleName, StringRef Filename) override {
 107     if (PrebuiltModuleFiles.insert({ModuleName.str(), Filename.str()}).second)
 108       NewModuleFiles.push_back(Filename.str());
 109   }
 110
 111   void visitModuleFile(StringRef Filename,
 112                        serialization::ModuleKind Kind) override {
 113     CurrentFile = Filename;
 114   }
 115
 116   bool ReadHeaderSearchPaths(const HeaderSearchOptions &HSOpts,
 117                              bool Complain) override {
 118     std::vector<std::string> VFSOverlayFiles = HSOpts.VFSOverlayFiles;
 119     PrebuiltModuleVFSMap.insert(
 120         {CurrentFile, llvm::StringSet<>(VFSOverlayFiles)});
 121     return checkHeaderSearchPaths(
 122         HSOpts, ExistingHSOpts, Complain ? &Diags : nullptr, ExistingLangOpts);
 123   }
 124
 125 private:
 126   PrebuiltModuleFilesT &PrebuiltModuleFiles;
 127   llvm::SmallVector<std::string> &NewModuleFiles;
 128   PrebuiltModuleVFSMapT &PrebuiltModuleVFSMap;
 129   const HeaderSearchOptions &ExistingHSOpts;
 130   const LangOptions &ExistingLangOpts;
 131   DiagnosticsEngine &Diags;
 132   std::string CurrentFile;
 133 };
 134
 135 /// Visit the given prebuilt module and collect all of the modules it
 136 /// transitively imports and contributing input files.
 137 static bool visitPrebuiltModule(StringRef PrebuiltModuleFilename,
 138                                 CompilerInstance &CI,
 139                                 PrebuiltModuleFilesT &ModuleFiles,
 140                                 PrebuiltModuleVFSMapT &PrebuiltModuleVFSMap,
 141                                 DiagnosticsEngine &Diags) {
 142   // List of module files to be processed.
 143   llvm::SmallVector<std::string> Worklist;
 144   PrebuiltModuleListener Listener(ModuleFiles, Worklist, PrebuiltModuleVFSMap,
 145                                   CI.getHeaderSearchOpts(), CI.getLangOpts(),
 146                                   Diags);
 147
 148   Listener.visitModuleFile(PrebuiltModuleFilename,
 149                            serialization::MK_ExplicitModule);
 150   if (ASTReader::readASTFileControlBlock(
 151           PrebuiltModuleFilename, CI.getFileManager(), CI.getModuleCache(),
 152           CI.getPCHContainerReader(),
 153           /*FindModuleFileExtensions=*/false, Listener,
 154           /*ValidateDiagnosticOptions=*/false, ASTReader::ARR_OutOfDate))
 155     return true;
 156
 157   while (!Worklist.empty()) {
 158     Listener.visitModuleFile(Worklist.back(), serialization::MK_ExplicitModule);
 159     if (ASTReader::readASTFileControlBlock(
 160             Worklist.pop_back_val(), CI.getFileManager(), CI.getModuleCache(),
 161             CI.getPCHContainerReader(),
 162             /*FindModuleFileExtensions=*/false, Listener,
 163             /*ValidateDiagnosticOptions=*/false))
 164       return true;
 165   }
 166   return false;
 167 }
 168
 169 /// Transform arbitrary file name into an object-like file name.
 170 static std::string makeObjFileName(StringRef FileName) {
 171   SmallString<128> ObjFileName(FileName);
 172   llvm::sys::path::replace_extension(ObjFileName, "o");
 173   return std::string(ObjFileName);
 174 }
 175
 176 /// Deduce the dependency target based on the output file and input files.
 177 static std::string
 178 deduceDepTarget(const std::string &OutputFile,
 179                 const SmallVectorImpl<FrontendInputFile> &InputFiles) {
 180   if (OutputFile != "-")
 181     return OutputFile;
 182
 183   if (InputFiles.empty() || !InputFiles.front().isFile())
 184     return "clang-scan-deps\\ dependency";
 185
 186   return makeObjFileName(InputFiles.front().getFile());
 187 }
 188
 189 /// Sanitize diagnostic options for dependency scan.
 190 static void sanitizeDiagOpts(DiagnosticOptions &DiagOpts) {
 191   // Don't print 'X warnings and Y errors generated'.
 192   DiagOpts.ShowCarets = false;
 193   // Don't write out diagnostic file.
 194   DiagOpts.DiagnosticSerializationFile.clear();
 195   // Don't emit warnings except for scanning specific warnings.
 196   // TODO: It would be useful to add a more principled way to ignore all
 197   //       warnings that come from source code. The issue is that we need to
 198   //       ignore warnings that could be surpressed by
 199   //       `#pragma clang diagnostic`, while still allowing some scanning
 200   //       warnings for things we're not ready to turn into errors yet.
 201   //       See `test/ClangScanDeps/diagnostic-pragmas.c` for an example.
 202   llvm::erase_if(DiagOpts.Warnings, [](StringRef Warning) {
 203     return llvm::StringSwitch<bool>(Warning)
 204         .Cases("pch-vfs-diff", "error=pch-vfs-diff", false)
 205         .StartsWith("no-error=", false)
 206         .Default(true);
 207   });
 208 }
 209
 210 // Clang implements -D and -U by splatting text into a predefines buffer. This
 211 // allows constructs such as `-DFඞ=3 "-D F\u{0D9E} 4 3 2”` to be accepted and
 212 // define the same macro, or adding C++ style comments before the macro name.
 213 //
 214 // This function checks that the first non-space characters in the macro
 215 // obviously form an identifier that can be uniqued on without lexing. Failing
 216 // to do this could lead to changing the final definition of a macro.
 217 //
 218 // We could set up a preprocessor and actually lex the name, but that's very
 219 // heavyweight for a situation that will almost never happen in practice.
 220 static std::optional<StringRef> getSimpleMacroName(StringRef Macro) {
 221   StringRef Name = Macro.split("=").first.ltrim(" \t");
 222   std::size_t I = 0;
 223
 224   auto FinishName = [&]() -> std::optional<StringRef> {
 225     StringRef SimpleName = Name.slice(0, I);
 226     if (SimpleName.empty())
 227       return std::nullopt;
 228     return SimpleName;
 229   };
 230
 231   for (; I != Name.size(); ++I) {
 232     switch (Name[I]) {
 233     case '(': // Start of macro parameter list
 234     case ' ': // End of macro name
 235     case '\t':
 236       return FinishName();
 237     case '_':
 238       continue;
 239     default:
 240       if (llvm::isAlnum(Name[I]))
 241         continue;
 242       return std::nullopt;
 243     }
 244   }
 245   return FinishName();
 246 }
 247
 248 static void canonicalizeDefines(PreprocessorOptions &PPOpts) {
 249   using MacroOpt = std::pair<StringRef, std::size_t>;
 250   std::vector<MacroOpt> SimpleNames;
 251   SimpleNames.reserve(PPOpts.Macros.size());
 252   std::size_t Index = 0;
 253   for (const auto &M : PPOpts.Macros) {
 254     auto SName = getSimpleMacroName(M.first);
 255     // Skip optimizing if we can't guarantee we can preserve relative order.
 256     if (!SName)
 257       return;
 258     SimpleNames.emplace_back(*SName, Index);
 259     ++Index;
 260   }
 261
 262   llvm::stable_sort(SimpleNames, llvm::less_first());
 263   // Keep the last instance of each macro name by going in reverse
 264   auto NewEnd = std::unique(
 265       SimpleNames.rbegin(), SimpleNames.rend(),
 266       [](const MacroOpt &A, const MacroOpt &B) { return A.first == B.first; });
 267   SimpleNames.erase(SimpleNames.begin(), NewEnd.base());
 268
 269   // Apply permutation.
 270   decltype(PPOpts.Macros) NewMacros;
 271   NewMacros.reserve(SimpleNames.size());
 272   for (std::size_t I = 0, E = SimpleNames.size(); I != E; ++I) {
 273     std::size_t OriginalIndex = SimpleNames[I].second;
 274     // We still emit undefines here as they may be undefining a predefined macro
 275     NewMacros.push_back(std::move(PPOpts.Macros[OriginalIndex]));
 276   }
 277   std::swap(PPOpts.Macros, NewMacros);
 278 }
 279
 280 /// A clang tool that runs the preprocessor in a mode that's optimized for
 281 /// dependency scanning for the given compiler invocation.
 282 class DependencyScanningAction : public tooling::ToolAction {
 283 public:
 284   DependencyScanningAction(
 285       StringRef WorkingDirectory, DependencyConsumer &Consumer,
 286       DependencyActionController &Controller,
 287       llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS,
 288       ScanningOutputFormat Format, ScanningOptimizations OptimizeArgs,
 289       bool EagerLoadModules, bool DisableFree,
 290       std::optional<StringRef> ModuleName = std::nullopt)
 291       : WorkingDirectory(WorkingDirectory), Consumer(Consumer),
 292         Controller(Controller), DepFS(std::move(DepFS)), Format(Format),
 293         OptimizeArgs(OptimizeArgs), EagerLoadModules(EagerLoadModules),
 294         DisableFree(DisableFree), ModuleName(ModuleName) {}
 295
 296   bool runInvocation(std::shared_ptr<CompilerInvocation> Invocation,
 297                      FileManager *DriverFileMgr,
 298                      std::shared_ptr<PCHContainerOperations> PCHContainerOps,
 299                      DiagnosticConsumer *DiagConsumer) override {
 300     // Make a deep copy of the original Clang invocation.
 301     CompilerInvocation OriginalInvocation(*Invocation);
 302     // Restore the value of DisableFree, which may be modified by Tooling.
 303     OriginalInvocation.getFrontendOpts().DisableFree = DisableFree;
 304     if (any(OptimizeArgs & ScanningOptimizations::Macros))
 305       canonicalizeDefines(OriginalInvocation.getPreprocessorOpts());
 306
 307     if (Scanned) {
 308       // Scanning runs once for the first -cc1 invocation in a chain of driver
 309       // jobs. For any dependent jobs, reuse the scanning result and just
 310       // update the LastCC1Arguments to correspond to the new invocation.
 311       // FIXME: to support multi-arch builds, each arch requires a separate scan
 312       setLastCC1Arguments(std::move(OriginalInvocation));
 313       return true;
 314     }
 315
 316     Scanned = true;
 317
 318     // Create a compiler instance to handle the actual work.
 319     ScanInstanceStorage.emplace(std::move(PCHContainerOps));
 320     CompilerInstance &ScanInstance = *ScanInstanceStorage;
 321     ScanInstance.setInvocation(std::move(Invocation));
 322
 323     // Create the compiler's actual diagnostics engine.
 324     sanitizeDiagOpts(ScanInstance.getDiagnosticOpts());
 325     ScanInstance.createDiagnostics(DriverFileMgr->getVirtualFileSystem(),
 326                                    DiagConsumer, /*ShouldOwnClient=*/false);
 327     if (!ScanInstance.hasDiagnostics())
 328       return false;
 329
 330     // Some DiagnosticConsumers require that finish() is called.
 331     auto DiagConsumerFinisher =
 332         llvm::make_scope_exit([DiagConsumer]() { DiagConsumer->finish(); });
 333
 334     ScanInstance.getPreprocessorOpts().AllowPCHWithDifferentModulesCachePath =
 335         true;
 336
 337     ScanInstance.getFrontendOpts().GenerateGlobalModuleIndex = false;
 338     ScanInstance.getFrontendOpts().UseGlobalModuleIndex = false;
 339     // This will prevent us compiling individual modules asynchronously since
 340     // FileManager is not thread-safe, but it does improve performance for now.
 341     ScanInstance.getFrontendOpts().ModulesShareFileManager = true;
 342     ScanInstance.getHeaderSearchOpts().ModuleFormat = "raw";
 343     ScanInstance.getHeaderSearchOpts().ModulesIncludeVFSUsage =
 344         any(OptimizeArgs & ScanningOptimizations::VFS);
 345
 346     // Support for virtual file system overlays.
 347     auto FS = createVFSFromCompilerInvocation(
 348         ScanInstance.getInvocation(), ScanInstance.getDiagnostics(),
 349         DriverFileMgr->getVirtualFileSystemPtr());
 350
 351     // Use the dependency scanning optimized file system if requested to do so.
 352     if (DepFS) {
 353       StringRef ModulesCachePath =
 354           ScanInstance.getHeaderSearchOpts().ModuleCachePath;
 355
 356       DepFS->resetBypassedPathPrefix();
 357       if (!ModulesCachePath.empty())
 358         DepFS->setBypassedPathPrefix(ModulesCachePath);
 359
 360       ScanInstance.getPreprocessorOpts().DependencyDirectivesForFile =
 361           [LocalDepFS = DepFS](FileEntryRef File)
 362           -> std::optional<ArrayRef<dependency_directives_scan::Directive>> {
 363         if (llvm::ErrorOr<EntryRef> Entry =
 364                 LocalDepFS->getOrCreateFileSystemEntry(File.getName()))
 365           if (LocalDepFS->ensureDirectiveTokensArePopulated(*Entry))
 366             return Entry->getDirectiveTokens();
 367         return std::nullopt;
 368       };
 369     }
 370
 371     // Create a new FileManager to match the invocation's FileSystemOptions.
 372     auto *FileMgr = ScanInstance.createFileManager(FS);
 373     ScanInstance.createSourceManager(*FileMgr);
 374
 375     // Store the list of prebuilt module files into header search options. This
 376     // will prevent the implicit build to create duplicate modules and will
 377     // force reuse of the existing prebuilt module files instead.
 378     PrebuiltModuleVFSMapT PrebuiltModuleVFSMap;
 379     if (!ScanInstance.getPreprocessorOpts().ImplicitPCHInclude.empty())
 380       if (visitPrebuiltModule(
 381               ScanInstance.getPreprocessorOpts().ImplicitPCHInclude,
 382               ScanInstance,
 383               ScanInstance.getHeaderSearchOpts().PrebuiltModuleFiles,
 384               PrebuiltModuleVFSMap, ScanInstance.getDiagnostics()))
 385         return false;
 386
 387     // Create the dependency collector that will collect the produced
 388     // dependencies.
 389     //
 390     // This also moves the existing dependency output options from the
 391     // invocation to the collector. The options in the invocation are reset,
 392     // which ensures that the compiler won't create new dependency collectors,
 393     // and thus won't write out the extra '.d' files to disk.
 394     auto Opts = std::make_unique<DependencyOutputOptions>();
 395     std::swap(*Opts, ScanInstance.getInvocation().getDependencyOutputOpts());
 396     // We need at least one -MT equivalent for the generator of make dependency
 397     // files to work.
 398     if (Opts->Targets.empty())
 399       Opts->Targets = {
 400           deduceDepTarget(ScanInstance.getFrontendOpts().OutputFile,
 401                           ScanInstance.getFrontendOpts().Inputs)};
 402     Opts->IncludeSystemHeaders = true;
 403
 404     switch (Format) {
 405     case ScanningOutputFormat::Make:
 406       ScanInstance.addDependencyCollector(
 407           std::make_shared<DependencyConsumerForwarder>(
 408               std::move(Opts), WorkingDirectory, Consumer));
 409       break;
 410     case ScanningOutputFormat::P1689:
 411     case ScanningOutputFormat::Full:
 412       MDC = std::make_shared<ModuleDepCollector>(
 413           std::move(Opts), ScanInstance, Consumer, Controller,
 414           OriginalInvocation, std::move(PrebuiltModuleVFSMap), OptimizeArgs,
 415           EagerLoadModules, Format == ScanningOutputFormat::P1689);
 416       ScanInstance.addDependencyCollector(MDC);
 417       break;
 418     }
 419
 420     // Consider different header search and diagnostic options to create
 421     // different modules. This avoids the unsound aliasing of module PCMs.
 422     //
 423     // TODO: Implement diagnostic bucketing to reduce the impact of strict
 424     // context hashing.
 425     ScanInstance.getHeaderSearchOpts().ModulesStrictContextHash = true;
 426     ScanInstance.getHeaderSearchOpts().ModulesSerializeOnlyPreprocessor = true;
 427     ScanInstance.getHeaderSearchOpts().ModulesSkipDiagnosticOptions = true;
 428     ScanInstance.getHeaderSearchOpts().ModulesSkipHeaderSearchPaths = true;
 429     ScanInstance.getHeaderSearchOpts().ModulesSkipPragmaDiagnosticMappings =
 430         true;
 431
 432     // Avoid some checks and module map parsing when loading PCM files.
 433     ScanInstance.getPreprocessorOpts().ModulesCheckRelocated = false;
 434
 435     std::unique_ptr<FrontendAction> Action;
 436
 437     if (Format == ScanningOutputFormat::P1689)
 438       Action = std::make_unique<PreprocessOnlyAction>();
 439     else if (ModuleName)
 440       Action = std::make_unique<GetDependenciesByModuleNameAction>(*ModuleName);
 441     else
 442       Action = std::make_unique<ReadPCHAndPreprocessAction>();
 443
 444     if (ScanInstance.getDiagnostics().hasErrorOccurred())
 445       return false;
 446
 447     // Each action is responsible for calling finish.
 448     DiagConsumerFinisher.release();
 449     const bool Result = ScanInstance.ExecuteAction(*Action);
 450
 451     if (Result)
 452       setLastCC1Arguments(std::move(OriginalInvocation));
 453
 454     // Propagate the statistics to the parent FileManager.
 455     DriverFileMgr->AddStats(ScanInstance.getFileManager());
 456
 457     return Result;
 458   }
 459
 460   bool hasScanned() const { return Scanned; }
 461
 462   /// Take the cc1 arguments corresponding to the most recent invocation used
 463   /// with this action. Any modifications implied by the discovered dependencies
 464   /// will have already been applied.
 465   std::vector<std::string> takeLastCC1Arguments() {
 466     std::vector<std::string> Result;
 467     std::swap(Result, LastCC1Arguments); // Reset LastCC1Arguments to empty.
 468     return Result;
 469   }
 470
 471 private:
 472   void setLastCC1Arguments(CompilerInvocation &&CI) {
 473     if (MDC)
 474       MDC->applyDiscoveredDependencies(CI);
 475     LastCC1Arguments = CI.getCC1CommandLine();
 476   }
 477
 478 private:
 479   StringRef WorkingDirectory;
 480   DependencyConsumer &Consumer;
 481   DependencyActionController &Controller;
 482   llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS;
 483   ScanningOutputFormat Format;
 484   ScanningOptimizations OptimizeArgs;
 485   bool EagerLoadModules;
 486   bool DisableFree;
 487   std::optional<StringRef> ModuleName;
 488   std::optional<CompilerInstance> ScanInstanceStorage;
 489   std::shared_ptr<ModuleDepCollector> MDC;
 490   std::vector<std::string> LastCC1Arguments;
 491   bool Scanned = false;
 492 };
 493
 494 } // end anonymous namespace
 495
 496 DependencyScanningWorker::DependencyScanningWorker(
 497     DependencyScanningService &Service,
 498     llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS)
 499     : Format(Service.getFormat()), OptimizeArgs(Service.getOptimizeArgs()),
 500       EagerLoadModules(Service.shouldEagerLoadModules()) {
 501   PCHContainerOps = std::make_shared<PCHContainerOperations>();
 502   // We need to read object files from PCH built outside the scanner.
 503   PCHContainerOps->registerReader(
 504       std::make_unique<ObjectFilePCHContainerReader>());
 505   // The scanner itself writes only raw ast files.
 506   PCHContainerOps->registerWriter(std::make_unique<RawPCHContainerWriter>());
 507
 508   if (Service.shouldTraceVFS())
 509     FS = llvm::makeIntrusiveRefCnt<llvm::vfs::TracingFileSystem>(std::move(FS));
 510
 511   switch (Service.getMode()) {
 512   case ScanningMode::DependencyDirectivesScan:
 513     DepFS =
 514         new DependencyScanningWorkerFilesystem(Service.getSharedCache(), FS);
 515     BaseFS = DepFS;
 516     break;
 517   case ScanningMode::CanonicalPreprocessing:
 518     DepFS = nullptr;
 519     BaseFS = FS;
 520     break;
 521   }
 522 }
 523
 524 llvm::Error DependencyScanningWorker::computeDependencies(
 525     StringRef WorkingDirectory, const std::vector<std::string> &CommandLine,
 526     DependencyConsumer &Consumer, DependencyActionController &Controller,
 527     std::optional<StringRef> ModuleName) {
 528   std::vector<const char *> CLI;
 529   for (const std::string &Arg : CommandLine)
 530     CLI.push_back(Arg.c_str());
 531   auto DiagOpts = CreateAndPopulateDiagOpts(CLI);
 532   sanitizeDiagOpts(*DiagOpts);
 533
 534   // Capture the emitted diagnostics and report them to the client
 535   // in the case of a failure.
 536   std::string DiagnosticOutput;
 537   llvm::raw_string_ostream DiagnosticsOS(DiagnosticOutput);
 538   TextDiagnosticPrinter DiagPrinter(DiagnosticsOS, DiagOpts.release());
 539
 540   if (computeDependencies(WorkingDirectory, CommandLine, Consumer, Controller,
 541                           DiagPrinter, ModuleName))
 542     return llvm::Error::success();
 543   return llvm::make_error<llvm::StringError>(DiagnosticsOS.str(),
 544                                              llvm::inconvertibleErrorCode());
 545 }
 546
 547 static bool forEachDriverJob(
 548     ArrayRef<std::string> ArgStrs, DiagnosticsEngine &Diags, FileManager &FM,
 549     llvm::function_ref<bool(const driver::Command &Cmd)> Callback) {
 550   SmallVector<const char *, 256> Argv;
 551   Argv.reserve(ArgStrs.size());
 552   for (const std::string &Arg : ArgStrs)
 553     Argv.push_back(Arg.c_str());
 554
 555   llvm::vfs::FileSystem *FS = &FM.getVirtualFileSystem();
 556
 557   std::unique_ptr<driver::Driver> Driver = std::make_unique<driver::Driver>(
 558       Argv[0], llvm::sys::getDefaultTargetTriple(), Diags,
 559       "clang LLVM compiler", FS);
 560   Driver->setTitle("clang_based_tool");
 561
 562   llvm::BumpPtrAllocator Alloc;
 563   bool CLMode = driver::IsClangCL(
 564       driver::getDriverMode(Argv[0], ArrayRef(Argv).slice(1)));
 565
 566   if (llvm::Error E = driver::expandResponseFiles(Argv, CLMode, Alloc, FS)) {
 567     Diags.Report(diag::err_drv_expand_response_file)
 568         << llvm::toString(std::move(E));
 569     return false;
 570   }
 571
 572   const std::unique_ptr<driver::Compilation> Compilation(
 573       Driver->BuildCompilation(llvm::ArrayRef(Argv)));
 574   if (!Compilation)
 575     return false;
 576
 577   if (Compilation->containsError())
 578     return false;
 579
 580   for (const driver::Command &Job : Compilation->getJobs()) {
 581     if (!Callback(Job))
 582       return false;
 583   }
 584   return true;
 585 }
 586
 587 static bool createAndRunToolInvocation(
 588     std::vector<std::string> CommandLine, DependencyScanningAction &Action,
 589     FileManager &FM,
 590     std::shared_ptr<clang::PCHContainerOperations> &PCHContainerOps,
 591     DiagnosticsEngine &Diags, DependencyConsumer &Consumer) {
 592
 593   // Save executable path before providing CommandLine to ToolInvocation
 594   std::string Executable = CommandLine[0];
 595   ToolInvocation Invocation(std::move(CommandLine), &Action, &FM,
 596                             PCHContainerOps);
 597   Invocation.setDiagnosticConsumer(Diags.getClient());
 598   Invocation.setDiagnosticOptions(&Diags.getDiagnosticOptions());
 599   if (!Invocation.run())
 600     return false;
 601
 602   std::vector<std::string> Args = Action.takeLastCC1Arguments();
 603   Consumer.handleBuildCommand({std::move(Executable), std::move(Args)});
 604   return true;
 605 }
 606
 607 bool DependencyScanningWorker::computeDependencies(
 608     StringRef WorkingDirectory, const std::vector<std::string> &CommandLine,
 609     DependencyConsumer &Consumer, DependencyActionController &Controller,
 610     DiagnosticConsumer &DC, std::optional<StringRef> ModuleName) {
 611   // Reset what might have been modified in the previous worker invocation.
 612   BaseFS->setCurrentWorkingDirectory(WorkingDirectory);
 613
 614   std::optional<std::vector<std::string>> ModifiedCommandLine;
 615   llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> ModifiedFS;
 616
 617   // If we're scanning based on a module name alone, we don't expect the client
 618   // to provide us with an input file. However, the driver really wants to have
 619   // one. Let's just make it up to make the driver happy.
 620   if (ModuleName) {
 621     auto OverlayFS =
 622         llvm::makeIntrusiveRefCnt<llvm::vfs::OverlayFileSystem>(BaseFS);
 623     auto InMemoryFS =
 624         llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
 625     InMemoryFS->setCurrentWorkingDirectory(WorkingDirectory);
 626     OverlayFS->pushOverlay(InMemoryFS);
 627     ModifiedFS = OverlayFS;
 628
 629     SmallString<128> FakeInputPath;
 630     // TODO: We should retry the creation if the path already exists.
 631     llvm::sys::fs::createUniquePath(*ModuleName + "-%%%%%%%%.input",
 632                                     FakeInputPath,
 633                                     /*MakeAbsolute=*/false);
 634     InMemoryFS->addFile(FakeInputPath, 0, llvm::MemoryBuffer::getMemBuffer(""));
 635
 636     ModifiedCommandLine = CommandLine;
 637     ModifiedCommandLine->emplace_back(FakeInputPath);
 638   }
 639
 640   const std::vector<std::string> &FinalCommandLine =
 641       ModifiedCommandLine ? *ModifiedCommandLine : CommandLine;
 642   auto &FinalFS = ModifiedFS ? ModifiedFS : BaseFS;
 643
 644   auto FileMgr =
 645       llvm::makeIntrusiveRefCnt<FileManager>(FileSystemOptions{}, FinalFS);
 646
 647   std::vector<const char *> FinalCCommandLine(FinalCommandLine.size(), nullptr);
 648   llvm::transform(FinalCommandLine, FinalCCommandLine.begin(),
 649                   [](const std::string &Str) { return Str.c_str(); });
 650
 651   auto DiagOpts = CreateAndPopulateDiagOpts(FinalCCommandLine);
 652   sanitizeDiagOpts(*DiagOpts);
 653   IntrusiveRefCntPtr<DiagnosticsEngine> Diags =
 654       CompilerInstance::createDiagnostics(*FinalFS, DiagOpts.release(), &DC,
 655                                           /*ShouldOwnClient=*/false);
 656
 657   // Although `Diagnostics` are used only for command-line parsing, the
 658   // custom `DiagConsumer` might expect a `SourceManager` to be present.
 659   SourceManager SrcMgr(*Diags, *FileMgr);
 660   Diags->setSourceManager(&SrcMgr);
 661   // DisableFree is modified by Tooling for running
 662   // in-process; preserve the original value, which is
 663   // always true for a driver invocation.
 664   bool DisableFree = true;
 665   DependencyScanningAction Action(WorkingDirectory, Consumer, Controller, DepFS,
 666                                   Format, OptimizeArgs, EagerLoadModules,
 667                                   DisableFree, ModuleName);
 668
 669   bool Success = false;
 670   if (FinalCommandLine[1] == "-cc1") {
 671     Success = createAndRunToolInvocation(FinalCommandLine, Action, *FileMgr,
 672                                          PCHContainerOps, *Diags, Consumer);
 673   } else {
 674     Success = forEachDriverJob(
 675         FinalCommandLine, *Diags, *FileMgr, [&](const driver::Command &Cmd) {
 676           if (StringRef(Cmd.getCreator().getName()) != "clang") {
 677             // Non-clang command. Just pass through to the dependency
 678             // consumer.
 679             Consumer.handleBuildCommand(
 680                 {Cmd.getExecutable(),
 681                  {Cmd.getArguments().begin(), Cmd.getArguments().end()}});
 682             return true;
 683           }
 684
 685           // Insert -cc1 comand line options into Argv
 686           std::vector<std::string> Argv;
 687           Argv.push_back(Cmd.getExecutable());
 688           Argv.insert(Argv.end(), Cmd.getArguments().begin(),
 689                       Cmd.getArguments().end());
 690
 691           // Create an invocation that uses the underlying file
 692           // system to ensure that any file system requests that
 693           // are made by the driver do not go through the
 694           // dependency scanning filesystem.
 695           return createAndRunToolInvocation(std::move(Argv), Action, *FileMgr,
 696                                             PCHContainerOps, *Diags, Consumer);
 697         });
 698   }
 699
 700   if (Success && !Action.hasScanned())
 701     Diags->Report(diag::err_fe_expected_compiler_job)
 702         << llvm::join(FinalCommandLine, " ");
 703   return Success && Action.hasScanned();
 704 }
 705
 706 DependencyActionController::~DependencyActionController() {}