1 //===- DependencyScanningWorker.cpp - clang-scan-deps worker --------------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 #include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h"
10 #include "clang/Basic/DiagnosticDriver.h"
11 #include "clang/Basic/DiagnosticFrontend.h"
12 #include "clang/CodeGen/ObjectFilePCHContainerOperations.h"
13 #include "clang/Driver/Compilation.h"
14 #include "clang/Driver/Driver.h"
15 #include "clang/Driver/Job.h"
16 #include "clang/Driver/Tool.h"
17 #include "clang/Frontend/CompilerInstance.h"
18 #include "clang/Frontend/CompilerInvocation.h"
19 #include "clang/Frontend/FrontendActions.h"
20 #include "clang/Frontend/TextDiagnosticPrinter.h"
21 #include "clang/Frontend/Utils.h"
22 #include "clang/Lex/PreprocessorOptions.h"
23 #include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
24 #include "clang/Tooling/DependencyScanning/ModuleDepCollector.h"
25 #include "clang/Tooling/Tooling.h"
26 #include "llvm/Support/Allocator.h"
27 #include "llvm/Support/Error.h"
28 #include "llvm/TargetParser/Host.h"
31 using namespace clang
;
32 using namespace tooling
;
33 using namespace dependencies
;
37 /// Forwards the gatherered dependencies to the consumer.
38 class DependencyConsumerForwarder
: public DependencyFileGenerator
{
40 DependencyConsumerForwarder(std::unique_ptr
<DependencyOutputOptions
> Opts
,
41 StringRef WorkingDirectory
, DependencyConsumer
&C
)
42 : DependencyFileGenerator(*Opts
), WorkingDirectory(WorkingDirectory
),
43 Opts(std::move(Opts
)), C(C
) {}
45 void finishedMainFile(DiagnosticsEngine
&Diags
) override
{
46 C
.handleDependencyOutputOpts(*Opts
);
47 llvm::SmallString
<256> CanonPath
;
48 for (const auto &File
: getDependencies()) {
50 llvm::sys::path::remove_dots(CanonPath
, /*remove_dot_dot=*/true);
51 llvm::sys::fs::make_absolute(WorkingDirectory
, CanonPath
);
52 C
.handleFileDependency(CanonPath
);
57 StringRef WorkingDirectory
;
58 std::unique_ptr
<DependencyOutputOptions
> Opts
;
59 DependencyConsumer
&C
;
62 using PrebuiltModuleFilesT
= decltype(HeaderSearchOptions::PrebuiltModuleFiles
);
64 /// A listener that collects the imported modules and optionally the input
66 class PrebuiltModuleListener
: public ASTReaderListener
{
68 PrebuiltModuleListener(PrebuiltModuleFilesT
&PrebuiltModuleFiles
,
69 llvm::SmallVector
<std::string
> &NewModuleFiles
)
70 : PrebuiltModuleFiles(PrebuiltModuleFiles
),
71 NewModuleFiles(NewModuleFiles
) {}
73 bool needsImportVisitation() const override
{ return true; }
75 void visitImport(StringRef ModuleName
, StringRef Filename
) override
{
76 if (PrebuiltModuleFiles
.insert({ModuleName
.str(), Filename
.str()}).second
)
77 NewModuleFiles
.push_back(Filename
.str());
81 PrebuiltModuleFilesT
&PrebuiltModuleFiles
;
82 llvm::SmallVector
<std::string
> &NewModuleFiles
;
85 /// Visit the given prebuilt module and collect all of the modules it
86 /// transitively imports and contributing input files.
87 static void visitPrebuiltModule(StringRef PrebuiltModuleFilename
,
89 PrebuiltModuleFilesT
&ModuleFiles
) {
90 // List of module files to be processed.
91 llvm::SmallVector
<std::string
> Worklist
{PrebuiltModuleFilename
.str()};
92 PrebuiltModuleListener
Listener(ModuleFiles
, Worklist
);
94 while (!Worklist
.empty())
95 ASTReader::readASTFileControlBlock(
96 Worklist
.pop_back_val(), CI
.getFileManager(), CI
.getModuleCache(),
97 CI
.getPCHContainerReader(),
98 /*FindModuleFileExtensions=*/false, Listener
,
99 /*ValidateDiagnosticOptions=*/false);
102 /// Transform arbitrary file name into an object-like file name.
103 static std::string
makeObjFileName(StringRef FileName
) {
104 SmallString
<128> ObjFileName(FileName
);
105 llvm::sys::path::replace_extension(ObjFileName
, "o");
106 return std::string(ObjFileName
.str());
109 /// Deduce the dependency target based on the output file and input files.
111 deduceDepTarget(const std::string
&OutputFile
,
112 const SmallVectorImpl
<FrontendInputFile
> &InputFiles
) {
113 if (OutputFile
!= "-")
116 if (InputFiles
.empty() || !InputFiles
.front().isFile())
117 return "clang-scan-deps\\ dependency";
119 return makeObjFileName(InputFiles
.front().getFile());
122 /// Sanitize diagnostic options for dependency scan.
123 static void sanitizeDiagOpts(DiagnosticOptions
&DiagOpts
) {
124 // Don't print 'X warnings and Y errors generated'.
125 DiagOpts
.ShowCarets
= false;
126 // Don't write out diagnostic file.
127 DiagOpts
.DiagnosticSerializationFile
.clear();
128 // Don't emit warnings as errors (and all other warnings too).
129 DiagOpts
.IgnoreWarnings
= true;
132 /// A clang tool that runs the preprocessor in a mode that's optimized for
133 /// dependency scanning for the given compiler invocation.
134 class DependencyScanningAction
: public tooling::ToolAction
{
136 DependencyScanningAction(
137 StringRef WorkingDirectory
, DependencyConsumer
&Consumer
,
138 DependencyActionController
&Controller
,
139 llvm::IntrusiveRefCntPtr
<DependencyScanningWorkerFilesystem
> DepFS
,
140 ScanningOutputFormat Format
, ScanningOptimizations OptimizeArgs
,
141 bool EagerLoadModules
, bool DisableFree
,
142 std::optional
<StringRef
> ModuleName
= std::nullopt
)
143 : WorkingDirectory(WorkingDirectory
), Consumer(Consumer
),
144 Controller(Controller
), DepFS(std::move(DepFS
)), Format(Format
),
145 OptimizeArgs(OptimizeArgs
), EagerLoadModules(EagerLoadModules
),
146 DisableFree(DisableFree
), ModuleName(ModuleName
) {}
148 bool runInvocation(std::shared_ptr
<CompilerInvocation
> Invocation
,
149 FileManager
*FileMgr
,
150 std::shared_ptr
<PCHContainerOperations
> PCHContainerOps
,
151 DiagnosticConsumer
*DiagConsumer
) override
{
152 // Make a deep copy of the original Clang invocation.
153 CompilerInvocation
OriginalInvocation(*Invocation
);
154 // Restore the value of DisableFree, which may be modified by Tooling.
155 OriginalInvocation
.getFrontendOpts().DisableFree
= DisableFree
;
158 // Scanning runs once for the first -cc1 invocation in a chain of driver
159 // jobs. For any dependent jobs, reuse the scanning result and just
160 // update the LastCC1Arguments to correspond to the new invocation.
161 // FIXME: to support multi-arch builds, each arch requires a separate scan
162 setLastCC1Arguments(std::move(OriginalInvocation
));
168 // Create a compiler instance to handle the actual work.
169 ScanInstanceStorage
.emplace(std::move(PCHContainerOps
));
170 CompilerInstance
&ScanInstance
= *ScanInstanceStorage
;
171 ScanInstance
.setInvocation(std::move(Invocation
));
173 // Create the compiler's actual diagnostics engine.
174 sanitizeDiagOpts(ScanInstance
.getDiagnosticOpts());
175 ScanInstance
.createDiagnostics(DiagConsumer
, /*ShouldOwnClient=*/false);
176 if (!ScanInstance
.hasDiagnostics())
179 ScanInstance
.getPreprocessorOpts().AllowPCHWithDifferentModulesCachePath
=
182 ScanInstance
.getFrontendOpts().GenerateGlobalModuleIndex
= false;
183 ScanInstance
.getFrontendOpts().UseGlobalModuleIndex
= false;
184 ScanInstance
.getFrontendOpts().ModulesShareFileManager
= false;
185 ScanInstance
.getHeaderSearchOpts().ModuleFormat
= "raw";
187 ScanInstance
.setFileManager(FileMgr
);
188 // Support for virtual file system overlays.
189 FileMgr
->setVirtualFileSystem(createVFSFromCompilerInvocation(
190 ScanInstance
.getInvocation(), ScanInstance
.getDiagnostics(),
191 FileMgr
->getVirtualFileSystemPtr()));
193 ScanInstance
.createSourceManager(*FileMgr
);
195 // Store the list of prebuilt module files into header search options. This
196 // will prevent the implicit build to create duplicate modules and will
197 // force reuse of the existing prebuilt module files instead.
198 if (!ScanInstance
.getPreprocessorOpts().ImplicitPCHInclude
.empty())
200 ScanInstance
.getPreprocessorOpts().ImplicitPCHInclude
, ScanInstance
,
201 ScanInstance
.getHeaderSearchOpts().PrebuiltModuleFiles
);
203 // Use the dependency scanning optimized file system if requested to do so.
205 llvm::IntrusiveRefCntPtr
<DependencyScanningWorkerFilesystem
> LocalDepFS
=
207 ScanInstance
.getPreprocessorOpts().DependencyDirectivesForFile
=
208 [LocalDepFS
= std::move(LocalDepFS
)](FileEntryRef File
)
209 -> std::optional
<ArrayRef
<dependency_directives_scan::Directive
>> {
210 if (llvm::ErrorOr
<EntryRef
> Entry
=
211 LocalDepFS
->getOrCreateFileSystemEntry(File
.getName()))
212 return Entry
->getDirectiveTokens();
217 // Create the dependency collector that will collect the produced
220 // This also moves the existing dependency output options from the
221 // invocation to the collector. The options in the invocation are reset,
222 // which ensures that the compiler won't create new dependency collectors,
223 // and thus won't write out the extra '.d' files to disk.
224 auto Opts
= std::make_unique
<DependencyOutputOptions
>();
225 std::swap(*Opts
, ScanInstance
.getInvocation().getDependencyOutputOpts());
226 // We need at least one -MT equivalent for the generator of make dependency
228 if (Opts
->Targets
.empty())
230 deduceDepTarget(ScanInstance
.getFrontendOpts().OutputFile
,
231 ScanInstance
.getFrontendOpts().Inputs
)};
232 Opts
->IncludeSystemHeaders
= true;
235 case ScanningOutputFormat::Make
:
236 ScanInstance
.addDependencyCollector(
237 std::make_shared
<DependencyConsumerForwarder
>(
238 std::move(Opts
), WorkingDirectory
, Consumer
));
240 case ScanningOutputFormat::P1689
:
241 case ScanningOutputFormat::Full
:
242 MDC
= std::make_shared
<ModuleDepCollector
>(
243 std::move(Opts
), ScanInstance
, Consumer
, Controller
,
244 OriginalInvocation
, OptimizeArgs
, EagerLoadModules
,
245 Format
== ScanningOutputFormat::P1689
);
246 ScanInstance
.addDependencyCollector(MDC
);
250 // Consider different header search and diagnostic options to create
251 // different modules. This avoids the unsound aliasing of module PCMs.
253 // TODO: Implement diagnostic bucketing to reduce the impact of strict
255 ScanInstance
.getHeaderSearchOpts().ModulesStrictContextHash
= true;
256 ScanInstance
.getHeaderSearchOpts().ModulesSkipDiagnosticOptions
= true;
257 ScanInstance
.getHeaderSearchOpts().ModulesSkipHeaderSearchPaths
= true;
258 ScanInstance
.getHeaderSearchOpts().ModulesSkipPragmaDiagnosticMappings
=
261 // Avoid some checks and module map parsing when loading PCM files.
262 ScanInstance
.getPreprocessorOpts().ModulesCheckRelocated
= false;
264 std::unique_ptr
<FrontendAction
> Action
;
267 Action
= std::make_unique
<GetDependenciesByModuleNameAction
>(*ModuleName
);
269 Action
= std::make_unique
<ReadPCHAndPreprocessAction
>();
271 const bool Result
= ScanInstance
.ExecuteAction(*Action
);
274 setLastCC1Arguments(std::move(OriginalInvocation
));
279 bool hasScanned() const { return Scanned
; }
281 /// Take the cc1 arguments corresponding to the most recent invocation used
282 /// with this action. Any modifications implied by the discovered dependencies
283 /// will have already been applied.
284 std::vector
<std::string
> takeLastCC1Arguments() {
285 std::vector
<std::string
> Result
;
286 std::swap(Result
, LastCC1Arguments
); // Reset LastCC1Arguments to empty.
291 void setLastCC1Arguments(CompilerInvocation
&&CI
) {
293 MDC
->applyDiscoveredDependencies(CI
);
294 LastCC1Arguments
= CI
.getCC1CommandLine();
298 StringRef WorkingDirectory
;
299 DependencyConsumer
&Consumer
;
300 DependencyActionController
&Controller
;
301 llvm::IntrusiveRefCntPtr
<DependencyScanningWorkerFilesystem
> DepFS
;
302 ScanningOutputFormat Format
;
303 ScanningOptimizations OptimizeArgs
;
304 bool EagerLoadModules
;
306 std::optional
<StringRef
> ModuleName
;
307 std::optional
<CompilerInstance
> ScanInstanceStorage
;
308 std::shared_ptr
<ModuleDepCollector
> MDC
;
309 std::vector
<std::string
> LastCC1Arguments
;
310 bool Scanned
= false;
313 } // end anonymous namespace
315 DependencyScanningWorker::DependencyScanningWorker(
316 DependencyScanningService
&Service
,
317 llvm::IntrusiveRefCntPtr
<llvm::vfs::FileSystem
> FS
)
318 : Format(Service
.getFormat()), OptimizeArgs(Service
.getOptimizeArgs()),
319 EagerLoadModules(Service
.shouldEagerLoadModules()) {
320 PCHContainerOps
= std::make_shared
<PCHContainerOperations
>();
321 // We need to read object files from PCH built outside the scanner.
322 PCHContainerOps
->registerReader(
323 std::make_unique
<ObjectFilePCHContainerReader
>());
324 // The scanner itself writes only raw ast files.
325 PCHContainerOps
->registerWriter(std::make_unique
<RawPCHContainerWriter
>());
327 switch (Service
.getMode()) {
328 case ScanningMode::DependencyDirectivesScan
:
330 new DependencyScanningWorkerFilesystem(Service
.getSharedCache(), FS
);
333 case ScanningMode::CanonicalPreprocessing
:
340 llvm::Error
DependencyScanningWorker::computeDependencies(
341 StringRef WorkingDirectory
, const std::vector
<std::string
> &CommandLine
,
342 DependencyConsumer
&Consumer
, DependencyActionController
&Controller
,
343 std::optional
<StringRef
> ModuleName
) {
344 std::vector
<const char *> CLI
;
345 for (const std::string
&Arg
: CommandLine
)
346 CLI
.push_back(Arg
.c_str());
347 auto DiagOpts
= CreateAndPopulateDiagOpts(CLI
);
348 sanitizeDiagOpts(*DiagOpts
);
350 // Capture the emitted diagnostics and report them to the client
351 // in the case of a failure.
352 std::string DiagnosticOutput
;
353 llvm::raw_string_ostream
DiagnosticsOS(DiagnosticOutput
);
354 TextDiagnosticPrinter
DiagPrinter(DiagnosticsOS
, DiagOpts
.release());
356 if (computeDependencies(WorkingDirectory
, CommandLine
, Consumer
, Controller
,
357 DiagPrinter
, ModuleName
))
358 return llvm::Error::success();
359 return llvm::make_error
<llvm::StringError
>(DiagnosticsOS
.str(),
360 llvm::inconvertibleErrorCode());
363 static bool forEachDriverJob(
364 ArrayRef
<std::string
> ArgStrs
, DiagnosticsEngine
&Diags
, FileManager
&FM
,
365 llvm::function_ref
<bool(const driver::Command
&Cmd
)> Callback
) {
366 SmallVector
<const char *, 256> Argv
;
367 Argv
.reserve(ArgStrs
.size());
368 for (const std::string
&Arg
: ArgStrs
)
369 Argv
.push_back(Arg
.c_str());
371 llvm::vfs::FileSystem
*FS
= &FM
.getVirtualFileSystem();
373 std::unique_ptr
<driver::Driver
> Driver
= std::make_unique
<driver::Driver
>(
374 Argv
[0], llvm::sys::getDefaultTargetTriple(), Diags
,
375 "clang LLVM compiler", FS
);
376 Driver
->setTitle("clang_based_tool");
378 llvm::BumpPtrAllocator Alloc
;
379 bool CLMode
= driver::IsClangCL(
380 driver::getDriverMode(Argv
[0], ArrayRef(Argv
).slice(1)));
382 if (llvm::Error E
= driver::expandResponseFiles(Argv
, CLMode
, Alloc
, FS
)) {
383 Diags
.Report(diag::err_drv_expand_response_file
)
384 << llvm::toString(std::move(E
));
388 const std::unique_ptr
<driver::Compilation
> Compilation(
389 Driver
->BuildCompilation(llvm::ArrayRef(Argv
)));
393 if (Compilation
->containsError())
396 for (const driver::Command
&Job
: Compilation
->getJobs()) {
403 static bool createAndRunToolInvocation(
404 std::vector
<std::string
> CommandLine
, DependencyScanningAction
&Action
,
406 std::shared_ptr
<clang::PCHContainerOperations
> &PCHContainerOps
,
407 DiagnosticsEngine
&Diags
, DependencyConsumer
&Consumer
) {
409 // Save executable path before providing CommandLine to ToolInvocation
410 std::string Executable
= CommandLine
[0];
411 ToolInvocation
Invocation(std::move(CommandLine
), &Action
, &FM
,
413 Invocation
.setDiagnosticConsumer(Diags
.getClient());
414 Invocation
.setDiagnosticOptions(&Diags
.getDiagnosticOptions());
415 if (!Invocation
.run())
418 std::vector
<std::string
> Args
= Action
.takeLastCC1Arguments();
419 Consumer
.handleBuildCommand({std::move(Executable
), std::move(Args
)});
423 bool DependencyScanningWorker::computeDependencies(
424 StringRef WorkingDirectory
, const std::vector
<std::string
> &CommandLine
,
425 DependencyConsumer
&Consumer
, DependencyActionController
&Controller
,
426 DiagnosticConsumer
&DC
, std::optional
<StringRef
> ModuleName
) {
427 // Reset what might have been modified in the previous worker invocation.
428 BaseFS
->setCurrentWorkingDirectory(WorkingDirectory
);
430 std::optional
<std::vector
<std::string
>> ModifiedCommandLine
;
431 llvm::IntrusiveRefCntPtr
<llvm::vfs::FileSystem
> ModifiedFS
;
433 // If we're scanning based on a module name alone, we don't expect the client
434 // to provide us with an input file. However, the driver really wants to have
435 // one. Let's just make it up to make the driver happy.
438 llvm::makeIntrusiveRefCnt
<llvm::vfs::OverlayFileSystem
>(BaseFS
);
440 llvm::makeIntrusiveRefCnt
<llvm::vfs::InMemoryFileSystem
>();
441 InMemoryFS
->setCurrentWorkingDirectory(WorkingDirectory
);
442 OverlayFS
->pushOverlay(InMemoryFS
);
443 ModifiedFS
= OverlayFS
;
445 SmallString
<128> FakeInputPath
;
446 // TODO: We should retry the creation if the path already exists.
447 llvm::sys::fs::createUniquePath(*ModuleName
+ "-%%%%%%%%.input",
449 /*MakeAbsolute=*/false);
450 InMemoryFS
->addFile(FakeInputPath
, 0, llvm::MemoryBuffer::getMemBuffer(""));
452 ModifiedCommandLine
= CommandLine
;
453 ModifiedCommandLine
->emplace_back(FakeInputPath
);
456 const std::vector
<std::string
> &FinalCommandLine
=
457 ModifiedCommandLine
? *ModifiedCommandLine
: CommandLine
;
458 auto &FinalFS
= ModifiedFS
? ModifiedFS
: BaseFS
;
460 FileSystemOptions FSOpts
;
461 FSOpts
.WorkingDir
= WorkingDirectory
.str();
462 auto FileMgr
= llvm::makeIntrusiveRefCnt
<FileManager
>(FSOpts
, FinalFS
);
464 std::vector
<const char *> FinalCCommandLine(FinalCommandLine
.size(), nullptr);
465 llvm::transform(FinalCommandLine
, FinalCCommandLine
.begin(),
466 [](const std::string
&Str
) { return Str
.c_str(); });
468 auto DiagOpts
= CreateAndPopulateDiagOpts(FinalCCommandLine
);
469 sanitizeDiagOpts(*DiagOpts
);
470 IntrusiveRefCntPtr
<DiagnosticsEngine
> Diags
=
471 CompilerInstance::createDiagnostics(DiagOpts
.release(), &DC
,
472 /*ShouldOwnClient=*/false);
474 // Although `Diagnostics` are used only for command-line parsing, the
475 // custom `DiagConsumer` might expect a `SourceManager` to be present.
476 SourceManager
SrcMgr(*Diags
, *FileMgr
);
477 Diags
->setSourceManager(&SrcMgr
);
478 // DisableFree is modified by Tooling for running
479 // in-process; preserve the original value, which is
480 // always true for a driver invocation.
481 bool DisableFree
= true;
482 DependencyScanningAction
Action(WorkingDirectory
, Consumer
, Controller
, DepFS
,
483 Format
, OptimizeArgs
, EagerLoadModules
,
484 DisableFree
, ModuleName
);
486 bool Success
= false;
487 if (FinalCommandLine
[1] == "-cc1") {
488 Success
= createAndRunToolInvocation(FinalCommandLine
, Action
, *FileMgr
,
489 PCHContainerOps
, *Diags
, Consumer
);
491 Success
= forEachDriverJob(
492 FinalCommandLine
, *Diags
, *FileMgr
, [&](const driver::Command
&Cmd
) {
493 if (StringRef(Cmd
.getCreator().getName()) != "clang") {
494 // Non-clang command. Just pass through to the dependency
496 Consumer
.handleBuildCommand(
497 {Cmd
.getExecutable(),
498 {Cmd
.getArguments().begin(), Cmd
.getArguments().end()}});
502 // Insert -cc1 comand line options into Argv
503 std::vector
<std::string
> Argv
;
504 Argv
.push_back(Cmd
.getExecutable());
505 Argv
.insert(Argv
.end(), Cmd
.getArguments().begin(),
506 Cmd
.getArguments().end());
508 // Create an invocation that uses the underlying file
509 // system to ensure that any file system requests that
510 // are made by the driver do not go through the
511 // dependency scanning filesystem.
512 return createAndRunToolInvocation(std::move(Argv
), Action
, *FileMgr
,
513 PCHContainerOps
, *Diags
, Consumer
);
517 if (Success
&& !Action
.hasScanned())
518 Diags
->Report(diag::err_fe_expected_compiler_job
)
519 << llvm::join(FinalCommandLine
, " ");
520 return Success
&& Action
.hasScanned();
523 DependencyActionController::~DependencyActionController() {}