[Flang] remove whole-archive option for AIX linker (#76039)
[llvm-project.git] / clang / lib / Tooling / DependencyScanning / DependencyScanningWorker.cpp
blobc54e6d523800b6b80048e11259efbe72c5818473
1 //===- DependencyScanningWorker.cpp - clang-scan-deps worker --------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
9 #include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h"
10 #include "clang/Basic/DiagnosticDriver.h"
11 #include "clang/Basic/DiagnosticFrontend.h"
12 #include "clang/CodeGen/ObjectFilePCHContainerOperations.h"
13 #include "clang/Driver/Compilation.h"
14 #include "clang/Driver/Driver.h"
15 #include "clang/Driver/Job.h"
16 #include "clang/Driver/Tool.h"
17 #include "clang/Frontend/CompilerInstance.h"
18 #include "clang/Frontend/CompilerInvocation.h"
19 #include "clang/Frontend/FrontendActions.h"
20 #include "clang/Frontend/TextDiagnosticPrinter.h"
21 #include "clang/Frontend/Utils.h"
22 #include "clang/Lex/PreprocessorOptions.h"
23 #include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
24 #include "clang/Tooling/DependencyScanning/ModuleDepCollector.h"
25 #include "clang/Tooling/Tooling.h"
26 #include "llvm/Support/Allocator.h"
27 #include "llvm/Support/Error.h"
28 #include "llvm/TargetParser/Host.h"
29 #include <optional>
31 using namespace clang;
32 using namespace tooling;
33 using namespace dependencies;
35 namespace {
37 /// Forwards the gatherered dependencies to the consumer.
38 class DependencyConsumerForwarder : public DependencyFileGenerator {
39 public:
40 DependencyConsumerForwarder(std::unique_ptr<DependencyOutputOptions> Opts,
41 StringRef WorkingDirectory, DependencyConsumer &C)
42 : DependencyFileGenerator(*Opts), WorkingDirectory(WorkingDirectory),
43 Opts(std::move(Opts)), C(C) {}
45 void finishedMainFile(DiagnosticsEngine &Diags) override {
46 C.handleDependencyOutputOpts(*Opts);
47 llvm::SmallString<256> CanonPath;
48 for (const auto &File : getDependencies()) {
49 CanonPath = File;
50 llvm::sys::path::remove_dots(CanonPath, /*remove_dot_dot=*/true);
51 llvm::sys::fs::make_absolute(WorkingDirectory, CanonPath);
52 C.handleFileDependency(CanonPath);
56 private:
57 StringRef WorkingDirectory;
58 std::unique_ptr<DependencyOutputOptions> Opts;
59 DependencyConsumer &C;
62 using PrebuiltModuleFilesT = decltype(HeaderSearchOptions::PrebuiltModuleFiles);
64 /// A listener that collects the imported modules and optionally the input
65 /// files.
66 class PrebuiltModuleListener : public ASTReaderListener {
67 public:
68 PrebuiltModuleListener(PrebuiltModuleFilesT &PrebuiltModuleFiles,
69 llvm::SmallVector<std::string> &NewModuleFiles)
70 : PrebuiltModuleFiles(PrebuiltModuleFiles),
71 NewModuleFiles(NewModuleFiles) {}
73 bool needsImportVisitation() const override { return true; }
75 void visitImport(StringRef ModuleName, StringRef Filename) override {
76 if (PrebuiltModuleFiles.insert({ModuleName.str(), Filename.str()}).second)
77 NewModuleFiles.push_back(Filename.str());
80 private:
81 PrebuiltModuleFilesT &PrebuiltModuleFiles;
82 llvm::SmallVector<std::string> &NewModuleFiles;
85 /// Visit the given prebuilt module and collect all of the modules it
86 /// transitively imports and contributing input files.
87 static void visitPrebuiltModule(StringRef PrebuiltModuleFilename,
88 CompilerInstance &CI,
89 PrebuiltModuleFilesT &ModuleFiles) {
90 // List of module files to be processed.
91 llvm::SmallVector<std::string> Worklist{PrebuiltModuleFilename.str()};
92 PrebuiltModuleListener Listener(ModuleFiles, Worklist);
94 while (!Worklist.empty())
95 ASTReader::readASTFileControlBlock(
96 Worklist.pop_back_val(), CI.getFileManager(), CI.getModuleCache(),
97 CI.getPCHContainerReader(),
98 /*FindModuleFileExtensions=*/false, Listener,
99 /*ValidateDiagnosticOptions=*/false);
102 /// Transform arbitrary file name into an object-like file name.
103 static std::string makeObjFileName(StringRef FileName) {
104 SmallString<128> ObjFileName(FileName);
105 llvm::sys::path::replace_extension(ObjFileName, "o");
106 return std::string(ObjFileName.str());
109 /// Deduce the dependency target based on the output file and input files.
110 static std::string
111 deduceDepTarget(const std::string &OutputFile,
112 const SmallVectorImpl<FrontendInputFile> &InputFiles) {
113 if (OutputFile != "-")
114 return OutputFile;
116 if (InputFiles.empty() || !InputFiles.front().isFile())
117 return "clang-scan-deps\\ dependency";
119 return makeObjFileName(InputFiles.front().getFile());
122 /// Sanitize diagnostic options for dependency scan.
123 static void sanitizeDiagOpts(DiagnosticOptions &DiagOpts) {
124 // Don't print 'X warnings and Y errors generated'.
125 DiagOpts.ShowCarets = false;
126 // Don't write out diagnostic file.
127 DiagOpts.DiagnosticSerializationFile.clear();
128 // Don't emit warnings as errors (and all other warnings too).
129 DiagOpts.IgnoreWarnings = true;
132 /// A clang tool that runs the preprocessor in a mode that's optimized for
133 /// dependency scanning for the given compiler invocation.
134 class DependencyScanningAction : public tooling::ToolAction {
135 public:
136 DependencyScanningAction(
137 StringRef WorkingDirectory, DependencyConsumer &Consumer,
138 DependencyActionController &Controller,
139 llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS,
140 ScanningOutputFormat Format, ScanningOptimizations OptimizeArgs,
141 bool EagerLoadModules, bool DisableFree,
142 std::optional<StringRef> ModuleName = std::nullopt)
143 : WorkingDirectory(WorkingDirectory), Consumer(Consumer),
144 Controller(Controller), DepFS(std::move(DepFS)), Format(Format),
145 OptimizeArgs(OptimizeArgs), EagerLoadModules(EagerLoadModules),
146 DisableFree(DisableFree), ModuleName(ModuleName) {}
148 bool runInvocation(std::shared_ptr<CompilerInvocation> Invocation,
149 FileManager *FileMgr,
150 std::shared_ptr<PCHContainerOperations> PCHContainerOps,
151 DiagnosticConsumer *DiagConsumer) override {
152 // Make a deep copy of the original Clang invocation.
153 CompilerInvocation OriginalInvocation(*Invocation);
154 // Restore the value of DisableFree, which may be modified by Tooling.
155 OriginalInvocation.getFrontendOpts().DisableFree = DisableFree;
157 if (Scanned) {
158 // Scanning runs once for the first -cc1 invocation in a chain of driver
159 // jobs. For any dependent jobs, reuse the scanning result and just
160 // update the LastCC1Arguments to correspond to the new invocation.
161 // FIXME: to support multi-arch builds, each arch requires a separate scan
162 setLastCC1Arguments(std::move(OriginalInvocation));
163 return true;
166 Scanned = true;
168 // Create a compiler instance to handle the actual work.
169 ScanInstanceStorage.emplace(std::move(PCHContainerOps));
170 CompilerInstance &ScanInstance = *ScanInstanceStorage;
171 ScanInstance.setInvocation(std::move(Invocation));
173 // Create the compiler's actual diagnostics engine.
174 sanitizeDiagOpts(ScanInstance.getDiagnosticOpts());
175 ScanInstance.createDiagnostics(DiagConsumer, /*ShouldOwnClient=*/false);
176 if (!ScanInstance.hasDiagnostics())
177 return false;
179 ScanInstance.getPreprocessorOpts().AllowPCHWithDifferentModulesCachePath =
180 true;
182 ScanInstance.getFrontendOpts().GenerateGlobalModuleIndex = false;
183 ScanInstance.getFrontendOpts().UseGlobalModuleIndex = false;
184 ScanInstance.getFrontendOpts().ModulesShareFileManager = false;
185 ScanInstance.getHeaderSearchOpts().ModuleFormat = "raw";
187 ScanInstance.setFileManager(FileMgr);
188 // Support for virtual file system overlays.
189 FileMgr->setVirtualFileSystem(createVFSFromCompilerInvocation(
190 ScanInstance.getInvocation(), ScanInstance.getDiagnostics(),
191 FileMgr->getVirtualFileSystemPtr()));
193 ScanInstance.createSourceManager(*FileMgr);
195 // Store the list of prebuilt module files into header search options. This
196 // will prevent the implicit build to create duplicate modules and will
197 // force reuse of the existing prebuilt module files instead.
198 if (!ScanInstance.getPreprocessorOpts().ImplicitPCHInclude.empty())
199 visitPrebuiltModule(
200 ScanInstance.getPreprocessorOpts().ImplicitPCHInclude, ScanInstance,
201 ScanInstance.getHeaderSearchOpts().PrebuiltModuleFiles);
203 // Use the dependency scanning optimized file system if requested to do so.
204 if (DepFS) {
205 llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> LocalDepFS =
206 DepFS;
207 ScanInstance.getPreprocessorOpts().DependencyDirectivesForFile =
208 [LocalDepFS = std::move(LocalDepFS)](FileEntryRef File)
209 -> std::optional<ArrayRef<dependency_directives_scan::Directive>> {
210 if (llvm::ErrorOr<EntryRef> Entry =
211 LocalDepFS->getOrCreateFileSystemEntry(File.getName()))
212 return Entry->getDirectiveTokens();
213 return std::nullopt;
217 // Create the dependency collector that will collect the produced
218 // dependencies.
220 // This also moves the existing dependency output options from the
221 // invocation to the collector. The options in the invocation are reset,
222 // which ensures that the compiler won't create new dependency collectors,
223 // and thus won't write out the extra '.d' files to disk.
224 auto Opts = std::make_unique<DependencyOutputOptions>();
225 std::swap(*Opts, ScanInstance.getInvocation().getDependencyOutputOpts());
226 // We need at least one -MT equivalent for the generator of make dependency
227 // files to work.
228 if (Opts->Targets.empty())
229 Opts->Targets = {
230 deduceDepTarget(ScanInstance.getFrontendOpts().OutputFile,
231 ScanInstance.getFrontendOpts().Inputs)};
232 Opts->IncludeSystemHeaders = true;
234 switch (Format) {
235 case ScanningOutputFormat::Make:
236 ScanInstance.addDependencyCollector(
237 std::make_shared<DependencyConsumerForwarder>(
238 std::move(Opts), WorkingDirectory, Consumer));
239 break;
240 case ScanningOutputFormat::P1689:
241 case ScanningOutputFormat::Full:
242 MDC = std::make_shared<ModuleDepCollector>(
243 std::move(Opts), ScanInstance, Consumer, Controller,
244 OriginalInvocation, OptimizeArgs, EagerLoadModules,
245 Format == ScanningOutputFormat::P1689);
246 ScanInstance.addDependencyCollector(MDC);
247 break;
250 // Consider different header search and diagnostic options to create
251 // different modules. This avoids the unsound aliasing of module PCMs.
253 // TODO: Implement diagnostic bucketing to reduce the impact of strict
254 // context hashing.
255 ScanInstance.getHeaderSearchOpts().ModulesStrictContextHash = true;
256 ScanInstance.getHeaderSearchOpts().ModulesSkipDiagnosticOptions = true;
257 ScanInstance.getHeaderSearchOpts().ModulesSkipHeaderSearchPaths = true;
258 ScanInstance.getHeaderSearchOpts().ModulesSkipPragmaDiagnosticMappings =
259 true;
261 // Avoid some checks and module map parsing when loading PCM files.
262 ScanInstance.getPreprocessorOpts().ModulesCheckRelocated = false;
264 std::unique_ptr<FrontendAction> Action;
266 if (ModuleName)
267 Action = std::make_unique<GetDependenciesByModuleNameAction>(*ModuleName);
268 else
269 Action = std::make_unique<ReadPCHAndPreprocessAction>();
271 const bool Result = ScanInstance.ExecuteAction(*Action);
273 if (Result)
274 setLastCC1Arguments(std::move(OriginalInvocation));
276 return Result;
279 bool hasScanned() const { return Scanned; }
281 /// Take the cc1 arguments corresponding to the most recent invocation used
282 /// with this action. Any modifications implied by the discovered dependencies
283 /// will have already been applied.
284 std::vector<std::string> takeLastCC1Arguments() {
285 std::vector<std::string> Result;
286 std::swap(Result, LastCC1Arguments); // Reset LastCC1Arguments to empty.
287 return Result;
290 private:
291 void setLastCC1Arguments(CompilerInvocation &&CI) {
292 if (MDC)
293 MDC->applyDiscoveredDependencies(CI);
294 LastCC1Arguments = CI.getCC1CommandLine();
297 private:
298 StringRef WorkingDirectory;
299 DependencyConsumer &Consumer;
300 DependencyActionController &Controller;
301 llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS;
302 ScanningOutputFormat Format;
303 ScanningOptimizations OptimizeArgs;
304 bool EagerLoadModules;
305 bool DisableFree;
306 std::optional<StringRef> ModuleName;
307 std::optional<CompilerInstance> ScanInstanceStorage;
308 std::shared_ptr<ModuleDepCollector> MDC;
309 std::vector<std::string> LastCC1Arguments;
310 bool Scanned = false;
313 } // end anonymous namespace
315 DependencyScanningWorker::DependencyScanningWorker(
316 DependencyScanningService &Service,
317 llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS)
318 : Format(Service.getFormat()), OptimizeArgs(Service.getOptimizeArgs()),
319 EagerLoadModules(Service.shouldEagerLoadModules()) {
320 PCHContainerOps = std::make_shared<PCHContainerOperations>();
321 // We need to read object files from PCH built outside the scanner.
322 PCHContainerOps->registerReader(
323 std::make_unique<ObjectFilePCHContainerReader>());
324 // The scanner itself writes only raw ast files.
325 PCHContainerOps->registerWriter(std::make_unique<RawPCHContainerWriter>());
327 switch (Service.getMode()) {
328 case ScanningMode::DependencyDirectivesScan:
329 DepFS =
330 new DependencyScanningWorkerFilesystem(Service.getSharedCache(), FS);
331 BaseFS = DepFS;
332 break;
333 case ScanningMode::CanonicalPreprocessing:
334 DepFS = nullptr;
335 BaseFS = FS;
336 break;
340 llvm::Error DependencyScanningWorker::computeDependencies(
341 StringRef WorkingDirectory, const std::vector<std::string> &CommandLine,
342 DependencyConsumer &Consumer, DependencyActionController &Controller,
343 std::optional<StringRef> ModuleName) {
344 std::vector<const char *> CLI;
345 for (const std::string &Arg : CommandLine)
346 CLI.push_back(Arg.c_str());
347 auto DiagOpts = CreateAndPopulateDiagOpts(CLI);
348 sanitizeDiagOpts(*DiagOpts);
350 // Capture the emitted diagnostics and report them to the client
351 // in the case of a failure.
352 std::string DiagnosticOutput;
353 llvm::raw_string_ostream DiagnosticsOS(DiagnosticOutput);
354 TextDiagnosticPrinter DiagPrinter(DiagnosticsOS, DiagOpts.release());
356 if (computeDependencies(WorkingDirectory, CommandLine, Consumer, Controller,
357 DiagPrinter, ModuleName))
358 return llvm::Error::success();
359 return llvm::make_error<llvm::StringError>(DiagnosticsOS.str(),
360 llvm::inconvertibleErrorCode());
363 static bool forEachDriverJob(
364 ArrayRef<std::string> ArgStrs, DiagnosticsEngine &Diags, FileManager &FM,
365 llvm::function_ref<bool(const driver::Command &Cmd)> Callback) {
366 SmallVector<const char *, 256> Argv;
367 Argv.reserve(ArgStrs.size());
368 for (const std::string &Arg : ArgStrs)
369 Argv.push_back(Arg.c_str());
371 llvm::vfs::FileSystem *FS = &FM.getVirtualFileSystem();
373 std::unique_ptr<driver::Driver> Driver = std::make_unique<driver::Driver>(
374 Argv[0], llvm::sys::getDefaultTargetTriple(), Diags,
375 "clang LLVM compiler", FS);
376 Driver->setTitle("clang_based_tool");
378 llvm::BumpPtrAllocator Alloc;
379 bool CLMode = driver::IsClangCL(
380 driver::getDriverMode(Argv[0], ArrayRef(Argv).slice(1)));
382 if (llvm::Error E = driver::expandResponseFiles(Argv, CLMode, Alloc, FS)) {
383 Diags.Report(diag::err_drv_expand_response_file)
384 << llvm::toString(std::move(E));
385 return false;
388 const std::unique_ptr<driver::Compilation> Compilation(
389 Driver->BuildCompilation(llvm::ArrayRef(Argv)));
390 if (!Compilation)
391 return false;
393 if (Compilation->containsError())
394 return false;
396 for (const driver::Command &Job : Compilation->getJobs()) {
397 if (!Callback(Job))
398 return false;
400 return true;
403 static bool createAndRunToolInvocation(
404 std::vector<std::string> CommandLine, DependencyScanningAction &Action,
405 FileManager &FM,
406 std::shared_ptr<clang::PCHContainerOperations> &PCHContainerOps,
407 DiagnosticsEngine &Diags, DependencyConsumer &Consumer) {
409 // Save executable path before providing CommandLine to ToolInvocation
410 std::string Executable = CommandLine[0];
411 ToolInvocation Invocation(std::move(CommandLine), &Action, &FM,
412 PCHContainerOps);
413 Invocation.setDiagnosticConsumer(Diags.getClient());
414 Invocation.setDiagnosticOptions(&Diags.getDiagnosticOptions());
415 if (!Invocation.run())
416 return false;
418 std::vector<std::string> Args = Action.takeLastCC1Arguments();
419 Consumer.handleBuildCommand({std::move(Executable), std::move(Args)});
420 return true;
423 bool DependencyScanningWorker::computeDependencies(
424 StringRef WorkingDirectory, const std::vector<std::string> &CommandLine,
425 DependencyConsumer &Consumer, DependencyActionController &Controller,
426 DiagnosticConsumer &DC, std::optional<StringRef> ModuleName) {
427 // Reset what might have been modified in the previous worker invocation.
428 BaseFS->setCurrentWorkingDirectory(WorkingDirectory);
430 std::optional<std::vector<std::string>> ModifiedCommandLine;
431 llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> ModifiedFS;
433 // If we're scanning based on a module name alone, we don't expect the client
434 // to provide us with an input file. However, the driver really wants to have
435 // one. Let's just make it up to make the driver happy.
436 if (ModuleName) {
437 auto OverlayFS =
438 llvm::makeIntrusiveRefCnt<llvm::vfs::OverlayFileSystem>(BaseFS);
439 auto InMemoryFS =
440 llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
441 InMemoryFS->setCurrentWorkingDirectory(WorkingDirectory);
442 OverlayFS->pushOverlay(InMemoryFS);
443 ModifiedFS = OverlayFS;
445 SmallString<128> FakeInputPath;
446 // TODO: We should retry the creation if the path already exists.
447 llvm::sys::fs::createUniquePath(*ModuleName + "-%%%%%%%%.input",
448 FakeInputPath,
449 /*MakeAbsolute=*/false);
450 InMemoryFS->addFile(FakeInputPath, 0, llvm::MemoryBuffer::getMemBuffer(""));
452 ModifiedCommandLine = CommandLine;
453 ModifiedCommandLine->emplace_back(FakeInputPath);
456 const std::vector<std::string> &FinalCommandLine =
457 ModifiedCommandLine ? *ModifiedCommandLine : CommandLine;
458 auto &FinalFS = ModifiedFS ? ModifiedFS : BaseFS;
460 FileSystemOptions FSOpts;
461 FSOpts.WorkingDir = WorkingDirectory.str();
462 auto FileMgr = llvm::makeIntrusiveRefCnt<FileManager>(FSOpts, FinalFS);
464 std::vector<const char *> FinalCCommandLine(FinalCommandLine.size(), nullptr);
465 llvm::transform(FinalCommandLine, FinalCCommandLine.begin(),
466 [](const std::string &Str) { return Str.c_str(); });
468 auto DiagOpts = CreateAndPopulateDiagOpts(FinalCCommandLine);
469 sanitizeDiagOpts(*DiagOpts);
470 IntrusiveRefCntPtr<DiagnosticsEngine> Diags =
471 CompilerInstance::createDiagnostics(DiagOpts.release(), &DC,
472 /*ShouldOwnClient=*/false);
474 // Although `Diagnostics` are used only for command-line parsing, the
475 // custom `DiagConsumer` might expect a `SourceManager` to be present.
476 SourceManager SrcMgr(*Diags, *FileMgr);
477 Diags->setSourceManager(&SrcMgr);
478 // DisableFree is modified by Tooling for running
479 // in-process; preserve the original value, which is
480 // always true for a driver invocation.
481 bool DisableFree = true;
482 DependencyScanningAction Action(WorkingDirectory, Consumer, Controller, DepFS,
483 Format, OptimizeArgs, EagerLoadModules,
484 DisableFree, ModuleName);
486 bool Success = false;
487 if (FinalCommandLine[1] == "-cc1") {
488 Success = createAndRunToolInvocation(FinalCommandLine, Action, *FileMgr,
489 PCHContainerOps, *Diags, Consumer);
490 } else {
491 Success = forEachDriverJob(
492 FinalCommandLine, *Diags, *FileMgr, [&](const driver::Command &Cmd) {
493 if (StringRef(Cmd.getCreator().getName()) != "clang") {
494 // Non-clang command. Just pass through to the dependency
495 // consumer.
496 Consumer.handleBuildCommand(
497 {Cmd.getExecutable(),
498 {Cmd.getArguments().begin(), Cmd.getArguments().end()}});
499 return true;
502 // Insert -cc1 comand line options into Argv
503 std::vector<std::string> Argv;
504 Argv.push_back(Cmd.getExecutable());
505 Argv.insert(Argv.end(), Cmd.getArguments().begin(),
506 Cmd.getArguments().end());
508 // Create an invocation that uses the underlying file
509 // system to ensure that any file system requests that
510 // are made by the driver do not go through the
511 // dependency scanning filesystem.
512 return createAndRunToolInvocation(std::move(Argv), Action, *FileMgr,
513 PCHContainerOps, *Diags, Consumer);
517 if (Success && !Action.hasScanned())
518 Diags->Report(diag::err_fe_expected_compiler_job)
519 << llvm::join(FinalCommandLine, " ");
520 return Success && Action.hasScanned();
523 DependencyActionController::~DependencyActionController() {}