Clang] Fix expansion of response files in -Wp after integrated-cc1 change
[llvm-project.git] / clang / tools / driver / driver.cpp
blob4457e40ff04bf072b14e98591ceb6a7e4d6ae7dd
1 //===-- driver.cpp - Clang GCC-Compatible Driver --------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This is the entry point to the clang driver; it is a thin wrapper
10 // for functionality in the Driver clang library.
12 //===----------------------------------------------------------------------===//
14 #include "clang/Driver/Driver.h"
15 #include "clang/Basic/DiagnosticOptions.h"
16 #include "clang/Basic/Stack.h"
17 #include "clang/Config/config.h"
18 #include "clang/Driver/Compilation.h"
19 #include "clang/Driver/DriverDiagnostic.h"
20 #include "clang/Driver/Options.h"
21 #include "clang/Driver/ToolChain.h"
22 #include "clang/Frontend/ChainedDiagnosticConsumer.h"
23 #include "clang/Frontend/CompilerInvocation.h"
24 #include "clang/Frontend/SerializedDiagnosticPrinter.h"
25 #include "clang/Frontend/TextDiagnosticPrinter.h"
26 #include "clang/Frontend/Utils.h"
27 #include "llvm/ADT/ArrayRef.h"
28 #include "llvm/ADT/SmallString.h"
29 #include "llvm/ADT/SmallVector.h"
30 #include "llvm/Option/ArgList.h"
31 #include "llvm/Option/OptTable.h"
32 #include "llvm/Option/Option.h"
33 #include "llvm/Support/CommandLine.h"
34 #include "llvm/Support/CrashRecoveryContext.h"
35 #include "llvm/Support/ErrorHandling.h"
36 #include "llvm/Support/FileSystem.h"
37 #include "llvm/Support/Host.h"
38 #include "llvm/Support/InitLLVM.h"
39 #include "llvm/Support/Path.h"
40 #include "llvm/Support/Process.h"
41 #include "llvm/Support/Program.h"
42 #include "llvm/Support/Regex.h"
43 #include "llvm/Support/Signals.h"
44 #include "llvm/Support/StringSaver.h"
45 #include "llvm/Support/TargetSelect.h"
46 #include "llvm/Support/Timer.h"
47 #include "llvm/Support/raw_ostream.h"
48 #include <memory>
49 #include <set>
50 #include <system_error>
51 using namespace clang;
52 using namespace clang::driver;
53 using namespace llvm::opt;
55 std::string GetExecutablePath(const char *Argv0, bool CanonicalPrefixes) {
56 if (!CanonicalPrefixes) {
57 SmallString<128> ExecutablePath(Argv0);
58 // Do a PATH lookup if Argv0 isn't a valid path.
59 if (!llvm::sys::fs::exists(ExecutablePath))
60 if (llvm::ErrorOr<std::string> P =
61 llvm::sys::findProgramByName(ExecutablePath))
62 ExecutablePath = *P;
63 return ExecutablePath.str();
66 // This just needs to be some symbol in the binary; C++ doesn't
67 // allow taking the address of ::main however.
68 void *P = (void*) (intptr_t) GetExecutablePath;
69 return llvm::sys::fs::getMainExecutable(Argv0, P);
72 static const char *GetStableCStr(std::set<std::string> &SavedStrings,
73 StringRef S) {
74 return SavedStrings.insert(S).first->c_str();
77 /// ApplyQAOverride - Apply a list of edits to the input argument lists.
78 ///
79 /// The input string is a space separate list of edits to perform,
80 /// they are applied in order to the input argument lists. Edits
81 /// should be one of the following forms:
82 ///
83 /// '#': Silence information about the changes to the command line arguments.
84 ///
85 /// '^': Add FOO as a new argument at the beginning of the command line.
86 ///
87 /// '+': Add FOO as a new argument at the end of the command line.
88 ///
89 /// 's/XXX/YYY/': Substitute the regular expression XXX with YYY in the command
90 /// line.
91 ///
92 /// 'xOPTION': Removes all instances of the literal argument OPTION.
93 ///
94 /// 'XOPTION': Removes all instances of the literal argument OPTION,
95 /// and the following argument.
96 ///
97 /// 'Ox': Removes all flags matching 'O' or 'O[sz0-9]' and adds 'Ox'
98 /// at the end of the command line.
99 ///
100 /// \param OS - The stream to write edit information to.
101 /// \param Args - The vector of command line arguments.
102 /// \param Edit - The override command to perform.
103 /// \param SavedStrings - Set to use for storing string representations.
104 static void ApplyOneQAOverride(raw_ostream &OS,
105 SmallVectorImpl<const char*> &Args,
106 StringRef Edit,
107 std::set<std::string> &SavedStrings) {
108 // This does not need to be efficient.
110 if (Edit[0] == '^') {
111 const char *Str =
112 GetStableCStr(SavedStrings, Edit.substr(1));
113 OS << "### Adding argument " << Str << " at beginning\n";
114 Args.insert(Args.begin() + 1, Str);
115 } else if (Edit[0] == '+') {
116 const char *Str =
117 GetStableCStr(SavedStrings, Edit.substr(1));
118 OS << "### Adding argument " << Str << " at end\n";
119 Args.push_back(Str);
120 } else if (Edit[0] == 's' && Edit[1] == '/' && Edit.endswith("/") &&
121 Edit.slice(2, Edit.size()-1).find('/') != StringRef::npos) {
122 StringRef MatchPattern = Edit.substr(2).split('/').first;
123 StringRef ReplPattern = Edit.substr(2).split('/').second;
124 ReplPattern = ReplPattern.slice(0, ReplPattern.size()-1);
126 for (unsigned i = 1, e = Args.size(); i != e; ++i) {
127 // Ignore end-of-line response file markers
128 if (Args[i] == nullptr)
129 continue;
130 std::string Repl = llvm::Regex(MatchPattern).sub(ReplPattern, Args[i]);
132 if (Repl != Args[i]) {
133 OS << "### Replacing '" << Args[i] << "' with '" << Repl << "'\n";
134 Args[i] = GetStableCStr(SavedStrings, Repl);
137 } else if (Edit[0] == 'x' || Edit[0] == 'X') {
138 auto Option = Edit.substr(1);
139 for (unsigned i = 1; i < Args.size();) {
140 if (Option == Args[i]) {
141 OS << "### Deleting argument " << Args[i] << '\n';
142 Args.erase(Args.begin() + i);
143 if (Edit[0] == 'X') {
144 if (i < Args.size()) {
145 OS << "### Deleting argument " << Args[i] << '\n';
146 Args.erase(Args.begin() + i);
147 } else
148 OS << "### Invalid X edit, end of command line!\n";
150 } else
151 ++i;
153 } else if (Edit[0] == 'O') {
154 for (unsigned i = 1; i < Args.size();) {
155 const char *A = Args[i];
156 // Ignore end-of-line response file markers
157 if (A == nullptr)
158 continue;
159 if (A[0] == '-' && A[1] == 'O' &&
160 (A[2] == '\0' ||
161 (A[3] == '\0' && (A[2] == 's' || A[2] == 'z' ||
162 ('0' <= A[2] && A[2] <= '9'))))) {
163 OS << "### Deleting argument " << Args[i] << '\n';
164 Args.erase(Args.begin() + i);
165 } else
166 ++i;
168 OS << "### Adding argument " << Edit << " at end\n";
169 Args.push_back(GetStableCStr(SavedStrings, '-' + Edit.str()));
170 } else {
171 OS << "### Unrecognized edit: " << Edit << "\n";
175 /// ApplyQAOverride - Apply a comma separate list of edits to the
176 /// input argument lists. See ApplyOneQAOverride.
177 static void ApplyQAOverride(SmallVectorImpl<const char*> &Args,
178 const char *OverrideStr,
179 std::set<std::string> &SavedStrings) {
180 raw_ostream *OS = &llvm::errs();
182 if (OverrideStr[0] == '#') {
183 ++OverrideStr;
184 OS = &llvm::nulls();
187 *OS << "### CCC_OVERRIDE_OPTIONS: " << OverrideStr << "\n";
189 // This does not need to be efficient.
191 const char *S = OverrideStr;
192 while (*S) {
193 const char *End = ::strchr(S, ' ');
194 if (!End)
195 End = S + strlen(S);
196 if (End != S)
197 ApplyOneQAOverride(*OS, Args, std::string(S, End), SavedStrings);
198 S = End;
199 if (*S != '\0')
200 ++S;
204 extern int cc1_main(ArrayRef<const char *> Argv, const char *Argv0,
205 void *MainAddr);
206 extern int cc1as_main(ArrayRef<const char *> Argv, const char *Argv0,
207 void *MainAddr);
208 extern int cc1gen_reproducer_main(ArrayRef<const char *> Argv,
209 const char *Argv0, void *MainAddr);
211 static void insertTargetAndModeArgs(const ParsedClangName &NameParts,
212 SmallVectorImpl<const char *> &ArgVector,
213 std::set<std::string> &SavedStrings) {
214 // Put target and mode arguments at the start of argument list so that
215 // arguments specified in command line could override them. Avoid putting
216 // them at index 0, as an option like '-cc1' must remain the first.
217 int InsertionPoint = 0;
218 if (ArgVector.size() > 0)
219 ++InsertionPoint;
221 if (NameParts.DriverMode) {
222 // Add the mode flag to the arguments.
223 ArgVector.insert(ArgVector.begin() + InsertionPoint,
224 GetStableCStr(SavedStrings, NameParts.DriverMode));
227 if (NameParts.TargetIsValid) {
228 const char *arr[] = {"-target", GetStableCStr(SavedStrings,
229 NameParts.TargetPrefix)};
230 ArgVector.insert(ArgVector.begin() + InsertionPoint,
231 std::begin(arr), std::end(arr));
235 static void getCLEnvVarOptions(std::string &EnvValue, llvm::StringSaver &Saver,
236 SmallVectorImpl<const char *> &Opts) {
237 llvm::cl::TokenizeWindowsCommandLine(EnvValue, Saver, Opts);
238 // The first instance of '#' should be replaced with '=' in each option.
239 for (const char *Opt : Opts)
240 if (char *NumberSignPtr = const_cast<char *>(::strchr(Opt, '#')))
241 *NumberSignPtr = '=';
244 static void SetBackdoorDriverOutputsFromEnvVars(Driver &TheDriver) {
245 // Handle CC_PRINT_OPTIONS and CC_PRINT_OPTIONS_FILE.
246 TheDriver.CCPrintOptions = !!::getenv("CC_PRINT_OPTIONS");
247 if (TheDriver.CCPrintOptions)
248 TheDriver.CCPrintOptionsFilename = ::getenv("CC_PRINT_OPTIONS_FILE");
250 // Handle CC_PRINT_HEADERS and CC_PRINT_HEADERS_FILE.
251 TheDriver.CCPrintHeaders = !!::getenv("CC_PRINT_HEADERS");
252 if (TheDriver.CCPrintHeaders)
253 TheDriver.CCPrintHeadersFilename = ::getenv("CC_PRINT_HEADERS_FILE");
255 // Handle CC_LOG_DIAGNOSTICS and CC_LOG_DIAGNOSTICS_FILE.
256 TheDriver.CCLogDiagnostics = !!::getenv("CC_LOG_DIAGNOSTICS");
257 if (TheDriver.CCLogDiagnostics)
258 TheDriver.CCLogDiagnosticsFilename = ::getenv("CC_LOG_DIAGNOSTICS_FILE");
261 static void FixupDiagPrefixExeName(TextDiagnosticPrinter *DiagClient,
262 const std::string &Path) {
263 // If the clang binary happens to be named cl.exe for compatibility reasons,
264 // use clang-cl.exe as the prefix to avoid confusion between clang and MSVC.
265 StringRef ExeBasename(llvm::sys::path::stem(Path));
266 if (ExeBasename.equals_lower("cl"))
267 ExeBasename = "clang-cl";
268 DiagClient->setPrefix(ExeBasename);
271 // This lets us create the DiagnosticsEngine with a properly-filled-out
272 // DiagnosticOptions instance.
273 static DiagnosticOptions *
274 CreateAndPopulateDiagOpts(ArrayRef<const char *> argv, bool &UseNewCC1Process) {
275 auto *DiagOpts = new DiagnosticOptions;
276 unsigned MissingArgIndex, MissingArgCount;
277 InputArgList Args = getDriverOptTable().ParseArgs(
278 argv.slice(1), MissingArgIndex, MissingArgCount);
279 // We ignore MissingArgCount and the return value of ParseDiagnosticArgs.
280 // Any errors that would be diagnosed here will also be diagnosed later,
281 // when the DiagnosticsEngine actually exists.
282 (void)ParseDiagnosticArgs(*DiagOpts, Args);
284 UseNewCC1Process =
285 Args.hasFlag(clang::driver::options::OPT_fno_integrated_cc1,
286 clang::driver::options::OPT_fintegrated_cc1,
287 /*Default=*/CLANG_SPAWN_CC1);
289 return DiagOpts;
292 static void SetInstallDir(SmallVectorImpl<const char *> &argv,
293 Driver &TheDriver, bool CanonicalPrefixes) {
294 // Attempt to find the original path used to invoke the driver, to determine
295 // the installed path. We do this manually, because we want to support that
296 // path being a symlink.
297 SmallString<128> InstalledPath(argv[0]);
299 // Do a PATH lookup, if there are no directory components.
300 if (llvm::sys::path::filename(InstalledPath) == InstalledPath)
301 if (llvm::ErrorOr<std::string> Tmp = llvm::sys::findProgramByName(
302 llvm::sys::path::filename(InstalledPath.str())))
303 InstalledPath = *Tmp;
305 // FIXME: We don't actually canonicalize this, we just make it absolute.
306 if (CanonicalPrefixes)
307 llvm::sys::fs::make_absolute(InstalledPath);
309 StringRef InstalledPathParent(llvm::sys::path::parent_path(InstalledPath));
310 if (llvm::sys::fs::exists(InstalledPathParent))
311 TheDriver.setInstalledDir(InstalledPathParent);
314 static int ExecuteCC1Tool(SmallVectorImpl<const char *> &ArgV) {
315 // If we call the cc1 tool from the clangDriver library (through
316 // Driver::CC1Main), we need to clean up the options usage count. The options
317 // are currently global, and they might have been used previously by the
318 // driver.
319 llvm::cl::ResetAllOptionOccurrences();
321 llvm::BumpPtrAllocator A;
322 llvm::StringSaver Saver(A);
323 llvm::cl::ExpandResponseFiles(Saver, &llvm::cl::TokenizeGNUCommandLine, ArgV,
324 /*MarkEOLs=*/false);
325 StringRef Tool = ArgV[1];
326 void *GetExecutablePathVP = (void *)(intptr_t)GetExecutablePath;
327 if (Tool == "-cc1")
328 return cc1_main(makeArrayRef(ArgV).slice(2), ArgV[0], GetExecutablePathVP);
329 if (Tool == "-cc1as")
330 return cc1as_main(makeArrayRef(ArgV).slice(2), ArgV[0],
331 GetExecutablePathVP);
332 if (Tool == "-cc1gen-reproducer")
333 return cc1gen_reproducer_main(makeArrayRef(ArgV).slice(2), ArgV[0],
334 GetExecutablePathVP);
335 // Reject unknown tools.
336 llvm::errs() << "error: unknown integrated tool '" << Tool << "'. "
337 << "Valid tools include '-cc1' and '-cc1as'.\n";
338 return 1;
341 int main(int argc_, const char **argv_) {
342 noteBottomOfStack();
343 llvm::InitLLVM X(argc_, argv_);
344 SmallVector<const char *, 256> argv(argv_, argv_ + argc_);
346 if (llvm::sys::Process::FixupStandardFileDescriptors())
347 return 1;
349 llvm::InitializeAllTargets();
350 auto TargetAndMode = ToolChain::getTargetAndModeFromProgramName(argv[0]);
352 llvm::BumpPtrAllocator A;
353 llvm::StringSaver Saver(A);
355 // Parse response files using the GNU syntax, unless we're in CL mode. There
356 // are two ways to put clang in CL compatibility mode: argv[0] is either
357 // clang-cl or cl, or --driver-mode=cl is on the command line. The normal
358 // command line parsing can't happen until after response file parsing, so we
359 // have to manually search for a --driver-mode=cl argument the hard way.
360 // Finally, our -cc1 tools don't care which tokenization mode we use because
361 // response files written by clang will tokenize the same way in either mode.
362 bool ClangCLMode = false;
363 if (StringRef(TargetAndMode.DriverMode).equals("--driver-mode=cl") ||
364 llvm::find_if(argv, [](const char *F) {
365 return F && strcmp(F, "--driver-mode=cl") == 0;
366 }) != argv.end()) {
367 ClangCLMode = true;
369 enum { Default, POSIX, Windows } RSPQuoting = Default;
370 for (const char *F : argv) {
371 if (strcmp(F, "--rsp-quoting=posix") == 0)
372 RSPQuoting = POSIX;
373 else if (strcmp(F, "--rsp-quoting=windows") == 0)
374 RSPQuoting = Windows;
377 // Determines whether we want nullptr markers in argv to indicate response
378 // files end-of-lines. We only use this for the /LINK driver argument with
379 // clang-cl.exe on Windows.
380 bool MarkEOLs = ClangCLMode;
382 llvm::cl::TokenizerCallback Tokenizer;
383 if (RSPQuoting == Windows || (RSPQuoting == Default && ClangCLMode))
384 Tokenizer = &llvm::cl::TokenizeWindowsCommandLine;
385 else
386 Tokenizer = &llvm::cl::TokenizeGNUCommandLine;
388 if (MarkEOLs && argv.size() > 1 && StringRef(argv[1]).startswith("-cc1"))
389 MarkEOLs = false;
390 llvm::cl::ExpandResponseFiles(Saver, Tokenizer, argv, MarkEOLs);
392 // Handle -cc1 integrated tools, even if -cc1 was expanded from a response
393 // file.
394 auto FirstArg = std::find_if(argv.begin() + 1, argv.end(),
395 [](const char *A) { return A != nullptr; });
396 if (FirstArg != argv.end() && StringRef(*FirstArg).startswith("-cc1")) {
397 // If -cc1 came from a response file, remove the EOL sentinels.
398 if (MarkEOLs) {
399 auto newEnd = std::remove(argv.begin(), argv.end(), nullptr);
400 argv.resize(newEnd - argv.begin());
402 return ExecuteCC1Tool(argv);
405 // Handle options that need handling before the real command line parsing in
406 // Driver::BuildCompilation()
407 bool CanonicalPrefixes = true;
408 for (int i = 1, size = argv.size(); i < size; ++i) {
409 // Skip end-of-line response file markers
410 if (argv[i] == nullptr)
411 continue;
412 if (StringRef(argv[i]) == "-no-canonical-prefixes") {
413 CanonicalPrefixes = false;
414 break;
418 // Handle CL and _CL_ which permits additional command line options to be
419 // prepended or appended.
420 if (ClangCLMode) {
421 // Arguments in "CL" are prepended.
422 llvm::Optional<std::string> OptCL = llvm::sys::Process::GetEnv("CL");
423 if (OptCL.hasValue()) {
424 SmallVector<const char *, 8> PrependedOpts;
425 getCLEnvVarOptions(OptCL.getValue(), Saver, PrependedOpts);
427 // Insert right after the program name to prepend to the argument list.
428 argv.insert(argv.begin() + 1, PrependedOpts.begin(), PrependedOpts.end());
430 // Arguments in "_CL_" are appended.
431 llvm::Optional<std::string> Opt_CL_ = llvm::sys::Process::GetEnv("_CL_");
432 if (Opt_CL_.hasValue()) {
433 SmallVector<const char *, 8> AppendedOpts;
434 getCLEnvVarOptions(Opt_CL_.getValue(), Saver, AppendedOpts);
436 // Insert at the end of the argument list to append.
437 argv.append(AppendedOpts.begin(), AppendedOpts.end());
441 std::set<std::string> SavedStrings;
442 // Handle CCC_OVERRIDE_OPTIONS, used for editing a command line behind the
443 // scenes.
444 if (const char *OverrideStr = ::getenv("CCC_OVERRIDE_OPTIONS")) {
445 // FIXME: Driver shouldn't take extra initial argument.
446 ApplyQAOverride(argv, OverrideStr, SavedStrings);
449 std::string Path = GetExecutablePath(argv[0], CanonicalPrefixes);
451 // Whether the cc1 tool should be called inside the current process, or if we
452 // should spawn a new clang subprocess (old behavior).
453 // Not having an additional process saves some execution time of Windows,
454 // and makes debugging and profiling easier.
455 bool UseNewCC1Process;
457 IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts =
458 CreateAndPopulateDiagOpts(argv, UseNewCC1Process);
460 TextDiagnosticPrinter *DiagClient
461 = new TextDiagnosticPrinter(llvm::errs(), &*DiagOpts);
462 FixupDiagPrefixExeName(DiagClient, Path);
464 IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
466 DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagClient);
468 if (!DiagOpts->DiagnosticSerializationFile.empty()) {
469 auto SerializedConsumer =
470 clang::serialized_diags::create(DiagOpts->DiagnosticSerializationFile,
471 &*DiagOpts, /*MergeChildRecords=*/true);
472 Diags.setClient(new ChainedDiagnosticConsumer(
473 Diags.takeClient(), std::move(SerializedConsumer)));
476 ProcessWarningOptions(Diags, *DiagOpts, /*ReportDiags=*/false);
478 Driver TheDriver(Path, llvm::sys::getDefaultTargetTriple(), Diags);
479 SetInstallDir(argv, TheDriver, CanonicalPrefixes);
480 TheDriver.setTargetAndMode(TargetAndMode);
482 insertTargetAndModeArgs(TargetAndMode, argv, SavedStrings);
484 SetBackdoorDriverOutputsFromEnvVars(TheDriver);
486 if (!UseNewCC1Process) {
487 TheDriver.CC1Main = &ExecuteCC1Tool;
488 // Ensure the CC1Command actually catches cc1 crashes
489 llvm::CrashRecoveryContext::Enable();
492 std::unique_ptr<Compilation> C(TheDriver.BuildCompilation(argv));
493 int Res = 1;
494 if (C && !C->containsError()) {
495 SmallVector<std::pair<int, const Command *>, 4> FailingCommands;
496 Res = TheDriver.ExecuteCompilation(*C, FailingCommands);
498 // Force a crash to test the diagnostics.
499 if (TheDriver.GenReproducer) {
500 Diags.Report(diag::err_drv_force_crash)
501 << !::getenv("FORCE_CLANG_DIAGNOSTICS_CRASH");
503 // Pretend that every command failed.
504 FailingCommands.clear();
505 for (const auto &J : C->getJobs())
506 if (const Command *C = dyn_cast<Command>(&J))
507 FailingCommands.push_back(std::make_pair(-1, C));
510 for (const auto &P : FailingCommands) {
511 int CommandRes = P.first;
512 const Command *FailingCommand = P.second;
513 if (!Res)
514 Res = CommandRes;
516 // If result status is < 0, then the driver command signalled an error.
517 // If result status is 70, then the driver command reported a fatal error.
518 // On Windows, abort will return an exit code of 3. In these cases,
519 // generate additional diagnostic information if possible.
520 bool DiagnoseCrash = CommandRes < 0 || CommandRes == 70;
521 #ifdef _WIN32
522 DiagnoseCrash |= CommandRes == 3;
523 #endif
524 if (DiagnoseCrash) {
525 TheDriver.generateCompilationDiagnostics(*C, *FailingCommand);
526 break;
531 Diags.getClient()->finish();
533 // If any timers were active but haven't been destroyed yet, print their
534 // results now. This happens in -disable-free mode.
535 llvm::TimerGroup::printAll(llvm::errs());
536 llvm::TimerGroup::clearAll();
538 #ifdef _WIN32
539 // Exit status should not be negative on Win32, unless abnormal termination.
540 // Once abnormal termination was caught, negative status should not be
541 // propagated.
542 if (Res < 0)
543 Res = 1;
544 #endif
546 // If we have multiple failing commands, we return the result of the first
547 // failing command.
548 return Res;