1 //===- Job.cpp - Command to Execute ---------------------------------------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 #include "clang/Driver/Job.h"
10 #include "clang/Basic/LLVM.h"
11 #include "clang/Driver/Driver.h"
12 #include "clang/Driver/DriverDiagnostic.h"
13 #include "clang/Driver/InputInfo.h"
14 #include "clang/Driver/Tool.h"
15 #include "clang/Driver/ToolChain.h"
16 #include "llvm/ADT/ArrayRef.h"
17 #include "llvm/ADT/SmallString.h"
18 #include "llvm/ADT/SmallVector.h"
19 #include "llvm/ADT/StringExtras.h"
20 #include "llvm/ADT/StringRef.h"
21 #include "llvm/ADT/StringSet.h"
22 #include "llvm/ADT/StringSwitch.h"
23 #include "llvm/Support/CrashRecoveryContext.h"
24 #include "llvm/Support/FileSystem.h"
25 #include "llvm/Support/Path.h"
26 #include "llvm/Support/PrettyStackTrace.h"
27 #include "llvm/Support/Program.h"
28 #include "llvm/Support/raw_ostream.h"
33 #include <system_error>
36 using namespace clang
;
37 using namespace driver
;
39 Command::Command(const Action
&Source
, const Tool
&Creator
,
40 ResponseFileSupport ResponseSupport
, const char *Executable
,
41 const llvm::opt::ArgStringList
&Arguments
,
42 ArrayRef
<InputInfo
> Inputs
, ArrayRef
<InputInfo
> Outputs
,
43 const char *PrependArg
)
44 : Source(Source
), Creator(Creator
), ResponseSupport(ResponseSupport
),
45 Executable(Executable
), PrependArg(PrependArg
), Arguments(Arguments
) {
46 for (const auto &II
: Inputs
)
48 InputInfoList
.push_back(II
);
49 for (const auto &II
: Outputs
)
51 OutputFilenames
.push_back(II
.getFilename());
54 /// Check if the compiler flag in question should be skipped when
55 /// emitting a reproducer. Also track how many arguments it has and if the
56 /// option is some kind of include path.
57 static bool skipArgs(const char *Flag
, bool HaveCrashVFS
, int &SkipNum
,
60 // These flags are all of the form -Flag <Arg> and are treated as two
61 // arguments. Therefore, we need to skip the flag and the next argument.
62 bool ShouldSkip
= llvm::StringSwitch
<bool>(Flag
)
63 .Cases("-MF", "-MT", "-MQ", "-serialize-diagnostic-file", true)
64 .Cases("-o", "-dependency-file", true)
65 .Cases("-fdebug-compilation-dir", "-diagnostic-log-file", true)
66 .Cases("-dwarf-debug-flags", "-ivfsoverlay", true)
71 // Some include flags shouldn't be skipped if we have a crash VFS
72 IsInclude
= llvm::StringSwitch
<bool>(Flag
)
73 .Cases("-include", "-header-include-file", true)
74 .Cases("-idirafter", "-internal-isystem", "-iwithprefix", true)
75 .Cases("-internal-externc-isystem", "-iprefix", true)
76 .Cases("-iwithprefixbefore", "-isystem", "-iquote", true)
77 .Cases("-isysroot", "-I", "-F", "-resource-dir", true)
78 .Cases("-iframework", "-include-pch", true)
83 // The remaining flags are treated as a single argument.
85 // These flags are all of the form -Flag and have no second argument.
86 ShouldSkip
= llvm::StringSwitch
<bool>(Flag
)
87 .Cases("-M", "-MM", "-MG", "-MP", "-MD", true)
96 // These flags are treated as a single argument (e.g., -F<Dir>).
97 StringRef
FlagRef(Flag
);
98 IsInclude
= FlagRef
.startswith("-F") || FlagRef
.startswith("-I");
100 return !HaveCrashVFS
;
101 if (FlagRef
.startswith("-fmodules-cache-path="))
108 void Command::writeResponseFile(raw_ostream
&OS
) const {
109 // In a file list, we only write the set of inputs to the response file
110 if (ResponseSupport
.ResponseKind
== ResponseFileSupport::RF_FileList
) {
111 for (const auto *Arg
: InputFileList
) {
117 // In regular response files, we send all arguments to the response file.
118 // Wrapping all arguments in double quotes ensures that both Unix tools and
119 // Windows tools understand the response file.
120 for (const auto *Arg
: Arguments
) {
123 for (; *Arg
!= '\0'; Arg
++) {
124 if (*Arg
== '\"' || *Arg
== '\\') {
134 void Command::buildArgvForResponseFile(
135 llvm::SmallVectorImpl
<const char *> &Out
) const {
136 // When not a file list, all arguments are sent to the response file.
137 // This leaves us to set the argv to a single parameter, requesting the tool
138 // to read the response file.
139 if (ResponseSupport
.ResponseKind
!= ResponseFileSupport::RF_FileList
) {
140 Out
.push_back(Executable
);
141 Out
.push_back(ResponseFileFlag
.c_str());
145 llvm::StringSet
<> Inputs
;
146 for (const auto *InputName
: InputFileList
)
147 Inputs
.insert(InputName
);
148 Out
.push_back(Executable
);
151 Out
.push_back(PrependArg
);
153 // In a file list, build args vector ignoring parameters that will go in the
154 // response file (elements of the InputFileList vector)
155 bool FirstInput
= true;
156 for (const auto *Arg
: Arguments
) {
157 if (Inputs
.count(Arg
) == 0) {
159 } else if (FirstInput
) {
161 Out
.push_back(ResponseSupport
.ResponseFlag
);
162 Out
.push_back(ResponseFile
);
167 /// Rewrite relative include-like flag paths to absolute ones.
169 rewriteIncludes(const llvm::ArrayRef
<const char *> &Args
, size_t Idx
,
171 llvm::SmallVectorImpl
<llvm::SmallString
<128>> &IncFlags
) {
172 using namespace llvm
;
175 auto getAbsPath
= [](StringRef InInc
, SmallVectorImpl
<char> &OutInc
) -> bool {
176 if (path::is_absolute(InInc
)) // Nothing to do here...
178 std::error_code EC
= fs::current_path(OutInc
);
181 path::append(OutInc
, InInc
);
185 SmallString
<128> NewInc
;
187 StringRef
FlagRef(Args
[Idx
+ NumArgs
- 1]);
188 assert((FlagRef
.startswith("-F") || FlagRef
.startswith("-I")) &&
189 "Expecting -I or -F");
190 StringRef Inc
= FlagRef
.slice(2, StringRef::npos
);
191 if (getAbsPath(Inc
, NewInc
)) {
192 SmallString
<128> NewArg(FlagRef
.slice(0, 2));
194 IncFlags
.push_back(std::move(NewArg
));
199 assert(NumArgs
== 2 && "Not expecting more than two arguments");
200 StringRef
Inc(Args
[Idx
+ NumArgs
- 1]);
201 if (!getAbsPath(Inc
, NewInc
))
203 IncFlags
.push_back(SmallString
<128>(Args
[Idx
]));
204 IncFlags
.push_back(std::move(NewInc
));
207 void Command::Print(raw_ostream
&OS
, const char *Terminator
, bool Quote
,
208 CrashReportInfo
*CrashInfo
) const {
209 // Always quote the exe.
211 llvm::sys::printArg(OS
, Executable
, /*Quote=*/true);
213 ArrayRef
<const char *> Args
= Arguments
;
214 SmallVector
<const char *, 128> ArgsRespFile
;
215 if (ResponseFile
!= nullptr) {
216 buildArgvForResponseFile(ArgsRespFile
);
217 Args
= ArrayRef
<const char *>(ArgsRespFile
).slice(1); // no executable name
218 } else if (PrependArg
) {
220 llvm::sys::printArg(OS
, PrependArg
, /*Quote=*/true);
223 bool HaveCrashVFS
= CrashInfo
&& !CrashInfo
->VFSPath
.empty();
224 for (size_t i
= 0, e
= Args
.size(); i
< e
; ++i
) {
225 const char *const Arg
= Args
[i
];
229 bool IsInclude
= false;
230 if (skipArgs(Arg
, HaveCrashVFS
, NumArgs
, IsInclude
)) {
235 // Relative includes need to be expanded to absolute paths.
236 if (HaveCrashVFS
&& IsInclude
) {
237 SmallVector
<SmallString
<128>, 2> NewIncFlags
;
238 rewriteIncludes(Args
, i
, NumArgs
, NewIncFlags
);
239 if (!NewIncFlags
.empty()) {
240 for (auto &F
: NewIncFlags
) {
242 llvm::sys::printArg(OS
, F
.c_str(), Quote
);
249 auto Found
= llvm::find_if(InputInfoList
, [&Arg
](const InputInfo
&II
) {
250 return II
.getFilename() == Arg
;
252 if (Found
!= InputInfoList
.end() &&
253 (i
== 0 || StringRef(Args
[i
- 1]) != "-main-file-name")) {
254 // Replace the input file name with the crashinfo's file name.
256 StringRef ShortName
= llvm::sys::path::filename(CrashInfo
->Filename
);
257 llvm::sys::printArg(OS
, ShortName
.str(), Quote
);
263 llvm::sys::printArg(OS
, Arg
, Quote
);
266 if (CrashInfo
&& HaveCrashVFS
) {
268 llvm::sys::printArg(OS
, "-ivfsoverlay", Quote
);
270 llvm::sys::printArg(OS
, CrashInfo
->VFSPath
.str(), Quote
);
272 // The leftover modules from the crash are stored in
273 // <name>.cache/vfs/modules
274 // Leave it untouched for pcm inspection and provide a clean/empty dir
275 // path to contain the future generated module cache:
276 // <name>.cache/vfs/repro-modules
277 SmallString
<128> RelModCacheDir
= llvm::sys::path::parent_path(
278 llvm::sys::path::parent_path(CrashInfo
->VFSPath
));
279 llvm::sys::path::append(RelModCacheDir
, "repro-modules");
281 std::string ModCachePath
= "-fmodules-cache-path=";
282 ModCachePath
.append(RelModCacheDir
.c_str());
285 llvm::sys::printArg(OS
, ModCachePath
, Quote
);
288 if (ResponseFile
!= nullptr) {
289 OS
<< "\n Arguments passed via response file:\n";
290 writeResponseFile(OS
);
291 // Avoiding duplicated newline terminator, since FileLists are
292 // newline-separated.
293 if (ResponseSupport
.ResponseKind
!= ResponseFileSupport::RF_FileList
)
295 OS
<< " (end of response file)";
301 void Command::setResponseFile(const char *FileName
) {
302 ResponseFile
= FileName
;
303 ResponseFileFlag
= ResponseSupport
.ResponseFlag
;
304 ResponseFileFlag
+= FileName
;
307 void Command::setEnvironment(llvm::ArrayRef
<const char *> NewEnvironment
) {
308 Environment
.reserve(NewEnvironment
.size() + 1);
309 Environment
.assign(NewEnvironment
.begin(), NewEnvironment
.end());
310 Environment
.push_back(nullptr);
313 void Command::setRedirectFiles(
314 const std::vector
<std::optional
<std::string
>> &Redirects
) {
315 RedirectFiles
= Redirects
;
318 void Command::PrintFileNames() const {
319 if (PrintInputFilenames
) {
320 for (const auto &Arg
: InputInfoList
)
321 llvm::outs() << llvm::sys::path::filename(Arg
.getFilename()) << "\n";
322 llvm::outs().flush();
326 int Command::Execute(ArrayRef
<std::optional
<StringRef
>> Redirects
,
327 std::string
*ErrMsg
, bool *ExecutionFailed
) const {
330 SmallVector
<const char *, 128> Argv
;
331 if (ResponseFile
== nullptr) {
332 Argv
.push_back(Executable
);
334 Argv
.push_back(PrependArg
);
335 Argv
.append(Arguments
.begin(), Arguments
.end());
336 Argv
.push_back(nullptr);
338 // If the command is too large, we need to put arguments in a response file.
339 std::string RespContents
;
340 llvm::raw_string_ostream
SS(RespContents
);
342 // Write file contents and build the Argv vector
343 writeResponseFile(SS
);
344 buildArgvForResponseFile(Argv
);
345 Argv
.push_back(nullptr);
348 // Save the response file in the appropriate encoding
349 if (std::error_code EC
= writeFileWithEncoding(
350 ResponseFile
, RespContents
, ResponseSupport
.ResponseEncoding
)) {
352 *ErrMsg
= EC
.message();
354 *ExecutionFailed
= true;
355 // Return -1 by convention (see llvm/include/llvm/Support/Program.h) to
356 // indicate the requested executable cannot be started.
361 std::optional
<ArrayRef
<StringRef
>> Env
;
362 std::vector
<StringRef
> ArgvVectorStorage
;
363 if (!Environment
.empty()) {
364 assert(Environment
.back() == nullptr &&
365 "Environment vector should be null-terminated by now");
366 ArgvVectorStorage
= llvm::toStringRefArray(Environment
.data());
367 Env
= ArrayRef(ArgvVectorStorage
);
370 auto Args
= llvm::toStringRefArray(Argv
.data());
372 // Use Job-specific redirect files if they are present.
373 if (!RedirectFiles
.empty()) {
374 std::vector
<std::optional
<StringRef
>> RedirectFilesOptional
;
375 for (const auto &Ele
: RedirectFiles
)
377 RedirectFilesOptional
.push_back(std::optional
<StringRef
>(*Ele
));
379 RedirectFilesOptional
.push_back(std::nullopt
);
381 return llvm::sys::ExecuteAndWait(Executable
, Args
, Env
,
382 ArrayRef(RedirectFilesOptional
),
383 /*secondsToWait=*/0, /*memoryLimit=*/0,
384 ErrMsg
, ExecutionFailed
, &ProcStat
);
387 return llvm::sys::ExecuteAndWait(Executable
, Args
, Env
, Redirects
,
388 /*secondsToWait*/ 0, /*memoryLimit*/ 0,
389 ErrMsg
, ExecutionFailed
, &ProcStat
);
392 CC1Command::CC1Command(const Action
&Source
, const Tool
&Creator
,
393 ResponseFileSupport ResponseSupport
,
394 const char *Executable
,
395 const llvm::opt::ArgStringList
&Arguments
,
396 ArrayRef
<InputInfo
> Inputs
, ArrayRef
<InputInfo
> Outputs
,
397 const char *PrependArg
)
398 : Command(Source
, Creator
, ResponseSupport
, Executable
, Arguments
, Inputs
,
399 Outputs
, PrependArg
) {
403 void CC1Command::Print(raw_ostream
&OS
, const char *Terminator
, bool Quote
,
404 CrashReportInfo
*CrashInfo
) const {
406 OS
<< " (in-process)\n";
407 Command::Print(OS
, Terminator
, Quote
, CrashInfo
);
410 int CC1Command::Execute(ArrayRef
<std::optional
<StringRef
>> Redirects
,
411 std::string
*ErrMsg
, bool *ExecutionFailed
) const {
412 // FIXME: Currently, if there're more than one job, we disable
413 // -fintegrate-cc1. If we're no longer a integrated-cc1 job, fallback to
414 // out-of-process execution. See discussion in https://reviews.llvm.org/D74447
416 return Command::Execute(Redirects
, ErrMsg
, ExecutionFailed
);
420 SmallVector
<const char *, 128> Argv
;
421 Argv
.push_back(getExecutable());
422 Argv
.append(getArguments().begin(), getArguments().end());
423 Argv
.push_back(nullptr);
424 Argv
.pop_back(); // The terminating null element shall not be part of the
425 // slice (main() behavior).
427 // This flag simply indicates that the program couldn't start, which isn't
430 *ExecutionFailed
= false;
432 llvm::CrashRecoveryContext CRC
;
433 CRC
.DumpStackAndCleanupOnFailure
= true;
435 const void *PrettyState
= llvm::SavePrettyStackState();
436 const Driver
&D
= getCreator().getToolChain().getDriver();
439 // Enter ExecuteCC1Tool() instead of starting up a new process
440 if (!CRC
.RunSafely([&]() { R
= D
.CC1Main(Argv
); })) {
441 llvm::RestorePrettyStackState(PrettyState
);
447 void CC1Command::setEnvironment(llvm::ArrayRef
<const char *> NewEnvironment
) {
448 // We don't support set a new environment when calling into ExecuteCC1Tool()
450 "The CC1Command doesn't support changing the environment vars!");
453 void JobList::Print(raw_ostream
&OS
, const char *Terminator
, bool Quote
,
454 CrashReportInfo
*CrashInfo
) const {
455 for (const auto &Job
: *this)
456 Job
.Print(OS
, Terminator
, Quote
, CrashInfo
);
459 void JobList::clear() { Jobs
.clear(); }