Add ICU message format support
[chromium-blink-merge.git] / tools / gn / setup.cc
blob033dab4fa15c1fc844e7be54834b852ec8f8b25e
1 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "tools/gn/setup.h"
7 #include <stdlib.h>
9 #include <algorithm>
10 #include <sstream>
12 #include "base/bind.h"
13 #include "base/command_line.h"
14 #include "base/files/file_path.h"
15 #include "base/files/file_util.h"
16 #include "base/process/launch.h"
17 #include "base/strings/string_split.h"
18 #include "base/strings/string_util.h"
19 #include "base/strings/utf_string_conversions.h"
20 #include "build/build_config.h"
21 #include "tools/gn/commands.h"
22 #include "tools/gn/filesystem_utils.h"
23 #include "tools/gn/input_file.h"
24 #include "tools/gn/parse_tree.h"
25 #include "tools/gn/parser.h"
26 #include "tools/gn/source_dir.h"
27 #include "tools/gn/source_file.h"
28 #include "tools/gn/standard_out.h"
29 #include "tools/gn/switches.h"
30 #include "tools/gn/tokenizer.h"
31 #include "tools/gn/trace.h"
32 #include "tools/gn/value.h"
34 #if defined(OS_WIN)
35 #include <windows.h>
36 #endif
38 extern const char kDotfile_Help[] =
39 ".gn file\n"
40 "\n"
41 " When gn starts, it will search the current directory and parent\n"
42 " directories for a file called \".gn\". This indicates the source root.\n"
43 " You can override this detection by using the --root command-line\n"
44 " argument\n"
45 "\n"
46 " The .gn file in the source root will be executed. The syntax is the\n"
47 " same as a buildfile, but with very limited build setup-specific\n"
48 " meaning.\n"
49 "\n"
50 " If you specify --root, by default GN will look for the file .gn in\n"
51 " that directory. If you want to specify a different file, you can\n"
52 " additionally pass --dotfile:\n"
53 "\n"
54 " gn gen out/Debug --root=/home/build --dotfile=/home/my_gn_file.gn\n"
55 "\n"
56 "Variables\n"
57 "\n"
58 " buildconfig [required]\n"
59 " Label of the build config file. This file will be used to set up\n"
60 " the build file execution environment for each toolchain.\n"
61 "\n"
62 " check_targets [optional]\n"
63 " A list of labels and label patterns that should be checked when\n"
64 " running \"gn check\" or \"gn gen --check\". If unspecified, all\n"
65 " targets will be checked. If it is the empty list, no targets will\n"
66 " be checked.\n"
67 "\n"
68 " The format of this list is identical to that of \"visibility\"\n"
69 " so see \"gn help visibility\" for examples.\n"
70 "\n"
71 " exec_script_whitelist [optional]\n"
72 " A list of .gn/.gni files (not labels) that have permission to call\n"
73 " the exec_script function. If this list is defined, calls to\n"
74 " exec_script will be checked against this list and GN will fail if\n"
75 " the current file isn't in the list.\n"
76 "\n"
77 " This is to allow the use of exec_script to be restricted since\n"
78 " is easy to use inappropriately. Wildcards are not supported.\n"
79 " Files in the secondary_source tree (if defined) should be\n"
80 " referenced by ignoring the secondary tree and naming them as if\n"
81 " they are in the main tree.\n"
82 "\n"
83 " If unspecified, the ability to call exec_script is unrestricted.\n"
84 "\n"
85 " Example:\n"
86 " exec_script_whitelist = [\n"
87 " \"//base/BUILD.gn\",\n"
88 " \"//build/my_config.gni\",\n"
89 " ]\n"
90 "\n"
91 " root [optional]\n"
92 " Label of the root build target. The GN build will start by loading\n"
93 " the build file containing this target name. This defaults to\n"
94 " \"//:\" which will cause the file //BUILD.gn to be loaded.\n"
95 "\n"
96 " secondary_source [optional]\n"
97 " Label of an alternate directory tree to find input files. When\n"
98 " searching for a BUILD.gn file (or the build config file discussed\n"
99 " above), the file will first be looked for in the source root.\n"
100 " If it's not found, the secondary source root will be checked\n"
101 " (which would contain a parallel directory hierarchy).\n"
102 "\n"
103 " This behavior is intended to be used when BUILD.gn files can't be\n"
104 " checked in to certain source directories for whatever reason.\n"
105 "\n"
106 " The secondary source root must be inside the main source tree.\n"
107 "\n"
108 "Example .gn file contents\n"
109 "\n"
110 " buildconfig = \"//build/config/BUILDCONFIG.gn\"\n"
111 "\n"
112 " check_targets = [\n"
113 " \"//doom_melon/*\", # Check everything in this subtree.\n"
114 " \"//tools:mind_controlling_ant\", # Check this specific target.\n"
115 " ]\n"
116 "\n"
117 " root = \"//:root\"\n"
118 "\n"
119 " secondary_source = \"//build/config/temporary_buildfiles/\"\n";
121 namespace {
123 const base::FilePath::CharType kGnFile[] = FILE_PATH_LITERAL(".gn");
125 base::FilePath FindDotFile(const base::FilePath& current_dir) {
126 base::FilePath try_this_file = current_dir.Append(kGnFile);
127 if (base::PathExists(try_this_file))
128 return try_this_file;
130 base::FilePath with_no_slash = current_dir.StripTrailingSeparators();
131 base::FilePath up_one_dir = with_no_slash.DirName();
132 if (up_one_dir == current_dir)
133 return base::FilePath(); // Got to the top.
135 return FindDotFile(up_one_dir);
138 // Called on any thread. Post the item to the builder on the main thread.
139 void ItemDefinedCallback(base::MessageLoop* main_loop,
140 scoped_refptr<Builder> builder,
141 scoped_ptr<Item> item) {
142 DCHECK(item);
143 main_loop->PostTask(FROM_HERE, base::Bind(&Builder::ItemDefined, builder,
144 base::Passed(&item)));
147 void DecrementWorkCount() {
148 g_scheduler->DecrementWorkCount();
151 } // namespace
153 const char Setup::kBuildArgFileName[] = "args.gn";
155 Setup::Setup()
156 : build_settings_(),
157 loader_(new LoaderImpl(&build_settings_)),
158 builder_(new Builder(loader_.get())),
159 root_build_file_("//BUILD.gn"),
160 check_public_headers_(false),
161 dotfile_settings_(&build_settings_, std::string()),
162 dotfile_scope_(&dotfile_settings_),
163 fill_arguments_(true) {
164 dotfile_settings_.set_toolchain_label(Label());
165 build_settings_.set_item_defined_callback(
166 base::Bind(&ItemDefinedCallback, scheduler_.main_loop(), builder_));
168 loader_->set_complete_callback(base::Bind(&DecrementWorkCount));
169 // The scheduler's main loop wasn't created when the Loader was created, so
170 // we need to set it now.
171 loader_->set_main_loop(scheduler_.main_loop());
174 Setup::~Setup() {
177 bool Setup::DoSetup(const std::string& build_dir, bool force_create) {
178 base::CommandLine* cmdline = base::CommandLine::ForCurrentProcess();
180 scheduler_.set_verbose_logging(cmdline->HasSwitch(switches::kVerbose));
181 if (cmdline->HasSwitch(switches::kTime) ||
182 cmdline->HasSwitch(switches::kTracelog))
183 EnableTracing();
185 ScopedTrace setup_trace(TraceItem::TRACE_SETUP, "DoSetup");
187 if (!FillSourceDir(*cmdline))
188 return false;
189 if (!RunConfigFile())
190 return false;
191 if (!FillOtherConfig(*cmdline))
192 return false;
194 // Must be after FillSourceDir to resolve.
195 if (!FillBuildDir(build_dir, !force_create))
196 return false;
198 // Check for unused variables in the .gn file.
199 Err err;
200 if (!dotfile_scope_.CheckForUnusedVars(&err)) {
201 err.PrintToStdout();
202 return false;
205 if (fill_arguments_) {
206 if (!FillArguments(*cmdline))
207 return false;
209 FillPythonPath();
211 return true;
214 bool Setup::Run() {
215 RunPreMessageLoop();
216 if (!scheduler_.Run())
217 return false;
218 return RunPostMessageLoop();
221 SourceFile Setup::GetBuildArgFile() const {
222 return SourceFile(build_settings_.build_dir().value() + kBuildArgFileName);
225 void Setup::RunPreMessageLoop() {
226 // Load the root build file.
227 loader_->Load(root_build_file_, LocationRange(), Label());
229 // Will be decremented with the loader is drained.
230 g_scheduler->IncrementWorkCount();
233 bool Setup::RunPostMessageLoop() {
234 Err err;
235 if (build_settings_.check_for_bad_items()) {
236 if (!builder_->CheckForBadItems(&err)) {
237 err.PrintToStdout();
238 return false;
241 if (!build_settings_.build_args().VerifyAllOverridesUsed(&err)) {
242 // TODO(brettw) implement a system of warnings. Until we have a better
243 // system, print the error but don't return failure.
244 err.PrintToStdout();
245 return true;
249 if (check_public_headers_) {
250 std::vector<const Target*> all_targets = builder_->GetAllResolvedTargets();
251 std::vector<const Target*> to_check;
252 if (check_patterns()) {
253 commands::FilterTargetsByPatterns(all_targets, *check_patterns(),
254 &to_check);
255 } else {
256 to_check = all_targets;
259 if (!commands::CheckPublicHeaders(&build_settings_, all_targets,
260 to_check, false)) {
261 return false;
265 // Write out tracing and timing if requested.
266 const base::CommandLine* cmdline = base::CommandLine::ForCurrentProcess();
267 if (cmdline->HasSwitch(switches::kTime))
268 PrintLongHelp(SummarizeTraces());
269 if (cmdline->HasSwitch(switches::kTracelog))
270 SaveTraces(cmdline->GetSwitchValuePath(switches::kTracelog));
272 return true;
275 bool Setup::FillArguments(const base::CommandLine& cmdline) {
276 // Use the args on the command line if specified, and save them. Do this even
277 // if the list is empty (this means clear any defaults).
278 if (cmdline.HasSwitch(switches::kArgs)) {
279 if (!FillArgsFromCommandLine(cmdline.GetSwitchValueASCII(switches::kArgs)))
280 return false;
281 SaveArgsToFile();
282 return true;
285 // No command line args given, use the arguments from the build dir (if any).
286 return FillArgsFromFile();
289 bool Setup::FillArgsFromCommandLine(const std::string& args) {
290 args_input_file_.reset(new InputFile(SourceFile()));
291 args_input_file_->SetContents(args);
292 args_input_file_->set_friendly_name("the command-line \"--args\"");
293 return FillArgsFromArgsInputFile();
296 bool Setup::FillArgsFromFile() {
297 ScopedTrace setup_trace(TraceItem::TRACE_SETUP, "Load args file");
299 SourceFile build_arg_source_file = GetBuildArgFile();
300 base::FilePath build_arg_file =
301 build_settings_.GetFullPath(build_arg_source_file);
303 std::string contents;
304 if (!base::ReadFileToString(build_arg_file, &contents))
305 return true; // File doesn't exist, continue with default args.
307 // Add a dependency on the build arguments file. If this changes, we want
308 // to re-generate the build.
309 g_scheduler->AddGenDependency(build_arg_file);
311 if (contents.empty())
312 return true; // Empty file, do nothing.
314 args_input_file_.reset(new InputFile(build_arg_source_file));
315 args_input_file_->SetContents(contents);
316 args_input_file_->set_friendly_name(
317 "build arg file (use \"gn args <out_dir>\" to edit)");
319 setup_trace.Done(); // Only want to count the load as part of the trace.
320 return FillArgsFromArgsInputFile();
323 bool Setup::FillArgsFromArgsInputFile() {
324 ScopedTrace setup_trace(TraceItem::TRACE_SETUP, "Parse args");
326 Err err;
327 args_tokens_ = Tokenizer::Tokenize(args_input_file_.get(), &err);
328 if (err.has_error()) {
329 err.PrintToStdout();
330 return false;
333 args_root_ = Parser::Parse(args_tokens_, &err);
334 if (err.has_error()) {
335 err.PrintToStdout();
336 return false;
339 Scope arg_scope(&dotfile_settings_);
340 args_root_->Execute(&arg_scope, &err);
341 if (err.has_error()) {
342 err.PrintToStdout();
343 return false;
346 // Save the result of the command args.
347 Scope::KeyValueMap overrides;
348 arg_scope.GetCurrentScopeValues(&overrides);
349 build_settings_.build_args().AddArgOverrides(overrides);
350 return true;
353 bool Setup::SaveArgsToFile() {
354 ScopedTrace setup_trace(TraceItem::TRACE_SETUP, "Save args file");
356 std::ostringstream stream;
357 for (const auto& pair : build_settings_.build_args().GetAllOverrides()) {
358 stream << pair.first.as_string() << " = " << pair.second.ToString(true);
359 stream << std::endl;
362 // For the first run, the build output dir might not be created yet, so do
363 // that so we can write a file into it. Ignore errors, we'll catch the error
364 // when we try to write a file to it below.
365 base::FilePath build_arg_file =
366 build_settings_.GetFullPath(GetBuildArgFile());
367 base::CreateDirectory(build_arg_file.DirName());
369 std::string contents = stream.str();
370 #if defined(OS_WIN)
371 // Use Windows lineendings for this file since it will often open in
372 // Notepad which can't handle Unix ones.
373 base::ReplaceSubstringsAfterOffset(&contents, 0, "\n", "\r\n");
374 #endif
375 if (base::WriteFile(build_arg_file, contents.c_str(),
376 static_cast<int>(contents.size())) == -1) {
377 Err(Location(), "Args file could not be written.",
378 "The file is \"" + FilePathToUTF8(build_arg_file) +
379 "\"").PrintToStdout();
380 return false;
383 // Add a dependency on the build arguments file. If this changes, we want
384 // to re-generate the build.
385 g_scheduler->AddGenDependency(build_arg_file);
387 return true;
390 bool Setup::FillSourceDir(const base::CommandLine& cmdline) {
391 // Find the .gn file.
392 base::FilePath root_path;
394 // Prefer the command line args to the config file.
395 base::FilePath relative_root_path =
396 cmdline.GetSwitchValuePath(switches::kRoot);
397 if (!relative_root_path.empty()) {
398 root_path = base::MakeAbsoluteFilePath(relative_root_path);
399 if (root_path.empty()) {
400 Err(Location(), "Root source path not found.",
401 "The path \"" + FilePathToUTF8(relative_root_path) +
402 "\" doesn't exist.").PrintToStdout();
403 return false;
406 // When --root is specified, an alternate --dotfile can also be set.
407 // --dotfile should be a real file path and not a "//foo" source-relative
408 // path.
409 base::FilePath dot_file_path =
410 cmdline.GetSwitchValuePath(switches::kDotfile);
411 if (dot_file_path.empty()) {
412 dotfile_name_ = root_path.Append(kGnFile);
413 } else {
414 dotfile_name_ = base::MakeAbsoluteFilePath(dot_file_path);
415 if (dotfile_name_.empty()) {
416 Err(Location(), "Could not load dotfile.",
417 "The file \"" + FilePathToUTF8(dot_file_path) +
418 "\" cound't be loaded.").PrintToStdout();
419 return false;
422 } else {
423 // In the default case, look for a dotfile and that also tells us where the
424 // source root is.
425 base::FilePath cur_dir;
426 base::GetCurrentDirectory(&cur_dir);
427 dotfile_name_ = FindDotFile(cur_dir);
428 if (dotfile_name_.empty()) {
429 Err(Location(), "Can't find source root.",
430 "I could not find a \".gn\" file in the current directory or any "
431 "parent,\nand the --root command-line argument was not specified.")
432 .PrintToStdout();
433 return false;
435 root_path = dotfile_name_.DirName();
438 if (scheduler_.verbose_logging())
439 scheduler_.Log("Using source root", FilePathToUTF8(root_path));
440 build_settings_.SetRootPath(root_path);
442 return true;
445 bool Setup::FillBuildDir(const std::string& build_dir, bool require_exists) {
446 Err err;
447 SourceDir resolved =
448 SourceDirForCurrentDirectory(build_settings_.root_path()).
449 ResolveRelativeDir(Value(nullptr, build_dir), &err,
450 build_settings_.root_path_utf8());
451 if (err.has_error()) {
452 err.PrintToStdout();
453 return false;
456 if (scheduler_.verbose_logging())
457 scheduler_.Log("Using build dir", resolved.value());
459 if (require_exists) {
460 base::FilePath build_dir_path = build_settings_.GetFullPath(resolved);
461 if (!base::PathExists(build_dir_path.Append(
462 FILE_PATH_LITERAL("build.ninja")))) {
463 Err(Location(), "Not a build directory.",
464 "This command requires an existing build directory. I interpreted "
465 "your input\n\"" + build_dir + "\" as:\n " +
466 FilePathToUTF8(build_dir_path) +
467 "\nwhich doesn't seem to contain a previously-generated build.")
468 .PrintToStdout();
469 return false;
473 build_settings_.SetBuildDir(resolved);
474 return true;
477 void Setup::FillPythonPath() {
478 // Trace this since it tends to be a bit slow on Windows.
479 ScopedTrace setup_trace(TraceItem::TRACE_SETUP, "Fill Python Path");
480 #if defined(OS_WIN)
481 // Find Python on the path so we can use the absolute path in the build.
482 const base::char16 kGetPython[] =
483 L"cmd.exe /c python -c \"import sys; print sys.executable\"";
484 std::string python_path;
485 if (base::GetAppOutput(kGetPython, &python_path)) {
486 base::TrimWhitespaceASCII(python_path, base::TRIM_ALL, &python_path);
487 if (scheduler_.verbose_logging())
488 scheduler_.Log("Found python", python_path);
489 } else {
490 scheduler_.Log("WARNING", "Could not find python on path, using "
491 "just \"python.exe\"");
492 python_path = "python.exe";
494 build_settings_.set_python_path(base::FilePath(base::UTF8ToUTF16(python_path))
495 .NormalizePathSeparatorsTo('/'));
496 #else
497 build_settings_.set_python_path(base::FilePath("python"));
498 #endif
501 bool Setup::RunConfigFile() {
502 if (scheduler_.verbose_logging())
503 scheduler_.Log("Got dotfile", FilePathToUTF8(dotfile_name_));
505 dotfile_input_file_.reset(new InputFile(SourceFile("//.gn")));
506 if (!dotfile_input_file_->Load(dotfile_name_)) {
507 Err(Location(), "Could not load dotfile.",
508 "The file \"" + FilePathToUTF8(dotfile_name_) + "\" cound't be loaded")
509 .PrintToStdout();
510 return false;
513 Err err;
514 dotfile_tokens_ = Tokenizer::Tokenize(dotfile_input_file_.get(), &err);
515 if (err.has_error()) {
516 err.PrintToStdout();
517 return false;
520 dotfile_root_ = Parser::Parse(dotfile_tokens_, &err);
521 if (err.has_error()) {
522 err.PrintToStdout();
523 return false;
526 dotfile_root_->Execute(&dotfile_scope_, &err);
527 if (err.has_error()) {
528 err.PrintToStdout();
529 return false;
532 return true;
535 bool Setup::FillOtherConfig(const base::CommandLine& cmdline) {
536 Err err;
537 SourceDir current_dir("//");
539 // Secondary source path, read from the config file if present.
540 // Read from the config file if present.
541 const Value* secondary_value =
542 dotfile_scope_.GetValue("secondary_source", true);
543 if (secondary_value) {
544 if (!secondary_value->VerifyTypeIs(Value::STRING, &err)) {
545 err.PrintToStdout();
546 return false;
548 build_settings_.SetSecondarySourcePath(
549 SourceDir(secondary_value->string_value()));
552 // Root build file.
553 const Value* root_value = dotfile_scope_.GetValue("root", true);
554 if (root_value) {
555 if (!root_value->VerifyTypeIs(Value::STRING, &err)) {
556 err.PrintToStdout();
557 return false;
560 Label root_target_label =
561 Label::Resolve(current_dir, Label(), *root_value, &err);
562 if (err.has_error()) {
563 err.PrintToStdout();
564 return false;
567 root_build_file_ = Loader::BuildFileForLabel(root_target_label);
570 // Build config file.
571 const Value* build_config_value =
572 dotfile_scope_.GetValue("buildconfig", true);
573 if (!build_config_value) {
574 Err(Location(), "No build config file.",
575 "Your .gn file (\"" + FilePathToUTF8(dotfile_name_) + "\")\n"
576 "didn't specify a \"buildconfig\" value.").PrintToStdout();
577 return false;
578 } else if (!build_config_value->VerifyTypeIs(Value::STRING, &err)) {
579 err.PrintToStdout();
580 return false;
582 build_settings_.set_build_config_file(
583 SourceFile(build_config_value->string_value()));
585 // Targets to check.
586 const Value* check_targets_value =
587 dotfile_scope_.GetValue("check_targets", true);
588 if (check_targets_value) {
589 // Fill the list of targets to check.
590 if (!check_targets_value->VerifyTypeIs(Value::LIST, &err)) {
591 err.PrintToStdout();
592 return false;
595 check_patterns_.reset(new std::vector<LabelPattern>);
596 for (const auto& item : check_targets_value->list_value()) {
597 check_patterns_->push_back(
598 LabelPattern::GetPattern(current_dir, item, &err));
599 if (err.has_error()) {
600 err.PrintToStdout();
601 return false;
606 // Fill exec_script_whitelist.
607 const Value* exec_script_whitelist_value =
608 dotfile_scope_.GetValue("exec_script_whitelist", true);
609 if (exec_script_whitelist_value) {
610 // Fill the list of targets to check.
611 if (!exec_script_whitelist_value->VerifyTypeIs(Value::LIST, &err)) {
612 err.PrintToStdout();
613 return false;
615 scoped_ptr<std::set<SourceFile>> whitelist(new std::set<SourceFile>);
616 for (const auto& item : exec_script_whitelist_value->list_value()) {
617 if (!item.VerifyTypeIs(Value::STRING, &err)) {
618 err.PrintToStdout();
619 return false;
621 whitelist->insert(current_dir.ResolveRelativeFile(item, &err));
622 if (err.has_error()) {
623 err.PrintToStdout();
624 return false;
627 build_settings_.set_exec_script_whitelist(whitelist.Pass());
630 return true;