tools/gn/setup.cc

   1 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "tools/gn/setup.h"
   6
   7 #include <stdlib.h>
   8
   9 #include <algorithm>
  10 #include <sstream>
  11
  12 #include "base/bind.h"
  13 #include "base/command_line.h"
  14 #include "base/files/file_path.h"
  15 #include "base/files/file_util.h"
  16 #include "base/process/launch.h"
  17 #include "base/strings/string_split.h"
  18 #include "base/strings/string_util.h"
  19 #include "base/strings/utf_string_conversions.h"
  20 #include "build/build_config.h"
  21 #include "tools/gn/commands.h"
  22 #include "tools/gn/filesystem_utils.h"
  23 #include "tools/gn/input_file.h"
  24 #include "tools/gn/parse_tree.h"
  25 #include "tools/gn/parser.h"
  26 #include "tools/gn/source_dir.h"
  27 #include "tools/gn/source_file.h"
  28 #include "tools/gn/standard_out.h"
  29 #include "tools/gn/tokenizer.h"
  30 #include "tools/gn/trace.h"
  31 #include "tools/gn/value.h"
  32
  33 #if defined(OS_WIN)
  34 #include <windows.h>
  35 #endif
  36
  37 extern const char kDotfile_Help[] =
  38     ".gn file\n"
  39     "\n"
  40     "  When gn starts, it will search the current directory and parent\n"
  41     "  directories for a file called \".gn\". This indicates the source root.\n"
  42     "  You can override this detection by using the --root command-line\n"
  43     "  argument\n"
  44     "\n"
  45     "  The .gn file in the source root will be executed. The syntax is the\n"
  46     "  same as a buildfile, but with very limited build setup-specific\n"
  47     "  meaning.\n"
  48     "\n"
  49     "  If you specify --root, by default GN will look for the file .gn in\n"
  50     "  that directory. If you want to specify a different file, you can\n"
  51     "  additionally pass --dotfile:\n"
  52     "\n"
  53     "    gn gen out/Debug --root=/home/build --dotfile=/home/my_gn_file.gn\n"
  54     "\n"
  55     "Variables\n"
  56     "\n"
  57     "  buildconfig [required]\n"
  58     "      Label of the build config file. This file will be used to set up\n"
  59     "      the build file execution environment for each toolchain.\n"
  60     "\n"
  61     "  root [optional]\n"
  62     "      Label of the root build target. The GN build will start by loading\n"
  63     "      the build file containing this target name. This defaults to\n"
  64     "      \"//:\" which will cause the file //BUILD.gn to be loaded.\n"
  65     "\n"
  66     "  secondary_source [optional]\n"
  67     "      Label of an alternate directory tree to find input files. When\n"
  68     "      searching for a BUILD.gn file (or the build config file discussed\n"
  69     "      above), the file will first be looked for in the source root.\n"
  70     "      If it's not found, the secondary source root will be checked\n"
  71     "      (which would contain a parallel directory hierarchy).\n"
  72     "\n"
  73     "      This behavior is intended to be used when BUILD.gn files can't be\n"
  74     "      checked in to certain source directories for whatever reason.\n"
  75     "\n"
  76     "      The secondary source root must be inside the main source tree.\n"
  77     "\n"
  78     "Example .gn file contents\n"
  79     "\n"
  80     "  buildconfig = \"//build/config/BUILDCONFIG.gn\"\n"
  81     "\n"
  82     "  root = \"//:root\"\n"
  83     "\n"
  84     "  secondary_source = \"//build/config/temporary_buildfiles/\"\n";
  85
  86 namespace {
  87
  88 // More logging.
  89 const char kSwitchVerbose[] = "v";
  90
  91 // Set build args.
  92 const char kSwitchArgs[] = "args";
  93
  94 // Set root dir.
  95 const char kSwitchRoot[] = "root";
  96
  97 // Set dotfile name.
  98 const char kSwitchDotfile[] = "dotfile";
  99
 100 // Enable timing.
 101 const char kTimeSwitch[] = "time";
 102
 103 const char kTracelogSwitch[] = "tracelog";
 104
 105 const base::FilePath::CharType kGnFile[] = FILE_PATH_LITERAL(".gn");
 106
 107 base::FilePath FindDotFile(const base::FilePath& current_dir) {
 108   base::FilePath try_this_file = current_dir.Append(kGnFile);
 109   if (base::PathExists(try_this_file))
 110     return try_this_file;
 111
 112   base::FilePath with_no_slash = current_dir.StripTrailingSeparators();
 113   base::FilePath up_one_dir = with_no_slash.DirName();
 114   if (up_one_dir == current_dir)
 115     return base::FilePath();  // Got to the top.
 116
 117   return FindDotFile(up_one_dir);
 118 }
 119
 120 // Called on any thread. Post the item to the builder on the main thread.
 121 void ItemDefinedCallback(base::MessageLoop* main_loop,
 122                          scoped_refptr<Builder> builder,
 123                          scoped_ptr<Item> item) {
 124   DCHECK(item);
 125   main_loop->PostTask(FROM_HERE, base::Bind(&Builder::ItemDefined, builder,
 126                                             base::Passed(&item)));
 127 }
 128
 129 void DecrementWorkCount() {
 130   g_scheduler->DecrementWorkCount();
 131 }
 132
 133 }  // namespace
 134
 135 // CommonSetup -----------------------------------------------------------------
 136
 137 const char CommonSetup::kBuildArgFileName[] = "args.gn";
 138
 139 CommonSetup::CommonSetup()
 140     : build_settings_(),
 141       loader_(new LoaderImpl(&build_settings_)),
 142       builder_(new Builder(loader_.get())),
 143       root_build_file_("//BUILD.gn"),
 144       check_for_bad_items_(true),
 145       check_for_unused_overrides_(true),
 146       check_public_headers_(false) {
 147   loader_->set_complete_callback(base::Bind(&DecrementWorkCount));
 148 }
 149
 150 CommonSetup::CommonSetup(const CommonSetup& other)
 151     : build_settings_(other.build_settings_),
 152       loader_(new LoaderImpl(&build_settings_)),
 153       builder_(new Builder(loader_.get())),
 154       root_build_file_(other.root_build_file_),
 155       check_for_bad_items_(other.check_for_bad_items_),
 156       check_for_unused_overrides_(other.check_for_unused_overrides_),
 157       check_public_headers_(other.check_public_headers_) {
 158   loader_->set_complete_callback(base::Bind(&DecrementWorkCount));
 159 }
 160
 161 CommonSetup::~CommonSetup() {
 162 }
 163
 164 void CommonSetup::RunPreMessageLoop() {
 165   // Load the root build file.
 166   loader_->Load(root_build_file_, LocationRange(), Label());
 167
 168   // Will be decremented with the loader is drained.
 169   g_scheduler->IncrementWorkCount();
 170 }
 171
 172 bool CommonSetup::RunPostMessageLoop() {
 173   Err err;
 174   if (check_for_bad_items_) {
 175     if (!builder_->CheckForBadItems(&err)) {
 176       err.PrintToStdout();
 177       return false;
 178     }
 179   }
 180
 181   if (check_for_unused_overrides_) {
 182     if (!build_settings_.build_args().VerifyAllOverridesUsed(&err)) {
 183       // TODO(brettw) implement a system of warnings. Until we have a better
 184       // system, print the error but don't return failure.
 185       err.PrintToStdout();
 186       return true;
 187     }
 188   }
 189
 190   if (check_public_headers_) {
 191     if (!commands::CheckPublicHeaders(&build_settings_,
 192                                       builder_->GetAllResolvedTargets(),
 193                                       std::vector<const Target*>(),
 194                                       false)) {
 195       return false;
 196     }
 197   }
 198
 199   // Write out tracing and timing if requested.
 200   const CommandLine* cmdline = CommandLine::ForCurrentProcess();
 201   if (cmdline->HasSwitch(kTimeSwitch))
 202     PrintLongHelp(SummarizeTraces());
 203   if (cmdline->HasSwitch(kTracelogSwitch))
 204     SaveTraces(cmdline->GetSwitchValuePath(kTracelogSwitch));
 205
 206   return true;
 207 }
 208
 209 // Setup -----------------------------------------------------------------------
 210
 211 Setup::Setup()
 212     : CommonSetup(),
 213       empty_settings_(&empty_build_settings_, std::string()),
 214       dotfile_scope_(&empty_settings_),
 215       fill_arguments_(true) {
 216   empty_settings_.set_toolchain_label(Label());
 217   build_settings_.set_item_defined_callback(
 218       base::Bind(&ItemDefinedCallback, scheduler_.main_loop(), builder_));
 219
 220   // The scheduler's main loop wasn't created when the Loader was created, so
 221   // we need to set it now.
 222   loader_->set_main_loop(scheduler_.main_loop());
 223 }
 224
 225 Setup::~Setup() {
 226 }
 227
 228 bool Setup::DoSetup(const std::string& build_dir, bool force_create) {
 229   CommandLine* cmdline = CommandLine::ForCurrentProcess();
 230
 231   scheduler_.set_verbose_logging(cmdline->HasSwitch(kSwitchVerbose));
 232   if (cmdline->HasSwitch(kTimeSwitch) ||
 233       cmdline->HasSwitch(kTracelogSwitch))
 234     EnableTracing();
 235
 236   ScopedTrace setup_trace(TraceItem::TRACE_SETUP, "DoSetup");
 237
 238   if (!FillSourceDir(*cmdline))
 239     return false;
 240   if (!RunConfigFile())
 241     return false;
 242   if (!FillOtherConfig(*cmdline))
 243     return false;
 244
 245   // Must be after FillSourceDir to resolve.
 246   if (!FillBuildDir(build_dir, !force_create))
 247     return false;
 248
 249   if (fill_arguments_) {
 250     if (!FillArguments(*cmdline))
 251       return false;
 252   }
 253   FillPythonPath();
 254
 255   return true;
 256 }
 257
 258 bool Setup::Run() {
 259   RunPreMessageLoop();
 260   if (!scheduler_.Run())
 261     return false;
 262   return RunPostMessageLoop();
 263 }
 264
 265 Scheduler* Setup::GetScheduler() {
 266   return &scheduler_;
 267 }
 268
 269 SourceFile Setup::GetBuildArgFile() const {
 270   return SourceFile(build_settings_.build_dir().value() + kBuildArgFileName);
 271 }
 272
 273 bool Setup::FillArguments(const CommandLine& cmdline) {
 274   // Use the args on the command line if specified, and save them. Do this even
 275   // if the list is empty (this means clear any defaults).
 276   if (cmdline.HasSwitch(kSwitchArgs)) {
 277     if (!FillArgsFromCommandLine(cmdline.GetSwitchValueASCII(kSwitchArgs)))
 278       return false;
 279     SaveArgsToFile();
 280     return true;
 281   }
 282
 283   // No command line args given, use the arguments from the build dir (if any).
 284   return FillArgsFromFile();
 285 }
 286
 287 bool Setup::FillArgsFromCommandLine(const std::string& args) {
 288   args_input_file_.reset(new InputFile(SourceFile()));
 289   args_input_file_->SetContents(args);
 290   args_input_file_->set_friendly_name("the command-line \"--args\"");
 291   return FillArgsFromArgsInputFile();
 292 }
 293
 294 bool Setup::FillArgsFromFile() {
 295   ScopedTrace setup_trace(TraceItem::TRACE_SETUP, "Load args file");
 296
 297   SourceFile build_arg_source_file = GetBuildArgFile();
 298   base::FilePath build_arg_file =
 299       build_settings_.GetFullPath(build_arg_source_file);
 300
 301   std::string contents;
 302   if (!base::ReadFileToString(build_arg_file, &contents))
 303     return true;  // File doesn't exist, continue with default args.
 304
 305   // Add a dependency on the build arguments file. If this changes, we want
 306   // to re-generate the build.
 307   g_scheduler->AddGenDependency(build_arg_file);
 308
 309   if (contents.empty())
 310     return true;  // Empty file, do nothing.
 311
 312   args_input_file_.reset(new InputFile(build_arg_source_file));
 313   args_input_file_->SetContents(contents);
 314   args_input_file_->set_friendly_name(
 315       "build arg file (use \"gn args <out_dir>\" to edit)");
 316
 317   setup_trace.Done();  // Only want to count the load as part of the trace.
 318   return FillArgsFromArgsInputFile();
 319 }
 320
 321 bool Setup::FillArgsFromArgsInputFile() {
 322   ScopedTrace setup_trace(TraceItem::TRACE_SETUP, "Parse args");
 323
 324   Err err;
 325   args_tokens_ = Tokenizer::Tokenize(args_input_file_.get(), &err);
 326   if (err.has_error()) {
 327     err.PrintToStdout();
 328     return false;
 329   }
 330
 331   args_root_ = Parser::Parse(args_tokens_, &err);
 332   if (err.has_error()) {
 333     err.PrintToStdout();
 334     return false;
 335   }
 336
 337   Scope arg_scope(&empty_settings_);
 338   args_root_->AsBlock()->ExecuteBlockInScope(&arg_scope, &err);
 339   if (err.has_error()) {
 340     err.PrintToStdout();
 341     return false;
 342   }
 343
 344   // Save the result of the command args.
 345   Scope::KeyValueMap overrides;
 346   arg_scope.GetCurrentScopeValues(&overrides);
 347   build_settings_.build_args().AddArgOverrides(overrides);
 348   return true;
 349 }
 350
 351 bool Setup::SaveArgsToFile() {
 352   ScopedTrace setup_trace(TraceItem::TRACE_SETUP, "Save args file");
 353
 354   Scope::KeyValueMap args = build_settings_.build_args().GetAllOverrides();
 355
 356   std::ostringstream stream;
 357   for (Scope::KeyValueMap::const_iterator i = args.begin();
 358        i != args.end(); ++i) {
 359     stream << i->first.as_string() << " = " << i->second.ToString(true);
 360     stream << std::endl;
 361   }
 362
 363   // For the first run, the build output dir might not be created yet, so do
 364   // that so we can write a file into it. Ignore errors, we'll catch the error
 365   // when we try to write a file to it below.
 366   base::FilePath build_arg_file =
 367       build_settings_.GetFullPath(GetBuildArgFile());
 368   base::CreateDirectory(build_arg_file.DirName());
 369
 370   std::string contents = stream.str();
 371 #if defined(OS_WIN)
 372   // Use Windows lineendings for this file since it will often open in
 373   // Notepad which can't handle Unix ones.
 374   ReplaceSubstringsAfterOffset(&contents, 0, "\n", "\r\n");
 375 #endif
 376   if (base::WriteFile(build_arg_file, contents.c_str(),
 377       static_cast<int>(contents.size())) == -1) {
 378     Err(Location(), "Args file could not be written.",
 379       "The file is \"" + FilePathToUTF8(build_arg_file) +
 380         "\"").PrintToStdout();
 381     return false;
 382   }
 383
 384   // Add a dependency on the build arguments file. If this changes, we want
 385   // to re-generate the build.
 386   g_scheduler->AddGenDependency(build_arg_file);
 387
 388   return true;
 389 }
 390
 391 bool Setup::FillSourceDir(const CommandLine& cmdline) {
 392   // Find the .gn file.
 393   base::FilePath root_path;
 394
 395   // Prefer the command line args to the config file.
 396   base::FilePath relative_root_path = cmdline.GetSwitchValuePath(kSwitchRoot);
 397   if (!relative_root_path.empty()) {
 398     root_path = base::MakeAbsoluteFilePath(relative_root_path);
 399     if (root_path.empty()) {
 400       Err(Location(), "Root source path not found.",
 401           "The path \"" + FilePathToUTF8(relative_root_path) +
 402           "\" doesn't exist.").PrintToStdout();
 403       return false;
 404     }
 405
 406     // When --root is specified, an alternate --dotfile can also be set.
 407     // --dotfile should be a real file path and not a "//foo" source-relative
 408     // path.
 409     base::FilePath dot_file_path = cmdline.GetSwitchValuePath(kSwitchDotfile);
 410     if (dot_file_path.empty()) {
 411       dotfile_name_ = root_path.Append(kGnFile);
 412     } else {
 413       dotfile_name_ = base::MakeAbsoluteFilePath(dot_file_path);
 414       if (dotfile_name_.empty()) {
 415         Err(Location(), "Could not load dotfile.",
 416             "The file \"" + FilePathToUTF8(dot_file_path) +
 417             "\" cound't be loaded.").PrintToStdout();
 418         return false;
 419       }
 420     }
 421   } else {
 422     // In the default case, look for a dotfile and that also tells us where the
 423     // source root is.
 424     base::FilePath cur_dir;
 425     base::GetCurrentDirectory(&cur_dir);
 426     dotfile_name_ = FindDotFile(cur_dir);
 427     if (dotfile_name_.empty()) {
 428       Err(Location(), "Can't find source root.",
 429           "I could not find a \".gn\" file in the current directory or any "
 430           "parent,\nand the --root command-line argument was not specified.")
 431           .PrintToStdout();
 432       return false;
 433     }
 434     root_path = dotfile_name_.DirName();
 435   }
 436
 437   if (scheduler_.verbose_logging())
 438     scheduler_.Log("Using source root", FilePathToUTF8(root_path));
 439   build_settings_.SetRootPath(root_path);
 440
 441   return true;
 442 }
 443
 444 bool Setup::FillBuildDir(const std::string& build_dir, bool require_exists) {
 445   SourceDir resolved =
 446       SourceDirForCurrentDirectory(build_settings_.root_path()).
 447           ResolveRelativeDir(build_dir);
 448   if (resolved.is_null()) {
 449     Err(Location(), "Couldn't resolve build directory.",
 450         "The build directory supplied (\"" + build_dir + "\") was not valid.").
 451         PrintToStdout();
 452     return false;
 453   }
 454
 455   if (scheduler_.verbose_logging())
 456     scheduler_.Log("Using build dir", resolved.value());
 457
 458   if (require_exists) {
 459     base::FilePath build_dir_path = build_settings_.GetFullPath(resolved);
 460     if (!base::PathExists(build_dir_path.Append(
 461             FILE_PATH_LITERAL("build.ninja")))) {
 462       Err(Location(), "Not a build directory.",
 463           "This command requires an existing build directory. I interpreted "
 464           "your input\n\"" + build_dir + "\" as:\n  " +
 465           FilePathToUTF8(build_dir_path) +
 466           "\nwhich doesn't seem to contain a previously-generated build.")
 467           .PrintToStdout();
 468       return false;
 469     }
 470   }
 471
 472   build_settings_.SetBuildDir(resolved);
 473   return true;
 474 }
 475
 476 void Setup::FillPythonPath() {
 477   // Trace this since it tends to be a bit slow on Windows.
 478   ScopedTrace setup_trace(TraceItem::TRACE_SETUP, "Fill Python Path");
 479 #if defined(OS_WIN)
 480   // Find Python on the path so we can use the absolute path in the build.
 481   const base::char16 kGetPython[] =
 482       L"cmd.exe /c python -c \"import sys; print sys.executable\"";
 483   std::string python_path;
 484   if (base::GetAppOutput(kGetPython, &python_path)) {
 485     base::TrimWhitespaceASCII(python_path, base::TRIM_ALL, &python_path);
 486     if (scheduler_.verbose_logging())
 487       scheduler_.Log("Found python", python_path);
 488   } else {
 489     scheduler_.Log("WARNING", "Could not find python on path, using "
 490         "just \"python.exe\"");
 491     python_path = "python.exe";
 492   }
 493   build_settings_.set_python_path(base::FilePath(base::UTF8ToUTF16(python_path))
 494                                       .NormalizePathSeparatorsTo('/'));
 495 #else
 496   build_settings_.set_python_path(base::FilePath("python"));
 497 #endif
 498 }
 499
 500 bool Setup::RunConfigFile() {
 501   if (scheduler_.verbose_logging())
 502     scheduler_.Log("Got dotfile", FilePathToUTF8(dotfile_name_));
 503
 504   dotfile_input_file_.reset(new InputFile(SourceFile("//.gn")));
 505   if (!dotfile_input_file_->Load(dotfile_name_)) {
 506     Err(Location(), "Could not load dotfile.",
 507         "The file \"" + FilePathToUTF8(dotfile_name_) + "\" cound't be loaded")
 508         .PrintToStdout();
 509     return false;
 510   }
 511
 512   Err err;
 513   dotfile_tokens_ = Tokenizer::Tokenize(dotfile_input_file_.get(), &err);
 514   if (err.has_error()) {
 515     err.PrintToStdout();
 516     return false;
 517   }
 518
 519   dotfile_root_ = Parser::Parse(dotfile_tokens_, &err);
 520   if (err.has_error()) {
 521     err.PrintToStdout();
 522     return false;
 523   }
 524
 525   dotfile_root_->AsBlock()->ExecuteBlockInScope(&dotfile_scope_, &err);
 526   if (err.has_error()) {
 527     err.PrintToStdout();
 528     return false;
 529   }
 530
 531   return true;
 532 }
 533
 534 bool Setup::FillOtherConfig(const CommandLine& cmdline) {
 535   Err err;
 536
 537   // Secondary source path, read from the config file if present.
 538   // Read from the config file if present.
 539   const Value* secondary_value =
 540       dotfile_scope_.GetValue("secondary_source", true);
 541   if (secondary_value) {
 542     if (!secondary_value->VerifyTypeIs(Value::STRING, &err)) {
 543       err.PrintToStdout();
 544       return false;
 545     }
 546     build_settings_.SetSecondarySourcePath(
 547         SourceDir(secondary_value->string_value()));
 548   }
 549
 550   // Root build file.
 551   const Value* root_value = dotfile_scope_.GetValue("root", true);
 552   if (root_value) {
 553     if (!root_value->VerifyTypeIs(Value::STRING, &err)) {
 554       err.PrintToStdout();
 555       return false;
 556     }
 557
 558     Label root_target_label =
 559         Label::Resolve(SourceDir("//"), Label(), *root_value, &err);
 560     if (err.has_error()) {
 561       err.PrintToStdout();
 562       return false;
 563     }
 564
 565     root_build_file_ = Loader::BuildFileForLabel(root_target_label);
 566   }
 567
 568   // Build config file.
 569   const Value* build_config_value =
 570       dotfile_scope_.GetValue("buildconfig", true);
 571   if (!build_config_value) {
 572     Err(Location(), "No build config file.",
 573         "Your .gn file (\"" + FilePathToUTF8(dotfile_name_) + "\")\n"
 574         "didn't specify a \"buildconfig\" value.").PrintToStdout();
 575     return false;
 576   } else if (!build_config_value->VerifyTypeIs(Value::STRING, &err)) {
 577     err.PrintToStdout();
 578     return false;
 579   }
 580   build_settings_.set_build_config_file(
 581       SourceFile(build_config_value->string_value()));
 582
 583   return true;
 584 }
 585
 586 // DependentSetup --------------------------------------------------------------
 587
 588 DependentSetup::DependentSetup(Setup* derive_from)
 589     : CommonSetup(*derive_from),
 590       scheduler_(derive_from->GetScheduler()) {
 591   build_settings_.set_item_defined_callback(
 592       base::Bind(&ItemDefinedCallback, scheduler_->main_loop(), builder_));
 593 }
 594
 595 DependentSetup::DependentSetup(DependentSetup* derive_from)
 596     : CommonSetup(*derive_from),
 597       scheduler_(derive_from->GetScheduler()) {
 598   build_settings_.set_item_defined_callback(
 599       base::Bind(&ItemDefinedCallback, scheduler_->main_loop(), builder_));
 600 }
 601
 602 DependentSetup::~DependentSetup() {
 603 }
 604
 605 Scheduler* DependentSetup::GetScheduler() {
 606   return scheduler_;
 607 }
 608
 609 void DependentSetup::RunPreMessageLoop() {
 610   CommonSetup::RunPreMessageLoop();
 611 }
 612
 613 bool DependentSetup::RunPostMessageLoop() {
 614   return CommonSetup::RunPostMessageLoop();
 615 }
 616