content/public/test/test_launcher.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/public/test/test_launcher.h"
   6
   7 #include <map>
   8 #include <string>
   9 #include <vector>
  10
  11 #include "base/command_line.h"
  12 #include "base/containers/hash_tables.h"
  13 #include "base/environment.h"
  14 #include "base/file_util.h"
  15 #include "base/files/scoped_temp_dir.h"
  16 #include "base/logging.h"
  17 #include "base/memory/linked_ptr.h"
  18 #include "base/memory/scoped_ptr.h"
  19 #include "base/message_loop/message_loop.h"
  20 #include "base/stl_util.h"
  21 #include "base/strings/string_number_conversions.h"
  22 #include "base/strings/string_util.h"
  23 #include "base/strings/utf_string_conversions.h"
  24 #include "base/test/launcher/test_launcher.h"
  25 #include "base/test/test_suite.h"
  26 #include "base/test/test_switches.h"
  27 #include "base/test/test_timeouts.h"
  28 #include "base/time/time.h"
  29 #include "content/public/app/content_main.h"
  30 #include "content/public/app/content_main_delegate.h"
  31 #include "content/public/app/startup_helper_win.h"
  32 #include "content/public/common/content_switches.h"
  33 #include "content/public/common/sandbox_init.h"
  34 #include "content/public/test/browser_test.h"
  35 #include "net/base/escape.h"
  36 #include "testing/gtest/include/gtest/gtest.h"
  37
  38 #if defined(OS_WIN)
  39 #include "base/base_switches.h"
  40 #include "content/common/sandbox_win.h"
  41 #include "sandbox/win/src/sandbox_factory.h"
  42 #include "sandbox/win/src/sandbox_types.h"
  43 #elif defined(OS_MACOSX)
  44 #include "base/mac/scoped_nsautorelease_pool.h"
  45 #endif
  46
  47 namespace content {
  48
  49 namespace {
  50
  51 // Tests with this prefix run before the same test without it, and use the same
  52 // profile. i.e. Foo.PRE_Test runs and then Foo.Test. This allows writing tests
  53 // that span browser restarts.
  54 const char kPreTestPrefix[] = "PRE_";
  55
  56 // Manual tests only run when --run-manual is specified. This allows writing
  57 // tests that don't run automatically but are still in the same test binary.
  58 // This is useful so that a team that wants to run a few tests doesn't have to
  59 // add a new binary that must be compiled on all builds.
  60 const char kManualTestPrefix[] = "MANUAL_";
  61
  62 TestLauncherDelegate* g_launcher_delegate;
  63
  64 std::string RemoveAnyPrePrefixes(const std::string& test_name) {
  65   std::string result(test_name);
  66   ReplaceSubstringsAfterOffset(&result, 0, kPreTestPrefix, std::string());
  67   return result;
  68 }
  69
  70 void PrintUsage() {
  71   fprintf(stdout,
  72           "Runs tests using the gtest framework, each batch of tests being\n"
  73           "run in their own process. Supported command-line flags:\n"
  74           "\n"
  75           " Common flags:\n"
  76           "  --gtest_filter=...\n"
  77           "    Runs a subset of tests (see --gtest_help for more info).\n"
  78           "\n"
  79           "  --help\n"
  80           "    Shows this message.\n"
  81           "\n"
  82           "  --gtest_help\n"
  83           "    Shows the gtest help message.\n"
  84           "\n"
  85           "  --test-launcher-jobs=N\n"
  86           "    Sets the number of parallel test jobs to N.\n"
  87           "\n"
  88           "  --single_process\n"
  89           "    Runs the tests and the launcher in the same process. Useful\n"
  90           "    for debugging a specific test in a debugger.\n"
  91           "\n"
  92           " Other flags:\n"
  93           "  --test-launcher-retry-limit=N\n"
  94           "    Sets the limit of test retries on failures to N.\n"
  95           "\n"
  96           "  --test-launcher-summary-output=PATH\n"
  97           "    Saves a JSON machine-readable summary of the run.\n"
  98           "\n"
  99           "  --test-launcher-print-test-stdio=auto|always|never\n"
 100           "    Controls when full test output is printed.\n"
 101           "    auto means to print it when the test failed.\n"
 102           "\n"
 103           "  --test-launcher-total-shards=N\n"
 104           "    Sets the total number of shards to N.\n"
 105           "\n"
 106           "  --test-launcher-shard-index=N\n"
 107           "    Sets the shard index to run to N (from 0 to TOTAL - 1).\n");
 108 }
 109
 110 // Implementation of base::TestLauncherDelegate. This is also a test launcher,
 111 // wrapping a lower-level test launcher with content-specific code.
 112 class WrapperTestLauncherDelegate : public base::TestLauncherDelegate {
 113  public:
 114   explicit WrapperTestLauncherDelegate(
 115       content::TestLauncherDelegate* launcher_delegate)
 116       : launcher_delegate_(launcher_delegate) {
 117     CHECK(temp_dir_.CreateUniqueTempDir());
 118   }
 119
 120   // base::TestLauncherDelegate:
 121   virtual void OnTestIterationStarting() OVERRIDE;
 122   virtual std::string GetTestNameForFiltering(
 123       const testing::TestCase* test_case,
 124       const testing::TestInfo* test_info) OVERRIDE;
 125   virtual bool ShouldRunTest(const testing::TestCase* test_case,
 126                              const testing::TestInfo* test_info) OVERRIDE;
 127   virtual size_t RunTests(base::TestLauncher* test_launcher,
 128                           const std::vector<std::string>& test_names) OVERRIDE;
 129   virtual size_t RetryTests(
 130       base::TestLauncher* test_launcher,
 131       const std::vector<std::string>& test_names) OVERRIDE;
 132
 133  private:
 134   void DoRunTest(base::TestLauncher* test_launcher,
 135                  const std::string& test_name);
 136
 137   // Launches test named |test_name| using parallel launcher,
 138   // given result of PRE_ test |pre_test_result|.
 139   void RunDependentTest(base::TestLauncher* test_launcher,
 140                         const std::string test_name,
 141                         const base::TestResult& pre_test_result);
 142
 143   // Callback to receive result of a test.
 144   void GTestCallback(
 145       base::TestLauncher* test_launcher,
 146       const std::string& test_name,
 147       int exit_code,
 148       const base::TimeDelta& elapsed_time,
 149       bool was_timeout,
 150       const std::string& output);
 151
 152   content::TestLauncherDelegate* launcher_delegate_;
 153
 154   // Store dependent test name (map is indexed by full test name).
 155   typedef std::map<std::string, std::string> DependentTestMap;
 156   DependentTestMap dependent_test_map_;
 157   DependentTestMap reverse_dependent_test_map_;
 158
 159   // Store unique data directory prefix for test names (without PRE_ prefixes).
 160   // PRE_ tests and tests that depend on them must share the same
 161   // data directory. Using test name as directory name leads to too long
 162   // names (exceeding UNIX_PATH_MAX, which creates a problem with
 163   // process_singleton_linux). Create a randomly-named temporary directory
 164   // and keep track of the names so that PRE_ tests can still re-use them.
 165   typedef std::map<std::string, base::FilePath> UserDataDirMap;
 166   UserDataDirMap user_data_dir_map_;
 167
 168   // Store names of all seen tests to properly handle PRE_ tests.
 169   std::set<std::string> all_test_names_;
 170
 171   // Temporary directory for user data directories.
 172   base::ScopedTempDir temp_dir_;
 173
 174   DISALLOW_COPY_AND_ASSIGN(WrapperTestLauncherDelegate);
 175 };
 176
 177 void WrapperTestLauncherDelegate::OnTestIterationStarting() {
 178   dependent_test_map_.clear();
 179   user_data_dir_map_.clear();
 180 }
 181
 182 std::string WrapperTestLauncherDelegate::GetTestNameForFiltering(
 183     const testing::TestCase* test_case,
 184     const testing::TestInfo* test_info) {
 185   return RemoveAnyPrePrefixes(
 186       std::string(test_case->name()) + "." + test_info->name());
 187 }
 188
 189 bool WrapperTestLauncherDelegate::ShouldRunTest(
 190     const testing::TestCase* test_case,
 191     const testing::TestInfo* test_info) {
 192   all_test_names_.insert(
 193       std::string(test_case->name()) + "." + test_info->name());
 194
 195   if (StartsWithASCII(test_info->name(), kManualTestPrefix, true) &&
 196       !CommandLine::ForCurrentProcess()->HasSwitch(kRunManualTestsFlag)) {
 197     return false;
 198   }
 199
 200   if (StartsWithASCII(test_info->name(), kPreTestPrefix, true)) {
 201     // We will actually run PRE_ tests, but to ensure they run on the same shard
 202     // as dependent tests, handle all these details internally.
 203     return false;
 204   }
 205
 206   return true;
 207 }
 208
 209 std::string GetPreTestName(const std::string& full_name) {
 210   size_t dot_pos = full_name.find('.');
 211   CHECK_NE(dot_pos, std::string::npos);
 212   std::string test_case_name = full_name.substr(0, dot_pos);
 213   std::string test_name = full_name.substr(dot_pos + 1);
 214   return test_case_name + "." + kPreTestPrefix + test_name;
 215 }
 216
 217 size_t WrapperTestLauncherDelegate::RunTests(
 218     base::TestLauncher* test_launcher,
 219     const std::vector<std::string>& test_names) {
 220   // Number of additional tests to run because of dependencies.
 221   size_t additional_tests_to_run_count = 0;
 222
 223   // Compute dependencies of tests to be run.
 224   for (size_t i = 0; i < test_names.size(); i++) {
 225     std::string full_name(test_names[i]);
 226     std::string pre_test_name(GetPreTestName(full_name));
 227
 228     while (ContainsKey(all_test_names_, pre_test_name)) {
 229       additional_tests_to_run_count++;
 230
 231       DCHECK(!ContainsKey(dependent_test_map_, pre_test_name));
 232       dependent_test_map_[pre_test_name] = full_name;
 233
 234       DCHECK(!ContainsKey(reverse_dependent_test_map_, full_name));
 235       reverse_dependent_test_map_[full_name] = pre_test_name;
 236
 237       full_name = pre_test_name;
 238       pre_test_name = GetPreTestName(pre_test_name);
 239     }
 240   }
 241
 242   for (size_t i = 0; i < test_names.size(); i++) {
 243     std::string full_name(test_names[i]);
 244
 245     // Make sure no PRE_ tests were requested explicitly.
 246     DCHECK_EQ(full_name, RemoveAnyPrePrefixes(full_name));
 247
 248     if (!ContainsKey(user_data_dir_map_, full_name)) {
 249       base::FilePath temp_dir;
 250       CHECK(base::CreateTemporaryDirInDir(temp_dir_.path(),
 251                                           FILE_PATH_LITERAL("d"), &temp_dir));
 252       user_data_dir_map_[full_name] = temp_dir;
 253     }
 254
 255     // If the test has any dependencies, get to the root and start with that.
 256     while (ContainsKey(reverse_dependent_test_map_, full_name))
 257       full_name = GetPreTestName(full_name);
 258
 259     DoRunTest(test_launcher, full_name);
 260   }
 261
 262   return test_names.size() + additional_tests_to_run_count;
 263 }
 264
 265 size_t WrapperTestLauncherDelegate::RetryTests(
 266     base::TestLauncher* test_launcher,
 267     const std::vector<std::string>& test_names) {
 268   // List of tests we can kick off right now, depending on no other tests.
 269   std::vector<std::string> tests_to_run_now;
 270
 271   // We retry at least the tests requested to retry.
 272   std::set<std::string> test_names_set(test_names.begin(), test_names.end());
 273
 274   // In the face of PRE_ tests, we need to retry the entire chain of tests,
 275   // from the very first one.
 276   for (size_t i = 0; i < test_names.size(); i++) {
 277     std::string test_name(test_names[i]);
 278     while (ContainsKey(reverse_dependent_test_map_, test_name)) {
 279       test_name = reverse_dependent_test_map_[test_name];
 280       test_names_set.insert(test_name);
 281     }
 282   }
 283
 284   // Discard user data directories from any previous runs. Start with
 285   // fresh state.
 286   for (UserDataDirMap::const_iterator i = user_data_dir_map_.begin();
 287        i != user_data_dir_map_.end();
 288        ++i) {
 289     // Delete temporary directories now to avoid using too much space in /tmp.
 290     if (!base::DeleteFile(i->second, true)) {
 291       LOG(WARNING) << "Failed to delete " << i->second.value();
 292     }
 293   }
 294   user_data_dir_map_.clear();
 295
 296   for (std::set<std::string>::const_iterator i = test_names_set.begin();
 297        i != test_names_set.end();
 298        ++i) {
 299     std::string full_name(*i);
 300
 301     // Make sure PRE_ tests and tests that depend on them share the same
 302     // data directory - based it on the test name without prefixes.
 303     std::string test_name_no_pre(RemoveAnyPrePrefixes(full_name));
 304     if (!ContainsKey(user_data_dir_map_, test_name_no_pre)) {
 305       base::FilePath temp_dir;
 306       CHECK(base::CreateTemporaryDirInDir(temp_dir_.path(),
 307                                           FILE_PATH_LITERAL("d"), &temp_dir));
 308       user_data_dir_map_[test_name_no_pre] = temp_dir;
 309     }
 310
 311     size_t dot_pos = full_name.find('.');
 312     CHECK_NE(dot_pos, std::string::npos);
 313     std::string test_case_name = full_name.substr(0, dot_pos);
 314     std::string test_name = full_name.substr(dot_pos + 1);
 315     std::string pre_test_name(
 316         test_case_name + "." + kPreTestPrefix + test_name);
 317     if (!ContainsKey(test_names_set, pre_test_name))
 318       tests_to_run_now.push_back(full_name);
 319   }
 320
 321   for (size_t i = 0; i < tests_to_run_now.size(); i++)
 322     DoRunTest(test_launcher, tests_to_run_now[i]);
 323
 324   return test_names_set.size();
 325 }
 326
 327 void WrapperTestLauncherDelegate::DoRunTest(base::TestLauncher* test_launcher,
 328                                             const std::string& test_name) {
 329   std::string test_name_no_pre(RemoveAnyPrePrefixes(test_name));
 330
 331   CommandLine cmd_line(*CommandLine::ForCurrentProcess());
 332   CHECK(launcher_delegate_->AdjustChildProcessCommandLine(
 333             &cmd_line, user_data_dir_map_[test_name_no_pre]));
 334
 335   CommandLine new_cmd_line(cmd_line.GetProgram());
 336   CommandLine::SwitchMap switches = cmd_line.GetSwitches();
 337
 338   // Strip out gtest_output flag because otherwise we would overwrite results
 339   // of the other tests.
 340   switches.erase(base::kGTestOutputFlag);
 341
 342   for (CommandLine::SwitchMap::const_iterator iter = switches.begin();
 343        iter != switches.end(); ++iter) {
 344     new_cmd_line.AppendSwitchNative(iter->first, iter->second);
 345   }
 346
 347   // Always enable disabled tests.  This method is not called with disabled
 348   // tests unless this flag was specified to the browser test executable.
 349   new_cmd_line.AppendSwitch("gtest_also_run_disabled_tests");
 350   new_cmd_line.AppendSwitchASCII("gtest_filter", test_name);
 351   new_cmd_line.AppendSwitch(kSingleProcessTestsFlag);
 352
 353   char* browser_wrapper = getenv("BROWSER_WRAPPER");
 354
 355   test_launcher->LaunchChildGTestProcess(
 356       new_cmd_line,
 357       browser_wrapper ? browser_wrapper : std::string(),
 358       TestTimeouts::action_max_timeout(),
 359       base::Bind(&WrapperTestLauncherDelegate::GTestCallback,
 360                  base::Unretained(this),
 361                  test_launcher,
 362                  test_name));
 363 }
 364
 365 void WrapperTestLauncherDelegate::RunDependentTest(
 366     base::TestLauncher* test_launcher,
 367     const std::string test_name,
 368     const base::TestResult& pre_test_result) {
 369   if (pre_test_result.status == base::TestResult::TEST_SUCCESS) {
 370     // Only run the dependent test if PRE_ test succeeded.
 371     DoRunTest(test_launcher, test_name);
 372   } else {
 373     // Otherwise skip the test.
 374     base::TestResult test_result;
 375     test_result.full_name = test_name;
 376     test_result.status = base::TestResult::TEST_SKIPPED;
 377     test_launcher->OnTestFinished(test_result);
 378
 379     if (ContainsKey(dependent_test_map_, test_name)) {
 380       RunDependentTest(test_launcher,
 381                        dependent_test_map_[test_name],
 382                        test_result);
 383     }
 384   }
 385 }
 386
 387 void WrapperTestLauncherDelegate::GTestCallback(
 388     base::TestLauncher* test_launcher,
 389     const std::string& test_name,
 390     int exit_code,
 391     const base::TimeDelta& elapsed_time,
 392     bool was_timeout,
 393     const std::string& output) {
 394   base::TestResult result;
 395   result.full_name = test_name;
 396
 397   // TODO(phajdan.jr): Recognize crashes.
 398   if (exit_code == 0)
 399     result.status = base::TestResult::TEST_SUCCESS;
 400   else if (was_timeout)
 401     result.status = base::TestResult::TEST_TIMEOUT;
 402   else
 403     result.status = base::TestResult::TEST_FAILURE;
 404
 405   result.elapsed_time = elapsed_time;
 406
 407   result.output_snippet = GetTestOutputSnippet(result, output);
 408
 409   if (ContainsKey(dependent_test_map_, test_name)) {
 410     RunDependentTest(test_launcher, dependent_test_map_[test_name], result);
 411   } else {
 412     // No other tests depend on this, we can delete the temporary directory now.
 413     // Do so to avoid too many temporary files using lots of disk space.
 414     std::string test_name_no_pre(RemoveAnyPrePrefixes(test_name));
 415     if (ContainsKey(user_data_dir_map_, test_name_no_pre)) {
 416       if (!base::DeleteFile(user_data_dir_map_[test_name_no_pre], true)) {
 417         LOG(WARNING) << "Failed to delete "
 418                      << user_data_dir_map_[test_name_no_pre].value();
 419       }
 420       user_data_dir_map_.erase(test_name_no_pre);
 421     }
 422   }
 423
 424   test_launcher->OnTestFinished(result);
 425 }
 426
 427 }  // namespace
 428
 429 const char kHelpFlag[]   = "help";
 430
 431 const char kLaunchAsBrowser[] = "as-browser";
 432
 433 // See kManualTestPrefix above.
 434 const char kRunManualTestsFlag[] = "run-manual";
 435
 436 const char kSingleProcessTestsFlag[]   = "single_process";
 437
 438
 439 TestLauncherDelegate::~TestLauncherDelegate() {
 440 }
 441
 442 bool ShouldRunContentMain() {
 443 #if defined(OS_WIN) || defined(OS_LINUX)
 444   CommandLine* command_line = CommandLine::ForCurrentProcess();
 445   return command_line->HasSwitch(switches::kProcessType) ||
 446          command_line->HasSwitch(kLaunchAsBrowser);
 447 #else
 448   return false;
 449 #endif  // defined(OS_WIN) || defined(OS_LINUX)
 450 }
 451
 452 int RunContentMain(int argc, char** argv,
 453                    TestLauncherDelegate* launcher_delegate) {
 454 #if defined(OS_WIN)
 455   sandbox::SandboxInterfaceInfo sandbox_info = {0};
 456   InitializeSandboxInfo(&sandbox_info);
 457   scoped_ptr<ContentMainDelegate> chrome_main_delegate(
 458       launcher_delegate->CreateContentMainDelegate());
 459   return ContentMain(GetModuleHandle(NULL),
 460                      &sandbox_info,
 461                      chrome_main_delegate.get());
 462 #elif defined(OS_LINUX)
 463   scoped_ptr<ContentMainDelegate> chrome_main_delegate(
 464       launcher_delegate->CreateContentMainDelegate());
 465   return ContentMain(argc, const_cast<const char**>(argv),
 466                      chrome_main_delegate.get());
 467 #endif  // defined(OS_WIN)
 468   NOTREACHED();
 469   return 0;
 470 }
 471
 472 int LaunchTests(TestLauncherDelegate* launcher_delegate,
 473                 int default_jobs,
 474                 int argc,
 475                 char** argv) {
 476   DCHECK(!g_launcher_delegate);
 477   g_launcher_delegate = launcher_delegate;
 478
 479   CommandLine::Init(argc, argv);
 480   const CommandLine* command_line = CommandLine::ForCurrentProcess();
 481
 482   if (command_line->HasSwitch(kHelpFlag)) {
 483     PrintUsage();
 484     return 0;
 485   }
 486
 487   if (command_line->HasSwitch(kSingleProcessTestsFlag) ||
 488       (command_line->HasSwitch(switches::kSingleProcess) &&
 489        command_line->HasSwitch(base::kGTestFilterFlag)) ||
 490       command_line->HasSwitch(base::kGTestListTestsFlag) ||
 491       command_line->HasSwitch(base::kGTestHelpFlag)) {
 492 #if defined(OS_WIN)
 493     if (command_line->HasSwitch(kSingleProcessTestsFlag)) {
 494       sandbox::SandboxInterfaceInfo sandbox_info;
 495       InitializeSandboxInfo(&sandbox_info);
 496       InitializeSandbox(&sandbox_info);
 497     }
 498 #endif
 499     return launcher_delegate->RunTestSuite(argc, argv);
 500   }
 501
 502   if (ShouldRunContentMain())
 503     return RunContentMain(argc, argv, launcher_delegate);
 504
 505   base::AtExitManager at_exit;
 506   testing::InitGoogleTest(&argc, argv);
 507   TestTimeouts::Initialize();
 508
 509   fprintf(stdout,
 510       "IMPORTANT DEBUGGING NOTE: each test is run inside its own process.\n"
 511       "For debugging a test inside a debugger, use the\n"
 512       "--gtest_filter=<your_test_name> flag along with either\n"
 513       "--single_process (to run the test in one launcher/browser process) or\n"
 514       "--single-process (to do the above, and also run Chrome in single-"
 515           "process mode).\n");
 516
 517   base::MessageLoopForIO message_loop;
 518
 519   WrapperTestLauncherDelegate delegate(launcher_delegate);
 520   base::TestLauncher launcher(&delegate, default_jobs);
 521   bool success = launcher.Run(argc, argv);
 522   return (success ? 0 : 1);
 523 }
 524
 525 TestLauncherDelegate* GetCurrentTestLauncherDelegate() {
 526   return g_launcher_delegate;
 527 }
 528
 529 }  // namespace content