base/test/launcher/unit_test_launcher.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "base/test/launcher/unit_test_launcher.h"
   6
   7 #include "base/base_switches.h"
   8 #include "base/bind.h"
   9 #include "base/callback_helpers.h"
  10 #include "base/command_line.h"
  11 #include "base/compiler_specific.h"
  12 #include "base/debug/debugger.h"
  13 #include "base/files/file_util.h"
  14 #include "base/files/scoped_temp_dir.h"
  15 #include "base/format_macros.h"
  16 #include "base/location.h"
  17 #include "base/message_loop/message_loop.h"
  18 #include "base/single_thread_task_runner.h"
  19 #include "base/stl_util.h"
  20 #include "base/strings/string_number_conversions.h"
  21 #include "base/strings/string_util.h"
  22 #include "base/sys_info.h"
  23 #include "base/test/gtest_xml_util.h"
  24 #include "base/test/launcher/test_launcher.h"
  25 #include "base/test/test_switches.h"
  26 #include "base/test/test_timeouts.h"
  27 #include "base/third_party/dynamic_annotations/dynamic_annotations.h"
  28 #include "base/thread_task_runner_handle.h"
  29 #include "base/threading/thread_checker.h"
  30 #include "testing/gtest/include/gtest/gtest.h"
  31
  32 namespace base {
  33
  34 namespace {
  35
  36 // This constant controls how many tests are run in a single batch by default.
  37 const size_t kDefaultTestBatchLimit = 10;
  38
  39 const char kHelpFlag[] = "help";
  40
  41 // Flag to run all tests in a single process.
  42 const char kSingleProcessTestsFlag[] = "single-process-tests";
  43
  44 void PrintUsage() {
  45   fprintf(stdout,
  46           "Runs tests using the gtest framework, each batch of tests being\n"
  47           "run in their own process. Supported command-line flags:\n"
  48           "\n"
  49           " Common flags:\n"
  50           "  --gtest_filter=...\n"
  51           "    Runs a subset of tests (see --gtest_help for more info).\n"
  52           "\n"
  53           "  --help\n"
  54           "    Shows this message.\n"
  55           "\n"
  56           "  --gtest_help\n"
  57           "    Shows the gtest help message.\n"
  58           "\n"
  59           "  --test-launcher-jobs=N\n"
  60           "    Sets the number of parallel test jobs to N.\n"
  61           "\n"
  62           "  --single-process-tests\n"
  63           "    Runs the tests and the launcher in the same process. Useful\n"
  64           "    for debugging a specific test in a debugger.\n"
  65           "\n"
  66           " Other flags:\n"
  67           "  --test-launcher-batch-limit=N\n"
  68           "    Sets the limit of test batch to run in a single process to N.\n"
  69           "\n"
  70           "  --test-launcher-debug-launcher\n"
  71           "    Disables autodetection of debuggers and similar tools,\n"
  72           "    making it possible to use them to debug launcher itself.\n"
  73           "\n"
  74           "  --test-launcher-retry-limit=N\n"
  75           "    Sets the limit of test retries on failures to N.\n"
  76           "\n"
  77           "  --test-launcher-summary-output=PATH\n"
  78           "    Saves a JSON machine-readable summary of the run.\n"
  79           "\n"
  80           "  --test-launcher-print-test-stdio=auto|always|never\n"
  81           "    Controls when full test output is printed.\n"
  82           "    auto means to print it when the test failed.\n"
  83           "\n"
  84           "  --test-launcher-total-shards=N\n"
  85           "    Sets the total number of shards to N.\n"
  86           "\n"
  87           "  --test-launcher-shard-index=N\n"
  88           "    Sets the shard index to run to N (from 0 to TOTAL - 1).\n");
  89   fflush(stdout);
  90 }
  91
  92 class DefaultUnitTestPlatformDelegate : public UnitTestPlatformDelegate {
  93  public:
  94   DefaultUnitTestPlatformDelegate() {
  95   }
  96
  97  private:
  98   // UnitTestPlatformDelegate:
  99   bool GetTests(std::vector<TestIdentifier>* output) override {
 100     *output = GetCompiledInTests();
 101     return true;
 102   }
 103
 104   bool CreateTemporaryFile(base::FilePath* path) override {
 105     if (!CreateNewTempDirectory(FilePath::StringType(), path))
 106       return false;
 107     *path = path->AppendASCII("test_results.xml");
 108     return true;
 109   }
 110
 111   CommandLine GetCommandLineForChildGTestProcess(
 112       const std::vector<std::string>& test_names,
 113       const base::FilePath& output_file) override {
 114     CommandLine new_cmd_line(*CommandLine::ForCurrentProcess());
 115
 116     new_cmd_line.AppendSwitchPath(switches::kTestLauncherOutput, output_file);
 117     new_cmd_line.AppendSwitchASCII(kGTestFilterFlag,
 118                                    JoinString(test_names, ":"));
 119     new_cmd_line.AppendSwitch(kSingleProcessTestsFlag);
 120
 121     return new_cmd_line;
 122   }
 123
 124   std::string GetWrapperForChildGTestProcess() override {
 125     return std::string();
 126   }
 127
 128   void RelaunchTests(TestLauncher* test_launcher,
 129                      const std::vector<std::string>& test_names,
 130                      int launch_flags) override {
 131     // Relaunch requested tests in parallel, but only use single
 132     // test per batch for more precise results (crashes, etc).
 133     for (const std::string& test_name : test_names) {
 134       std::vector<std::string> batch;
 135       batch.push_back(test_name);
 136       RunUnitTestsBatch(test_launcher, this, batch, launch_flags);
 137     }
 138   }
 139
 140   DISALLOW_COPY_AND_ASSIGN(DefaultUnitTestPlatformDelegate);
 141 };
 142
 143 bool GetSwitchValueAsInt(const std::string& switch_name, int* result) {
 144   if (!CommandLine::ForCurrentProcess()->HasSwitch(switch_name))
 145     return true;
 146
 147   std::string switch_value =
 148       CommandLine::ForCurrentProcess()->GetSwitchValueASCII(switch_name);
 149   if (!StringToInt(switch_value, result) || *result < 1) {
 150     LOG(ERROR) << "Invalid value for " << switch_name << ": " << switch_value;
 151     return false;
 152   }
 153
 154   return true;
 155 }
 156
 157 int LaunchUnitTestsInternal(const RunTestSuiteCallback& run_test_suite,
 158                             int default_jobs,
 159                             bool use_job_objects,
 160                             const Closure& gtest_init) {
 161 #if defined(OS_ANDROID)
 162   // We can't easily fork on Android, just run the test suite directly.
 163   return run_test_suite.Run();
 164 #else
 165   bool force_single_process = false;
 166   if (CommandLine::ForCurrentProcess()->HasSwitch(
 167           switches::kTestLauncherDebugLauncher)) {
 168     fprintf(stdout, "Forcing test launcher debugging mode.\n");
 169     fflush(stdout);
 170   } else {
 171     if (base::debug::BeingDebugged()) {
 172       fprintf(stdout,
 173               "Debugger detected, switching to single process mode.\n"
 174               "Pass --test-launcher-debug-launcher to debug the launcher "
 175               "itself.\n");
 176       fflush(stdout);
 177       force_single_process = true;
 178     }
 179   }
 180
 181   if (CommandLine::ForCurrentProcess()->HasSwitch(kGTestHelpFlag) ||
 182       CommandLine::ForCurrentProcess()->HasSwitch(kGTestListTestsFlag) ||
 183       CommandLine::ForCurrentProcess()->HasSwitch(kSingleProcessTestsFlag) ||
 184       CommandLine::ForCurrentProcess()->HasSwitch(
 185           switches::kTestChildProcess) ||
 186       force_single_process) {
 187     return run_test_suite.Run();
 188   }
 189 #endif
 190
 191   if (CommandLine::ForCurrentProcess()->HasSwitch(kHelpFlag)) {
 192     PrintUsage();
 193     return 0;
 194   }
 195
 196   base::TimeTicks start_time(base::TimeTicks::Now());
 197
 198   gtest_init.Run();
 199   TestTimeouts::Initialize();
 200
 201   int batch_limit = kDefaultTestBatchLimit;
 202   if (!GetSwitchValueAsInt(switches::kTestLauncherBatchLimit, &batch_limit))
 203     return 1;
 204
 205   fprintf(stdout,
 206           "IMPORTANT DEBUGGING NOTE: batches of tests are run inside their\n"
 207           "own process. For debugging a test inside a debugger, use the\n"
 208           "--gtest_filter=<your_test_name> flag along with\n"
 209           "--single-process-tests.\n");
 210   fflush(stdout);
 211
 212   MessageLoopForIO message_loop;
 213
 214   DefaultUnitTestPlatformDelegate platform_delegate;
 215   UnitTestLauncherDelegate delegate(
 216       &platform_delegate, batch_limit, use_job_objects);
 217   base::TestLauncher launcher(&delegate, default_jobs);
 218   bool success = launcher.Run();
 219
 220   fprintf(stdout, "Tests took %" PRId64 " seconds.\n",
 221           (base::TimeTicks::Now() - start_time).InSeconds());
 222   fflush(stdout);
 223
 224   return (success ? 0 : 1);
 225 }
 226
 227 void InitGoogleTestChar(int* argc, char** argv) {
 228   testing::InitGoogleTest(argc, argv);
 229 }
 230
 231 #if defined(OS_WIN)
 232 void InitGoogleTestWChar(int* argc, wchar_t** argv) {
 233   testing::InitGoogleTest(argc, argv);
 234 }
 235 #endif  // defined(OS_WIN)
 236
 237 // Interprets test results and reports to the test launcher. Returns true
 238 // on success.
 239 bool ProcessTestResults(
 240     TestLauncher* test_launcher,
 241     const std::vector<std::string>& test_names,
 242     const base::FilePath& output_file,
 243     const std::string& output,
 244     int exit_code,
 245     bool was_timeout,
 246     std::vector<std::string>* tests_to_relaunch) {
 247   std::vector<TestResult> test_results;
 248   bool crashed = false;
 249   bool have_test_results =
 250       ProcessGTestOutput(output_file, &test_results, &crashed);
 251
 252   bool called_any_callback = false;
 253
 254   if (have_test_results) {
 255     // TODO(phajdan.jr): Check for duplicates and mismatches between
 256     // the results we got from XML file and tests we intended to run.
 257     std::map<std::string, TestResult> results_map;
 258     for (size_t i = 0; i < test_results.size(); i++)
 259       results_map[test_results[i].full_name] = test_results[i];
 260
 261     bool had_interrupted_test = false;
 262
 263     // Results to be reported back to the test launcher.
 264     std::vector<TestResult> final_results;
 265
 266     for (size_t i = 0; i < test_names.size(); i++) {
 267       if (ContainsKey(results_map, test_names[i])) {
 268         TestResult test_result = results_map[test_names[i]];
 269         if (test_result.status == TestResult::TEST_CRASH) {
 270           had_interrupted_test = true;
 271
 272           if (was_timeout) {
 273             // Fix up the test status: we forcibly kill the child process
 274             // after the timeout, so from XML results it looks just like
 275             // a crash.
 276             test_result.status = TestResult::TEST_TIMEOUT;
 277           }
 278         } else if (test_result.status == TestResult::TEST_SUCCESS ||
 279                    test_result.status == TestResult::TEST_FAILURE) {
 280           // We run multiple tests in a batch with a timeout applied
 281           // to the entire batch. It is possible that with other tests
 282           // running quickly some tests take longer than the per-test timeout.
 283           // For consistent handling of tests independent of order and other
 284           // factors, mark them as timing out.
 285           if (test_result.elapsed_time >
 286               TestTimeouts::test_launcher_timeout()) {
 287             test_result.status = TestResult::TEST_TIMEOUT;
 288           }
 289         }
 290         test_result.output_snippet = GetTestOutputSnippet(test_result, output);
 291         final_results.push_back(test_result);
 292       } else if (had_interrupted_test) {
 293         tests_to_relaunch->push_back(test_names[i]);
 294       } else {
 295         // TODO(phajdan.jr): Explicitly pass the info that the test didn't
 296         // run for a mysterious reason.
 297         LOG(ERROR) << "no test result for " << test_names[i];
 298         TestResult test_result;
 299         test_result.full_name = test_names[i];
 300         test_result.status = TestResult::TEST_UNKNOWN;
 301         test_result.output_snippet = GetTestOutputSnippet(test_result, output);
 302         final_results.push_back(test_result);
 303       }
 304     }
 305
 306     // TODO(phajdan.jr): Handle the case where processing XML output
 307     // indicates a crash but none of the test results is marked as crashing.
 308
 309     if (final_results.empty())
 310       return false;
 311
 312     bool has_non_success_test = false;
 313     for (size_t i = 0; i < final_results.size(); i++) {
 314       if (final_results[i].status != TestResult::TEST_SUCCESS) {
 315         has_non_success_test = true;
 316         break;
 317       }
 318     }
 319
 320     if (!has_non_success_test && exit_code != 0) {
 321       // This is a bit surprising case: all tests are marked as successful,
 322       // but the exit code was not zero. This can happen e.g. under memory
 323       // tools that report leaks this way. Mark all tests as a failure on exit,
 324       // and for more precise info they'd need to be retried serially.
 325       for (size_t i = 0; i < final_results.size(); i++)
 326         final_results[i].status = TestResult::TEST_FAILURE_ON_EXIT;
 327     }
 328
 329     for (size_t i = 0; i < final_results.size(); i++) {
 330       // Fix the output snippet after possible changes to the test result.
 331       final_results[i].output_snippet =
 332           GetTestOutputSnippet(final_results[i], output);
 333       test_launcher->OnTestFinished(final_results[i]);
 334       called_any_callback = true;
 335     }
 336   } else {
 337     fprintf(stdout,
 338             "Failed to get out-of-band test success data, "
 339             "dumping full stdio below:\n%s\n",
 340             output.c_str());
 341     fflush(stdout);
 342
 343     // We do not have reliable details about test results (parsing test
 344     // stdout is known to be unreliable), apply the executable exit code
 345     // to all tests.
 346     // TODO(phajdan.jr): Be smarter about this, e.g. retry each test
 347     // individually.
 348     for (size_t i = 0; i < test_names.size(); i++) {
 349       TestResult test_result;
 350       test_result.full_name = test_names[i];
 351       test_result.status = TestResult::TEST_UNKNOWN;
 352       test_launcher->OnTestFinished(test_result);
 353       called_any_callback = true;
 354     }
 355   }
 356
 357   return called_any_callback;
 358 }
 359
 360 // TODO(phajdan.jr): Pass parameters directly with C++11 variadic templates.
 361 struct GTestCallbackState {
 362   TestLauncher* test_launcher;
 363   UnitTestPlatformDelegate* platform_delegate;
 364   std::vector<std::string> test_names;
 365   int launch_flags;
 366   FilePath output_file;
 367 };
 368
 369 void GTestCallback(
 370     const GTestCallbackState& callback_state,
 371     int exit_code,
 372     const TimeDelta& elapsed_time,
 373     bool was_timeout,
 374     const std::string& output) {
 375   std::vector<std::string> tests_to_relaunch;
 376   ProcessTestResults(callback_state.test_launcher, callback_state.test_names,
 377                      callback_state.output_file, output, exit_code, was_timeout,
 378                      &tests_to_relaunch);
 379
 380   if (!tests_to_relaunch.empty()) {
 381     callback_state.platform_delegate->RelaunchTests(
 382         callback_state.test_launcher,
 383         tests_to_relaunch,
 384         callback_state.launch_flags);
 385   }
 386
 387   // The temporary file's directory is also temporary.
 388   DeleteFile(callback_state.output_file.DirName(), true);
 389 }
 390
 391 void SerialGTestCallback(
 392     const GTestCallbackState& callback_state,
 393     const std::vector<std::string>& test_names,
 394     int exit_code,
 395     const TimeDelta& elapsed_time,
 396     bool was_timeout,
 397     const std::string& output) {
 398   std::vector<std::string> tests_to_relaunch;
 399   bool called_any_callbacks =
 400       ProcessTestResults(callback_state.test_launcher,
 401                          callback_state.test_names, callback_state.output_file,
 402                          output, exit_code, was_timeout, &tests_to_relaunch);
 403
 404   // There is only one test, there cannot be other tests to relaunch
 405   // due to a crash.
 406   DCHECK(tests_to_relaunch.empty());
 407
 408   // There is only one test, we should have called back with its result.
 409   DCHECK(called_any_callbacks);
 410
 411   // The temporary file's directory is also temporary.
 412   DeleteFile(callback_state.output_file.DirName(), true);
 413
 414   ThreadTaskRunnerHandle::Get()->PostTask(
 415       FROM_HERE, Bind(&RunUnitTestsSerially, callback_state.test_launcher,
 416                       callback_state.platform_delegate, test_names,
 417                       callback_state.launch_flags));
 418 }
 419
 420 }  // namespace
 421
 422 int LaunchUnitTests(int argc,
 423                     char** argv,
 424                     const RunTestSuiteCallback& run_test_suite) {
 425   CommandLine::Init(argc, argv);
 426   return LaunchUnitTestsInternal(run_test_suite, SysInfo::NumberOfProcessors(),
 427                                  true, Bind(&InitGoogleTestChar, &argc, argv));
 428 }
 429
 430 int LaunchUnitTestsSerially(int argc,
 431                             char** argv,
 432                             const RunTestSuiteCallback& run_test_suite) {
 433   CommandLine::Init(argc, argv);
 434   return LaunchUnitTestsInternal(run_test_suite, 1, true,
 435                                  Bind(&InitGoogleTestChar, &argc, argv));
 436 }
 437
 438 #if defined(OS_WIN)
 439 int LaunchUnitTests(int argc,
 440                     wchar_t** argv,
 441                     bool use_job_objects,
 442                     const RunTestSuiteCallback& run_test_suite) {
 443   // Windows CommandLine::Init ignores argv anyway.
 444   CommandLine::Init(argc, NULL);
 445   return LaunchUnitTestsInternal(run_test_suite, SysInfo::NumberOfProcessors(),
 446                                  use_job_objects,
 447                                  Bind(&InitGoogleTestWChar, &argc, argv));
 448 }
 449 #endif  // defined(OS_WIN)
 450
 451 void RunUnitTestsSerially(
 452     TestLauncher* test_launcher,
 453     UnitTestPlatformDelegate* platform_delegate,
 454     const std::vector<std::string>& test_names,
 455     int launch_flags) {
 456   if (test_names.empty())
 457     return;
 458
 459   std::vector<std::string> new_test_names(test_names);
 460   std::string test_name(new_test_names.back());
 461   new_test_names.pop_back();
 462
 463   // Create a dedicated temporary directory to store the xml result data
 464   // per run to ensure clean state and make it possible to launch multiple
 465   // processes in parallel.
 466   base::FilePath output_file;
 467   CHECK(platform_delegate->CreateTemporaryFile(&output_file));
 468
 469   std::vector<std::string> current_test_names;
 470   current_test_names.push_back(test_name);
 471   CommandLine cmd_line(platform_delegate->GetCommandLineForChildGTestProcess(
 472       current_test_names, output_file));
 473
 474   GTestCallbackState callback_state;
 475   callback_state.test_launcher = test_launcher;
 476   callback_state.platform_delegate = platform_delegate;
 477   callback_state.test_names = current_test_names;
 478   callback_state.launch_flags = launch_flags;
 479   callback_state.output_file = output_file;
 480
 481   test_launcher->LaunchChildGTestProcess(
 482       cmd_line,
 483       platform_delegate->GetWrapperForChildGTestProcess(),
 484       TestTimeouts::test_launcher_timeout(),
 485       launch_flags,
 486       Bind(&SerialGTestCallback, callback_state, new_test_names));
 487 }
 488
 489 void RunUnitTestsBatch(
 490     TestLauncher* test_launcher,
 491     UnitTestPlatformDelegate* platform_delegate,
 492     const std::vector<std::string>& test_names,
 493     int launch_flags) {
 494   if (test_names.empty())
 495     return;
 496
 497   // Create a dedicated temporary directory to store the xml result data
 498   // per run to ensure clean state and make it possible to launch multiple
 499   // processes in parallel.
 500   base::FilePath output_file;
 501   CHECK(platform_delegate->CreateTemporaryFile(&output_file));
 502
 503   CommandLine cmd_line(platform_delegate->GetCommandLineForChildGTestProcess(
 504       test_names, output_file));
 505
 506   // Adjust the timeout depending on how many tests we're running
 507   // (note that e.g. the last batch of tests will be smaller).
 508   // TODO(phajdan.jr): Consider an adaptive timeout, which can change
 509   // depending on how many tests ran and how many remain.
 510   // Note: do NOT parse child's stdout to do that, it's known to be
 511   // unreliable (e.g. buffering issues can mix up the output).
 512   base::TimeDelta timeout =
 513       test_names.size() * TestTimeouts::test_launcher_timeout();
 514
 515   GTestCallbackState callback_state;
 516   callback_state.test_launcher = test_launcher;
 517   callback_state.platform_delegate = platform_delegate;
 518   callback_state.test_names = test_names;
 519   callback_state.launch_flags = launch_flags;
 520   callback_state.output_file = output_file;
 521
 522   test_launcher->LaunchChildGTestProcess(
 523       cmd_line,
 524       platform_delegate->GetWrapperForChildGTestProcess(),
 525       timeout,
 526       launch_flags,
 527       Bind(&GTestCallback, callback_state));
 528 }
 529
 530 UnitTestLauncherDelegate::UnitTestLauncherDelegate(
 531     UnitTestPlatformDelegate* platform_delegate,
 532     size_t batch_limit,
 533     bool use_job_objects)
 534     : platform_delegate_(platform_delegate),
 535       batch_limit_(batch_limit),
 536       use_job_objects_(use_job_objects) {
 537 }
 538
 539 UnitTestLauncherDelegate::~UnitTestLauncherDelegate() {
 540   DCHECK(thread_checker_.CalledOnValidThread());
 541 }
 542
 543 bool UnitTestLauncherDelegate::GetTests(std::vector<TestIdentifier>* output) {
 544   DCHECK(thread_checker_.CalledOnValidThread());
 545   return platform_delegate_->GetTests(output);
 546 }
 547
 548 bool UnitTestLauncherDelegate::ShouldRunTest(const std::string& test_case_name,
 549                                              const std::string& test_name) {
 550   DCHECK(thread_checker_.CalledOnValidThread());
 551
 552   // There is no additional logic to disable specific tests.
 553   return true;
 554 }
 555
 556 size_t UnitTestLauncherDelegate::RunTests(
 557     TestLauncher* test_launcher,
 558     const std::vector<std::string>& test_names) {
 559   DCHECK(thread_checker_.CalledOnValidThread());
 560
 561   int launch_flags = use_job_objects_ ? TestLauncher::USE_JOB_OBJECTS : 0;
 562
 563   std::vector<std::string> batch;
 564   for (size_t i = 0; i < test_names.size(); i++) {
 565     batch.push_back(test_names[i]);
 566
 567     // Use 0 to indicate unlimited batch size.
 568     if (batch.size() >= batch_limit_ && batch_limit_ != 0) {
 569       RunUnitTestsBatch(test_launcher, platform_delegate_, batch, launch_flags);
 570       batch.clear();
 571     }
 572   }
 573
 574   RunUnitTestsBatch(test_launcher, platform_delegate_, batch, launch_flags);
 575
 576   return test_names.size();
 577 }
 578
 579 size_t UnitTestLauncherDelegate::RetryTests(
 580     TestLauncher* test_launcher,
 581     const std::vector<std::string>& test_names) {
 582   ThreadTaskRunnerHandle::Get()->PostTask(
 583       FROM_HERE,
 584       Bind(&RunUnitTestsSerially, test_launcher, platform_delegate_, test_names,
 585            use_job_objects_ ? TestLauncher::USE_JOB_OBJECTS : 0));
 586   return test_names.size();
 587 }
 588
 589 }  // namespace base