base/test/launcher/unit_test_launcher.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "base/test/launcher/unit_test_launcher.h"
   6
   7 #include "base/bind.h"
   8 #include "base/callback_helpers.h"
   9 #include "base/command_line.h"
  10 #include "base/compiler_specific.h"
  11 #include "base/debug/debugger.h"
  12 #include "base/files/file_util.h"
  13 #include "base/files/scoped_temp_dir.h"
  14 #include "base/format_macros.h"
  15 #include "base/message_loop/message_loop.h"
  16 #include "base/stl_util.h"
  17 #include "base/strings/string_number_conversions.h"
  18 #include "base/strings/string_util.h"
  19 #include "base/sys_info.h"
  20 #include "base/test/gtest_xml_util.h"
  21 #include "base/test/launcher/test_launcher.h"
  22 #include "base/test/test_switches.h"
  23 #include "base/test/test_timeouts.h"
  24 #include "base/third_party/dynamic_annotations/dynamic_annotations.h"
  25 #include "base/threading/thread_checker.h"
  26 #include "testing/gtest/include/gtest/gtest.h"
  27
  28 namespace base {
  29
  30 namespace {
  31
  32 // This constant controls how many tests are run in a single batch by default.
  33 const size_t kDefaultTestBatchLimit = 10;
  34
  35 const char kHelpFlag[] = "help";
  36
  37 // Flag to run all tests in a single process.
  38 const char kSingleProcessTestsFlag[] = "single-process-tests";
  39
  40 void PrintUsage() {
  41   fprintf(stdout,
  42           "Runs tests using the gtest framework, each batch of tests being\n"
  43           "run in their own process. Supported command-line flags:\n"
  44           "\n"
  45           " Common flags:\n"
  46           "  --gtest_filter=...\n"
  47           "    Runs a subset of tests (see --gtest_help for more info).\n"
  48           "\n"
  49           "  --help\n"
  50           "    Shows this message.\n"
  51           "\n"
  52           "  --gtest_help\n"
  53           "    Shows the gtest help message.\n"
  54           "\n"
  55           "  --test-launcher-jobs=N\n"
  56           "    Sets the number of parallel test jobs to N.\n"
  57           "\n"
  58           "  --single-process-tests\n"
  59           "    Runs the tests and the launcher in the same process. Useful\n"
  60           "    for debugging a specific test in a debugger.\n"
  61           "\n"
  62           " Other flags:\n"
  63           "  --test-launcher-batch-limit=N\n"
  64           "    Sets the limit of test batch to run in a single process to N.\n"
  65           "\n"
  66           "  --test-launcher-debug-launcher\n"
  67           "    Disables autodetection of debuggers and similar tools,\n"
  68           "    making it possible to use them to debug launcher itself.\n"
  69           "\n"
  70           "  --test-launcher-retry-limit=N\n"
  71           "    Sets the limit of test retries on failures to N.\n"
  72           "\n"
  73           "  --test-launcher-summary-output=PATH\n"
  74           "    Saves a JSON machine-readable summary of the run.\n"
  75           "\n"
  76           "  --test-launcher-print-test-stdio=auto|always|never\n"
  77           "    Controls when full test output is printed.\n"
  78           "    auto means to print it when the test failed.\n"
  79           "\n"
  80           "  --test-launcher-total-shards=N\n"
  81           "    Sets the total number of shards to N.\n"
  82           "\n"
  83           "  --test-launcher-shard-index=N\n"
  84           "    Sets the shard index to run to N (from 0 to TOTAL - 1).\n");
  85   fflush(stdout);
  86 }
  87
  88 class DefaultUnitTestPlatformDelegate : public UnitTestPlatformDelegate {
  89  public:
  90   DefaultUnitTestPlatformDelegate() {
  91   }
  92
  93  private:
  94   // UnitTestPlatformDelegate:
  95   bool GetTests(std::vector<SplitTestName>* output) override {
  96     *output = GetCompiledInTests();
  97     return true;
  98   }
  99
 100   bool CreateTemporaryFile(base::FilePath* path) override {
 101     if (!CreateNewTempDirectory(FilePath::StringType(), path))
 102       return false;
 103     *path = path->AppendASCII("test_results.xml");
 104     return true;
 105   }
 106
 107   CommandLine GetCommandLineForChildGTestProcess(
 108       const std::vector<std::string>& test_names,
 109       const base::FilePath& output_file) override {
 110     CommandLine new_cmd_line(*CommandLine::ForCurrentProcess());
 111
 112     new_cmd_line.AppendSwitchPath(switches::kTestLauncherOutput, output_file);
 113     new_cmd_line.AppendSwitchASCII(kGTestFilterFlag,
 114                                    JoinString(test_names, ":"));
 115     new_cmd_line.AppendSwitch(kSingleProcessTestsFlag);
 116
 117     return new_cmd_line;
 118   }
 119
 120   std::string GetWrapperForChildGTestProcess() override {
 121     return std::string();
 122   }
 123
 124   void RelaunchTests(TestLauncher* test_launcher,
 125                      const std::vector<std::string>& test_names,
 126                      int launch_flags) override {
 127     // Relaunch requested tests in parallel, but only use single
 128     // test per batch for more precise results (crashes, etc).
 129     for (const std::string& test_name : test_names) {
 130       std::vector<std::string> batch;
 131       batch.push_back(test_name);
 132       RunUnitTestsBatch(test_launcher, this, batch, launch_flags);
 133     }
 134   }
 135
 136   DISALLOW_COPY_AND_ASSIGN(DefaultUnitTestPlatformDelegate);
 137 };
 138
 139 bool GetSwitchValueAsInt(const std::string& switch_name, int* result) {
 140   if (!CommandLine::ForCurrentProcess()->HasSwitch(switch_name))
 141     return true;
 142
 143   std::string switch_value =
 144       CommandLine::ForCurrentProcess()->GetSwitchValueASCII(switch_name);
 145   if (!StringToInt(switch_value, result) || *result < 1) {
 146     LOG(ERROR) << "Invalid value for " << switch_name << ": " << switch_value;
 147     return false;
 148   }
 149
 150   return true;
 151 }
 152
 153 int LaunchUnitTestsInternal(const RunTestSuiteCallback& run_test_suite,
 154                             int default_jobs,
 155                             bool use_job_objects,
 156                             const Closure& gtest_init) {
 157 #if defined(OS_ANDROID)
 158   // We can't easily fork on Android, just run the test suite directly.
 159   return run_test_suite.Run();
 160 #else
 161   bool force_single_process = false;
 162   if (CommandLine::ForCurrentProcess()->HasSwitch(
 163           switches::kTestLauncherDebugLauncher)) {
 164     fprintf(stdout, "Forcing test launcher debugging mode.\n");
 165     fflush(stdout);
 166   } else {
 167     if (base::debug::BeingDebugged()) {
 168       fprintf(stdout,
 169               "Debugger detected, switching to single process mode.\n"
 170               "Pass --test-launcher-debug-launcher to debug the launcher "
 171               "itself.\n");
 172       fflush(stdout);
 173       force_single_process = true;
 174     }
 175   }
 176
 177   if (CommandLine::ForCurrentProcess()->HasSwitch(kGTestHelpFlag) ||
 178       CommandLine::ForCurrentProcess()->HasSwitch(kGTestListTestsFlag) ||
 179       CommandLine::ForCurrentProcess()->HasSwitch(kSingleProcessTestsFlag) ||
 180       force_single_process) {
 181     return run_test_suite.Run();
 182   }
 183 #endif
 184
 185   if (CommandLine::ForCurrentProcess()->HasSwitch(kHelpFlag)) {
 186     PrintUsage();
 187     return 0;
 188   }
 189
 190   base::TimeTicks start_time(base::TimeTicks::Now());
 191
 192   gtest_init.Run();
 193   TestTimeouts::Initialize();
 194
 195   int batch_limit = kDefaultTestBatchLimit;
 196   if (!GetSwitchValueAsInt(switches::kTestLauncherBatchLimit, &batch_limit))
 197     return 1;
 198
 199   fprintf(stdout,
 200           "IMPORTANT DEBUGGING NOTE: batches of tests are run inside their\n"
 201           "own process. For debugging a test inside a debugger, use the\n"
 202           "--gtest_filter=<your_test_name> flag along with\n"
 203           "--single-process-tests.\n");
 204   fflush(stdout);
 205
 206   MessageLoopForIO message_loop;
 207
 208   DefaultUnitTestPlatformDelegate platform_delegate;
 209   UnitTestLauncherDelegate delegate(
 210       &platform_delegate, batch_limit, use_job_objects);
 211   base::TestLauncher launcher(&delegate, default_jobs);
 212   bool success = launcher.Run();
 213
 214   fprintf(stdout, "Tests took %" PRId64 " seconds.\n",
 215           (base::TimeTicks::Now() - start_time).InSeconds());
 216   fflush(stdout);
 217
 218   return (success ? 0 : 1);
 219 }
 220
 221 void InitGoogleTestChar(int* argc, char** argv) {
 222   testing::InitGoogleTest(argc, argv);
 223 }
 224
 225 #if defined(OS_WIN)
 226 void InitGoogleTestWChar(int* argc, wchar_t** argv) {
 227   testing::InitGoogleTest(argc, argv);
 228 }
 229 #endif  // defined(OS_WIN)
 230
 231 // Interprets test results and reports to the test launcher. Returns true
 232 // on success.
 233 bool ProcessTestResults(
 234     TestLauncher* test_launcher,
 235     const std::vector<std::string>& test_names,
 236     const base::FilePath& output_file,
 237     const std::string& output,
 238     int exit_code,
 239     bool was_timeout,
 240     std::vector<std::string>* tests_to_relaunch) {
 241   std::vector<TestResult> test_results;
 242   bool crashed = false;
 243   bool have_test_results =
 244       ProcessGTestOutput(output_file, &test_results, &crashed);
 245
 246   bool called_any_callback = false;
 247
 248   if (have_test_results) {
 249     // TODO(phajdan.jr): Check for duplicates and mismatches between
 250     // the results we got from XML file and tests we intended to run.
 251     std::map<std::string, TestResult> results_map;
 252     for (size_t i = 0; i < test_results.size(); i++)
 253       results_map[test_results[i].full_name] = test_results[i];
 254
 255     bool had_interrupted_test = false;
 256
 257     // Results to be reported back to the test launcher.
 258     std::vector<TestResult> final_results;
 259
 260     for (size_t i = 0; i < test_names.size(); i++) {
 261       if (ContainsKey(results_map, test_names[i])) {
 262         TestResult test_result = results_map[test_names[i]];
 263         if (test_result.status == TestResult::TEST_CRASH) {
 264           had_interrupted_test = true;
 265
 266           if (was_timeout) {
 267             // Fix up the test status: we forcibly kill the child process
 268             // after the timeout, so from XML results it looks just like
 269             // a crash.
 270             test_result.status = TestResult::TEST_TIMEOUT;
 271           }
 272         } else if (test_result.status == TestResult::TEST_SUCCESS ||
 273                    test_result.status == TestResult::TEST_FAILURE) {
 274           // We run multiple tests in a batch with a timeout applied
 275           // to the entire batch. It is possible that with other tests
 276           // running quickly some tests take longer than the per-test timeout.
 277           // For consistent handling of tests independent of order and other
 278           // factors, mark them as timing out.
 279           if (test_result.elapsed_time >
 280               TestTimeouts::test_launcher_timeout()) {
 281             test_result.status = TestResult::TEST_TIMEOUT;
 282           }
 283         }
 284         test_result.output_snippet = GetTestOutputSnippet(test_result, output);
 285         final_results.push_back(test_result);
 286       } else if (had_interrupted_test) {
 287         tests_to_relaunch->push_back(test_names[i]);
 288       } else {
 289         // TODO(phajdan.jr): Explicitly pass the info that the test didn't
 290         // run for a mysterious reason.
 291         LOG(ERROR) << "no test result for " << test_names[i];
 292         TestResult test_result;
 293         test_result.full_name = test_names[i];
 294         test_result.status = TestResult::TEST_UNKNOWN;
 295         test_result.output_snippet = GetTestOutputSnippet(test_result, output);
 296         final_results.push_back(test_result);
 297       }
 298     }
 299
 300     // TODO(phajdan.jr): Handle the case where processing XML output
 301     // indicates a crash but none of the test results is marked as crashing.
 302
 303     if (final_results.empty())
 304       return false;
 305
 306     bool has_non_success_test = false;
 307     for (size_t i = 0; i < final_results.size(); i++) {
 308       if (final_results[i].status != TestResult::TEST_SUCCESS) {
 309         has_non_success_test = true;
 310         break;
 311       }
 312     }
 313
 314     if (!has_non_success_test && exit_code != 0) {
 315       // This is a bit surprising case: all tests are marked as successful,
 316       // but the exit code was not zero. This can happen e.g. under memory
 317       // tools that report leaks this way. Mark all tests as a failure on exit,
 318       // and for more precise info they'd need to be retried serially.
 319       for (size_t i = 0; i < final_results.size(); i++)
 320         final_results[i].status = TestResult::TEST_FAILURE_ON_EXIT;
 321     }
 322
 323     for (size_t i = 0; i < final_results.size(); i++) {
 324       // Fix the output snippet after possible changes to the test result.
 325       final_results[i].output_snippet =
 326           GetTestOutputSnippet(final_results[i], output);
 327       test_launcher->OnTestFinished(final_results[i]);
 328       called_any_callback = true;
 329     }
 330   } else {
 331     fprintf(stdout,
 332             "Failed to get out-of-band test success data, "
 333             "dumping full stdio below:\n%s\n",
 334             output.c_str());
 335     fflush(stdout);
 336
 337     // We do not have reliable details about test results (parsing test
 338     // stdout is known to be unreliable), apply the executable exit code
 339     // to all tests.
 340     // TODO(phajdan.jr): Be smarter about this, e.g. retry each test
 341     // individually.
 342     for (size_t i = 0; i < test_names.size(); i++) {
 343       TestResult test_result;
 344       test_result.full_name = test_names[i];
 345       test_result.status = TestResult::TEST_UNKNOWN;
 346       test_launcher->OnTestFinished(test_result);
 347       called_any_callback = true;
 348     }
 349   }
 350
 351   return called_any_callback;
 352 }
 353
 354 // TODO(phajdan.jr): Pass parameters directly with C++11 variadic templates.
 355 struct GTestCallbackState {
 356   TestLauncher* test_launcher;
 357   UnitTestPlatformDelegate* platform_delegate;
 358   std::vector<std::string> test_names;
 359   int launch_flags;
 360   FilePath output_file;
 361 };
 362
 363 void GTestCallback(
 364     const GTestCallbackState& callback_state,
 365     int exit_code,
 366     const TimeDelta& elapsed_time,
 367     bool was_timeout,
 368     const std::string& output) {
 369   std::vector<std::string> tests_to_relaunch;
 370   ProcessTestResults(callback_state.test_launcher, callback_state.test_names,
 371                      callback_state.output_file, output, exit_code, was_timeout,
 372                      &tests_to_relaunch);
 373
 374   if (!tests_to_relaunch.empty()) {
 375     callback_state.platform_delegate->RelaunchTests(
 376         callback_state.test_launcher,
 377         tests_to_relaunch,
 378         callback_state.launch_flags);
 379   }
 380
 381   // The temporary file's directory is also temporary.
 382   DeleteFile(callback_state.output_file.DirName(), true);
 383 }
 384
 385 void SerialGTestCallback(
 386     const GTestCallbackState& callback_state,
 387     const std::vector<std::string>& test_names,
 388     int exit_code,
 389     const TimeDelta& elapsed_time,
 390     bool was_timeout,
 391     const std::string& output) {
 392   std::vector<std::string> tests_to_relaunch;
 393   bool called_any_callbacks =
 394       ProcessTestResults(callback_state.test_launcher,
 395                          callback_state.test_names, callback_state.output_file,
 396                          output, exit_code, was_timeout, &tests_to_relaunch);
 397
 398   // There is only one test, there cannot be other tests to relaunch
 399   // due to a crash.
 400   DCHECK(tests_to_relaunch.empty());
 401
 402   // There is only one test, we should have called back with its result.
 403   DCHECK(called_any_callbacks);
 404
 405   // The temporary file's directory is also temporary.
 406   DeleteFile(callback_state.output_file.DirName(), true);
 407
 408   MessageLoop::current()->PostTask(
 409       FROM_HERE,
 410       Bind(&RunUnitTestsSerially,
 411            callback_state.test_launcher,
 412            callback_state.platform_delegate,
 413            test_names,
 414            callback_state.launch_flags));
 415 }
 416
 417 }  // namespace
 418
 419 int LaunchUnitTests(int argc,
 420                     char** argv,
 421                     const RunTestSuiteCallback& run_test_suite) {
 422   CommandLine::Init(argc, argv);
 423   return LaunchUnitTestsInternal(run_test_suite, SysInfo::NumberOfProcessors(),
 424                                  true, Bind(&InitGoogleTestChar, &argc, argv));
 425 }
 426
 427 int LaunchUnitTestsSerially(int argc,
 428                             char** argv,
 429                             const RunTestSuiteCallback& run_test_suite) {
 430   CommandLine::Init(argc, argv);
 431   return LaunchUnitTestsInternal(run_test_suite, 1, true,
 432                                  Bind(&InitGoogleTestChar, &argc, argv));
 433 }
 434
 435 #if defined(OS_WIN)
 436 int LaunchUnitTests(int argc,
 437                     wchar_t** argv,
 438                     bool use_job_objects,
 439                     const RunTestSuiteCallback& run_test_suite) {
 440   // Windows CommandLine::Init ignores argv anyway.
 441   CommandLine::Init(argc, NULL);
 442   return LaunchUnitTestsInternal(run_test_suite, SysInfo::NumberOfProcessors(),
 443                                  use_job_objects,
 444                                  Bind(&InitGoogleTestWChar, &argc, argv));
 445 }
 446 #endif  // defined(OS_WIN)
 447
 448 void RunUnitTestsSerially(
 449     TestLauncher* test_launcher,
 450     UnitTestPlatformDelegate* platform_delegate,
 451     const std::vector<std::string>& test_names,
 452     int launch_flags) {
 453   if (test_names.empty())
 454     return;
 455
 456   std::vector<std::string> new_test_names(test_names);
 457   std::string test_name(new_test_names.back());
 458   new_test_names.pop_back();
 459
 460   // Create a dedicated temporary directory to store the xml result data
 461   // per run to ensure clean state and make it possible to launch multiple
 462   // processes in parallel.
 463   base::FilePath output_file;
 464   CHECK(platform_delegate->CreateTemporaryFile(&output_file));
 465
 466   std::vector<std::string> current_test_names;
 467   current_test_names.push_back(test_name);
 468   CommandLine cmd_line(platform_delegate->GetCommandLineForChildGTestProcess(
 469       current_test_names, output_file));
 470
 471   GTestCallbackState callback_state;
 472   callback_state.test_launcher = test_launcher;
 473   callback_state.platform_delegate = platform_delegate;
 474   callback_state.test_names = current_test_names;
 475   callback_state.launch_flags = launch_flags;
 476   callback_state.output_file = output_file;
 477
 478   test_launcher->LaunchChildGTestProcess(
 479       cmd_line,
 480       platform_delegate->GetWrapperForChildGTestProcess(),
 481       TestTimeouts::test_launcher_timeout(),
 482       launch_flags,
 483       Bind(&SerialGTestCallback, callback_state, new_test_names));
 484 }
 485
 486 void RunUnitTestsBatch(
 487     TestLauncher* test_launcher,
 488     UnitTestPlatformDelegate* platform_delegate,
 489     const std::vector<std::string>& test_names,
 490     int launch_flags) {
 491   if (test_names.empty())
 492     return;
 493
 494   // Create a dedicated temporary directory to store the xml result data
 495   // per run to ensure clean state and make it possible to launch multiple
 496   // processes in parallel.
 497   base::FilePath output_file;
 498   CHECK(platform_delegate->CreateTemporaryFile(&output_file));
 499
 500   CommandLine cmd_line(platform_delegate->GetCommandLineForChildGTestProcess(
 501       test_names, output_file));
 502
 503   // Adjust the timeout depending on how many tests we're running
 504   // (note that e.g. the last batch of tests will be smaller).
 505   // TODO(phajdan.jr): Consider an adaptive timeout, which can change
 506   // depending on how many tests ran and how many remain.
 507   // Note: do NOT parse child's stdout to do that, it's known to be
 508   // unreliable (e.g. buffering issues can mix up the output).
 509   base::TimeDelta timeout =
 510       test_names.size() * TestTimeouts::test_launcher_timeout();
 511
 512   GTestCallbackState callback_state;
 513   callback_state.test_launcher = test_launcher;
 514   callback_state.platform_delegate = platform_delegate;
 515   callback_state.test_names = test_names;
 516   callback_state.launch_flags = launch_flags;
 517   callback_state.output_file = output_file;
 518
 519   test_launcher->LaunchChildGTestProcess(
 520       cmd_line,
 521       platform_delegate->GetWrapperForChildGTestProcess(),
 522       timeout,
 523       launch_flags,
 524       Bind(&GTestCallback, callback_state));
 525 }
 526
 527 UnitTestLauncherDelegate::UnitTestLauncherDelegate(
 528     UnitTestPlatformDelegate* platform_delegate,
 529     size_t batch_limit,
 530     bool use_job_objects)
 531     : platform_delegate_(platform_delegate),
 532       batch_limit_(batch_limit),
 533       use_job_objects_(use_job_objects) {
 534 }
 535
 536 UnitTestLauncherDelegate::~UnitTestLauncherDelegate() {
 537   DCHECK(thread_checker_.CalledOnValidThread());
 538 }
 539
 540 bool UnitTestLauncherDelegate::GetTests(std::vector<SplitTestName>* output) {
 541   DCHECK(thread_checker_.CalledOnValidThread());
 542   return platform_delegate_->GetTests(output);
 543 }
 544
 545 bool UnitTestLauncherDelegate::ShouldRunTest(const std::string& test_case_name,
 546                                              const std::string& test_name) {
 547   DCHECK(thread_checker_.CalledOnValidThread());
 548
 549   // There is no additional logic to disable specific tests.
 550   return true;
 551 }
 552
 553 size_t UnitTestLauncherDelegate::RunTests(
 554     TestLauncher* test_launcher,
 555     const std::vector<std::string>& test_names) {
 556   DCHECK(thread_checker_.CalledOnValidThread());
 557
 558   int launch_flags = use_job_objects_ ? TestLauncher::USE_JOB_OBJECTS : 0;
 559
 560   std::vector<std::string> batch;
 561   for (size_t i = 0; i < test_names.size(); i++) {
 562     batch.push_back(test_names[i]);
 563
 564     // Use 0 to indicate unlimited batch size.
 565     if (batch.size() >= batch_limit_ && batch_limit_ != 0) {
 566       RunUnitTestsBatch(test_launcher, platform_delegate_, batch, launch_flags);
 567       batch.clear();
 568     }
 569   }
 570
 571   RunUnitTestsBatch(test_launcher, platform_delegate_, batch, launch_flags);
 572
 573   return test_names.size();
 574 }
 575
 576 size_t UnitTestLauncherDelegate::RetryTests(
 577     TestLauncher* test_launcher,
 578     const std::vector<std::string>& test_names) {
 579   MessageLoop::current()->PostTask(
 580       FROM_HERE,
 581       Bind(&RunUnitTestsSerially,
 582            test_launcher,
 583            platform_delegate_,
 584            test_names,
 585            use_job_objects_ ? TestLauncher::USE_JOB_OBJECTS : 0));
 586   return test_names.size();
 587 }
 588
 589 }  // namespace base