base/test/launcher/unit_test_launcher.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "base/test/launcher/unit_test_launcher.h"
   6
   7 #include "base/bind.h"
   8 #include "base/callback_helpers.h"
   9 #include "base/command_line.h"
  10 #include "base/compiler_specific.h"
  11 #include "base/debug/debugger.h"
  12 #include "base/files/file_util.h"
  13 #include "base/files/scoped_temp_dir.h"
  14 #include "base/format_macros.h"
  15 #include "base/location.h"
  16 #include "base/message_loop/message_loop.h"
  17 #include "base/single_thread_task_runner.h"
  18 #include "base/stl_util.h"
  19 #include "base/strings/string_number_conversions.h"
  20 #include "base/strings/string_util.h"
  21 #include "base/sys_info.h"
  22 #include "base/test/gtest_xml_util.h"
  23 #include "base/test/launcher/test_launcher.h"
  24 #include "base/test/test_switches.h"
  25 #include "base/test/test_timeouts.h"
  26 #include "base/third_party/dynamic_annotations/dynamic_annotations.h"
  27 #include "base/thread_task_runner_handle.h"
  28 #include "base/threading/thread_checker.h"
  29 #include "testing/gtest/include/gtest/gtest.h"
  30
  31 namespace base {
  32
  33 namespace {
  34
  35 // This constant controls how many tests are run in a single batch by default.
  36 const size_t kDefaultTestBatchLimit = 10;
  37
  38 const char kHelpFlag[] = "help";
  39
  40 // Flag to run all tests in a single process.
  41 const char kSingleProcessTestsFlag[] = "single-process-tests";
  42
  43 void PrintUsage() {
  44   fprintf(stdout,
  45           "Runs tests using the gtest framework, each batch of tests being\n"
  46           "run in their own process. Supported command-line flags:\n"
  47           "\n"
  48           " Common flags:\n"
  49           "  --gtest_filter=...\n"
  50           "    Runs a subset of tests (see --gtest_help for more info).\n"
  51           "\n"
  52           "  --help\n"
  53           "    Shows this message.\n"
  54           "\n"
  55           "  --gtest_help\n"
  56           "    Shows the gtest help message.\n"
  57           "\n"
  58           "  --test-launcher-jobs=N\n"
  59           "    Sets the number of parallel test jobs to N.\n"
  60           "\n"
  61           "  --single-process-tests\n"
  62           "    Runs the tests and the launcher in the same process. Useful\n"
  63           "    for debugging a specific test in a debugger.\n"
  64           "\n"
  65           " Other flags:\n"
  66           "  --test-launcher-batch-limit=N\n"
  67           "    Sets the limit of test batch to run in a single process to N.\n"
  68           "\n"
  69           "  --test-launcher-debug-launcher\n"
  70           "    Disables autodetection of debuggers and similar tools,\n"
  71           "    making it possible to use them to debug launcher itself.\n"
  72           "\n"
  73           "  --test-launcher-retry-limit=N\n"
  74           "    Sets the limit of test retries on failures to N.\n"
  75           "\n"
  76           "  --test-launcher-summary-output=PATH\n"
  77           "    Saves a JSON machine-readable summary of the run.\n"
  78           "\n"
  79           "  --test-launcher-print-test-stdio=auto|always|never\n"
  80           "    Controls when full test output is printed.\n"
  81           "    auto means to print it when the test failed.\n"
  82           "\n"
  83           "  --test-launcher-total-shards=N\n"
  84           "    Sets the total number of shards to N.\n"
  85           "\n"
  86           "  --test-launcher-shard-index=N\n"
  87           "    Sets the shard index to run to N (from 0 to TOTAL - 1).\n");
  88   fflush(stdout);
  89 }
  90
  91 class DefaultUnitTestPlatformDelegate : public UnitTestPlatformDelegate {
  92  public:
  93   DefaultUnitTestPlatformDelegate() {
  94   }
  95
  96  private:
  97   // UnitTestPlatformDelegate:
  98   bool GetTests(std::vector<TestIdentifier>* output) override {
  99     *output = GetCompiledInTests();
 100     return true;
 101   }
 102
 103   bool CreateTemporaryFile(base::FilePath* path) override {
 104     if (!CreateNewTempDirectory(FilePath::StringType(), path))
 105       return false;
 106     *path = path->AppendASCII("test_results.xml");
 107     return true;
 108   }
 109
 110   CommandLine GetCommandLineForChildGTestProcess(
 111       const std::vector<std::string>& test_names,
 112       const base::FilePath& output_file) override {
 113     CommandLine new_cmd_line(*CommandLine::ForCurrentProcess());
 114
 115     new_cmd_line.AppendSwitchPath(switches::kTestLauncherOutput, output_file);
 116     new_cmd_line.AppendSwitchASCII(kGTestFilterFlag,
 117                                    JoinString(test_names, ":"));
 118     new_cmd_line.AppendSwitch(kSingleProcessTestsFlag);
 119
 120     return new_cmd_line;
 121   }
 122
 123   std::string GetWrapperForChildGTestProcess() override {
 124     return std::string();
 125   }
 126
 127   void RelaunchTests(TestLauncher* test_launcher,
 128                      const std::vector<std::string>& test_names,
 129                      int launch_flags) override {
 130     // Relaunch requested tests in parallel, but only use single
 131     // test per batch for more precise results (crashes, etc).
 132     for (const std::string& test_name : test_names) {
 133       std::vector<std::string> batch;
 134       batch.push_back(test_name);
 135       RunUnitTestsBatch(test_launcher, this, batch, launch_flags);
 136     }
 137   }
 138
 139   DISALLOW_COPY_AND_ASSIGN(DefaultUnitTestPlatformDelegate);
 140 };
 141
 142 bool GetSwitchValueAsInt(const std::string& switch_name, int* result) {
 143   if (!CommandLine::ForCurrentProcess()->HasSwitch(switch_name))
 144     return true;
 145
 146   std::string switch_value =
 147       CommandLine::ForCurrentProcess()->GetSwitchValueASCII(switch_name);
 148   if (!StringToInt(switch_value, result) || *result < 1) {
 149     LOG(ERROR) << "Invalid value for " << switch_name << ": " << switch_value;
 150     return false;
 151   }
 152
 153   return true;
 154 }
 155
 156 int LaunchUnitTestsInternal(const RunTestSuiteCallback& run_test_suite,
 157                             int default_jobs,
 158                             bool use_job_objects,
 159                             const Closure& gtest_init) {
 160 #if defined(OS_ANDROID)
 161   // We can't easily fork on Android, just run the test suite directly.
 162   return run_test_suite.Run();
 163 #else
 164   bool force_single_process = false;
 165   if (CommandLine::ForCurrentProcess()->HasSwitch(
 166           switches::kTestLauncherDebugLauncher)) {
 167     fprintf(stdout, "Forcing test launcher debugging mode.\n");
 168     fflush(stdout);
 169   } else {
 170     if (base::debug::BeingDebugged()) {
 171       fprintf(stdout,
 172               "Debugger detected, switching to single process mode.\n"
 173               "Pass --test-launcher-debug-launcher to debug the launcher "
 174               "itself.\n");
 175       fflush(stdout);
 176       force_single_process = true;
 177     }
 178   }
 179
 180   if (CommandLine::ForCurrentProcess()->HasSwitch(kGTestHelpFlag) ||
 181       CommandLine::ForCurrentProcess()->HasSwitch(kGTestListTestsFlag) ||
 182       CommandLine::ForCurrentProcess()->HasSwitch(kSingleProcessTestsFlag) ||
 183       force_single_process) {
 184     return run_test_suite.Run();
 185   }
 186 #endif
 187
 188   if (CommandLine::ForCurrentProcess()->HasSwitch(kHelpFlag)) {
 189     PrintUsage();
 190     return 0;
 191   }
 192
 193   base::TimeTicks start_time(base::TimeTicks::Now());
 194
 195   gtest_init.Run();
 196   TestTimeouts::Initialize();
 197
 198   int batch_limit = kDefaultTestBatchLimit;
 199   if (!GetSwitchValueAsInt(switches::kTestLauncherBatchLimit, &batch_limit))
 200     return 1;
 201
 202   fprintf(stdout,
 203           "IMPORTANT DEBUGGING NOTE: batches of tests are run inside their\n"
 204           "own process. For debugging a test inside a debugger, use the\n"
 205           "--gtest_filter=<your_test_name> flag along with\n"
 206           "--single-process-tests.\n");
 207   fflush(stdout);
 208
 209   MessageLoopForIO message_loop;
 210
 211   DefaultUnitTestPlatformDelegate platform_delegate;
 212   UnitTestLauncherDelegate delegate(
 213       &platform_delegate, batch_limit, use_job_objects);
 214   base::TestLauncher launcher(&delegate, default_jobs);
 215   bool success = launcher.Run();
 216
 217   fprintf(stdout, "Tests took %" PRId64 " seconds.\n",
 218           (base::TimeTicks::Now() - start_time).InSeconds());
 219   fflush(stdout);
 220
 221   return (success ? 0 : 1);
 222 }
 223
 224 void InitGoogleTestChar(int* argc, char** argv) {
 225   testing::InitGoogleTest(argc, argv);
 226 }
 227
 228 #if defined(OS_WIN)
 229 void InitGoogleTestWChar(int* argc, wchar_t** argv) {
 230   testing::InitGoogleTest(argc, argv);
 231 }
 232 #endif  // defined(OS_WIN)
 233
 234 // Interprets test results and reports to the test launcher. Returns true
 235 // on success.
 236 bool ProcessTestResults(
 237     TestLauncher* test_launcher,
 238     const std::vector<std::string>& test_names,
 239     const base::FilePath& output_file,
 240     const std::string& output,
 241     int exit_code,
 242     bool was_timeout,
 243     std::vector<std::string>* tests_to_relaunch) {
 244   std::vector<TestResult> test_results;
 245   bool crashed = false;
 246   bool have_test_results =
 247       ProcessGTestOutput(output_file, &test_results, &crashed);
 248
 249   bool called_any_callback = false;
 250
 251   if (have_test_results) {
 252     // TODO(phajdan.jr): Check for duplicates and mismatches between
 253     // the results we got from XML file and tests we intended to run.
 254     std::map<std::string, TestResult> results_map;
 255     for (size_t i = 0; i < test_results.size(); i++)
 256       results_map[test_results[i].full_name] = test_results[i];
 257
 258     bool had_interrupted_test = false;
 259
 260     // Results to be reported back to the test launcher.
 261     std::vector<TestResult> final_results;
 262
 263     for (size_t i = 0; i < test_names.size(); i++) {
 264       if (ContainsKey(results_map, test_names[i])) {
 265         TestResult test_result = results_map[test_names[i]];
 266         if (test_result.status == TestResult::TEST_CRASH) {
 267           had_interrupted_test = true;
 268
 269           if (was_timeout) {
 270             // Fix up the test status: we forcibly kill the child process
 271             // after the timeout, so from XML results it looks just like
 272             // a crash.
 273             test_result.status = TestResult::TEST_TIMEOUT;
 274           }
 275         } else if (test_result.status == TestResult::TEST_SUCCESS ||
 276                    test_result.status == TestResult::TEST_FAILURE) {
 277           // We run multiple tests in a batch with a timeout applied
 278           // to the entire batch. It is possible that with other tests
 279           // running quickly some tests take longer than the per-test timeout.
 280           // For consistent handling of tests independent of order and other
 281           // factors, mark them as timing out.
 282           if (test_result.elapsed_time >
 283               TestTimeouts::test_launcher_timeout()) {
 284             test_result.status = TestResult::TEST_TIMEOUT;
 285           }
 286         }
 287         test_result.output_snippet = GetTestOutputSnippet(test_result, output);
 288         final_results.push_back(test_result);
 289       } else if (had_interrupted_test) {
 290         tests_to_relaunch->push_back(test_names[i]);
 291       } else {
 292         // TODO(phajdan.jr): Explicitly pass the info that the test didn't
 293         // run for a mysterious reason.
 294         LOG(ERROR) << "no test result for " << test_names[i];
 295         TestResult test_result;
 296         test_result.full_name = test_names[i];
 297         test_result.status = TestResult::TEST_UNKNOWN;
 298         test_result.output_snippet = GetTestOutputSnippet(test_result, output);
 299         final_results.push_back(test_result);
 300       }
 301     }
 302
 303     // TODO(phajdan.jr): Handle the case where processing XML output
 304     // indicates a crash but none of the test results is marked as crashing.
 305
 306     if (final_results.empty())
 307       return false;
 308
 309     bool has_non_success_test = false;
 310     for (size_t i = 0; i < final_results.size(); i++) {
 311       if (final_results[i].status != TestResult::TEST_SUCCESS) {
 312         has_non_success_test = true;
 313         break;
 314       }
 315     }
 316
 317     if (!has_non_success_test && exit_code != 0) {
 318       // This is a bit surprising case: all tests are marked as successful,
 319       // but the exit code was not zero. This can happen e.g. under memory
 320       // tools that report leaks this way. Mark all tests as a failure on exit,
 321       // and for more precise info they'd need to be retried serially.
 322       for (size_t i = 0; i < final_results.size(); i++)
 323         final_results[i].status = TestResult::TEST_FAILURE_ON_EXIT;
 324     }
 325
 326     for (size_t i = 0; i < final_results.size(); i++) {
 327       // Fix the output snippet after possible changes to the test result.
 328       final_results[i].output_snippet =
 329           GetTestOutputSnippet(final_results[i], output);
 330       test_launcher->OnTestFinished(final_results[i]);
 331       called_any_callback = true;
 332     }
 333   } else {
 334     fprintf(stdout,
 335             "Failed to get out-of-band test success data, "
 336             "dumping full stdio below:\n%s\n",
 337             output.c_str());
 338     fflush(stdout);
 339
 340     // We do not have reliable details about test results (parsing test
 341     // stdout is known to be unreliable), apply the executable exit code
 342     // to all tests.
 343     // TODO(phajdan.jr): Be smarter about this, e.g. retry each test
 344     // individually.
 345     for (size_t i = 0; i < test_names.size(); i++) {
 346       TestResult test_result;
 347       test_result.full_name = test_names[i];
 348       test_result.status = TestResult::TEST_UNKNOWN;
 349       test_launcher->OnTestFinished(test_result);
 350       called_any_callback = true;
 351     }
 352   }
 353
 354   return called_any_callback;
 355 }
 356
 357 // TODO(phajdan.jr): Pass parameters directly with C++11 variadic templates.
 358 struct GTestCallbackState {
 359   TestLauncher* test_launcher;
 360   UnitTestPlatformDelegate* platform_delegate;
 361   std::vector<std::string> test_names;
 362   int launch_flags;
 363   FilePath output_file;
 364 };
 365
 366 void GTestCallback(
 367     const GTestCallbackState& callback_state,
 368     int exit_code,
 369     const TimeDelta& elapsed_time,
 370     bool was_timeout,
 371     const std::string& output) {
 372   std::vector<std::string> tests_to_relaunch;
 373   ProcessTestResults(callback_state.test_launcher, callback_state.test_names,
 374                      callback_state.output_file, output, exit_code, was_timeout,
 375                      &tests_to_relaunch);
 376
 377   if (!tests_to_relaunch.empty()) {
 378     callback_state.platform_delegate->RelaunchTests(
 379         callback_state.test_launcher,
 380         tests_to_relaunch,
 381         callback_state.launch_flags);
 382   }
 383
 384   // The temporary file's directory is also temporary.
 385   DeleteFile(callback_state.output_file.DirName(), true);
 386 }
 387
 388 void SerialGTestCallback(
 389     const GTestCallbackState& callback_state,
 390     const std::vector<std::string>& test_names,
 391     int exit_code,
 392     const TimeDelta& elapsed_time,
 393     bool was_timeout,
 394     const std::string& output) {
 395   std::vector<std::string> tests_to_relaunch;
 396   bool called_any_callbacks =
 397       ProcessTestResults(callback_state.test_launcher,
 398                          callback_state.test_names, callback_state.output_file,
 399                          output, exit_code, was_timeout, &tests_to_relaunch);
 400
 401   // There is only one test, there cannot be other tests to relaunch
 402   // due to a crash.
 403   DCHECK(tests_to_relaunch.empty());
 404
 405   // There is only one test, we should have called back with its result.
 406   DCHECK(called_any_callbacks);
 407
 408   // The temporary file's directory is also temporary.
 409   DeleteFile(callback_state.output_file.DirName(), true);
 410
 411   ThreadTaskRunnerHandle::Get()->PostTask(
 412       FROM_HERE, Bind(&RunUnitTestsSerially, callback_state.test_launcher,
 413                       callback_state.platform_delegate, test_names,
 414                       callback_state.launch_flags));
 415 }
 416
 417 }  // namespace
 418
 419 int LaunchUnitTests(int argc,
 420                     char** argv,
 421                     const RunTestSuiteCallback& run_test_suite) {
 422   CommandLine::Init(argc, argv);
 423   return LaunchUnitTestsInternal(run_test_suite, SysInfo::NumberOfProcessors(),
 424                                  true, Bind(&InitGoogleTestChar, &argc, argv));
 425 }
 426
 427 int LaunchUnitTestsSerially(int argc,
 428                             char** argv,
 429                             const RunTestSuiteCallback& run_test_suite) {
 430   CommandLine::Init(argc, argv);
 431   return LaunchUnitTestsInternal(run_test_suite, 1, true,
 432                                  Bind(&InitGoogleTestChar, &argc, argv));
 433 }
 434
 435 #if defined(OS_WIN)
 436 int LaunchUnitTests(int argc,
 437                     wchar_t** argv,
 438                     bool use_job_objects,
 439                     const RunTestSuiteCallback& run_test_suite) {
 440   // Windows CommandLine::Init ignores argv anyway.
 441   CommandLine::Init(argc, NULL);
 442   return LaunchUnitTestsInternal(run_test_suite, SysInfo::NumberOfProcessors(),
 443                                  use_job_objects,
 444                                  Bind(&InitGoogleTestWChar, &argc, argv));
 445 }
 446 #endif  // defined(OS_WIN)
 447
 448 void RunUnitTestsSerially(
 449     TestLauncher* test_launcher,
 450     UnitTestPlatformDelegate* platform_delegate,
 451     const std::vector<std::string>& test_names,
 452     int launch_flags) {
 453   if (test_names.empty())
 454     return;
 455
 456   std::vector<std::string> new_test_names(test_names);
 457   std::string test_name(new_test_names.back());
 458   new_test_names.pop_back();
 459
 460   // Create a dedicated temporary directory to store the xml result data
 461   // per run to ensure clean state and make it possible to launch multiple
 462   // processes in parallel.
 463   base::FilePath output_file;
 464   CHECK(platform_delegate->CreateTemporaryFile(&output_file));
 465
 466   std::vector<std::string> current_test_names;
 467   current_test_names.push_back(test_name);
 468   CommandLine cmd_line(platform_delegate->GetCommandLineForChildGTestProcess(
 469       current_test_names, output_file));
 470
 471   GTestCallbackState callback_state;
 472   callback_state.test_launcher = test_launcher;
 473   callback_state.platform_delegate = platform_delegate;
 474   callback_state.test_names = current_test_names;
 475   callback_state.launch_flags = launch_flags;
 476   callback_state.output_file = output_file;
 477
 478   test_launcher->LaunchChildGTestProcess(
 479       cmd_line,
 480       platform_delegate->GetWrapperForChildGTestProcess(),
 481       TestTimeouts::test_launcher_timeout(),
 482       launch_flags,
 483       Bind(&SerialGTestCallback, callback_state, new_test_names));
 484 }
 485
 486 void RunUnitTestsBatch(
 487     TestLauncher* test_launcher,
 488     UnitTestPlatformDelegate* platform_delegate,
 489     const std::vector<std::string>& test_names,
 490     int launch_flags) {
 491   if (test_names.empty())
 492     return;
 493
 494   // Create a dedicated temporary directory to store the xml result data
 495   // per run to ensure clean state and make it possible to launch multiple
 496   // processes in parallel.
 497   base::FilePath output_file;
 498   CHECK(platform_delegate->CreateTemporaryFile(&output_file));
 499
 500   CommandLine cmd_line(platform_delegate->GetCommandLineForChildGTestProcess(
 501       test_names, output_file));
 502
 503   // Adjust the timeout depending on how many tests we're running
 504   // (note that e.g. the last batch of tests will be smaller).
 505   // TODO(phajdan.jr): Consider an adaptive timeout, which can change
 506   // depending on how many tests ran and how many remain.
 507   // Note: do NOT parse child's stdout to do that, it's known to be
 508   // unreliable (e.g. buffering issues can mix up the output).
 509   base::TimeDelta timeout =
 510       test_names.size() * TestTimeouts::test_launcher_timeout();
 511
 512   GTestCallbackState callback_state;
 513   callback_state.test_launcher = test_launcher;
 514   callback_state.platform_delegate = platform_delegate;
 515   callback_state.test_names = test_names;
 516   callback_state.launch_flags = launch_flags;
 517   callback_state.output_file = output_file;
 518
 519   test_launcher->LaunchChildGTestProcess(
 520       cmd_line,
 521       platform_delegate->GetWrapperForChildGTestProcess(),
 522       timeout,
 523       launch_flags,
 524       Bind(&GTestCallback, callback_state));
 525 }
 526
 527 UnitTestLauncherDelegate::UnitTestLauncherDelegate(
 528     UnitTestPlatformDelegate* platform_delegate,
 529     size_t batch_limit,
 530     bool use_job_objects)
 531     : platform_delegate_(platform_delegate),
 532       batch_limit_(batch_limit),
 533       use_job_objects_(use_job_objects) {
 534 }
 535
 536 UnitTestLauncherDelegate::~UnitTestLauncherDelegate() {
 537   DCHECK(thread_checker_.CalledOnValidThread());
 538 }
 539
 540 bool UnitTestLauncherDelegate::GetTests(std::vector<TestIdentifier>* output) {
 541   DCHECK(thread_checker_.CalledOnValidThread());
 542   return platform_delegate_->GetTests(output);
 543 }
 544
 545 bool UnitTestLauncherDelegate::ShouldRunTest(const std::string& test_case_name,
 546                                              const std::string& test_name) {
 547   DCHECK(thread_checker_.CalledOnValidThread());
 548
 549   // There is no additional logic to disable specific tests.
 550   return true;
 551 }
 552
 553 size_t UnitTestLauncherDelegate::RunTests(
 554     TestLauncher* test_launcher,
 555     const std::vector<std::string>& test_names) {
 556   DCHECK(thread_checker_.CalledOnValidThread());
 557
 558   int launch_flags = use_job_objects_ ? TestLauncher::USE_JOB_OBJECTS : 0;
 559
 560   std::vector<std::string> batch;
 561   for (size_t i = 0; i < test_names.size(); i++) {
 562     batch.push_back(test_names[i]);
 563
 564     // Use 0 to indicate unlimited batch size.
 565     if (batch.size() >= batch_limit_ && batch_limit_ != 0) {
 566       RunUnitTestsBatch(test_launcher, platform_delegate_, batch, launch_flags);
 567       batch.clear();
 568     }
 569   }
 570
 571   RunUnitTestsBatch(test_launcher, platform_delegate_, batch, launch_flags);
 572
 573   return test_names.size();
 574 }
 575
 576 size_t UnitTestLauncherDelegate::RetryTests(
 577     TestLauncher* test_launcher,
 578     const std::vector<std::string>& test_names) {
 579   ThreadTaskRunnerHandle::Get()->PostTask(
 580       FROM_HERE,
 581       Bind(&RunUnitTestsSerially, test_launcher, platform_delegate_, test_names,
 582            use_job_objects_ ? TestLauncher::USE_JOB_OBJECTS : 0));
 583   return test_names.size();
 584 }
 585
 586 }  // namespace base