base/test/launcher/unit_test_launcher.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "base/test/launcher/unit_test_launcher.h"
   6
   7 #include "base/bind.h"
   8 #include "base/callback_helpers.h"
   9 #include "base/command_line.h"
  10 #include "base/compiler_specific.h"
  11 #include "base/debug/debugger.h"
  12 #include "base/file_util.h"
  13 #include "base/files/scoped_temp_dir.h"
  14 #include "base/format_macros.h"
  15 #include "base/message_loop/message_loop.h"
  16 #include "base/stl_util.h"
  17 #include "base/strings/string_number_conversions.h"
  18 #include "base/strings/string_util.h"
  19 #include "base/sys_info.h"
  20 #include "base/test/gtest_xml_util.h"
  21 #include "base/test/launcher/test_launcher.h"
  22 #include "base/test/test_switches.h"
  23 #include "base/test/test_timeouts.h"
  24 #include "base/third_party/dynamic_annotations/dynamic_annotations.h"
  25 #include "base/threading/thread_checker.h"
  26 #include "testing/gtest/include/gtest/gtest.h"
  27
  28 namespace base {
  29
  30 namespace {
  31
  32 // This constant controls how many tests are run in a single batch by default.
  33 const size_t kDefaultTestBatchLimit = 10;
  34
  35 const char kHelpFlag[] = "help";
  36
  37 // Flag to run all tests in a single process.
  38 const char kSingleProcessTestsFlag[] = "single-process-tests";
  39
  40 void PrintUsage() {
  41   fprintf(stdout,
  42           "Runs tests using the gtest framework, each batch of tests being\n"
  43           "run in their own process. Supported command-line flags:\n"
  44           "\n"
  45           " Common flags:\n"
  46           "  --gtest_filter=...\n"
  47           "    Runs a subset of tests (see --gtest_help for more info).\n"
  48           "\n"
  49           "  --help\n"
  50           "    Shows this message.\n"
  51           "\n"
  52           "  --gtest_help\n"
  53           "    Shows the gtest help message.\n"
  54           "\n"
  55           "  --test-launcher-jobs=N\n"
  56           "    Sets the number of parallel test jobs to N.\n"
  57           "\n"
  58           "  --single-process-tests\n"
  59           "    Runs the tests and the launcher in the same process. Useful\n"
  60           "    for debugging a specific test in a debugger.\n"
  61           "\n"
  62           " Other flags:\n"
  63           "  --test-launcher-batch-limit=N\n"
  64           "    Sets the limit of test batch to run in a single process to N.\n"
  65           "\n"
  66           "  --test-launcher-debug-launcher\n"
  67           "    Disables autodetection of debuggers and similar tools,\n"
  68           "    making it possible to use them to debug launcher itself.\n"
  69           "\n"
  70           "  --test-launcher-retry-limit=N\n"
  71           "    Sets the limit of test retries on failures to N.\n"
  72           "\n"
  73           "  --test-launcher-summary-output=PATH\n"
  74           "    Saves a JSON machine-readable summary of the run.\n"
  75           "\n"
  76           "  --test-launcher-print-test-stdio=auto|always|never\n"
  77           "    Controls when full test output is printed.\n"
  78           "    auto means to print it when the test failed.\n"
  79           "\n"
  80           "  --test-launcher-total-shards=N\n"
  81           "    Sets the total number of shards to N.\n"
  82           "\n"
  83           "  --test-launcher-shard-index=N\n"
  84           "    Sets the shard index to run to N (from 0 to TOTAL - 1).\n");
  85   fflush(stdout);
  86 }
  87
  88 // Returns command line for child GTest process based on the command line
  89 // of current process. |test_names| is a vector of test full names
  90 // (e.g. "A.B"), |output_file| is path to the GTest XML output file.
  91 CommandLine GetCommandLineForChildGTestProcess(
  92     const std::vector<std::string>& test_names,
  93     const base::FilePath& output_file) {
  94   CommandLine new_cmd_line(*CommandLine::ForCurrentProcess());
  95
  96   new_cmd_line.AppendSwitchPath(switches::kTestLauncherOutput, output_file);
  97   new_cmd_line.AppendSwitchASCII(kGTestFilterFlag, JoinString(test_names, ":"));
  98   new_cmd_line.AppendSwitch(kSingleProcessTestsFlag);
  99
 100   return new_cmd_line;
 101 }
 102
 103 class UnitTestLauncherDelegate : public TestLauncherDelegate {
 104  public:
 105   explicit UnitTestLauncherDelegate(size_t batch_limit, bool use_job_objects)
 106       : batch_limit_(batch_limit),
 107         use_job_objects_(use_job_objects) {
 108   }
 109
 110   virtual ~UnitTestLauncherDelegate() {
 111     DCHECK(thread_checker_.CalledOnValidThread());
 112   }
 113
 114  private:
 115   struct GTestCallbackState {
 116     TestLauncher* test_launcher;
 117     std::vector<std::string> test_names;
 118     FilePath output_file;
 119   };
 120
 121   virtual bool ShouldRunTest(const testing::TestCase* test_case,
 122                              const testing::TestInfo* test_info) OVERRIDE {
 123     DCHECK(thread_checker_.CalledOnValidThread());
 124
 125     // There is no additional logic to disable specific tests.
 126     return true;
 127   }
 128
 129   virtual size_t RunTests(TestLauncher* test_launcher,
 130                           const std::vector<std::string>& test_names) OVERRIDE {
 131     DCHECK(thread_checker_.CalledOnValidThread());
 132
 133     std::vector<std::string> batch;
 134     for (size_t i = 0; i < test_names.size(); i++) {
 135       batch.push_back(test_names[i]);
 136
 137       if (batch.size() >= batch_limit_) {
 138         RunBatch(test_launcher, batch);
 139         batch.clear();
 140       }
 141     }
 142
 143     RunBatch(test_launcher, batch);
 144
 145     return test_names.size();
 146   }
 147
 148   virtual size_t RetryTests(
 149       TestLauncher* test_launcher,
 150       const std::vector<std::string>& test_names) OVERRIDE {
 151     MessageLoop::current()->PostTask(
 152         FROM_HERE,
 153         Bind(&UnitTestLauncherDelegate::RunSerially,
 154              Unretained(this),
 155              test_launcher,
 156              test_names));
 157     return test_names.size();
 158   }
 159
 160   void RunSerially(TestLauncher* test_launcher,
 161                    const std::vector<std::string>& test_names) {
 162     if (test_names.empty())
 163       return;
 164
 165     std::vector<std::string> new_test_names(test_names);
 166     std::string test_name(new_test_names.back());
 167     new_test_names.pop_back();
 168
 169     // Create a dedicated temporary directory to store the xml result data
 170     // per run to ensure clean state and make it possible to launch multiple
 171     // processes in parallel.
 172     base::FilePath output_file;
 173     CHECK(CreateNewTempDirectory(FilePath::StringType(), &output_file));
 174     output_file = output_file.AppendASCII("test_results.xml");
 175
 176     std::vector<std::string> current_test_names;
 177     current_test_names.push_back(test_name);
 178     CommandLine cmd_line(
 179         GetCommandLineForChildGTestProcess(current_test_names, output_file));
 180
 181     GTestCallbackState callback_state;
 182     callback_state.test_launcher = test_launcher;
 183     callback_state.test_names = current_test_names;
 184     callback_state.output_file = output_file;
 185
 186     test_launcher->LaunchChildGTestProcess(
 187         cmd_line,
 188         std::string(),
 189         TestTimeouts::test_launcher_timeout(),
 190         use_job_objects_,
 191         Bind(&UnitTestLauncherDelegate::SerialGTestCallback,
 192              Unretained(this),
 193              callback_state,
 194              new_test_names));
 195   }
 196
 197   void RunBatch(TestLauncher* test_launcher,
 198                 const std::vector<std::string>& test_names) {
 199     DCHECK(thread_checker_.CalledOnValidThread());
 200
 201     if (test_names.empty())
 202       return;
 203
 204     // Create a dedicated temporary directory to store the xml result data
 205     // per run to ensure clean state and make it possible to launch multiple
 206     // processes in parallel.
 207     base::FilePath output_file;
 208     CHECK(CreateNewTempDirectory(FilePath::StringType(), &output_file));
 209     output_file = output_file.AppendASCII("test_results.xml");
 210
 211     CommandLine cmd_line(
 212         GetCommandLineForChildGTestProcess(test_names, output_file));
 213
 214     // Adjust the timeout depending on how many tests we're running
 215     // (note that e.g. the last batch of tests will be smaller).
 216     // TODO(phajdan.jr): Consider an adaptive timeout, which can change
 217     // depending on how many tests ran and how many remain.
 218     // Note: do NOT parse child's stdout to do that, it's known to be
 219     // unreliable (e.g. buffering issues can mix up the output).
 220     base::TimeDelta timeout =
 221         test_names.size() * TestTimeouts::test_launcher_timeout();
 222
 223     GTestCallbackState callback_state;
 224     callback_state.test_launcher = test_launcher;
 225     callback_state.test_names = test_names;
 226     callback_state.output_file = output_file;
 227
 228     test_launcher->LaunchChildGTestProcess(
 229         cmd_line,
 230         std::string(),
 231         timeout,
 232         use_job_objects_,
 233         Bind(&UnitTestLauncherDelegate::GTestCallback,
 234              Unretained(this),
 235              callback_state));
 236   }
 237
 238   void GTestCallback(const GTestCallbackState& callback_state,
 239                      int exit_code,
 240                      const TimeDelta& elapsed_time,
 241                      bool was_timeout,
 242                      const std::string& output) {
 243     DCHECK(thread_checker_.CalledOnValidThread());
 244     std::vector<std::string> tests_to_relaunch;
 245     ProcessTestResults(callback_state.test_launcher,
 246                        callback_state.test_names,
 247                        callback_state.output_file,
 248                        output,
 249                        exit_code,
 250                        was_timeout,
 251                        &tests_to_relaunch);
 252
 253     // Relaunch requested tests in parallel, but only use single
 254     // test per batch for more precise results (crashes, test passes
 255     // but non-zero exit codes etc).
 256     for (size_t i = 0; i < tests_to_relaunch.size(); i++) {
 257       std::vector<std::string> batch;
 258       batch.push_back(tests_to_relaunch[i]);
 259       RunBatch(callback_state.test_launcher, batch);
 260     }
 261
 262     // The temporary file's directory is also temporary.
 263     DeleteFile(callback_state.output_file.DirName(), true);
 264   }
 265
 266   void SerialGTestCallback(const GTestCallbackState& callback_state,
 267                            const std::vector<std::string>& test_names,
 268                            int exit_code,
 269                            const TimeDelta& elapsed_time,
 270                            bool was_timeout,
 271                            const std::string& output) {
 272     DCHECK(thread_checker_.CalledOnValidThread());
 273     std::vector<std::string> tests_to_relaunch;
 274     bool called_any_callbacks =
 275         ProcessTestResults(callback_state.test_launcher,
 276                            callback_state.test_names,
 277                            callback_state.output_file,
 278                            output,
 279                            exit_code,
 280                            was_timeout,
 281                            &tests_to_relaunch);
 282
 283     // There is only one test, there cannot be other tests to relaunch
 284     // due to a crash.
 285     DCHECK(tests_to_relaunch.empty());
 286
 287     // There is only one test, we should have called back with its result.
 288     DCHECK(called_any_callbacks);
 289
 290     // The temporary file's directory is also temporary.
 291     DeleteFile(callback_state.output_file.DirName(), true);
 292
 293     MessageLoop::current()->PostTask(
 294         FROM_HERE,
 295         Bind(&UnitTestLauncherDelegate::RunSerially,
 296              Unretained(this),
 297              callback_state.test_launcher,
 298              test_names));
 299   }
 300
 301   static bool ProcessTestResults(
 302       TestLauncher* test_launcher,
 303       const std::vector<std::string>& test_names,
 304       const base::FilePath& output_file,
 305       const std::string& output,
 306       int exit_code,
 307       bool was_timeout,
 308       std::vector<std::string>* tests_to_relaunch) {
 309     std::vector<TestResult> test_results;
 310     bool crashed = false;
 311     bool have_test_results =
 312         ProcessGTestOutput(output_file, &test_results, &crashed);
 313
 314     bool called_any_callback = false;
 315
 316     if (have_test_results) {
 317       // TODO(phajdan.jr): Check for duplicates and mismatches between
 318       // the results we got from XML file and tests we intended to run.
 319       std::map<std::string, TestResult> results_map;
 320       for (size_t i = 0; i < test_results.size(); i++)
 321         results_map[test_results[i].full_name] = test_results[i];
 322
 323       bool had_interrupted_test = false;
 324
 325       // Results to be reported back to the test launcher.
 326       std::vector<TestResult> final_results;
 327
 328       for (size_t i = 0; i < test_names.size(); i++) {
 329         if (ContainsKey(results_map, test_names[i])) {
 330           TestResult test_result = results_map[test_names[i]];
 331           if (test_result.status == TestResult::TEST_CRASH) {
 332             had_interrupted_test = true;
 333
 334             if (was_timeout) {
 335               // Fix up the test status: we forcibly kill the child process
 336               // after the timeout, so from XML results it looks just like
 337               // a crash.
 338               test_result.status = TestResult::TEST_TIMEOUT;
 339             }
 340           } else if (test_result.status == TestResult::TEST_SUCCESS ||
 341                      test_result.status == TestResult::TEST_FAILURE) {
 342             // We run multiple tests in a batch with a timeout applied
 343             // to the entire batch. It is possible that with other tests
 344             // running quickly some tests take longer than the per-test timeout.
 345             // For consistent handling of tests independent of order and other
 346             // factors, mark them as timing out.
 347             if (test_result.elapsed_time >
 348                 TestTimeouts::test_launcher_timeout()) {
 349               test_result.status = TestResult::TEST_TIMEOUT;
 350             }
 351           }
 352           test_result.output_snippet =
 353               GetTestOutputSnippet(test_result, output);
 354           final_results.push_back(test_result);
 355         } else if (had_interrupted_test) {
 356           tests_to_relaunch->push_back(test_names[i]);
 357         } else {
 358           // TODO(phajdan.jr): Explicitly pass the info that the test didn't
 359           // run for a mysterious reason.
 360           LOG(ERROR) << "no test result for " << test_names[i];
 361           TestResult test_result;
 362           test_result.full_name = test_names[i];
 363           test_result.status = TestResult::TEST_UNKNOWN;
 364           test_result.output_snippet =
 365               GetTestOutputSnippet(test_result, output);
 366           final_results.push_back(test_result);
 367         }
 368       }
 369
 370       // TODO(phajdan.jr): Handle the case where processing XML output
 371       // indicates a crash but none of the test results is marked as crashing.
 372
 373       if (final_results.empty())
 374         return false;
 375
 376       bool has_non_success_test = false;
 377       for (size_t i = 0; i < final_results.size(); i++) {
 378         if (final_results[i].status != TestResult::TEST_SUCCESS) {
 379           has_non_success_test = true;
 380           break;
 381         }
 382       }
 383
 384       if (!has_non_success_test && exit_code != 0) {
 385         // This is a bit surprising case: all tests are marked as successful,
 386         // but the exit code was not zero. This can happen e.g. under memory
 387         // tools that report leaks this way.
 388
 389         if (final_results.size() == 1) {
 390           // Easy case. One test only so we know the non-zero exit code
 391           // was caused by that one test.
 392           final_results[0].status = TestResult::TEST_FAILURE_ON_EXIT;
 393         } else {
 394           // Harder case. Discard the results and request relaunching all
 395           // tests without batching. This will trigger above branch on
 396           // relaunch leading to more precise results.
 397           LOG(WARNING) << "Not sure which test caused non-zero exit code, "
 398                        << "relaunching all of them without batching.";
 399
 400           for (size_t i = 0; i < final_results.size(); i++)
 401             tests_to_relaunch->push_back(final_results[i].full_name);
 402
 403           return false;
 404         }
 405       }
 406
 407       for (size_t i = 0; i < final_results.size(); i++) {
 408         // Fix the output snippet after possible changes to the test result.
 409         final_results[i].output_snippet =
 410             GetTestOutputSnippet(final_results[i], output);
 411         test_launcher->OnTestFinished(final_results[i]);
 412         called_any_callback = true;
 413       }
 414     } else {
 415       fprintf(stdout,
 416               "Failed to get out-of-band test success data, "
 417               "dumping full stdio below:\n%s\n",
 418               output.c_str());
 419       fflush(stdout);
 420
 421       // We do not have reliable details about test results (parsing test
 422       // stdout is known to be unreliable), apply the executable exit code
 423       // to all tests.
 424       // TODO(phajdan.jr): Be smarter about this, e.g. retry each test
 425       // individually.
 426       for (size_t i = 0; i < test_names.size(); i++) {
 427         TestResult test_result;
 428         test_result.full_name = test_names[i];
 429         test_result.status = TestResult::TEST_UNKNOWN;
 430         test_launcher->OnTestFinished(test_result);
 431         called_any_callback = true;
 432       }
 433     }
 434
 435     return called_any_callback;
 436   }
 437
 438   ThreadChecker thread_checker_;
 439
 440   // Maximum number of tests to run in a single batch.
 441   size_t batch_limit_;
 442
 443   // Determines whether we use job objects on Windows.
 444   bool use_job_objects_;
 445 };
 446
 447 bool GetSwitchValueAsInt(const std::string& switch_name, int* result) {
 448   if (!CommandLine::ForCurrentProcess()->HasSwitch(switch_name))
 449     return true;
 450
 451   std::string switch_value =
 452       CommandLine::ForCurrentProcess()->GetSwitchValueASCII(switch_name);
 453   if (!StringToInt(switch_value, result) || *result < 1) {
 454     LOG(ERROR) << "Invalid value for " << switch_name << ": " << switch_value;
 455     return false;
 456   }
 457
 458   return true;
 459 }
 460
 461 int LaunchUnitTestsInternal(const RunTestSuiteCallback& run_test_suite,
 462                             int default_jobs,
 463                             bool use_job_objects,
 464                             const Closure& gtest_init) {
 465 #if defined(OS_ANDROID)
 466   // We can't easily fork on Android, just run the test suite directly.
 467   return run_test_suite.Run();
 468 #else
 469   bool force_single_process = false;
 470   if (CommandLine::ForCurrentProcess()->HasSwitch(
 471           switches::kTestLauncherDebugLauncher)) {
 472     fprintf(stdout, "Forcing test launcher debugging mode.\n");
 473     fflush(stdout);
 474   } else {
 475     if (base::debug::BeingDebugged()) {
 476       fprintf(stdout,
 477               "Debugger detected, switching to single process mode.\n"
 478               "Pass --test-launcher-debug-launcher to debug the launcher "
 479               "itself.\n");
 480       fflush(stdout);
 481       force_single_process = true;
 482     }
 483
 484     if (RunningOnValgrind()) {
 485       fprintf(stdout,
 486               "Valgrind detected, switching to single process mode.\n"
 487               "Pass --test-launcher-debug-launcher to valgrind the launcher "
 488               "itself.\n");
 489       fflush(stdout);
 490       force_single_process = true;
 491     }
 492   }
 493
 494   if (CommandLine::ForCurrentProcess()->HasSwitch(kGTestHelpFlag) ||
 495       CommandLine::ForCurrentProcess()->HasSwitch(kGTestListTestsFlag) ||
 496       CommandLine::ForCurrentProcess()->HasSwitch(kSingleProcessTestsFlag) ||
 497       force_single_process) {
 498     return run_test_suite.Run();
 499   }
 500 #endif
 501
 502   if (CommandLine::ForCurrentProcess()->HasSwitch(kHelpFlag)) {
 503     PrintUsage();
 504     return 0;
 505   }
 506
 507   base::TimeTicks start_time(base::TimeTicks::Now());
 508
 509   gtest_init.Run();
 510   TestTimeouts::Initialize();
 511
 512   int batch_limit = kDefaultTestBatchLimit;
 513   if (!GetSwitchValueAsInt(switches::kTestLauncherBatchLimit, &batch_limit))
 514     return 1;
 515
 516   fprintf(stdout,
 517           "IMPORTANT DEBUGGING NOTE: batches of tests are run inside their\n"
 518           "own process. For debugging a test inside a debugger, use the\n"
 519           "--gtest_filter=<your_test_name> flag along with\n"
 520           "--single-process-tests.\n");
 521   fflush(stdout);
 522
 523   MessageLoopForIO message_loop;
 524
 525   UnitTestLauncherDelegate delegate(batch_limit, use_job_objects);
 526   base::TestLauncher launcher(&delegate, default_jobs);
 527   bool success = launcher.Run();
 528
 529   fprintf(stdout,
 530           "Tests took %" PRId64 " seconds.\n",
 531           (base::TimeTicks::Now() - start_time).InSeconds());
 532   fflush(stdout);
 533
 534   return (success ? 0 : 1);
 535 }
 536
 537 void InitGoogleTestChar(int* argc, char** argv) {
 538   testing::InitGoogleTest(argc, argv);
 539 }
 540
 541 #if defined(OS_WIN)
 542 void InitGoogleTestWChar(int* argc, wchar_t** argv) {
 543   testing::InitGoogleTest(argc, argv);
 544 }
 545 #endif  // defined(OS_WIN)
 546
 547 }  // namespace
 548
 549 int LaunchUnitTests(int argc,
 550                     char** argv,
 551                     const RunTestSuiteCallback& run_test_suite) {
 552   CommandLine::Init(argc, argv);
 553   return LaunchUnitTestsInternal(
 554       run_test_suite,
 555       SysInfo::NumberOfProcessors(),
 556       true,
 557       Bind(&InitGoogleTestChar, &argc, argv));
 558 }
 559
 560 int LaunchUnitTestsSerially(int argc,
 561                             char** argv,
 562                             const RunTestSuiteCallback& run_test_suite) {
 563   CommandLine::Init(argc, argv);
 564   return LaunchUnitTestsInternal(
 565       run_test_suite,
 566       1,
 567       true,
 568       Bind(&InitGoogleTestChar, &argc, argv));
 569 }
 570
 571 #if defined(OS_WIN)
 572 int LaunchUnitTests(int argc,
 573                     wchar_t** argv,
 574                     bool use_job_objects,
 575                     const RunTestSuiteCallback& run_test_suite) {
 576   // Windows CommandLine::Init ignores argv anyway.
 577   CommandLine::Init(argc, NULL);
 578   return LaunchUnitTestsInternal(
 579       run_test_suite,
 580       SysInfo::NumberOfProcessors(),
 581       use_job_objects,
 582       Bind(&InitGoogleTestWChar, &argc, argv));
 583 }
 584 #endif  // defined(OS_WIN)
 585
 586 }  // namespace base