base/test/launcher/test_launcher.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "base/test/launcher/test_launcher.h"
   6
   7 #if defined(OS_POSIX)
   8 #include <fcntl.h>
   9 #endif
  10
  11 #include "base/at_exit.h"
  12 #include "base/bind.h"
  13 #include "base/command_line.h"
  14 #include "base/environment.h"
  15 #include "base/files/file_path.h"
  16 #include "base/files/file_util.h"
  17 #include "base/files/scoped_file.h"
  18 #include "base/format_macros.h"
  19 #include "base/hash.h"
  20 #include "base/lazy_instance.h"
  21 #include "base/logging.h"
  22 #include "base/memory/scoped_ptr.h"
  23 #include "base/message_loop/message_loop.h"
  24 #include "base/process/kill.h"
  25 #include "base/process/launch.h"
  26 #include "base/strings/string_number_conversions.h"
  27 #include "base/strings/string_split.h"
  28 #include "base/strings/string_util.h"
  29 #include "base/strings/stringize_macros.h"
  30 #include "base/strings/stringprintf.h"
  31 #include "base/strings/utf_string_conversions.h"
  32 #include "base/test/gtest_util.h"
  33 #include "base/test/launcher/test_results_tracker.h"
  34 #include "base/test/sequenced_worker_pool_owner.h"
  35 #include "base/test/test_switches.h"
  36 #include "base/test/test_timeouts.h"
  37 #include "base/threading/thread_checker.h"
  38 #include "base/time/time.h"
  39 #include "testing/gtest/include/gtest/gtest.h"
  40
  41 #if defined(OS_MACOSX)
  42 #include "base/mac/scoped_nsautorelease_pool.h"
  43 #endif
  44
  45 #if defined(OS_WIN)
  46 #include "base/win/windows_version.h"
  47 #endif
  48
  49 namespace base {
  50
  51 // See https://groups.google.com/a/chromium.org/d/msg/chromium-dev/nkdTP7sstSc/uT3FaE_sgkAJ .
  52 using ::operator<<;
  53
  54 // The environment variable name for the total number of test shards.
  55 const char kTestTotalShards[] = "GTEST_TOTAL_SHARDS";
  56 // The environment variable name for the test shard index.
  57 const char kTestShardIndex[] = "GTEST_SHARD_INDEX";
  58
  59 namespace {
  60
  61 // Global tag for test runs where the results are incomplete or unreliable
  62 // for any reason, e.g. early exit because of too many broken tests.
  63 const char kUnreliableResultsTag[] = "UNRELIABLE_RESULTS";
  64
  65 // Maximum time of no output after which we print list of processes still
  66 // running. This deliberately doesn't use TestTimeouts (which is otherwise
  67 // a recommended solution), because they can be increased. This would defeat
  68 // the purpose of this timeout, which is 1) to avoid buildbot "no output for
  69 // X seconds" timeout killing the process 2) help communicate status of
  70 // the test launcher to people looking at the output (no output for a long
  71 // time is mysterious and gives no info about what is happening) 3) help
  72 // debugging in case the process hangs anyway.
  73 const int kOutputTimeoutSeconds = 15;
  74
  75 // Limit of output snippet lines when printing to stdout.
  76 // Avoids flooding the logs with amount of output that gums up
  77 // the infrastructure.
  78 const size_t kOutputSnippetLinesLimit = 5000;
  79
  80 // Set of live launch test processes with corresponding lock (it is allowed
  81 // for callers to launch processes on different threads).
  82 LazyInstance<std::map<ProcessHandle, CommandLine> > g_live_processes
  83     = LAZY_INSTANCE_INITIALIZER;
  84 LazyInstance<Lock> g_live_processes_lock = LAZY_INSTANCE_INITIALIZER;
  85
  86 #if defined(OS_POSIX)
  87 // Self-pipe that makes it possible to do complex shutdown handling
  88 // outside of the signal handler.
  89 int g_shutdown_pipe[2] = { -1, -1 };
  90
  91 void ShutdownPipeSignalHandler(int signal) {
  92   HANDLE_EINTR(write(g_shutdown_pipe[1], "q", 1));
  93 }
  94
  95 void KillSpawnedTestProcesses() {
  96   // Keep the lock until exiting the process to prevent further processes
  97   // from being spawned.
  98   AutoLock lock(g_live_processes_lock.Get());
  99
 100   fprintf(stdout,
 101           "Sending SIGTERM to %" PRIuS " child processes... ",
 102           g_live_processes.Get().size());
 103   fflush(stdout);
 104
 105   for (std::map<ProcessHandle, CommandLine>::iterator i =
 106            g_live_processes.Get().begin();
 107        i != g_live_processes.Get().end();
 108        ++i) {
 109     // Send the signal to entire process group.
 110     kill((-1) * (i->first), SIGTERM);
 111   }
 112
 113   fprintf(stdout,
 114           "done.\nGiving processes a chance to terminate cleanly... ");
 115   fflush(stdout);
 116
 117   PlatformThread::Sleep(TimeDelta::FromMilliseconds(500));
 118
 119   fprintf(stdout, "done.\n");
 120   fflush(stdout);
 121
 122   fprintf(stdout,
 123           "Sending SIGKILL to %" PRIuS " child processes... ",
 124           g_live_processes.Get().size());
 125   fflush(stdout);
 126
 127   for (std::map<ProcessHandle, CommandLine>::iterator i =
 128            g_live_processes.Get().begin();
 129        i != g_live_processes.Get().end();
 130        ++i) {
 131     // Send the signal to entire process group.
 132     kill((-1) * (i->first), SIGKILL);
 133   }
 134
 135   fprintf(stdout, "done.\n");
 136   fflush(stdout);
 137 }
 138
 139 // I/O watcher for the reading end of the self-pipe above.
 140 // Terminates any launched child processes and exits the process.
 141 class SignalFDWatcher : public MessageLoopForIO::Watcher {
 142  public:
 143   SignalFDWatcher() {
 144   }
 145
 146   void OnFileCanReadWithoutBlocking(int fd) override {
 147     fprintf(stdout, "\nCaught signal. Killing spawned test processes...\n");
 148     fflush(stdout);
 149
 150     KillSpawnedTestProcesses();
 151
 152     // The signal would normally kill the process, so exit now.
 153     exit(1);
 154   }
 155
 156   void OnFileCanWriteWithoutBlocking(int fd) override { NOTREACHED(); }
 157
 158  private:
 159   DISALLOW_COPY_AND_ASSIGN(SignalFDWatcher);
 160 };
 161 #endif  // defined(OS_POSIX)
 162
 163 // Parses the environment variable var as an Int32.  If it is unset, returns
 164 // true.  If it is set, unsets it then converts it to Int32 before
 165 // returning it in |result|.  Returns true on success.
 166 bool TakeInt32FromEnvironment(const char* const var, int32* result) {
 167   scoped_ptr<Environment> env(Environment::Create());
 168   std::string str_val;
 169
 170   if (!env->GetVar(var, &str_val))
 171     return true;
 172
 173   if (!env->UnSetVar(var)) {
 174     LOG(ERROR) << "Invalid environment: we could not unset " << var << ".\n";
 175     return false;
 176   }
 177
 178   if (!StringToInt(str_val, result)) {
 179     LOG(ERROR) << "Invalid environment: " << var << " is not an integer.\n";
 180     return false;
 181   }
 182
 183   return true;
 184 }
 185
 186 // Unsets the environment variable |name| and returns true on success.
 187 // Also returns true if the variable just doesn't exist.
 188 bool UnsetEnvironmentVariableIfExists(const std::string& name) {
 189   scoped_ptr<Environment> env(Environment::Create());
 190   std::string str_val;
 191
 192   if (!env->GetVar(name.c_str(), &str_val))
 193     return true;
 194
 195   return env->UnSetVar(name.c_str());
 196 }
 197
 198 // Returns true if bot mode has been requested, i.e. defaults optimized
 199 // for continuous integration bots. This way developers don't have to remember
 200 // special command-line flags.
 201 bool BotModeEnabled() {
 202   scoped_ptr<Environment> env(Environment::Create());
 203   return CommandLine::ForCurrentProcess()->HasSwitch(
 204       switches::kTestLauncherBotMode) ||
 205       env->HasVar("CHROMIUM_TEST_LAUNCHER_BOT_MODE");
 206 }
 207
 208 // Returns command line command line after gtest-specific processing
 209 // and applying |wrapper|.
 210 CommandLine PrepareCommandLineForGTest(const CommandLine& command_line,
 211                                        const std::string& wrapper) {
 212   CommandLine new_command_line(command_line.GetProgram());
 213   CommandLine::SwitchMap switches = command_line.GetSwitches();
 214
 215   // Strip out gtest_repeat flag - this is handled by the launcher process.
 216   switches.erase(kGTestRepeatFlag);
 217
 218   // Don't try to write the final XML report in child processes.
 219   switches.erase(kGTestOutputFlag);
 220
 221   for (CommandLine::SwitchMap::const_iterator iter = switches.begin();
 222        iter != switches.end(); ++iter) {
 223     new_command_line.AppendSwitchNative((*iter).first, (*iter).second);
 224   }
 225
 226   // Prepend wrapper after last CommandLine quasi-copy operation. CommandLine
 227   // does not really support removing switches well, and trying to do that
 228   // on a CommandLine with a wrapper is known to break.
 229   // TODO(phajdan.jr): Give it a try to support CommandLine removing switches.
 230 #if defined(OS_WIN)
 231   new_command_line.PrependWrapper(ASCIIToUTF16(wrapper));
 232 #elif defined(OS_POSIX)
 233   new_command_line.PrependWrapper(wrapper);
 234 #endif
 235
 236   return new_command_line;
 237 }
 238
 239 // Launches a child process using |command_line|. If the child process is still
 240 // running after |timeout|, it is terminated and |*was_timeout| is set to true.
 241 // Returns exit code of the process.
 242 int LaunchChildTestProcessWithOptions(const CommandLine& command_line,
 243                                       const LaunchOptions& options,
 244                                       int flags,
 245                                       TimeDelta timeout,
 246                                       bool* was_timeout) {
 247 #if defined(OS_POSIX)
 248   // Make sure an option we rely on is present - see LaunchChildGTestProcess.
 249   DCHECK(options.new_process_group);
 250 #endif
 251
 252   LaunchOptions new_options(options);
 253
 254 #if defined(OS_WIN)
 255   DCHECK(!new_options.job_handle);
 256
 257   win::ScopedHandle job_handle;
 258   if (flags & TestLauncher::USE_JOB_OBJECTS) {
 259     job_handle.Set(CreateJobObject(NULL, NULL));
 260     if (!job_handle.IsValid()) {
 261       LOG(ERROR) << "Could not create JobObject.";
 262       return -1;
 263     }
 264
 265     DWORD job_flags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE;
 266
 267     // Allow break-away from job since sandbox and few other places rely on it
 268     // on Windows versions prior to Windows 8 (which supports nested jobs).
 269     if (win::GetVersion() < win::VERSION_WIN8 &&
 270         flags & TestLauncher::ALLOW_BREAKAWAY_FROM_JOB) {
 271       job_flags |= JOB_OBJECT_LIMIT_BREAKAWAY_OK;
 272     }
 273
 274     if (!SetJobObjectLimitFlags(job_handle.Get(), job_flags)) {
 275       LOG(ERROR) << "Could not SetJobObjectLimitFlags.";
 276       return -1;
 277     }
 278
 279     new_options.job_handle = job_handle.Get();
 280   }
 281 #endif  // defined(OS_WIN)
 282
 283 #if defined(OS_LINUX)
 284   // To prevent accidental privilege sharing to an untrusted child, processes
 285   // are started with PR_SET_NO_NEW_PRIVS. Do not set that here, since this
 286   // new child will be privileged and trusted.
 287   new_options.allow_new_privs = true;
 288 #endif
 289
 290   Process process;
 291
 292   {
 293     // Note how we grab the lock before the process possibly gets created.
 294     // This ensures that when the lock is held, ALL the processes are registered
 295     // in the set.
 296     AutoLock lock(g_live_processes_lock.Get());
 297
 298     process = LaunchProcess(command_line, new_options);
 299     if (!process.IsValid())
 300       return -1;
 301
 302     // TODO(rvargas) crbug.com/417532: Don't store process handles.
 303     g_live_processes.Get().insert(std::make_pair(process.Handle(),
 304                                                  command_line));
 305   }
 306
 307   int exit_code = 0;
 308   if (!process.WaitForExitWithTimeout(timeout, &exit_code)) {
 309     *was_timeout = true;
 310     exit_code = -1;  // Set a non-zero exit code to signal a failure.
 311
 312     // Ensure that the process terminates.
 313     KillProcess(process.Handle(), -1, true);
 314   }
 315
 316   {
 317     // Note how we grab the log before issuing a possibly broad process kill.
 318     // Other code parts that grab the log kill processes, so avoid trying
 319     // to do that twice and trigger all kinds of log messages.
 320     AutoLock lock(g_live_processes_lock.Get());
 321
 322 #if defined(OS_POSIX)
 323     if (exit_code != 0) {
 324       // On POSIX, in case the test does not exit cleanly, either due to a crash
 325       // or due to it timing out, we need to clean up any child processes that
 326       // it might have created. On Windows, child processes are automatically
 327       // cleaned up using JobObjects.
 328       KillProcessGroup(process.Handle());
 329     }
 330 #endif
 331
 332     g_live_processes.Get().erase(process.Handle());
 333   }
 334
 335   return exit_code;
 336 }
 337
 338 void RunCallback(
 339     const TestLauncher::LaunchChildGTestProcessCallback& callback,
 340     int exit_code,
 341     const TimeDelta& elapsed_time,
 342     bool was_timeout,
 343     const std::string& output) {
 344   callback.Run(exit_code, elapsed_time, was_timeout, output);
 345 }
 346
 347 void DoLaunchChildTestProcess(
 348     const CommandLine& command_line,
 349     TimeDelta timeout,
 350     int flags,
 351     bool redirect_stdio,
 352     scoped_refptr<MessageLoopProxy> message_loop_proxy,
 353     const TestLauncher::LaunchChildGTestProcessCallback& callback) {
 354   TimeTicks start_time = TimeTicks::Now();
 355
 356   // Redirect child process output to a file.
 357   FilePath output_file;
 358   CHECK(CreateTemporaryFile(&output_file));
 359
 360   LaunchOptions options;
 361 #if defined(OS_WIN)
 362   win::ScopedHandle handle;
 363
 364   if (redirect_stdio) {
 365     // Make the file handle inheritable by the child.
 366     SECURITY_ATTRIBUTES sa_attr;
 367     sa_attr.nLength = sizeof(SECURITY_ATTRIBUTES);
 368     sa_attr.lpSecurityDescriptor = NULL;
 369     sa_attr.bInheritHandle = TRUE;
 370
 371     handle.Set(CreateFile(output_file.value().c_str(),
 372                           GENERIC_WRITE,
 373                           FILE_SHARE_READ | FILE_SHARE_DELETE,
 374                           &sa_attr,
 375                           OPEN_EXISTING,
 376                           FILE_ATTRIBUTE_TEMPORARY,
 377                           NULL));
 378     CHECK(handle.IsValid());
 379     options.inherit_handles = true;
 380     options.stdin_handle = INVALID_HANDLE_VALUE;
 381     options.stdout_handle = handle.Get();
 382     options.stderr_handle = handle.Get();
 383   }
 384 #elif defined(OS_POSIX)
 385   options.new_process_group = true;
 386
 387   FileHandleMappingVector fds_mapping;
 388   ScopedFD output_file_fd;
 389
 390   if (redirect_stdio) {
 391     output_file_fd.reset(open(output_file.value().c_str(), O_RDWR));
 392     CHECK(output_file_fd.is_valid());
 393
 394     fds_mapping.push_back(std::make_pair(output_file_fd.get(), STDOUT_FILENO));
 395     fds_mapping.push_back(std::make_pair(output_file_fd.get(), STDERR_FILENO));
 396     options.fds_to_remap = &fds_mapping;
 397   }
 398 #endif
 399
 400   bool was_timeout = false;
 401   int exit_code = LaunchChildTestProcessWithOptions(
 402       command_line, options, flags, timeout, &was_timeout);
 403
 404   if (redirect_stdio) {
 405 #if defined(OS_WIN)
 406     FlushFileBuffers(handle.Get());
 407     handle.Close();
 408 #elif defined(OS_POSIX)
 409     output_file_fd.reset();
 410 #endif
 411   }
 412
 413   std::string output_file_contents;
 414   CHECK(ReadFileToString(output_file, &output_file_contents));
 415
 416   if (!DeleteFile(output_file, false)) {
 417     // This needs to be non-fatal at least for Windows.
 418     LOG(WARNING) << "Failed to delete " << output_file.AsUTF8Unsafe();
 419   }
 420
 421   // Run target callback on the thread it was originating from, not on
 422   // a worker pool thread.
 423   message_loop_proxy->PostTask(
 424       FROM_HERE,
 425       Bind(&RunCallback,
 426            callback,
 427            exit_code,
 428            TimeTicks::Now() - start_time,
 429            was_timeout,
 430            output_file_contents));
 431 }
 432
 433 }  // namespace
 434
 435 const char kGTestFilterFlag[] = "gtest_filter";
 436 const char kGTestHelpFlag[]   = "gtest_help";
 437 const char kGTestListTestsFlag[] = "gtest_list_tests";
 438 const char kGTestRepeatFlag[] = "gtest_repeat";
 439 const char kGTestRunDisabledTestsFlag[] = "gtest_also_run_disabled_tests";
 440 const char kGTestOutputFlag[] = "gtest_output";
 441
 442 TestLauncherDelegate::~TestLauncherDelegate() {
 443 }
 444
 445 TestLauncher::TestLauncher(TestLauncherDelegate* launcher_delegate,
 446                            size_t parallel_jobs)
 447     : launcher_delegate_(launcher_delegate),
 448       total_shards_(1),
 449       shard_index_(0),
 450       cycles_(1),
 451       test_started_count_(0),
 452       test_finished_count_(0),
 453       test_success_count_(0),
 454       test_broken_count_(0),
 455       retry_count_(0),
 456       retry_limit_(0),
 457       run_result_(true),
 458       watchdog_timer_(FROM_HERE,
 459                       TimeDelta::FromSeconds(kOutputTimeoutSeconds),
 460                       this,
 461                       &TestLauncher::OnOutputTimeout),
 462       parallel_jobs_(parallel_jobs) {
 463 }
 464
 465 TestLauncher::~TestLauncher() {
 466   if (worker_pool_owner_)
 467     worker_pool_owner_->pool()->Shutdown();
 468 }
 469
 470 bool TestLauncher::Run() {
 471   if (!Init())
 472     return false;
 473
 474   // Value of |cycles_| changes after each iteration. Keep track of the
 475   // original value.
 476   int requested_cycles = cycles_;
 477
 478 #if defined(OS_POSIX)
 479   CHECK_EQ(0, pipe(g_shutdown_pipe));
 480
 481   struct sigaction action;
 482   memset(&action, 0, sizeof(action));
 483   sigemptyset(&action.sa_mask);
 484   action.sa_handler = &ShutdownPipeSignalHandler;
 485
 486   CHECK_EQ(0, sigaction(SIGINT, &action, NULL));
 487   CHECK_EQ(0, sigaction(SIGQUIT, &action, NULL));
 488   CHECK_EQ(0, sigaction(SIGTERM, &action, NULL));
 489
 490   MessageLoopForIO::FileDescriptorWatcher controller;
 491   SignalFDWatcher watcher;
 492
 493   CHECK(MessageLoopForIO::current()->WatchFileDescriptor(
 494             g_shutdown_pipe[0],
 495             true,
 496             MessageLoopForIO::WATCH_READ,
 497             &controller,
 498             &watcher));
 499 #endif  // defined(OS_POSIX)
 500
 501   // Start the watchdog timer.
 502   watchdog_timer_.Reset();
 503
 504   MessageLoop::current()->PostTask(
 505       FROM_HERE,
 506       Bind(&TestLauncher::RunTestIteration, Unretained(this)));
 507
 508   MessageLoop::current()->Run();
 509
 510   if (requested_cycles != 1)
 511     results_tracker_.PrintSummaryOfAllIterations();
 512
 513   MaybeSaveSummaryAsJSON();
 514
 515   return run_result_;
 516 }
 517
 518 void TestLauncher::LaunchChildGTestProcess(
 519     const CommandLine& command_line,
 520     const std::string& wrapper,
 521     TimeDelta timeout,
 522     int flags,
 523     const LaunchChildGTestProcessCallback& callback) {
 524   DCHECK(thread_checker_.CalledOnValidThread());
 525
 526   // Record the exact command line used to launch the child.
 527   CommandLine new_command_line(
 528       PrepareCommandLineForGTest(command_line, wrapper));
 529
 530   // When running in parallel mode we need to redirect stdio to avoid mixed-up
 531   // output. We also always redirect on the bots to get the test output into
 532   // JSON summary.
 533   bool redirect_stdio = (parallel_jobs_ > 1) || BotModeEnabled();
 534
 535   worker_pool_owner_->pool()->PostWorkerTask(
 536       FROM_HERE,
 537       Bind(&DoLaunchChildTestProcess,
 538            new_command_line,
 539            timeout,
 540            flags,
 541            redirect_stdio,
 542            MessageLoopProxy::current(),
 543            Bind(&TestLauncher::OnLaunchTestProcessFinished,
 544                 Unretained(this),
 545                 callback)));
 546 }
 547
 548 void TestLauncher::OnTestFinished(const TestResult& result) {
 549   ++test_finished_count_;
 550
 551   bool print_snippet = false;
 552   std::string print_test_stdio("auto");
 553   if (CommandLine::ForCurrentProcess()->HasSwitch(
 554           switches::kTestLauncherPrintTestStdio)) {
 555     print_test_stdio = CommandLine::ForCurrentProcess()->GetSwitchValueASCII(
 556         switches::kTestLauncherPrintTestStdio);
 557   }
 558   if (print_test_stdio == "auto") {
 559     print_snippet = (result.status != TestResult::TEST_SUCCESS);
 560   } else if (print_test_stdio == "always") {
 561     print_snippet = true;
 562   } else if (print_test_stdio == "never") {
 563     print_snippet = false;
 564   } else {
 565     LOG(WARNING) << "Invalid value of " << switches::kTestLauncherPrintTestStdio
 566                  << ": " << print_test_stdio;
 567   }
 568   if (print_snippet) {
 569     std::vector<std::string> snippet_lines;
 570     SplitString(result.output_snippet, '\n', &snippet_lines);
 571     if (snippet_lines.size() > kOutputSnippetLinesLimit) {
 572       size_t truncated_size = snippet_lines.size() - kOutputSnippetLinesLimit;
 573       snippet_lines.erase(
 574           snippet_lines.begin(),
 575           snippet_lines.begin() + truncated_size);
 576       snippet_lines.insert(snippet_lines.begin(), "<truncated>");
 577     }
 578     fprintf(stdout, "%s", JoinString(snippet_lines, "\n").c_str());
 579     fflush(stdout);
 580   }
 581
 582   if (result.status == TestResult::TEST_SUCCESS) {
 583     ++test_success_count_;
 584   } else {
 585     tests_to_retry_.insert(result.full_name);
 586   }
 587
 588   results_tracker_.AddTestResult(result);
 589
 590   // TODO(phajdan.jr): Align counter (padding).
 591   std::string status_line(
 592       StringPrintf("[%" PRIuS "/%" PRIuS "] %s ",
 593                    test_finished_count_,
 594                    test_started_count_,
 595                    result.full_name.c_str()));
 596   if (result.completed()) {
 597     status_line.append(StringPrintf("(%" PRId64 " ms)",
 598                                     result.elapsed_time.InMilliseconds()));
 599   } else if (result.status == TestResult::TEST_TIMEOUT) {
 600     status_line.append("(TIMED OUT)");
 601   } else if (result.status == TestResult::TEST_CRASH) {
 602     status_line.append("(CRASHED)");
 603   } else if (result.status == TestResult::TEST_SKIPPED) {
 604     status_line.append("(SKIPPED)");
 605   } else if (result.status == TestResult::TEST_UNKNOWN) {
 606     status_line.append("(UNKNOWN)");
 607   } else {
 608     // Fail very loudly so it's not ignored.
 609     CHECK(false) << "Unhandled test result status: " << result.status;
 610   }
 611   fprintf(stdout, "%s\n", status_line.c_str());
 612   fflush(stdout);
 613
 614   // We just printed a status line, reset the watchdog timer.
 615   watchdog_timer_.Reset();
 616
 617   // Do not waste time on timeouts. We include tests with unknown results here
 618   // because sometimes (e.g. hang in between unit tests) that's how a timeout
 619   // gets reported.
 620   if (result.status == TestResult::TEST_TIMEOUT ||
 621       result.status == TestResult::TEST_UNKNOWN) {
 622     test_broken_count_++;
 623   }
 624   size_t broken_threshold =
 625       std::max(static_cast<size_t>(20), test_started_count_ / 10);
 626   if (test_broken_count_ >= broken_threshold) {
 627     fprintf(stdout, "Too many badly broken tests (%" PRIuS "), exiting now.\n",
 628             test_broken_count_);
 629     fflush(stdout);
 630
 631 #if defined(OS_POSIX)
 632     KillSpawnedTestProcesses();
 633 #endif  // defined(OS_POSIX)
 634
 635     results_tracker_.AddGlobalTag("BROKEN_TEST_EARLY_EXIT");
 636     results_tracker_.AddGlobalTag(kUnreliableResultsTag);
 637     MaybeSaveSummaryAsJSON();
 638
 639     exit(1);
 640   }
 641
 642   if (test_finished_count_ != test_started_count_)
 643     return;
 644
 645   if (tests_to_retry_.empty() || retry_count_ >= retry_limit_) {
 646     OnTestIterationFinished();
 647     return;
 648   }
 649
 650   if (tests_to_retry_.size() >= broken_threshold) {
 651     fprintf(stdout,
 652             "Too many failing tests (%" PRIuS "), skipping retries.\n",
 653             tests_to_retry_.size());
 654     fflush(stdout);
 655
 656     results_tracker_.AddGlobalTag("BROKEN_TEST_SKIPPED_RETRIES");
 657     results_tracker_.AddGlobalTag(kUnreliableResultsTag);
 658
 659     OnTestIterationFinished();
 660     return;
 661   }
 662
 663   retry_count_++;
 664
 665   std::vector<std::string> test_names(tests_to_retry_.begin(),
 666                                       tests_to_retry_.end());
 667
 668   tests_to_retry_.clear();
 669
 670   size_t retry_started_count = launcher_delegate_->RetryTests(this, test_names);
 671   if (retry_started_count == 0) {
 672     // Signal failure, but continue to run all requested test iterations.
 673     // With the summary of all iterations at the end this is a good default.
 674     run_result_ = false;
 675
 676     OnTestIterationFinished();
 677     return;
 678   }
 679
 680   fprintf(stdout, "Retrying %" PRIuS " test%s (retry #%" PRIuS ")\n",
 681           retry_started_count,
 682           retry_started_count > 1 ? "s" : "",
 683           retry_count_);
 684   fflush(stdout);
 685
 686   test_started_count_ += retry_started_count;
 687 }
 688
 689 bool TestLauncher::Init() {
 690   const CommandLine* command_line = CommandLine::ForCurrentProcess();
 691
 692   // Initialize sharding. Command line takes precedence over legacy environment
 693   // variables.
 694   if (command_line->HasSwitch(switches::kTestLauncherTotalShards) &&
 695       command_line->HasSwitch(switches::kTestLauncherShardIndex)) {
 696     if (!StringToInt(
 697             command_line->GetSwitchValueASCII(
 698                 switches::kTestLauncherTotalShards),
 699             &total_shards_)) {
 700       LOG(ERROR) << "Invalid value for " << switches::kTestLauncherTotalShards;
 701       return false;
 702     }
 703     if (!StringToInt(
 704             command_line->GetSwitchValueASCII(
 705                 switches::kTestLauncherShardIndex),
 706             &shard_index_)) {
 707       LOG(ERROR) << "Invalid value for " << switches::kTestLauncherShardIndex;
 708       return false;
 709     }
 710     fprintf(stdout,
 711             "Using sharding settings from command line. This is shard %d/%d\n",
 712             shard_index_, total_shards_);
 713     fflush(stdout);
 714   } else {
 715     if (!TakeInt32FromEnvironment(kTestTotalShards, &total_shards_))
 716       return false;
 717     if (!TakeInt32FromEnvironment(kTestShardIndex, &shard_index_))
 718       return false;
 719     fprintf(stdout,
 720             "Using sharding settings from environment. This is shard %d/%d\n",
 721             shard_index_, total_shards_);
 722     fflush(stdout);
 723   }
 724   if (shard_index_ < 0 ||
 725       total_shards_ < 0 ||
 726       shard_index_ >= total_shards_) {
 727     LOG(ERROR) << "Invalid sharding settings: we require 0 <= "
 728                << kTestShardIndex << " < " << kTestTotalShards
 729                << ", but you have " << kTestShardIndex << "=" << shard_index_
 730                << ", " << kTestTotalShards << "=" << total_shards_ << ".\n";
 731     return false;
 732   }
 733
 734   // Make sure we don't pass any sharding-related environment to the child
 735   // processes. This test launcher implements the sharding completely.
 736   CHECK(UnsetEnvironmentVariableIfExists("GTEST_TOTAL_SHARDS"));
 737   CHECK(UnsetEnvironmentVariableIfExists("GTEST_SHARD_INDEX"));
 738
 739   if (command_line->HasSwitch(kGTestRepeatFlag) &&
 740       !StringToInt(command_line->GetSwitchValueASCII(kGTestRepeatFlag),
 741                    &cycles_)) {
 742     LOG(ERROR) << "Invalid value for " << kGTestRepeatFlag;
 743     return false;
 744   }
 745
 746   if (command_line->HasSwitch(switches::kTestLauncherRetryLimit)) {
 747     int retry_limit = -1;
 748     if (!StringToInt(command_line->GetSwitchValueASCII(
 749                          switches::kTestLauncherRetryLimit), &retry_limit) ||
 750         retry_limit < 0) {
 751       LOG(ERROR) << "Invalid value for " << switches::kTestLauncherRetryLimit;
 752       return false;
 753     }
 754
 755     retry_limit_ = retry_limit;
 756   } else if (!command_line->HasSwitch(kGTestFilterFlag) || BotModeEnabled()) {
 757     // Retry failures 3 times by default if we are running all of the tests or
 758     // in bot mode.
 759     retry_limit_ = 3;
 760   }
 761
 762   if (command_line->HasSwitch(switches::kTestLauncherJobs)) {
 763     int jobs = -1;
 764     if (!StringToInt(command_line->GetSwitchValueASCII(
 765                          switches::kTestLauncherJobs), &jobs) ||
 766         jobs < 0) {
 767       LOG(ERROR) << "Invalid value for " << switches::kTestLauncherJobs;
 768       return false;
 769     }
 770
 771     parallel_jobs_ = jobs;
 772   } else if (command_line->HasSwitch(kGTestFilterFlag) && !BotModeEnabled()) {
 773     // Do not run jobs in parallel by default if we are running a subset of
 774     // the tests and if bot mode is off.
 775     parallel_jobs_ = 1;
 776   }
 777
 778   fprintf(stdout, "Using %" PRIuS " parallel jobs.\n", parallel_jobs_);
 779   fflush(stdout);
 780   worker_pool_owner_.reset(
 781       new SequencedWorkerPoolOwner(parallel_jobs_, "test_launcher"));
 782
 783   if (command_line->HasSwitch(switches::kTestLauncherFilterFile) &&
 784       command_line->HasSwitch(kGTestFilterFlag)) {
 785     LOG(ERROR) << "Only one of --test-launcher-filter-file and --gtest_filter "
 786                << "at a time is allowed.";
 787     return false;
 788   }
 789
 790   if (command_line->HasSwitch(switches::kTestLauncherFilterFile)) {
 791     std::string filter;
 792     if (!ReadFileToString(
 793             command_line->GetSwitchValuePath(switches::kTestLauncherFilterFile),
 794             &filter)) {
 795       LOG(ERROR) << "Failed to read the filter file.";
 796       return false;
 797     }
 798
 799     std::vector<std::string> filter_lines;
 800     SplitString(filter, '\n', &filter_lines);
 801     for (size_t i = 0; i < filter_lines.size(); i++) {
 802       if (filter_lines[i].empty())
 803         continue;
 804
 805       if (filter_lines[i][0] == '-')
 806         negative_test_filter_.push_back(filter_lines[i].substr(1));
 807       else
 808         positive_test_filter_.push_back(filter_lines[i]);
 809     }
 810   } else {
 811     // Split --gtest_filter at '-', if there is one, to separate into
 812     // positive filter and negative filter portions.
 813     std::string filter = command_line->GetSwitchValueASCII(kGTestFilterFlag);
 814     size_t dash_pos = filter.find('-');
 815     if (dash_pos == std::string::npos) {
 816       SplitString(filter, ':', &positive_test_filter_);
 817     } else {
 818       // Everything up to the dash.
 819       SplitString(filter.substr(0, dash_pos), ':', &positive_test_filter_);
 820
 821       // Everything after the dash.
 822       SplitString(filter.substr(dash_pos + 1), ':', &negative_test_filter_);
 823     }
 824   }
 825
 826   if (!launcher_delegate_->GetTests(&tests_)) {
 827     LOG(ERROR) << "Failed to get list of tests.";
 828     return false;
 829   }
 830
 831   if (!results_tracker_.Init(*command_line)) {
 832     LOG(ERROR) << "Failed to initialize test results tracker.";
 833     return 1;
 834   }
 835
 836 #if defined(NDEBUG)
 837   results_tracker_.AddGlobalTag("MODE_RELEASE");
 838 #else
 839   results_tracker_.AddGlobalTag("MODE_DEBUG");
 840 #endif
 841
 842   // Operating systems (sorted alphabetically).
 843   // Note that they can deliberately overlap, e.g. OS_LINUX is a subset
 844   // of OS_POSIX.
 845 #if defined(OS_ANDROID)
 846   results_tracker_.AddGlobalTag("OS_ANDROID");
 847 #endif
 848
 849 #if defined(OS_BSD)
 850   results_tracker_.AddGlobalTag("OS_BSD");
 851 #endif
 852
 853 #if defined(OS_FREEBSD)
 854   results_tracker_.AddGlobalTag("OS_FREEBSD");
 855 #endif
 856
 857 #if defined(OS_IOS)
 858   results_tracker_.AddGlobalTag("OS_IOS");
 859 #endif
 860
 861 #if defined(OS_LINUX)
 862   results_tracker_.AddGlobalTag("OS_LINUX");
 863 #endif
 864
 865 #if defined(OS_MACOSX)
 866   results_tracker_.AddGlobalTag("OS_MACOSX");
 867 #endif
 868
 869 #if defined(OS_NACL)
 870   results_tracker_.AddGlobalTag("OS_NACL");
 871 #endif
 872
 873 #if defined(OS_OPENBSD)
 874   results_tracker_.AddGlobalTag("OS_OPENBSD");
 875 #endif
 876
 877 #if defined(OS_POSIX)
 878   results_tracker_.AddGlobalTag("OS_POSIX");
 879 #endif
 880
 881 #if defined(OS_SOLARIS)
 882   results_tracker_.AddGlobalTag("OS_SOLARIS");
 883 #endif
 884
 885 #if defined(OS_WIN)
 886   results_tracker_.AddGlobalTag("OS_WIN");
 887 #endif
 888
 889   // CPU-related tags.
 890 #if defined(ARCH_CPU_32_BITS)
 891   results_tracker_.AddGlobalTag("CPU_32_BITS");
 892 #endif
 893
 894 #if defined(ARCH_CPU_64_BITS)
 895   results_tracker_.AddGlobalTag("CPU_64_BITS");
 896 #endif
 897
 898   return true;
 899 }
 900
 901 void TestLauncher::RunTests() {
 902   std::vector<std::string> test_names;
 903   for (size_t i = 0; i < tests_.size(); i++) {
 904     std::string test_name = FormatFullTestName(
 905         tests_[i].first, tests_[i].second);
 906
 907     results_tracker_.AddTest(test_name);
 908
 909     const CommandLine* command_line = CommandLine::ForCurrentProcess();
 910     if (test_name.find("DISABLED") != std::string::npos) {
 911       results_tracker_.AddDisabledTest(test_name);
 912
 913       // Skip disabled tests unless explicitly requested.
 914       if (!command_line->HasSwitch(kGTestRunDisabledTestsFlag))
 915         continue;
 916     }
 917
 918     if (!launcher_delegate_->ShouldRunTest(tests_[i].first, tests_[i].second))
 919       continue;
 920
 921     // Skip the test that doesn't match the filter (if given).
 922     if (!positive_test_filter_.empty()) {
 923       bool found = false;
 924       for (size_t k = 0; k < positive_test_filter_.size(); ++k) {
 925         if (MatchPattern(test_name, positive_test_filter_[k])) {
 926           found = true;
 927           break;
 928         }
 929       }
 930
 931       if (!found)
 932         continue;
 933     }
 934     bool excluded = false;
 935     for (size_t k = 0; k < negative_test_filter_.size(); ++k) {
 936       if (MatchPattern(test_name, negative_test_filter_[k])) {
 937         excluded = true;
 938         break;
 939       }
 940     }
 941     if (excluded)
 942       continue;
 943
 944     if (Hash(test_name) % total_shards_ != static_cast<uint32>(shard_index_))
 945       continue;
 946
 947     test_names.push_back(test_name);
 948   }
 949
 950   test_started_count_ = launcher_delegate_->RunTests(this, test_names);
 951
 952   if (test_started_count_ == 0) {
 953     fprintf(stdout, "0 tests run\n");
 954     fflush(stdout);
 955
 956     // No tests have actually been started, so kick off the next iteration.
 957     MessageLoop::current()->PostTask(
 958         FROM_HERE,
 959         Bind(&TestLauncher::RunTestIteration, Unretained(this)));
 960   }
 961 }
 962
 963 void TestLauncher::RunTestIteration() {
 964   if (cycles_ == 0) {
 965     MessageLoop::current()->Quit();
 966     return;
 967   }
 968
 969   // Special value "-1" means "repeat indefinitely".
 970   cycles_ = (cycles_ == -1) ? cycles_ : cycles_ - 1;
 971
 972   test_started_count_ = 0;
 973   test_finished_count_ = 0;
 974   test_success_count_ = 0;
 975   test_broken_count_ = 0;
 976   retry_count_ = 0;
 977   tests_to_retry_.clear();
 978   results_tracker_.OnTestIterationStarting();
 979
 980   MessageLoop::current()->PostTask(
 981       FROM_HERE, Bind(&TestLauncher::RunTests, Unretained(this)));
 982 }
 983
 984 void TestLauncher::MaybeSaveSummaryAsJSON() {
 985   const CommandLine* command_line = CommandLine::ForCurrentProcess();
 986   if (command_line->HasSwitch(switches::kTestLauncherSummaryOutput)) {
 987     FilePath summary_path(command_line->GetSwitchValuePath(
 988                               switches::kTestLauncherSummaryOutput));
 989     if (!results_tracker_.SaveSummaryAsJSON(summary_path)) {
 990       LOG(ERROR) << "Failed to save test launcher output summary.";
 991     }
 992   }
 993 }
 994
 995 void TestLauncher::OnLaunchTestProcessFinished(
 996     const LaunchChildGTestProcessCallback& callback,
 997     int exit_code,
 998     const TimeDelta& elapsed_time,
 999     bool was_timeout,
1000     const std::string& output) {
1001   DCHECK(thread_checker_.CalledOnValidThread());
1002
1003   callback.Run(exit_code, elapsed_time, was_timeout, output);
1004 }
1005
1006 void TestLauncher::OnTestIterationFinished() {
1007   TestResultsTracker::TestStatusMap tests_by_status(
1008       results_tracker_.GetTestStatusMapForCurrentIteration());
1009   if (!tests_by_status[TestResult::TEST_UNKNOWN].empty())
1010     results_tracker_.AddGlobalTag(kUnreliableResultsTag);
1011
1012   // When we retry tests, success is determined by having nothing more
1013   // to retry (everything eventually passed), as opposed to having
1014   // no failures at all.
1015   if (tests_to_retry_.empty()) {
1016     fprintf(stdout, "SUCCESS: all tests passed.\n");
1017     fflush(stdout);
1018   } else {
1019     // Signal failure, but continue to run all requested test iterations.
1020     // With the summary of all iterations at the end this is a good default.
1021     run_result_ = false;
1022   }
1023
1024   results_tracker_.PrintSummaryOfCurrentIteration();
1025
1026   // Kick off the next iteration.
1027   MessageLoop::current()->PostTask(
1028       FROM_HERE,
1029       Bind(&TestLauncher::RunTestIteration, Unretained(this)));
1030 }
1031
1032 void TestLauncher::OnOutputTimeout() {
1033   DCHECK(thread_checker_.CalledOnValidThread());
1034
1035   AutoLock lock(g_live_processes_lock.Get());
1036
1037   fprintf(stdout, "Still waiting for the following processes to finish:\n");
1038
1039   for (std::map<ProcessHandle, CommandLine>::iterator i =
1040            g_live_processes.Get().begin();
1041        i != g_live_processes.Get().end();
1042        ++i) {
1043 #if defined(OS_WIN)
1044     fwprintf(stdout, L"\t%s\n", i->second.GetCommandLineString().c_str());
1045 #else
1046     fprintf(stdout, "\t%s\n", i->second.GetCommandLineString().c_str());
1047 #endif
1048   }
1049
1050   fflush(stdout);
1051
1052   // Arm the timer again - otherwise it would fire only once.
1053   watchdog_timer_.Reset();
1054 }
1055
1056 std::string GetTestOutputSnippet(const TestResult& result,
1057                                  const std::string& full_output) {
1058   size_t run_pos = full_output.find(std::string("[ RUN      ] ") +
1059                                     result.full_name);
1060   if (run_pos == std::string::npos)
1061     return std::string();
1062
1063   size_t end_pos = full_output.find(std::string("[  FAILED  ] ") +
1064                                     result.full_name,
1065                                     run_pos);
1066   // Only clip the snippet to the "OK" message if the test really
1067   // succeeded. It still might have e.g. crashed after printing it.
1068   if (end_pos == std::string::npos &&
1069       result.status == TestResult::TEST_SUCCESS) {
1070     end_pos = full_output.find(std::string("[       OK ] ") +
1071                                result.full_name,
1072                                run_pos);
1073   }
1074   if (end_pos != std::string::npos) {
1075     size_t newline_pos = full_output.find("\n", end_pos);
1076     if (newline_pos != std::string::npos)
1077       end_pos = newline_pos + 1;
1078   }
1079
1080   std::string snippet(full_output.substr(run_pos));
1081   if (end_pos != std::string::npos)
1082     snippet = full_output.substr(run_pos, end_pos - run_pos);
1083
1084   return snippet;
1085 }
1086
1087 }  // namespace base