tools/sharding_supervisor/sharding_supervisor_old.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Shards a given test suite and runs the shards in parallel.
   7
   8 ShardingSupervisor is called to process the command line options and creates
   9 the specified number of worker threads. These threads then run each shard of
  10 the test in a separate process and report on the results. When all the shards
  11 have been completed, the supervisor reprints any lines indicating a test
  12 failure for convenience. If only one shard is to be run, a single subprocess
  13 is started for that shard and the output is identical to gtest's output.
  14 """
  15
  16 import itertools
  17 import optparse
  18 import os
  19 import Queue
  20 import random
  21 import re
  22 import sys
  23 import threading
  24
  25 from stdio_buffer import StdioBuffer
  26 from xml.dom import minidom
  27
  28 # Add tools/ to path
  29 BASE_PATH = os.path.dirname(os.path.abspath(__file__))
  30 sys.path.append(os.path.join(BASE_PATH, ".."))
  31 try:
  32   import find_depot_tools  # pylint: disable=F0401,W0611
  33   # Fixes a bug in Windows where some shards die upon starting
  34   # TODO(charleslee): actually fix this bug
  35   import subprocess2 as subprocess
  36 except ImportError:
  37   # Unable to find depot_tools, so just use standard subprocess
  38   import subprocess
  39
  40 SS_USAGE = "python %prog [options] path/to/test [gtest_args]"
  41 SS_DEFAULT_NUM_CORES = 4
  42 SS_DEFAULT_SHARDS_PER_CORE = 5  # num_shards = cores * SHARDS_PER_CORE
  43 SS_DEFAULT_RUNS_PER_CORE = 1  # num_workers = cores * RUNS_PER_CORE
  44 SS_DEFAULT_RETRY_PERCENT = 5  # --retry-failed ignored if more than 5% fail
  45 SS_DEFAULT_TIMEOUT = 530  # Slightly less than buildbot's default 600 seconds
  46
  47
  48 def DetectNumCores():
  49   """Detects the number of cores on the machine.
  50
  51   Returns:
  52     The number of cores on the machine or DEFAULT_NUM_CORES if it could not
  53     be found.
  54   """
  55   try:
  56     # Override on some Chromium Valgrind bots.
  57     if "CHROME_VALGRIND_NUMCPUS" in os.environ:
  58       return int(os.environ["CHROME_VALGRIND_NUMCPUS"])
  59     # Linux, Unix, MacOS
  60     if hasattr(os, "sysconf"):
  61       if "SC_NPROCESSORS_ONLN" in os.sysconf_names:
  62         # Linux, Unix
  63         return int(os.sysconf("SC_NPROCESSORS_ONLN"))
  64       else:
  65         # OSX
  66         return int(os.popen2("sysctl -n hw.ncpu")[1].read())
  67     # Windows
  68     return int(os.environ["NUMBER_OF_PROCESSORS"])
  69   except ValueError:
  70     return SS_DEFAULT_NUM_CORES
  71
  72
  73 def GetGTestOutput(args):
  74   """Extracts gtest_output from the args. Returns none if not present."""
  75
  76   for arg in args:
  77     if '--gtest_output=' in arg:
  78       return arg.split('=')[1]
  79   return None
  80
  81
  82 def AppendToGTestOutput(gtest_args, value):
  83   args = gtest_args[:]
  84   current_value = GetGTestOutput(args)
  85   if not current_value:
  86     return gtest_args
  87
  88   current_arg = '--gtest_output=' + current_value
  89   args.remove(current_arg)
  90   args.append('--gtest_output=' + current_value + value)
  91   return args
  92
  93
  94 def RemoveGTestOutput(gtest_args):
  95   args = gtest_args[:]
  96   current_value = GetGTestOutput(args)
  97   if not current_value:
  98     return gtest_args
  99
 100   args.remove('--gtest_output=' + current_value)
 101   return args
 102
 103
 104 def AppendToXML(final_xml, generic_path, shard):
 105   """Combine the shard xml file with the final xml file."""
 106
 107   path = generic_path + str(shard)
 108
 109   try:
 110     with open(path) as shard_xml_file:
 111       shard_xml = minidom.parse(shard_xml_file)
 112   except IOError:
 113     # If the shard crashed, gtest will not have generated an xml file.
 114     return final_xml
 115
 116   if not final_xml:
 117     # Out final xml is empty, let's prepopulate it with the first one we see.
 118     return shard_xml
 119
 120   shard_node = shard_xml.documentElement
 121   final_node = final_xml.documentElement
 122
 123   testcases = shard_node.getElementsByTagName('testcase')
 124   final_testcases = final_node.getElementsByTagName('testcase')
 125
 126   final_testsuites = final_node.getElementsByTagName('testsuite')
 127   final_testsuites_by_name = dict(
 128       (suite.getAttribute('name'), suite) for suite in final_testsuites)
 129
 130   for testcase in testcases:
 131     name = testcase.getAttribute('name')
 132     classname = testcase.getAttribute('classname')
 133     failures = testcase.getElementsByTagName('failure')
 134     status = testcase.getAttribute('status')
 135     elapsed = testcase.getAttribute('time')
 136
 137     # don't bother updating the final xml if there is no data.
 138     if status == 'notrun':
 139       continue
 140
 141     # Look in our final xml to see if it's there.
 142     # There has to be a better way...
 143     merged_into_final_testcase = False
 144     for final_testcase in final_testcases:
 145       final_name = final_testcase.getAttribute('name')
 146       final_classname = final_testcase.getAttribute('classname')
 147       if final_name == name and final_classname == classname:
 148         # We got the same entry.
 149         final_testcase.setAttribute('status', status)
 150         final_testcase.setAttribute('time', elapsed)
 151         for failure in failures:
 152           final_testcase.appendChild(failure)
 153         merged_into_final_testcase = True
 154
 155     # We couldn't find an existing testcase to merge the results into, so we
 156     # copy the node into the existing test suite.
 157     if not merged_into_final_testcase:
 158       testsuite = testcase.parentNode
 159       final_testsuite = final_testsuites_by_name[testsuite.getAttribute('name')]
 160       final_testsuite.appendChild(testcase)
 161
 162   return final_xml
 163
 164
 165 def RunShard(test, total_shards, index, gtest_args, stdout, stderr):
 166   """Runs a single test shard in a subprocess.
 167
 168   Returns:
 169     The Popen object representing the subprocess handle.
 170   """
 171   args = [test]
 172
 173   # If there is a gtest_output
 174   test_args = AppendToGTestOutput(gtest_args, str(index))
 175   args.extend(test_args)
 176   env = os.environ.copy()
 177   env["GTEST_TOTAL_SHARDS"] = str(total_shards)
 178   env["GTEST_SHARD_INDEX"] = str(index)
 179
 180   # Use a unique log file for each shard
 181   # Allows ui_tests to be run in parallel on the same machine
 182   env["CHROME_LOG_FILE"] = "chrome_log_%d" % index
 183
 184   return subprocess.Popen(
 185       args, stdout=stdout,
 186       stderr=stderr,
 187       env=env,
 188       bufsize=0,
 189       universal_newlines=True)
 190
 191
 192 class ShardRunner(threading.Thread):
 193   """Worker thread that manages a single shard at a time.
 194
 195   Attributes:
 196     supervisor: The ShardingSupervisor that this worker reports to.
 197     counter: Called to get the next shard index to run.
 198     test_start: Regex that detects when a test runs.
 199     test_ok: Regex that detects a passing test.
 200     test_fail: Regex that detects a failing test.
 201     current_test: The name of the currently running test.
 202   """
 203
 204   def __init__(self, supervisor, counter, test_start, test_ok, test_fail):
 205     """Inits ShardRunner and sets the current test to nothing."""
 206     threading.Thread.__init__(self)
 207     self.supervisor = supervisor
 208     self.counter = counter
 209     self.test_start = test_start
 210     self.test_ok = test_ok
 211     self.test_fail = test_fail
 212     self.current_test = ""
 213
 214   def ReportFailure(self, description, index, test_name):
 215     """Assembles and reports a failure line to be printed later."""
 216     log_line = "%s (%i): %s\n" % (description, index, test_name)
 217     self.supervisor.LogTestFailure(log_line)
 218
 219   def ProcessLine(self, index, line):
 220     """Checks a shard output line for test status, and reports a failure or
 221     incomplete test if needed.
 222     """
 223     results = self.test_start.search(line)
 224     if results:
 225       if self.current_test:
 226         self.ReportFailure("INCOMPLETE", index, self.current_test)
 227       self.current_test = results.group(1)
 228       self.supervisor.IncrementTestCount()
 229       return
 230
 231     results = self.test_ok.search(line)
 232     if results:
 233       self.current_test = ""
 234       return
 235
 236     results = self.test_fail.search(line)
 237     if results:
 238       self.ReportFailure("FAILED", index, results.group(1))
 239       self.current_test = ""
 240
 241   def run(self):
 242     """Runs shards and outputs the results.
 243
 244     Gets the next shard index from the supervisor, runs it in a subprocess,
 245     and collects the output. The output is read character by character in
 246     case the shard crashes without an ending newline. Each line is processed
 247     as it is finished.
 248     """
 249     while True:
 250       try:
 251         index = self.counter.get_nowait()
 252       except Queue.Empty:
 253         break
 254       shard_running = True
 255       shard = RunShard(
 256           self.supervisor.test, self.supervisor.total_shards, index,
 257           self.supervisor.gtest_args, subprocess.PIPE, subprocess.PIPE)
 258       buf = StdioBuffer(shard)
 259       # Spawn two threads to collect stdio output
 260       stdout_collector_thread = buf.handle_pipe(sys.stdout, shard.stdout)
 261       stderr_collector_thread = buf.handle_pipe(sys.stderr, shard.stderr)
 262       while shard_running:
 263         pipe, line = buf.readline()
 264         if pipe is None and line is None:
 265           shard_running = False
 266         if not line and not shard_running:
 267           break
 268         self.ProcessLine(index, line)
 269         self.supervisor.LogOutputLine(index, line, pipe)
 270       stdout_collector_thread.join()
 271       stderr_collector_thread.join()
 272       if self.current_test:
 273         self.ReportFailure("INCOMPLETE", index, self.current_test)
 274       self.supervisor.ShardIndexCompleted(index)
 275       if shard.returncode != 0:
 276         self.supervisor.LogShardFailure(index)
 277
 278
 279 class ShardingSupervisor(object):
 280   """Supervisor object that handles the worker threads.
 281
 282   Attributes:
 283     test: Name of the test to shard.
 284     num_shards_to_run: Total number of shards to split the test into.
 285     num_runs: Total number of worker threads to create for running shards.
 286     color: Indicates which coloring mode to use in the output.
 287     original_order: True if shard output should be printed as it comes.
 288     prefix: True if each line should indicate the shard index.
 289     retry_percent: Integer specifying the max percent of tests to retry.
 290     gtest_args: The options to pass to gtest.
 291     failed_tests: List of statements from shard output indicating a failure.
 292     failed_shards: List of shards that contained failing tests.
 293     shards_completed: List of flags indicating which shards have finished.
 294     shard_output: Buffer that stores output from each shard as (stdio, line).
 295     test_counter: Stores the total number of tests run.
 296     total_slaves: Total number of slaves running this test.
 297     slave_index: Current slave to run tests for.
 298
 299   If total_slaves is set, we run only a subset of the tests. This is meant to be
 300   used when we want to shard across machines as well as across cpus. In that
 301   case the number of shards to execute will be the same, but they will be
 302   smaller, as the total number of shards in the test suite will be multiplied
 303   by 'total_slaves'.
 304
 305   For example, if you are on a quad core machine, the sharding supervisor by
 306   default will use 20 shards for the whole suite. However, if you set
 307   total_slaves to 2, it will split the suite in 40 shards and will only run
 308   shards [0-19] or shards [20-39] depending if you set slave_index to 0 or 1.
 309   """
 310
 311   SHARD_COMPLETED = object()
 312
 313   def __init__(self, test, num_shards_to_run, num_runs, color, original_order,
 314                prefix, retry_percent, timeout, total_slaves, slave_index,
 315                gtest_args):
 316     """Inits ShardingSupervisor with given options and gtest arguments."""
 317     self.test = test
 318     # Number of shards to run locally.
 319     self.num_shards_to_run = num_shards_to_run
 320     # Total shards in the test suite running across all slaves.
 321     self.total_shards = num_shards_to_run * total_slaves
 322     self.slave_index = slave_index
 323     self.num_runs = num_runs
 324     self.color = color
 325     self.original_order = original_order
 326     self.prefix = prefix
 327     self.retry_percent = retry_percent
 328     self.timeout = timeout
 329     self.gtest_args = gtest_args
 330     self.failed_tests = []
 331     self.failed_shards = []
 332     self.shards_completed = [False] * self.num_shards_to_run
 333     self.shard_output = [Queue.Queue() for _ in range(self.num_shards_to_run)]
 334     self.test_counter = itertools.count()
 335
 336   def ShardTest(self):
 337     """Runs the test and manages the worker threads.
 338
 339     Runs the test and outputs a summary at the end. All the tests in the
 340     suite are run by creating (cores * runs_per_core) threads and
 341     (cores * shards_per_core) shards. When all the worker threads have
 342     finished, the lines saved in failed_tests are printed again. If enabled,
 343     and failed tests that do not have FLAKY or FAILS in their names are run
 344     again, serially, and the results are printed.
 345
 346     Returns:
 347       1 if some unexpected (not FLAKY or FAILS) tests failed, 0 otherwise.
 348     """
 349
 350     # Regular expressions for parsing GTest logs. Test names look like
 351     #   SomeTestCase.SomeTest
 352     #   SomeName/SomeTestCase.SomeTest/1
 353     # This regex also matches SomeName.SomeTest/1 and
 354     # SomeName/SomeTestCase.SomeTest, which should be harmless.
 355     test_name_regex = r"((\w+/)?\w+\.\w+(/\d+)?)"
 356
 357     # Regex for filtering out ANSI escape codes when using color.
 358     ansi_regex = r"(?:\x1b\[.*?[a-zA-Z])?"
 359
 360     test_start = re.compile(
 361         ansi_regex + r"\[\s+RUN\s+\] " + ansi_regex + test_name_regex)
 362     test_ok = re.compile(
 363         ansi_regex + r"\[\s+OK\s+\] " + ansi_regex + test_name_regex)
 364     test_fail = re.compile(
 365         ansi_regex + r"\[\s+FAILED\s+\] " + ansi_regex + test_name_regex)
 366
 367     workers = []
 368     counter = Queue.Queue()
 369     start_point = self.num_shards_to_run * self.slave_index
 370     for i in range(start_point, start_point + self.num_shards_to_run):
 371       counter.put(i)
 372
 373     for i in range(self.num_runs):
 374       worker = ShardRunner(
 375           self, counter, test_start, test_ok, test_fail)
 376       worker.start()
 377       workers.append(worker)
 378     if self.original_order:
 379       for worker in workers:
 380         worker.join()
 381     else:
 382       self.WaitForShards()
 383
 384     # All the shards are done.  Merge all the XML files and generate the
 385     # main one.
 386     output_arg = GetGTestOutput(self.gtest_args)
 387     if output_arg:
 388       xml, xml_path = output_arg.split(':', 1)
 389       assert(xml == 'xml')
 390       final_xml = None
 391       for i in range(start_point, start_point + self.num_shards_to_run):
 392         final_xml = AppendToXML(final_xml, xml_path, i)
 393
 394       if final_xml:
 395         with open(xml_path, 'w') as final_file:
 396           final_xml.writexml(final_file)
 397
 398     num_failed = len(self.failed_shards)
 399     if num_failed > 0:
 400       self.failed_shards.sort()
 401       self.WriteText(sys.stdout,
 402                      "\nFAILED SHARDS: %s\n" % str(self.failed_shards),
 403                      "\x1b[1;5;31m")
 404     else:
 405       self.WriteText(sys.stdout, "\nALL SHARDS PASSED!\n", "\x1b[1;5;32m")
 406     self.PrintSummary(self.failed_tests)
 407     if self.retry_percent < 0:
 408       return len(self.failed_shards) > 0
 409
 410     self.failed_tests = [x for x in self.failed_tests if x.find("FAILS_") < 0]
 411     self.failed_tests = [x for x in self.failed_tests if x.find("FLAKY_") < 0]
 412     if not self.failed_tests:
 413       return 0
 414     return self.RetryFailedTests()
 415
 416   def LogTestFailure(self, line):
 417     """Saves a line in the lsit of failed tests to be printed at the end."""
 418     if line not in self.failed_tests:
 419       self.failed_tests.append(line)
 420
 421   def LogShardFailure(self, index):
 422     """Records that a test in the given shard has failed."""
 423     self.failed_shards.append(index)
 424
 425   def WaitForShards(self):
 426     """Prints the output from each shard in consecutive order, waiting for
 427     the current shard to finish before starting on the next shard.
 428     """
 429     try:
 430       for shard_index in range(self.num_shards_to_run):
 431         while True:
 432           try:
 433             _, line = self.shard_output[shard_index].get(True, self.timeout)
 434           except Queue.Empty:
 435             # Shard timed out, notice failure and move on.
 436             self.LogShardFailure(shard_index)
 437             # TODO(maruel): Print last test. It'd be simpler to have the
 438             # processing in the main thread.
 439             # TODO(maruel): Make sure the worker thread terminates.
 440             sys.stdout.write('TIMED OUT\n\n')
 441             self.LogTestFailure(
 442                 'FAILURE: SHARD %d TIMED OUT; %d seconds' % (
 443                     shard_index, self.timeout))
 444             break
 445           if line is self.SHARD_COMPLETED:
 446             break
 447           sys.stdout.write(line)
 448     except:
 449       sys.stdout.flush()
 450       print 'CAUGHT EXCEPTION: dumping remaining data:'
 451       for shard_index in range(self.num_shards_to_run):
 452         while True:
 453           try:
 454             _, line = self.shard_output[shard_index].get(False)
 455           except Queue.Empty:
 456             # Shard timed out, notice failure and move on.
 457             self.LogShardFailure(shard_index)
 458             break
 459           if line is self.SHARD_COMPLETED:
 460             break
 461           sys.stdout.write(line)
 462       raise
 463
 464   def LogOutputLine(self, index, line, pipe=sys.stdout):
 465     """Either prints the shard output line immediately or saves it in the
 466     output buffer, depending on the settings. Also optionally adds a prefix.
 467     Adds a (sys.stdout, line) or (sys.stderr, line) tuple in the output queue.
 468     """
 469     # Fix up the index.
 470     array_index = index - (self.num_shards_to_run * self.slave_index)
 471     if self.prefix:
 472       line = "%i>%s" % (index, line)
 473     if self.original_order:
 474       pipe.write(line)
 475     else:
 476       self.shard_output[array_index].put((pipe, line))
 477
 478   def IncrementTestCount(self):
 479     """Increments the number of tests run. This is relevant to the
 480     --retry-percent option.
 481     """
 482     self.test_counter.next()
 483
 484   def ShardIndexCompleted(self, index):
 485     """Records that a shard has finished so the output from the next shard
 486     can now be printed.
 487     """
 488     # Fix up the index.
 489     array_index = index - (self.num_shards_to_run * self.slave_index)
 490     self.shard_output[array_index].put((sys.stdout, self.SHARD_COMPLETED))
 491
 492   def RetryFailedTests(self):
 493     """Reruns any failed tests serially and prints another summary of the
 494     results if no more than retry_percent failed.
 495     """
 496     num_tests_run = self.test_counter.next()
 497     if len(self.failed_tests) > self.retry_percent * num_tests_run:
 498       sys.stdout.write("\nNOT RETRYING FAILED TESTS (too many failed)\n")
 499       return 1
 500     self.WriteText(sys.stdout, "\nRETRYING FAILED TESTS:\n", "\x1b[1;5;33m")
 501     sharded_description = re.compile(r": (?:\d+>)?(.*)")
 502     gtest_filters = [sharded_description.search(line).group(1)
 503                      for line in self.failed_tests]
 504     sys.stdout.write("\nRETRY GTEST FILTERS: %r\n" % gtest_filters)
 505     failed_retries = []
 506
 507     for test_filter in gtest_filters:
 508       args = [self.test, "--gtest_filter=" + test_filter]
 509       # Don't update the xml output files during retry.
 510       stripped_gtests_args = RemoveGTestOutput(self.gtest_args)
 511       args.extend(stripped_gtests_args)
 512       sys.stdout.write("\nRETRY COMMAND: %r\n" % args)
 513       rerun = subprocess.Popen(args, stdout=sys.stdout, stderr=sys.stderr)
 514       rerun.wait()
 515       if rerun.returncode != 0:
 516         failed_retries.append(test_filter)
 517
 518     self.WriteText(sys.stdout, "RETRY RESULTS:\n", "\x1b[1;5;33m")
 519     self.PrintSummary(failed_retries)
 520     return len(failed_retries) > 0
 521
 522   def PrintSummary(self, failed_tests):
 523     """Prints a summary of the test results.
 524
 525     If any shards had failing tests, the list is sorted and printed. Then all
 526     the lines that indicate a test failure are reproduced.
 527     """
 528     if failed_tests:
 529       self.WriteText(sys.stdout, "FAILED TESTS:\n", "\x1b[1;5;31m")
 530       for line in failed_tests:
 531         sys.stdout.write(line)
 532     else:
 533       self.WriteText(sys.stdout, "ALL TESTS PASSED!\n", "\x1b[1;5;32m")
 534
 535   def WriteText(self, pipe, text, ansi):
 536     """Writes the text to the pipe with the ansi escape code, if colored
 537     output is set, for Unix systems.
 538     """
 539     if self.color:
 540       pipe.write(ansi)
 541     pipe.write(text)
 542     if self.color:
 543       pipe.write("\x1b[m")
 544
 545
 546 def main():
 547   parser = optparse.OptionParser(usage=SS_USAGE)
 548   parser.add_option(
 549       "-n", "--shards_per_core", type="int", default=SS_DEFAULT_SHARDS_PER_CORE,
 550       help="number of shards to generate per CPU")
 551   parser.add_option(
 552       "-r", "--runs_per_core", type="int", default=SS_DEFAULT_RUNS_PER_CORE,
 553       help="number of shards to run in parallel per CPU")
 554   parser.add_option(
 555       "-c", "--color", action="store_true",
 556       default=sys.platform != "win32" and sys.stdout.isatty(),
 557       help="force color output, also used by gtest if --gtest_color is not"
 558       " specified")
 559   parser.add_option(
 560       "--no-color", action="store_false", dest="color",
 561       help="disable color output")
 562   parser.add_option(
 563       "-s", "--runshard", type="int", help="single shard index to run")
 564   parser.add_option(
 565       "--reorder", action="store_true",
 566       help="ensure that all output from an earlier shard is printed before"
 567       " output from a later shard")
 568   # TODO(charleslee): for backwards compatibility with master.cfg file
 569   parser.add_option(
 570       "--original-order", action="store_true",
 571       help="print shard output in its orginal jumbled order of execution"
 572       " (useful for debugging flaky tests)")
 573   parser.add_option(
 574       "--prefix", action="store_true",
 575       help="prefix each line of shard output with 'N>', where N is the shard"
 576       " index (forced True when --original-order is True)")
 577   parser.add_option(
 578       "--random-seed", action="store_true",
 579       help="shuffle the tests with a random seed value")
 580   parser.add_option(
 581       "--retry-failed", action="store_true",
 582       help="retry tests that did not pass serially")
 583   parser.add_option(
 584       "--retry-percent", type="int",
 585       default=SS_DEFAULT_RETRY_PERCENT,
 586       help="ignore --retry-failed if more than this percent fail [0, 100]"
 587       " (default = %i)" % SS_DEFAULT_RETRY_PERCENT)
 588   parser.add_option(
 589       "-t", "--timeout", type="int", default=SS_DEFAULT_TIMEOUT,
 590       help="timeout in seconds to wait for a shard (default=%default s)")
 591   parser.add_option(
 592       "--total-slaves", type="int", default=1,
 593       help="if running a subset, number of slaves sharing the test")
 594   parser.add_option(
 595       "--slave-index", type="int", default=0,
 596       help="if running a subset, index of the slave to run tests for")
 597
 598   parser.disable_interspersed_args()
 599   (options, args) = parser.parse_args()
 600
 601   if not args:
 602     parser.error("You must specify a path to test!")
 603   if not os.path.exists(args[0]):
 604     parser.error("%s does not exist!" % args[0])
 605
 606   num_cores = DetectNumCores()
 607
 608   if options.shards_per_core < 1:
 609     parser.error("You must have at least 1 shard per core!")
 610   num_shards_to_run = num_cores * options.shards_per_core
 611
 612   if options.runs_per_core < 1:
 613     parser.error("You must have at least 1 run per core!")
 614   num_runs = num_cores * options.runs_per_core
 615
 616   test = args[0]
 617   gtest_args = ["--gtest_color=%s" % {
 618       True: "yes", False: "no"}[options.color]] + args[1:]
 619
 620   if options.original_order:
 621     options.prefix = True
 622
 623   # TODO(charleslee): for backwards compatibility with buildbot's log_parser
 624   if options.reorder:
 625     options.original_order = False
 626     options.prefix = True
 627
 628   if options.random_seed:
 629     seed = random.randint(1, 99999)
 630     gtest_args.extend(["--gtest_shuffle", "--gtest_random_seed=%i" % seed])
 631
 632   if options.retry_failed:
 633     if options.retry_percent < 0 or options.retry_percent > 100:
 634       parser.error("Retry percent must be an integer [0, 100]!")
 635   else:
 636     options.retry_percent = -1
 637
 638   if options.runshard != None:
 639     # run a single shard and exit
 640     if (options.runshard < 0 or options.runshard >= num_shards_to_run):
 641       parser.error("Invalid shard number given parameters!")
 642     shard = RunShard(
 643         test, num_shards_to_run, options.runshard, gtest_args, None, None)
 644     shard.communicate()
 645     return shard.poll()
 646
 647   # When running browser_tests, load the test binary into memory before running
 648   # any tests. This is needed to prevent loading it from disk causing the first
 649   # run tests to timeout flakily. See: http://crbug.com/124260
 650   if "browser_tests" in test:
 651     args = [test]
 652     args.extend(gtest_args)
 653     args.append("--warmup")
 654     result = subprocess.call(args,
 655                              bufsize=0,
 656                              universal_newlines=True)
 657     # If the test fails, don't run anything else.
 658     if result != 0:
 659       return result
 660
 661   # shard and run the whole test
 662   ss = ShardingSupervisor(
 663       test, num_shards_to_run, num_runs, options.color,
 664       options.original_order, options.prefix, options.retry_percent,
 665       options.timeout, options.total_slaves, options.slave_index, gtest_args)
 666   return ss.ShardTest()
 667
 668
 669 if __name__ == "__main__":
 670   sys.exit(main())