build/android/pylib/gtest/test_runner.py

   1 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import logging
   6 import os
   7 import re
   8
   9 from pylib import pexpect
  10 from pylib import ports
  11 from pylib.base import base_test_result
  12 from pylib.base import base_test_runner
  13 from pylib.device import device_errors
  14 from pylib.local import local_test_server_spawner
  15 from pylib.perf import perf_control
  16
  17 # Test case statuses.
  18 RE_RUN = re.compile('\\[ RUN      \\] ?(.*)\r\n')
  19 RE_FAIL = re.compile('\\[  FAILED  \\] ?(.*?)( \\((\\d+) ms\\))?\r\r\n')
  20 RE_OK = re.compile('\\[       OK \\] ?(.*?)( \\((\\d+) ms\\))?\r\r\n')
  21
  22 # Test run statuses.
  23 RE_PASSED = re.compile('\\[  PASSED  \\] ?(.*)\r\n')
  24 RE_RUNNER_FAIL = re.compile('\\[ RUNNER_FAILED \\] ?(.*)\r\n')
  25 # Signal handlers are installed before starting tests
  26 # to output the CRASHED marker when a crash happens.
  27 RE_CRASH = re.compile('\\[ CRASHED      \\](.*)\r\n')
  28
  29 # Bots that don't output anything for 20 minutes get timed out, so that's our
  30 # hard cap.
  31 _INFRA_STDOUT_TIMEOUT = 20 * 60
  32
  33
  34 def _TestSuiteRequiresMockTestServer(suite_name):
  35   """Returns True if the test suite requires mock test server."""
  36   tests_require_net_test_server = ['unit_tests', 'net_unittests',
  37                                    'components_browsertests',
  38                                    'content_unittests',
  39                                    'content_browsertests']
  40   return (suite_name in
  41           tests_require_net_test_server)
  42
  43 def _TestSuiteRequiresHighPerfMode(suite_name):
  44   """Returns True if the test suite requires high performance mode."""
  45   return 'perftests' in suite_name
  46
  47 class TestRunner(base_test_runner.BaseTestRunner):
  48   def __init__(self, test_options, device, test_package):
  49     """Single test suite attached to a single device.
  50
  51     Args:
  52       test_options: A GTestOptions object.
  53       device: Device to run the tests.
  54       test_package: An instance of TestPackage class.
  55     """
  56
  57     super(TestRunner, self).__init__(device, test_options.tool)
  58
  59     self.test_package = test_package
  60     self.test_package.tool = self.tool
  61     self._test_arguments = test_options.test_arguments
  62
  63     timeout = test_options.timeout
  64     if timeout == 0:
  65       timeout = 60
  66     # On a VM (e.g. chromium buildbots), this timeout is way too small.
  67     if os.environ.get('BUILDBOT_SLAVENAME'):
  68       timeout = timeout * 2
  69
  70     self._timeout = min(timeout * self.tool.GetTimeoutScale(),
  71                         _INFRA_STDOUT_TIMEOUT)
  72     if _TestSuiteRequiresHighPerfMode(self.test_package.suite_name):
  73       self._perf_controller = perf_control.PerfControl(self.device)
  74
  75     if _TestSuiteRequiresMockTestServer(self.test_package.suite_name):
  76       self._servers = [
  77           local_test_server_spawner.LocalTestServerSpawner(
  78               ports.AllocateTestServerPort(), self.device, self.tool)]
  79     else:
  80       self._servers = []
  81
  82   #override
  83   def InstallTestPackage(self):
  84     self.test_package.Install(self.device)
  85
  86   def _ParseTestOutput(self, p):
  87     """Process the test output.
  88
  89     Args:
  90       p: An instance of pexpect spawn class.
  91
  92     Returns:
  93       A TestRunResults object.
  94     """
  95     results = base_test_result.TestRunResults()
  96
  97     log = ''
  98     try:
  99       while True:
 100         full_test_name = None
 101
 102         found = p.expect([RE_RUN, RE_PASSED, RE_RUNNER_FAIL],
 103                          timeout=self._timeout)
 104         if found == 1:  # RE_PASSED
 105           break
 106         elif found == 2:  # RE_RUNNER_FAIL
 107           break
 108         else:  # RE_RUN
 109           full_test_name = p.match.group(1).replace('\r', '')
 110           found = p.expect([RE_OK, RE_FAIL, RE_CRASH], timeout=self._timeout)
 111           log = p.before.replace('\r', '')
 112           if found == 0:  # RE_OK
 113             if full_test_name == p.match.group(1).replace('\r', ''):
 114               duration_ms = int(p.match.group(3)) if p.match.group(3) else 0
 115               results.AddResult(base_test_result.BaseTestResult(
 116                   full_test_name, base_test_result.ResultType.PASS,
 117                   duration=duration_ms, log=log))
 118           elif found == 2:  # RE_CRASH
 119             results.AddResult(base_test_result.BaseTestResult(
 120                 full_test_name, base_test_result.ResultType.CRASH,
 121                 log=log))
 122             break
 123           else:  # RE_FAIL
 124             duration_ms = int(p.match.group(3)) if p.match.group(3) else 0
 125             results.AddResult(base_test_result.BaseTestResult(
 126                 full_test_name, base_test_result.ResultType.FAIL,
 127                 duration=duration_ms, log=log))
 128     except pexpect.EOF:
 129       logging.error('Test terminated - EOF')
 130       # We're here because either the device went offline, or the test harness
 131       # crashed without outputting the CRASHED marker (crbug.com/175538).
 132       if not self.device.IsOnline():
 133         raise device_errors.DeviceUnreachableError(
 134             'Device %s went offline.' % str(self.device))
 135       if full_test_name:
 136         results.AddResult(base_test_result.BaseTestResult(
 137             full_test_name, base_test_result.ResultType.CRASH,
 138             log=p.before.replace('\r', '')))
 139     except pexpect.TIMEOUT:
 140       logging.error('Test terminated after %d second timeout.',
 141                     self._timeout)
 142       if full_test_name:
 143         results.AddResult(base_test_result.BaseTestResult(
 144             full_test_name, base_test_result.ResultType.TIMEOUT,
 145             log=p.before.replace('\r', '')))
 146     finally:
 147       p.close()
 148
 149     ret_code = self.test_package.GetGTestReturnCode(self.device)
 150     if ret_code:
 151       logging.critical(
 152           'gtest exit code: %d\npexpect.before: %s\npexpect.after: %s',
 153           ret_code, p.before, p.after)
 154
 155     return results
 156
 157   #override
 158   def RunTest(self, test):
 159     test_results = base_test_result.TestRunResults()
 160     if not test:
 161       return test_results, None
 162
 163     try:
 164       self.test_package.ClearApplicationState(self.device)
 165       self.test_package.CreateCommandLineFileOnDevice(
 166           self.device, test, self._test_arguments)
 167       test_results = self._ParseTestOutput(
 168           self.test_package.SpawnTestProcess(self.device))
 169     finally:
 170       for s in self._servers:
 171         s.Reset()
 172     # Calculate unknown test results.
 173     all_tests = set(test.split(':'))
 174     all_tests_ran = set([t.GetName() for t in test_results.GetAll()])
 175     unknown_tests = all_tests - all_tests_ran
 176     test_results.AddResults(
 177         [base_test_result.BaseTestResult(t, base_test_result.ResultType.UNKNOWN)
 178          for t in unknown_tests])
 179     retry = ':'.join([t.GetName() for t in test_results.GetNotPass()])
 180     return test_results, retry
 181
 182   #override
 183   def SetUp(self):
 184     """Sets up necessary test enviroment for the test suite."""
 185     super(TestRunner, self).SetUp()
 186     for s in self._servers:
 187       s.SetUp()
 188     if _TestSuiteRequiresHighPerfMode(self.test_package.suite_name):
 189       self._perf_controller.SetHighPerfMode()
 190     self.tool.SetupEnvironment()
 191
 192   #override
 193   def TearDown(self):
 194     """Cleans up the test enviroment for the test suite."""
 195     for s in self._servers:
 196       s.TearDown()
 197     if _TestSuiteRequiresHighPerfMode(self.test_package.suite_name):
 198       self._perf_controller.SetDefaultPerfMode()
 199     self.test_package.ClearApplicationState(self.device)
 200     self.tool.CleanUpEnvironment()
 201     super(TestRunner, self).TearDown()