build/android/pylib/gtest/test_runner.py

   1 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import logging
   6 import os
   7 import re
   8
   9 from pylib import constants
  10 from pylib import pexpect
  11 from pylib.base import base_test_result
  12 from pylib.base import base_test_runner
  13 from pylib.device import device_errors
  14 from pylib.perf import perf_control
  15
  16
  17 def _TestSuiteRequiresMockTestServer(suite_name):
  18   """Returns True if the test suite requires mock test server."""
  19   tests_require_net_test_server = ['unit_tests', 'net_unittests',
  20                                    'content_unittests',
  21                                    'content_browsertests']
  22   return (suite_name in
  23           tests_require_net_test_server)
  24
  25 def _TestSuiteRequiresHighPerfMode(suite_name):
  26   """Returns True if the test suite requires high performance mode."""
  27   return 'perftests' in suite_name
  28
  29 class TestRunner(base_test_runner.BaseTestRunner):
  30   def __init__(self, test_options, device, test_package):
  31     """Single test suite attached to a single device.
  32
  33     Args:
  34       test_options: A GTestOptions object.
  35       device: Device to run the tests.
  36       test_package: An instance of TestPackage class.
  37     """
  38
  39     super(TestRunner, self).__init__(device, test_options.tool,
  40                                      test_options.push_deps,
  41                                      test_options.cleanup_test_files)
  42
  43     self.test_package = test_package
  44     self.test_package.tool = self.tool
  45     self._test_arguments = test_options.test_arguments
  46
  47     timeout = test_options.timeout
  48     if timeout == 0:
  49       timeout = 60
  50     # On a VM (e.g. chromium buildbots), this timeout is way too small.
  51     if os.environ.get('BUILDBOT_SLAVENAME'):
  52       timeout = timeout * 2
  53
  54     self._timeout = timeout * self.tool.GetTimeoutScale()
  55     if _TestSuiteRequiresHighPerfMode(self.test_package.suite_name):
  56       self._perf_controller = perf_control.PerfControl(self.device)
  57
  58   #override
  59   def InstallTestPackage(self):
  60     self.test_package.Install(self.device)
  61
  62   #override
  63   def PushDataDeps(self):
  64     self.device.WaitUntilFullyBooted(timeout=20)
  65     self.tool.CopyFiles()
  66     if os.path.exists(constants.ISOLATE_DEPS_DIR):
  67       # TODO(frankf): linux_dumper_unittest_helper needs to be in the same dir
  68       # as breakpad_unittests exe. Find a better way to do this.
  69       if self.test_package.suite_name == 'breakpad_unittests':
  70         device_dir = constants.TEST_EXECUTABLE_DIR
  71       else:
  72         device_dir = self.device.GetExternalStoragePath()
  73       self.device.PushChangedFiles(
  74           [(os.path.join(constants.ISOLATE_DEPS_DIR, p),
  75             os.path.join(device_dir, p))
  76            for p in os.listdir(constants.ISOLATE_DEPS_DIR)])
  77
  78   def _ParseTestOutput(self, p):
  79     """Process the test output.
  80
  81     Args:
  82       p: An instance of pexpect spawn class.
  83
  84     Returns:
  85       A TestRunResults object.
  86     """
  87     results = base_test_result.TestRunResults()
  88
  89     # Test case statuses.
  90     re_run = re.compile('\[ RUN      \] ?(.*)\r\n')
  91     re_fail = re.compile('\[  FAILED  \] ?(.*)\r\n')
  92     re_ok = re.compile('\[       OK \] ?(.*?) .*\r\n')
  93
  94     # Test run statuses.
  95     re_passed = re.compile('\[  PASSED  \] ?(.*)\r\n')
  96     re_runner_fail = re.compile('\[ RUNNER_FAILED \] ?(.*)\r\n')
  97     # Signal handlers are installed before starting tests
  98     # to output the CRASHED marker when a crash happens.
  99     re_crash = re.compile('\[ CRASHED      \](.*)\r\n')
 100
 101     log = ''
 102     try:
 103       while True:
 104         full_test_name = None
 105         found = p.expect([re_run, re_passed, re_runner_fail],
 106                          timeout=self._timeout)
 107         if found == 1:  # re_passed
 108           break
 109         elif found == 2:  # re_runner_fail
 110           break
 111         else:  # re_run
 112           full_test_name = p.match.group(1).replace('\r', '')
 113           found = p.expect([re_ok, re_fail, re_crash], timeout=self._timeout)
 114           log = p.before.replace('\r', '')
 115           if found == 0:  # re_ok
 116             if full_test_name == p.match.group(1).replace('\r', ''):
 117               results.AddResult(base_test_result.BaseTestResult(
 118                   full_test_name, base_test_result.ResultType.PASS,
 119                   log=log))
 120           elif found == 2:  # re_crash
 121             results.AddResult(base_test_result.BaseTestResult(
 122                 full_test_name, base_test_result.ResultType.CRASH,
 123                 log=log))
 124             break
 125           else:  # re_fail
 126             results.AddResult(base_test_result.BaseTestResult(
 127                 full_test_name, base_test_result.ResultType.FAIL, log=log))
 128     except pexpect.EOF:
 129       logging.error('Test terminated - EOF')
 130       # We're here because either the device went offline, or the test harness
 131       # crashed without outputting the CRASHED marker (crbug.com/175538).
 132       if not self.device.IsOnline():
 133         raise device_errors.DeviceUnreachableError(
 134             'Device %s went offline.' % str(self.device))
 135       if full_test_name:
 136         results.AddResult(base_test_result.BaseTestResult(
 137             full_test_name, base_test_result.ResultType.CRASH,
 138             log=p.before.replace('\r', '')))
 139     except pexpect.TIMEOUT:
 140       logging.error('Test terminated after %d second timeout.',
 141                     self._timeout)
 142       if full_test_name:
 143         results.AddResult(base_test_result.BaseTestResult(
 144             full_test_name, base_test_result.ResultType.TIMEOUT,
 145             log=p.before.replace('\r', '')))
 146     finally:
 147       p.close()
 148
 149     ret_code = self.test_package.GetGTestReturnCode(self.device)
 150     if ret_code:
 151       logging.critical(
 152           'gtest exit code: %d\npexpect.before: %s\npexpect.after: %s',
 153           ret_code, p.before, p.after)
 154
 155     return results
 156
 157   #override
 158   def RunTest(self, test):
 159     test_results = base_test_result.TestRunResults()
 160     if not test:
 161       return test_results, None
 162
 163     try:
 164       self.test_package.ClearApplicationState(self.device)
 165       self.test_package.CreateCommandLineFileOnDevice(
 166           self.device, test, self._test_arguments)
 167       test_results = self._ParseTestOutput(
 168           self.test_package.SpawnTestProcess(self.device))
 169     finally:
 170       self.CleanupSpawningServerState()
 171     # Calculate unknown test results.
 172     all_tests = set(test.split(':'))
 173     all_tests_ran = set([t.GetName() for t in test_results.GetAll()])
 174     unknown_tests = all_tests - all_tests_ran
 175     test_results.AddResults(
 176         [base_test_result.BaseTestResult(t, base_test_result.ResultType.UNKNOWN)
 177          for t in unknown_tests])
 178     retry = ':'.join([t.GetName() for t in test_results.GetNotPass()])
 179     return test_results, retry
 180
 181   #override
 182   def SetUp(self):
 183     """Sets up necessary test enviroment for the test suite."""
 184     super(TestRunner, self).SetUp()
 185     if _TestSuiteRequiresMockTestServer(self.test_package.suite_name):
 186       self.LaunchChromeTestServerSpawner()
 187     if _TestSuiteRequiresHighPerfMode(self.test_package.suite_name):
 188       self._perf_controller.SetHighPerfMode()
 189     self.tool.SetupEnvironment()
 190
 191   #override
 192   def TearDown(self):
 193     """Cleans up the test enviroment for the test suite."""
 194     if _TestSuiteRequiresHighPerfMode(self.test_package.suite_name):
 195       self._perf_controller.SetDefaultPerfMode()
 196     self.test_package.ClearApplicationState(self.device)
 197     self.tool.CleanUpEnvironment()
 198     super(TestRunner, self).TearDown()