build/android/pylib/host_driven/test_runner.py

   1 # Copyright 2013 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """Runs host-driven tests on a particular device."""
   6
   7 import logging
   8 import sys
   9 import time
  10 import traceback
  11
  12 from pylib.base import base_test_result
  13 from pylib.base import base_test_runner
  14 from pylib.host_driven import test_case
  15 from pylib.instrumentation import test_result
  16
  17
  18 class HostDrivenExceptionTestResult(test_result.InstrumentationTestResult):
  19   """Test result corresponding to a python exception in a host-driven test."""
  20
  21   def __init__(self, test_name, start_date_ms, exc_info):
  22     """Constructs a HostDrivenExceptionTestResult object.
  23
  24     Args:
  25       test_name: name of the test which raised an exception.
  26       start_date_ms: the starting time for the test.
  27       exc_info: exception info, ostensibly from sys.exc_info().
  28     """
  29     exc_type, exc_value, exc_traceback = exc_info
  30     trace_info = ''.join(traceback.format_exception(exc_type, exc_value,
  31                                                     exc_traceback))
  32     log_msg = 'Exception:\n' + trace_info
  33     duration_ms = (int(time.time()) * 1000) - start_date_ms
  34
  35     super(HostDrivenExceptionTestResult, self).__init__(
  36         test_name,
  37         base_test_result.ResultType.FAIL,
  38         start_date_ms,
  39         duration_ms,
  40         log=str(exc_type) + ' ' + log_msg)
  41
  42
  43 class HostDrivenTestRunner(base_test_runner.BaseTestRunner):
  44   """Orchestrates running a set of host-driven tests.
  45
  46   Any Python exceptions in the tests are caught and translated into a failed
  47   result, rather than being re-raised on the main thread.
  48   """
  49
  50   # TODO(jbudorick): Remove cleanup_test_files once it's no longer used.
  51   # pylint: disable=unused-argument
  52   #override
  53   def __init__(self, device, shard_index, tool, cleanup_test_files=None):
  54     """Creates a new HostDrivenTestRunner.
  55
  56     Args:
  57       device: Attached android device.
  58       shard_index: Shard index.
  59       tool: Name of the Valgrind tool.
  60       cleanup_test_files: Deprecated.
  61     """
  62
  63     super(HostDrivenTestRunner, self).__init__(device, tool)
  64
  65     # The shard index affords the ability to create unique port numbers (e.g.
  66     # DEFAULT_PORT + shard_index) if the test so wishes.
  67     self.shard_index = shard_index
  68
  69   # pylint: enable=unused-argument
  70
  71   #override
  72   def RunTest(self, test):
  73     """Sets up and runs a test case.
  74
  75     Args:
  76       test: An object which is ostensibly a subclass of HostDrivenTestCase.
  77
  78     Returns:
  79       A TestRunResults object which contains the result produced by the test
  80       and, in the case of a failure, the test that should be retried.
  81     """
  82
  83     assert isinstance(test, test_case.HostDrivenTestCase)
  84
  85     start_date_ms = int(time.time()) * 1000
  86     exception_raised = False
  87
  88     try:
  89       test.SetUp(self.device, self.shard_index)
  90     except Exception:
  91       logging.exception(
  92           'Caught exception while trying to run SetUp() for test: ' +
  93           test.tagged_name)
  94       # Tests whose SetUp() method has failed are likely to fail, or at least
  95       # yield invalid results.
  96       exc_info = sys.exc_info()
  97       results = base_test_result.TestRunResults()
  98       results.AddResult(HostDrivenExceptionTestResult(
  99           test.tagged_name, start_date_ms, exc_info))
 100       return results, test
 101
 102     try:
 103       results = test.Run()
 104     except Exception:
 105       # Setting this lets TearDown() avoid stomping on our stack trace from
 106       # Run() should TearDown() also raise an exception.
 107       exception_raised = True
 108       logging.exception('Caught exception while trying to run test: ' +
 109                         test.tagged_name)
 110       exc_info = sys.exc_info()
 111       results = base_test_result.TestRunResults()
 112       results.AddResult(HostDrivenExceptionTestResult(
 113           test.tagged_name, start_date_ms, exc_info))
 114
 115     try:
 116       test.TearDown()
 117     except Exception:
 118       logging.exception(
 119           'Caught exception while trying run TearDown() for test: ' +
 120           test.tagged_name)
 121       if not exception_raised:
 122         # Don't stomp the error during the test if TearDown blows up. This is a
 123         # trade-off: if the test fails, this will mask any problem with TearDown
 124         # until the test is fixed.
 125         exc_info = sys.exc_info()
 126         results = base_test_result.TestRunResults()
 127         results.AddResult(HostDrivenExceptionTestResult(
 128             test.tagged_name, start_date_ms, exc_info))
 129
 130     if not results.DidRunPass():
 131       return results, test
 132     else:
 133       return results, None