build/android/pylib/host_driven/test_runner.py

   1 # Copyright 2013 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """Runs host-driven tests on a particular device."""
   6
   7 import logging
   8 import sys
   9 import time
  10 import traceback
  11
  12 from pylib.base import base_test_result
  13 from pylib.base import base_test_runner
  14 from pylib.host_driven import test_case
  15 from pylib.instrumentation import test_result
  16
  17
  18 class HostDrivenExceptionTestResult(test_result.InstrumentationTestResult):
  19   """Test result corresponding to a python exception in a host-driven test."""
  20
  21   def __init__(self, test_name, start_date_ms, exc_info):
  22     """Constructs a HostDrivenExceptionTestResult object.
  23
  24     Args:
  25       test_name: name of the test which raised an exception.
  26       start_date_ms: the starting time for the test.
  27       exc_info: exception info, ostensibly from sys.exc_info().
  28     """
  29     exc_type, exc_value, exc_traceback = exc_info
  30     trace_info = ''.join(traceback.format_exception(exc_type, exc_value,
  31                                                     exc_traceback))
  32     log_msg = 'Exception:\n' + trace_info
  33     duration_ms = (int(time.time()) * 1000) - start_date_ms
  34
  35     super(HostDrivenExceptionTestResult, self).__init__(
  36         test_name,
  37         base_test_result.ResultType.FAIL,
  38         start_date_ms,
  39         duration_ms,
  40         log=str(exc_type) + ' ' + log_msg)
  41
  42
  43 class HostDrivenTestRunner(base_test_runner.BaseTestRunner):
  44   """Orchestrates running a set of host-driven tests.
  45
  46   Any Python exceptions in the tests are caught and translated into a failed
  47   result, rather than being re-raised on the main thread.
  48   """
  49
  50   #override
  51   def __init__(self, device, shard_index, tool, cleanup_test_files):
  52     """Creates a new HostDrivenTestRunner.
  53
  54     Args:
  55       device: Attached android device.
  56       shard_index: Shard index.
  57       tool: Name of the Valgrind tool.
  58       cleanup_test_files: Whether or not to cleanup test files on device.
  59     """
  60
  61     super(HostDrivenTestRunner, self).__init__(device, tool, cleanup_test_files)
  62
  63     # The shard index affords the ability to create unique port numbers (e.g.
  64     # DEFAULT_PORT + shard_index) if the test so wishes.
  65     self.shard_index = shard_index
  66
  67   #override
  68   def RunTest(self, test):
  69     """Sets up and runs a test case.
  70
  71     Args:
  72       test: An object which is ostensibly a subclass of HostDrivenTestCase.
  73
  74     Returns:
  75       A TestRunResults object which contains the result produced by the test
  76       and, in the case of a failure, the test that should be retried.
  77     """
  78
  79     assert isinstance(test, test_case.HostDrivenTestCase)
  80
  81     start_date_ms = int(time.time()) * 1000
  82     exception_raised = False
  83
  84     try:
  85       test.SetUp(str(self.device), self.shard_index, self._cleanup_test_files)
  86     except Exception:
  87       logging.exception(
  88           'Caught exception while trying to run SetUp() for test: ' +
  89           test.tagged_name)
  90       # Tests whose SetUp() method has failed are likely to fail, or at least
  91       # yield invalid results.
  92       exc_info = sys.exc_info()
  93       results = base_test_result.TestRunResults()
  94       results.AddResult(HostDrivenExceptionTestResult(
  95           test.tagged_name, start_date_ms, exc_info))
  96       return results, test
  97
  98     try:
  99       results = test.Run()
 100     except Exception:
 101       # Setting this lets TearDown() avoid stomping on our stack trace from
 102       # Run() should TearDown() also raise an exception.
 103       exception_raised = True
 104       logging.exception('Caught exception while trying to run test: ' +
 105                         test.tagged_name)
 106       exc_info = sys.exc_info()
 107       results = base_test_result.TestRunResults()
 108       results.AddResult(HostDrivenExceptionTestResult(
 109           test.tagged_name, start_date_ms, exc_info))
 110
 111     try:
 112       test.TearDown()
 113     except Exception:
 114       logging.exception(
 115           'Caught exception while trying run TearDown() for test: ' +
 116           test.tagged_name)
 117       if not exception_raised:
 118         # Don't stomp the error during the test if TearDown blows up. This is a
 119         # trade-off: if the test fails, this will mask any problem with TearDown
 120         # until the test is fixed.
 121         exc_info = sys.exc_info()
 122         results = base_test_result.TestRunResults()
 123         results.AddResult(HostDrivenExceptionTestResult(
 124             test.tagged_name, start_date_ms, exc_info))
 125
 126     if not results.DidRunPass():
 127       return results, test
 128     else:
 129       return results, None