build/android/pylib/base/base_test_result.py

   1 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """Module containing base test results classes."""
   6
   7 import threading
   8
   9
  10 class ResultType(object):
  11   """Class enumerating test types."""
  12   PASS = 'PASS'
  13   SKIP = 'SKIP'
  14   FAIL = 'FAIL'
  15   CRASH = 'CRASH'
  16   TIMEOUT = 'TIMEOUT'
  17   UNKNOWN = 'UNKNOWN'
  18
  19   @staticmethod
  20   def GetTypes():
  21     """Get a list of all test types."""
  22     return [ResultType.PASS, ResultType.SKIP, ResultType.FAIL,
  23             ResultType.CRASH, ResultType.TIMEOUT, ResultType.UNKNOWN]
  24
  25
  26 class BaseTestResult(object):
  27   """Base class for a single test result."""
  28
  29   def __init__(self, name, test_type, duration=0, log=''):
  30     """Construct a BaseTestResult.
  31
  32     Args:
  33       name: Name of the test which defines uniqueness.
  34       test_type: Type of the test result as defined in ResultType.
  35       duration: Time it took for the test to run in milliseconds.
  36       log: An optional string listing any errors.
  37     """
  38     assert name
  39     assert test_type in ResultType.GetTypes()
  40     self._name = name
  41     self._test_type = test_type
  42     self._duration = duration
  43     self._log = log
  44
  45   def __str__(self):
  46     return self._name
  47
  48   def __repr__(self):
  49     return self._name
  50
  51   def __cmp__(self, other):
  52     # pylint: disable=W0212
  53     return cmp(self._name, other._name)
  54
  55   def __hash__(self):
  56     return hash(self._name)
  57
  58   def SetName(self, name):
  59     """Set the test name.
  60
  61     Because we're putting this into a set, this should only be used if moving
  62     this test result into another set.
  63     """
  64     self._name = name
  65
  66   def GetName(self):
  67     """Get the test name."""
  68     return self._name
  69
  70   def SetType(self, test_type):
  71     """Set the test result type."""
  72     assert test_type in ResultType.GetTypes()
  73     self._test_type = test_type
  74
  75   def GetType(self):
  76     """Get the test result type."""
  77     return self._test_type
  78
  79   def GetDuration(self):
  80     """Get the test duration."""
  81     return self._duration
  82
  83   def SetLog(self, log):
  84     """Set the test log."""
  85     self._log = log
  86
  87   def GetLog(self):
  88     """Get the test log."""
  89     return self._log
  90
  91
  92 class TestRunResults(object):
  93   """Set of results for a test run."""
  94
  95   def __init__(self):
  96     self._results = set()
  97     self._results_lock = threading.RLock()
  98
  99   def GetLogs(self):
 100     """Get the string representation of all test logs."""
 101     with self._results_lock:
 102       s = []
 103       for test_type in ResultType.GetTypes():
 104         if test_type != ResultType.PASS:
 105           for t in sorted(self._GetType(test_type)):
 106             log = t.GetLog()
 107             if log:
 108               s.append('[%s] %s:' % (test_type, t))
 109               s.append(log)
 110       return '\n'.join(s)
 111
 112   def GetGtestForm(self):
 113     """Get the gtest string representation of this object."""
 114     with self._results_lock:
 115       s = []
 116       plural = lambda n, s, p: '%d %s' % (n, p if n != 1 else s)
 117       tests = lambda n: plural(n, 'test', 'tests')
 118
 119       s.append('[==========] %s ran.' % (tests(len(self.GetAll()))))
 120       s.append('[  PASSED  ] %s.' % (tests(len(self.GetPass()))))
 121
 122       skipped = self.GetSkip()
 123       if skipped:
 124         s.append('[  SKIPPED ] Skipped %s, listed below:' % tests(len(skipped)))
 125         for t in sorted(skipped):
 126           s.append('[  SKIPPED ] %s' % str(t))
 127
 128       all_failures = self.GetFail().union(self.GetCrash(), self.GetTimeout(),
 129           self.GetUnknown())
 130       if all_failures:
 131         s.append('[  FAILED  ] %s, listed below:' % tests(len(all_failures)))
 132         for t in sorted(self.GetFail()):
 133           s.append('[  FAILED  ] %s' % str(t))
 134         for t in sorted(self.GetCrash()):
 135           s.append('[  FAILED  ] %s (CRASHED)' % str(t))
 136         for t in sorted(self.GetTimeout()):
 137           s.append('[  FAILED  ] %s (TIMEOUT)' % str(t))
 138         for t in sorted(self.GetUnknown()):
 139           s.append('[  FAILED  ] %s (UNKNOWN)' % str(t))
 140         s.append('')
 141         s.append(plural(len(all_failures), 'FAILED TEST', 'FAILED TESTS'))
 142       return '\n'.join(s)
 143
 144   def GetShortForm(self):
 145     """Get the short string representation of this object."""
 146     with self._results_lock:
 147       s = []
 148       s.append('ALL: %d' % len(self._results))
 149       for test_type in ResultType.GetTypes():
 150         s.append('%s: %d' % (test_type, len(self._GetType(test_type))))
 151       return ''.join([x.ljust(15) for x in s])
 152
 153   def __str__(self):
 154     return self.GetGtestForm()
 155
 156   def AddResult(self, result):
 157     """Add |result| to the set.
 158
 159     Args:
 160       result: An instance of BaseTestResult.
 161     """
 162     assert isinstance(result, BaseTestResult)
 163     with self._results_lock:
 164       self._results.add(result)
 165
 166   def AddResults(self, results):
 167     """Add |results| to the set.
 168
 169     Args:
 170       results: An iterable of BaseTestResult objects.
 171     """
 172     with self._results_lock:
 173       for t in results:
 174         self.AddResult(t)
 175
 176   def AddTestRunResults(self, results):
 177     """Add the set of test results from |results|.
 178
 179     Args:
 180       results: An instance of TestRunResults.
 181     """
 182     assert isinstance(results, TestRunResults)
 183     with self._results_lock:
 184       # pylint: disable=W0212
 185       self._results.update(results._results)
 186
 187   def GetAll(self):
 188     """Get the set of all test results."""
 189     with self._results_lock:
 190       return self._results.copy()
 191
 192   def _GetType(self, test_type):
 193     """Get the set of test results with the given test type."""
 194     with self._results_lock:
 195       return set(t for t in self._results if t.GetType() == test_type)
 196
 197   def GetPass(self):
 198     """Get the set of all passed test results."""
 199     return self._GetType(ResultType.PASS)
 200
 201   def GetSkip(self):
 202     """Get the set of all skipped test results."""
 203     return self._GetType(ResultType.SKIP)
 204
 205   def GetFail(self):
 206     """Get the set of all failed test results."""
 207     return self._GetType(ResultType.FAIL)
 208
 209   def GetCrash(self):
 210     """Get the set of all crashed test results."""
 211     return self._GetType(ResultType.CRASH)
 212
 213   def GetTimeout(self):
 214     """Get the set of all timed out test results."""
 215     return self._GetType(ResultType.TIMEOUT)
 216
 217   def GetUnknown(self):
 218     """Get the set of all unknown test results."""
 219     return self._GetType(ResultType.UNKNOWN)
 220
 221   def GetNotPass(self):
 222     """Get the set of all non-passed test results."""
 223     return self.GetAll() - self.GetPass()
 224
 225   def DidRunPass(self):
 226     """Return whether the test run was successful."""
 227     return not self.GetNotPass() - self.GetSkip()
 228