framework/results.py

   1 # Permission is hereby granted, free of charge, to any person
   2 # obtaining a copy of this software and associated documentation
   3 # files (the "Software"), to deal in the Software without
   4 # restriction, including without limitation the rights to use,
   5 # copy, modify, merge, publish, distribute, sublicense, and/or
   6 # sell copies of the Software, and to permit persons to whom the
   7 # Software is furnished to do so, subject to the following
   8 # conditions:
   9 #
  10 # This permission notice shall be included in all copies or
  11 # substantial portions of the Software.
  12 #
  13 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
  14 # KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
  15 # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
  16 # PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE AUTHOR(S) BE
  17 # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
  18 # AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
  19 # OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  20 # DEALINGS IN THE SOFTWARE.
  21
  22 """ Module for results generation """
  23
  24 from __future__ import (
  25     absolute_import, division, print_function, unicode_literals
  26 )
  27 import collections
  28 import copy
  29 import datetime
  30
  31 import six
  32
  33 from framework import status, exceptions, grouptools, compat
  34
  35 __all__ = [
  36     'TestrunResult',
  37     'TestResult',
  38 ]
  39
  40
  41 class Subtests(collections.MutableMapping):
  42     """A dict-like object that stores Statuses as values."""
  43     def __init__(self, dict_=None):
  44         self.__container = {}
  45
  46         if dict_ is not None:
  47             self.update(dict_)
  48
  49     def __setitem__(self, name, value):
  50         self.__container[name] = status.status_lookup(value)
  51
  52     def __getitem__(self, name):
  53         return self.__container[name]
  54
  55     def __delitem__(self, name):
  56         del self.__container[name]
  57
  58     def __iter__(self):
  59         return iter(self.__container)
  60
  61     def __len__(self):
  62         return len(self.__container)
  63
  64     def __repr__(self):
  65         return repr(self.__container)
  66
  67     def to_json(self):
  68         res = dict(self)
  69         res['__type__'] = 'Subtests'
  70         return res
  71
  72     @classmethod
  73     def from_dict(cls, dict_):
  74         if '__type__' in dict_:
  75             del dict_['__type__']
  76
  77         res = cls(dict_)
  78
  79         return res
  80
  81
  82 class StringDescriptor(object):  # pylint: disable=too-few-public-methods
  83     """A Shared data descriptor class for TestResult.
  84
  85     This provides a property that can be passed a str or unicode, but always
  86     returns a unicode object.
  87
  88     """
  89     def __init__(self, name, default=six.text_type()):
  90         assert isinstance(default, six.text_type)
  91         self.__name = name
  92         self.__default = default
  93
  94     def __get__(self, instance, cls):
  95         return getattr(instance, self.__name, self.__default)
  96
  97     def __set__(self, instance, value):
  98         if isinstance(value, six.binary_type):
  99             setattr(instance, self.__name, value.decode('utf-8', 'replace'))
 100         elif isinstance(value, six.text_type):
 101             setattr(instance, self.__name, value)
 102         else:
 103             raise TypeError('{} attribute must be a unicode or bytes instance, '
 104                             'but was {}.'.format(self.__name, type(value)))
 105
 106     def __delete__(self, instance):
 107         raise NotImplementedError
 108
 109
 110 class TimeAttribute(object):
 111     """Attribute of TestResult for time.
 112
 113     This attribute provides a couple of nice helpers. It stores the start and
 114     end time and provides methods for getting the total and delta of the times.
 115
 116     """
 117     __slots__ = ['start', 'end']
 118
 119     def __init__(self, start=0.0, end=0.0):
 120         self.start = start
 121         self.end = end
 122
 123     @property
 124     def total(self):
 125         return self.end - self.start
 126
 127     @property
 128     def delta(self):
 129         return str(datetime.timedelta(seconds=self.total))
 130
 131     def to_json(self):
 132         return {
 133             'start': self.start,
 134             'end': self.end,
 135             '__type__': 'TimeAttribute',
 136         }
 137
 138     @classmethod
 139     def from_dict(cls, dict_):
 140         dict_ = copy.copy(dict_)
 141
 142         if '__type__' in dict_:
 143             del dict_['__type__']
 144         return cls(**dict_)
 145
 146
 147 class TestResult(object):
 148     """An object represting the result of a single test."""
 149     __slots__ = ['returncode', '_err', '_out', 'time', 'command', 'traceback',
 150                  'environment', 'subtests', 'dmesg', '__result', 'images',
 151                  'exception', 'pid']
 152     err = StringDescriptor('_err')
 153     out = StringDescriptor('_out')
 154
 155     def __init__(self, result=None):
 156         self.returncode = None
 157         self.time = TimeAttribute()
 158         self.command = str()
 159         self.environment = str()
 160         self.subtests = Subtests()
 161         self.dmesg = str()
 162         self.images = None
 163         self.traceback = None
 164         self.exception = None
 165         self.pid = None
 166         if result:
 167             self.result = result
 168         else:
 169             self.__result = status.NOTRUN
 170
 171     @property
 172     def result(self):
 173         """Return the result of the test.
 174
 175         If there are subtests return the "worst" value of those subtests. If
 176         there are not return the stored value of the test. There is an
 177         exception to this rule, and that's if the status is crash; since this
 178         status is set by the framework, and can be generated even when some or
 179         all unit tests pass.
 180
 181         """
 182         if self.subtests and self.__result != status.CRASH:
 183             return max(six.itervalues(self.subtests))
 184         return self.__result
 185
 186     @result.setter
 187     def result(self, new):
 188         try:
 189             self.__result = status.status_lookup(new)
 190         except exceptions.PiglitInternalError as e:
 191             raise exceptions.PiglitFatalError(str(e))
 192
 193     def to_json(self):
 194         """Return the TestResult as a json serializable object."""
 195         obj = {
 196             '__type__': 'TestResult',
 197             'command': self.command,
 198             'environment': self.environment,
 199             'err': self.err,
 200             'out': self.out,
 201             'result': self.result,
 202             'returncode': self.returncode,
 203             'subtests': self.subtests,
 204             'time': self.time,
 205             'exception': self.exception,
 206             'traceback': self.traceback,
 207             'dmesg': self.dmesg,
 208             'pid': self.pid,
 209         }
 210         return obj
 211
 212     @classmethod
 213     def from_dict(cls, dict_):
 214         """Load an already generated result in dictionary form.
 215
 216         This is used as an alternate constructor which converts an existing
 217         dictionary into a TestResult object. It converts a key 'result' into a
 218         status.Status object
 219
 220         """
 221         # pylint will say that assining to inst.out or inst.err is a non-slot
 222         # because self.err and self.out are descriptors, methods that act like
 223         # variables. Just silence pylint
 224         # pylint: disable=assigning-non-slot
 225         inst = cls()
 226
 227         for each in ['returncode', 'command', 'exception', 'environment',
 228                      'time', 'traceback', 'result', 'dmesg', 'pid']:
 229             if each in dict_:
 230                 setattr(inst, each, dict_[each])
 231
 232         if 'subtests' in dict_:
 233             for name, value in six.iteritems(dict_['subtests']):
 234                 inst.subtests[name] = value
 235
 236         # out and err must be set manually to avoid replacing the setter
 237         if 'out' in dict_:
 238             inst.out = dict_['out']
 239
 240         if 'err' in dict_:
 241             inst.err = dict_['err']
 242
 243         return inst
 244
 245     def update(self, dict_):
 246         """Update the results and subtests fields from a piglit test.
 247
 248         Native piglit tests output their data as valid json, and piglit uses
 249         the json module to parse this data. This method consumes that raw
 250         dictionary data and updates itself.
 251
 252         """
 253         if 'result' in dict_:
 254             self.result = dict_['result']
 255         elif 'subtest' in dict_:
 256             self.subtests.update(dict_['subtest'])
 257
 258
 259 @compat.python_2_bool_compatible
 260 class Totals(dict):
 261     def __init__(self, *args, **kwargs):
 262         super(Totals, self).__init__(*args, **kwargs)
 263         for each in status.ALL:
 264             self[str(each)] = 0
 265
 266     def __bool__(self):
 267         # Since totals are prepopulated, calling 'if not <Totals instance>'
 268         # will always result in True, this will cause it to return True only if
 269         # one of the values is not zero
 270         for each in six.itervalues(self):
 271             if each != 0:
 272                 return True
 273         return False
 274
 275     def to_json(self):
 276         """Convert totals to a json object."""
 277         result = copy.copy(self)
 278         result['__type__'] = 'Totals'
 279         return result
 280
 281     @classmethod
 282     def from_dict(cls, dict_):
 283         """Convert a dictionary into a Totals object."""
 284         return cls(dict_)
 285
 286
 287 class TestrunResult(object):
 288     """The result of a single piglit run."""
 289     def __init__(self):
 290         self.name = None
 291         self.uname = None
 292         self.options = None
 293         self.glxinfo = None
 294         self.wglinfo = None
 295         self.clinfo = None
 296         self.lspci = None
 297         self.time_elapsed = TimeAttribute()
 298         self.tests = collections.OrderedDict()
 299         self.totals = collections.defaultdict(Totals)
 300
 301     def get_result(self, key):
 302         """Get the result of a test or subtest.
 303
 304         If neither a test nor a subtest instance exist, then raise the original
 305         KeyError generated from looking up <key> in the tests attribute. It is
 306         the job of the caller to handle this error.
 307
 308         Arguments:
 309         key -- the key name of the test to return
 310
 311         """
 312         try:
 313             return self.tests[key].result
 314         except KeyError as e:
 315             name, test = grouptools.splitname(key)
 316             try:
 317                 return self.tests[name].subtests[test]
 318             except KeyError:
 319                 raise e
 320
 321     def calculate_group_totals(self):
 322         """Calculate the number of pases, fails, etc at each level."""
 323         for name, result in six.iteritems(self.tests):
 324             # If there are subtests treat the test as if it is a group instead
 325             # of a test.
 326             if result.subtests:
 327                 for res in six.itervalues(result.subtests):
 328                     res = str(res)
 329                     temp = name
 330
 331                     self.totals[temp][res] += 1
 332                     while temp:
 333                         temp = grouptools.groupname(temp)
 334                         self.totals[temp][res] += 1
 335                     self.totals['root'][res] += 1
 336             else:
 337                 res = str(result.result)
 338                 while name:
 339                     name = grouptools.groupname(name)
 340                     self.totals[name][res] += 1
 341                 self.totals['root'][res] += 1
 342
 343     def to_json(self):
 344         if not self.totals:
 345             self.calculate_group_totals()
 346         rep = copy.copy(self.__dict__)
 347         rep['__type__'] = 'TestrunResult'
 348         return rep
 349
 350     @classmethod
 351     def from_dict(cls, dict_, _no_totals=False):
 352         """Convert a dictionary into a TestrunResult.
 353
 354         This method is meant to be used for loading results from json or
 355         similar formats
 356
 357         _no_totals is not meant to be used externally, it allows us to control
 358         the generation of totals when loading old results formats.
 359
 360         """
 361         res = cls()
 362         for name in ['name', 'uname', 'options', 'glxinfo', 'wglinfo', 'lspci',
 363                      'time_elapsed', 'tests', 'totals', 'results_version',
 364                      'clinfo']:
 365             value = dict_.get(name)
 366             if value:
 367                 setattr(res, name, value)
 368
 369         if not res.totals and not _no_totals:
 370             res.calculate_group_totals()
 371
 372         return res