framework/results.py

   1 # Permission is hereby granted, free of charge, to any person
   2 # obtaining a copy of this software and associated documentation
   3 # files (the "Software"), to deal in the Software without
   4 # restriction, including without limitation the rights to use,
   5 # copy, modify, merge, publish, distribute, sublicense, and/or
   6 # sell copies of the Software, and to permit persons to whom the
   7 # Software is furnished to do so, subject to the following
   8 # conditions:
   9 #
  10 # This permission notice shall be included in all copies or
  11 # substantial portions of the Software.
  12 #
  13 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
  14 # KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
  15 # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
  16 # PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE AUTHOR(S) BE
  17 # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
  18 # AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
  19 # OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  20 # DEALINGS IN THE SOFTWARE.
  21
  22 """ Module for results generation """
  23
  24 from __future__ import (
  25     absolute_import, division, print_function, unicode_literals
  26 )
  27 import collections
  28 import copy
  29 import datetime
  30
  31 import six
  32
  33 from framework import status, exceptions, grouptools, compat
  34
  35 __all__ = [
  36     'TestrunResult',
  37     'TestResult',
  38 ]
  39
  40
  41 class Subtests(collections.MutableMapping):
  42     """A dict-like object that stores Statuses as values."""
  43     def __init__(self, dict_=None):
  44         self.__container = collections.OrderedDict()
  45
  46         if dict_ is not None:
  47             self.update(dict_)
  48
  49     def __setitem__(self, name, value):
  50         self.__container[name.lower()] = status.status_lookup(value)
  51
  52     def __getitem__(self, name):
  53         return self.__container[name.lower()]
  54
  55     def __delitem__(self, name):
  56         del self.__container[name.lower()]
  57
  58     def __iter__(self):
  59         return iter(self.__container)
  60
  61     def __len__(self):
  62         return len(self.__container)
  63
  64     def __repr__(self):
  65         return repr(self.__container)
  66
  67     def to_json(self):
  68         res = dict(self)
  69         res['__type__'] = 'Subtests'
  70         return res
  71
  72     @classmethod
  73     def from_dict(cls, dict_):
  74         if '__type__' in dict_:
  75             del dict_['__type__']
  76
  77         res = cls(dict_)
  78
  79         return res
  80
  81
  82 class StringDescriptor(object):  # pylint: disable=too-few-public-methods
  83     """A Shared data descriptor class for TestResult.
  84
  85     This provides a property that can be passed a str or unicode, but always
  86     returns a unicode object.
  87
  88     """
  89     def __init__(self, name, default=six.text_type()):
  90         assert isinstance(default, six.text_type)
  91         self.__name = name
  92         self.__default = default
  93
  94     def __get__(self, instance, cls):
  95         return getattr(instance, self.__name, self.__default)
  96
  97     def __set__(self, instance, value):
  98         if isinstance(value, six.binary_type):
  99             setattr(instance, self.__name, value.decode('utf-8', 'replace'))
 100         elif isinstance(value, six.text_type):
 101             setattr(instance, self.__name, value)
 102         else:
 103             raise TypeError('{} attribute must be a unicode or bytes instance, '
 104                             'but was {}.'.format(self.__name, type(value)))
 105
 106     def __delete__(self, instance):
 107         raise NotImplementedError
 108
 109
 110 class TimeAttribute(object):
 111     """Attribute of TestResult for time.
 112
 113     This attribute provides a couple of nice helpers. It stores the start and
 114     end time and provides methods for getting the total and delta of the times.
 115
 116     """
 117     __slots__ = ['start', 'end']
 118
 119     def __init__(self, start=0.0, end=0.0):
 120         self.start = start
 121         self.end = end
 122
 123     @property
 124     def total(self):
 125         return self.end - self.start
 126
 127     @property
 128     def delta(self):
 129         return str(datetime.timedelta(seconds=self.total))
 130
 131     def to_json(self):
 132         return {
 133             'start': self.start,
 134             'end': self.end,
 135             '__type__': 'TimeAttribute',
 136         }
 137
 138     @classmethod
 139     def from_dict(cls, dict_):
 140         dict_ = copy.copy(dict_)
 141
 142         if '__type__' in dict_:
 143             del dict_['__type__']
 144         return cls(**dict_)
 145
 146
 147 class TestResult(object):
 148     """An object represting the result of a single test."""
 149     __slots__ = ['returncode', '_err', '_out', 'time', 'command', 'traceback',
 150                  'environment', 'subtests', 'dmesg', '__result', 'images',
 151                  'exception', 'pid']
 152     err = StringDescriptor('_err')
 153     out = StringDescriptor('_out')
 154
 155     def __init__(self, result=None):
 156         self.returncode = None
 157         self.time = TimeAttribute()
 158         self.command = str()
 159         self.environment = str()
 160         self.subtests = Subtests()
 161         self.dmesg = str()
 162         self.images = None
 163         self.traceback = None
 164         self.exception = None
 165         self.pid = []
 166         if result:
 167             self.result = result
 168         else:
 169             self.__result = status.NOTRUN
 170
 171     @property
 172     def result(self):
 173         """Return the result of the test.
 174
 175         If there are subtests return the "worst" value of those subtests. If
 176         there are not return the stored value of the test. There is an
 177         exception to this rule, and that's if the status is crash; since this
 178         status is set by the framework, and can be generated even when some or
 179         all unit tests pass.
 180
 181         """
 182         if self.subtests and self.__result != status.CRASH:
 183             return max(six.itervalues(self.subtests))
 184         return self.__result
 185
 186     @property
 187     def raw_result(self):
 188         """Get the result of the test without taking subtests into account."""
 189         return self.__result
 190
 191     @result.setter
 192     def result(self, new):
 193         try:
 194             self.__result = status.status_lookup(new)
 195         except exceptions.PiglitInternalError as e:
 196             raise exceptions.PiglitFatalError(str(e))
 197
 198     def to_json(self):
 199         """Return the TestResult as a json serializable object."""
 200         obj = {
 201             '__type__': 'TestResult',
 202             'command': self.command,
 203             'environment': self.environment,
 204             'err': self.err,
 205             'out': self.out,
 206             'result': self.result,
 207             'returncode': self.returncode,
 208             'subtests': self.subtests.to_json(),
 209             'time': self.time.to_json(),
 210             'exception': self.exception,
 211             'traceback': self.traceback,
 212             'dmesg': self.dmesg,
 213             'pid': self.pid,
 214         }
 215         return obj
 216
 217     @classmethod
 218     def from_dict(cls, dict_):
 219         """Load an already generated result in dictionary form.
 220
 221         This is used as an alternate constructor which converts an existing
 222         dictionary into a TestResult object. It converts a key 'result' into a
 223         status.Status object
 224
 225         """
 226         # pylint will say that assining to inst.out or inst.err is a non-slot
 227         # because self.err and self.out are descriptors, methods that act like
 228         # variables. Just silence pylint
 229         # pylint: disable=assigning-non-slot
 230         inst = cls()
 231
 232         for each in ['returncode', 'command', 'exception', 'environment',
 233                      'traceback', 'dmesg', 'pid', 'result']:
 234             if each in dict_:
 235                 setattr(inst, each, dict_[each])
 236
 237         # Set special instances
 238         if 'subtests' in dict_:
 239             inst.subtests = Subtests.from_dict(dict_['subtests'])
 240         if 'time' in dict_:
 241             inst.time = TimeAttribute.from_dict(dict_['time'])
 242
 243         # out and err must be set manually to avoid replacing the setter
 244         if 'out' in dict_:
 245             inst.out = dict_['out']
 246         if 'err' in dict_:
 247             inst.err = dict_['err']
 248
 249         return inst
 250
 251     def update(self, dict_):
 252         """Update the results and subtests fields from a piglit test.
 253
 254         Native piglit tests output their data as valid json, and piglit uses
 255         the json module to parse this data. This method consumes that raw
 256         dictionary data and updates itself.
 257
 258         """
 259         if 'result' in dict_:
 260             self.result = dict_['result']
 261         elif 'subtest' in dict_:
 262             self.subtests.update(dict_['subtest'])
 263
 264
 265 @compat.python_2_bool_compatible
 266 class Totals(dict):
 267     def __init__(self, *args, **kwargs):
 268         super(Totals, self).__init__(*args, **kwargs)
 269         for each in status.ALL:
 270             each = six.text_type(each)
 271             if each not in self:
 272                 self[each] = 0
 273
 274     def __bool__(self):
 275         # Since totals are prepopulated, calling 'if not <Totals instance>'
 276         # will always result in True, this will cause it to return True only if
 277         # one of the values is not zero
 278         for each in six.itervalues(self):
 279             if each != 0:
 280                 return True
 281         return False
 282
 283     def to_json(self):
 284         """Convert totals to a json object."""
 285         result = copy.copy(self)
 286         result['__type__'] = 'Totals'
 287         return result
 288
 289     @classmethod
 290     def from_dict(cls, dict_):
 291         """Convert a dictionary into a Totals object."""
 292         tots = cls(dict_)
 293         if '__type__' in tots:
 294             del tots['__type__']
 295         return tots
 296
 297
 298 class TestrunResult(object):
 299     """The result of a single piglit run."""
 300     def __init__(self):
 301         self.name = None
 302         self.uname = None
 303         self.options = {}
 304         self.glxinfo = None
 305         self.wglinfo = None
 306         self.clinfo = None
 307         self.lspci = None
 308         self.time_elapsed = TimeAttribute()
 309         self.tests = collections.OrderedDict()
 310         self.totals = collections.defaultdict(Totals)
 311
 312     def get_result(self, key):
 313         """Get the result of a test or subtest.
 314
 315         If neither a test nor a subtest instance exist, then raise the original
 316         KeyError generated from looking up <key> in the tests attribute. It is
 317         the job of the caller to handle this error.
 318
 319         Arguments:
 320         key -- the key name of the test to return
 321
 322         """
 323         try:
 324             return self.tests[key].result
 325         except KeyError as e:
 326             name, test = grouptools.splitname(key)
 327             try:
 328                 return self.tests[name].subtests[test]
 329             except KeyError:
 330                 raise e
 331
 332     def calculate_group_totals(self):
 333         """Calculate the number of passes, fails, etc at each level."""
 334         for name, result in six.iteritems(self.tests):
 335             # If there are subtests treat the test as if it is a group instead
 336             # of a test.
 337             if result.subtests:
 338                 for res in six.itervalues(result.subtests):
 339                     res = str(res)
 340                     temp = name
 341
 342                     self.totals[temp][res] += 1
 343                     while temp:
 344                         temp = grouptools.groupname(temp)
 345                         self.totals[temp][res] += 1
 346                     self.totals['root'][res] += 1
 347             else:
 348                 res = str(result.result)
 349                 while name:
 350                     name = grouptools.groupname(name)
 351                     self.totals[name][res] += 1
 352                 self.totals['root'][res] += 1
 353
 354     def to_json(self):
 355         if not self.totals:
 356             self.calculate_group_totals()
 357         rep = copy.copy(self.__dict__)
 358         rep['tests'] = collections.OrderedDict((k, t.to_json())
 359                        for k, t in six.iteritems(self.tests))
 360         rep['__type__'] = 'TestrunResult'
 361         return rep
 362
 363     @classmethod
 364     def from_dict(cls, dict_, _no_totals=False):
 365         """Convert a dictionary into a TestrunResult.
 366
 367         This method is meant to be used for loading results from json or
 368         similar formats
 369
 370         _no_totals is not meant to be used externally, it allows us to control
 371         the generation of totals when loading old results formats.
 372
 373         """
 374         res = cls()
 375         for name in ['name', 'uname', 'options', 'glxinfo', 'wglinfo', 'lspci',
 376                      'results_version', 'clinfo']:
 377             value = dict_.get(name)
 378             if value:
 379                 setattr(res, name, value)
 380
 381         # Since this is used to load partial metadata when writing final test
 382         # results there is no guarantee that this will have a "time_elapsed"
 383         # key
 384         if 'time_elapsed' in dict_:
 385             setattr(res, 'time_elapsed',
 386                     TimeAttribute.from_dict(dict_['time_elapsed']))
 387         res.tests = collections.OrderedDict((n, TestResult.from_dict(t))
 388                     for n, t in six.iteritems(dict_['tests']))
 389
 390         if not 'totals' in dict_ and not _no_totals:
 391             res.calculate_group_totals()
 392         else:
 393             res.totals = {n: Totals.from_dict(t) for n, t in
 394                           six.iteritems(dict_['totals'])}
 395
 396         return res