unittests/framework/test_results.py

   1 # encoding=utf-8
   2 # Copyright (c) 2014-2016, 2019 Intel Corporation
   3
   4 # Permission is hereby granted, free of charge, to any person obtaining a copy
   5 # of this software and associated documentation files (the "Software"), to deal
   6 # in the Software without restriction, including without limitation the rights
   7 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   8 # copies of the Software, and to permit persons to whom the Software is
   9 # furnished to do so, subject to the following conditions:
  10
  11 # The above copyright notice and this permission notice shall be included in
  12 # all copies or substantial portions of the Software.
  13
  14 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  17 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  18 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  19 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  20 # SOFTWARE.
  21
  22 """Tests for the results module."""
  23
  24 import pytest
  25
  26 from framework import exceptions
  27 from framework import grouptools
  28 from framework import results
  29 from framework import status
  30
  31 from .backends import shared
  32
  33 # pylint: disable=no-self-use
  34
  35
  36 class TestSubtests(object):
  37     """Tests for the Subtest class."""
  38
  39     @pytest.fixture
  40     def subtest(self):
  41         return results.Subtests()
  42
  43     def test_convert_statuses(self, subtest):
  44         """results.Subtests.__setitem__: converts strings to statues."""
  45         subtest['foo'] = 'pass'
  46         assert subtest['foo'] is status.PASS
  47
  48     def test_to_json(self, subtest):
  49         """results.Subtests.to_json: sets values properly."""
  50         baseline = {
  51             'foo': status.PASS,
  52             'bar': status.CRASH,
  53             '__type__': 'Subtests',
  54         }
  55
  56         subtest['foo'] = status.PASS
  57         subtest['bar'] = status.CRASH
  58
  59         assert baseline == subtest.to_json()
  60
  61     def test_from_dict(self, subtest):
  62         """results.Subtests.from_dict: restores values properly"""
  63         subtest['foo'] = status.PASS
  64         subtest['bar'] = status.CRASH
  65
  66         test = results.Subtests.from_dict(subtest.to_json())
  67
  68         # Subtests don't have equality methods, so convert them to dicts before
  69         # comparing.
  70         assert dict(subtest) == dict(test)
  71
  72     def test_from_dict_instance(self, subtest):
  73         """results.Subtests.from_dict: restores values properly"""
  74         subtest['foo'] = status.PASS
  75         test = results.Subtests.from_dict(subtest.to_json())
  76
  77         assert test['foo'] is status.PASS
  78
  79
  80 class TestTestResult(object):
  81     """Tests for the TestResult class."""
  82
  83     class TestFromDict(object):
  84         """Tests for TestResult.from_dict."""
  85
  86         def test_inst(self):
  87             """results.TestResult.from_dict: returns a TestResult."""
  88             test = results.TestResult.from_dict({'result': 'pass'})
  89             assert isinstance(test, results.TestResult)
  90
  91         class TestAttributes(object):
  92             """Tests attribute restoration for the from_dict method."""
  93
  94             @classmethod
  95             def setup_class(cls):
  96                 """Setup state for all tests."""
  97                 cls.dict = {
  98                     'returncode': 100,
  99                     'err': 'this is an err',
 100                     'out': 'this is some text',
 101                     'time': {
 102                         'start': 0.5,
 103                         'end': 0.9,
 104                     },
 105                     'environment': 'some env stuff',
 106                     'subtests': {
 107                         'a': 'pass',
 108                         'b': 'fail',
 109                     },
 110                     'result': 'crash',
 111                     'exception': 'an exception',
 112                     'dmesg': 'this is dmesg',
 113                     'pid': [1934],
 114                 }
 115
 116                 cls.test = results.TestResult.from_dict(cls.dict)
 117
 118             def test_returncode(self):
 119                 """sets returncode properly."""
 120                 assert self.test.returncode == self.dict['returncode']
 121
 122             def test_err(self):
 123                 """sets err properly."""
 124                 assert self.test.err == self.dict['err']
 125
 126             def test_out(self):
 127                 """sets out properly."""
 128                 assert self.test.out == self.dict['out']
 129
 130             def test_time(self):
 131                 """sets time properly.
 132
 133                 Ultimatley time needs to be converted to a TimeAttribute object,
 134                 not a dictionary, however, that functionality is handled by
 135                 backends.json.piglit_decoder, not by the from_dict method of
 136                 TestResult. So in this case the test is that the object is a
 137                 dictionary, not a TimeAttribute because this method shouldn't
 138                 make the conversion.
 139
 140                 """
 141                 # pylint: disable=unsubscriptable-object
 142                 assert self.test.time.start == self.dict['time']['start']
 143                 assert self.test.time.end == self.dict['time']['end']
 144
 145             def test_environment(self):
 146                 """sets environment properly."""
 147                 assert self.test.environment == self.dict['environment']
 148
 149             def test_exception(self):
 150                 """sets exception properly."""
 151                 assert self.test.exception == self.dict['exception']
 152
 153             def test_subtests(self):
 154                 """sets subtests properly."""
 155                 assert self.test.subtests == self.dict['subtests']
 156
 157             def test_subtests_type(self):
 158                 """subtests are Status instances."""
 159                 assert self.test.subtests['a'] is status.PASS
 160                 assert self.test.subtests['b'] is status.FAIL
 161
 162             def test_dmesg(self):
 163                 """sets dmesg properly."""
 164                 assert self.test.dmesg == self.dict['dmesg']
 165
 166             def test_pid(self):
 167                 """sets pid properly."""
 168                 assert self.test.pid == self.dict['pid']
 169
 170         class TestResult(object):
 171             """Tests for TestResult.result getter and setter methods."""
 172
 173             @pytest.fixture
 174             def result(self):
 175                 return results.TestResult('pass')
 176
 177             def test_getter_no_subtests(self, result):
 178                 """Getter returns the result when there are no subtests."""
 179                 assert result.result is status.PASS
 180
 181             def test_getter_with_subtests(self, result):
 182                 """Getter returns worst subtest when subtests are present."""
 183                 result.subtests['a'] = status.PASS
 184                 result.subtests['b'] = status.CRASH
 185
 186                 assert result.result is status.CRASH
 187
 188             def test_getter_subtests_crash(self, result):
 189                 """Doesn't mask crashes.
 190
 191                 Crash is somewhat unique in that subtests can run and pass, but
 192                 then the test can crash before all subtests have been run.
 193                 """
 194                 result.result = status.CRASH
 195                 result.subtests['a'] = status.PASS
 196                 result.subtests['b'] = status.SKIP
 197
 198                 assert result.result is status.CRASH
 199
 200             def test_setter(self, result):
 201                 """setter makes the result a status."""
 202                 result.result = status.FAIL
 203                 assert result.result is status.FAIL
 204
 205             def test_setter_invalid(self, result):
 206                 """setter raises PiglitFatalError for invalid values."""
 207                 with pytest.raises(exceptions.PiglitFatalError):
 208                     result.result = 'foo'
 209
 210     class TestToJson(object):
 211         """Tests for the attributes of the to_json method."""
 212
 213         @classmethod
 214         def setup_class(cls):
 215             """Setup state for all tests."""
 216             test = results.TestResult()
 217             test.returncode = 100
 218             test.err = 'this is an err'
 219             test.out = 'this is some text'
 220             test.time.start = 0.3
 221             test.time.end = 0.5
 222             test.environment = 'some env stuff'
 223             test.subtests.update({
 224                 'a': 'pass',
 225                 'b': 'fail'})
 226             test.result = 'crash'
 227             test.exception = 'an exception'
 228             test.dmesg = 'this is dmesg'
 229             test.pid = 1934
 230             test.traceback = 'a traceback'
 231
 232             cls.test = test
 233             cls.json = test.to_json()
 234
 235         def test_returncode(self):
 236             """results.TestResult.to_json: sets the returncode correctly"""
 237             assert self.test.returncode == self.json['returncode']
 238
 239         def test_err(self):
 240             """results.TestResult.to_json: sets the err correctly"""
 241             assert self.test.err == self.json['err']
 242
 243         def test_out(self):
 244             """results.TestResult.to_json: sets the out correctly"""
 245             assert self.test.out == self.json['out']
 246
 247         def test_exception(self):
 248             """results.TestResult.to_json: sets the exception correctly"""
 249             assert self.test.exception == self.json['exception']
 250
 251         def test_time(self):
 252             """results.TestResult.to_json: sets the time correctly"""
 253             # pylint: disable=unsubscriptable-object
 254             assert self.test.time.start == self.json['time']['start']
 255             assert self.test.time.end == self.json['time']['end']
 256
 257         def test_environment(self):
 258             """results.TestResult.to_json: sets the environment correctly"""
 259             assert self.test.environment == self.json['environment']
 260
 261         def test_subtests(self):
 262             """results.TestResult.to_json: sets the subtests correctly"""
 263             assert self.test.subtests['a'] == self.json['subtests']['a']
 264             assert self.test.subtests['b'] == self.json['subtests']['b']
 265             assert self.json['subtests']['__type__']
 266
 267         def test_type(self):
 268             """results.TestResult.to_json: adds the __type__ hint"""
 269             assert self.json['__type__'] == 'TestResult'
 270
 271         def test_dmesg(self):
 272             """results.TestResult.to_json: Adds the dmesg attribute"""
 273             assert self.test.dmesg == self.json['dmesg']
 274
 275         def test_pid(self):
 276             """results.TestResult.to_json: Adds the pid attribute"""
 277             assert self.test.pid == self.json['pid']
 278
 279         def test_traceback(self):
 280             """results.TestResult.to_json: Adds the traceback attribute"""
 281             assert self.test.traceback == self.json['traceback']
 282
 283     class TestUpdate(object):
 284         """Tests for TestResult.update."""
 285
 286         def test_no_subtests(self):
 287             """results.TestResult.update: result is updated"""
 288             test = results.TestResult('pass')
 289             test.update({'result': 'incomplete'})
 290             assert test.result == 'incomplete'
 291
 292         def test_with_subtests(self):
 293             """results.TestResult.update: subests are updated"""
 294             test = results.TestResult('pass')
 295             test.update({'subtest': {'result': 'incomplete'}})
 296             assert test.subtests['result'] == 'incomplete'
 297
 298     class TestTotals(object):
 299         """Test the totals generated by TestrunResult.calculate_group_totals().
 300         """
 301         @classmethod
 302         def setup_class(cls):
 303             """setup state for all tests."""
 304             pass_ = results.TestResult('pass')
 305             fail = results.TestResult('fail')
 306             crash = results.TestResult('crash')
 307             skip = results.TestResult('skip')
 308             tr = results.TestrunResult()
 309             tr.tests = {
 310                 'oink': pass_,
 311                 grouptools.join('foo', 'bar'): fail,
 312                 grouptools.join('foo', 'foo', 'bar'): crash,
 313                 grouptools.join('foo', 'foo', 'oink'): skip,
 314             }
 315
 316             tr.calculate_group_totals()
 317             cls.test = tr.totals
 318
 319         def test_root(self):
 320             """The root is correct."""
 321             root = results.Totals()
 322             root['pass'] += 1
 323             root['fail'] += 1
 324             root['crash'] += 1
 325             root['skip'] += 1
 326
 327             assert dict(self.test['root']) == dict(root)
 328
 329         def test_recurse(self):
 330             """Recurses correctly."""
 331             expected = results.Totals()
 332             expected['fail'] += 1
 333             expected['crash'] += 1
 334             expected['skip'] += 1
 335             assert dict(self.test['foo']) == dict(expected)
 336
 337         def test_two_parents(self):
 338             """Handles multiple parents correctly."""
 339             expected = results.Totals()
 340             expected['crash'] += 1
 341             expected['skip'] += 1
 342             assert dict(self.test[grouptools.join('foo', 'foo')]) == \
 343                 dict(expected)
 344
 345     class TestTotalsWithSubtests(object):
 346         """results.TestrunResult.totals: Tests with subtests are handled
 347         correctly.
 348         """
 349
 350         @classmethod
 351         def setup_class(cls):
 352             """Setup all tests."""
 353             tr = results.TestResult('crash')
 354             tr.subtests['foo'] = status.PASS
 355             tr.subtests['bar'] = status.CRASH
 356             tr.subtests['oink'] = status.FAIL
 357
 358             run = results.TestrunResult()
 359             run.tests[grouptools.join('sub', 'test')] = tr
 360             run.calculate_group_totals()
 361
 362             cls.test = run.totals
 363
 364         def test_root(self):
 365             """The root is correct with subtests."""
 366             expect = results.Totals()
 367             expect['pass'] += 1
 368             expect['crash'] += 1
 369             expect['fail'] += 1
 370             assert dict(self.test['root']) == dict(expect)
 371
 372         def test_node(self):
 373             """Tests with subtests are treated as groups."""
 374             key = grouptools.join('sub', 'test')
 375             assert key in self.test
 376
 377         def test_node_values(self):
 378             """Tests with subtests values are correct."""
 379             expect = results.Totals()
 380             expect['pass'] += 1
 381             expect['crash'] += 1
 382             expect['fail'] += 1
 383             assert dict(self.test[grouptools.join('sub', 'test')]) == \
 384                 dict(expect)
 385
 386
 387 class TestStringDescriptor(object):
 388     """Test class for StringDescriptor."""
 389
 390     @pytest.fixture
 391     def test(self):
 392         class Test(object):
 393             val = results.StringDescriptor('test')
 394
 395         return Test()
 396
 397     def test_get_default(self, test):
 398         """results.StringDescriptor.__get__: returns default when unset"""
 399         assert test.val == ''
 400
 401     def test_set_no_replace(self, test):
 402         """results.StringDescriptor.__set__: instance is not replaced
 403
 404         This works by setting the value to a valid value (either bytes or str)
 405         and then trying to set it to an invalid value (int). If the assignment
 406         succeeds then the instance has been replaced, if not, it hasn't.
 407
 408         """
 409         test.val = 'foo'
 410         with pytest.raises(TypeError):
 411             test.val = 1  # pylint: disable=redefined-variable-type
 412
 413     def test_set_str(self, test):
 414         """results.StringDescriptor.__set__: str is stored directly"""
 415         inst = 'foo'
 416         test.val = inst
 417         assert test.val == inst
 418
 419     def test_set_bytes(self, test):
 420         """results.StringDescriptor.__set__: converts bytes to str"""
 421         inst = b'foo'
 422         test.val = inst
 423         assert test.val == 'foo'
 424
 425     def test_set_str_unicode_literals(self, test):
 426         """results.StringDescriptor.__set__: handles unicode literals in strs.
 427         """
 428         test.val = '\ufffd'
 429         assert test.val == '�'
 430
 431     def test_delete(self, test):
 432         """results.StringDescriptor.__delete__: raises NotImplementedError"""
 433         with pytest.raises(NotImplementedError):
 434             del test.val
 435
 436
 437 class TestTotals(object):
 438     """Tests for the totals class."""
 439
 440     def test_totals_false(self):
 441         """bool() returns False when all values are 0."""
 442         assert not bool(results.Totals())
 443
 444     # The tuple is required because of the timeout status, which conflicts with
 445     # the timeout pylint plugin
 446     @pytest.mark.parametrize(
 447         "key", ((x, ) for x in results.Totals().keys()))
 448     def test_totals_true(self, key):
 449         """bool() returns True when any value is not 0."""
 450         test = results.Totals()
 451         test[key[0]] += 1
 452         assert bool(test)
 453
 454
 455 class TestTestrunResult(object):
 456     """Tests for the TestrunResult class."""
 457
 458     class TestToJson(object):
 459         """Test TestrunResult.to_json method."""
 460
 461         @classmethod
 462         def setup_class(cls):
 463             """Setup values used by all tests."""
 464             test = results.TestrunResult()
 465             test.info = {
 466                 'system': {
 467                     'uname': 'this is uname',
 468                     'glxinfo': 'glxinfo',
 469                     'clinfo': 'clinfo',
 470                     'wglinfo': 'wglinfo',
 471                     'lspci': 'this is lspci',
 472                 }
 473             }
 474             test.name = 'name'
 475             test.options = {'some': 'option'}
 476             test.time_elapsed.end = 1.23
 477             test.tests = {'a test': results.TestResult('pass')}
 478
 479             cls.test = test.to_json()
 480
 481         def test_name(self):
 482             """name is properly encoded."""
 483             assert self.test['name'] == 'name'
 484
 485         def test_info(self):
 486             assert self.test['info'] == {
 487                 'system': {
 488                     'uname': 'this is uname',
 489                     'glxinfo': 'glxinfo',
 490                     'clinfo': 'clinfo',
 491                     'wglinfo': 'wglinfo',
 492                     'lspci': 'this is lspci',
 493                 }
 494             }
 495
 496         def test_options(self):
 497             """options is properly encoded."""
 498             assert dict(self.test['options']) == {'some': 'option'}
 499
 500         def test_time(self):
 501             """time_elapsed is properly encoded."""
 502             assert self.test['time_elapsed'].end == 1.23
 503
 504         def test_tests(self):
 505             """tests is properly encoded."""
 506             assert self.test['tests']['a test']['result'] == 'pass'
 507
 508         def test_type(self):
 509             """__type__ is added."""
 510             assert self.test['__type__'] == 'TestrunResult'
 511
 512     class TestFromDict(object):
 513         """Tests for TestrunResult.from_dict."""
 514
 515         @pytest.fixture(scope="module")
 516         def inst(self):
 517             return results.TestrunResult.from_dict(shared.JSON)
 518
 519         @pytest.mark.parametrize("attrib", [
 520             'name', 'results_version', 'info', 'options',
 521         ])
 522         def test_attribs_restored(self, attrib, inst):
 523             """tests for basic attributes."""
 524             assert shared.JSON[attrib] == getattr(inst, attrib)
 525
 526         def test_tests(self, inst):
 527             """tests is restored correctly."""
 528             assert inst.tests.keys() == shared.JSON['tests'].keys()
 529
 530         def test_test_type(self, inst):
 531             """tests is restored correctly."""
 532             assert isinstance(
 533                 inst.tests['spec@!opengl 1.0@gl-1.0-readpixsanity'],
 534                 results.TestResult)
 535
 536         def test_totals(self, inst):
 537             """totals is restored correctly."""
 538             baseline = shared.JSON['totals'].copy()
 539             for s in baseline.values():
 540                 del s['__type__']
 541             assert baseline == dict(inst.totals)
 542
 543         def test_time_elapsed(self, inst):
 544             """time_elapsed is restored correctly."""
 545             assert inst.time_elapsed.start == \
 546                 shared.JSON['time_elapsed']['start']
 547             assert inst.time_elapsed.end == \
 548                 shared.JSON['time_elapsed']['end']
 549
 550     class TestGetResult(object):
 551         """Tests for TestrunResult.get_result."""
 552
 553         @classmethod
 554         def setup_class(cls):
 555             """setup state for all tests."""
 556             tr = results.TestResult('crash')
 557             tr.subtests['foo'] = status.PASS
 558
 559             run = results.TestrunResult()
 560             run.tests['sub'] = tr
 561             run.tests['test'] = results.TestResult('pass')
 562             run.calculate_group_totals()
 563
 564             cls.inst = run
 565
 566         def test_get_test(self):
 567             """gets non-subtests."""
 568             assert self.inst.get_result('test') == 'pass'
 569
 570         def test_get_subtest(self):
 571             """gets subtests."""
 572             assert self.inst.get_result(grouptools.join('sub', 'foo')) == 'pass'
 573
 574         def test_get_nonexist(self):
 575             """raises KeyError if test doesn't exist."""
 576             with pytest.raises(KeyError):
 577                 self.inst.get_result('fooobar')
 578
 579
 580 class TestTimeAttribute(object):
 581     """Tests for the TimeAttribute class."""
 582
 583     def test_to_json(self):
 584         """returns expected dictionary."""
 585         baseline = {'start': 0.1, 'end': 1.0}
 586         test = results.TimeAttribute(**baseline)
 587         baseline['__type__'] = 'TimeAttribute'
 588
 589         assert baseline == test.to_json()
 590
 591     def test_from_dict(self):
 592         """returns expected value."""
 593         # Type is included because to_json() adds it.
 594         baseline = {'start': 0.1, 'end': 1.0, '__type__': 'TimeAttribute'}
 595         test = results.TimeAttribute.from_dict(baseline).to_json()
 596
 597         assert baseline == test
 598
 599     def test_total(self):
 600         """returns the difference between end and start."""
 601         test = results.TimeAttribute(1.0, 5.0)
 602         assert test.total == 4.0
 603
 604     def test_delta(self):
 605         """results.TimeAttribute.delta: returns the delta of the values"""
 606         test = results.TimeAttribute(1.0, 5.0)
 607         assert test.delta == '0:00:04'