unittests/framework/test/test_deqp.py

   1 # coding=utf-8
   2 # Copyright (c) 2015-2016, 2019 Intel Corporation
   3
   4 # Permission is hereby granted, free of charge, to any person obtaining a copy
   5 # of this software and associated documentation files (the "Software"), to deal
   6 # in the Software without restriction, including without limitation the rights
   7 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   8 # copies of the Software, and to permit persons to whom the Software is
   9 # furnished to do so, subject to the following conditions:
  10
  11 # The above copyright notice and this permission notice shall be included in
  12 # all copies or substantial portions of the Software.
  13
  14 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  17 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  18 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  19 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  20 # SOFTWARE.
  21
  22 """Tests for the dEQP integration in framework.
  23
  24 This tests the core framework shared code, and not the individual packages in
  25 tests
  26
  27 """
  28
  29 import textwrap
  30
  31 import pytest
  32
  33 from framework import exceptions
  34 from framework import grouptools
  35 from framework import profile
  36 from framework import status
  37 from framework.test import deqp
  38
  39 # pylint:disable=invalid-name,no-self-use
  40
  41
  42 class _DEQPTestTest(deqp.DEQPBaseTest):
  43     deqp_bin = 'deqp.bin'
  44     extra_args = ['extra']
  45
  46
  47 class TestMakeProfile(object):
  48     """Test deqp.make_profile."""
  49
  50     @classmethod
  51     def setup_class(cls):
  52         cls.profile = deqp.make_profile(['this.is.a.deqp.test'], _DEQPTestTest)
  53
  54     def test_returns_profile(self):
  55         """deqp.make_profile: returns a TestProfile."""
  56         assert isinstance(self.profile, profile.TestProfile)
  57
  58     def test_replaces_separator(self):
  59         """deqp.make_profile: replaces '.' with grouptools.separator"""
  60         expected = grouptools.join('this', 'is', 'a', 'deqp', 'test')
  61         assert expected in self.profile.test_list
  62
  63
  64 class TestIterDeqpTestCases(object):
  65     """Tests for iter_deqp_test_cases."""
  66
  67     def _do_test(self, write, expect, tmpdir):
  68         """Run the actual test."""
  69         p = tmpdir.join('foo')
  70         p.write(write)
  71         gen = deqp.iter_deqp_test_cases(str(p))
  72         assert next(gen) == expect
  73
  74     def test_test_cases(self, tmpdir):
  75         """Correctly detects a test line."""
  76         self._do_test('TEST: a.deqp.test', 'a.deqp.test', tmpdir)
  77
  78     def test_test_group(self, tmpdir):
  79         """Correctly detects a group line."""
  80         self._do_test('GROUP: a group\nTEST: a.deqp.test', 'a.deqp.test',
  81                       tmpdir)
  82
  83     def test_bad_entry(self, tmpdir):
  84         """A PiglitFatalException is raised if a line is not a TEST or GROUP.
  85         """
  86         with pytest.raises(exceptions.PiglitFatalError):
  87             self._do_test('this will fail', None, tmpdir)
  88
  89
  90 class TestDEQPBaseTest(object):
  91     """Test the DEQPBaseTest class."""
  92
  93     @classmethod
  94     def setup_class(cls):
  95         cls.test = _DEQPTestTest('a.deqp.test')
  96
  97     def test_command_adds_extra_args(self):
  98         assert self.test.command[-1] == 'extra'
  99
 100     class TestInterpretResultReturncodes(object):
 101         """Test the interpret_result method's returncode handling."""
 102
 103         @classmethod
 104         def setup_class(cls):
 105             cls.test = _DEQPTestTest('a.deqp.test')
 106
 107         def test_crash(self):
 108             """deqp.DEQPBaseTest.interpret_result: if returncode is < 0 stauts
 109             is crash.
 110             """
 111             self.test.result.returncode = -9
 112             self.test.interpret_result()
 113             assert self.test.result.result is status.CRASH
 114
 115         def test_returncode_fail(self):
 116             """deqp.DEQPBaseTest.interpret_result: if returncode is > 0 result
 117             is fail.
 118             """
 119             self.test.result.returncode = 1
 120             self.test.interpret_result()
 121             assert self.test.result.result is status.FAIL
 122
 123         def test_fallthrough(self):
 124             """deqp.DEQPBaseTest.interpret_result: if no case is hit set to
 125             fail.
 126             """
 127             self.test.result.returncode = 0
 128             self.test.result.out = ''
 129             self.test.interpret_result()
 130             assert self.test.result.result is status.FAIL
 131
 132         def test_windows_returncode_3(self, mocker):
 133             """deqp.DEQPBaseTest.interpret_result: on windows returncode 3 is
 134             crash.
 135             """
 136             mocker.patch('framework.test.base.sys.platform', 'win32')
 137             self.test.result.returncode = 3
 138             self.test.interpret_result()
 139             assert self.test.result.result is status.CRASH
 140
 141     class TestDEQPBaseTestIntepretResultOutput(object):
 142         """Tests for DEQPBaseTest.__find_map."""
 143
 144         inst = None
 145         __OUT = textwrap.dedent("""\
 146             dEQP Core 2014.x (0xcafebabe) starting..
 147               target implementation = 'DRM'
 148
 149             Test case 'dEQP-GLES2.functional.shaders.conversions.vector_to_vector.vec3_to_ivec3_fragment'..
 150             Vertex shader compile time = 0.129000 ms
 151             Fragment shader compile time = 0.264000 ms
 152             Link time = 0.814000 ms
 153             Test case duration in microseconds = 487155 us
 154               {stat} ({stat})
 155
 156             DONE!
 157
 158             Test run totals:
 159               Passed:        {pass_}/1 (100.0%)
 160               Failed:        {fail}/1 (0.0%)
 161               Not supported: {supp}/1 (0.0%)
 162               Warnings:      {warn}/1 (0.0%)
 163             Test run was ABORTED!
 164         """)
 165
 166         def __gen_stdout(self, stat):
 167             """Make a string that looks like DEQP output."""
 168             assert stat in ['Fail', 'NotSupported', 'Pass', 'QualityWarning',
 169                             'InternalError', 'Crash', 'ResourceError']
 170
 171             return self.__OUT.format(
 172                 stat=stat,
 173                 pass_=1 if stat == 'Pass' else 0,
 174                 fail=1 if stat in ['Crash', 'Fail', 'ResourceError'] else 0,
 175                 supp=1 if stat == 'InternalError' else 0,
 176                 warn=1 if stat == 'QualityWarning' else 0,
 177             )
 178
 179         def setup(self):
 180             self.inst = _DEQPTestTest('a.deqp.test')
 181             self.inst.result.returncode = 0
 182
 183         def test_fail(self):
 184             """test.deqp.DEQPBaseTest.interpret_result: when Fail in result the
 185             result is 'fail'.
 186             """
 187             self.inst.result.out = self.__gen_stdout('Fail')
 188             self.inst.interpret_result()
 189             assert self.inst.result.result is status.FAIL
 190
 191         def test_pass(self):
 192             """test.deqp.DEQPBaseTest.interpret_result: when Pass in result the
 193             result is 'Pass'.
 194             """
 195             self.inst.result.out = self.__gen_stdout('Pass')
 196             self.inst.interpret_result()
 197             assert self.inst.result.result is status.PASS
 198
 199         def test_warn(self):
 200             """test.deqp.DEQPBaseTest.interpret_result: when QualityWarning in
 201             result the result is 'warn'.
 202             """
 203             self.inst.result.out = self.__gen_stdout('QualityWarning')
 204             self.inst.interpret_result()
 205             assert self.inst.result.result is status.WARN
 206
 207         def test_error(self):
 208             """test.deqp.DEQPBaseTest.interpret_result: when InternalError in
 209             result the result is 'fail'.
 210             """
 211             self.inst.result.out = self.__gen_stdout('InternalError')
 212             self.inst.interpret_result()
 213             assert self.inst.result.result is status.FAIL
 214
 215         def test_crash(self):
 216             """test.deqp.DEQPBaseTest.interpret_result: when InternalError in
 217             result the result is 'crash'.
 218             """
 219             self.inst.result.out = self.__gen_stdout('Crash')
 220             self.inst.interpret_result()
 221             assert self.inst.result.result is status.CRASH
 222
 223         def test_skip(self):
 224             """test.deqp.DEQPBaseTest.interpret_result: when NotSupported in
 225             result the result is 'skip'.
 226             """
 227             self.inst.result.out = self.__gen_stdout('NotSupported')
 228             self.inst.interpret_result()
 229             assert self.inst.result.result is status.SKIP
 230
 231         def test_resourceerror(self):
 232             """test.deqp.DEQPBaseTest.interpret_result: when ResourceError in
 233             result the result is 'crash'.
 234             """
 235             self.inst.result.out = self.__gen_stdout('ResourceError')
 236             self.inst.interpret_result()
 237             assert self.inst.result.result is status.CRASH
 238
 239
 240 class TestGenMustpassTests(object):
 241     """Tests for the gen_mustpass_tests function."""
 242
 243     _txt = """\
 244 dEQP.piglit.group1.test1
 245 dEQP.piglit.group1.test2
 246 dEQP.piglit.nested.group2.test3
 247 dEQP.piglit.nested.group2.test4
 248     """
 249
 250     def test_basic(self, tmpdir):
 251         p = tmpdir.join('foo.txt')
 252         p.write(self._txt)
 253         tests = set(deqp.gen_mustpass_tests(str(p)))
 254         print(tests)
 255         assert tests == {
 256             'dEQP.piglit.group1.test1',
 257             'dEQP.piglit.group1.test2',
 258             'dEQP.piglit.nested.group2.test3',
 259             'dEQP.piglit.nested.group2.test4',
 260         }