framework/test/piglit_test.py

   1 # coding=utf-8
   2 #
   3 # Copyright © 2021 Collabora Ltd.
   4 #
   5 # Permission is hereby granted, free of charge, to any person
   6 # obtaining a copy of this software and associated documentation
   7 # files (the "Software"), to deal in the Software without
   8 # restriction, including without limitation the rights to use,
   9 # copy, modify, merge, publish, distribute, sublicense, and/or
  10 # sell copies of the Software, and to permit persons to whom the
  11 # Software is furnished to do so, subject to the following
  12 # conditions:
  13 #
  14 # This permission notice shall be included in all copies or
  15 # substantial portions of the Software.
  16 #
  17 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
  18 # KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
  19 # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
  20 # PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE AUTHOR(S) BE
  21 # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
  22 # AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
  23 # OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  24 # DEALINGS IN THE SOFTWARE.
  25
  26 """ Module provides a base class for Tests """
  27
  28 import glob
  29 import os
  30 import sys
  31 try:
  32     import simplejson as json
  33 except ImportError:
  34     import json
  35
  36 from framework import core, options
  37 from framework import status
  38 from .base import Test, WindowResizeMixin, ValgrindMixin, TestIsSkip
  39
  40
  41 __all__ = [
  42     'PiglitCLTest',
  43     'PiglitGLTest',
  44     'PiglitBaseTest',
  45     'PiglitReplayerTest',
  46     'VkRunnerTest',
  47     'CL_CONCURRENT',
  48     'ROOT_DIR',
  49     'TEST_BIN_DIR',
  50 ]
  51
  52 if 'PIGLIT_BUILD_DIR' in os.environ:
  53     ROOT_DIR = os.environ['PIGLIT_BUILD_DIR']
  54 else:
  55     ROOT_DIR = os.path.normpath(os.path.join(os.path.dirname(__file__), '../..'))
  56
  57 TEST_BIN_DIR = os.path.normpath(os.path.join(ROOT_DIR, 'bin'))
  58
  59 CL_CONCURRENT = bool(not sys.platform.startswith('linux') or
  60                      glob.glob('/dev/dri/render*'))
  61
  62
  63 class PiglitBaseTest(ValgrindMixin, Test):
  64     """
  65     PiglitTest: Run a "native" piglit test executable
  66
  67     Expect one line prefixed PIGLIT: in the output, which contains a result
  68     dictionary. The plain output is appended to this dictionary
  69     """
  70     def __init__(self, command, run_concurrent=True, **kwargs):
  71         super(PiglitBaseTest, self).__init__(command, run_concurrent, **kwargs)
  72
  73     @Test.command.getter
  74     def command(self):
  75         command = super(PiglitBaseTest, self).command
  76
  77         def fixup_bin_path(c):
  78             # Prepend TEST_BIN_DIR to the path.
  79             if c == self._command[0]:
  80                 return os.path.join(TEST_BIN_DIR, c)
  81             else:
  82                 return c
  83
  84         return [fixup_bin_path(c) for c in command]
  85
  86     def interpret_result(self):
  87         out = []
  88
  89         for each in self.result.out.split('\n'):
  90             if each.startswith('PIGLIT:'):
  91                 deserial = json.loads(each[8:])
  92                 if 'enumerate subtests' in deserial:
  93                     for n in deserial['enumerate subtests']:
  94                         self.result.subtests[n] = status.NOTRUN
  95                 else:
  96                     self.result.update(deserial)
  97             else:
  98                 out.append(each)
  99
 100         self.result.out = '\n'.join(out)
 101
 102         # XXX: There are a number of tests that now enumerate their subtests,
 103         # but don't properly report skip for all of them when the skip due to a
 104         # missing feature. We should fix these tests to do the right thing, but
 105         # for the moment this work around will suffice to keep things running.
 106         if self.result.raw_result is status.SKIP and self.result.subtests:
 107             for k, v in self.result.subtests.items():
 108                 if v is status.NOTRUN:
 109                     self.result.subtests[k] = status.SKIP
 110
 111         super(PiglitBaseTest, self).interpret_result()
 112
 113
 114 class PiglitGLTest(WindowResizeMixin, PiglitBaseTest):
 115     """ OpenGL specific Piglit test class
 116
 117     This Subclass provides provides an is_skip() implementation that skips glx
 118     tests on non-glx platforms
 119
 120     This class also provides two additional keyword arguments, require_platform
 121     and exclude_platforms. require_platforms may be set to a list of platforms
 122     which the test requires to run. This should be resereved for platform
 123     specific tests, such as GLX specific tests, or EGL specific tests. Multiple
 124     platforms are allowed because EGL can be fulfilled by multiple platforms.
 125     exclude_platforms is a list of platforms a test should not be run on, this
 126     is useful for tests that are valid on more than one platform, but not on
 127     all of them. This will probably be mainly used to exclude gbm. These
 128     options are mutually exclusive.
 129
 130     """
 131     def __init__(self, command, require_platforms=None, exclude_platforms=None,
 132                  **kwargs):
 133         # TODO: There is a design flaw in python2, keyword args can be
 134         # fulfilled as positional arguments. This sounds really great, until
 135         # you realize that because of it you cannot use the splat operator with
 136         # args and create new keyword arguments.
 137         # What we really want is __init__(self, *args, new_arg=None, **kwargs),
 138         # but this doesn't work in python2. In python3 thanks to PEP3102, you
 139         # can in fact do just that
 140         # The work around is to explicitly pass the arguments down.
 141         super(PiglitGLTest, self).__init__(command, **kwargs)
 142
 143         assert not (require_platforms and exclude_platforms)
 144
 145         if not require_platforms or set(require_platforms).issubset(
 146                 set(core.PLATFORMS)):
 147             self.require_platforms = require_platforms or []
 148         else:
 149             raise Exception("Error: require_platform is not valid")
 150
 151         if (not exclude_platforms or
 152                 set(exclude_platforms).issubset(set(core.PLATFORMS))):
 153             self.exclude_platforms = exclude_platforms or []
 154         else:
 155             raise Exception("Error: exclude_platforms is not valid")
 156
 157     def is_skip(self):
 158         """ Native Piglit-test specific skip checking
 159
 160         If the platform for the run doesn't support glx (either directly as
 161         glx or through the hybrid glx/x11_egl setup that is default), then skip
 162         any glx specific tests.
 163
 164         """
 165         platform = options.OPTIONS.env['PIGLIT_PLATFORM']
 166         if self.require_platforms and platform not in self.require_platforms:
 167             raise TestIsSkip(
 168                 'Test requires one of the following platforms "{}" '
 169                 'but the platform is "{}"'.format(
 170                     self.require_platforms, platform))
 171         elif self.exclude_platforms and platform in self.exclude_platforms:
 172             raise TestIsSkip(
 173                 'Test cannot be run on any of the following platforms "{}" '
 174                 'and the platform is "{}"'.format(
 175                     self.exclude_platforms, platform))
 176         super(PiglitGLTest, self).is_skip()
 177
 178     @PiglitBaseTest.command.getter
 179     def command(self):
 180         """ Automatically add -auto and -fbo as appropriate """
 181         if not self.run_concurrent:
 182             return self.keys() + super(PiglitGLTest, self).command + ['-auto']
 183         else:
 184             return self.keys() + super(PiglitGLTest, self).command + ['-auto', '-fbo']
 185
 186     @command.setter
 187     def command(self, new):
 188         self._command = [n for n in new if n not in ['-auto', '-fbo']]
 189
 190
 191 class ASMParserTest(PiglitBaseTest):
 192
 193     """Test class for ASM parser tests."""
 194
 195     def __init__(self, type_, filename, env=None):
 196         super(ASMParserTest, self).__init__(['asmparsertest', type_], env=env)
 197         self.filename = filename
 198
 199     @PiglitBaseTest.command.getter
 200     def command(self):
 201         command = super(ASMParserTest, self).command
 202         return self.keys() + command + [os.path.join(ROOT_DIR, self.filename)]
 203
 204
 205 class BuiltInConstantsTest(PiglitBaseTest):
 206
 207     """Test class for handling built in constants tests."""
 208
 209     @PiglitBaseTest.command.getter
 210     def command(self):
 211         command = super(BuiltInConstantsTest, self).command
 212
 213         command[1] = os.path.join(ROOT_DIR, 'tests', command[1])
 214         return self.keys() + command;
 215
 216
 217 class PiglitCLTest(PiglitBaseTest):  # pylint: disable=too-few-public-methods
 218     """ OpenCL specific Test class.
 219
 220     Set concurrency based on CL requirements.
 221
 222     """
 223     def __init__(self, command, run_concurrent=CL_CONCURRENT, **kwargs):
 224         if self.timeout is None:
 225             self.timeout = 60
 226         super(PiglitCLTest, self).__init__(command, run_concurrent, **kwargs)
 227
 228
 229 class CLProgramTester(PiglitCLTest):
 230
 231     """Class for cl-program-tester tests."""
 232
 233     def __init__(self, filename, **kwargs):
 234         super(CLProgramTester, self).__init__(['cl-program-tester'], **kwargs)
 235         self.filename = filename
 236
 237     @PiglitCLTest.command.getter
 238     def command(self):
 239         command = super(CLProgramTester, self).command
 240         return self.keys() + command + [os.path.join(ROOT_DIR, self.filename)]
 241
 242
 243 class VkRunnerTest(PiglitBaseTest):
 244     """ Make a PiglitTest instance for a VkRunner shader test file """
 245
 246     def __init__(self, filename, env=None):
 247         vkrunner_bin = core.get_option('PIGLIT_VKRUNNER_BINARY',
 248                                        ('vkrunner', 'bin'),
 249                                        default='vkrunner')
 250
 251         super(VkRunnerTest, self).__init__(
 252             [vkrunner_bin],
 253             run_concurrent=True,
 254             env=env)
 255
 256         self.filename = filename
 257
 258     @PiglitBaseTest.command.getter
 259     def command(self):
 260         # self._command is used because we don't want PiglitBaseTest
 261         # to prepend TEST_BIN_DIR so that it will look for vkrunner in
 262         # the search path.
 263         return self.keys() + self._command + [os.path.join(ROOT_DIR, self.filename)]
 264
 265
 266 class PiglitReplayerTest(PiglitBaseTest):
 267     """ Make a PiglitTest instance for a Replayer test
 268
 269     Runs a single replayer test.
 270
 271     Arguments:
 272     extra_args -- to pass to replayer
 273
 274     """
 275
 276     RESULTS_PATH = None
 277
 278     def __init__(self, subcommand, extra_args, **kwargs):
 279         super(PiglitReplayerTest, self).__init__(
 280             ['replayer.py', subcommand, 'trace'], **kwargs)
 281         self.extra_args = extra_args
 282
 283     @PiglitBaseTest.command.getter
 284     def command(self):
 285         command = super(PiglitReplayerTest, self).command
 286         if self.RESULTS_PATH is not None:
 287             command += ['--output', self.RESULTS_PATH]
 288         return self.keys() + command + self.extra_args
 289
 290     def interpret_result(self):
 291         # Python's unhandled exceptions use "1" as exit code value.  We want to
 292         # interpret this as a CRASH. Therefore, we change the exit code value
 293         # to a negative one, which is the one the Test base class interprets as
 294         # a CRASH.
 295         if self.result.returncode == 1:
 296             self.result.returncode = -1
 297
 298         super(PiglitReplayerTest, self).interpret_result()