framework/test/piglit_test.py

   1 # coding=utf-8
   2 #
   3 # Copyright © 2021 Collabora Ltd.
   4 #
   5 # Permission is hereby granted, free of charge, to any person
   6 # obtaining a copy of this software and associated documentation
   7 # files (the "Software"), to deal in the Software without
   8 # restriction, including without limitation the rights to use,
   9 # copy, modify, merge, publish, distribute, sublicense, and/or
  10 # sell copies of the Software, and to permit persons to whom the
  11 # Software is furnished to do so, subject to the following
  12 # conditions:
  13 #
  14 # This permission notice shall be included in all copies or
  15 # substantial portions of the Software.
  16 #
  17 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
  18 # KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
  19 # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
  20 # PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE AUTHOR(S) BE
  21 # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
  22 # AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
  23 # OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  24 # DEALINGS IN THE SOFTWARE.
  25
  26 """ Module provides a base class for Tests """
  27
  28 import glob
  29 import os
  30 import sys
  31 import json
  32
  33 from framework import core, options
  34 from framework import status
  35 from .base import Test, WindowResizeMixin, ValgrindMixin, TestIsSkip
  36
  37
  38 __all__ = [
  39     'PiglitCLTest',
  40     'PiglitGLTest',
  41     'PiglitBaseTest',
  42     'PiglitReplayerTest',
  43     'VkRunnerTest',
  44     'CL_CONCURRENT',
  45     'ROOT_DIR',
  46     'TEST_BIN_DIR',
  47 ]
  48
  49 if 'PIGLIT_BUILD_DIR' in os.environ:
  50     ROOT_DIR = os.environ['PIGLIT_BUILD_DIR']
  51 else:
  52     ROOT_DIR = os.path.normpath(os.path.join(os.path.dirname(__file__), '../..'))
  53
  54 TEST_BIN_DIR = os.path.normpath(os.path.join(ROOT_DIR, 'bin'))
  55
  56 CL_CONCURRENT = bool(not sys.platform.startswith('linux') or
  57                      glob.glob('/dev/dri/render*'))
  58
  59
  60 class PiglitBaseTest(ValgrindMixin, Test):
  61     """
  62     PiglitTest: Run a "native" piglit test executable
  63
  64     Expect one line prefixed PIGLIT: in the output, which contains a result
  65     dictionary. The plain output is appended to this dictionary
  66     """
  67     def __init__(self, command, run_concurrent=True, **kwargs):
  68         super(PiglitBaseTest, self).__init__(command, run_concurrent, **kwargs)
  69
  70     @Test.command.getter
  71     def command(self):
  72         command = super(PiglitBaseTest, self).command
  73
  74         def fixup_bin_path(c):
  75             # Prepend TEST_BIN_DIR to the path.
  76             if c == self._command[0]:
  77                 return os.path.join(TEST_BIN_DIR, c)
  78             else:
  79                 return c
  80
  81         return [fixup_bin_path(c) for c in command]
  82
  83     def interpret_result(self):
  84         out = []
  85
  86         for each in self.result.out.split('\n'):
  87             if each.startswith('PIGLIT:'):
  88                 deserial = json.loads(each[8:])
  89                 if 'enumerate subtests' in deserial:
  90                     for n in deserial['enumerate subtests']:
  91                         self.result.subtests[n] = status.NOTRUN
  92                 else:
  93                     self.result.update(deserial)
  94             else:
  95                 out.append(each)
  96
  97         self.result.out = '\n'.join(out)
  98
  99         # XXX: There are a number of tests that now enumerate their subtests,
 100         # but don't properly report skip for all of them when the skip due to a
 101         # missing feature. We should fix these tests to do the right thing, but
 102         # for the moment this work around will suffice to keep things running.
 103         if self.result.raw_result is status.SKIP and self.result.subtests:
 104             for k, v in self.result.subtests.items():
 105                 if v is status.NOTRUN:
 106                     self.result.subtests[k] = status.SKIP
 107
 108         super(PiglitBaseTest, self).interpret_result()
 109
 110
 111 class PiglitGLTest(WindowResizeMixin, PiglitBaseTest):
 112     """ OpenGL specific Piglit test class
 113
 114     This Subclass provides provides an is_skip() implementation that skips glx
 115     tests on non-glx platforms
 116
 117     This class also provides two additional keyword arguments, require_platform
 118     and exclude_platforms. require_platforms may be set to a list of platforms
 119     which the test requires to run. This should be resereved for platform
 120     specific tests, such as GLX specific tests, or EGL specific tests. Multiple
 121     platforms are allowed because EGL can be fulfilled by multiple platforms.
 122     exclude_platforms is a list of platforms a test should not be run on, this
 123     is useful for tests that are valid on more than one platform, but not on
 124     all of them. This will probably be mainly used to exclude gbm. These
 125     options are mutually exclusive.
 126
 127     """
 128     def __init__(self, command, require_platforms=None, exclude_platforms=None,
 129                  **kwargs):
 130         # TODO: There is a design flaw in python2, keyword args can be
 131         # fulfilled as positional arguments. This sounds really great, until
 132         # you realize that because of it you cannot use the splat operator with
 133         # args and create new keyword arguments.
 134         # What we really want is __init__(self, *args, new_arg=None, **kwargs),
 135         # but this doesn't work in python2. In python3 thanks to PEP3102, you
 136         # can in fact do just that
 137         # The work around is to explicitly pass the arguments down.
 138         super(PiglitGLTest, self).__init__(command, **kwargs)
 139
 140         assert not (require_platforms and exclude_platforms)
 141
 142         if not require_platforms or set(require_platforms).issubset(
 143                 set(core.PLATFORMS)):
 144             self.require_platforms = require_platforms or []
 145         else:
 146             raise Exception("Error: require_platform is not valid")
 147
 148         if (not exclude_platforms or
 149                 set(exclude_platforms).issubset(set(core.PLATFORMS))):
 150             self.exclude_platforms = exclude_platforms or []
 151         else:
 152             raise Exception("Error: exclude_platforms is not valid")
 153
 154     def is_skip(self):
 155         """ Native Piglit-test specific skip checking
 156
 157         If the platform for the run doesn't support glx (either directly as
 158         glx or through the hybrid glx/x11_egl setup that is default), then skip
 159         any glx specific tests.
 160
 161         """
 162         platform = options.OPTIONS.env['PIGLIT_PLATFORM']
 163         if self.require_platforms and platform not in self.require_platforms:
 164             raise TestIsSkip(
 165                 'Test requires one of the following platforms "{}" '
 166                 'but the platform is "{}"'.format(
 167                     self.require_platforms, platform))
 168         elif self.exclude_platforms and platform in self.exclude_platforms:
 169             raise TestIsSkip(
 170                 'Test cannot be run on any of the following platforms "{}" '
 171                 'and the platform is "{}"'.format(
 172                     self.exclude_platforms, platform))
 173         super(PiglitGLTest, self).is_skip()
 174
 175     @PiglitBaseTest.command.getter
 176     def command(self):
 177         """ Automatically add -auto and -fbo as appropriate """
 178         if not self.run_concurrent:
 179             return self.keys() + super(PiglitGLTest, self).command + ['-auto']
 180         else:
 181             return self.keys() + super(PiglitGLTest, self).command + ['-auto', '-fbo']
 182
 183     @command.setter
 184     def command(self, new):
 185         self._command = [n for n in new if n not in ['-auto', '-fbo']]
 186
 187
 188 class ASMParserTest(PiglitBaseTest):
 189
 190     """Test class for ASM parser tests."""
 191
 192     def __init__(self, type_, filename, env=None):
 193         super(ASMParserTest, self).__init__(['asmparsertest', type_], env=env)
 194         self.filename = filename
 195
 196     @PiglitBaseTest.command.getter
 197     def command(self):
 198         command = super(ASMParserTest, self).command
 199         return self.keys() + command + [os.path.join(ROOT_DIR, self.filename)]
 200
 201
 202 class BuiltInConstantsTest(PiglitBaseTest):
 203
 204     """Test class for handling built in constants tests."""
 205
 206     @PiglitBaseTest.command.getter
 207     def command(self):
 208         command = super(BuiltInConstantsTest, self).command
 209
 210         command[1] = os.path.join(ROOT_DIR, 'tests', command[1])
 211         return self.keys() + command;
 212
 213
 214 class PiglitCLTest(PiglitBaseTest):  # pylint: disable=too-few-public-methods
 215     """ OpenCL specific Test class.
 216
 217     Set concurrency based on CL requirements.
 218
 219     """
 220     def __init__(self, command, run_concurrent=CL_CONCURRENT, **kwargs):
 221         if self.timeout is None:
 222             self.timeout = 60
 223         super(PiglitCLTest, self).__init__(command, run_concurrent, **kwargs)
 224
 225
 226 class CLProgramTester(PiglitCLTest):
 227
 228     """Class for cl-program-tester tests."""
 229
 230     def __init__(self, filename, **kwargs):
 231         super(CLProgramTester, self).__init__(['cl-program-tester'], **kwargs)
 232         self.filename = filename
 233
 234     @PiglitCLTest.command.getter
 235     def command(self):
 236         command = super(CLProgramTester, self).command
 237         return self.keys() + command + [os.path.join(ROOT_DIR, self.filename)]
 238
 239
 240 class VkRunnerTest(PiglitBaseTest):
 241     """ Make a PiglitTest instance for a VkRunner shader test file """
 242
 243     def __init__(self, filename, env=None):
 244         vkrunner_bin = core.get_option('PIGLIT_VKRUNNER_BINARY',
 245                                        ('vkrunner', 'bin'),
 246                                        default='vkrunner')
 247
 248         super(VkRunnerTest, self).__init__(
 249             [vkrunner_bin],
 250             run_concurrent=True,
 251             env=env)
 252
 253         self.filename = filename
 254
 255     @PiglitBaseTest.command.getter
 256     def command(self):
 257         # self._command is used because we don't want PiglitBaseTest
 258         # to prepend TEST_BIN_DIR so that it will look for vkrunner in
 259         # the search path.
 260         return self.keys() + self._command + [os.path.join(ROOT_DIR, self.filename)]
 261
 262
 263 class PiglitReplayerTest(PiglitBaseTest):
 264     """ Make a PiglitTest instance for a Replayer test
 265
 266     Runs a single replayer test.
 267
 268     Arguments:
 269     extra_args -- to pass to replayer
 270
 271     """
 272
 273     RESULTS_PATH = None
 274
 275     def __init__(self, subcommand, extra_args, **kwargs):
 276         super(PiglitReplayerTest, self).__init__(
 277             ['replayer.py', subcommand, 'trace'], **kwargs)
 278         self.extra_args = extra_args
 279
 280     @PiglitBaseTest.command.getter
 281     def command(self):
 282         command = super(PiglitReplayerTest, self).command
 283         if self.RESULTS_PATH is not None:
 284             command += ['--output', self.RESULTS_PATH]
 285         return self.keys() + command + self.extra_args
 286
 287     def interpret_result(self):
 288         # Python's unhandled exceptions use "1" as exit code value.  We want to
 289         # interpret this as a CRASH. Therefore, we change the exit code value
 290         # to a negative one, which is the one the Test base class interprets as
 291         # a CRASH.
 292         if self.result.returncode == 1:
 293             self.result.returncode = -1
 294
 295         super(PiglitReplayerTest, self).interpret_result()