framework/test/shader_test.py

   1 # coding=utf-8
   2 # Copyright (C) 2012, 2014-2016, 2019 Intel Corporation
   3 #
   4 # Permission is hereby granted, free of charge, to any person
   5 # obtaining a copy of this software and associated documentation
   6 # files (the "Software"), to deal in the Software without
   7 # restriction, including without limitation the rights to use,
   8 # copy, modify, merge, publish, distribute, sublicense, and/or
   9 # sell copies of the Software, and to permit persons to whom the
  10 # Software is furnished to do so, subject to the following
  11 # conditions:
  12 #
  13 # This permission notice shall be included in all copies or
  14 # substantial portions of the Software.
  15 #
  16 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
  17 # KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
  18 # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
  19 # PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE AUTHOR(S) BE
  20 # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
  21 # AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
  22 # OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  23 # DEALINGS IN THE SOFTWARE.
  24
  25 """ This module enables running shader tests. """
  26
  27 import io
  28 import os
  29 import re
  30
  31 from framework import exceptions
  32 from framework import status
  33 from framework import options
  34 from .base import ReducedProcessMixin, TestIsSkip
  35 from .opengl import FastSkipMixin, FastSkip
  36 from .piglit_test import PiglitBaseTest, ROOT_DIR
  37
  38 __all__ = [
  39     'ShaderTest',
  40 ]
  41
  42
  43 class Parser(object):
  44     """An object responsible for parsing a shader_test file."""
  45
  46     _is_gl = re.compile(r'GL (<|<=|=|>=|>) \d\.\d')
  47     _match_gl_version = re.compile(
  48         r'^GL\s+(?P<profile>(ES|CORE|COMPAT))?\s*(?P<op>(<|<=|=|>=|>))\s*(?P<ver>\d\.\d)')
  49     _match_glsl_version = re.compile(
  50         r'^GLSL\s+(?P<es>ES)?\s*(?P<op>(<|<=|=|>=|>))\s*(?P<ver>\d\.\d+)')
  51
  52     def __init__(self, filename):
  53         self.filename = filename
  54         self.extensions = set()
  55         self.api_version = 0.0
  56         self.shader_version = 0.0
  57         self.api = None
  58         self.prog = None
  59         self.__op = None
  60         self.__sl_op = None
  61
  62     def parse(self):
  63         # Iterate over the lines in shader file looking for the config section.
  64         # By using a generator this can be split into two for loops at minimal
  65         # cost. The first one looks for the start of the config block or raises
  66         # an exception. The second looks for the GL version or raises an
  67         # exception
  68         with io.open(os.path.join(ROOT_DIR, self.filename), 'r', encoding='utf-8') as shader_file:
  69             lines = (l for l in shader_file.readlines())
  70
  71             # Find the config section
  72             for line in lines:
  73                 # We need to find the first line of the configuration file, as
  74                 # soon as we do then we can move on to getting the
  75                 # configuration. The first line needs to be parsed by the next
  76                 # block.
  77                 if line.startswith('[require]'):
  78                     break
  79             else:
  80                 raise exceptions.PiglitFatalError(
  81                     "In file {}: Config block not found".format(self.filename))
  82
  83         for line in lines:
  84             if line.startswith('GL_'):
  85                 line = line.strip()
  86                 if not (line.startswith('GL_MAX') or line.startswith('GL_NUM')):
  87                     self.extensions.add(line)
  88                     if line == 'GL_ARB_compatibility':
  89                         assert self.api is None or self.api == 'compat'
  90                         self.api = 'compat'
  91                 continue
  92
  93             # Find any GLES requirements.
  94             if not self.api_version:
  95                 m = self._match_gl_version.match(line)
  96                 if m:
  97                     self.__op = m.group('op')
  98                     self.api_version = float(m.group('ver'))
  99                     if m.group('profile') == 'ES':
 100                         assert self.api is None or self.api == 'gles2'
 101                         self.api = 'gles2'
 102                     elif m.group('profile') == 'COMPAT':
 103                         assert self.api is None or self.api == 'compat'
 104                         self.api = 'compat'
 105                     elif self.api_version >= 3.1:
 106                         assert self.api is None or self.api == 'core'
 107                         self.api = 'core'
 108                     continue
 109
 110             if not self.shader_version:
 111                 # Find any GLSL requirements
 112                 m = self._match_glsl_version.match(line)
 113                 if m:
 114                     self.__sl_op = m.group('op')
 115                     self.shader_version = float(m.group('ver'))
 116                     if m.group('es'):
 117                         assert self.api is None or self.api == 'gles2'
 118                         self.api = 'gles2'
 119                     continue
 120
 121             if line.startswith('['):
 122                 if not self.api:
 123                     # Because this is inferred rather than explicitly declared
 124                     # check this after al other requirements are parsed. It's
 125                     # possible that a test can declare glsl >= 1.30 and GL >=
 126                     # 4.0
 127                     if self.shader_version < 1.4:
 128                         self.api = 'compat'
 129                     else:
 130                         self.api = 'core'
 131                 break
 132
 133         # Select the correct binary to run the test, but be as conservative as
 134         # possible by always selecting the lowest version that meets the
 135         # criteria.
 136         if self.api == 'gles2':
 137             if self.__op in ['<', '<='] or (
 138                     self.__op in ['=', '>='] and self.api_version is not None
 139                     and self.api_version < 3):
 140                 self.prog = 'shader_runner_gles2'
 141             else:
 142                 self.prog = 'shader_runner_gles3'
 143         else:
 144             self.prog = 'shader_runner'
 145
 146
 147 class ShaderTest(FastSkipMixin, PiglitBaseTest):
 148     """ Parse a shader test file and return a PiglitTest instance
 149
 150     This function parses a shader test to determine if it's a GL, GLES2 or
 151     GLES3 test, and then returns a PiglitTest setup properly.
 152
 153     """
 154
 155     def __init__(self, command, api=None, extensions=set(),
 156                   shader_version=None, api_version=None, env=None, **kwargs):
 157         super(ShaderTest, self).__init__(
 158             command,
 159             run_concurrent=True,
 160             api=api,
 161             extensions=extensions,
 162             shader_version=shader_version,
 163             api_version=api_version,
 164             env=env)
 165
 166     @classmethod
 167     def new(cls, filename, installed_name=None):
 168         """Parse an XML file and create a new instance.
 169
 170         :param str filename: The name of the file to parse
 171         :param str installed_name: The relative path to the file when installed
 172             if not the same as the parsed name
 173         """
 174         parser = Parser(filename)
 175         parser.parse()
 176
 177         return cls(
 178             [parser.prog, installed_name or filename],
 179             run_concurrent=True,
 180             api=parser.api,
 181             extensions=parser.extensions,
 182             shader_version=parser.shader_version,
 183             api_version=parser.api_version)
 184
 185     @PiglitBaseTest.command.getter
 186     def command(self):
 187         """ Add -auto, -fbo and -glsl (if needed) to the test command """
 188
 189         command = super(ShaderTest, self).command
 190         shaderfile = os.path.join(ROOT_DIR, self._command[1])
 191
 192         if options.OPTIONS.force_glsl:
 193             return self.keys() + [command[0]] + [shaderfile, '-auto', '-fbo', '-glsl']
 194         else:
 195             return self.keys() + [command[0]] + [shaderfile, '-auto', '-fbo']
 196
 197     @command.setter
 198     def command(self, new):
 199         self._command = [n for n in new if n not in ['-auto', '-fbo']]
 200
 201
 202 class MultiShaderTest(ReducedProcessMixin, PiglitBaseTest):
 203     """A Shader class that can run more than one test at a time.
 204
 205     This class can call shader_runner with multiple shader_files at a time, and
 206     interpret the results, as well as handle pre-mature exit through crashes or
 207     from breaking import assupmtions in the utils about skipping.
 208
 209     Arguments:
 210     filenames -- a list of absolute paths to shader test files
 211     """
 212
 213     def __init__(self, prog, files, subtests, skips, env=None):
 214         super(MultiShaderTest, self).__init__(
 215             [prog] + files,
 216             subtests=subtests,
 217             run_concurrent=True,
 218             env=env)
 219
 220         self.prog = prog
 221         self.files = files
 222         self.subtests = subtests
 223         self.skips = [FastSkip(**s) for s in skips]
 224
 225     @classmethod
 226     def new(cls, filenames, installednames=None):
 227         # TODO
 228         assert filenames
 229         prog = None
 230         subtests = []
 231         skips = []
 232
 233         # Walk each subtest, and either add it to the list of tests to run, or
 234         # determine it is skip, and set the result of that test in the subtests
 235         # dictionary to skip without adding it to the list of tests to run.
 236         for each in filenames:
 237             parser = Parser(each)
 238             parser.parse()
 239             subtests.append(os.path.basename(os.path.splitext(each)[0]).lower())
 240
 241             if prog is not None:
 242                 # This allows mixing GLES2 and GLES3 shader test files
 243                 # together. Since GLES2 profiles can be promoted to GLES3, this
 244                 # is fine.
 245                 if parser.prog != prog:
 246                     # Pylint can't figure out that prog is not None.
 247                     if 'gles' in parser.prog and 'gles' in prog:  # pylint: disable=unsupported-membership-test
 248                         prog = max(parser.prog, prog)
 249                     else:
 250                         # The only way we can get here is if one is GLES and
 251                         # one is not, since there is only one desktop runner
 252                         # thus it will never fail the is parser.prog != prog
 253                         # check
 254                         raise exceptions.PiglitInternalError(
 255                             'GLES and GL shaders in the same command!\n'
 256                             'Cannot pick a shader_runner binary!\n'
 257                             'in file: {}'.format(os.path.dirname(each)))
 258             else:
 259                 prog = parser.prog
 260
 261             skips.append({
 262                 'extensions': parser.extensions,
 263                 'api_version': parser.api_version,
 264                 'shader_version': parser.shader_version,
 265                 'api': parser.api,
 266             })
 267
 268         return cls(prog, installednames or filenames, subtests, skips)
 269
 270     def _process_skips(self):
 271         r_files = []
 272         r_subtests = []
 273         r_skips = []
 274         for f, s, k in zip(self.files, self.subtests, self.skips):
 275             try:
 276                 k.test()
 277             except TestIsSkip:
 278                 r_skips.append(s)
 279             else:
 280                 r_files.append(f)
 281                 r_subtests.append(s)
 282
 283         assert len(r_subtests) + len(r_skips) == len(self.files), \
 284             'not all tests accounted for'
 285
 286         for name in r_skips:
 287             self.result.subtests[name] = status.SKIP
 288
 289         self._expected = r_subtests
 290         self._command = [self._command[0]] + r_files
 291
 292     def run(self):
 293         self._process_skips()
 294         super(MultiShaderTest, self).run()
 295
 296     @PiglitBaseTest.command.getter
 297     def command(self):
 298         command = super(MultiShaderTest, self).command
 299         shaderfiles = (x for x in command[1:] if not x.startswith('-'))
 300         shaderfiles = [os.path.join(ROOT_DIR, s) for s in shaderfiles]
 301         return self.keys() + [command[0]] + shaderfiles + ['-auto', '-report-subtests']
 302
 303     def _is_subtest(self, line):
 304         return line.startswith('PIGLIT TEST:')
 305
 306     def _resume(self, current):
 307         command = [self.command[0]]
 308         command.extend(self.command[current + 1:])
 309         return command
 310
 311     def _stop_status(self):
 312         # If the lower level framework skips then return a status for that
 313         # subtest as skip, and resume.
 314         if self.result.out.endswith('PIGLIT: {"result": "skip" }\n'):
 315             return status.SKIP
 316         if self.result.returncode > 0:
 317             return status.FAIL
 318         return status.CRASH
 319
 320     def _is_cherry(self):
 321         # Due to the way that piglt is architected if a particular feature
 322         # isn't supported it causes the test to exit with status 0. There is no
 323         # straightforward way to fix this, so we work around it by looking for
 324         # the message that feature provides and marking the test as not
 325         # "cherry" when it is found at the *end* of stdout. (We don't want to
 326         # match other places or we'll end up in an infinite loop)
 327         return (
 328             self.result.returncode == 0 and not
 329             self.result.out.endswith(
 330                 'not supported on this implementation\n') and not
 331             self.result.out.endswith(
 332                 'PIGLIT: {"result": "skip" }\n'))