tests/xts.py

   1 # Copyright (c) 2013-2014 Intel Corporation
   2 #
   3 # Permission is hereby granted, free of charge, to any person
   4 # obtaining a copy of this software and associated documentation
   5 # files (the "Software"), to deal in the Software without
   6 # restriction, including without limitation the rights to use,
   7 # copy, modify, merge, publish, distribute, sublicense, and/or
   8 # sell copies of the Software, and to permit persons to whom the
   9 # Software is furnished to do so, subject to the following
  10 # conditions:
  11 #
  12 # This permission notice shall be included in all copies or
  13 # substantial portions of the Software.
  14 #
  15 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
  16 # KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
  17 # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
  18 # PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE AUTHOR(S) BE
  19 # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
  20 # AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
  21 # OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  22 # DEALINGS IN THE SOFTWARE.
  23
  24 """ Test integreation for the X Test Suite """
  25
  26 from __future__ import (
  27     absolute_import, division, print_function, unicode_literals
  28 )
  29 import os
  30 import re
  31 import subprocess
  32 import itertools
  33
  34 from framework import grouptools, exceptions, core
  35 from framework.profile import TestProfile, Test
  36
  37 __all__ = ['profile']
  38
  39 X_TEST_SUITE = core.PIGLIT_CONFIG.required_get('xts', 'path')
  40
  41
  42 class XTSProfile(TestProfile):  # pylint: disable=too-few-public-methods
  43     """ A subclass of TestProfile that provides a setup hook for XTS """
  44
  45     def setup(self):
  46         """This hook sets the XTSTest.results_path variable.
  47
  48         Setting this variable allows images created by XTS to moved into the
  49         results directory.
  50         """
  51         XTSTest.RESULTS_PATH = self.results_dir
  52
  53         try:
  54             os.mkdir(os.path.join(self.results_dir, 'images'))
  55         except OSError as e:
  56             # If the exception is not 'directory already exists', raise the
  57             # exception
  58             if e.errno != 17:
  59                 raise
  60
  61
  62 class XTSTest(Test):  # pylint: disable=too-few-public-methods
  63     """ X Test Suite class
  64
  65     Runs a single test or subtest from XTS.
  66
  67     Arguments:
  68     name -- the name of the test
  69     testname -- the name of the test file
  70     testnum -- the number of the test file
  71
  72     """
  73     RESULTS_PATH = None
  74
  75     def __init__(self, name, testname, testdir, testnum):
  76         super(XTSTest, self).__init__(
  77             ['./' + os.path.basename(name), '-i', str(testnum)])
  78         # Path relative to XTSTest.RESULTS_PATH (which is not
  79         # initialized at init time) to store any test-specific files.
  80         # We need to store into the results directory to protect
  81         # against races when multiple piglit-run.py -t xts commands
  82         # are running (as in the X Server's make check).
  83         self.testdir = testdir
  84         self.testname = '{0}-{1}'.format(testname, testnum)
  85         self.cwd = os.path.dirname(os.path.realpath(name))
  86         self.env.update(
  87             {"XT_RESET_DELAY": '0',
  88              "XT_FONTPATH_GOOD": '/usr/share/fonts/X11/misc',
  89              "XT_FONTPATH": os.path.join(X_TEST_SUITE, 'xts5', 'fonts'),
  90              # XXX: Are the next 3 necissary?
  91              "XT_LOCAL": 'Yes',
  92              "XT_TCP": 'No',
  93              "XT_DISPLAYHOST": ''})
  94
  95     def run(self):
  96         # We only get the RESULTS_PATH after the profile has been set
  97         # up, so we can't do it in init.
  98         self.test_results_file = os.path.join(XTSTest.RESULTS_PATH,
  99                                               self.testdir,
 100                                               self.testname)
 101         self.env.update({"TET_RESFILE": self.test_results_file})
 102         super(XTSTest, self).run()
 103
 104     def _process_log_for_images(self, log):
 105         """ Parse the image logfile """
 106         images = []
 107         search = re.compile('See file (Err[0-9]+.err)')
 108
 109         for line in log.splitlines():
 110             match = search.search(line)
 111             if match is not None:
 112                 # Can we parse any other useful information out to give a
 113                 # better description of each image?
 114                 desc = match.group(1)
 115
 116                 # The error logs are text, with a header with width, height,
 117                 # and depth, then run-length-encoded pixel values (in
 118                 # hexadecimal).  Use xtsttopng to convert the error log to a
 119                 # pair of PNGs so we can put them in the summary.
 120                 command = ['xtsttopng', os.path.join(XTSTest.RESULTS_PATH,
 121                                                      self.testdir,
 122                                                      match.group(1))]
 123                 try:
 124                     out = subprocess.check_output(command, cwd=self.cwd)
 125                 except OSError:
 126                     images.append({'image_desc': 'image processing failed'})
 127                     continue
 128
 129                 # Each Err*.err log contains a rendered image, and a reference
 130                 # image that it was compared to.  We relocate the to our tree
 131                 # with more useful names.  (Otherwise, since tests generate
 132                 # error logs with numbers sequentially starting from 0, each
 133                 # subtest with an error would overwrite the previous test's
 134                 # images).
 135                 ref_path = os.path.join(
 136                     self.RESULTS_PATH, 'images', '{1}-{2}-ref.png'.format(
 137                         self.testname, match.group(1)))
 138                 render_path = os.path.join(
 139                     self.RESULTS_PATH, 'images', '{1}-{2}-render.png'.format(
 140                         self.testname, match.group(1)))
 141
 142                 split = out.splitlines()
 143                 os.rename(os.path.join(self.cwd, split[0]), render_path)
 144                 os.rename(os.path.join(self.cwd, split[1]), ref_path)
 145
 146                 images.append({'image_desc': desc,
 147                                'image_ref': ref_path,
 148                                'image_render': render_path})
 149
 150         return images
 151
 152     def interpret_result(self):
 153         super(XTSTest, self).interpret_result()
 154
 155         try:
 156             with open(self.test_results_file, 'r') as rfile:
 157                 log = rfile.read()
 158                 self.result.out = log
 159                 os.remove(self.test_results_file)
 160         except IOError:
 161             self.result.err = "No results file found"
 162             log = ""
 163
 164         if self.result.returncode == 0:
 165             if re.search('FAIL', self.result.out) is not None:
 166                 self.result.result = 'fail'
 167             elif re.search('PASS', self.result.out) is not None:
 168                 self.result.result = 'pass'
 169             else:
 170                 self.result.result = 'fail'
 171         elif self.result.returncode == 77:
 172             self.result.result = 'skip'
 173         elif self.result.returncode == 1:
 174             if re.search('Could not open all VSW5 fonts', log):
 175                 self.result.result = 'warn'
 176             else:
 177                 self.result.result = 'fail'
 178
 179         self.result.images = self._process_log_for_images(log)
 180
 181
 182 class RendercheckTest(Test):
 183     def __init__(self, args):
 184         super(RendercheckTest, self).__init__(['rendercheck'] + args)
 185         self.testname = "rendercheck " + " ".join(args)
 186
 187     def interpret_result(self):
 188         super(RendercheckTest, self).interpret_result()
 189
 190         if self.result.returncode == 0:
 191             self.result.result = 'pass'
 192         elif self.result.returncode == 77:
 193             self.result.result = 'skip'
 194
 195
 196 def _populate_profile_xts(profile):
 197     fpath = os.path.join(X_TEST_SUITE, 'xts5')
 198     for dirpath, _, filenames in os.walk(fpath):
 199         for fname in filenames:
 200             # only look at the .m test files
 201             testname, ext = os.path.splitext(fname)
 202             if ext != '.m':
 203                 continue
 204
 205             # incrementing number generator
 206             counts = itertools.count(start=1)
 207
 208             # Walk the file looking for >>ASSERTION, each of these corresponds
 209             # to a generated subtest, there can be multiple subtests per .m
 210             # file
 211             with open(os.path.join(dirpath, fname), 'r') as rfile:
 212                 for line in rfile:
 213                     if line.startswith('>>ASSERTION'):
 214                         num = next(counts)
 215                         group = grouptools.join(
 216                             grouptools.from_path(
 217                                 os.path.relpath(dirpath, X_TEST_SUITE)),
 218                             testname, str(num))
 219
 220                         profile.test_list[group] = XTSTest(
 221                             os.path.join(dirpath, testname),
 222                             testname,
 223                             os.path.relpath(dirpath, X_TEST_SUITE),
 224                             num)
 225
 226
 227 def _add_rendercheck_test(profile, path, args):
 228     test = RendercheckTest(args.split(' '))
 229     group_path = 'rendercheck/' + path
 230     group = grouptools.join(*(group_path.split('/')))
 231     profile.test_list[group] = test
 232
 233
 234 def _populate_profile_rendercheck(profile):
 235     _add_rendercheck_test(profile, 'blend/All/a8r8g8b8', '-t blend -f a8r8g8b8')
 236     _add_rendercheck_test(profile, 'blend/All/x8r8g8b8', '-t blend -f a8r8g8b8,x8r8g8b8')
 237     _add_rendercheck_test(profile, 'blend/All/a2r10g10b10', '-t blend -f a8r8g8b8,a2r10g10b10')
 238     _add_rendercheck_test(profile, 'blend/Clear', '-t blend -o clear')
 239     _add_rendercheck_test(profile, 'blend/Src', '-t blend -o src')
 240     _add_rendercheck_test(profile, 'blend/Over', '-t blend -o over')
 241     _add_rendercheck_test(profile, 'composite/All/a8r8g8b8', '-t composite -f a8r8g8b8')
 242     _add_rendercheck_test(profile, 'composite/All/x8r8g8b8', '-t composite -f a8r8g8b8,x8r8g8b8')
 243     _add_rendercheck_test(profile, 'composite/All/a2r10g10b10', '-t composite -f a8r8g8b8,a2r10g10b10')
 244     _add_rendercheck_test(profile, 'ca composite/All/a8r8g8b8', '-t cacomposite -f a8r8g8b8')
 245     _add_rendercheck_test(profile, 'ca composite/All/a8', '-t cacomposite -f a8r8g8b8,a8')
 246     _add_rendercheck_test(profile, 'ca composite/All/x8r8g8b8', '-t cacomposite -f a8r8g8b8,x8r8g8b8')
 247     _add_rendercheck_test(profile, 'ca composite/All/a2r10g10b10', '-t cacomposite -f a8r8g8b8,a2r10g10b10')
 248     _add_rendercheck_test(profile, 'fill', '-t fill')
 249     _add_rendercheck_test(profile, 'bug7366', '-t bug7366')
 250     _add_rendercheck_test(profile, 'destination coordinates', '-t dcoords')
 251     _add_rendercheck_test(profile, 'source coordinates', '-t scoords')
 252     _add_rendercheck_test(profile, 'mask coordinates', '-t mcoords')
 253     _add_rendercheck_test(profile, 'translated source coordinates', '-t tscoords')
 254     _add_rendercheck_test(profile, 'translated mask coordinates', '-t tmcoords')
 255     _add_rendercheck_test(profile, 'triangles', '-t triangles')
 256     _add_rendercheck_test(profile, 'LibreOffice xRGB', '-t libreoffice_xrgb')
 257     _add_rendercheck_test(profile, 'GTK ARGB vs xBGR', '-t gtk_argb_xbgr')
 258
 259
 260 def _populate_profile():
 261     """ Populate the profile attribute """
 262     # Add all tests to the profile
 263     profile = XTSProfile()  # pylint: disable=redefined-outer-name
 264     _populate_profile_xts(profile)
 265     _populate_profile_rendercheck(profile)
 266     return profile
 267
 268
 269 if not os.path.exists(X_TEST_SUITE):
 270     raise exceptions.PiglitFatalError('XTest suite not found.')
 271
 272 profile = _populate_profile()  # pylint: disable=invalid-name