tests/xts.py

   1 # coding=utf-8
   2 # Copyright (c) 2013-2014 Intel Corporation
   3 #
   4 # Permission is hereby granted, free of charge, to any person
   5 # obtaining a copy of this software and associated documentation
   6 # files (the "Software"), to deal in the Software without
   7 # restriction, including without limitation the rights to use,
   8 # copy, modify, merge, publish, distribute, sublicense, and/or
   9 # sell copies of the Software, and to permit persons to whom the
  10 # Software is furnished to do so, subject to the following
  11 # conditions:
  12 #
  13 # This permission notice shall be included in all copies or
  14 # substantial portions of the Software.
  15 #
  16 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
  17 # KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
  18 # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
  19 # PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE AUTHOR(S) BE
  20 # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
  21 # AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
  22 # OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  23 # DEALINGS IN THE SOFTWARE.
  24
  25 """ Test integreation for the X Test Suite """
  26
  27 import os
  28 import re
  29 import subprocess
  30 import itertools
  31
  32 from framework import grouptools, exceptions, core
  33 from framework.profile import TestProfile, Test
  34
  35 __all__ = ['profile']
  36
  37 X_TEST_SUITE = core.PIGLIT_CONFIG.required_get('xts', 'path')
  38
  39
  40 class XTSProfile(TestProfile):  # pylint: disable=too-few-public-methods
  41     """ A subclass of TestProfile that provides a setup hook for XTS """
  42
  43     def setup(self):
  44         """This hook sets the XTSTest.results_path variable.
  45
  46         Setting this variable allows images created by XTS to be moved into the
  47         results directory.
  48         """
  49         XTSTest.RESULTS_PATH = self.results_dir
  50
  51         try:
  52             os.mkdir(os.path.join(self.results_dir, 'images'))
  53         except OSError as e:
  54             # If the exception is not 'directory already exists', raise the
  55             # exception
  56             if e.errno != 17:
  57                 raise
  58
  59
  60 class XTSTest(Test):  # pylint: disable=too-few-public-methods
  61     """ X Test Suite class
  62
  63     Runs a single test or subtest from XTS.
  64
  65     Arguments:
  66     name -- the name of the test
  67     testname -- the name of the test file
  68     testnum -- the number of the test file
  69
  70     """
  71     RESULTS_PATH = None
  72
  73     def __init__(self, name, testname, testdir, testnum):
  74         super(XTSTest, self).__init__(
  75             ['./' + os.path.basename(name), '-i', str(testnum)])
  76         # Path relative to XTSTest.RESULTS_PATH (which is not
  77         # initialized at init time) to store any test-specific files.
  78         # We need to store into the results directory to protect
  79         # against races when multiple piglit-run.py -t xts commands
  80         # are running (as in the X Server's make check).
  81         self.testdir = testdir
  82         self.testname = '{0}-{1}'.format(testname, testnum)
  83         self.cwd = os.path.dirname(os.path.realpath(name))
  84         self.env.update(
  85             {"XT_RESET_DELAY": '0',
  86              "XT_FONTPATH_GOOD": '/usr/share/fonts/X11/misc',
  87              "XT_FONTPATH": os.path.join(X_TEST_SUITE, 'xts5', 'fonts'),
  88              # XXX: Are the next 3 necessary?
  89              "XT_LOCAL": 'Yes',
  90              "XT_TCP": 'No',
  91              "XT_DISPLAYHOST": ''})
  92
  93     def run(self):
  94         # We only get the RESULTS_PATH after the profile has been set
  95         # up, so we can't do it in init.
  96         self.test_results_file = os.path.join(XTSTest.RESULTS_PATH,
  97                                               self.testdir,
  98                                               self.testname)
  99         self.env.update({"TET_RESFILE": self.test_results_file})
 100         super(XTSTest, self).run()
 101
 102     def _process_log_for_images(self, log):
 103         """ Parse the image logfile """
 104         images = []
 105         search = re.compile('See file (Err[0-9]+.err)')
 106
 107         for line in log.splitlines():
 108             match = search.search(line)
 109             if match is not None:
 110                 # Can we parse any other useful information out to give a
 111                 # better description of each image?
 112                 desc = match.group(1)
 113
 114                 # The error logs are text, with a header with width, height,
 115                 # and depth, then run-length-encoded pixel values (in
 116                 # hexadecimal).  Use xtsttopng to convert the error log to a
 117                 # pair of PNGs so we can put them in the summary.
 118                 command = ['xtsttopng', os.path.join(XTSTest.RESULTS_PATH,
 119                                                      self.testdir,
 120                                                      match.group(1))]
 121                 try:
 122                     out = subprocess.check_output(command, cwd=self.cwd)
 123                 except OSError:
 124                     images.append({'image_desc': 'image processing failed',
 125                                    'image_ref': None,
 126                                    'image_render': None})
 127                     continue
 128
 129                 # Each Err*.err log contains a rendered image, and a reference
 130                 # image that it was compared to.  We relocate the to our tree
 131                 # with more useful names.  (Otherwise, since tests generate
 132                 # error logs with numbers sequentially starting from 0, each
 133                 # subtest with an error would overwrite the previous test's
 134                 # images).
 135                 ref_path = os.path.join(
 136                     self.RESULTS_PATH, 'images', '{1}-{2}-ref.png'.format(
 137                         self.testname, match.group(1)))
 138                 render_path = os.path.join(
 139                     self.RESULTS_PATH, 'images', '{1}-{2}-render.png'.format(
 140                         self.testname, match.group(1)))
 141
 142                 split = out.splitlines()
 143                 os.rename(os.path.join(self.cwd, split[0]), render_path)
 144                 os.rename(os.path.join(self.cwd, split[1]), ref_path)
 145
 146                 images.append({'image_desc': desc,
 147                                'image_ref': ref_path,
 148                                'image_render': render_path})
 149
 150         return images
 151
 152     def interpret_result(self):
 153         super(XTSTest, self).interpret_result()
 154
 155         try:
 156             with open(self.test_results_file, 'r') as rfile:
 157                 log = rfile.read()
 158                 self.result.out = log
 159                 os.remove(self.test_results_file)
 160         except IOError:
 161             self.result.err = "No results file found"
 162             log = ""
 163
 164         if self.result.returncode == 0:
 165             if re.search('FAIL', self.result.out) is not None:
 166                 self.result.result = 'fail'
 167             elif re.search('PASS', self.result.out) is not None:
 168                 self.result.result = 'pass'
 169             else:
 170                 self.result.result = 'fail'
 171         elif self.result.returncode == 77:
 172             self.result.result = 'skip'
 173         elif self.result.returncode == 1:
 174             if re.search('Could not open all VSW5 fonts', log):
 175                 self.result.result = 'warn'
 176             else:
 177                 self.result.result = 'fail'
 178
 179         self.result.images = self._process_log_for_images(log)
 180
 181
 182 class RendercheckTest(Test):
 183     def __init__(self, args):
 184         super(RendercheckTest, self).__init__(['rendercheck'] + args)
 185         self.testname = "rendercheck " + " ".join(args)
 186
 187     def interpret_result(self):
 188         super(RendercheckTest, self).interpret_result()
 189
 190         if self.result.returncode == 0:
 191             self.result.result = 'pass'
 192         elif self.result.returncode == 77:
 193             self.result.result = 'skip'
 194
 195
 196 def _populate_profile_xts(profile):
 197     fpath = os.path.join(X_TEST_SUITE, 'xts5')
 198     for dirpath, _, filenames in os.walk(fpath):
 199         for fname in filenames:
 200             # only look at the .m test files
 201             testname, ext = os.path.splitext(fname)
 202             if ext != '.m':
 203                 continue
 204
 205             # incrementing number generator
 206             counts = itertools.count(start=1)
 207
 208             # Walk the file looking for >>ASSERTION, each of these corresponds
 209             # to a generated subtest, there can be multiple subtests per .m
 210             # file
 211             with open(os.path.join(dirpath, fname), 'r') as rfile:
 212                 for line in rfile:
 213                     if line.startswith('>>ASSERTION'):
 214                         num = next(counts)
 215                         group = grouptools.join(
 216                             grouptools.from_path(
 217                                 os.path.relpath(dirpath, X_TEST_SUITE)),
 218                             testname, str(num))
 219
 220                         profile.test_list[group] = XTSTest(
 221                             os.path.join(dirpath, testname),
 222                             testname,
 223                             os.path.relpath(dirpath, X_TEST_SUITE),
 224                             num)
 225
 226
 227 def _add_rendercheck_test(profile, path, args):
 228     test = RendercheckTest(args.split(' '))
 229     group_path = 'rendercheck/' + path
 230     group = grouptools.join(*(group_path.split('/')))
 231     profile.test_list[group] = test
 232
 233
 234 def _populate_profile_rendercheck(profile):
 235     _add_rendercheck_test(profile, 'blend/All/a8r8g8b8', '-t blend -f a8r8g8b8')
 236     _add_rendercheck_test(profile, 'blend/All/x8r8g8b8', '-t blend -f a8r8g8b8,x8r8g8b8')
 237     _add_rendercheck_test(profile, 'blend/All/a2r10g10b10', '-t blend -f a8r8g8b8,a2r10g10b10')
 238     _add_rendercheck_test(profile, 'blend/Clear', '-t blend -o clear')
 239     _add_rendercheck_test(profile, 'blend/Src', '-t blend -o src')
 240     _add_rendercheck_test(profile, 'blend/Over', '-t blend -o over')
 241     _add_rendercheck_test(profile, 'composite/All/a8r8g8b8', '-t composite -f a8r8g8b8')
 242     _add_rendercheck_test(profile, 'composite/All/x8r8g8b8', '-t composite -f a8r8g8b8,x8r8g8b8')
 243     _add_rendercheck_test(profile, 'composite/All/a2r10g10b10', '-t composite -f a8r8g8b8,a2r10g10b10')
 244     _add_rendercheck_test(profile, 'ca composite/All/a8r8g8b8', '-t cacomposite -f a8r8g8b8')
 245     _add_rendercheck_test(profile, 'ca composite/All/a8', '-t cacomposite -f a8r8g8b8,a8')
 246     _add_rendercheck_test(profile, 'ca composite/All/x8r8g8b8', '-t cacomposite -f a8r8g8b8,x8r8g8b8')
 247     _add_rendercheck_test(profile, 'ca composite/All/a2r10g10b10', '-t cacomposite -f a8r8g8b8,a2r10g10b10')
 248     _add_rendercheck_test(profile, 'fill', '-t fill')
 249     _add_rendercheck_test(profile, 'bug7366', '-t bug7366')
 250     _add_rendercheck_test(profile, 'destination coordinates', '-t dcoords')
 251     _add_rendercheck_test(profile, 'source coordinates', '-t scoords')
 252     _add_rendercheck_test(profile, 'mask coordinates', '-t mcoords')
 253     _add_rendercheck_test(profile, 'translated source coordinates', '-t tscoords')
 254     _add_rendercheck_test(profile, 'translated mask coordinates', '-t tmcoords')
 255     _add_rendercheck_test(profile, 'triangles', '-t triangles')
 256     _add_rendercheck_test(profile, 'LibreOffice xRGB', '-t libreoffice_xrgb')
 257     _add_rendercheck_test(profile, 'GTK ARGB vs xBGR', '-t gtk_argb_xbgr')
 258
 259
 260 def _populate_profile():
 261     """ Populate the profile attribute """
 262     # Add all tests to the profile
 263     profile = XTSProfile()  # pylint: disable=redefined-outer-name
 264     _populate_profile_xts(profile)
 265     _populate_profile_rendercheck(profile)
 266     return profile
 267
 268
 269 if not os.path.exists(X_TEST_SUITE):
 270     raise exceptions.PiglitFatalError('XTest suite not found.')
 271
 272 profile = _populate_profile()  # pylint: disable=invalid-name