tests/xts.py

   1 # coding=utf-8
   2 # Copyright (c) 2013-2014 Intel Corporation
   3 #
   4 # Permission is hereby granted, free of charge, to any person
   5 # obtaining a copy of this software and associated documentation
   6 # files (the "Software"), to deal in the Software without
   7 # restriction, including without limitation the rights to use,
   8 # copy, modify, merge, publish, distribute, sublicense, and/or
   9 # sell copies of the Software, and to permit persons to whom the
  10 # Software is furnished to do so, subject to the following
  11 # conditions:
  12 #
  13 # This permission notice shall be included in all copies or
  14 # substantial portions of the Software.
  15 #
  16 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
  17 # KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
  18 # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
  19 # PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE AUTHOR(S) BE
  20 # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
  21 # AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
  22 # OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  23 # DEALINGS IN THE SOFTWARE.
  24
  25 """ Test integreation for the X Test Suite """
  26
  27 import os
  28 import re
  29 import subprocess
  30 import itertools
  31
  32 from framework import grouptools, exceptions, core
  33 from framework.profile import TestProfile, Test
  34
  35 __all__ = ['profile']
  36
  37 X_TEST_SUITE = core.PIGLIT_CONFIG.required_get('xts', 'path')
  38
  39
  40 class XTSProfile(TestProfile):  # pylint: disable=too-few-public-methods
  41     """ A subclass of TestProfile that provides a setup hook for XTS """
  42
  43     def setup(self):
  44         """This hook sets the XTSTest.results_path variable.
  45
  46         Setting this variable allows images created by XTS to moved into the
  47         results directory.
  48         """
  49         XTSTest.RESULTS_PATH = self.results_dir
  50
  51         try:
  52             os.mkdir(os.path.join(self.results_dir, 'images'))
  53         except OSError as e:
  54             # If the exception is not 'directory already exists', raise the
  55             # exception
  56             if e.errno != 17:
  57                 raise
  58
  59
  60 class XTSTest(Test):  # pylint: disable=too-few-public-methods
  61     """ X Test Suite class
  62
  63     Runs a single test or subtest from XTS.
  64
  65     Arguments:
  66     name -- the name of the test
  67     testname -- the name of the test file
  68     testnum -- the number of the test file
  69
  70     """
  71     RESULTS_PATH = None
  72
  73     def __init__(self, name, testname, testdir, testnum):
  74         super(XTSTest, self).__init__(
  75             ['./' + os.path.basename(name), '-i', str(testnum)])
  76         # Path relative to XTSTest.RESULTS_PATH (which is not
  77         # initialized at init time) to store any test-specific files.
  78         # We need to store into the results directory to protect
  79         # against races when multiple piglit-run.py -t xts commands
  80         # are running (as in the X Server's make check).
  81         self.testdir = testdir
  82         self.testname = '{0}-{1}'.format(testname, testnum)
  83         self.cwd = os.path.dirname(os.path.realpath(name))
  84         self.env.update(
  85             {"XT_RESET_DELAY": '0',
  86              "XT_FONTPATH_GOOD": '/usr/share/fonts/X11/misc',
  87              "XT_FONTPATH": os.path.join(X_TEST_SUITE, 'xts5', 'fonts'),
  88              # XXX: Are the next 3 necissary?
  89              "XT_LOCAL": 'Yes',
  90              "XT_TCP": 'No',
  91              "XT_DISPLAYHOST": ''})
  92
  93     def run(self):
  94         # We only get the RESULTS_PATH after the profile has been set
  95         # up, so we can't do it in init.
  96         self.test_results_file = os.path.join(XTSTest.RESULTS_PATH,
  97                                               self.testdir,
  98                                               self.testname)
  99         self.env.update({"TET_RESFILE": self.test_results_file})
 100         super(XTSTest, self).run()
 101
 102     def _process_log_for_images(self, log):
 103         """ Parse the image logfile """
 104         images = []
 105         search = re.compile('See file (Err[0-9]+.err)')
 106
 107         for line in log.splitlines():
 108             match = search.search(line)
 109             if match is not None:
 110                 # Can we parse any other useful information out to give a
 111                 # better description of each image?
 112                 desc = match.group(1)
 113
 114                 # The error logs are text, with a header with width, height,
 115                 # and depth, then run-length-encoded pixel values (in
 116                 # hexadecimal).  Use xtsttopng to convert the error log to a
 117                 # pair of PNGs so we can put them in the summary.
 118                 command = ['xtsttopng', os.path.join(XTSTest.RESULTS_PATH,
 119                                                      self.testdir,
 120                                                      match.group(1))]
 121                 try:
 122                     out = subprocess.check_output(command, cwd=self.cwd)
 123                 except OSError:
 124                     images.append({'image_desc': 'image processing failed'})
 125                     continue
 126
 127                 # Each Err*.err log contains a rendered image, and a reference
 128                 # image that it was compared to.  We relocate the to our tree
 129                 # with more useful names.  (Otherwise, since tests generate
 130                 # error logs with numbers sequentially starting from 0, each
 131                 # subtest with an error would overwrite the previous test's
 132                 # images).
 133                 ref_path = os.path.join(
 134                     self.RESULTS_PATH, 'images', '{1}-{2}-ref.png'.format(
 135                         self.testname, match.group(1)))
 136                 render_path = os.path.join(
 137                     self.RESULTS_PATH, 'images', '{1}-{2}-render.png'.format(
 138                         self.testname, match.group(1)))
 139
 140                 split = out.splitlines()
 141                 os.rename(os.path.join(self.cwd, split[0]), render_path)
 142                 os.rename(os.path.join(self.cwd, split[1]), ref_path)
 143
 144                 images.append({'image_desc': desc,
 145                                'image_ref': ref_path,
 146                                'image_render': render_path})
 147
 148         return images
 149
 150     def interpret_result(self):
 151         super(XTSTest, self).interpret_result()
 152
 153         try:
 154             with open(self.test_results_file, 'r') as rfile:
 155                 log = rfile.read()
 156                 self.result.out = log
 157                 os.remove(self.test_results_file)
 158         except IOError:
 159             self.result.err = "No results file found"
 160             log = ""
 161
 162         if self.result.returncode == 0:
 163             if re.search('FAIL', self.result.out) is not None:
 164                 self.result.result = 'fail'
 165             elif re.search('PASS', self.result.out) is not None:
 166                 self.result.result = 'pass'
 167             else:
 168                 self.result.result = 'fail'
 169         elif self.result.returncode == 77:
 170             self.result.result = 'skip'
 171         elif self.result.returncode == 1:
 172             if re.search('Could not open all VSW5 fonts', log):
 173                 self.result.result = 'warn'
 174             else:
 175                 self.result.result = 'fail'
 176
 177         self.result.images = self._process_log_for_images(log)
 178
 179
 180 class RendercheckTest(Test):
 181     def __init__(self, args):
 182         super(RendercheckTest, self).__init__(['rendercheck'] + args)
 183         self.testname = "rendercheck " + " ".join(args)
 184
 185     def interpret_result(self):
 186         super(RendercheckTest, self).interpret_result()
 187
 188         if self.result.returncode == 0:
 189             self.result.result = 'pass'
 190         elif self.result.returncode == 77:
 191             self.result.result = 'skip'
 192
 193
 194 def _populate_profile_xts(profile):
 195     fpath = os.path.join(X_TEST_SUITE, 'xts5')
 196     for dirpath, _, filenames in os.walk(fpath):
 197         for fname in filenames:
 198             # only look at the .m test files
 199             testname, ext = os.path.splitext(fname)
 200             if ext != '.m':
 201                 continue
 202
 203             # incrementing number generator
 204             counts = itertools.count(start=1)
 205
 206             # Walk the file looking for >>ASSERTION, each of these corresponds
 207             # to a generated subtest, there can be multiple subtests per .m
 208             # file
 209             with open(os.path.join(dirpath, fname), 'r') as rfile:
 210                 for line in rfile:
 211                     if line.startswith('>>ASSERTION'):
 212                         num = next(counts)
 213                         group = grouptools.join(
 214                             grouptools.from_path(
 215                                 os.path.relpath(dirpath, X_TEST_SUITE)),
 216                             testname, str(num))
 217
 218                         profile.test_list[group] = XTSTest(
 219                             os.path.join(dirpath, testname),
 220                             testname,
 221                             os.path.relpath(dirpath, X_TEST_SUITE),
 222                             num)
 223
 224
 225 def _add_rendercheck_test(profile, path, args):
 226     test = RendercheckTest(args.split(' '))
 227     group_path = 'rendercheck/' + path
 228     group = grouptools.join(*(group_path.split('/')))
 229     profile.test_list[group] = test
 230
 231
 232 def _populate_profile_rendercheck(profile):
 233     _add_rendercheck_test(profile, 'blend/All/a8r8g8b8', '-t blend -f a8r8g8b8')
 234     _add_rendercheck_test(profile, 'blend/All/x8r8g8b8', '-t blend -f a8r8g8b8,x8r8g8b8')
 235     _add_rendercheck_test(profile, 'blend/All/a2r10g10b10', '-t blend -f a8r8g8b8,a2r10g10b10')
 236     _add_rendercheck_test(profile, 'blend/Clear', '-t blend -o clear')
 237     _add_rendercheck_test(profile, 'blend/Src', '-t blend -o src')
 238     _add_rendercheck_test(profile, 'blend/Over', '-t blend -o over')
 239     _add_rendercheck_test(profile, 'composite/All/a8r8g8b8', '-t composite -f a8r8g8b8')
 240     _add_rendercheck_test(profile, 'composite/All/x8r8g8b8', '-t composite -f a8r8g8b8,x8r8g8b8')
 241     _add_rendercheck_test(profile, 'composite/All/a2r10g10b10', '-t composite -f a8r8g8b8,a2r10g10b10')
 242     _add_rendercheck_test(profile, 'ca composite/All/a8r8g8b8', '-t cacomposite -f a8r8g8b8')
 243     _add_rendercheck_test(profile, 'ca composite/All/a8', '-t cacomposite -f a8r8g8b8,a8')
 244     _add_rendercheck_test(profile, 'ca composite/All/x8r8g8b8', '-t cacomposite -f a8r8g8b8,x8r8g8b8')
 245     _add_rendercheck_test(profile, 'ca composite/All/a2r10g10b10', '-t cacomposite -f a8r8g8b8,a2r10g10b10')
 246     _add_rendercheck_test(profile, 'fill', '-t fill')
 247     _add_rendercheck_test(profile, 'bug7366', '-t bug7366')
 248     _add_rendercheck_test(profile, 'destination coordinates', '-t dcoords')
 249     _add_rendercheck_test(profile, 'source coordinates', '-t scoords')
 250     _add_rendercheck_test(profile, 'mask coordinates', '-t mcoords')
 251     _add_rendercheck_test(profile, 'translated source coordinates', '-t tscoords')
 252     _add_rendercheck_test(profile, 'translated mask coordinates', '-t tmcoords')
 253     _add_rendercheck_test(profile, 'triangles', '-t triangles')
 254     _add_rendercheck_test(profile, 'LibreOffice xRGB', '-t libreoffice_xrgb')
 255     _add_rendercheck_test(profile, 'GTK ARGB vs xBGR', '-t gtk_argb_xbgr')
 256
 257
 258 def _populate_profile():
 259     """ Populate the profile attribute """
 260     # Add all tests to the profile
 261     profile = XTSProfile()  # pylint: disable=redefined-outer-name
 262     _populate_profile_xts(profile)
 263     _populate_profile_rendercheck(profile)
 264     return profile
 265
 266
 267 if not os.path.exists(X_TEST_SUITE):
 268     raise exceptions.PiglitFatalError('XTest suite not found.')
 269
 270 profile = _populate_profile()  # pylint: disable=invalid-name