framework/backends/junit.py

   1 # coding=utf-8
   2 # Copyright (c) 2014-2016, 2019 Intel Corporation
   3 # Copyright © 2020 Valve Corporation.
   4
   5 # Permission is hereby granted, free of charge, to any person obtaining a copy
   6 # of this software and associated documentation files (the "Software"), to deal
   7 # in the Software without restriction, including without limitation the rights
   8 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   9 # copies of the Software, and to permit persons to whom the Software is
  10 # furnished to do so, subject to the following conditions:
  11
  12 # The above copyright notice and this permission notice shall be included in
  13 # all copies or substantial portions of the Software.
  14
  15 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  18 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  20 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  21 # SOFTWARE.
  22
  23 """ Module implementing a JUnitBackend for piglit """
  24
  25 import os.path
  26 import re
  27 import shutil
  28 try:
  29     from lxml import etree
  30 except ImportError:
  31     import xml.etree.ElementTree as etree
  32 try:
  33     import simplejson as json
  34 except ImportError:
  35     import json
  36
  37 from framework import grouptools, results, exceptions
  38 from framework.core import PIGLIT_CONFIG
  39 from .abstract import FileBackend
  40 from .register import Registry
  41
  42 __all__ = [
  43     'REGISTRY',
  44     'JUnitBackend',
  45 ]
  46
  47
  48 _JUNIT_SPECIAL_NAMES = ('api', 'search')
  49
  50 _PID_STR = "pid: "
  51 _START_TIME_STR = "start time: "
  52 _END_TIME_STR = "end time: "
  53
  54 # XML cannot include certain characters. This regex matches the "invalid XML
  55 # text character range".
  56 _FORBIDDEN_XML_TEXT_CHARS_RE = re.compile(
  57     u'[^\u0009\u000A\u000D\u0020-\uD7FF\uE000-\uFFFD\U00010000-\U0010FFFF]+'
  58 )
  59
  60
  61 def escape_forbidden_xml_text_chars(val, replacement=''):
  62     """
  63     Strip invalid XML text characters.
  64     """
  65     # See: https://github.com/html5lib/html5lib-python/issues/96
  66     #
  67     # The XML 1.0 spec defines the valid character range as:
  68     # Char ::= #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF]
  69     #
  70     # Sources:
  71     # https://www.w3.org/TR/REC-xml/#charsets,
  72     # https://lsimons.wordpress.com/2011/03/17/stripping-illegal-characters-out-of-xml-in-python/
  73
  74     return _FORBIDDEN_XML_TEXT_CHARS_RE.sub(replacement, val)
  75
  76
  77 def junit_escape(name):
  78     name = name.replace('.', '_')
  79     if name in _JUNIT_SPECIAL_NAMES:
  80         name += '_'
  81     return name
  82
  83
  84 class JUnitWriter(object):
  85     """A class that provides a write mechanism for junit tests."""
  86
  87     def __init__(self, test_suffix, efail, ecrash):
  88         self._test_suffix = test_suffix
  89         self._expected_crashes = ecrash
  90         self._expected_failures = efail
  91
  92     @staticmethod
  93     def _make_names(name):
  94         """Takes a name from piglit (using grouptools.SEPARATOR and returns a
  95         split classnam, testname pair in junit format.
  96         """
  97         classname, testname = grouptools.splitname(name)
  98         classname = classname.split(grouptools.SEPARATOR)
  99         classname = [junit_escape(e) for e in classname]
 100         classname = '.'.join(classname)
 101
 102         # Add the test to the piglit group rather than directly to the root
 103         # group, this allows piglit junit to be used in conjunction with other
 104         # piglit
 105         # TODO: It would be nice if other suites integrating with piglit could
 106         # set different root names.
 107         classname = 'piglit.' + classname
 108
 109         return (classname, junit_escape(testname))
 110
 111     @staticmethod
 112     def _set_xml_err(element, data, expected_result):
 113         """Adds the 'system-err' element."""
 114         err = etree.SubElement(element, 'system-err')
 115         # We cannot control what is in the error output. Let's escape the
 116         # forbidden XML characters.
 117         err.text = escape_forbidden_xml_text_chars(data.err)
 118         err.text += '\n\n{}{}\n{}{}\n{}{}\n'.format(
 119             _PID_STR, data.pid,
 120             _START_TIME_STR, data.time.start,
 121             _END_TIME_STR, data.time.end)
 122
 123         if data.result in ['fail', 'dmesg-warn', 'dmesg-fail']:
 124             if expected_result == "failure":
 125                 err.text += "\n\nWARN: passing test as an expected failure"
 126             elif expected_result == 'error':
 127                 err.text += \
 128                     "\n\nERROR: Test should have been crash but was failure"
 129         elif data.result in ['crash', 'timeout']:
 130             if expected_result == "error":
 131                 err.text += "\n\nWARN: passing test as an expected crash"
 132             elif expected_result == 'failure':
 133                 err.text += \
 134                     "\n\nERROR: Test should have been failure but was crash"
 135         elif expected_result != "pass":
 136             err.text += "\n\nERROR: This test passed when it "\
 137                         "expected {0}".format(expected_result)
 138
 139     @staticmethod
 140     def _make_result(element, result, expected_result):
 141         """Adds the skipped, failure, or error element."""
 142         res = None
 143         # If the result is skip, then just add the skipped message and go on
 144         if result == 'incomplete':
 145             res = etree.SubElement(element, 'failure',
 146                                    message='Incomplete run.')
 147         elif result in ['fail', 'dmesg-warn', 'dmesg-fail']:
 148             if expected_result == "failure":
 149                 res = etree.SubElement(element, 'skipped',
 150                                        message='expected failure')
 151             elif expected_result == 'error':
 152                 res = etree.SubElement(element, 'failure',
 153                                        message='expected crash, but got '
 154                                                'failure')
 155             else:
 156                 res = etree.SubElement(element, 'failure')
 157         elif result in ['crash', 'timeout']:
 158             if expected_result == "error":
 159                 res = etree.SubElement(element, 'skipped',
 160                                        message='expected crash')
 161             elif expected_result == 'failure':
 162                 res = etree.SubElement(element, 'error',
 163                                        message='expected failure, but got '
 164                                                'error')
 165             else:
 166                 res = etree.SubElement(element, 'error')
 167         elif expected_result != "pass":
 168             res = etree.SubElement(element, 'failure',
 169                                    message="expected {}, but got {}".format(
 170                                        expected_result, result))
 171         elif result == 'skip':
 172             res = etree.SubElement(element, 'skipped')
 173
 174         # Add the piglit type to the failure result
 175         if res is not None:
 176             res.attrib['type'] = str(result)
 177
 178     def _make_root(self, testname, classname, data):
 179         """Creates and returns the root element."""
 180         element = etree.Element('testcase',
 181                                 name=self._make_full_test_name(testname),
 182                                 classname=classname,
 183                                 # Incomplete will not have a time.
 184                                 time=str(data.time.total),
 185                                 status=str(data.result))
 186
 187         return element
 188
 189     def _make_full_test_name(self, testname):
 190         # Jenkins will display special pages when the test has certain names.
 191         # https://jenkins-ci.org/issue/18062
 192         # https://jenkins-ci.org/issue/19810
 193         # The testname variable is used in the calculate_result closure, and
 194         # must not have the suffix appended.
 195         return testname + self._test_suffix
 196
 197     def _expected_result(self, name):
 198         """Get the expected result of the test."""
 199         name = name.replace("=", ".").replace(":", ".")
 200         expected_result = "pass"
 201
 202         if name in self._expected_failures:
 203             expected_result = "failure"
 204             # a test can either fail or crash, but not both
 205             assert name not in self._expected_crashes
 206
 207         if name in self._expected_crashes:
 208             expected_result = "error"
 209
 210         return expected_result
 211
 212     def __call__(self, f, name, data):
 213         classname, testname = self._make_names(name)
 214         element = self._make_root(testname, classname, data)
 215         expected_result = self._expected_result(
 216             '{}.{}'.format(classname, testname).lower())
 217
 218         # If this is an incomplete status then none of these values will be
 219         # available, nor
 220         if data.result != 'incomplete':
 221             self._set_xml_err(element, data, expected_result)
 222
 223             # Add stdout
 224             out = etree.SubElement(element, 'system-out')
 225             # We cannot control what is in the output. Let's escape the
 226             # forbidden XML characters.
 227             out.text = escape_forbidden_xml_text_chars(data.out)
 228
 229             # Prepend command line to stdout
 230             out.text = data.command + '\n' + out.text
 231
 232         self._make_result(element, data.result, expected_result)
 233
 234         f.write(str(etree.tostring(element).decode('utf-8')))
 235
 236
 237 class JUnitSubtestWriter(JUnitWriter):
 238     """A JUnitWriter derived class that treats subtest at testsuites.
 239
 240     This class will turn a piglit test with subtests into a testsuite element
 241     with each subtest as a testcase element. This subclass is needed because
 242     not all JUnit readers (like the JUnit plugin for Jenkins) handle nested
 243     testsuites correctly.
 244     """
 245
 246     def _make_root(self, testname, classname, data):
 247         if data.subtests:
 248             testname = '{}.{}'.format(classname, testname)
 249             element = etree.Element('testsuite',
 250                                     name=testname,
 251                                     time=str(data.time.total),
 252                                     tests=str(len(data.subtests)))
 253             for test, result in data.subtests.items():
 254                 etree.SubElement(element,
 255                                  'testcase',
 256                                  name=self._make_full_test_name(test),
 257                                  classname=testname,
 258                                  status=str(result))
 259
 260         else:
 261             element = super(JUnitSubtestWriter, self)._make_root(
 262                 testname, classname, data)
 263         return element
 264
 265     def __call__(self, f, name, data):
 266         classname, testname = self._make_names(name)
 267         element = self._make_root(testname, classname, data)
 268
 269         # If this is an incomplete status then none of these values will be
 270         # available, nor
 271         if data.result != 'incomplete':
 272             self._set_xml_err(element, data, 'pass')
 273
 274             # Add stdout
 275             out = etree.SubElement(element, 'system-out')
 276             # We cannot control what is in the output. Let's escape the
 277             # forbidden XML characters.
 278             out.text = escape_forbidden_xml_text_chars(data.out)
 279             # Prepend command line to stdout
 280             out.text = data.command + '\n' + out.text
 281
 282             if data.subtests:
 283                 for subname, result in data.subtests.items():
 284                     # replace special characters and make case insensitive
 285                     elem = element.find('.//testcase[@name="{}"]'.format(
 286                         self._make_full_test_name(subname)))
 287                     assert elem is not None
 288                     self._make_result(
 289                         elem, result,
 290                         self._expected_result('{}.{}.{}'.format(
 291                             classname, testname, subname).lower()))
 292             else:
 293                 self._make_result(element, data.result,
 294                                   self._expected_result('{}.{}'.format(
 295                                       classname, testname).lower()))
 296         else:
 297             self._make_result(element, data.result,
 298                               self._expected_result('{}.{}'.format(
 299                                   classname, testname).lower()))
 300
 301         f.write(str(etree.tostring(element).decode('utf-8')))
 302
 303
 304 class JUnitBackend(FileBackend):
 305     """ Backend that produces ANT JUnit XML
 306
 307     Based on the following schema:
 308     https://svn.jenkins-ci.org/trunk/hudson/dtkit/dtkit-format/dtkit-junit-model/src/main/resources/com/thalesgroup/dtkit/junit/model/xsd/junit-7.xsd
 309
 310     """
 311     _file_extension = 'xml'
 312     _write = None  # this silences the abstract-not-subclassed warning
 313
 314     def __init__(self, dest, junit_suffix='', junit_subtests=False, **options):
 315         super(JUnitBackend, self).__init__(dest, **options)
 316
 317         # make dictionaries of all test names expected to crash/fail
 318         # for quick lookup when writing results.  Use lower-case to
 319         # provide case insensitive matches.
 320         expected_failures = {}
 321         if PIGLIT_CONFIG.has_section("expected-failures"):
 322             for fail, _ in PIGLIT_CONFIG.items("expected-failures"):
 323                 expected_failures[fail.lower()] = True
 324         expected_crashes = {}
 325         if PIGLIT_CONFIG.has_section("expected-crashes"):
 326             for fail, _ in PIGLIT_CONFIG.items("expected-crashes"):
 327                 expected_crashes[fail.lower()] = True
 328
 329         if not junit_subtests:
 330             self._write = JUnitWriter(
 331                 junit_suffix, expected_failures, expected_crashes)
 332         else:
 333             self._write = JUnitSubtestWriter(  # pylint: disable=redefined-variable-type
 334                 junit_suffix, expected_failures, expected_crashes)
 335
 336     def initialize(self, metadata):
 337         """ Do nothing
 338
 339         Junit doesn't support restore, and doesn't have an initial metadata
 340         block to write, so all this method does is create the tests directory
 341
 342         """
 343         tests = os.path.join(self._dest, 'tests')
 344         if os.path.exists(tests):
 345             shutil.rmtree(tests)
 346         os.mkdir(tests)
 347
 348     def finalize(self, metadata=None):
 349         """ Scoop up all of the individual pieces and put them together """
 350         root = etree.Element('testsuites')
 351         piglit = etree.Element('testsuite', name='piglit')
 352         root.append(piglit)
 353         for each in os.listdir(os.path.join(self._dest, 'tests')):
 354             with open(os.path.join(self._dest, 'tests', each), 'r') as f:
 355                 # parse returns an element tree, and that's not what we want,
 356                 # we want the first (and only) Element node
 357                 # If the element cannot be properly parsed then consider it a
 358                 # failed transaction and ignore it.
 359                 try:
 360                     piglit.append(etree.parse(f).getroot())
 361                 except etree.ParseError:
 362                     continue
 363
 364         num_tests = len(piglit)
 365         if not num_tests:
 366             raise exceptions.PiglitUserError(
 367                 'No tests were run, not writing a result file',
 368                 exitcode=2)
 369
 370         # set the test count by counting the number of tests.
 371         # This must be unicode (py3 str)
 372         piglit.attrib['tests'] = str(num_tests)
 373
 374
 375         with open(os.path.join(self._dest, 'results.xml'), 'w') as f:
 376             f.write("<?xml version='1.0' encoding='utf-8'?>\n")
 377             # lxml has a pretty print we want to use
 378             if etree.__name__ == 'lxml.etree':
 379                 out = etree.tostring(root, pretty_print=True)
 380             else:
 381                 out = etree.tostring(root)
 382             f.write(out.decode('utf-8'))
 383
 384         shutil.rmtree(os.path.join(self._dest, 'tests'))
 385
 386
 387 def _load(results_file):
 388     """Load a junit results instance and return a TestrunResult.
 389
 390     It's worth noting that junit is not as descriptive as piglit's own json
 391     format, so some data structures will be empty compared to json.
 392
 393     This tries to not make too many assumptions about the structure of the
 394     JUnit document.
 395
 396     """
 397     run_result = results.TestrunResult()
 398
 399     splitpath = os.path.splitext(results_file)[0].split(os.path.sep)
 400     if splitpath[-1] != 'results':
 401         run_result.name = splitpath[-1]
 402     elif len(splitpath) > 1:
 403         run_result.name = splitpath[-2]
 404     else:
 405         run_result.name = 'junit result'
 406
 407     tree = etree.parse(results_file).getroot().find('.//testsuite')
 408     for test in tree.iterfind('testcase'):
 409         result = results.TestResult()
 410         # Take the class name minus the 'piglit.' element, replace junit's '.'
 411         # separator with piglit's separator, and join the group and test names
 412         name = test.attrib['name']
 413         if 'classname' in test.attrib:
 414             name = grouptools.join(test.attrib['classname'], name)
 415         name = name.replace('.', grouptools.SEPARATOR)
 416         is_piglit = False
 417         if name.startswith("piglit"):
 418             is_piglit = True
 419             name = name.split(grouptools.SEPARATOR, 1)[1]
 420
 421         # Remove the trailing _ if they were added (such as to api and search)
 422         if name.endswith('_'):
 423             name = name[:-1]
 424
 425         result.result = test.attrib['status']
 426
 427         # This is the fallback path, we'll try to overwrite this with the value
 428         # in stderr
 429         result.time = results.TimeAttribute()
 430         if 'time' in test.attrib:
 431             result.time = results.TimeAttribute(end=float(test.attrib['time']))
 432         syserr = test.find('system-err')
 433         if syserr is not None:
 434             reversed_err = syserr.text.split('\n')
 435             reversed_err.reverse()
 436         else:
 437             reversed_err = []
 438
 439         # The command is prepended to system-out, so we need to separate those
 440         # into two separate elements
 441         out_tag = test.find('system-out')
 442         if out_tag is not None:
 443             if is_piglit:
 444                 out = out_tag.text.split('\n')
 445                 result.command = out[0]
 446                 result.out = '\n'.join(out[1:])
 447             else:
 448                 result.out = out_tag.text
 449
 450         err_list = []
 451         skip = 0
 452         # Try to get the values in stderr for time and pid
 453         for line in reversed_err:
 454             if skip > 0:
 455                 if line == '':
 456                     skip -= 1
 457                     continue
 458                 else:
 459                     skip = 0
 460             if line.startswith(_START_TIME_STR):
 461                 result.time.start = float(line[len(_START_TIME_STR):])
 462                 continue
 463             elif line.startswith(_END_TIME_STR):
 464                 result.time.end = float(line[len(_END_TIME_STR):])
 465                 continue
 466             elif line.startswith(_PID_STR):
 467                 result.pid = json.loads(line[len(_PID_STR):])
 468                 skip = 2
 469                 continue
 470             err_list.append(line)
 471
 472         err_list.reverse()
 473         result.err = "\n".join(err_list)
 474
 475
 476         run_result.tests[name] = result
 477
 478     run_result.calculate_group_totals()
 479
 480     return run_result
 481
 482
 483 def load(results_dir, compression):  # pylint: disable=unused-argument
 484     """Searches for a results file and returns a TestrunResult.
 485
 486     wraps _load and searches for the result file.
 487
 488     """
 489     if not os.path.isdir(results_dir):
 490         return _load(results_dir)
 491     elif os.path.exists(os.path.join(results_dir, 'tests')):
 492         raise NotImplementedError('resume support of junit not implemented')
 493     elif os.path.exists(os.path.join(results_dir, 'results.xml')):
 494         return _load(os.path.join(results_dir, 'results.xml'))
 495     else:
 496         raise exceptions.PiglitFatalError("No results found")
 497
 498
 499 def write_results(results, file_, junit_subtests=False):
 500     """Write the values of the results out to a file."""
 501
 502     if not junit_subtests:
 503         writer = JUnitWriter('', {}, {})
 504     else:
 505         writer = JUnitSubtestWriter('', {}, {})
 506
 507     with open(file_, 'w') as f:
 508         f.write("<?xml version='1.0' encoding='utf-8'?>\n")
 509         f.write('<testsuites><testsuite name="piglit" tests="{}">'.format(
 510             len(results.tests)))
 511         for k, v in results.tests.items():
 512             writer(f, k, v)
 513         f.write("</testsuite></testsuites>")
 514
 515     return False
 516
 517
 518 REGISTRY = Registry(
 519     extensions=['.xml'],
 520     backend=JUnitBackend,
 521     load=load,
 522     meta=lambda x: x,  # The venerable no-op function
 523     write=write_results,
 524 )