framework/backends/junit.py

   1 # coding=utf-8
   2 # Copyright (c) 2014-2016, 2019 Intel Corporation
   3 # Copyright © 2020 Valve Corporation.
   4
   5 # Permission is hereby granted, free of charge, to any person obtaining a copy
   6 # of this software and associated documentation files (the "Software"), to deal
   7 # in the Software without restriction, including without limitation the rights
   8 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   9 # copies of the Software, and to permit persons to whom the Software is
  10 # furnished to do so, subject to the following conditions:
  11
  12 # The above copyright notice and this permission notice shall be included in
  13 # all copies or substantial portions of the Software.
  14
  15 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  18 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  20 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  21 # SOFTWARE.
  22
  23 """ Module implementing a JUnitBackend for piglit """
  24
  25 import os.path
  26 import re
  27 import shutil
  28 try:
  29     from lxml import etree
  30 except ImportError:
  31     import xml.etree.ElementTree as etree
  32 import json
  33
  34 from framework import grouptools, results, exceptions
  35 from framework.core import PIGLIT_CONFIG
  36 from .abstract import FileBackend
  37 from .register import Registry
  38
  39 __all__ = [
  40     'REGISTRY',
  41     'JUnitBackend',
  42 ]
  43
  44
  45 _JUNIT_SPECIAL_NAMES = ('api', 'search')
  46
  47 _PID_STR = "pid: "
  48 _START_TIME_STR = "start time: "
  49 _END_TIME_STR = "end time: "
  50
  51 # XML cannot include certain characters. This regex matches the "invalid XML
  52 # text character range".
  53 _FORBIDDEN_XML_TEXT_CHARS_RE = re.compile(
  54     u'[^\u0009\u000A\u000D\u0020-\uD7FF\uE000-\uFFFD\U00010000-\U0010FFFF]+'
  55 )
  56
  57
  58 def escape_forbidden_xml_text_chars(val, replacement=''):
  59     """
  60     Strip invalid XML text characters.
  61     """
  62     # See: https://github.com/html5lib/html5lib-python/issues/96
  63     #
  64     # The XML 1.0 spec defines the valid character range as:
  65     # Char ::= #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF]
  66     #
  67     # Sources:
  68     # https://www.w3.org/TR/REC-xml/#charsets,
  69     # https://lsimons.wordpress.com/2011/03/17/stripping-illegal-characters-out-of-xml-in-python/
  70
  71     return _FORBIDDEN_XML_TEXT_CHARS_RE.sub(replacement, val)
  72
  73
  74 def junit_escape(name):
  75     name = name.replace('.', '_')
  76     if name in _JUNIT_SPECIAL_NAMES:
  77         name += '_'
  78     return name
  79
  80
  81 class JUnitWriter(object):
  82     """A class that provides a write mechanism for junit tests."""
  83
  84     def __init__(self, test_suffix, efail, ecrash):
  85         self._test_suffix = test_suffix
  86         self._expected_crashes = ecrash
  87         self._expected_failures = efail
  88
  89     @staticmethod
  90     def _make_names(name):
  91         """Takes a name from piglit (using grouptools.SEPARATOR and returns a
  92         split classnam, testname pair in junit format.
  93         """
  94         classname, testname = grouptools.splitname(name)
  95         classname = classname.split(grouptools.SEPARATOR)
  96         classname = [junit_escape(e) for e in classname]
  97         classname = '.'.join(classname)
  98
  99         # Add the test to the piglit group rather than directly to the root
 100         # group, this allows piglit junit to be used in conjunction with other
 101         # piglit
 102         # TODO: It would be nice if other suites integrating with piglit could
 103         # set different root names.
 104         classname = 'piglit.' + classname
 105
 106         return (classname, junit_escape(testname))
 107
 108     @staticmethod
 109     def _set_xml_err(element, data, expected_result):
 110         """Adds the 'system-err' element."""
 111         err = etree.SubElement(element, 'system-err')
 112         # We cannot control what is in the error output. Let's escape the
 113         # forbidden XML characters.
 114         err.text = escape_forbidden_xml_text_chars(data.err)
 115         err.text += '\n\n{}{}\n{}{}\n{}{}\n'.format(
 116             _PID_STR, data.pid,
 117             _START_TIME_STR, data.time.start,
 118             _END_TIME_STR, data.time.end)
 119
 120         if data.result in ['fail', 'dmesg-warn', 'dmesg-fail']:
 121             if expected_result == "failure":
 122                 err.text += "\n\nWARN: passing test as an expected failure"
 123             elif expected_result == 'error':
 124                 err.text += \
 125                     "\n\nERROR: Test should have been crash but was failure"
 126         elif data.result in ['crash', 'timeout']:
 127             if expected_result == "error":
 128                 err.text += "\n\nWARN: passing test as an expected crash"
 129             elif expected_result == 'failure':
 130                 err.text += \
 131                     "\n\nERROR: Test should have been failure but was crash"
 132         elif expected_result != "pass":
 133             err.text += "\n\nERROR: This test passed when it "\
 134                         "expected {0}".format(expected_result)
 135
 136     @staticmethod
 137     def _make_result(element, result, expected_result):
 138         """Adds the skipped, failure, or error element."""
 139         res = None
 140         if result == 'incomplete':
 141             if expected_result != "error":
 142                 res = etree.SubElement(element, 'failure',
 143                                        message='Incomplete run.')
 144             else:
 145                 res = etree.SubElement(element, 'skipped',
 146                                        message='expected incomplete')
 147         elif result in ['fail', 'dmesg-warn', 'dmesg-fail']:
 148             if expected_result == "failure":
 149                 res = etree.SubElement(element, 'skipped',
 150                                        message='expected failure')
 151             elif expected_result == 'error':
 152                 res = etree.SubElement(element, 'failure',
 153                                        message='expected crash, but got '
 154                                                'failure')
 155             else:
 156                 res = etree.SubElement(element, 'failure')
 157         elif result in ['crash', 'timeout']:
 158             if expected_result == "error":
 159                 res = etree.SubElement(element, 'skipped',
 160                                        message='expected crash')
 161             elif expected_result == 'failure':
 162                 res = etree.SubElement(element, 'error',
 163                                        message='expected failure, but got '
 164                                                'error')
 165             else:
 166                 res = etree.SubElement(element, 'error')
 167         elif expected_result != "pass":
 168             res = etree.SubElement(element, 'failure',
 169                                    message="expected {}, but got {}".format(
 170                                        expected_result, result))
 171         elif result == 'skip':
 172             # If the result is skip, then just add the skipped message and go on
 173             res = etree.SubElement(element, 'skipped')
 174
 175         # Add the piglit type to the failure result
 176         if res is not None:
 177             res.attrib['type'] = str(result)
 178
 179     def _make_root(self, testname, classname, data):
 180         """Creates and returns the root element."""
 181         element = etree.Element('testcase',
 182                                 name=self._make_full_test_name(testname),
 183                                 classname=classname,
 184                                 # Incomplete will not have a time.
 185                                 time=str(data.time.total),
 186                                 status=str(data.result))
 187
 188         return element
 189
 190     def _make_full_test_name(self, testname):
 191         # Jenkins will display special pages when the test has certain names.
 192         # https://jenkins-ci.org/issue/18062
 193         # https://jenkins-ci.org/issue/19810
 194         # The testname variable is used in the calculate_result closure, and
 195         # must not have the suffix appended.
 196         return testname + self._test_suffix
 197
 198     def _expected_result(self, name):
 199         """Get the expected result of the test."""
 200         name = name.replace("=", ".").replace(":", ".")
 201         expected_result = "pass"
 202
 203         if name in self._expected_failures:
 204             expected_result = "failure"
 205             # a test can either fail or crash, but not both
 206             assert name not in self._expected_crashes
 207
 208         if name in self._expected_crashes:
 209             expected_result = "error"
 210
 211         return expected_result
 212
 213     def __call__(self, f, name, data):
 214         classname, testname = self._make_names(name)
 215         element = self._make_root(testname, classname, data)
 216         expected_result = self._expected_result(
 217             '{}.{}'.format(classname, testname).lower())
 218
 219         # If this is an incomplete status then none of these values will be
 220         # available, nor
 221         if data.result != 'incomplete':
 222             self._set_xml_err(element, data, expected_result)
 223
 224             # Add stdout
 225             out = etree.SubElement(element, 'system-out')
 226             # We cannot control what is in the output. Let's escape the
 227             # forbidden XML characters.
 228             out.text = escape_forbidden_xml_text_chars(data.out)
 229
 230             # Prepend command line to stdout
 231             out.text = data.command + '\n' + out.text
 232
 233         self._make_result(element, data.result, expected_result)
 234
 235         f.write(str(etree.tostring(element).decode('utf-8')))
 236
 237
 238 class JUnitSubtestWriter(JUnitWriter):
 239     """A JUnitWriter derived class that treats subtest at testsuites.
 240
 241     This class will turn a piglit test with subtests into a testsuite element
 242     with each subtest as a testcase element. This subclass is needed because
 243     not all JUnit readers (like the JUnit plugin for Jenkins) handle nested
 244     testsuites correctly.
 245     """
 246
 247     def _make_root(self, testname, classname, data):
 248         if data.subtests:
 249             testname = '{}.{}'.format(classname, testname)
 250             element = etree.Element('testsuite',
 251                                     name=testname,
 252                                     time=str(data.time.total),
 253                                     tests=str(len(data.subtests)))
 254             for test, result in data.subtests.items():
 255                 etree.SubElement(element,
 256                                  'testcase',
 257                                  name=self._make_full_test_name(test),
 258                                  classname=testname,
 259                                  status=str(result))
 260
 261         else:
 262             element = super(JUnitSubtestWriter, self)._make_root(
 263                 testname, classname, data)
 264         return element
 265
 266     def __call__(self, f, name, data):
 267         classname, testname = self._make_names(name)
 268         element = self._make_root(testname, classname, data)
 269
 270         # If this is an incomplete status then none of these values will be
 271         # available, nor
 272         if data.result != 'incomplete':
 273             self._set_xml_err(element, data, 'pass')
 274
 275             # Add stdout
 276             out = etree.SubElement(element, 'system-out')
 277             # We cannot control what is in the output. Let's escape the
 278             # forbidden XML characters.
 279             out.text = escape_forbidden_xml_text_chars(data.out)
 280             # Prepend command line to stdout
 281             out.text = data.command + '\n' + out.text
 282
 283             if data.subtests:
 284                 for subname, result in data.subtests.items():
 285                     # replace special characters and make case insensitive
 286                     elem = element.find('.//testcase[@name="{}"]'.format(
 287                         self._make_full_test_name(subname)))
 288                     assert elem is not None
 289                     self._make_result(
 290                         elem, result,
 291                         self._expected_result('{}.{}.{}'.format(
 292                             classname, testname, subname).lower()))
 293             else:
 294                 self._make_result(element, data.result,
 295                                   self._expected_result('{}.{}'.format(
 296                                       classname, testname).lower()))
 297         else:
 298             self._make_result(element, data.result,
 299                               self._expected_result('{}.{}'.format(
 300                                   classname, testname).lower()))
 301
 302         f.write(str(etree.tostring(element).decode('utf-8')))
 303
 304
 305 class JUnitBackend(FileBackend):
 306     """ Backend that produces ANT JUnit XML
 307
 308     Based on the following schema:
 309     https://svn.jenkins-ci.org/trunk/hudson/dtkit/dtkit-format/dtkit-junit-model/src/main/resources/com/thalesgroup/dtkit/junit/model/xsd/junit-7.xsd
 310
 311     """
 312     _file_extension = 'xml'
 313     _write = None  # this silences the abstract-not-subclassed warning
 314
 315     def __init__(self, dest, junit_suffix='', junit_subtests=False, **options):
 316         super(JUnitBackend, self).__init__(dest, **options)
 317
 318         # make dictionaries of all test names expected to crash/fail
 319         # for quick lookup when writing results.  Use lower-case to
 320         # provide case insensitive matches.
 321         expected_failures = {}
 322         if PIGLIT_CONFIG.has_section("expected-failures"):
 323             for fail, _ in PIGLIT_CONFIG.items("expected-failures"):
 324                 expected_failures[fail.lower()] = True
 325         expected_crashes = {}
 326         if PIGLIT_CONFIG.has_section("expected-crashes"):
 327             for fail, _ in PIGLIT_CONFIG.items("expected-crashes"):
 328                 expected_crashes[fail.lower()] = True
 329
 330         if not junit_subtests:
 331             self._write = JUnitWriter(
 332                 junit_suffix, expected_failures, expected_crashes)
 333         else:
 334             self._write = JUnitSubtestWriter(  # pylint: disable=redefined-variable-type
 335                 junit_suffix, expected_failures, expected_crashes)
 336
 337     def initialize(self, metadata):
 338         """ Do nothing
 339
 340         Junit doesn't support restore, and doesn't have an initial metadata
 341         block to write, so all this method does is create the tests directory
 342
 343         """
 344         tests = os.path.join(self._dest, 'tests')
 345         if os.path.exists(tests):
 346             shutil.rmtree(tests)
 347         os.mkdir(tests)
 348
 349     def finalize(self, metadata=None):
 350         """ Scoop up all of the individual pieces and put them together """
 351         root = etree.Element('testsuites')
 352         piglit = etree.Element('testsuite', name='piglit')
 353         root.append(piglit)
 354         for each in os.listdir(os.path.join(self._dest, 'tests')):
 355             with open(os.path.join(self._dest, 'tests', each), 'r') as f:
 356                 # parse returns an element tree, and that's not what we want,
 357                 # we want the first (and only) Element node
 358                 # If the element cannot be properly parsed then consider it a
 359                 # failed transaction and ignore it.
 360                 try:
 361                     piglit.append(etree.parse(f).getroot())
 362                 except etree.ParseError:
 363                     continue
 364
 365         num_tests = len(piglit)
 366         if not num_tests:
 367             raise exceptions.PiglitUserError(
 368                 'No tests were run, not writing a result file',
 369                 exitcode=2)
 370
 371         # set the test count by counting the number of tests.
 372         # This must be unicode (py3 str)
 373         piglit.attrib['tests'] = str(num_tests)
 374
 375
 376         with open(os.path.join(self._dest, 'results.xml'), 'w') as f:
 377             f.write("<?xml version='1.0' encoding='utf-8'?>\n")
 378             # lxml has a pretty print we want to use
 379             if etree.__name__ == 'lxml.etree':
 380                 out = etree.tostring(root, pretty_print=True)
 381             else:
 382                 out = etree.tostring(root)
 383             f.write(out.decode('utf-8'))
 384
 385         shutil.rmtree(os.path.join(self._dest, 'tests'))
 386
 387
 388 def _load(results_file):
 389     """Load a junit results instance and return a TestrunResult.
 390
 391     It's worth noting that junit is not as descriptive as piglit's own json
 392     format, so some data structures will be empty compared to json.
 393
 394     This tries to not make too many assumptions about the structure of the
 395     JUnit document.
 396
 397     """
 398     run_result = results.TestrunResult()
 399
 400     splitpath = os.path.splitext(results_file)[0].split(os.path.sep)
 401     if splitpath[-1] != 'results':
 402         run_result.name = splitpath[-1]
 403     elif len(splitpath) > 1:
 404         run_result.name = splitpath[-2]
 405     else:
 406         run_result.name = 'junit result'
 407
 408     tree = etree.parse(results_file).getroot().find('.//testsuite')
 409     for test in tree.iterfind('testcase'):
 410         result = results.TestResult()
 411         # Take the class name minus the 'piglit.' element, replace junit's '.'
 412         # separator with piglit's separator, and join the group and test names
 413         name = test.attrib['name']
 414         if 'classname' in test.attrib:
 415             name = grouptools.join(test.attrib['classname'], name)
 416         name = name.replace('.', grouptools.SEPARATOR)
 417         is_piglit = False
 418         if name.startswith("piglit"):
 419             is_piglit = True
 420             name = name.split(grouptools.SEPARATOR, 1)[1]
 421
 422         # Remove the trailing _ if they were added (such as to api and search)
 423         if name.endswith('_'):
 424             name = name[:-1]
 425
 426         result.result = test.attrib['status']
 427
 428         # This is the fallback path, we'll try to overwrite this with the value
 429         # in stderr
 430         result.time = results.TimeAttribute()
 431         if 'time' in test.attrib:
 432             result.time = results.TimeAttribute(end=float(test.attrib['time']))
 433         syserr = test.find('system-err')
 434         if syserr is not None:
 435             reversed_err = syserr.text.split('\n')
 436             reversed_err.reverse()
 437         else:
 438             reversed_err = []
 439
 440         # The command is prepended to system-out, so we need to separate those
 441         # into two separate elements
 442         out_tag = test.find('system-out')
 443         if out_tag is not None:
 444             if is_piglit:
 445                 out = out_tag.text.split('\n')
 446                 result.command = out[0]
 447                 result.out = '\n'.join(out[1:])
 448             else:
 449                 result.out = out_tag.text
 450
 451         err_list = []
 452         skip = 0
 453         # Try to get the values in stderr for time and pid
 454         for line in reversed_err:
 455             if skip > 0:
 456                 if line == '':
 457                     skip -= 1
 458                     continue
 459                 else:
 460                     skip = 0
 461             if line.startswith(_START_TIME_STR):
 462                 result.time.start = float(line[len(_START_TIME_STR):])
 463                 continue
 464             elif line.startswith(_END_TIME_STR):
 465                 result.time.end = float(line[len(_END_TIME_STR):])
 466                 continue
 467             elif line.startswith(_PID_STR):
 468                 result.pid = json.loads(line[len(_PID_STR):])
 469                 skip = 2
 470                 continue
 471             err_list.append(line)
 472
 473         err_list.reverse()
 474         result.err = "\n".join(err_list)
 475
 476
 477         run_result.tests[name] = result
 478
 479     run_result.calculate_group_totals()
 480
 481     return run_result
 482
 483
 484 def load(results_dir, compression):  # pylint: disable=unused-argument
 485     """Searches for a results file and returns a TestrunResult.
 486
 487     wraps _load and searches for the result file.
 488
 489     """
 490     if not os.path.isdir(results_dir):
 491         return _load(results_dir)
 492     elif os.path.exists(os.path.join(results_dir, 'tests')):
 493         raise NotImplementedError('resume support of junit not implemented')
 494     elif os.path.exists(os.path.join(results_dir, 'results.xml')):
 495         return _load(os.path.join(results_dir, 'results.xml'))
 496     else:
 497         raise exceptions.PiglitFatalError("No results found")
 498
 499
 500 def write_results(results, file_, junit_subtests=False):
 501     """Write the values of the results out to a file."""
 502
 503     if not junit_subtests:
 504         writer = JUnitWriter('', {}, {})
 505     else:
 506         writer = JUnitSubtestWriter('', {}, {})
 507
 508     with open(file_, 'w') as f:
 509         f.write("<?xml version='1.0' encoding='utf-8'?>\n")
 510         f.write('<testsuites><testsuite name="piglit" tests="{}">'.format(
 511             len(results.tests)))
 512         for k, v in results.tests.items():
 513             writer(f, k, v)
 514         f.write("</testsuite></testsuites>")
 515
 516     return False
 517
 518
 519 REGISTRY = Registry(
 520     extensions=['.xml'],
 521     backend=JUnitBackend,
 522     load=load,
 523     meta=lambda x: x,  # The venerable no-op function
 524     write=write_results,
 525 )