unittests/framework/backends/test_junit.py

   1 # coding=utf-8
   2 # Copyright (c) 2014, 2016 Intel Corporation
   3
   4 # Permission is hereby granted, free of charge, to any person obtaining a copy
   5 # of this software and associated documentation files (the "Software"), to deal
   6 # in the Software without restriction, including without limitation the rights
   7 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   8 # copies of the Software, and to permit persons to whom the Software is
   9 # furnished to do so, subject to the following conditions:
  10
  11 # The above copyright notice and this permission notice shall be included in
  12 # all copies or substantial portions of the Software.
  13
  14 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  17 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  18 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  19 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  20 # SOFTWARE.
  21
  22 """Tests for the Junit backend package."""
  23
  24 import os
  25 import textwrap
  26 try:
  27     from lxml import etree
  28 except ImportError:
  29     import xml.etree.ElementTree as etree
  30 try:
  31     import mock
  32 except ImportError:
  33     from unittest import mock
  34
  35 import pytest
  36
  37 from framework import backends
  38 from framework import grouptools
  39 from framework import results
  40 from framework import status
  41
  42 from . import shared
  43
  44 # pylint: disable=no-self-use
  45
  46 JUNIT_SCHEMA = os.path.join(os.path.dirname(__file__), 'schema', 'junit-7.xsd')
  47
  48 _XML = """\
  49 <?xml version='1.0' encoding='utf-8'?>
  50   <testsuites>
  51     <testsuite name="piglit" tests="1">
  52       <testcase classname="piglit.foo.bar" name="a-test" status="pass" time="1.12345">
  53         <system-out>this/is/a/command\nThis is stdout</system-out>
  54         <system-err>this is stderr
  55
  56
  57 pid: [1934]
  58 start time: 1.0
  59 end time: 4.5
  60         </system-err>
  61       </testcase>
  62     </testsuite>
  63   </testsuites>
  64 """
  65
  66 @pytest.fixture(autouse=True, scope="module")
  67 def mock_compression():
  68     with mock.patch('framework.backends.compression.get_mode',
  69                     mock.Mock(return_value='none')):
  70         yield
  71
  72
  73 class TestProtectedLoad(object):
  74     """Tests for the _load method."""
  75
  76     def test_default_name(self, tmpdir):
  77         """backends.junit._load: uses 'junit result' for name as fallback."""
  78         tmpdir.chdir()
  79         with open('results.xml', 'w') as f:
  80             f.write(_XML)
  81         test = backends.junit.REGISTRY.load('results.xml', 'none')
  82
  83         assert test.name == 'junit result'
  84
  85     def test_file_name(self, tmpdir):
  86         """backends.junit._load: uses the filename for name if filename !=
  87         'results'
  88         """
  89         p = tmpdir.join('foobar.xml')
  90         p.write(_XML)
  91         test = backends.junit.REGISTRY.load(str(p), 'none')
  92         assert test.name == 'foobar'
  93
  94     def test_folder_name(self, tmpdir):
  95         """backends.junit._load: uses the folder name if the result is
  96         'results.'
  97         """
  98         tmpdir.mkdir('foo')
  99         p = tmpdir.join('foo', 'results.xml')
 100         p.write(_XML)
 101         test = backends.junit.REGISTRY.load(str(p), 'none')
 102
 103         assert test.name == 'foo'
 104
 105     class TestReturned(object):
 106         """Test that the returned object is as expected."""
 107
 108         testname = grouptools.join('foo', 'bar', 'a-test')
 109
 110         @pytest.fixture
 111         def result(self, tmpdir):
 112             p = tmpdir.join('test.xml')
 113             p.write(_XML)
 114             return backends.junit._load(str(p))
 115
 116         def test_testrunresult(self, result):
 117             """backends.junit._load: returns a TestrunResult instance."""
 118             assert isinstance(result, results.TestrunResult)
 119
 120         def test_replace_sep(self, result):
 121             """backends.junit._load: replaces '.' with grouptools.SEPARATOR."""
 122             assert self.testname in result.tests
 123
 124         def test_testresult_instance(self, result):
 125             """backends.junit._load: replaces result with TestResult instance.
 126             """
 127             assert isinstance(result.tests[self.testname], results.TestResult)
 128
 129         def test_status_instance(self, result):
 130             """backends.junit._load: a status is found and loaded."""
 131             assert isinstance(result.tests[self.testname].result,
 132                               status.Status)
 133
 134         def test_time(self, result):
 135             time = result.tests[self.testname].time
 136             assert isinstance(time, results.TimeAttribute)
 137             assert time.start == 1.0
 138             assert time.end == 4.5
 139
 140         def test_command(self, result):
 141             """backends.junit._load: command is loaded correctly."""
 142             assert result.tests[self.testname].command == 'this/is/a/command'
 143
 144         def test_out(self, result):
 145             """backends.junit._load: stdout is loaded correctly."""
 146             assert result.tests[self.testname].out == 'This is stdout'
 147
 148         def test_err(self, result):
 149             """backends.junit._load: stderr is loaded correctly."""
 150             assert result.tests[self.testname].err.strip() == 'this is stderr'
 151
 152         def test_totals(self, result):
 153             """backends.junit._load: Totals are calculated."""
 154             assert bool(result)
 155
 156         def test_pid(self, result):
 157             """backends.junit._load: pid is loaded correctly."""
 158             assert result.tests[self.testname].pid == [1934]
 159
 160
 161 class TestJUnitBackend(object):
 162     """Tests for the JUnitBackend class."""
 163
 164     class TestFinalize(object):
 165         """Tests for the finalize method."""
 166
 167         def test_skips_illformed_tests(self, tmpdir):
 168             """backends.junit.JUnitBackend: skips illformed tests"""
 169             result = results.TestResult()
 170             result.time.end = 1.2345
 171             result.result = 'pass'
 172             result.out = 'this is stdout'
 173             result.err = 'this is stderr'
 174             result.command = 'foo'
 175
 176             test = backends.junit.JUnitBackend(str(tmpdir))
 177             test.initialize(shared.INITIAL_METADATA)
 178             with test.write_test(grouptools.join('a', 'group', 'test1')) as t:
 179                 t(result)
 180             tmpdir.join('tests', '1.xml').write('bad data')
 181
 182             test.finalize()
 183
 184
 185 class TestJUnitWriter(object):
 186     """Tests for the JUnitWriter class."""
 187
 188     def test_junit_replace(self, tmpdir):
 189         """backends.junit.JUnitBackend.write_test: grouptools.SEPARATOR is
 190         replaced with '.'.
 191         """
 192         result = results.TestResult()
 193         result.time.end = 1.2345
 194         result.result = 'pass'
 195         result.out = 'this is stdout'
 196         result.err = 'this is stderr'
 197         result.command = 'foo'
 198
 199         test = backends.junit.JUnitBackend(str(tmpdir))
 200         test.initialize(shared.INITIAL_METADATA)
 201         with test.write_test(grouptools.join('a', 'group', 'test1')) as t:
 202             t(result)
 203         test.finalize()
 204
 205         test_value = etree.parse(str(tmpdir.join('results.xml')))
 206         test_value = test_value.getroot()
 207
 208         assert test_value.find('.//testcase').attrib['classname'] == \
 209             'piglit.a.group'
 210
 211     class TestValid(object):
 212         @pytest.fixture
 213         def test_file(self, tmpdir):
 214             tmpdir.mkdir('foo')
 215             p = tmpdir.join('foo')
 216
 217             result = results.TestResult()
 218             result.time.end = 1.2345
 219             result.result = 'pass'
 220             result.out = 'this is stdout'
 221             result.err = 'this is stderr'
 222             result.command = 'foo'
 223             result.pid = 1034
 224
 225             test = backends.junit.JUnitBackend(str(p))
 226             test.initialize(shared.INITIAL_METADATA)
 227             with test.write_test(grouptools.join('a', 'group', 'test1')) as t:
 228                 t(result)
 229
 230             result.result = 'fail'
 231             with test.write_test(grouptools.join('a', 'test', 'test1')) as t:
 232                 t(result)
 233             test.finalize()
 234
 235             return str(p.join('results.xml'))
 236
 237         def test_xml_well_formed(self, test_file):
 238             """backends.junit.JUnitBackend.write_test: produces well formed xml."""
 239             etree.parse(test_file)
 240
 241         @pytest.mark.skipif(etree.__name__ != 'lxml.etree',
 242                             reason="This test requires lxml")
 243         def test_xml_valid(self, test_file):
 244             """backends.junit.JUnitBackend.write_test: produces valid JUnit xml."""
 245             # This XMLSchema class is unique to lxml
 246             schema = etree.XMLSchema(file=JUNIT_SCHEMA)  # pylint: disable=no-member
 247             with open(test_file, 'r') as f:
 248                 assert schema.validate(etree.parse(f))
 249
 250
 251 class TestJUnitSubtestWriter(object):
 252     """Tests for the JUnitWriter class."""
 253
 254     def test_junit_replace(self, tmpdir):
 255         """backends.junit.JUnitBackend.write_test: grouptools.SEPARATOR is
 256         replaced with '.'.
 257         """
 258         result = results.TestResult()
 259         result.time.end = 1.2345
 260         result.out = 'this is stdout'
 261         result.err = 'this is stderr'
 262         result.command = 'foo'
 263         result.subtests['foo'] = 'pass'
 264         result.subtests['bar'] = 'fail'
 265
 266         test = backends.junit.JUnitBackend(str(tmpdir),
 267                                            junit_subtests=True)
 268         test.initialize(shared.INITIAL_METADATA)
 269         with test.write_test(grouptools.join('a', 'group', 'test1')) as t:
 270             t(result)
 271         test.finalize()
 272
 273         test_value = etree.parse(str(tmpdir.join('results.xml')))
 274         test_value = test_value.getroot()
 275
 276         assert test_value.find('.//testsuite/testsuite').attrib['name'] == \
 277             'piglit.a.group.test1'
 278
 279     def test_junit_replace_suffix(self, tmpdir):
 280         """backends.junit.JUnitBackend.write_test: grouptools.SEPARATOR is
 281         replaced with '.'.
 282         """
 283         result = results.TestResult()
 284         result.time.end = 1.2345
 285         result.out = 'this is stdout'
 286         result.err = 'this is stderr'
 287         result.command = 'foo'
 288         result.subtests['foo'] = 'pass'
 289         result.subtests['bar'] = 'fail'
 290
 291         test = backends.junit.JUnitBackend(str(tmpdir),
 292                                            junit_subtests=True,
 293                                            junit_suffix='.foo')
 294         test.initialize(shared.INITIAL_METADATA)
 295         with test.write_test(grouptools.join('a', 'group', 'test1')) as t:
 296             t(result)
 297         test.finalize()
 298
 299         test_value = etree.parse(str(tmpdir.join('results.xml')))
 300         test_value = test_value.getroot()
 301
 302         suite = test_value.find('.//testsuite/testsuite')
 303         assert suite.attrib['name'] == 'piglit.a.group.test1'
 304         assert suite.find('.//testcase[@name="{}"]'.format('foo.foo')) is not None
 305
 306     def test_subtest_skip(self, tmpdir):
 307         result = results.TestResult()
 308         result.time.end = 1.2345
 309         result.out = 'this is stdout'
 310         result.err = 'this is stderr'
 311         result.command = 'foo'
 312         result.subtests['foo'] = 'pass'
 313         result.subtests['bar'] = 'skip'
 314
 315         test = backends.junit.JUnitBackend(str(tmpdir),
 316                                            junit_subtests=True)
 317         test.initialize(shared.INITIAL_METADATA)
 318         with test.write_test(grouptools.join('a', 'group', 'test1')) as t:
 319             t(result)
 320         test.finalize()
 321
 322         test_value = etree.parse(str(tmpdir.join('results.xml')))
 323         test_value = test_value.getroot()
 324
 325         suite = test_value.find('.//testsuite/testsuite')
 326         assert suite.attrib['name'] == 'piglit.a.group.test1'
 327         assert suite.find('.//testcase[@name="{}"]/skipped'.format('bar')) \
 328             is not None
 329
 330     def test_result_skip(self, tmpdir):
 331         result = results.TestResult()
 332         result.time.end = 1.2345
 333         result.out = 'this is stdout'
 334         result.err = 'this is stderr'
 335         result.command = 'foo'
 336         result.result = 'skip'
 337
 338         test = backends.junit.JUnitBackend(str(tmpdir),
 339                                            junit_subtests=True)
 340         test.initialize(shared.INITIAL_METADATA)
 341         with test.write_test(grouptools.join('a', 'group', 'test1')) as t:
 342             t(result)
 343         test.finalize()
 344
 345         test_value = etree.parse(str(tmpdir.join('results.xml')))
 346         test_value = test_value.getroot()
 347
 348         elem = test_value.find('.//testsuite/testcase[@name="test1"]/skipped')
 349         assert elem is not None
 350
 351     def test_classname(self, tmpdir):
 352         result = results.TestResult()
 353         result.time.end = 1.2345
 354         result.out = 'this is stdout'
 355         result.err = 'this is stderr'
 356         result.command = 'foo'
 357         result.subtests['foo'] = 'pass'
 358         result.subtests['bar'] = 'skip'
 359
 360         test = backends.junit.JUnitBackend(str(tmpdir),
 361                                            junit_subtests=True)
 362         test.initialize(shared.INITIAL_METADATA)
 363         with test.write_test(grouptools.join('a', 'group', 'test1')) as t:
 364             t(result)
 365         test.finalize()
 366
 367         test_value = etree.parse(str(tmpdir.join('results.xml')))
 368         test_value = test_value.getroot()
 369
 370         suite = test_value.find('.//testsuite/testsuite')
 371         assert suite.find('.//testcase[@classname="piglit.a.group.test1"]') \
 372             is not None
 373
 374     class TestValid(object):
 375         @pytest.fixture
 376         def test_file(self, tmpdir):
 377             tmpdir.mkdir('foo')
 378             p = tmpdir.join('foo')
 379
 380             result = results.TestResult()
 381             result.time.end = 1.2345
 382             result.out = 'this is stdout'
 383             result.err = 'this is stderr'
 384             result.command = 'foo'
 385             result.pid = 1034
 386             result.subtests['foo'] = 'pass'
 387             result.subtests['bar'] = 'fail'
 388
 389             test = backends.junit.JUnitBackend(str(p),
 390                                                junit_subtests=True)
 391             test.initialize(shared.INITIAL_METADATA)
 392             with test.write_test(grouptools.join('a', 'group', 'test1')) as t:
 393                 t(result)
 394
 395             result.result = 'fail'
 396             with test.write_test(grouptools.join('a', 'test', 'test1')) as t:
 397                 t(result)
 398             test.finalize()
 399
 400             return str(p.join('results.xml'))
 401
 402         def test_xml_well_formed(self, test_file):
 403             """backends.junit.JUnitBackend.write_test: produces well formed xml."""
 404             etree.parse(test_file)
 405
 406         @pytest.mark.skipif(etree.__name__ != 'lxml.etree',
 407                             reason="This test requires lxml")
 408         def test_xml_valid(self, test_file):
 409             """backends.junit.JUnitBackend.write_test: produces valid JUnit xml."""
 410             # This XMLSchema class is unique to lxml
 411             schema = etree.XMLSchema(file=JUNIT_SCHEMA)  # pylint: disable=no-member
 412             with open(test_file, 'r') as f:
 413                 assert schema.validate(etree.parse(f))
 414
 415
 416 class TestWriteResults(object):
 417     """Tests for the write_results function."""
 418
 419     @classmethod
 420     def setup_class(cls):
 421         """Setup values used by all tests."""
 422         test = results.TestrunResult()
 423         test.info = {
 424             'system': {
 425                 'uname': 'this is uname',
 426                 'glxinfo': 'glxinfo',
 427                 'clinfo': 'clinfo',
 428                 'wglinfo': 'wglinfo',
 429                 'lspci': 'this is lspci',
 430             }
 431         }
 432         test.name = 'name'
 433         test.options = {'some': 'option'}
 434         test.time_elapsed.end = 1.23
 435         another_test = results.TestResult()
 436         another_test.subtests['foo'] = 'pass'
 437         another_test.subtests['bar'] = 'skip'
 438         test.tests = {'a test': results.TestResult('pass'),
 439                       'another test': another_test}
 440
 441         cls.test = test
 442
 443     @pytest.mark.parametrize('filepath', [
 444         ('foo.xml'),
 445         ('bar.xml'),
 446     ])
 447     def test_write(self, filepath, tmpdir):
 448         """backends.json.write_results: writes a TestrunResult into a filepath.
 449         """
 450         p = tmpdir.join(filepath)
 451         assert not backends.junit.write_results(self.test, str(p))
 452         assert p.check()
 453         p.remove()
 454         assert not backends.junit.write_results(self.test, str(p),
 455                                                 junit_subtests=True)
 456         assert p.check()
 457
 458     def test_xml_well_formed(self, tmpdir):
 459         """backends.json.write_results: writes a TestrunResult into a well
 460         formed xml.
 461         """
 462         p = tmpdir.join('foo.xml')
 463         assert not backends.junit.write_results(self.test, str(p))
 464         etree.parse(str(p))
 465         p.remove()
 466         assert not backends.junit.write_results(self.test, str(p),
 467                                                 junit_subtests=True)
 468         etree.parse(str(p))
 469
 470     @pytest.mark.skipif(etree.__name__ != 'lxml.etree',
 471                         reason="This test requires lxml")
 472     def test_xml_valid(self, tmpdir):
 473         """backends.json.write_results: writes a TestrunResult into a well
 474         formed JUnit.
 475         """
 476         # This XMLSchema class is unique to lxml
 477         schema = etree.XMLSchema(file=JUNIT_SCHEMA)  # pylint: disable=no-member
 478         p = tmpdir.join('foo.xml')
 479         assert not backends.junit.write_results(self.test, str(p))
 480         with p.open('r') as f:
 481             assert schema.validate(etree.parse(f))
 482         p.remove()
 483         assert not backends.junit.write_results(self.test, str(p),
 484                                                 junit_subtests=True)
 485         with p.open('r') as f:
 486             assert schema.validate(etree.parse(f))
 487
 488     def test_inst(self, tmpdir):
 489         """backends.junit.write_results: test that the written JUnit loads
 490         as a proper TestrunResult.
 491         """
 492         p = tmpdir.join('foo.xml')
 493         assert not backends.junit.write_results(self.test, str(p))
 494         assert isinstance(backends.junit.load(str(p), 'none'),
 495                           results.TestrunResult)
 496         p.remove()
 497         assert not backends.junit.write_results(self.test, str(p),
 498                                                 junit_subtests=True)
 499         assert isinstance(backends.junit.load(str(p), 'none'),
 500                           results.TestrunResult)
 501
 502     @pytest.mark.parametrize('filepath, exception', [
 503         ('', FileNotFoundError),
 504         (None, TypeError),
 505     ])
 506     def test_bogus_filepath(self, filepath, exception):
 507         """backends.junit.write_results: raise with bogus filepaths
 508         for the output.
 509         """
 510         with pytest.raises(exception):
 511             assert not backends.junit.write_results(self.test, filepath)
 512
 513         with pytest.raises(exception):
 514             assert not backends.junit.write_results(self.test, filepath,
 515                                                     junit_subtests=True)