third_party/cython/src/Cython/Tests/xmlrunner.py

   1 # -*- coding: utf-8 -*-
   2
   3 """unittest-xml-reporting is a PyUnit-based TestRunner that can export test
   4 results to XML files that can be consumed by a wide range of tools, such as
   5 build systems, IDEs and Continuous Integration servers.
   6
   7 This module provides the XMLTestRunner class, which is heavily based on the
   8 default TextTestRunner. This makes the XMLTestRunner very simple to use.
   9
  10 The script below, adapted from the unittest documentation, shows how to use
  11 XMLTestRunner in a very simple way. In fact, the only difference between this
  12 script and the original one is the last line:
  13
  14 import random
  15 import unittest
  16 import xmlrunner
  17
  18 class TestSequenceFunctions(unittest.TestCase):
  19     def setUp(self):
  20         self.seq = range(10)
  21
  22     def test_shuffle(self):
  23         # make sure the shuffled sequence does not lose any elements
  24         random.shuffle(self.seq)
  25         self.seq.sort()
  26         self.assertEqual(self.seq, range(10))
  27
  28     def test_choice(self):
  29         element = random.choice(self.seq)
  30         self.assert_(element in self.seq)
  31
  32     def test_sample(self):
  33         self.assertRaises(ValueError, random.sample, self.seq, 20)
  34         for element in random.sample(self.seq, 5):
  35             self.assert_(element in self.seq)
  36
  37 if __name__ == '__main__':
  38     unittest.main(testRunner=xmlrunner.XMLTestRunner(output='test-reports'))
  39 """
  40
  41 import os
  42 import sys
  43 import time
  44 from unittest import TestResult, _TextTestResult, TextTestRunner
  45 from cStringIO import StringIO
  46 import xml.dom.minidom
  47
  48
  49 class XMLDocument(xml.dom.minidom.Document):
  50     def createCDATAOrText(self, data):
  51         if ']]>' in data:
  52             return self.createTextNode(data)
  53         return self.createCDATASection(data)
  54
  55
  56 class _TestInfo(object):
  57     """This class is used to keep useful information about the execution of a
  58     test method.
  59     """
  60
  61     # Possible test outcomes
  62     (SUCCESS, FAILURE, ERROR) = range(3)
  63
  64     def __init__(self, test_result, test_method, outcome=SUCCESS, err=None):
  65         "Create a new instance of _TestInfo."
  66         self.test_result = test_result
  67         self.test_method = test_method
  68         self.outcome = outcome
  69         self.err = err
  70         self.stdout = test_result.stdout and test_result.stdout.getvalue().strip() or ''
  71         self.stderr = test_result.stdout and test_result.stderr.getvalue().strip() or ''
  72
  73     def get_elapsed_time(self):
  74         """Return the time that shows how long the test method took to
  75         execute.
  76         """
  77         return self.test_result.stop_time - self.test_result.start_time
  78
  79     def get_description(self):
  80         "Return a text representation of the test method."
  81         return self.test_result.getDescription(self.test_method)
  82
  83     def get_error_info(self):
  84         """Return a text representation of an exception thrown by a test
  85         method.
  86         """
  87         if not self.err:
  88             return ''
  89         if sys.version_info < (2,4):
  90             return self.test_result._exc_info_to_string(self.err)
  91         else:
  92             return self.test_result._exc_info_to_string(
  93                 self.err, self.test_method)
  94
  95
  96 class _XMLTestResult(_TextTestResult):
  97     """A test result class that can express test results in a XML report.
  98
  99     Used by XMLTestRunner.
 100     """
 101     def __init__(self, stream=sys.stderr, descriptions=1, verbosity=1, \
 102         elapsed_times=True):
 103         "Create a new instance of _XMLTestResult."
 104         _TextTestResult.__init__(self, stream, descriptions, verbosity)
 105         self.successes = []
 106         self.callback = None
 107         self.elapsed_times = elapsed_times
 108         self.output_patched = False
 109
 110     def _prepare_callback(self, test_info, target_list, verbose_str,
 111         short_str):
 112         """Append a _TestInfo to the given target list and sets a callback
 113         method to be called by stopTest method.
 114         """
 115         target_list.append(test_info)
 116         def callback():
 117             """This callback prints the test method outcome to the stream,
 118             as well as the elapsed time.
 119             """
 120
 121             # Ignore the elapsed times for a more reliable unit testing
 122             if not self.elapsed_times:
 123                 self.start_time = self.stop_time = 0
 124
 125             if self.showAll:
 126                 self.stream.writeln('(%.3fs) %s' % \
 127                     (test_info.get_elapsed_time(), verbose_str))
 128             elif self.dots:
 129                 self.stream.write(short_str)
 130         self.callback = callback
 131
 132     def _patch_standard_output(self):
 133         """Replace the stdout and stderr streams with string-based streams
 134         in order to capture the tests' output.
 135         """
 136         if not self.output_patched:
 137             (self.old_stdout, self.old_stderr) = (sys.stdout, sys.stderr)
 138             self.output_patched = True
 139         (sys.stdout, sys.stderr) = (self.stdout, self.stderr) = \
 140             (StringIO(), StringIO())
 141
 142     def _restore_standard_output(self):
 143         "Restore the stdout and stderr streams."
 144         (sys.stdout, sys.stderr) = (self.old_stdout, self.old_stderr)
 145         self.output_patched = False
 146
 147     def startTest(self, test):
 148         "Called before execute each test method."
 149         self._patch_standard_output()
 150         self.start_time = time.time()
 151         TestResult.startTest(self, test)
 152
 153         if self.showAll:
 154             self.stream.write('  ' + self.getDescription(test))
 155             self.stream.write(" ... ")
 156
 157     def stopTest(self, test):
 158         "Called after execute each test method."
 159         self._restore_standard_output()
 160         _TextTestResult.stopTest(self, test)
 161         self.stop_time = time.time()
 162
 163         if self.callback and callable(self.callback):
 164             self.callback()
 165             self.callback = None
 166
 167     def addSuccess(self, test):
 168         "Called when a test executes successfully."
 169         self._prepare_callback(_TestInfo(self, test),
 170                                self.successes, 'OK', '.')
 171
 172     def addFailure(self, test, err):
 173         "Called when a test method fails."
 174         self._prepare_callback(_TestInfo(self, test, _TestInfo.FAILURE, err),
 175                                self.failures, 'FAIL', 'F')
 176
 177     def addError(self, test, err):
 178         "Called when a test method raises an error."
 179         self._prepare_callback(_TestInfo(self, test, _TestInfo.ERROR, err),
 180                                self.errors, 'ERROR', 'E')
 181
 182     def printErrorList(self, flavour, errors):
 183         "Write some information about the FAIL or ERROR to the stream."
 184         for test_info in errors:
 185             if isinstance(test_info, tuple):
 186                 test_info, exc_info = test_info
 187             self.stream.writeln(self.separator1)
 188             self.stream.writeln('%s [%.3fs]: %s' % (
 189                 flavour, test_info.get_elapsed_time(),
 190                 test_info.get_description()))
 191             self.stream.writeln(self.separator2)
 192             self.stream.writeln('%s' % test_info.get_error_info())
 193
 194     def _get_info_by_testcase(self):
 195         """This method organizes test results by TestCase module. This
 196         information is used during the report generation, where a XML report
 197         will be generated for each TestCase.
 198         """
 199         tests_by_testcase = {}
 200
 201         for tests in (self.successes, self.failures, self.errors):
 202             for test_info in tests:
 203                 testcase = type(test_info.test_method)
 204
 205                 # Ignore module name if it is '__main__'
 206                 module = testcase.__module__ + '.'
 207                 if module == '__main__.':
 208                     module = ''
 209                 testcase_name = module + testcase.__name__
 210
 211                 if testcase_name not in tests_by_testcase:
 212                     tests_by_testcase[testcase_name] = []
 213                 tests_by_testcase[testcase_name].append(test_info)
 214
 215         return tests_by_testcase
 216
 217     def _report_testsuite(suite_name, tests, xml_document):
 218         "Appends the testsuite section to the XML document."
 219         testsuite = xml_document.createElement('testsuite')
 220         xml_document.appendChild(testsuite)
 221
 222         testsuite.setAttribute('name', str(suite_name))
 223         testsuite.setAttribute('tests', str(len(tests)))
 224
 225         testsuite.setAttribute('time', '%.3f' %
 226             sum([e.get_elapsed_time() for e in tests]))
 227
 228         failures = len([1 for e in tests if e.outcome == _TestInfo.FAILURE])
 229         testsuite.setAttribute('failures', str(failures))
 230
 231         errors = len([1 for e in tests if e.outcome == _TestInfo.ERROR])
 232         testsuite.setAttribute('errors', str(errors))
 233
 234         return testsuite
 235
 236     _report_testsuite = staticmethod(_report_testsuite)
 237
 238     def _report_testcase(suite_name, test_result, xml_testsuite, xml_document):
 239         "Appends a testcase section to the XML document."
 240         testcase = xml_document.createElement('testcase')
 241         xml_testsuite.appendChild(testcase)
 242
 243         testcase.setAttribute('classname', str(suite_name))
 244         testcase.setAttribute('name', test_result.test_method.shortDescription()
 245                               or getattr(test_result.test_method, '_testMethodName',
 246                                          str(test_result.test_method)))
 247         testcase.setAttribute('time', '%.3f' % test_result.get_elapsed_time())
 248
 249         if (test_result.outcome != _TestInfo.SUCCESS):
 250             elem_name = ('failure', 'error')[test_result.outcome-1]
 251             failure = xml_document.createElement(elem_name)
 252             testcase.appendChild(failure)
 253
 254             failure.setAttribute('type', str(test_result.err[0].__name__))
 255             failure.setAttribute('message', str(test_result.err[1]))
 256
 257             error_info = test_result.get_error_info()
 258             failureText = xml_document.createCDATAOrText(error_info)
 259             failure.appendChild(failureText)
 260
 261     _report_testcase = staticmethod(_report_testcase)
 262
 263     def _report_output(test_runner, xml_testsuite, xml_document, stdout, stderr):
 264         "Appends the system-out and system-err sections to the XML document."
 265         systemout = xml_document.createElement('system-out')
 266         xml_testsuite.appendChild(systemout)
 267
 268         systemout_text = xml_document.createCDATAOrText(stdout)
 269         systemout.appendChild(systemout_text)
 270
 271         systemerr = xml_document.createElement('system-err')
 272         xml_testsuite.appendChild(systemerr)
 273
 274         systemerr_text = xml_document.createCDATAOrText(stderr)
 275         systemerr.appendChild(systemerr_text)
 276
 277     _report_output = staticmethod(_report_output)
 278
 279     def generate_reports(self, test_runner):
 280         "Generates the XML reports to a given XMLTestRunner object."
 281         all_results = self._get_info_by_testcase()
 282
 283         if type(test_runner.output) == str and not \
 284             os.path.exists(test_runner.output):
 285             os.makedirs(test_runner.output)
 286
 287         for suite, tests in all_results.items():
 288             doc = XMLDocument()
 289
 290             # Build the XML file
 291             testsuite = _XMLTestResult._report_testsuite(suite, tests, doc)
 292             stdout, stderr = [], []
 293             for test in tests:
 294                 _XMLTestResult._report_testcase(suite, test, testsuite, doc)
 295                 if test.stdout:
 296                     stdout.extend(['*****************', test.get_description(), test.stdout])
 297                 if test.stderr:
 298                     stderr.extend(['*****************', test.get_description(), test.stderr])
 299             _XMLTestResult._report_output(test_runner, testsuite, doc,
 300                                           '\n'.join(stdout), '\n'.join(stderr))
 301             xml_content = doc.toprettyxml(indent='\t')
 302
 303             if type(test_runner.output) is str:
 304                 report_file = open('%s%sTEST-%s.xml' % \
 305                     (test_runner.output, os.sep, suite), 'w')
 306                 try:
 307                     report_file.write(xml_content)
 308                 finally:
 309                     report_file.close()
 310             else:
 311                 # Assume that test_runner.output is a stream
 312                 test_runner.output.write(xml_content)
 313
 314
 315 class XMLTestRunner(TextTestRunner):
 316     """A test runner class that outputs the results in JUnit like XML files.
 317     """
 318     def __init__(self, output='.', stream=sys.stderr, descriptions=True, \
 319         verbose=False, elapsed_times=True):
 320         "Create a new instance of XMLTestRunner."
 321         verbosity = (1, 2)[verbose]
 322         TextTestRunner.__init__(self, stream, descriptions, verbosity)
 323         self.output = output
 324         self.elapsed_times = elapsed_times
 325
 326     def _make_result(self):
 327         """Create the TestResult object which will be used to store
 328         information about the executed tests.
 329         """
 330         return _XMLTestResult(self.stream, self.descriptions, \
 331             self.verbosity, self.elapsed_times)
 332
 333     def run(self, test):
 334         "Run the given test case or test suite."
 335         # Prepare the test execution
 336         result = self._make_result()
 337
 338         # Print a nice header
 339         self.stream.writeln()
 340         self.stream.writeln('Running tests...')
 341         self.stream.writeln(result.separator2)
 342
 343         # Execute tests
 344         start_time = time.time()
 345         test(result)
 346         stop_time = time.time()
 347         time_taken = stop_time - start_time
 348
 349         # Print results
 350         result.printErrors()
 351         self.stream.writeln(result.separator2)
 352         run = result.testsRun
 353         self.stream.writeln("Ran %d test%s in %.3fs" %
 354             (run, run != 1 and "s" or "", time_taken))
 355         self.stream.writeln()
 356
 357         # Error traces
 358         if not result.wasSuccessful():
 359             self.stream.write("FAILED (")
 360             failed, errored = (len(result.failures), len(result.errors))
 361             if failed:
 362                 self.stream.write("failures=%d" % failed)
 363             if errored:
 364                 if failed:
 365                     self.stream.write(", ")
 366                 self.stream.write("errors=%d" % errored)
 367             self.stream.writeln(")")
 368         else:
 369             self.stream.writeln("OK")
 370
 371         # Generate reports
 372         self.stream.writeln()
 373         self.stream.writeln('Generating XML reports...')
 374         result.generate_reports(self)
 375
 376         return result