.ci/generate_test_report.py

   1 # Script to parse many JUnit XML result files and send a report to the buildkite
   2 # agent as an annotation.
   3 #
   4 # To run the unittests:
   5 # python3 -m unittest discover -p generate_test_report.py
   6
   7 import argparse
   8 import subprocess
   9 import unittest
  10 from io import StringIO
  11 from junitparser import JUnitXml, Failure
  12 from textwrap import dedent
  13
  14
  15 def junit_from_xml(xml):
  16     return JUnitXml.fromfile(StringIO(xml))
  17
  18
  19 class TestReports(unittest.TestCase):
  20     def test_title_only(self):
  21         self.assertEqual(_generate_report("Foo", []), ("", None))
  22
  23     def test_no_tests_in_testsuite(self):
  24         self.assertEqual(
  25             _generate_report(
  26                 "Foo",
  27                 [
  28                     junit_from_xml(
  29                         dedent(
  30                             """\
  31           <?xml version="1.0" encoding="UTF-8"?>
  32           <testsuites time="0.00">
  33           <testsuite name="Empty" tests="0" failures="0" skipped="0" time="0.00">
  34           </testsuite>
  35           </testsuites>"""
  36                         )
  37                     )
  38                 ],
  39             ),
  40             ("", None),
  41         )
  42
  43     def test_no_failures(self):
  44         self.assertEqual(
  45             _generate_report(
  46                 "Foo",
  47                 [
  48                     junit_from_xml(
  49                         dedent(
  50                             """\
  51           <?xml version="1.0" encoding="UTF-8"?>
  52           <testsuites time="0.00">
  53           <testsuite name="Passed" tests="1" failures="0" skipped="0" time="0.00">
  54           <testcase classname="Bar/test_1" name="test_1" time="0.00"/>
  55           </testsuite>
  56           </testsuites>"""
  57                         )
  58                     )
  59                 ],
  60             ),
  61             (
  62                 dedent(
  63                     """\
  64               # Foo
  65
  66               * 1 test passed"""
  67                 ),
  68                 "success",
  69             ),
  70         )
  71
  72     def test_report_single_file_single_testsuite(self):
  73         self.assertEqual(
  74             _generate_report(
  75                 "Foo",
  76                 [
  77                     junit_from_xml(
  78                         dedent(
  79                             """\
  80           <?xml version="1.0" encoding="UTF-8"?>
  81           <testsuites time="8.89">
  82           <testsuite name="Bar" tests="4" failures="2" skipped="1" time="410.63">
  83           <testcase classname="Bar/test_1" name="test_1" time="0.02"/>
  84           <testcase classname="Bar/test_2" name="test_2" time="0.02">
  85             <skipped message="Reason"/>
  86           </testcase>
  87           <testcase classname="Bar/test_3" name="test_3" time="0.02">
  88             <failure><![CDATA[Output goes here]]></failure>
  89           </testcase>
  90           <testcase classname="Bar/test_4" name="test_4" time="0.02">
  91             <failure><![CDATA[Other output goes here]]></failure>
  92           </testcase>
  93           </testsuite>
  94           </testsuites>"""
  95                         )
  96                     )
  97                 ],
  98             ),
  99             (
 100                 dedent(
 101                     """\
 102           # Foo
 103
 104           * 1 test passed
 105           * 1 test skipped
 106           * 2 tests failed
 107
 108           ## Failed Tests
 109           (click to see output)
 110
 111           ### Bar
 112           <details>
 113           <summary>Bar/test_3/test_3</summary>
 114
 115           ```
 116           Output goes here
 117           ```
 118           </details>
 119           <details>
 120           <summary>Bar/test_4/test_4</summary>
 121
 122           ```
 123           Other output goes here
 124           ```
 125           </details>"""
 126                 ),
 127                 "error",
 128             ),
 129         )
 130
 131     MULTI_SUITE_OUTPUT = (
 132         dedent(
 133             """\
 134         # ABC and DEF
 135
 136         * 1 test passed
 137         * 1 test skipped
 138         * 2 tests failed
 139
 140         ## Failed Tests
 141         (click to see output)
 142
 143         ### ABC
 144         <details>
 145         <summary>ABC/test_2/test_2</summary>
 146
 147         ```
 148         ABC/test_2 output goes here
 149         ```
 150         </details>
 151
 152         ### DEF
 153         <details>
 154         <summary>DEF/test_2/test_2</summary>
 155
 156         ```
 157         DEF/test_2 output goes here
 158         ```
 159         </details>"""
 160         ),
 161         "error",
 162     )
 163
 164     def test_report_single_file_multiple_testsuites(self):
 165         self.assertEqual(
 166             _generate_report(
 167                 "ABC and DEF",
 168                 [
 169                     junit_from_xml(
 170                         dedent(
 171                             """\
 172           <?xml version="1.0" encoding="UTF-8"?>
 173           <testsuites time="8.89">
 174           <testsuite name="ABC" tests="2" failures="1" skipped="0" time="410.63">
 175           <testcase classname="ABC/test_1" name="test_1" time="0.02"/>
 176           <testcase classname="ABC/test_2" name="test_2" time="0.02">
 177             <failure><![CDATA[ABC/test_2 output goes here]]></failure>
 178           </testcase>
 179           </testsuite>
 180           <testsuite name="DEF" tests="2" failures="1" skipped="1" time="410.63">
 181           <testcase classname="DEF/test_1" name="test_1" time="0.02">
 182             <skipped message="reason"/>
 183           </testcase>
 184           <testcase classname="DEF/test_2" name="test_2" time="0.02">
 185             <failure><![CDATA[DEF/test_2 output goes here]]></failure>
 186           </testcase>
 187           </testsuite>
 188           </testsuites>"""
 189                         )
 190                     )
 191                 ],
 192             ),
 193             self.MULTI_SUITE_OUTPUT,
 194         )
 195
 196     def test_report_multiple_files_multiple_testsuites(self):
 197         self.assertEqual(
 198             _generate_report(
 199                 "ABC and DEF",
 200                 [
 201                     junit_from_xml(
 202                         dedent(
 203                             """\
 204           <?xml version="1.0" encoding="UTF-8"?>
 205           <testsuites time="8.89">
 206           <testsuite name="ABC" tests="2" failures="1" skipped="0" time="410.63">
 207           <testcase classname="ABC/test_1" name="test_1" time="0.02"/>
 208           <testcase classname="ABC/test_2" name="test_2" time="0.02">
 209             <failure><![CDATA[ABC/test_2 output goes here]]></failure>
 210           </testcase>
 211           </testsuite>
 212           </testsuites>"""
 213                         )
 214                     ),
 215                     junit_from_xml(
 216                         dedent(
 217                             """\
 218           <?xml version="1.0" encoding="UTF-8"?>
 219           <testsuites time="8.89">
 220           <testsuite name="DEF" tests="2" failures="1" skipped="1" time="410.63">
 221           <testcase classname="DEF/test_1" name="test_1" time="0.02">
 222             <skipped message="reason"/>
 223           </testcase>
 224           <testcase classname="DEF/test_2" name="test_2" time="0.02">
 225             <failure><![CDATA[DEF/test_2 output goes here]]></failure>
 226           </testcase>
 227           </testsuite>
 228           </testsuites>"""
 229                         )
 230                     ),
 231                 ],
 232             ),
 233             self.MULTI_SUITE_OUTPUT,
 234         )
 235
 236     def test_report_dont_list_failures(self):
 237         self.assertEqual(
 238             _generate_report(
 239                 "Foo",
 240                 [
 241                     junit_from_xml(
 242                         dedent(
 243                             """\
 244           <?xml version="1.0" encoding="UTF-8"?>
 245           <testsuites time="0.02">
 246           <testsuite name="Bar" tests="1" failures="1" skipped="0" time="0.02">
 247           <testcase classname="Bar/test_1" name="test_1" time="0.02">
 248             <failure><![CDATA[Output goes here]]></failure>
 249           </testcase>
 250           </testsuite>
 251           </testsuites>"""
 252                         )
 253                     )
 254                 ],
 255                 list_failures=False,
 256             ),
 257             (
 258                 dedent(
 259                     """\
 260           # Foo
 261
 262           * 1 test failed
 263
 264           Failed tests and their output was too large to report. Download the build's log file to see the details."""
 265                 ),
 266                 "error",
 267             ),
 268         )
 269
 270     def test_report_size_limit(self):
 271         self.assertEqual(
 272             _generate_report(
 273                 "Foo",
 274                 [
 275                     junit_from_xml(
 276                         dedent(
 277                             """\
 278           <?xml version="1.0" encoding="UTF-8"?>
 279           <testsuites time="0.02">
 280           <testsuite name="Bar" tests="1" failures="1" skipped="0" time="0.02">
 281           <testcase classname="Bar/test_1" name="test_1" time="0.02">
 282             <failure><![CDATA[Some long output goes here...]]></failure>
 283           </testcase>
 284           </testsuite>
 285           </testsuites>"""
 286                         )
 287                     )
 288                 ],
 289                 size_limit=128,
 290             ),
 291             (
 292                 dedent(
 293                     """\
 294           # Foo
 295
 296           * 1 test failed
 297
 298           Failed tests and their output was too large to report. Download the build's log file to see the details."""
 299                 ),
 300                 "error",
 301             ),
 302         )
 303
 304
 305 # Set size_limit to limit the byte size of the report. The default is 1MB as this
 306 # is the most that can be put into an annotation. If the generated report exceeds
 307 # this limit and failures are listed, it will be generated again without failures
 308 # listed. This minimal report will always fit into an annotation.
 309 # If include failures is False, total number of test will be reported but their names
 310 # and output will not be.
 311 def _generate_report(title, junit_objects, size_limit=1024 * 1024, list_failures=True):
 312     if not junit_objects:
 313         return ("", "success")
 314
 315     failures = {}
 316     tests_run = 0
 317     tests_skipped = 0
 318     tests_failed = 0
 319
 320     for results in junit_objects:
 321         for testsuite in results:
 322             tests_run += testsuite.tests
 323             tests_skipped += testsuite.skipped
 324             tests_failed += testsuite.failures
 325
 326             for test in testsuite:
 327                 if (
 328                     not test.is_passed
 329                     and test.result
 330                     and isinstance(test.result[0], Failure)
 331                 ):
 332                     if failures.get(testsuite.name) is None:
 333                         failures[testsuite.name] = []
 334                     failures[testsuite.name].append(
 335                         (test.classname + "/" + test.name, test.result[0].text)
 336                     )
 337
 338     if not tests_run:
 339         return ("", style)
 340
 341     style = "error" if tests_failed else "success"
 342     report = [f"# {title}", ""]
 343
 344     tests_passed = tests_run - tests_skipped - tests_failed
 345
 346     def plural(num_tests):
 347         return "test" if num_tests == 1 else "tests"
 348
 349     if tests_passed:
 350         report.append(f"* {tests_passed} {plural(tests_passed)} passed")
 351     if tests_skipped:
 352         report.append(f"* {tests_skipped} {plural(tests_skipped)} skipped")
 353     if tests_failed:
 354         report.append(f"* {tests_failed} {plural(tests_failed)} failed")
 355
 356     if not list_failures:
 357         report.extend(
 358             [
 359                 "",
 360                 "Failed tests and their output was too large to report. "
 361                 "Download the build's log file to see the details.",
 362             ]
 363         )
 364     elif failures:
 365         report.extend(["", "## Failed Tests", "(click to see output)"])
 366
 367         for testsuite_name, failures in failures.items():
 368             report.extend(["", f"### {testsuite_name}"])
 369             for name, output in failures:
 370                 report.extend(
 371                     [
 372                         "<details>",
 373                         f"<summary>{name}</summary>",
 374                         "",
 375                         "```",
 376                         output,
 377                         "```",
 378                         "</details>",
 379                     ]
 380                 )
 381
 382     report = "\n".join(report)
 383     if len(report.encode("utf-8")) > size_limit:
 384         return _generate_report(title, junit_objects, size_limit, list_failures=False)
 385
 386     return report, style
 387
 388
 389 def generate_report(title, junit_files):
 390     return _generate_report(title, [JUnitXml.fromfile(p) for p in junit_files])
 391
 392
 393 if __name__ == "__main__":
 394     parser = argparse.ArgumentParser()
 395     parser.add_argument(
 396         "title", help="Title of the test report, without Markdown formatting."
 397     )
 398     parser.add_argument("context", help="Annotation context to write to.")
 399     parser.add_argument("junit_files", help="Paths to JUnit report files.", nargs="*")
 400     args = parser.parse_args()
 401
 402     report, style = generate_report(args.title, args.junit_files)
 403
 404     if report:
 405         p = subprocess.Popen(
 406             [
 407                 "buildkite-agent",
 408                 "annotate",
 409                 "--context",
 410                 args.context,
 411                 "--style",
 412                 style,
 413             ],
 414             stdin=subprocess.PIPE,
 415             stderr=subprocess.PIPE,
 416             universal_newlines=True,
 417         )
 418
 419         # The report can be larger than the buffer for command arguments so we send
 420         # it over stdin instead.
 421         _, err = p.communicate(input=report)
 422         if p.returncode:
 423             raise RuntimeError(f"Failed to send report to buildkite-agent:\n{err}")