.ci/generate_test_report.py

   1 # Script to parse many JUnit XML result files and send a report to the buildkite
   2 # agent as an annotation.
   3 #
   4 # To run the unittests:
   5 # python3 -m unittest discover -p generate_test_report.py
   6
   7 import argparse
   8 import os
   9 import subprocess
  10 import unittest
  11 from io import StringIO
  12 from junitparser import JUnitXml, Failure
  13 from textwrap import dedent
  14
  15
  16 def junit_from_xml(xml):
  17     return JUnitXml.fromfile(StringIO(xml))
  18
  19
  20 class TestReports(unittest.TestCase):
  21     def test_title_only(self):
  22         self.assertEqual(_generate_report("Foo", []), ("", "success"))
  23
  24     def test_no_tests_in_testsuite(self):
  25         self.assertEqual(
  26             _generate_report(
  27                 "Foo",
  28                 [
  29                     junit_from_xml(
  30                         dedent(
  31                             """\
  32           <?xml version="1.0" encoding="UTF-8"?>
  33           <testsuites time="0.00">
  34           <testsuite name="Empty" tests="0" failures="0" skipped="0" time="0.00">
  35           </testsuite>
  36           </testsuites>"""
  37                         )
  38                     )
  39                 ],
  40             ),
  41             ("", None),
  42         )
  43
  44     def test_no_failures(self):
  45         self.assertEqual(
  46             _generate_report(
  47                 "Foo",
  48                 [
  49                     junit_from_xml(
  50                         dedent(
  51                             """\
  52           <?xml version="1.0" encoding="UTF-8"?>
  53           <testsuites time="0.00">
  54           <testsuite name="Passed" tests="1" failures="0" skipped="0" time="0.00">
  55           <testcase classname="Bar/test_1" name="test_1" time="0.00"/>
  56           </testsuite>
  57           </testsuites>"""
  58                         )
  59                     )
  60                 ],
  61             ),
  62             (
  63                 dedent(
  64                     """\
  65               # Foo
  66
  67               * 1 test passed"""
  68                 ),
  69                 "success",
  70             ),
  71         )
  72
  73     def test_report_single_file_single_testsuite(self):
  74         self.assertEqual(
  75             _generate_report(
  76                 "Foo",
  77                 [
  78                     junit_from_xml(
  79                         dedent(
  80                             """\
  81           <?xml version="1.0" encoding="UTF-8"?>
  82           <testsuites time="8.89">
  83           <testsuite name="Bar" tests="4" failures="2" skipped="1" time="410.63">
  84           <testcase classname="Bar/test_1" name="test_1" time="0.02"/>
  85           <testcase classname="Bar/test_2" name="test_2" time="0.02">
  86             <skipped message="Reason"/>
  87           </testcase>
  88           <testcase classname="Bar/test_3" name="test_3" time="0.02">
  89             <failure><![CDATA[Output goes here]]></failure>
  90           </testcase>
  91           <testcase classname="Bar/test_4" name="test_4" time="0.02">
  92             <failure><![CDATA[Other output goes here]]></failure>
  93           </testcase>
  94           </testsuite>
  95           </testsuites>"""
  96                         )
  97                     )
  98                 ],
  99             ),
 100             (
 101                 dedent(
 102                     """\
 103           # Foo
 104
 105           * 1 test passed
 106           * 1 test skipped
 107           * 2 tests failed
 108
 109           ## Failed Tests
 110           (click to see output)
 111
 112           ### Bar
 113           <details>
 114           <summary>Bar/test_3/test_3</summary>
 115
 116           ```
 117           Output goes here
 118           ```
 119           </details>
 120           <details>
 121           <summary>Bar/test_4/test_4</summary>
 122
 123           ```
 124           Other output goes here
 125           ```
 126           </details>"""
 127                 ),
 128                 "error",
 129             ),
 130         )
 131
 132     MULTI_SUITE_OUTPUT = (
 133         dedent(
 134             """\
 135         # ABC and DEF
 136
 137         * 1 test passed
 138         * 1 test skipped
 139         * 2 tests failed
 140
 141         ## Failed Tests
 142         (click to see output)
 143
 144         ### ABC
 145         <details>
 146         <summary>ABC/test_2/test_2</summary>
 147
 148         ```
 149         ABC/test_2 output goes here
 150         ```
 151         </details>
 152
 153         ### DEF
 154         <details>
 155         <summary>DEF/test_2/test_2</summary>
 156
 157         ```
 158         DEF/test_2 output goes here
 159         ```
 160         </details>"""
 161         ),
 162         "error",
 163     )
 164
 165     def test_report_single_file_multiple_testsuites(self):
 166         self.assertEqual(
 167             _generate_report(
 168                 "ABC and DEF",
 169                 [
 170                     junit_from_xml(
 171                         dedent(
 172                             """\
 173           <?xml version="1.0" encoding="UTF-8"?>
 174           <testsuites time="8.89">
 175           <testsuite name="ABC" tests="2" failures="1" skipped="0" time="410.63">
 176           <testcase classname="ABC/test_1" name="test_1" time="0.02"/>
 177           <testcase classname="ABC/test_2" name="test_2" time="0.02">
 178             <failure><![CDATA[ABC/test_2 output goes here]]></failure>
 179           </testcase>
 180           </testsuite>
 181           <testsuite name="DEF" tests="2" failures="1" skipped="1" time="410.63">
 182           <testcase classname="DEF/test_1" name="test_1" time="0.02">
 183             <skipped message="reason"/>
 184           </testcase>
 185           <testcase classname="DEF/test_2" name="test_2" time="0.02">
 186             <failure><![CDATA[DEF/test_2 output goes here]]></failure>
 187           </testcase>
 188           </testsuite>
 189           </testsuites>"""
 190                         )
 191                     )
 192                 ],
 193             ),
 194             self.MULTI_SUITE_OUTPUT,
 195         )
 196
 197     def test_report_multiple_files_multiple_testsuites(self):
 198         self.assertEqual(
 199             _generate_report(
 200                 "ABC and DEF",
 201                 [
 202                     junit_from_xml(
 203                         dedent(
 204                             """\
 205           <?xml version="1.0" encoding="UTF-8"?>
 206           <testsuites time="8.89">
 207           <testsuite name="ABC" tests="2" failures="1" skipped="0" time="410.63">
 208           <testcase classname="ABC/test_1" name="test_1" time="0.02"/>
 209           <testcase classname="ABC/test_2" name="test_2" time="0.02">
 210             <failure><![CDATA[ABC/test_2 output goes here]]></failure>
 211           </testcase>
 212           </testsuite>
 213           </testsuites>"""
 214                         )
 215                     ),
 216                     junit_from_xml(
 217                         dedent(
 218                             """\
 219           <?xml version="1.0" encoding="UTF-8"?>
 220           <testsuites time="8.89">
 221           <testsuite name="DEF" tests="2" failures="1" skipped="1" time="410.63">
 222           <testcase classname="DEF/test_1" name="test_1" time="0.02">
 223             <skipped message="reason"/>
 224           </testcase>
 225           <testcase classname="DEF/test_2" name="test_2" time="0.02">
 226             <failure><![CDATA[DEF/test_2 output goes here]]></failure>
 227           </testcase>
 228           </testsuite>
 229           </testsuites>"""
 230                         )
 231                     ),
 232                 ],
 233             ),
 234             self.MULTI_SUITE_OUTPUT,
 235         )
 236
 237     def test_report_dont_list_failures(self):
 238         self.assertEqual(
 239             _generate_report(
 240                 "Foo",
 241                 [
 242                     junit_from_xml(
 243                         dedent(
 244                             """\
 245           <?xml version="1.0" encoding="UTF-8"?>
 246           <testsuites time="0.02">
 247           <testsuite name="Bar" tests="1" failures="1" skipped="0" time="0.02">
 248           <testcase classname="Bar/test_1" name="test_1" time="0.02">
 249             <failure><![CDATA[Output goes here]]></failure>
 250           </testcase>
 251           </testsuite>
 252           </testsuites>"""
 253                         )
 254                     )
 255                 ],
 256                 list_failures=False,
 257             ),
 258             (
 259                 dedent(
 260                     """\
 261           # Foo
 262
 263           * 1 test failed
 264
 265           Failed tests and their output was too large to report. Download the build's log file to see the details."""
 266                 ),
 267                 "error",
 268             ),
 269         )
 270
 271     def test_report_dont_list_failures_link_to_log(self):
 272         self.assertEqual(
 273             _generate_report(
 274                 "Foo",
 275                 [
 276                     junit_from_xml(
 277                         dedent(
 278                             """\
 279           <?xml version="1.0" encoding="UTF-8"?>
 280           <testsuites time="0.02">
 281           <testsuite name="Bar" tests="1" failures="1" skipped="0" time="0.02">
 282           <testcase classname="Bar/test_1" name="test_1" time="0.02">
 283             <failure><![CDATA[Output goes here]]></failure>
 284           </testcase>
 285           </testsuite>
 286           </testsuites>"""
 287                         )
 288                     )
 289                 ],
 290                 list_failures=False,
 291                 buildkite_info={
 292                     "BUILDKITE_ORGANIZATION_SLUG": "organization_slug",
 293                     "BUILDKITE_PIPELINE_SLUG": "pipeline_slug",
 294                     "BUILDKITE_BUILD_NUMBER": "build_number",
 295                     "BUILDKITE_JOB_ID": "job_id",
 296                 },
 297             ),
 298             (
 299                 dedent(
 300                     """\
 301           # Foo
 302
 303           * 1 test failed
 304
 305           Failed tests and their output was too large to report. [Download](https://buildkite.com/organizations/organization_slug/pipelines/pipeline_slug/builds/build_number/jobs/job_id/download.txt) the build's log file to see the details."""
 306                 ),
 307                 "error",
 308             ),
 309         )
 310
 311     def test_report_size_limit(self):
 312         self.assertEqual(
 313             _generate_report(
 314                 "Foo",
 315                 [
 316                     junit_from_xml(
 317                         dedent(
 318                             """\
 319           <?xml version="1.0" encoding="UTF-8"?>
 320           <testsuites time="0.02">
 321           <testsuite name="Bar" tests="1" failures="1" skipped="0" time="0.02">
 322           <testcase classname="Bar/test_1" name="test_1" time="0.02">
 323             <failure><![CDATA[Some long output goes here...]]></failure>
 324           </testcase>
 325           </testsuite>
 326           </testsuites>"""
 327                         )
 328                     )
 329                 ],
 330                 size_limit=128,
 331             ),
 332             (
 333                 dedent(
 334                     """\
 335           # Foo
 336
 337           * 1 test failed
 338
 339           Failed tests and their output was too large to report. Download the build's log file to see the details."""
 340                 ),
 341                 "error",
 342             ),
 343         )
 344
 345
 346 # Set size_limit to limit the byte size of the report. The default is 1MB as this
 347 # is the most that can be put into an annotation. If the generated report exceeds
 348 # this limit and failures are listed, it will be generated again without failures
 349 # listed. This minimal report will always fit into an annotation.
 350 # If include failures is False, total number of test will be reported but their names
 351 # and output will not be.
 352 def _generate_report(
 353     title,
 354     junit_objects,
 355     size_limit=1024 * 1024,
 356     list_failures=True,
 357     buildkite_info=None,
 358 ):
 359     if not junit_objects:
 360         return ("", "success")
 361
 362     failures = {}
 363     tests_run = 0
 364     tests_skipped = 0
 365     tests_failed = 0
 366
 367     for results in junit_objects:
 368         for testsuite in results:
 369             tests_run += testsuite.tests
 370             tests_skipped += testsuite.skipped
 371             tests_failed += testsuite.failures
 372
 373             for test in testsuite:
 374                 if (
 375                     not test.is_passed
 376                     and test.result
 377                     and isinstance(test.result[0], Failure)
 378                 ):
 379                     if failures.get(testsuite.name) is None:
 380                         failures[testsuite.name] = []
 381                     failures[testsuite.name].append(
 382                         (test.classname + "/" + test.name, test.result[0].text)
 383                     )
 384
 385     if not tests_run:
 386         return ("", None)
 387
 388     style = "error" if tests_failed else "success"
 389     report = [f"# {title}", ""]
 390
 391     tests_passed = tests_run - tests_skipped - tests_failed
 392
 393     def plural(num_tests):
 394         return "test" if num_tests == 1 else "tests"
 395
 396     if tests_passed:
 397         report.append(f"* {tests_passed} {plural(tests_passed)} passed")
 398     if tests_skipped:
 399         report.append(f"* {tests_skipped} {plural(tests_skipped)} skipped")
 400     if tests_failed:
 401         report.append(f"* {tests_failed} {plural(tests_failed)} failed")
 402
 403     if not list_failures:
 404         if buildkite_info is not None:
 405             log_url = (
 406                 "https://buildkite.com/organizations/{BUILDKITE_ORGANIZATION_SLUG}/"
 407                 "pipelines/{BUILDKITE_PIPELINE_SLUG}/builds/{BUILDKITE_BUILD_NUMBER}/"
 408                 "jobs/{BUILDKITE_JOB_ID}/download.txt".format(**buildkite_info)
 409             )
 410             download_text = f"[Download]({log_url})"
 411         else:
 412             download_text = "Download"
 413
 414         report.extend(
 415             [
 416                 "",
 417                 "Failed tests and their output was too large to report. "
 418                 f"{download_text} the build's log file to see the details.",
 419             ]
 420         )
 421     elif failures:
 422         report.extend(["", "## Failed Tests", "(click to see output)"])
 423
 424         for testsuite_name, failures in failures.items():
 425             report.extend(["", f"### {testsuite_name}"])
 426             for name, output in failures:
 427                 report.extend(
 428                     [
 429                         "<details>",
 430                         f"<summary>{name}</summary>",
 431                         "",
 432                         "```",
 433                         output,
 434                         "```",
 435                         "</details>",
 436                     ]
 437                 )
 438
 439     report = "\n".join(report)
 440     if len(report.encode("utf-8")) > size_limit:
 441         return _generate_report(
 442             title,
 443             junit_objects,
 444             size_limit,
 445             list_failures=False,
 446             buildkite_info=buildkite_info,
 447         )
 448
 449     return report, style
 450
 451
 452 def generate_report(title, junit_files, buildkite_info):
 453     return _generate_report(
 454         title,
 455         [JUnitXml.fromfile(p) for p in junit_files],
 456         buildkite_info=buildkite_info,
 457     )
 458
 459
 460 if __name__ == "__main__":
 461     parser = argparse.ArgumentParser()
 462     parser.add_argument(
 463         "title", help="Title of the test report, without Markdown formatting."
 464     )
 465     parser.add_argument("context", help="Annotation context to write to.")
 466     parser.add_argument("junit_files", help="Paths to JUnit report files.", nargs="*")
 467     args = parser.parse_args()
 468
 469     # All of these are required to build a link to download the log file.
 470     env_var_names = [
 471         "BUILDKITE_ORGANIZATION_SLUG",
 472         "BUILDKITE_PIPELINE_SLUG",
 473         "BUILDKITE_BUILD_NUMBER",
 474         "BUILDKITE_JOB_ID",
 475     ]
 476     buildkite_info = {k: v for k, v in os.environ.items() if k in env_var_names}
 477     if len(buildkite_info) != len(env_var_names):
 478         buildkite_info = None
 479
 480     report, style = generate_report(args.title, args.junit_files, buildkite_info)
 481
 482     if report:
 483         p = subprocess.Popen(
 484             [
 485                 "buildkite-agent",
 486                 "annotate",
 487                 "--context",
 488                 args.context,
 489                 "--style",
 490                 style,
 491             ],
 492             stdin=subprocess.PIPE,
 493             stderr=subprocess.PIPE,
 494             universal_newlines=True,
 495         )
 496
 497         # The report can be larger than the buffer for command arguments so we send
 498         # it over stdin instead.
 499         _, err = p.communicate(input=report)
 500         if p.returncode:
 501             raise RuntimeError(f"Failed to send report to buildkite-agent:\n{err}")