dev-support/report-flakies.py

   1 #!/usr/bin/env python
   2 ##
   3 # Licensed to the Apache Software Foundation (ASF) under one
   4 # or more contributor license agreements.  See the NOTICE file
   5 # distributed with this work for additional information
   6 # regarding copyright ownership.  The ASF licenses this file
   7 # to you under the Apache License, Version 2.0 (the
   8 # "License"); you may not use this file except in compliance
   9 # with the License.  You may obtain a copy of the License at
  10 #
  11 #     http://www.apache.org/licenses/LICENSE-2.0
  12 #
  13 # Unless required by applicable law or agreed to in writing, software
  14 # distributed under the License is distributed on an "AS IS" BASIS,
  15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16 # See the License for the specific language governing permissions and
  17 # limitations under the License.
  18
  19 # pylint: disable=invalid-name
  20 # To disable 'invalid constant name' warnings.
  21 # pylint: disable=import-error
  22 # Testing environment may not have all dependencies.
  23
  24 """
  25 This script uses Jenkins REST api to collect test result(s) of given build/builds and generates
  26 flakyness data about unittests.
  27 Print help: report-flakies.py -h
  28 """
  29
  30 import argparse
  31 import logging
  32 import os
  33 import time
  34 from collections import OrderedDict
  35 from jinja2 import Template
  36
  37 import requests
  38
  39 import findHangingTests
  40
  41 parser = argparse.ArgumentParser()
  42 parser.add_argument(
  43     '--urls', metavar='URL', action='append', required=True,
  44     help='Urls to analyze, which can refer to simple projects, multi-configuration projects or '
  45          'individual build run.')
  46 parser.add_argument('--excluded-builds', metavar='n1,n2', action='append',
  47                     help='List of build numbers to exclude (or "None"). Not required, '
  48                          'but if specified, number of uses should be same as that of --urls '
  49                          'since the values are matched.')
  50 parser.add_argument('--max-builds', metavar='n', action='append', type=int,
  51                     help='The maximum number of builds to use (if available on jenkins). Specify '
  52                          '0 to analyze all builds. Not required, but if specified, number of uses '
  53                          'should be same as that of --urls since the values are matched.')
  54 parser.add_argument('--is-yetus', metavar='True/False', action='append', choices=['True', 'False'],
  55                     help='True, if build is yetus style i.e. look for maven output in artifacts; '
  56                          'False, if maven output is in <url>/consoleText itself.')
  57 parser.add_argument(
  58     "--mvn", action="store_true",
  59     help="Writes two strings for including/excluding these flaky tests using maven flags. These "
  60          "strings are written to files so they can be saved as artifacts and easily imported in "
  61          "other projects. Also writes timeout and failing tests in separate files for "
  62          "reference.")
  63 parser.add_argument("-v", "--verbose", help="Prints more logs.", action="store_true")
  64 args = parser.parse_args()
  65
  66 logging.basicConfig()
  67 logger = logging.getLogger(__name__)
  68 if args.verbose:
  69     logger.setLevel(logging.INFO)
  70
  71
  72 def get_bad_tests(build_url, is_yetus):
  73     """
  74     Given url of an executed build, analyzes its maven output, and returns
  75     [list of all tests, list of timeout tests, list of failed tests].
  76     Returns None if can't get maven output from the build or if there is any other error.
  77     """
  78     logger.info("Analyzing %s", build_url)
  79     response = requests.get(build_url + "/api/json").json()
  80     if response["building"]:
  81         logger.info("Skipping this build since it is in progress.")
  82         return {}
  83     console_url = None
  84     if is_yetus:
  85         for artifact in response["artifacts"]:
  86             if artifact["fileName"] == "patch-unit-root.txt":
  87                 console_url = build_url + "/artifact/" + artifact["relativePath"]
  88                 break
  89         if console_url is None:
  90             logger.info("Can't find 'patch-unit-root.txt' artifact for Yetus build %s\n. Ignoring "
  91                         "this build.", build_url)
  92             return
  93     else:
  94         console_url = build_url + "/consoleText"
  95     build_result = findHangingTests.get_bad_tests(console_url)
  96     if not build_result:
  97         logger.info("Ignoring build %s", build_url)
  98         return
  99     return build_result
 100
 101
 102 def expand_multi_config_projects(cli_args):
 103     """
 104     If any url is of type multi-configuration project (i.e. has key 'activeConfigurations'),
 105     get urls for individual jobs.
 106     """
 107     job_urls = cli_args.urls
 108     excluded_builds_arg = cli_args.excluded_builds
 109     max_builds_arg = cli_args.max_builds
 110     is_yetus_arg = cli_args.is_yetus
 111     if excluded_builds_arg is not None and len(excluded_builds_arg) != len(job_urls):
 112         raise Exception("Number of --excluded-builds arguments should be same as that of --urls "
 113                         "since values are matched.")
 114     if max_builds_arg is not None and len(max_builds_arg) != len(job_urls):
 115         raise Exception("Number of --max-builds arguments should be same as that of --urls "
 116                         "since values are matched.")
 117     final_expanded_urls = []
 118     for (i, job_url) in enumerate(job_urls):
 119         max_builds = 10000  # Some high number
 120         is_yetus = False
 121         if is_yetus_arg is not None:
 122             is_yetus = is_yetus_arg[i] == "True"
 123         if max_builds_arg is not None and max_builds_arg[i] != 0:
 124             max_builds = int(max_builds_arg[i])
 125         excluded_builds = []
 126         if excluded_builds_arg is not None and excluded_builds_arg[i] != "None":
 127             excluded_builds = [int(x) for x in excluded_builds_arg[i].split(",")]
 128         response = requests.get(job_url + "/api/json").json()
 129         if response.has_key("activeConfigurations"):
 130             for config in response["activeConfigurations"]:
 131                 final_expanded_urls.append({'url':config["url"], 'max_builds': max_builds,
 132                                             'excludes': excluded_builds, 'is_yetus': is_yetus})
 133         else:
 134             final_expanded_urls.append({'url':job_url, 'max_builds': max_builds,
 135                                         'excludes': excluded_builds, 'is_yetus': is_yetus})
 136     return final_expanded_urls
 137
 138
 139 # Set of timeout/failed tests across all given urls.
 140 all_timeout_tests = set()
 141 all_failed_tests = set()
 142 all_hanging_tests = set()
 143 # Contains { <url> : { <bad_test> : { 'all': [<build ids>], 'failed': [<build ids>],
 144 #                                     'timeout': [<build ids>], 'hanging': [<builds ids>] } } }
 145 url_to_bad_test_results = OrderedDict()
 146 # Contains { <url> : [run_ids] }
 147 # Used for common min/max build ids when generating sparklines.
 148 url_to_build_ids = OrderedDict()
 149
 150 # Iterates over each url, gets test results and prints flaky tests.
 151 expanded_urls = expand_multi_config_projects(args)
 152 for url_max_build in expanded_urls:
 153     url = url_max_build["url"]
 154     excludes = url_max_build["excludes"]
 155     json_response = requests.get(url + "/api/json").json()
 156     if json_response.has_key("builds"):
 157         builds = json_response["builds"]
 158         logger.info("Analyzing job: %s", url)
 159     else:
 160         builds = [{'number': json_response["id"], 'url': url}]
 161         logger.info("Analyzing build : %s", url)
 162     build_id_to_results = {}
 163     num_builds = 0
 164     url_to_build_ids[url] = []
 165     build_ids_without_tests_run = []
 166     for build in builds:
 167         build_id = build["number"]
 168         if build_id in excludes:
 169             continue
 170         result = get_bad_tests(build["url"], url_max_build['is_yetus'])
 171         if not result:
 172             continue
 173         if len(result[0]) > 0:
 174             build_id_to_results[build_id] = result
 175         else:
 176             build_ids_without_tests_run.append(build_id)
 177         num_builds += 1
 178         url_to_build_ids[url].append(build_id)
 179         if num_builds == url_max_build["max_builds"]:
 180             break
 181     url_to_build_ids[url].sort()
 182
 183     # Collect list of bad tests.
 184     bad_tests = set()
 185     for build in build_id_to_results:
 186         [_, failed_tests, timeout_tests, hanging_tests] = build_id_to_results[build]
 187         all_timeout_tests.update(timeout_tests)
 188         all_failed_tests.update(failed_tests)
 189         all_hanging_tests.update(hanging_tests)
 190         # Note that timedout tests are already included in failed tests.
 191         bad_tests.update(failed_tests.union(hanging_tests))
 192
 193     # For each bad test, get build ids where it ran, timed out, failed or hanged.
 194     test_to_build_ids = {key : {'all' : set(), 'timeout': set(), 'failed': set(),
 195                                 'hanging' : set(), 'bad_count' : 0}
 196                          for key in bad_tests}
 197     for build in build_id_to_results:
 198         [all_tests, failed_tests, timeout_tests, hanging_tests] = build_id_to_results[build]
 199         for bad_test in test_to_build_ids:
 200             is_bad = False
 201             if all_tests.issuperset([bad_test]):
 202                 test_to_build_ids[bad_test]["all"].add(build)
 203             if timeout_tests.issuperset([bad_test]):
 204                 test_to_build_ids[bad_test]['timeout'].add(build)
 205                 is_bad = True
 206             if failed_tests.issuperset([bad_test]):
 207                 test_to_build_ids[bad_test]['failed'].add(build)
 208                 is_bad = True
 209             if hanging_tests.issuperset([bad_test]):
 210                 test_to_build_ids[bad_test]['hanging'].add(build)
 211                 is_bad = True
 212             if is_bad:
 213                 test_to_build_ids[bad_test]['bad_count'] += 1
 214
 215     # Calculate flakyness % and successful builds for each test. Also sort build ids.
 216     for bad_test in test_to_build_ids:
 217         test_result = test_to_build_ids[bad_test]
 218         test_result['flakyness'] = test_result['bad_count'] * 100.0 / len(test_result['all'])
 219         test_result['success'] = (test_result['all'].difference(
 220             test_result['failed'].union(test_result['hanging'])))
 221         for key in ['all', 'timeout', 'failed', 'hanging', 'success']:
 222             test_result[key] = sorted(test_result[key])
 223
 224
 225     # Sort tests in descending order by flakyness.
 226     sorted_test_to_build_ids = OrderedDict(
 227         sorted(test_to_build_ids.iteritems(), key=lambda x: x[1]['flakyness'], reverse=True))
 228     url_to_bad_test_results[url] = sorted_test_to_build_ids
 229
 230     if len(sorted_test_to_build_ids) > 0:
 231         print "URL: {}".format(url)
 232         print "{:>60}  {:10}  {:25}  {}".format(
 233             "Test Name", "Total Runs", "Bad Runs(failed/timeout/hanging)", "Flakyness")
 234         for bad_test in sorted_test_to_build_ids:
 235             test_status = sorted_test_to_build_ids[bad_test]
 236             print "{:>60}  {:10}  {:7} ( {:4} / {:5} / {:5} )  {:2.0f}%".format(
 237                 bad_test, len(test_status['all']), test_status['bad_count'],
 238                 len(test_status['failed']), len(test_status['timeout']),
 239                 len(test_status['hanging']), test_status['flakyness'])
 240     else:
 241         print "No flaky tests founds."
 242         if len(url_to_build_ids[url]) == len(build_ids_without_tests_run):
 243             print "None of the analyzed builds have test result."
 244
 245     print "Builds analyzed: {}".format(url_to_build_ids[url])
 246     print "Builds without any test runs: {}".format(build_ids_without_tests_run)
 247     print ""
 248
 249
 250 all_bad_tests = all_hanging_tests.union(all_failed_tests)
 251 if args.mvn:
 252     includes = ",".join(all_bad_tests)
 253     with open("./includes", "w") as inc_file:
 254         inc_file.write(includes)
 255
 256     excludes = ["**/{0}.java".format(bad_test) for bad_test in all_bad_tests]
 257     with open("./excludes", "w") as exc_file:
 258         exc_file.write(",".join(excludes))
 259
 260     with open("./timeout", "w") as timeout_file:
 261         timeout_file.write(",".join(all_timeout_tests))
 262
 263     with open("./failed", "w") as failed_file:
 264         failed_file.write(",".join(all_failed_tests))
 265
 266 dev_support_dir = os.path.dirname(os.path.abspath(__file__))
 267 with open(os.path.join(dev_support_dir, "flaky-dashboard-template.html"), "r") as f:
 268     template = Template(f.read())
 269
 270 with open("dashboard.html", "w") as f:
 271     datetime = time.strftime("%m/%d/%Y %H:%M:%S")
 272     f.write(template.render(datetime=datetime, bad_tests_count=len(all_bad_tests),
 273                             results=url_to_bad_test_results, build_ids=url_to_build_ids))