dev-support/report-flakies.py

   1 #!/usr/bin/env python
   2 ##
   3 # Licensed to the Apache Software Foundation (ASF) under one
   4 # or more contributor license agreements.  See the NOTICE file
   5 # distributed with this work for additional information
   6 # regarding copyright ownership.  The ASF licenses this file
   7 # to you under the Apache License, Version 2.0 (the
   8 # "License"); you may not use this file except in compliance
   9 # with the License.  You may obtain a copy of the License at
  10 #
  11 #     http://www.apache.org/licenses/LICENSE-2.0
  12 #
  13 # Unless required by applicable law or agreed to in writing, software
  14 # distributed under the License is distributed on an "AS IS" BASIS,
  15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16 # See the License for the specific language governing permissions and
  17 # limitations under the License.
  18
  19 # pylint: disable=invalid-name
  20 # To disable 'invalid constant name' warnings.
  21 # pylint: disable=import-error
  22 # Testing environment may not have all dependencies.
  23
  24 """
  25 This script uses Jenkins REST api to collect test result(s) of given build/builds and generates
  26 flakyness data about unittests.
  27 Print help: report-flakies.py -h
  28 """
  29
  30 import argparse
  31 import logging
  32 import os
  33 import time
  34 from collections import OrderedDict
  35 from jinja2 import Template
  36
  37 import requests
  38
  39 import findHangingTests
  40
  41 parser = argparse.ArgumentParser()
  42 parser.add_argument(
  43     '--urls', metavar='URL', action='append', required=True,
  44     help='Urls to analyze, which can refer to simple projects, multi-configuration projects or '
  45          'individual build run.')
  46 parser.add_argument('--excluded-builds', metavar='n1,n2', action='append',
  47                     help='List of build numbers to exclude (or "None"). Not required, '
  48                          'but if specified, number of uses should be same as that of --urls '
  49                          'since the values are matched.')
  50 parser.add_argument('--max-builds', metavar='n', action='append', type=int,
  51                     help='The maximum number of builds to use (if available on jenkins). Specify '
  52                          '0 to analyze all builds. Not required, but if specified, number of uses '
  53                          'should be same as that of --urls since the values are matched.')
  54 parser.add_argument(
  55     "--mvn", action="store_true",
  56     help="Writes two strings for including/excluding these flaky tests using maven flags. These "
  57          "strings are written to files so they can be saved as artifacts and easily imported in "
  58          "other projects. Also writes timeout and failing tests in separate files for "
  59          "reference.")
  60 parser.add_argument("-v", "--verbose", help="Prints more logs.", action="store_true")
  61 args = parser.parse_args()
  62
  63 logging.basicConfig()
  64 logger = logging.getLogger(__name__)
  65 if args.verbose:
  66     logger.setLevel(logging.INFO)
  67
  68
  69 def get_bad_tests(build_url):
  70     """
  71     Given url of an executed build, analyzes its console text, and returns
  72     [list of all tests, list of timeout tests, list of failed tests].
  73     Returns None if can't get console text or if there is any other error.
  74     """
  75     logger.info("Analyzing %s", build_url)
  76     response = requests.get(build_url + "/api/json").json()
  77     if response["building"]:
  78         logger.info("Skipping this build since it is in progress.")
  79         return {}
  80     console_url = build_url + "/consoleText"
  81     build_result = findHangingTests.get_bad_tests(console_url)
  82     if not build_result:
  83         logger.info("Ignoring build %s", build_url)
  84         return
  85     return build_result
  86
  87
  88 def expand_multi_config_projects(cli_args):
  89     """
  90     If any url is of type multi-configuration project (i.e. has key 'activeConfigurations'),
  91     get urls for individual jobs.
  92     """
  93     job_urls = cli_args.urls
  94     excluded_builds_arg = cli_args.excluded_builds
  95     max_builds_arg = cli_args.max_builds
  96     if excluded_builds_arg is not None and len(excluded_builds_arg) != len(job_urls):
  97         raise Exception("Number of --excluded-builds arguments should be same as that of --urls "
  98                         "since values are matched.")
  99     if max_builds_arg is not None and len(max_builds_arg) != len(job_urls):
 100         raise Exception("Number of --max-builds arguments should be same as that of --urls "
 101                         "since values are matched.")
 102     final_expanded_urls = []
 103     for (i, job_url) in enumerate(job_urls):
 104         max_builds = 10000  # Some high number
 105         if max_builds_arg is not None and max_builds_arg[i] != 0:
 106             max_builds = int(max_builds_arg[i])
 107         excluded_builds = []
 108         if excluded_builds_arg is not None and excluded_builds_arg[i] != "None":
 109             excluded_builds = [int(x) for x in excluded_builds_arg[i].split(",")]
 110         response = requests.get(job_url + "/api/json").json()
 111         if response.has_key("activeConfigurations"):
 112             for config in response["activeConfigurations"]:
 113                 final_expanded_urls.append({'url':config["url"], 'max_builds': max_builds,
 114                                             'excludes': excluded_builds})
 115         else:
 116             final_expanded_urls.append({'url':job_url, 'max_builds': max_builds,
 117                                         'excludes': excluded_builds})
 118     return final_expanded_urls
 119
 120
 121 # Set of timeout/failed tests across all given urls.
 122 all_timeout_tests = set()
 123 all_failed_tests = set()
 124 all_hanging_tests = set()
 125 # Contains { <url> : { <bad_test> : { 'all': [<build ids>], 'failed': [<build ids>],
 126 #                                     'timeout': [<build ids>], 'hanging': [<builds ids>] } } }
 127 url_to_bad_test_results = OrderedDict()
 128 # Contains { <url> : [run_ids] }
 129 # Used for common min/max build ids when generating sparklines.
 130 url_to_build_ids = OrderedDict()
 131
 132 # Iterates over each url, gets test results and prints flaky tests.
 133 expanded_urls = expand_multi_config_projects(args)
 134 for url_max_build in expanded_urls:
 135     url = url_max_build["url"]
 136     excludes = url_max_build["excludes"]
 137     json_response = requests.get(url + "/api/json").json()
 138     if json_response.has_key("builds"):
 139         builds = json_response["builds"]
 140         logger.info("Analyzing job: %s", url)
 141     else:
 142         builds = [{'number' : json_response["id"], 'url': url}]
 143         logger.info("Analyzing build : %s", url)
 144     build_id_to_results = {}
 145     num_builds = 0
 146     url_to_build_ids[url] = []
 147     build_ids_without_tests_run = []
 148     for build in builds:
 149         build_id = build["number"]
 150         if build_id in excludes:
 151             continue
 152         result = get_bad_tests(build["url"])
 153         if not result:
 154             continue
 155         if len(result[0]) > 0:
 156             build_id_to_results[build_id] = result
 157         else:
 158             build_ids_without_tests_run.append(build_id)
 159         num_builds += 1
 160         url_to_build_ids[url].append(build_id)
 161         if num_builds == url_max_build["max_builds"]:
 162             break
 163     url_to_build_ids[url].sort()
 164
 165     # Collect list of bad tests.
 166     bad_tests = set()
 167     for build in build_id_to_results:
 168         [_, failed_tests, timeout_tests, hanging_tests] = build_id_to_results[build]
 169         all_timeout_tests.update(timeout_tests)
 170         all_failed_tests.update(failed_tests)
 171         all_hanging_tests.update(hanging_tests)
 172         # Note that timedout tests are already included in failed tests.
 173         bad_tests.update(failed_tests.union(hanging_tests))
 174
 175     # For each bad test, get build ids where it ran, timed out, failed or hanged.
 176     test_to_build_ids = {key : {'all' : set(), 'timeout': set(), 'failed': set(),
 177                                 'hanging' : set(), 'bad_count' : 0}
 178                          for key in bad_tests}
 179     for build in build_id_to_results:
 180         [all_tests, failed_tests, timeout_tests, hanging_tests] = build_id_to_results[build]
 181         for bad_test in test_to_build_ids:
 182             is_bad = False
 183             if all_tests.issuperset([bad_test]):
 184                 test_to_build_ids[bad_test]["all"].add(build)
 185             if timeout_tests.issuperset([bad_test]):
 186                 test_to_build_ids[bad_test]['timeout'].add(build)
 187                 is_bad = True
 188             if failed_tests.issuperset([bad_test]):
 189                 test_to_build_ids[bad_test]['failed'].add(build)
 190                 is_bad = True
 191             if hanging_tests.issuperset([bad_test]):
 192                 test_to_build_ids[bad_test]['hanging'].add(build)
 193                 is_bad = True
 194             if is_bad:
 195                 test_to_build_ids[bad_test]['bad_count'] += 1
 196
 197     # Calculate flakyness % and successful builds for each test. Also sort build ids.
 198     for bad_test in test_to_build_ids:
 199         test_result = test_to_build_ids[bad_test]
 200         test_result['flakyness'] = test_result['bad_count'] * 100.0 / len(test_result['all'])
 201         test_result['success'] = (test_result['all'].difference(
 202             test_result['failed'].union(test_result['hanging'])))
 203         for key in ['all', 'timeout', 'failed', 'hanging', 'success']:
 204             test_result[key] = sorted(test_result[key])
 205
 206
 207     # Sort tests in descending order by flakyness.
 208     sorted_test_to_build_ids = OrderedDict(
 209         sorted(test_to_build_ids.iteritems(), key=lambda x: x[1]['flakyness'], reverse=True))
 210     url_to_bad_test_results[url] = sorted_test_to_build_ids
 211
 212     if len(sorted_test_to_build_ids) > 0:
 213         print "URL: {}".format(url)
 214         print "{:>60}  {:10}  {:25}  {}".format(
 215             "Test Name", "Total Runs", "Bad Runs(failed/timeout/hanging)", "Flakyness")
 216         for bad_test in sorted_test_to_build_ids:
 217             test_status = sorted_test_to_build_ids[bad_test]
 218             print "{:>60}  {:10}  {:7} ( {:4} / {:5} / {:5} )  {:2.0f}%".format(
 219                 bad_test, len(test_status['all']), test_status['bad_count'],
 220                 len(test_status['failed']), len(test_status['timeout']),
 221                 len(test_status['hanging']), test_status['flakyness'])
 222     else:
 223         print "No flaky tests founds."
 224         if len(url_to_build_ids[url]) == len(build_ids_without_tests_run):
 225             print "None of the analyzed builds have test result."
 226
 227     print "Builds analyzed: {}".format(url_to_build_ids[url])
 228     print "Builds without any test runs: {}".format(build_ids_without_tests_run)
 229     print ""
 230
 231
 232 all_bad_tests = all_hanging_tests.union(all_failed_tests)
 233 if args.mvn:
 234     includes = ",".join(all_bad_tests)
 235     with open("./includes", "w") as inc_file:
 236         inc_file.write(includes)
 237
 238     excludes = ["**/{0}.java".format(bad_test) for bad_test in all_bad_tests]
 239     with open("./excludes", "w") as exc_file:
 240         exc_file.write(",".join(excludes))
 241
 242     with open("./timeout", "w") as timeout_file:
 243         timeout_file.write(",".join(all_timeout_tests))
 244
 245     with open("./failed", "w") as failed_file:
 246         failed_file.write(",".join(all_failed_tests))
 247
 248 dev_support_dir = os.path.dirname(os.path.abspath(__file__))
 249 with open(os.path.join(dev_support_dir, "flaky-dashboard-template.html"), "r") as f:
 250     template = Template(f.read())
 251
 252 with open("dashboard.html", "w") as f:
 253     datetime = time.strftime("%m/%d/%Y %H:%M:%S")
 254     f.write(template.render(datetime=datetime, bad_tests_count=len(all_bad_tests),
 255                             results=url_to_bad_test_results, build_ids=url_to_build_ids))