dev-support/flaky-tests/findHangingTests.py

   1 #!/usr/bin/env python
   2 ##
   3 # Licensed to the Apache Software Foundation (ASF) under one
   4 # or more contributor license agreements.  See the NOTICE file
   5 # distributed with this work for additional information
   6 # regarding copyright ownership.  The ASF licenses this file
   7 # to you under the Apache License, Version 2.0 (the
   8 # "License"); you may not use this file except in compliance
   9 # with the License.  You may obtain a copy of the License at
  10 #
  11 #     http://www.apache.org/licenses/LICENSE-2.0
  12 #
  13 # Unless required by applicable law or agreed to in writing, software
  14 # distributed under the License is distributed on an "AS IS" BASIS,
  15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16 # See the License for the specific language governing permissions and
  17 # limitations under the License.
  18
  19 # pylint: disable=invalid-name
  20 # To disable 'invalid constant name' warnings.
  21
  22 """
  23 # Script to find hanging test from Jenkins build output
  24 # usage: ./findHangingTests.py <url of Jenkins build console>
  25 """
  26
  27 import re
  28 import sys
  29 import requests
  30
  31 # If any of these strings appear in the console output, it's a build one should probably ignore
  32 # for analyzing failed/hanging tests.
  33 BAD_RUN_STRINGS = [
  34     "Slave went offline during the build",  # Machine went down, can't do anything about it.
  35     "The forked VM terminated without properly saying goodbye",  # JVM crashed.
  36 ]
  37
  38
  39 def get_bad_tests(console_url):
  40     """
  41     Returns [[all tests], [failed tests], [timeout tests], [hanging tests]] if successfully gets
  42     the build information.
  43     If there is error getting console text or if there are blacklisted strings in console text,
  44     then returns None.
  45     """
  46     response = requests.get(console_url)
  47     if response.status_code != 200:
  48         print "Error getting consoleText. Response = {} {}".format(
  49             response.status_code, response.reason)
  50         return
  51
  52     # All tests: All testcases which were run.
  53     # Hanging test: A testcase which started but never finished.
  54     # Failed test: Testcase which encountered any kind of failure. It can be failing atomic tests,
  55     #   timed out tests, etc
  56     # Timeout test: A Testcase which encountered timeout. Naturally, all timeout tests will be
  57     #   included in failed tests.
  58     all_tests_set = set()
  59     hanging_tests_set = set()
  60     failed_tests_set = set()
  61     timeout_tests_set = set()
  62     for line in response.content.splitlines():
  63         result1 = re.findall("Running org.apache.hadoop.hbase.(.*)", line)
  64         if len(result1) == 1:
  65             test_case = result1[0]
  66             if test_case in all_tests_set:
  67                 print ("ERROR! Multiple tests with same name '{}'. Might get wrong results "
  68                        "for this test.".format(test_case))
  69             else:
  70                 hanging_tests_set.add(test_case)
  71                 all_tests_set.add(test_case)
  72         result2 = re.findall("Tests run:.*?- in org.apache.hadoop.hbase.(.*)", line)
  73         if len(result2) == 1:
  74             test_case = result2[0]
  75             if "FAILURE!" in line:
  76                 failed_tests_set.add(test_case)
  77             if test_case not in hanging_tests_set:
  78                 print ("ERROR! No test '{}' found in hanging_tests. Might get wrong results "
  79                        "for this test. This may also happen if maven is set to retry failing "
  80                        "tests.".format(test_case))
  81             else:
  82                 hanging_tests_set.remove(test_case)
  83         result3 = re.match("^\\s+(\\w*).*\\sTestTimedOut", line)
  84         if result3:
  85             test_case = result3.group(1)
  86             timeout_tests_set.add(test_case)
  87         for bad_string in BAD_RUN_STRINGS:
  88             if re.match(".*" + bad_string + ".*", line):
  89                 print "Bad string found in build:\n > {}".format(line)
  90     print "Result > total tests: {:4}   failed : {:4}  timedout : {:4}  hanging : {:4}".format(
  91         len(all_tests_set), len(failed_tests_set), len(timeout_tests_set), len(hanging_tests_set))
  92     return [all_tests_set, failed_tests_set, timeout_tests_set, hanging_tests_set]
  93
  94 if __name__ == "__main__":
  95     if len(sys.argv) != 2:
  96         print "ERROR : Provide the jenkins job console URL as the only argument."
  97         sys.exit(1)
  98
  99     print "Fetching {}".format(sys.argv[1])
 100     result = get_bad_tests(sys.argv[1])
 101     if not result:
 102         sys.exit(1)
 103     [all_tests, failed_tests, timedout_tests, hanging_tests] = result
 104
 105     print "Found {} hanging tests:".format(len(hanging_tests))
 106     for test in hanging_tests:
 107         print test
 108     print "\n"
 109     print "Found {} failed tests of which {} timed out:".format(
 110         len(failed_tests), len(timedout_tests))
 111     for test in failed_tests:
 112         print "{0} {1}".format(test, ("(Timed Out)" if test in timedout_tests else ""))
 113
 114     print ("\nA test may have had 0 or more atomic test failures before it timed out. So a "
 115            "'Timed Out' test may have other errors too.")