gdb/testsuite/analyze-racy-logs.py

   1 #!/usr/bin/env python
   2
   3 # Copyright (C) 2016-2021 Free Software Foundation, Inc.
   4 #
   5 # This file is part of GDB.
   6 #
   7 # This program is free software; you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation; either version 3 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19
  20
  21 # This program is used to analyze the test results (i.e., *.sum files)
  22 # generated by GDB's testsuite, and print the testcases that are found
  23 # to be racy.
  24 #
  25 # Racy testcases are considered as being testcases which can
  26 # intermittently FAIL (or PASS) when run two or more times
  27 # consecutively, i.e., tests whose results are not deterministic.
  28 #
  29 # This program is invoked when the user runs "make check" and
  30 # specifies the RACY_ITER environment variable.
  31
  32 import sys
  33 import os
  34 import re
  35
  36 # The (global) dictionary that stores the associations between a *.sum
  37 # file and its results.  The data inside it will be stored as:
  38 #
  39 # files_and_tests = { 'file1.sum' : { 'PASS' : { 'test1', 'test2' ... },
  40 #                                     'FAIL' : { 'test5', 'test6' ... },
  41 #                                     ...
  42 #                                   },
  43 #                   { 'file2.sum' : { 'PASS' : { 'test1', 'test3' ... },
  44 #                                   ...
  45 #                                   }
  46 #                   }
  47
  48 files_and_tests = dict()
  49
  50 # The relatioships between various states of the same tests that
  51 # should be ignored.  For example, if the same test PASSes on a
  52 # testcase run but KFAILs on another, this test should be considered
  53 # racy because a known-failure is...  known.
  54
  55 ignore_relations = {"PASS": "KFAIL"}
  56
  57 # We are interested in lines that start with '.?(PASS|FAIL)'.  In
  58 # other words, we don't process errors (maybe we should).
  59
  60 sum_matcher = re.compile("^(.?(PASS|FAIL)): (.*)$")
  61
  62
  63 def parse_sum_line(line, dic):
  64     """Parse a single LINE from a sumfile, and store the results in the
  65     dictionary referenced by DIC."""
  66     global sum_matcher
  67
  68     line = line.rstrip()
  69     m = re.match(sum_matcher, line)
  70     if m:
  71         result = m.group(1)
  72         test_name = m.group(3)
  73         # Remove tail parentheses.  These are likely to be '(timeout)'
  74         # and other extra information that will only confuse us.
  75         test_name = re.sub("(\s+)?\(.*$", "", test_name)
  76         if result not in dic.keys():
  77             dic[result] = set()
  78         if test_name in dic[result]:
  79             # If the line is already present in the dictionary, then
  80             # we include a unique identifier in the end of it, in the
  81             # form or '<<N>>' (where N is a number >= 2).  This is
  82             # useful because the GDB testsuite is full of non-unique
  83             # test messages; however, if you process the racy summary
  84             # file you will also need to perform this same operation
  85             # in order to identify the racy test.
  86             i = 2
  87             while True:
  88                 nname = test_name + " <<" + str(i) + ">>"
  89                 if nname not in dic[result]:
  90                     break
  91                 i += 1
  92             test_name = nname
  93         dic[result].add(test_name)
  94
  95
  96 def read_sum_files(files):
  97     """Read the sumfiles (passed as a list in the FILES variable), and
  98     process each one, filling the FILES_AND_TESTS global dictionary with
  99     information about them."""
 100     global files_and_tests
 101
 102     for x in files:
 103         with open(x, "r") as f:
 104             files_and_tests[x] = dict()
 105             for line in f.readlines():
 106                 parse_sum_line(line, files_and_tests[x])
 107
 108
 109 def identify_racy_tests():
 110     """Identify and print the racy tests.  This function basically works
 111     on sets, and the idea behind it is simple.  It takes all the sets that
 112     refer to the same result (for example, all the sets that contain PASS
 113     tests), and compare them.  If a test is present in all PASS sets, then
 114     it is not racy.  Otherwise, it is.
 115
 116     This function does that for all sets (PASS, FAIL, KPASS, KFAIL, etc.),
 117     and then print a sorted list (without duplicates) of all the tests
 118     that were found to be racy."""
 119     global files_and_tests
 120
 121     # First, construct two dictionaries that will hold one set of
 122     # testcases for each state (PASS, FAIL, etc.).
 123     #
 124     # Each set in NONRACY_TESTS will contain only the non-racy
 125     # testcases for that state.  A non-racy testcase is a testcase
 126     # that has the same state in all test runs.
 127     #
 128     # Each set in ALL_TESTS will contain all tests, racy or not, for
 129     # that state.
 130     nonracy_tests = dict()
 131     all_tests = dict()
 132     for f in files_and_tests:
 133         for state in files_and_tests[f]:
 134             try:
 135                 nonracy_tests[state] &= files_and_tests[f][state].copy()
 136             except KeyError:
 137                 nonracy_tests[state] = files_and_tests[f][state].copy()
 138
 139             try:
 140                 all_tests[state] |= files_and_tests[f][state].copy()
 141             except KeyError:
 142                 all_tests[state] = files_and_tests[f][state].copy()
 143
 144     # Now, we eliminate the tests that are present in states that need
 145     # to be ignored.  For example, tests both in the PASS and KFAIL
 146     # states should not be considered racy.
 147     ignored_tests = set()
 148     for s1, s2 in ignore_relations.iteritems():
 149         try:
 150             ignored_tests |= all_tests[s1] & all_tests[s2]
 151         except:
 152             continue
 153
 154     racy_tests = set()
 155     for f in files_and_tests:
 156         for state in files_and_tests[f]:
 157             racy_tests |= files_and_tests[f][state] - nonracy_tests[state]
 158
 159     racy_tests = racy_tests - ignored_tests
 160
 161     # Print the header.
 162     print "\t\t=== gdb racy tests ===\n"
 163
 164     # Print each test.
 165     for line in sorted(racy_tests):
 166         print line
 167
 168     # Print the summary.
 169     print "\n"
 170     print "\t\t=== gdb Summary ===\n"
 171     print "# of racy tests:\t\t%d" % len(racy_tests)
 172
 173
 174 if __name__ == "__main__":
 175     if len(sys.argv) < 3:
 176         # It only makes sense to invoke this program if you pass two
 177         # or more files to be analyzed.
 178         sys.exit("Usage: %s [FILE] [FILE] ..." % sys.argv[0])
 179     read_sum_files(sys.argv[1:])
 180     identify_racy_tests()
 181     exit(0)