gdb/testsuite/analyze-racy-logs.py

   1 #!/usr/bin/env python3
   2
   3 # Copyright (C) 2016-2024 Free Software Foundation, Inc.
   4 #
   5 # This file is part of GDB.
   6 #
   7 # This program is free software; you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation; either version 3 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19
  20
  21 # This program is used to analyze the test results (i.e., *.sum files)
  22 # generated by GDB's testsuite, and print the testcases that are found
  23 # to be racy.
  24 #
  25 # Racy testcases are considered as being testcases which can
  26 # intermittently FAIL (or PASS) when run two or more times
  27 # consecutively, i.e., tests whose results are not deterministic.
  28 #
  29 # This program is invoked when the user runs "make check" and
  30 # specifies the RACY_ITER environment variable.
  31
  32 import re
  33 import sys
  34
  35 # The (global) dictionary that stores the associations between a *.sum
  36 # file and its results.  The data inside it will be stored as:
  37 #
  38 # files_and_tests = { 'file1.sum' : { 'PASS' : { 'test1', 'test2' ... },
  39 #                                     'FAIL' : { 'test5', 'test6' ... },
  40 #                                     ...
  41 #                                   },
  42 #                   { 'file2.sum' : { 'PASS' : { 'test1', 'test3' ... },
  43 #                                   ...
  44 #                                   }
  45 #                   }
  46
  47 files_and_tests: dict[str, dict[str, set[str]]] = dict()
  48
  49 # The relatioships between various states of the same tests that
  50 # should be ignored.  For example, if the same test PASSes on a
  51 # testcase run but KFAILs on another, this test should be considered
  52 # racy because a known-failure is...  known.
  53
  54 ignore_relations = {"PASS": "KFAIL"}
  55
  56 # We are interested in lines that start with '.?(PASS|FAIL)'.  In
  57 # other words, we don't process errors (maybe we should).
  58
  59 sum_matcher = re.compile("^(.?(PASS|FAIL)): (.*)$")
  60
  61
  62 def parse_sum_line(line: str, dic: dict[str, set[str]]):
  63     """Parse a single LINE from a sumfile, and store the results in the
  64     dictionary referenced by DIC."""
  65     global sum_matcher
  66
  67     line = line.rstrip()
  68     m = re.match(sum_matcher, line)
  69     if m:
  70         result = m.group(1)
  71         test_name = m.group(3)
  72         # Remove tail parentheses.  These are likely to be '(timeout)'
  73         # and other extra information that will only confuse us.
  74         test_name = re.sub(r"(\s+)?\(.*$", "", test_name)
  75         if result not in dic.keys():
  76             dic[result] = set()
  77         if test_name in dic[result]:
  78             # If the line is already present in the dictionary, then
  79             # we include a unique identifier in the end of it, in the
  80             # form or '<<N>>' (where N is a number >= 2).  This is
  81             # useful because the GDB testsuite is full of non-unique
  82             # test messages; however, if you process the racy summary
  83             # file you will also need to perform this same operation
  84             # in order to identify the racy test.
  85             i = 2
  86             while True:
  87                 nname = test_name + " <<" + str(i) + ">>"
  88                 if nname not in dic[result]:
  89                     break
  90                 i += 1
  91             test_name = nname
  92         dic[result].add(test_name)
  93
  94
  95 def read_sum_files(files: list[str]):
  96     """Read the sumfiles (passed as a list in the FILES variable), and
  97     process each one, filling the FILES_AND_TESTS global dictionary with
  98     information about them."""
  99     global files_and_tests
 100
 101     for x in files:
 102         with open(x, "r") as f:
 103             files_and_tests[x] = dict()
 104             for line in f.readlines():
 105                 parse_sum_line(line, files_and_tests[x])
 106
 107
 108 def identify_racy_tests():
 109     """Identify and print the racy tests.  This function basically works
 110     on sets, and the idea behind it is simple.  It takes all the sets that
 111     refer to the same result (for example, all the sets that contain PASS
 112     tests), and compare them.  If a test is present in all PASS sets, then
 113     it is not racy.  Otherwise, it is.
 114
 115     This function does that for all sets (PASS, FAIL, KPASS, KFAIL, etc.),
 116     and then print a sorted list (without duplicates) of all the tests
 117     that were found to be racy."""
 118     global files_and_tests
 119
 120     # First, construct two dictionaries that will hold one set of
 121     # testcases for each state (PASS, FAIL, etc.).
 122     #
 123     # Each set in NONRACY_TESTS will contain only the non-racy
 124     # testcases for that state.  A non-racy testcase is a testcase
 125     # that has the same state in all test runs.
 126     #
 127     # Each set in ALL_TESTS will contain all tests, racy or not, for
 128     # that state.
 129     nonracy_tests: dict[str, set[str]] = dict()
 130     all_tests: dict[str, set[str]] = dict()
 131     for f in files_and_tests:
 132         for state in files_and_tests[f]:
 133             try:
 134                 nonracy_tests[state] &= files_and_tests[f][state].copy()
 135             except KeyError:
 136                 nonracy_tests[state] = files_and_tests[f][state].copy()
 137
 138             try:
 139                 all_tests[state] |= files_and_tests[f][state].copy()
 140             except KeyError:
 141                 all_tests[state] = files_and_tests[f][state].copy()
 142
 143     # Now, we eliminate the tests that are present in states that need
 144     # to be ignored.  For example, tests both in the PASS and KFAIL
 145     # states should not be considered racy.
 146     ignored_tests: set[str] = set()
 147     for s1, s2 in ignore_relations.items():
 148         try:
 149             ignored_tests |= all_tests[s1] & all_tests[s2]
 150         except:
 151             continue
 152
 153     racy_tests: set[str] = set()
 154     for f in files_and_tests:
 155         for state in files_and_tests[f]:
 156             racy_tests |= files_and_tests[f][state] - nonracy_tests[state]
 157
 158     racy_tests = racy_tests - ignored_tests
 159
 160     # Print the header.
 161     print("\t\t=== gdb racy tests ===\n")
 162
 163     # Print each test.
 164     for line in sorted(racy_tests):
 165         print(line)
 166
 167     # Print the summary.
 168     print("\n")
 169     print("\t\t=== gdb Summary ===\n")
 170     print("# of racy tests:\t\t%d" % len(racy_tests))
 171
 172
 173 if __name__ == "__main__":
 174     if len(sys.argv) < 3:
 175         # It only makes sense to invoke this program if you pass two
 176         # or more files to be analyzed.
 177         sys.exit("Usage: %s [FILE] [FILE] ..." % sys.argv[0])
 178     read_sum_files(sys.argv[1:])
 179     identify_racy_tests()
 180     exit(0)