llvm/utils/release/findRegressions-simple.py

   1 #!/usr/bin/env python
   2
   3 from __future__ import print_function
   4 import re, string, sys, os, time, math
   5
   6 DEBUG = 0
   7
   8 (tp, exp) = ("compile", "exec")
   9
  10
  11 def parse(file):
  12     f = open(file, "r")
  13     d = f.read()
  14
  15     # Cleanup weird stuff
  16     d = re.sub(r",\d+:\d", "", d)
  17
  18     r = re.findall(r"TEST-(PASS|FAIL|RESULT.*?):\s+(.*?)\s+(.*?)\r*\n", d)
  19
  20     test = {}
  21     fname = ""
  22     for t in r:
  23         if DEBUG:
  24             print(t)
  25
  26         if t[0] == "PASS" or t[0] == "FAIL":
  27             tmp = t[2].split("llvm-test/")
  28
  29             if DEBUG:
  30                 print(tmp)
  31
  32             if len(tmp) == 2:
  33                 fname = tmp[1].strip("\r\n")
  34             else:
  35                 fname = tmp[0].strip("\r\n")
  36
  37             if fname not in test:
  38                 test[fname] = {}
  39
  40             test[fname][t[1] + " state"] = t[0]
  41             test[fname][t[1] + " time"] = float("nan")
  42         else:
  43             try:
  44                 n = t[0].split("RESULT-")[1]
  45
  46                 if DEBUG:
  47                     print("n == ", n)
  48
  49                 if n == "compile-success":
  50                     test[fname]["compile time"] = float(
  51                         t[2].split("program")[1].strip("\r\n")
  52                     )
  53
  54                 elif n == "exec-success":
  55                     test[fname]["exec time"] = float(
  56                         t[2].split("program")[1].strip("\r\n")
  57                     )
  58                     if DEBUG:
  59                         print(test[fname][string.replace(n, "-success", "")])
  60
  61                 else:
  62                     # print "ERROR!"
  63                     sys.exit(1)
  64
  65             except:
  66                 continue
  67
  68     return test
  69
  70
  71 # Diff results and look for regressions.
  72 def diffResults(d_old, d_new):
  73     regressions = {}
  74     passes = {}
  75     removed = ""
  76
  77     for x in ["compile state", "compile time", "exec state", "exec time"]:
  78         regressions[x] = ""
  79         passes[x] = ""
  80
  81     for t in sorted(d_old.keys()):
  82         if t in d_new:
  83
  84             # Check if the test passed or failed.
  85             for x in ["compile state", "compile time", "exec state", "exec time"]:
  86
  87                 if x not in d_old[t] and x not in d_new[t]:
  88                     continue
  89
  90                 if x in d_old[t]:
  91                     if x in d_new[t]:
  92
  93                         if d_old[t][x] == "PASS":
  94                             if d_new[t][x] != "PASS":
  95                                 regressions[x] += t + "\n"
  96                         else:
  97                             if d_new[t][x] == "PASS":
  98                                 passes[x] += t + "\n"
  99
 100                     else:
 101                         regressions[x] += t + "\n"
 102
 103                 if x == "compile state" or x == "exec state":
 104                     continue
 105
 106                 # For execution time, if there is no result it's a fail.
 107                 if x not in d_old[t] and x not in d_new[t]:
 108                     continue
 109                 elif x not in d_new[t]:
 110                     regressions[x] += t + "\n"
 111                 elif x not in d_old[t]:
 112                     passes[x] += t + "\n"
 113
 114                 if math.isnan(d_old[t][x]) and math.isnan(d_new[t][x]):
 115                     continue
 116
 117                 elif math.isnan(d_old[t][x]) and not math.isnan(d_new[t][x]):
 118                     passes[x] += t + "\n"
 119
 120                 elif not math.isnan(d_old[t][x]) and math.isnan(d_new[t][x]):
 121                     regressions[x] += t + ": NaN%\n"
 122
 123                 if (
 124                     d_new[t][x] > d_old[t][x]
 125                     and d_old[t][x] > 0.0
 126                     and (d_new[t][x] - d_old[t][x]) / d_old[t][x] > 0.05
 127                 ):
 128                     regressions[x] += (
 129                         t
 130                         + ": "
 131                         + "{0:.1f}".format(
 132                             100 * (d_new[t][x] - d_old[t][x]) / d_old[t][x]
 133                         )
 134                         + "%\n"
 135                     )
 136
 137         else:
 138             removed += t + "\n"
 139
 140     if len(regressions["compile state"]) != 0:
 141         print("REGRESSION: Compilation Failed")
 142         print(regressions["compile state"])
 143
 144     if len(regressions["exec state"]) != 0:
 145         print("REGRESSION: Execution Failed")
 146         print(regressions["exec state"])
 147
 148     if len(regressions["compile time"]) != 0:
 149         print("REGRESSION: Compilation Time")
 150         print(regressions["compile time"])
 151
 152     if len(regressions["exec time"]) != 0:
 153         print("REGRESSION: Execution Time")
 154         print(regressions["exec time"])
 155
 156     if len(passes["compile state"]) != 0:
 157         print("NEW PASSES: Compilation")
 158         print(passes["compile state"])
 159
 160     if len(passes["exec state"]) != 0:
 161         print("NEW PASSES: Execution")
 162         print(passes["exec state"])
 163
 164     if len(removed) != 0:
 165         print("REMOVED TESTS")
 166         print(removed)
 167
 168
 169 # Main
 170 if len(sys.argv) < 3:
 171     print("Usage:", sys.argv[0], "<old log> <new log>")
 172     sys.exit(-1)
 173
 174 d_old = parse(sys.argv[1])
 175 d_new = parse(sys.argv[2])
 176
 177 diffResults(d_old, d_new)