llvm/utils/release/findRegressions-nightly.py

   1 #!/usr/bin/env python
   2 from __future__ import print_function
   3
   4 import re, string, sys, os, time
   5
   6 DEBUG = 0
   7 testDirName = "llvm-test"
   8 test = ["compile", "llc", "jit", "cbe"]
   9 exectime = [
  10     "llc-time",
  11     "jit-time",
  12     "cbe-time",
  13 ]
  14 comptime = ["llc", "jit-comptime", "compile"]
  15
  16 (tp, exp) = ("compileTime_", "executeTime_")
  17
  18
  19 def parse(file):
  20     f = open(file, "r")
  21     d = f.read()
  22
  23     # Cleanup weird stuff
  24     d = re.sub(r",\d+:\d", "", d)
  25
  26     r = re.findall(r"TEST-(PASS|FAIL|RESULT.*?):\s+(.*?)\s+(.*?)\r*\n", d)
  27
  28     test = {}
  29     fname = ""
  30     for t in r:
  31         if DEBUG:
  32             print(t)
  33         if t[0] == "PASS" or t[0] == "FAIL":
  34             tmp = t[2].split(testDirName)
  35
  36             if DEBUG:
  37                 print(tmp)
  38
  39             if len(tmp) == 2:
  40                 fname = tmp[1].strip("\r\n")
  41             else:
  42                 fname = tmp[0].strip("\r\n")
  43
  44             if fname not in test:
  45                 test[fname] = {}
  46
  47             for k in test:
  48                 test[fname][k] = "NA"
  49                 test[fname][t[1]] = t[0]
  50                 if DEBUG:
  51                     print(test[fname][t[1]])
  52         else:
  53             try:
  54                 n = t[0].split("RESULT-")[1]
  55
  56                 if DEBUG:
  57                     print(n)
  58
  59                 if n == "llc" or n == "jit-comptime" or n == "compile":
  60                     test[fname][tp + n] = float(t[2].split(" ")[2])
  61                     if DEBUG:
  62                         print(test[fname][tp + n])
  63
  64                 elif n.endswith("-time"):
  65                     test[fname][exp + n] = float(t[2].strip("\r\n"))
  66                     if DEBUG:
  67                         print(test[fname][exp + n])
  68
  69                 else:
  70                     print("ERROR!")
  71                     sys.exit(1)
  72
  73             except:
  74                 continue
  75
  76     return test
  77
  78
  79 # Diff results and look for regressions.
  80 def diffResults(d_old, d_new):
  81
  82     for t in sorted(d_old.keys()):
  83         if DEBUG:
  84             print(t)
  85
  86         if t in d_new:
  87
  88             # Check if the test passed or failed.
  89             for x in test:
  90                 if x in d_old[t]:
  91                     if x in d_new[t]:
  92                         if d_old[t][x] == "PASS":
  93                             if d_new[t][x] != "PASS":
  94                                 print(t + " *** REGRESSION (" + x + ")\n")
  95                         else:
  96                             if d_new[t][x] == "PASS":
  97                                 print(t + " * NEW PASS (" + x + ")\n")
  98
  99                     else:
 100                         print(t + "*** REGRESSION (" + x + ")\n")
 101
 102                 # For execution time, if there is no result, its a fail.
 103                 for x in exectime:
 104                     if tp + x in d_old[t]:
 105                         if tp + x not in d_new[t]:
 106                             print(t + " *** REGRESSION (" + tp + x + ")\n")
 107
 108                     else:
 109                         if tp + x in d_new[t]:
 110                             print(t + " * NEW PASS (" + tp + x + ")\n")
 111
 112                 for x in comptime:
 113                     if exp + x in d_old[t]:
 114                         if exp + x not in d_new[t]:
 115                             print(t + " *** REGRESSION (" + exp + x + ")\n")
 116
 117                     else:
 118                         if exp + x in d_new[t]:
 119                             print(t + " * NEW PASS (" + exp + x + ")\n")
 120
 121         else:
 122             print(t + ": Removed from test-suite.\n")
 123
 124
 125 # Main
 126 if len(sys.argv) < 3:
 127     print("Usage:", sys.argv[0], "<old log> <new log>")
 128     sys.exit(-1)
 129
 130 d_old = parse(sys.argv[1])
 131 d_new = parse(sys.argv[2])
 132
 133
 134 diffResults(d_old, d_new)