utils/release/findRegressions-simple.py

   1 #!/usr/bin/env python
   2
   3 from __future__ import print_function
   4 import re, string, sys, os, time, math
   5
   6 DEBUG = 0
   7
   8 (tp, exp) = ('compile', 'exec')
   9
  10 def parse(file):
  11   f = open(file, 'r')
  12   d = f.read()
  13
  14   # Cleanup weird stuff
  15   d = re.sub(r',\d+:\d', '', d)
  16
  17   r = re.findall(r'TEST-(PASS|FAIL|RESULT.*?):\s+(.*?)\s+(.*?)\r*\n', d)
  18
  19   test = {}
  20   fname = ''
  21   for t in r:
  22     if DEBUG:
  23       print(t)
  24
  25     if t[0] == 'PASS' or t[0] == 'FAIL' :
  26       tmp = t[2].split('llvm-test/')
  27
  28       if DEBUG:
  29         print(tmp)
  30
  31       if len(tmp) == 2:
  32         fname = tmp[1].strip('\r\n')
  33       else:
  34         fname = tmp[0].strip('\r\n')
  35
  36       if fname not in test:
  37         test[fname] = {}
  38
  39       test[fname][t[1] + ' state'] = t[0]
  40       test[fname][t[1] + ' time'] = float('nan')
  41     else :
  42       try:
  43         n = t[0].split('RESULT-')[1]
  44
  45         if DEBUG:
  46           print("n == ", n);
  47
  48         if n == 'compile-success':
  49           test[fname]['compile time'] = float(t[2].split('program')[1].strip('\r\n'))
  50
  51         elif n == 'exec-success':
  52           test[fname]['exec time'] = float(t[2].split('program')[1].strip('\r\n'))
  53           if DEBUG:
  54             print(test[fname][string.replace(n, '-success', '')])
  55
  56         else :
  57           # print "ERROR!"
  58           sys.exit(1)
  59
  60       except:
  61           continue
  62
  63   return test
  64
  65 # Diff results and look for regressions.
  66 def diffResults(d_old, d_new):
  67   regressions = {}
  68   passes = {}
  69   removed = ''
  70
  71   for x in ['compile state', 'compile time', 'exec state', 'exec time']:
  72     regressions[x] = ''
  73     passes[x] = ''
  74
  75   for t in sorted(d_old.keys()) :
  76     if t in d_new:
  77
  78       # Check if the test passed or failed.
  79       for x in ['compile state', 'compile time', 'exec state', 'exec time']:
  80
  81         if x not in d_old[t] and x not in d_new[t]:
  82           continue
  83
  84         if x in d_old[t]:
  85           if x in d_new[t]:
  86
  87             if d_old[t][x] == 'PASS':
  88               if d_new[t][x] != 'PASS':
  89                 regressions[x] += t + "\n"
  90             else:
  91               if d_new[t][x] == 'PASS':
  92                 passes[x] += t + "\n"
  93
  94           else :
  95             regressions[x] += t + "\n"
  96
  97         if x == 'compile state' or x == 'exec state':
  98           continue
  99
 100         # For execution time, if there is no result it's a fail.
 101         if x not in d_old[t] and x not in d_new[t]:
 102           continue
 103         elif x not in d_new[t]:
 104           regressions[x] += t + "\n"
 105         elif x not in d_old[t]:
 106           passes[x] += t + "\n"
 107
 108         if math.isnan(d_old[t][x]) and math.isnan(d_new[t][x]):
 109           continue
 110
 111         elif math.isnan(d_old[t][x]) and not math.isnan(d_new[t][x]):
 112           passes[x] += t + "\n"
 113
 114         elif not math.isnan(d_old[t][x]) and math.isnan(d_new[t][x]):
 115           regressions[x] += t + ": NaN%\n"
 116
 117         if d_new[t][x] > d_old[t][x] and d_old[t][x] > 0.0 and \
 118               (d_new[t][x] - d_old[t][x]) / d_old[t][x] > .05:
 119           regressions[x] += t + ": " + "{0:.1f}".format(100 * (d_new[t][x] - d_old[t][x]) / d_old[t][x]) + "%\n"
 120
 121     else :
 122       removed += t + "\n"
 123
 124   if len(regressions['compile state']) != 0:
 125     print('REGRESSION: Compilation Failed')
 126     print(regressions['compile state'])
 127
 128   if len(regressions['exec state']) != 0:
 129     print('REGRESSION: Execution Failed')
 130     print(regressions['exec state'])
 131
 132   if len(regressions['compile time']) != 0:
 133     print('REGRESSION: Compilation Time')
 134     print(regressions['compile time'])
 135
 136   if len(regressions['exec time']) != 0:
 137     print('REGRESSION: Execution Time')
 138     print(regressions['exec time'])
 139
 140   if len(passes['compile state']) != 0:
 141     print('NEW PASSES: Compilation')
 142     print(passes['compile state'])
 143
 144   if len(passes['exec state']) != 0:
 145     print('NEW PASSES: Execution')
 146     print(passes['exec state'])
 147
 148   if len(removed) != 0:
 149     print('REMOVED TESTS')
 150     print(removed)
 151
 152 # Main
 153 if len(sys.argv) < 3 :
 154   print('Usage:', sys.argv[0], '<old log> <new log>')
 155   sys.exit(-1)
 156
 157 d_old = parse(sys.argv[1])
 158 d_new = parse(sys.argv[2])
 159
 160 diffResults(d_old, d_new)