utils/release/findRegressions-nightly.py

   1 #!/usr/bin/env python
   2 from __future__ import print_function
   3
   4 import re, string, sys, os, time
   5
   6 DEBUG = 0
   7 testDirName = 'llvm-test'
   8 test      = ['compile', 'llc', 'jit', 'cbe']
   9 exectime     = ['llc-time', 'jit-time', 'cbe-time',]
  10 comptime     = ['llc', 'jit-comptime', 'compile']
  11
  12 (tp, exp) = ('compileTime_', 'executeTime_')
  13
  14 def parse(file):
  15   f=open(file, 'r')
  16   d = f.read()
  17
  18   #Cleanup weird stuff
  19   d = re.sub(r',\d+:\d','', d)
  20
  21   r = re.findall(r'TEST-(PASS|FAIL|RESULT.*?):\s+(.*?)\s+(.*?)\r*\n', d)
  22
  23   test = {}
  24   fname = ''
  25   for t in r:
  26     if DEBUG:
  27       print(t)
  28     if t[0] == 'PASS' or t[0] == 'FAIL' :
  29       tmp = t[2].split(testDirName)
  30
  31       if DEBUG:
  32         print(tmp)
  33
  34       if len(tmp) == 2:
  35         fname = tmp[1].strip('\r\n')
  36       else:
  37         fname = tmp[0].strip('\r\n')
  38
  39       if fname not in test :
  40         test[fname] = {}
  41
  42       for k in test:
  43         test[fname][k] = 'NA'
  44         test[fname][t[1]] = t[0]
  45         if DEBUG:
  46           print(test[fname][t[1]])
  47     else :
  48       try:
  49         n = t[0].split('RESULT-')[1]
  50
  51         if DEBUG:
  52           print(n);
  53
  54         if n == 'llc' or n == 'jit-comptime' or n == 'compile':
  55           test[fname][tp + n] = float(t[2].split(' ')[2])
  56           if DEBUG:
  57             print(test[fname][tp + n])
  58
  59         elif n.endswith('-time') :
  60             test[fname][exp + n] = float(t[2].strip('\r\n'))
  61             if DEBUG:
  62               print(test[fname][exp + n])
  63
  64         else :
  65           print("ERROR!")
  66           sys.exit(1)
  67
  68       except:
  69           continue
  70
  71   return test
  72
  73 # Diff results and look for regressions.
  74 def diffResults(d_old, d_new):
  75
  76   for t in sorted(d_old.keys()) :
  77     if DEBUG:
  78       print(t)
  79
  80     if t in d_new :
  81
  82       # Check if the test passed or failed.
  83       for x in test:
  84         if x in d_old[t]:
  85           if x in d_new[t]:
  86             if d_old[t][x] == 'PASS':
  87               if d_new[t][x] != 'PASS':
  88                 print(t + " *** REGRESSION (" + x + ")\n")
  89             else:
  90               if d_new[t][x] == 'PASS':
  91                 print(t + " * NEW PASS (" + x + ")\n")
  92
  93           else :
  94             print(t + "*** REGRESSION (" + x + ")\n")
  95
  96         # For execution time, if there is no result, its a fail.
  97         for x in exectime:
  98           if tp + x in d_old[t]:
  99             if tp + x not in d_new[t]:
 100               print(t + " *** REGRESSION (" + tp + x + ")\n")
 101
 102           else :
 103             if tp + x in d_new[t]:
 104               print(t + " * NEW PASS (" + tp + x + ")\n")
 105
 106
 107         for x in comptime:
 108           if exp + x in d_old[t]:
 109             if exp + x not in d_new[t]:
 110               print(t + " *** REGRESSION (" + exp + x + ")\n")
 111
 112           else :
 113             if exp + x in d_new[t]:
 114               print(t + " * NEW PASS (" + exp + x + ")\n")
 115
 116     else :
 117       print(t + ": Removed from test-suite.\n")
 118
 119
 120 #Main
 121 if len(sys.argv) < 3 :
 122     print('Usage:', sys.argv[0], \
 123           '<old log> <new log>')
 124     sys.exit(-1)
 125
 126 d_old = parse(sys.argv[1])
 127 d_new = parse(sys.argv[2])
 128
 129
 130 diffResults(d_old, d_new)
 131
 132