sdcc/support/regression/collate-results.py

   1 from __future__ import print_function
   2
   3 import sys, re, io
   4 #import string
   5
   6 """Simple script that scans all of the test suite results text fed in
   7 through stdin and summarises the total number of failures, test
   8 points, and test cases."""
   9
  10 # Function to parse numbers
  11 def number_from_string(str_value):
  12   if str_value.find("x") != -1:
  13     return int(str_value.replace("0x", "").replace("x", ""), 16);
  14   else:
  15     return int(str_value)
  16
  17 # Read in everything
  18 if sys.version_info[0]<3:
  19     safe_stdin = sys.stdin
  20 else:
  21     safe_stdin = io.TextIOWrapper(sys.stdin.buffer, encoding="latin-1")
  22 lines = safe_stdin.readlines()
  23
  24 # Init the running totals
  25 failures = 0
  26 cases = 0
  27 tests = 0
  28 bytes = 0
  29 ticks = 0
  30 invalid = 0
  31 stack_overflow = 0
  32 halt = 0
  33 unmatch = 0
  34 flag = 0
  35 messagelog = []
  36 exlist = ["bug663539"]
  37
  38 # hack for valdiag
  39 name = ""
  40
  41 for line in lines:
  42
  43     m = re.match(r'^Simulation started,', line)
  44     if (m):
  45         flag = 0
  46         name = ""
  47
  48     # --- Running: gen/ucz80/longor/longor
  49     m = re.match(r'^--- Running: (.*)$', line)
  50     if (m):
  51         #take the name only if not a whitespace, this happens if simulator stops when calling to print the name (stack overflow)
  52         stripped_name = m.group(1).strip()
  53         if stripped_name != "":
  54           name = stripped_name
  55
  56     # in case the test program crashes before the "--- Running" message
  57     m = re.match(r'^[0-9]+ words read from (.*).ihx', line)
  58     if (m):
  59         name = m.group(1)
  60
  61     # '--- Summary: f/t/c: ...', where f = # failures, t = # test points,
  62     # c = # test cases.
  63     if (re.search(r'^--- Summary:', line)):
  64         try:
  65             if line.count(':') == 1:
  66               (summary, data) = re.split(r':', line)
  67             else:
  68               (summary, data, rest) = re.split(r':', line)
  69
  70             (nfailures, ntests, ncases) = re.split(r'/', data)
  71             failures = failures + number_from_string(nfailures)
  72             tests = tests + number_from_string(ntests)
  73             cases = cases + number_from_string(ncases)
  74         except ValueError:
  75             print("Parsing error at ", name)
  76             print("Bad summary line: ", line)
  77             nfailures = '1'
  78         if (number_from_string(nfailures)):
  79             messagelog.append("Failure: %s" % name)
  80         flag = 1
  81
  82     # '--- Simulator: b/t: ...', where b = # bytes, t = # ticks
  83     if (re.search(r'^--- Simulator:', line)):
  84         try:
  85             (simulator, data, rest) = re.split(r':', line)
  86             (nbytes, nticks) = re.split(r'/', data)
  87         except ValueError:
  88             print("Bad simulator line", line)
  89         else:
  90             bytes = bytes + int(nbytes)
  91             ticks = ticks + int(nticks)
  92         if (flag != 1):
  93             for e in exlist:
  94                 if (e in name):
  95                     flag = 2
  96         if (flag == 0):
  97             unmatch += 1
  98             messagelog.append("abnormal stop: %s" % name)
  99         flag = 0
 100
 101     # Stop at 0x000228: (106) Invalid instruction 0x00fd
 102     if (re.search(r'Invalid instruction', line) or re.search(r'unknown instruction', line)):
 103         invalid += 1
 104         messagelog.append("Invalid instruction: %s" % name)
 105
 106     # Stop at 0xXXXXXX: (103) Stack overflow
 107     if (re.search(r'Stack overflow', line)):
 108         stack_overflow += 1
 109         messagelog.append("Stack overflow: %s" % name)
 110
 111     # HALT instruction
 112     if (re.search(r'HALT instruction', line) or re.search(r'Halt instruction', line) or re.search(r'halt instruction', line)):
 113         halt += 1
 114         messagelog.append("HALT instruction: %s" % name)
 115
 116     # --- FAIL: "timeout, simulation killed" in xx/xx/testfile.c
 117     m = re.search(r'simulation killed',line)
 118     if (m):
 119       name = line.split()[-1]
 120       name = '.'.join(name.split('.')[0:-1])
 121
 122
 123 if (len(sys.argv) > 1):
 124     print("Summary for '%s':" % sys.argv[1], end=' ')
 125 if (unmatch > 0):
 126     print("%d abnormal stops (" % unmatch, end=' ')
 127     if (invalid > 0):
 128         print("%d invalid instructions," % invalid, end=' ')
 129     if (stack_overflow > 0):
 130         print("%d Stack overflows," % stack_overflow, end=' ')
 131     if (halt > 0):
 132         print("%d HALT instructions," % halt, end=' ')
 133     print("),", end=' ')
 134 print("%.0f failures, %.0f tests, %.0f test cases, %.0f bytes, %.0f ticks" % (failures, tests, cases, bytes, ticks))
 135 for msg in messagelog:
 136   print("  ",msg)
 137 print()