travis/nasm-t.py

   1 #!/usr/bin/python3
   2
   3 import subprocess
   4 import argparse
   5 import difflib
   6 import filecmp
   7 import fnmatch
   8 import json
   9 import sys
  10 import re
  11 import os
  12
  13 fmtr_class = argparse.ArgumentDefaultsHelpFormatter
  14 parser = argparse.ArgumentParser(prog = 'nasm-t.py',
  15                                  formatter_class=fmtr_class)
  16
  17 parser.add_argument('-d', '--directory',
  18                     dest = 'dir', default = './travis/test',
  19                     help = 'Directory with tests')
  20
  21 parser.add_argument('--nasm',
  22                     dest = 'nasm', default = './nasm',
  23                     help = 'Nasm executable to use')
  24
  25 parser.add_argument('--hexdump',
  26                     dest = 'hexdump', default = '/usr/bin/hexdump',
  27                     help = 'Hexdump executable to use')
  28
  29 sp = parser.add_subparsers(dest = 'cmd')
  30 for cmd in ['run']:
  31     spp = sp.add_parser(cmd, help = 'Run test cases')
  32     spp.add_argument('-t', '--test',
  33                      dest = 'test',
  34                      help = 'Run the selected test only',
  35                      required = False)
  36
  37 for cmd in ['list']:
  38     spp = sp.add_parser(cmd, help = 'List test cases')
  39
  40 for cmd in ['update']:
  41     spp = sp.add_parser(cmd, help = 'Update test cases with new compiler')
  42     spp.add_argument('-t', '--test',
  43                      dest = 'test',
  44                      help = 'Update the selected test only',
  45                      required = False)
  46
  47 args = parser.parse_args()
  48
  49 if args.cmd == None:
  50     parser.print_help()
  51     sys.exit(1)
  52
  53 def read_stdfile(path):
  54     with open(path, "rb") as f:
  55         data = f.read().decode("utf-8").strip("\n")
  56         f.close()
  57         return data
  58
  59 #
  60 # Check if descriptor has mandatory fields
  61 def is_valid_desc(desc):
  62     if desc == None:
  63         return False
  64     if 'description' not in desc:
  65         return False
  66     if desc['description'] == "":
  67         return False
  68     return True
  69
  70 #
  71 # Expand ref/id in descriptors array
  72 def expand_templates(desc_array):
  73     desc_ids = { }
  74     for d in desc_array:
  75         if 'id' in d:
  76             desc_ids[d['id']] = d
  77     for i, d in enumerate(desc_array):
  78         if 'ref' in d and d['ref'] in desc_ids:
  79             ref = desc_ids[d['ref']]
  80             own = d.copy()
  81             desc_array[i] = ref.copy()
  82             for k, v in own.items():
  83                 desc_array[i][k] = v
  84             del desc_array[i]['id']
  85     return desc_array
  86
  87 def prepare_desc(desc, basedir, name, path):
  88     if not is_valid_desc(desc):
  89         return False
  90     #
  91     # Put private fields
  92     desc['_base-dir'] = basedir
  93     desc['_json-file'] = name
  94     desc['_json-path'] = path
  95     desc['_test-name'] = basedir + os.sep + name[:-5]
  96     #
  97     # If no target provided never update
  98     if 'target' not in desc:
  99         desc['target'] = []
 100         desc['update'] = 'false'
 101     #
 102     # Which code to expect when nasm finishes
 103     desc['_wait'] = 0
 104     if 'error' in desc:
 105         if desc['error'] == 'expected':
 106             desc['_wait'] = 1
 107     #
 108     # Walk over targets and generate match templates
 109     # if were not provided yet
 110     for d in desc['target']:
 111         if 'output' in d and not 'match' in d:
 112             d['match'] = d['output'] + ".t"
 113     return True
 114
 115 def read_json(path):
 116     desc = None
 117     try:
 118         with open(path, "rb") as f:
 119             try:
 120                 desc = json.loads(f.read().decode("utf-8").strip("\n"))
 121             except:
 122                 desc = None
 123             finally:
 124                 f.close()
 125     except:
 126         pass
 127     return desc
 128
 129 def read_desc(basedir, name):
 130     path = basedir + os.sep + name
 131     desc = read_json(path)
 132     desc_array = []
 133     if type(desc) == dict:
 134         if prepare_desc(desc, basedir, name, path) == True:
 135             desc_array += [desc]
 136     elif type(desc) == list:
 137         expand_templates(desc)
 138         for de in desc:
 139             if prepare_desc(de, basedir, name, path) == True:
 140                 desc_array += [de]
 141     return desc_array
 142
 143 def collect_test_desc_from_file(path):
 144     if not fnmatch.fnmatch(path, '*.json'):
 145         path += '.json'
 146     basedir = os.path.dirname(path)
 147     filename = os.path.basename(path)
 148     return read_desc(basedir, filename)
 149
 150 def collect_test_desc_from_dir(basedir):
 151     desc_array = []
 152     if os.path.isdir(basedir):
 153         for filename in os.listdir(basedir):
 154             if os.path.isdir(basedir + os.sep + filename):
 155                 desc_array += collect_test_desc_from_dir(basedir + os.sep + filename)
 156             elif fnmatch.fnmatch(filename, '*.json'):
 157                 desc = read_desc(basedir, filename)
 158                 if desc == None:
 159                     continue
 160                 desc_array += desc
 161         desc_array.sort(key=lambda x: x['_test-name'])
 162     return desc_array
 163
 164 if args.cmd == 'list':
 165     fmt_entry = '%-32s %s'
 166     desc_array = collect_test_desc_from_dir(args.dir)
 167     print(fmt_entry % ('Name', 'Description'))
 168     for desc in desc_array:
 169         print(fmt_entry % (desc['_test-name'], desc['description']))
 170
 171 def test_abort(test, message):
 172     print("\t%s: %s" % (test, message))
 173     print("=== Test %s ABORT ===" % (test))
 174     sys.exit(1)
 175     return False
 176
 177 def test_fail(test, message):
 178     print("\t%s: %s" % (test, message))
 179     print("=== Test %s FAIL ===" % (test))
 180     return False
 181
 182 def test_skip(test, message):
 183     print("\t%s: %s" % (test, message))
 184     print("=== Test %s SKIP ===" % (test))
 185     return True
 186
 187 def test_over(test):
 188     print("=== Test %s ERROR OVER ===" % (test))
 189     return True
 190
 191 def test_pass(test):
 192     print("=== Test %s PASS ===" % (test))
 193     return True
 194
 195 def test_updated(test):
 196     print("=== Test %s UPDATED ===" % (test))
 197     return True
 198
 199 def run_hexdump(path):
 200     p = subprocess.Popen([args.hexdump, "-C", path],
 201                          stdout = subprocess.PIPE,
 202                          close_fds = True)
 203     if p.wait() == 0:
 204         return p
 205     return None
 206
 207 def show_std(stdname, data):
 208     print("\t--- %s" % (stdname))
 209     for i in data.split("\n"):
 210         print("\t%s" % i)
 211     print("\t---")
 212
 213 def cmp_std(from_name, from_data, match_name, match_data):
 214     if from_data != match_data:
 215         print("\t--- %s" % (from_name))
 216         for i in from_data.split("\n"):
 217             print("\t%s" % i)
 218         print("\t--- %s" % (match_name))
 219         for i in match_data.split("\n"):
 220             print("\t%s" % i)
 221
 222         diff = difflib.unified_diff(from_data.split("\n"), match_data.split("\n"),
 223                                     fromfile = from_name, tofile = match_name)
 224         for i in diff:
 225             print("\t%s" % i.strip("\n"))
 226         print("\t---")
 227         return False
 228     return True
 229
 230 def show_diff(test, patha, pathb):
 231     pa = run_hexdump(patha)
 232     pb = run_hexdump(pathb)
 233     if pa == None or pb == None:
 234         return test_fail(test, "Can't create dumps")
 235     sa = pa.stdout.read().decode("utf-8").strip("\n")
 236     sb = pb.stdout.read().decode("utf-8").strip("\n")
 237     print("\t--- hexdump %s" % (patha))
 238     for i in sa.split("\n"):
 239         print("\t%s" % i)
 240     print("\t--- hexdump %s" % (pathb))
 241     for i in sb.split("\n"):
 242         print("\t%s" % i)
 243     pa.stdout.close()
 244     pb.stdout.close()
 245
 246     diff = difflib.unified_diff(sa.split("\n"), sb.split("\n"),
 247                                 fromfile = patha, tofile = pathb)
 248     for i in diff:
 249         print("\t%s" % i.strip("\n"))
 250     print("\t---")
 251     return True
 252
 253 def prepare_run_opts(desc):
 254     opts = []
 255
 256     if 'format' in desc:
 257         opts += ['-f', desc['format']]
 258     if 'option' in desc:
 259         opts += desc['option'].split(" ")
 260     for t in desc['target']:
 261         if 'output' in t:
 262             if 'option' in t:
 263                 opts += t['option'].split(" ") + [desc['_base-dir'] + os.sep + t['output']]
 264             else:
 265                 opts += ['-o', desc['_base-dir'] + os.sep + t['output']]
 266         if 'stdout' in t or 'stderr' in t:
 267             if 'option' in t:
 268                 opts += t['option'].split(" ")
 269     if 'source' in desc:
 270         opts += [desc['_base-dir'] + os.sep + desc['source']]
 271     return opts
 272
 273 def exec_nasm(desc):
 274     print("\tProcessing %s" % (desc['_test-name']))
 275     opts = [args.nasm] + prepare_run_opts(desc)
 276
 277     nasm_env = os.environ.copy()
 278     nasm_env['NASM_TEST_RUN'] = 'y'
 279
 280     desc_env = desc.get('environ')
 281     if desc_env:
 282         for i in desc_env:
 283             v = i.split('=')
 284             if len(v) == 2:
 285                 nasm_env[v[0]] = v[1]
 286             else:
 287                 nasm_env[v[0]] = None
 288
 289     print("\tExecuting %s" % (" ".join(opts)))
 290     pnasm = subprocess.Popen(opts,
 291                              stdout = subprocess.PIPE,
 292                              stderr = subprocess.PIPE,
 293                              close_fds = True,
 294                              env = nasm_env)
 295     if pnasm == None:
 296         test_fail(desc['_test-name'], "Unable to execute test")
 297         return None
 298
 299     stderr = pnasm.stderr.read(4194304).decode("utf-8").strip("\n")
 300     stdout = pnasm.stdout.read(4194304).decode("utf-8").strip("\n")
 301
 302     pnasm.stdout.close()
 303     pnasm.stderr.close()
 304
 305     wait_rc = pnasm.wait();
 306
 307     if desc['_wait'] != wait_rc:
 308         if stdout != "":
 309             show_std("stdout", stdout)
 310         if stderr != "":
 311             show_std("stderr", stderr)
 312         test_fail(desc['_test-name'],
 313                   "Unexpected ret code: " + str(wait_rc))
 314         return None, None, None
 315     return pnasm, stdout, stderr
 316
 317 def test_run(desc):
 318     print("=== Running %s ===" % (desc['_test-name']))
 319
 320     pnasm, stdout, stderr = exec_nasm(desc)
 321     if pnasm == None:
 322         return False
 323
 324     for t in desc['target']:
 325         if 'output' in t:
 326             output = desc['_base-dir'] + os.sep + t['output']
 327             match = desc['_base-dir'] + os.sep + t['match']
 328             if desc['_wait'] == 1:
 329                 continue
 330             print("\tComparing %s %s" % (output, match))
 331             if filecmp.cmp(match, output) == False:
 332                 show_diff(desc['_test-name'], match, output)
 333                 return test_fail(desc['_test-name'], match + " and " + output + " files are different")
 334         elif 'stdout' in t:
 335             print("\tComparing stdout")
 336             match = desc['_base-dir'] + os.sep + t['stdout']
 337             match_data = read_stdfile(match)
 338             if match_data == None:
 339                 return test_fail(test, "Can't read " + match)
 340             if cmp_std(match, match_data, 'stdout', stdout) == False:
 341                 return test_fail(desc['_test-name'], "Stdout mismatch")
 342             else:
 343                 stdout = ""
 344         elif 'stderr' in t:
 345             print("\tComparing stderr")
 346             match = desc['_base-dir'] + os.sep + t['stderr']
 347             match_data = read_stdfile(match)
 348             if match_data == None:
 349                 return test_fail(test, "Can't read " + match)
 350             if cmp_std(match, match_data, 'stderr', stderr) == False:
 351                 return test_fail(desc['_test-name'], "Stderr mismatch")
 352             else:
 353                 stderr = ""
 354
 355     if stdout != "":
 356         show_std("stdout", stdout)
 357         return test_fail(desc['_test-name'], "Stdout is not empty")
 358
 359     if stderr != "":
 360         show_std("stderr", stderr)
 361         return test_fail(desc['_test-name'], "Stderr is not empty")
 362
 363     return test_pass(desc['_test-name'])
 364
 365 #
 366 # Compile sources and generate new targets
 367 def test_update(desc):
 368     print("=== Updating %s ===" % (desc['_test-name']))
 369
 370     if 'update' in desc and desc['update'] == 'false':
 371         return test_skip(desc['_test-name'], "No output provided")
 372
 373     pnasm, stdout, stderr = exec_nasm(desc)
 374     if pnasm == None:
 375         return False
 376
 377     for t in desc['target']:
 378         if 'output' in t:
 379             output = desc['_base-dir'] + os.sep + t['output']
 380             match = desc['_base-dir'] + os.sep + t['match']
 381             print("\tMoving %s to %s" % (output, match))
 382             os.rename(output, match)
 383         if 'stdout' in t:
 384             match = desc['_base-dir'] + os.sep + t['stdout']
 385             print("\tMoving %s to %s" % ('stdout', match))
 386             with open(match, "wb") as f:
 387                 f.write(stdout.encode("utf-8"))
 388                 f.close()
 389         if 'stderr' in t:
 390             match = desc['_base-dir'] + os.sep + t['stderr']
 391             print("\tMoving %s to %s" % ('stderr', match))
 392             with open(match, "wb") as f:
 393                 f.write(stderr.encode("utf-8"))
 394                 f.close()
 395
 396     return test_updated(desc['_test-name'])
 397
 398 if args.cmd == 'run':
 399     desc_array = []
 400     if args.test == None:
 401         desc_array = collect_test_desc_from_dir(args.dir)
 402     else:
 403         desc_array = collect_test_desc_from_file(args.test)
 404         if len(desc_array) == 0:
 405             test_abort(args.test, "Can't obtain test descriptors")
 406
 407     for desc in desc_array:
 408         if test_run(desc) == False:
 409             if 'error' in desc and desc['error'] == 'over':
 410                 test_over(desc['_test-name'])
 411             else:
 412                 test_abort(desc['_test-name'], "Error detected")
 413
 414 if args.cmd == 'update':
 415     desc_array = []
 416     if args.test == None:
 417         desc_array = collect_test_desc_from_dir(args.dir)
 418     else:
 419         desc_array = collect_test_desc_from_file(args.test)
 420         if len(desc_array) == 0:
 421             test_abort(args.test, "Can't obtain a test descriptors")
 422
 423     for desc in desc_array:
 424         if test_update(desc) == False:
 425             if 'error' in desc and desc['error'] == 'over':
 426                 test_over(desc['_test-name'])
 427             else:
 428                 test_abort(desc['_test-name'], "Error detected")