utils/lit/TestRunner.py

   1 import os, signal, subprocess, sys
   2 import StringIO
   3
   4 import ShUtil
   5 import Test
   6 import Util
   7
   8 def executeCommand(command, cwd=None, env=None):
   9     p = subprocess.Popen(command, cwd=cwd,
  10                          stdin=subprocess.PIPE,
  11                          stdout=subprocess.PIPE,
  12                          stderr=subprocess.PIPE,
  13                          env=env)
  14     out,err = p.communicate()
  15     exitCode = p.wait()
  16
  17     # Detect Ctrl-C in subprocess.
  18     if exitCode == -signal.SIGINT:
  19         raise KeyboardInterrupt
  20
  21     return out, err, exitCode
  22
  23 def executeShCmd(cmd, cfg, cwd, results):
  24     if isinstance(cmd, ShUtil.Seq):
  25         if cmd.op == ';':
  26             res = executeShCmd(cmd.lhs, cfg, cwd, results)
  27             return executeShCmd(cmd.rhs, cfg, cwd, results)
  28
  29         if cmd.op == '&':
  30             raise NotImplementedError,"unsupported test command: '&'"
  31
  32         if cmd.op == '||':
  33             res = executeShCmd(cmd.lhs, cfg, cwd, results)
  34             if res != 0:
  35                 res = executeShCmd(cmd.rhs, cfg, cwd, results)
  36             return res
  37         if cmd.op == '&&':
  38             res = executeShCmd(cmd.lhs, cfg, cwd, results)
  39             if res is None:
  40                 return res
  41
  42             if res == 0:
  43                 res = executeShCmd(cmd.rhs, cfg, cwd, results)
  44             return res
  45
  46         raise ValueError,'Unknown shell command: %r' % cmd.op
  47
  48     assert isinstance(cmd, ShUtil.Pipeline)
  49     procs = []
  50     input = subprocess.PIPE
  51     for j in cmd.commands:
  52         redirects = [(0,), (1,), (2,)]
  53         for r in j.redirects:
  54             if r[0] == ('>',2):
  55                 redirects[2] = [r[1], 'w', None]
  56             elif r[0] == ('>&',2) and r[1] in '012':
  57                 redirects[2] = redirects[int(r[1])]
  58             elif r[0] == ('>&',) or r[0] == ('&>',):
  59                 redirects[1] = redirects[2] = [r[1], 'w', None]
  60             elif r[0] == ('>',):
  61                 redirects[1] = [r[1], 'w', None]
  62             elif r[0] == ('<',):
  63                 redirects[0] = [r[1], 'r', None]
  64             else:
  65                 raise NotImplementedError,"Unsupported redirect: %r" % (r,)
  66
  67         final_redirects = []
  68         for index,r in enumerate(redirects):
  69             if r == (0,):
  70                 result = input
  71             elif r == (1,):
  72                 if index == 0:
  73                     raise NotImplementedError,"Unsupported redirect for stdin"
  74                 elif index == 1:
  75                     result = subprocess.PIPE
  76                 else:
  77                     result = subprocess.STDOUT
  78             elif r == (2,):
  79                 if index != 2:
  80                     raise NotImplementedError,"Unsupported redirect on stdout"
  81                 result = subprocess.PIPE
  82             else:
  83                 if r[2] is None:
  84                     r[2] = open(r[0], r[1])
  85                 result = r[2]
  86             final_redirects.append(result)
  87
  88         stdin, stdout, stderr = final_redirects
  89
  90         # If stderr wants to come from stdout, but stdout isn't a pipe, then put
  91         # stderr on a pipe and treat it as stdout.
  92         if (stderr == subprocess.STDOUT and stdout != subprocess.PIPE):
  93             stderr = subprocess.PIPE
  94             stderrIsStdout = True
  95         else:
  96             stderrIsStdout = False
  97         procs.append(subprocess.Popen(j.args, cwd=cwd,
  98                                       stdin = stdin,
  99                                       stdout = stdout,
 100                                       stderr = stderr,
 101                                       env = cfg.environment,
 102                                       close_fds = True))
 103
 104         # Immediately close stdin for any process taking stdin from us.
 105         if stdin == subprocess.PIPE:
 106             procs[-1].stdin.close()
 107             procs[-1].stdin = None
 108
 109         # Update the current stdin source.
 110         if stdout == subprocess.PIPE:
 111             input = procs[-1].stdout
 112         elif stderrIsStdout:
 113             input = procs[-1].stderr
 114         else:
 115             input = subprocess.PIPE
 116
 117     # FIXME: There is a potential for deadlock here, when we have a pipe and
 118     # some process other than the last one ends up blocked on stderr.
 119     procData = [None] * len(procs)
 120     procData[-1] = procs[-1].communicate()
 121     for i in range(len(procs) - 1):
 122         if procs[i].stdout is not None:
 123             out = procs[i].stdout.read()
 124         else:
 125             out = ''
 126         if procs[i].stderr is not None:
 127             err = procs[i].stderr.read()
 128         else:
 129             err = ''
 130         procData[i] = (out,err)
 131
 132     exitCode = None
 133     for i,(out,err) in enumerate(procData):
 134         res = procs[i].wait()
 135         # Detect Ctrl-C in subprocess.
 136         if res == -signal.SIGINT:
 137             raise KeyboardInterrupt
 138
 139         results.append((cmd.commands[i], out, err, res))
 140         if cmd.pipe_err:
 141             # Python treats the exit code as a signed char.
 142             if res < 0:
 143                 exitCode = min(exitCode, res)
 144             else:
 145                 exitCode = max(exitCode, res)
 146         else:
 147             exitCode = res
 148
 149     if cmd.negate:
 150         exitCode = not exitCode
 151
 152     return exitCode
 153
 154 def executeScriptInternal(test, litConfig, tmpBase, commands, cwd):
 155     ln = ' &&\n'.join(commands)
 156     try:
 157         cmd = ShUtil.ShParser(ln, litConfig.isWindows).parse()
 158     except:
 159         return (Test.FAIL, "shell parser error on: %r" % ln)
 160
 161     results = []
 162     exitCode = executeShCmd(cmd, test.config, cwd, results)
 163
 164     out = err = ''
 165     for i,(cmd, cmd_out,cmd_err,res) in enumerate(results):
 166         out += 'Command %d: %s\n' % (i, ' '.join('"%s"' % s for s in cmd.args))
 167         out += 'Command %d Result: %r\n' % (i, res)
 168         out += 'Command %d Output:\n%s\n\n' % (i, cmd_out)
 169         out += 'Command %d Stderr:\n%s\n\n' % (i, cmd_err)
 170
 171     return out, err, exitCode
 172
 173 def executeTclScriptInternal(test, litConfig, tmpBase, commands, cwd):
 174     import TclUtil
 175     cmds = []
 176     for ln in commands:
 177         # Given the unfortunate way LLVM's test are written, the line gets
 178         # backslash substitution done twice.
 179         ln = TclUtil.TclLexer(ln).lex_unquoted(process_all = True)
 180
 181         try:
 182             tokens = list(TclUtil.TclLexer(ln).lex())
 183         except:
 184             return (Test.FAIL, "Tcl lexer error on: %r" % ln)
 185
 186         # Validate there are no control tokens.
 187         for t in tokens:
 188             if not isinstance(t, str):
 189                 return (Test.FAIL,
 190                         "Invalid test line: %r containing %r" % (ln, t))
 191
 192         try:
 193             cmds.append(TclUtil.TclExecCommand(tokens).parse_pipeline())
 194         except:
 195             return (Test.FAIL, "Tcl 'exec' parse error on: %r" % ln)
 196
 197     cmd = cmds[0]
 198     for c in cmds[1:]:
 199         cmd = ShUtil.Seq(cmd, '&&', c)
 200
 201     if litConfig.useTclAsSh:
 202         script = tmpBase + '.script'
 203
 204         # Write script file
 205         f = open(script,'w')
 206         print >>f, 'set -o pipefail'
 207         cmd.toShell(f, pipefail = True)
 208         f.close()
 209
 210         if 0:
 211             print >>sys.stdout, cmd
 212             print >>sys.stdout, open(script).read()
 213             print >>sys.stdout
 214             return '', '', 0
 215
 216         command = ['/bin/bash', script]
 217         out,err,exitCode = executeCommand(command, cwd=cwd,
 218                                           env=test.config.environment)
 219
 220         # Tcl commands fail on standard error output.
 221         if err:
 222             exitCode = 1
 223             out = 'Command has output on stderr!\n\n' + out
 224
 225         return out,err,exitCode
 226     else:
 227         results = []
 228         exitCode = executeShCmd(cmd, test.config, cwd, results)
 229
 230     out = err = ''
 231
 232     # Tcl commands fail on standard error output.
 233     if [True for _,_,err,res in results if err]:
 234         exitCode = 1
 235         out += 'Command has output on stderr!\n\n'
 236
 237     for i,(cmd, cmd_out, cmd_err, res) in enumerate(results):
 238         out += 'Command %d: %s\n' % (i, ' '.join('"%s"' % s for s in cmd.args))
 239         out += 'Command %d Result: %r\n' % (i, res)
 240         out += 'Command %d Output:\n%s\n\n' % (i, cmd_out)
 241         out += 'Command %d Stderr:\n%s\n\n' % (i, cmd_err)
 242
 243     return out, err, exitCode
 244
 245 def executeScript(test, litConfig, tmpBase, commands, cwd):
 246     script = tmpBase + '.script'
 247     if litConfig.isWindows:
 248         script += '.bat'
 249
 250     # Write script file
 251     f = open(script,'w')
 252     if litConfig.isWindows:
 253         f.write('\nif %ERRORLEVEL% NEQ 0 EXIT\n'.join(commands))
 254     else:
 255         f.write(' &&\n'.join(commands))
 256     f.write('\n')
 257     f.close()
 258
 259     if litConfig.isWindows:
 260         command = ['cmd','/c', script]
 261     else:
 262         command = ['/bin/sh', script]
 263         if litConfig.useValgrind:
 264             # FIXME: Running valgrind on sh is overkill. We probably could just
 265             # run on clang with no real loss.
 266             valgrindArgs = ['valgrind', '-q',
 267                             '--tool=memcheck', '--trace-children=yes',
 268                             '--error-exitcode=123']
 269             valgrindArgs.extend(litConfig.valgrindArgs)
 270
 271             command = valgrindArgs + command
 272
 273     return executeCommand(command, cwd=cwd, env=test.config.environment)
 274
 275 def parseIntegratedTestScript(test, xfailHasColon, requireAndAnd):
 276     """parseIntegratedTestScript - Scan an LLVM/Clang style integrated test
 277     script and extract the lines to 'RUN' as well as 'XFAIL' and 'XTARGET'
 278     information. The RUN lines also will have variable substitution performed.
 279     """
 280
 281     # Get the temporary location, this is always relative to the test suite
 282     # root, not test source root.
 283     #
 284     # FIXME: This should not be here?
 285     sourcepath = test.getSourcePath()
 286     execpath = test.getExecPath()
 287     execdir,execbase = os.path.split(execpath)
 288     tmpBase = os.path.join(execdir, 'Output', execbase)
 289
 290     # We use #_MARKER_# to hide %% while we do the other substitutions.
 291     substitutions = [('%%', '#_MARKER_#')]
 292     substitutions.extend(test.config.substitutions)
 293     substitutions.extend([('%s', sourcepath),
 294                           ('%S', os.path.dirname(sourcepath)),
 295                           ('%p', os.path.dirname(sourcepath)),
 296                           ('%t', tmpBase + '.tmp'),
 297                           # FIXME: Remove this once we kill DejaGNU.
 298                           ('%abs_tmp', tmpBase + '.tmp'),
 299                           ('#_MARKER_#', '%')])
 300
 301     # Collect the test lines from the script.
 302     script = []
 303     xfails = []
 304     xtargets = []
 305     for ln in open(sourcepath):
 306         if 'RUN:' in ln:
 307             # Isolate the command to run.
 308             index = ln.index('RUN:')
 309             ln = ln[index+4:]
 310
 311             # Trim trailing whitespace.
 312             ln = ln.rstrip()
 313
 314             # Collapse lines with trailing '\\'.
 315             if script and script[-1][-1] == '\\':
 316                 script[-1] = script[-1][:-1] + ln
 317             else:
 318                 script.append(ln)
 319         elif xfailHasColon and 'XFAIL:' in ln:
 320             items = ln[ln.index('XFAIL:') + 6:].split(',')
 321             xfails.extend([s.strip() for s in items])
 322         elif not xfailHasColon and 'XFAIL' in ln:
 323             items = ln[ln.index('XFAIL') + 5:].split(',')
 324             xfails.extend([s.strip() for s in items])
 325         elif 'XTARGET:' in ln:
 326             items = ln[ln.index('XTARGET:') + 8:].split(',')
 327             xtargets.extend([s.strip() for s in items])
 328         elif 'END.' in ln:
 329             # Check for END. lines.
 330             if ln[ln.index('END.'):].strip() == 'END.':
 331                 break
 332
 333     # Apply substitutions to the script.
 334     def processLine(ln):
 335         # Apply substitutions
 336         for a,b in substitutions:
 337             ln = ln.replace(a,b)
 338
 339         # Strip the trailing newline and any extra whitespace.
 340         return ln.strip()
 341     script = map(processLine, script)
 342
 343     # Verify the script contains a run line.
 344     if not script:
 345         return (Test.UNRESOLVED, "Test has no run line!")
 346
 347     if script[-1][-1] == '\\':
 348         return (Test.UNRESOLVED, "Test has unterminated run lines (with '\\')")
 349
 350     # Validate interior lines for '&&', a lovely historical artifact.
 351     if requireAndAnd:
 352         for i in range(len(script) - 1):
 353             ln = script[i]
 354
 355             if not ln.endswith('&&'):
 356                 return (Test.FAIL,
 357                         ("MISSING \'&&\': %s\n"  +
 358                          "FOLLOWED BY   : %s\n") % (ln, script[i + 1]))
 359
 360             # Strip off '&&'
 361             script[i] = ln[:-2]
 362
 363     return script,xfails,xtargets,tmpBase,execdir
 364
 365 def formatTestOutput(status, out, err, exitCode, script):
 366     output = StringIO.StringIO()
 367     print >>output, "Script:"
 368     print >>output, "--"
 369     print >>output, '\n'.join(script)
 370     print >>output, "--"
 371     print >>output, "Exit Code: %r" % exitCode
 372     print >>output, "Command Output (stdout):"
 373     print >>output, "--"
 374     output.write(out)
 375     print >>output, "--"
 376     print >>output, "Command Output (stderr):"
 377     print >>output, "--"
 378     output.write(err)
 379     print >>output, "--"
 380     return (status, output.getvalue())
 381
 382 def executeTclTest(test, litConfig):
 383     if test.config.unsupported:
 384         return (Test.UNSUPPORTED, 'Test is unsupported')
 385
 386     res = parseIntegratedTestScript(test, True, False)
 387     if len(res) == 2:
 388         return res
 389
 390     script, xfails, xtargets, tmpBase, execdir = res
 391
 392     if litConfig.noExecute:
 393         return (Test.PASS, '')
 394
 395     # Create the output directory if it does not already exist.
 396     Util.mkdir_p(os.path.dirname(tmpBase))
 397
 398     res = executeTclScriptInternal(test, litConfig, tmpBase, script, execdir)
 399     if len(res) == 2:
 400         return res
 401
 402     isXFail = False
 403     for item in xfails:
 404         if item == '*' or item in test.suite.config.target_triple:
 405             isXFail = True
 406             break
 407
 408     # If this is XFAIL, see if it is expected to pass on this target.
 409     if isXFail:
 410         for item in xtargets:
 411             if item == '*' or item in test.suite.config.target_triple:
 412                 isXFail = False
 413                 break
 414
 415     out,err,exitCode = res
 416     if isXFail:
 417         ok = exitCode != 0
 418         status = (Test.XPASS, Test.XFAIL)[ok]
 419     else:
 420         ok = exitCode == 0
 421         status = (Test.FAIL, Test.PASS)[ok]
 422
 423     if ok:
 424         return (status,'')
 425
 426     return formatTestOutput(status, out, err, exitCode, script)
 427
 428 def executeShTest(test, litConfig, useExternalSh, requireAndAnd):
 429     if test.config.unsupported:
 430         return (Test.UNSUPPORTED, 'Test is unsupported')
 431
 432     res = parseIntegratedTestScript(test, False, requireAndAnd)
 433     if len(res) == 2:
 434         return res
 435
 436     script, xfails, xtargets, tmpBase, execdir = res
 437
 438     if litConfig.noExecute:
 439         return (Test.PASS, '')
 440
 441     # Create the output directory if it does not already exist.
 442     Util.mkdir_p(os.path.dirname(tmpBase))
 443
 444     if useExternalSh:
 445         res = executeScript(test, litConfig, tmpBase, script, execdir)
 446     else:
 447         res = executeScriptInternal(test, litConfig, tmpBase, script, execdir)
 448     if len(res) == 2:
 449         return res
 450
 451     out,err,exitCode = res
 452     if xfails:
 453         ok = exitCode != 0
 454         status = (Test.XPASS, Test.XFAIL)[ok]
 455     else:
 456         ok = exitCode == 0
 457         status = (Test.FAIL, Test.PASS)[ok]
 458
 459     if ok:
 460         return (status,'')
 461
 462     return formatTestOutput(status, out, err, exitCode, script)