utils/lit/TestRunner.py

   1 import os, signal, subprocess, sys
   2 import StringIO
   3
   4 import ShUtil
   5 import Test
   6 import Util
   7
   8 def executeCommand(command, cwd=None, env=None):
   9     p = subprocess.Popen(command, cwd=cwd,
  10                          stdin=subprocess.PIPE,
  11                          stdout=subprocess.PIPE,
  12                          stderr=subprocess.PIPE,
  13                          env=env)
  14     out,err = p.communicate()
  15     exitCode = p.wait()
  16
  17     # Detect Ctrl-C in subprocess.
  18     if exitCode == -signal.SIGINT:
  19         raise KeyboardInterrupt
  20
  21     return out, err, exitCode
  22
  23 def executeShCmd(cmd, cfg, cwd, results):
  24     if isinstance(cmd, ShUtil.Seq):
  25         if cmd.op == ';':
  26             res = executeShCmd(cmd.lhs, cfg, cwd, results)
  27             return executeShCmd(cmd.rhs, cfg, cwd, results)
  28
  29         if cmd.op == '&':
  30             raise NotImplementedError,"unsupported test command: '&'"
  31
  32         if cmd.op == '||':
  33             res = executeShCmd(cmd.lhs, cfg, cwd, results)
  34             if res != 0:
  35                 res = executeShCmd(cmd.rhs, cfg, cwd, results)
  36             return res
  37         if cmd.op == '&&':
  38             res = executeShCmd(cmd.lhs, cfg, cwd, results)
  39             if res is None:
  40                 return res
  41
  42             if res == 0:
  43                 res = executeShCmd(cmd.rhs, cfg, cwd, results)
  44             return res
  45
  46         raise ValueError,'Unknown shell command: %r' % cmd.op
  47
  48     assert isinstance(cmd, ShUtil.Pipeline)
  49     procs = []
  50     input = subprocess.PIPE
  51     for j in cmd.commands:
  52         redirects = [(0,), (1,), (2,)]
  53         for r in j.redirects:
  54             if r[0] == ('>',2):
  55                 redirects[2] = [r[1], 'w', None]
  56             elif r[0] == ('>&',2) and r[1] in '012':
  57                 redirects[2] = redirects[int(r[1])]
  58             elif r[0] == ('>&',) or r[0] == ('&>',):
  59                 redirects[1] = redirects[2] = [r[1], 'w', None]
  60             elif r[0] == ('>',):
  61                 redirects[1] = [r[1], 'w', None]
  62             elif r[0] == ('<',):
  63                 redirects[0] = [r[1], 'r', None]
  64             else:
  65                 raise NotImplementedError,"Unsupported redirect: %r" % (r,)
  66
  67         final_redirects = []
  68         for index,r in enumerate(redirects):
  69             if r == (0,):
  70                 result = input
  71             elif r == (1,):
  72                 if index == 0:
  73                     raise NotImplementedError,"Unsupported redirect for stdin"
  74                 elif index == 1:
  75                     result = subprocess.PIPE
  76                 else:
  77                     result = subprocess.STDOUT
  78             elif r == (2,):
  79                 if index != 2:
  80                     raise NotImplementedError,"Unsupported redirect on stdout"
  81                 result = subprocess.PIPE
  82             else:
  83                 if r[2] is None:
  84                     r[2] = open(r[0], r[1])
  85                 result = r[2]
  86             final_redirects.append(result)
  87
  88         stdin, stdout, stderr = final_redirects
  89
  90         # If stderr wants to come from stdout, but stdout isn't a pipe, then put
  91         # stderr on a pipe and treat it as stdout.
  92         if (stderr == subprocess.STDOUT and stdout != subprocess.PIPE):
  93             stderr = subprocess.PIPE
  94             stderrIsStdout = True
  95         else:
  96             stderrIsStdout = False
  97         procs.append(subprocess.Popen(j.args, cwd=cwd,
  98                                       stdin = stdin,
  99                                       stdout = stdout,
 100                                       stderr = stderr,
 101                                       env = cfg.environment,
 102                                       close_fds = True))
 103
 104         # Immediately close stdin for any process taking stdin from us.
 105         if stdin == subprocess.PIPE:
 106             procs[-1].stdin.close()
 107             procs[-1].stdin = None
 108
 109         # Update the current stdin source.
 110         if stdout == subprocess.PIPE:
 111             input = procs[-1].stdout
 112         elif stderrIsStdout:
 113             input = procs[-1].stderr
 114         else:
 115             input = subprocess.PIPE
 116
 117     # FIXME: There is a potential for deadlock here, when we have a pipe and
 118     # some process other than the last one ends up blocked on stderr.
 119     procData = [None] * len(procs)
 120     procData[-1] = procs[-1].communicate()
 121     for i in range(len(procs) - 1):
 122         if procs[i].stdout is not None:
 123             out = procs[i].stdout.read()
 124         else:
 125             out = ''
 126         if procs[i].stderr is not None:
 127             err = procs[i].stderr.read()
 128         else:
 129             err = ''
 130         procData[i] = (out,err)
 131
 132     exitCode = None
 133     for i,(out,err) in enumerate(procData):
 134         res = procs[i].wait()
 135         # Detect Ctrl-C in subprocess.
 136         if res == -signal.SIGINT:
 137             raise KeyboardInterrupt
 138
 139         results.append((cmd.commands[i], out, err, res))
 140         if cmd.pipe_err:
 141             # Python treats the exit code as a signed char.
 142             if res < 0:
 143                 exitCode = min(exitCode, res)
 144             else:
 145                 exitCode = max(exitCode, res)
 146         else:
 147             exitCode = res
 148
 149     if cmd.negate:
 150         exitCode = not exitCode
 151
 152     return exitCode
 153
 154 def executeScriptInternal(test, litConfig, tmpBase, commands, cwd):
 155     ln = ' &&\n'.join(commands)
 156     try:
 157         cmd = ShUtil.ShParser(ln, litConfig.isWindows).parse()
 158     except:
 159         return (Test.FAIL, "shell parser error on: %r" % ln)
 160
 161     results = []
 162     exitCode = executeShCmd(cmd, test.config, cwd, results)
 163
 164     out = err = ''
 165     for i,(cmd, cmd_out,cmd_err,res) in enumerate(results):
 166         out += 'Command %d: %s\n' % (i, ' '.join('"%s"' % s for s in cmd.args))
 167         out += 'Command %d Result: %r\n' % (i, res)
 168         out += 'Command %d Output:\n%s\n\n' % (i, cmd_out)
 169         out += 'Command %d Stderr:\n%s\n\n' % (i, cmd_err)
 170
 171     return out, err, exitCode
 172
 173 def executeTclScriptInternal(test, litConfig, tmpBase, commands, cwd):
 174     import TclUtil
 175     cmds = []
 176     for ln in commands:
 177         # Given the unfortunate way LLVM's test are written, the line gets
 178         # backslash substitution done twice.
 179         ln = TclUtil.TclLexer(ln).lex_unquoted(process_all = True)
 180
 181         try:
 182             tokens = list(TclUtil.TclLexer(ln).lex())
 183         except:
 184             return (Test.FAIL, "Tcl lexer error on: %r" % ln)
 185
 186         # Validate there are no control tokens.
 187         for t in tokens:
 188             if not isinstance(t, str):
 189                 return (Test.FAIL,
 190                         "Invalid test line: %r containing %r" % (ln, t))
 191
 192         try:
 193             cmds.append(TclUtil.TclExecCommand(tokens).parse_pipeline())
 194         except:
 195             return (Test.FAIL, "Tcl 'exec' parse error on: %r" % ln)
 196
 197     cmd = cmds[0]
 198     for c in cmds[1:]:
 199         cmd = ShUtil.Seq(cmd, '&&', c)
 200
 201     if litConfig.useTclAsSh:
 202         script = tmpBase + '.script'
 203
 204         # Write script file
 205         f = open(script,'w')
 206         print >>f, 'set -o pipefail'
 207         cmd.toShell(f, pipefail = True)
 208         f.close()
 209
 210         if 0:
 211             print >>sys.stdout, cmd
 212             print >>sys.stdout, open(script).read()
 213             print >>sys.stdout
 214             return '', '', 0
 215
 216         command = ['/bin/bash', script]
 217         out,err,exitCode = executeCommand(command, cwd=cwd,
 218                                           env=test.config.environment)
 219
 220         # Tcl commands fail on standard error output.
 221         if err:
 222             exitCode = 1
 223             out = 'Command has output on stderr!\n\n' + out
 224
 225         return out,err,exitCode
 226     else:
 227         results = []
 228         exitCode = executeShCmd(cmd, test.config, cwd, results)
 229
 230     out = err = ''
 231
 232     # Tcl commands fail on standard error output.
 233     if [True for _,_,err,res in results if err]:
 234         exitCode = 1
 235         out += 'Command has output on stderr!\n\n'
 236
 237     for i,(cmd, cmd_out, cmd_err, res) in enumerate(results):
 238         out += 'Command %d: %s\n' % (i, ' '.join('"%s"' % s for s in cmd.args))
 239         out += 'Command %d Result: %r\n' % (i, res)
 240         out += 'Command %d Output:\n%s\n\n' % (i, cmd_out)
 241         out += 'Command %d Stderr:\n%s\n\n' % (i, cmd_err)
 242
 243     return out, err, exitCode
 244
 245 def executeScript(test, litConfig, tmpBase, commands, cwd):
 246     script = tmpBase + '.script'
 247     if litConfig.isWindows:
 248         script += '.bat'
 249
 250     # Write script file
 251     f = open(script,'w')
 252     if litConfig.isWindows:
 253         f.write('\nif %ERRORLEVEL% NEQ 0 EXIT\n'.join(commands))
 254     else:
 255         f.write(' &&\n'.join(commands))
 256     f.write('\n')
 257     f.close()
 258
 259     if litConfig.isWindows:
 260         command = ['cmd','/c', script]
 261     else:
 262         command = ['/bin/sh', script]
 263         if litConfig.useValgrind:
 264             # FIXME: Running valgrind on sh is overkill. We probably could just
 265             # run on clang with no real loss.
 266             valgrindArgs = ['valgrind', '-q',
 267                             '--tool=memcheck', '--trace-children=yes',
 268                             '--error-exitcode=123']
 269             valgrindArgs.extend(litConfig.valgrindArgs)
 270
 271             command = valgrindArgs + command
 272
 273     return executeCommand(command, cwd=cwd, env=test.config.environment)
 274
 275 def parseIntegratedTestScript(test, xfailHasColon, requireAndAnd):
 276     """parseIntegratedTestScript - Scan an LLVM/Clang style integrated test
 277     script and extract the lines to 'RUN' as well as 'XFAIL' and 'XTARGET'
 278     information. The RUN lines also will have variable substitution performed.
 279     """
 280
 281     # Get the temporary location, this is always relative to the test suite
 282     # root, not test source root.
 283     #
 284     # FIXME: This should not be here?
 285     sourcepath = test.getSourcePath()
 286     execpath = test.getExecPath()
 287     execdir,execbase = os.path.split(execpath)
 288     tmpBase = os.path.join(execdir, 'Output', execbase)
 289
 290     # We use #_MARKER_# to hide %% while we do the other substitutions.
 291     substitutions = [('%%', '#_MARKER_#')]
 292     substitutions.extend(test.config.substitutions)
 293     substitutions.extend([('%s', sourcepath),
 294                           ('%S', os.path.dirname(sourcepath)),
 295                           ('%p', os.path.dirname(sourcepath)),
 296                           ('%t', tmpBase + '.tmp'),
 297                           ('#_MARKER_#', '%')])
 298
 299     # Collect the test lines from the script.
 300     script = []
 301     xfails = []
 302     xtargets = []
 303     for ln in open(sourcepath):
 304         if 'RUN:' in ln:
 305             # Isolate the command to run.
 306             index = ln.index('RUN:')
 307             ln = ln[index+4:]
 308
 309             # Trim trailing whitespace.
 310             ln = ln.rstrip()
 311
 312             # Collapse lines with trailing '\\'.
 313             if script and script[-1][-1] == '\\':
 314                 script[-1] = script[-1][:-1] + ln
 315             else:
 316                 script.append(ln)
 317         elif xfailHasColon and 'XFAIL:' in ln:
 318             items = ln[ln.index('XFAIL:') + 6:].split(',')
 319             xfails.extend([s.strip() for s in items])
 320         elif not xfailHasColon and 'XFAIL' in ln:
 321             items = ln[ln.index('XFAIL') + 5:].split(',')
 322             xfails.extend([s.strip() for s in items])
 323         elif 'XTARGET:' in ln:
 324             items = ln[ln.index('XTARGET:') + 8:].split(',')
 325             xtargets.extend([s.strip() for s in items])
 326         elif 'END.' in ln:
 327             # Check for END. lines.
 328             if ln[ln.index('END.'):].strip() == 'END.':
 329                 break
 330
 331     # Apply substitutions to the script.
 332     def processLine(ln):
 333         # Apply substitutions
 334         for a,b in substitutions:
 335             ln = ln.replace(a,b)
 336
 337         # Strip the trailing newline and any extra whitespace.
 338         return ln.strip()
 339     script = map(processLine, script)
 340
 341     # Verify the script contains a run line.
 342     if not script:
 343         return (Test.UNRESOLVED, "Test has no run line!")
 344
 345     if script[-1][-1] == '\\':
 346         return (Test.UNRESOLVED, "Test has unterminated run lines (with '\\')")
 347
 348     # Validate interior lines for '&&', a lovely historical artifact.
 349     if requireAndAnd:
 350         for i in range(len(script) - 1):
 351             ln = script[i]
 352
 353             if not ln.endswith('&&'):
 354                 return (Test.FAIL,
 355                         ("MISSING \'&&\': %s\n"  +
 356                          "FOLLOWED BY   : %s\n") % (ln, script[i + 1]))
 357
 358             # Strip off '&&'
 359             script[i] = ln[:-2]
 360
 361     return script,xfails,xtargets,tmpBase,execdir
 362
 363 def formatTestOutput(status, out, err, exitCode, script):
 364     output = StringIO.StringIO()
 365     print >>output, "Script:"
 366     print >>output, "--"
 367     print >>output, '\n'.join(script)
 368     print >>output, "--"
 369     print >>output, "Exit Code: %r" % exitCode
 370     print >>output, "Command Output (stdout):"
 371     print >>output, "--"
 372     output.write(out)
 373     print >>output, "--"
 374     print >>output, "Command Output (stderr):"
 375     print >>output, "--"
 376     output.write(err)
 377     print >>output, "--"
 378     return (status, output.getvalue())
 379
 380 def executeTclTest(test, litConfig):
 381     if test.config.unsupported:
 382         return (Test.UNSUPPORTED, 'Test is unsupported')
 383
 384     res = parseIntegratedTestScript(test, True, False)
 385     if len(res) == 2:
 386         return res
 387
 388     script, xfails, xtargets, tmpBase, execdir = res
 389
 390     if litConfig.noExecute:
 391         return (Test.PASS, '')
 392
 393     # Create the output directory if it does not already exist.
 394     Util.mkdir_p(os.path.dirname(tmpBase))
 395
 396     res = executeTclScriptInternal(test, litConfig, tmpBase, script, execdir)
 397     if len(res) == 2:
 398         return res
 399
 400     isXFail = False
 401     for item in xfails:
 402         if item == '*' or item in test.suite.config.target_triple:
 403             isXFail = True
 404             break
 405
 406     # If this is XFAIL, see if it is expected to pass on this target.
 407     if isXFail:
 408         for item in xtargets:
 409             if item == '*' or item in test.suite.config.target_triple:
 410                 isXFail = False
 411                 break
 412
 413     out,err,exitCode = res
 414     if isXFail:
 415         ok = exitCode != 0
 416         status = (Test.XPASS, Test.XFAIL)[ok]
 417     else:
 418         ok = exitCode == 0
 419         status = (Test.FAIL, Test.PASS)[ok]
 420
 421     if ok:
 422         return (status,'')
 423
 424     return formatTestOutput(status, out, err, exitCode, script)
 425
 426 def executeShTest(test, litConfig, useExternalSh, requireAndAnd):
 427     if test.config.unsupported:
 428         return (Test.UNSUPPORTED, 'Test is unsupported')
 429
 430     res = parseIntegratedTestScript(test, False, requireAndAnd)
 431     if len(res) == 2:
 432         return res
 433
 434     script, xfails, xtargets, tmpBase, execdir = res
 435
 436     if litConfig.noExecute:
 437         return (Test.PASS, '')
 438
 439     # Create the output directory if it does not already exist.
 440     Util.mkdir_p(os.path.dirname(tmpBase))
 441
 442     if useExternalSh:
 443         res = executeScript(test, litConfig, tmpBase, script, execdir)
 444     else:
 445         res = executeScriptInternal(test, litConfig, tmpBase, script, execdir)
 446     if len(res) == 2:
 447         return res
 448
 449     out,err,exitCode = res
 450     if xfails:
 451         ok = exitCode != 0
 452         status = (Test.XPASS, Test.XFAIL)[ok]
 453     else:
 454         ok = exitCode == 0
 455         status = (Test.FAIL, Test.PASS)[ok]
 456
 457     if ok:
 458         return (status,'')
 459
 460     return formatTestOutput(status, out, err, exitCode, script)