utils/lit/lit.py

   1 #!/usr/bin/env python
   2
   3 """
   4 lit - LLVM Integrated Tester.
   5
   6 See lit.pod for more information.
   7 """
   8
   9 import math, os, platform, random, re, sys, time, threading, traceback
  10
  11 import ProgressBar
  12 import TestRunner
  13 import Util
  14
  15 from TestingConfig import TestingConfig
  16 import LitConfig
  17 import Test
  18
  19 # FIXME: Rename to 'config.lit', 'site.lit', and 'local.lit' ?
  20 kConfigName = 'lit.cfg'
  21 kSiteConfigName = 'lit.site.cfg'
  22 kLocalConfigName = 'lit.local.cfg'
  23
  24 class TestingProgressDisplay:
  25     def __init__(self, opts, numTests, progressBar=None):
  26         self.opts = opts
  27         self.numTests = numTests
  28         self.current = None
  29         self.lock = threading.Lock()
  30         self.progressBar = progressBar
  31         self.completed = 0
  32
  33     def update(self, test):
  34         # Avoid locking overhead in quiet mode
  35         if self.opts.quiet and not test.result.isFailure:
  36             self.completed += 1
  37             return
  38
  39         # Output lock.
  40         self.lock.acquire()
  41         try:
  42             self.handleUpdate(test)
  43         finally:
  44             self.lock.release()
  45
  46     def finish(self):
  47         if self.progressBar:
  48             self.progressBar.clear()
  49         elif self.opts.quiet:
  50             pass
  51         elif self.opts.succinct:
  52             sys.stdout.write('\n')
  53
  54     def handleUpdate(self, test):
  55         self.completed += 1
  56         if self.progressBar:
  57             self.progressBar.update(float(self.completed)/self.numTests,
  58                                     test.getFullName())
  59
  60         if self.opts.succinct and not test.result.isFailure:
  61             return
  62
  63         if self.progressBar:
  64             self.progressBar.clear()
  65
  66         print '%s: %s (%d of %d)' % (test.result.name, test.getFullName(),
  67                                      self.completed, self.numTests)
  68
  69         if test.result.isFailure and self.opts.showOutput:
  70             print "%s TEST '%s' FAILED %s" % ('*'*20, test.getFullName(),
  71                                               '*'*20)
  72             print test.output
  73             print "*" * 20
  74
  75         sys.stdout.flush()
  76
  77 class TestProvider:
  78     def __init__(self, tests, maxTime):
  79         self.maxTime = maxTime
  80         self.iter = iter(tests)
  81         self.lock = threading.Lock()
  82         self.startTime = time.time()
  83
  84     def get(self):
  85         # Check if we have run out of time.
  86         if self.maxTime is not None:
  87             if time.time() - self.startTime > self.maxTime:
  88                 return None
  89
  90         # Otherwise take the next test.
  91         self.lock.acquire()
  92         try:
  93             item = self.iter.next()
  94         except StopIteration:
  95             item = None
  96         self.lock.release()
  97         return item
  98
  99 class Tester(threading.Thread):
 100     def __init__(self, litConfig, provider, display):
 101         threading.Thread.__init__(self)
 102         self.litConfig = litConfig
 103         self.provider = provider
 104         self.display = display
 105
 106     def run(self):
 107         while 1:
 108             item = self.provider.get()
 109             if item is None:
 110                 break
 111             self.runTest(item)
 112
 113     def runTest(self, test):
 114         result = None
 115         startTime = time.time()
 116         try:
 117             result, output = test.config.test_format.execute(test,
 118                                                              self.litConfig)
 119         except KeyboardInterrupt:
 120             # This is a sad hack. Unfortunately subprocess goes
 121             # bonkers with ctrl-c and we start forking merrily.
 122             print '\nCtrl-C detected, goodbye.'
 123             os.kill(0,9)
 124         except:
 125             if self.litConfig.debug:
 126                 raise
 127             result = Test.UNRESOLVED
 128             output = 'Exception during script execution:\n'
 129             output += traceback.format_exc()
 130             output += '\n'
 131         elapsed = time.time() - startTime
 132
 133         test.setResult(result, output, elapsed)
 134         self.display.update(test)
 135
 136 def dirContainsTestSuite(path):
 137     cfgpath = os.path.join(path, kSiteConfigName)
 138     if os.path.exists(cfgpath):
 139         return cfgpath
 140     cfgpath = os.path.join(path, kConfigName)
 141     if os.path.exists(cfgpath):
 142         return cfgpath
 143
 144 def getTestSuite(item, litConfig, cache):
 145     """getTestSuite(item, litConfig, cache) -> (suite, relative_path)
 146
 147     Find the test suite containing @arg item.
 148
 149     @retval (None, ...) - Indicates no test suite contains @arg item.
 150     @retval (suite, relative_path) - The suite that @arg item is in, and its
 151     relative path inside that suite.
 152     """
 153     def search1(path):
 154         # Check for a site config or a lit config.
 155         cfgpath = dirContainsTestSuite(path)
 156
 157         # If we didn't find a config file, keep looking.
 158         if not cfgpath:
 159             parent,base = os.path.split(path)
 160             if parent == path:
 161                 return (None, ())
 162
 163             ts, relative = search(parent)
 164             return (ts, relative + (base,))
 165
 166         # We found a config file, load it.
 167         if litConfig.debug:
 168             litConfig.note('loading suite config %r' % cfgpath)
 169
 170         cfg = TestingConfig.frompath(cfgpath, None, litConfig, mustExist = True)
 171         source_root = os.path.realpath(cfg.test_source_root or path)
 172         exec_root = os.path.realpath(cfg.test_exec_root or path)
 173         return Test.TestSuite(cfg.name, source_root, exec_root, cfg), ()
 174
 175     def search(path):
 176         # Check for an already instantiated test suite.
 177         res = cache.get(path)
 178         if res is None:
 179             cache[path] = res = search1(path)
 180         return res
 181
 182     # Canonicalize the path.
 183     item = os.path.realpath(item)
 184
 185     # Skip files and virtual components.
 186     components = []
 187     while not os.path.isdir(item):
 188         parent,base = os.path.split(item)
 189         if parent == item:
 190             return (None, ())
 191         components.append(base)
 192         item = parent
 193     components.reverse()
 194
 195     ts, relative = search(item)
 196     return ts, tuple(relative + tuple(components))
 197
 198 def getLocalConfig(ts, path_in_suite, litConfig, cache):
 199     def search1(path_in_suite):
 200         # Get the parent config.
 201         if not path_in_suite:
 202             parent = ts.config
 203         else:
 204             parent = search(path_in_suite[:-1])
 205
 206         # Load the local configuration.
 207         source_path = ts.getSourcePath(path_in_suite)
 208         cfgpath = os.path.join(source_path, kLocalConfigName)
 209         if litConfig.debug:
 210             litConfig.note('loading local config %r' % cfgpath)
 211         return TestingConfig.frompath(cfgpath, parent, litConfig,
 212                                     mustExist = False,
 213                                     config = parent.clone(cfgpath))
 214
 215     def search(path_in_suite):
 216         key = (ts, path_in_suite)
 217         res = cache.get(key)
 218         if res is None:
 219             cache[key] = res = search1(path_in_suite)
 220         return res
 221
 222     return search(path_in_suite)
 223
 224 def getTests(path, litConfig, testSuiteCache, localConfigCache):
 225     # Find the test suite for this input and its relative path.
 226     ts,path_in_suite = getTestSuite(path, litConfig, testSuiteCache)
 227     if ts is None:
 228         litConfig.warning('unable to find test suite for %r' % path)
 229         return ()
 230
 231     if litConfig.debug:
 232         litConfig.note('resolved input %r to %r::%r' % (path, ts.name,
 233                                                         path_in_suite))
 234
 235     return getTestsInSuite(ts, path_in_suite, litConfig,
 236                            testSuiteCache, localConfigCache)
 237
 238 def getTestsInSuite(ts, path_in_suite, litConfig,
 239                     testSuiteCache, localConfigCache):
 240     # Check that the source path exists (errors here are reported by the
 241     # caller).
 242     source_path = ts.getSourcePath(path_in_suite)
 243     if not os.path.exists(source_path):
 244         return
 245
 246     # Check if the user named a test directly.
 247     if not os.path.isdir(source_path):
 248         lc = getLocalConfig(ts, path_in_suite[:-1], litConfig, localConfigCache)
 249         yield Test.Test(ts, path_in_suite, lc)
 250         return
 251
 252     # Otherwise we have a directory to search for tests, start by getting the
 253     # local configuration.
 254     lc = getLocalConfig(ts, path_in_suite, litConfig, localConfigCache)
 255
 256     # Search for tests.
 257     for res in lc.test_format.getTestsInDirectory(ts, path_in_suite,
 258                                                   litConfig, lc):
 259         yield res
 260
 261     # Search subdirectories.
 262     for filename in os.listdir(source_path):
 263         # FIXME: This doesn't belong here?
 264         if filename == 'Output' or filename in lc.excludes:
 265             continue
 266
 267         filepath = os.path.join(source_path, filename)
 268         if os.path.isdir(filepath):
 269             # If this directory contains a test suite, reload it.
 270             if dirContainsTestSuite(filepath):
 271                 for res in getTests(filepath, litConfig,
 272                                     testSuiteCache, localConfigCache):
 273                     yield res
 274             else:
 275                 # Otherwise, continue loading from inside this test suite.
 276                 for res in getTestsInSuite(ts, path_in_suite + (filename,),
 277                                            litConfig, testSuiteCache,
 278                                            localConfigCache):
 279                     yield res
 280
 281 def runTests(numThreads, litConfig, provider, display):
 282     # If only using one testing thread, don't use threads at all; this lets us
 283     # profile, among other things.
 284     if numThreads == 1:
 285         t = Tester(litConfig, provider, display)
 286         t.run()
 287         return
 288
 289     # Otherwise spin up the testing threads and wait for them to finish.
 290     testers = [Tester(litConfig, provider, display)
 291                for i in range(numThreads)]
 292     for t in testers:
 293         t.start()
 294     try:
 295         for t in testers:
 296             t.join()
 297     except KeyboardInterrupt:
 298         sys.exit(2)
 299
 300 def main():
 301     global options
 302     from optparse import OptionParser, OptionGroup
 303     parser = OptionParser("usage: %prog [options] {file-or-path}")
 304
 305     parser.add_option("-j", "--threads", dest="numThreads", metavar="N",
 306                       help="Number of testing threads",
 307                       type=int, action="store", default=None)
 308
 309     group = OptionGroup(parser, "Output Format")
 310     # FIXME: I find these names very confusing, although I like the
 311     # functionality.
 312     group.add_option("-q", "--quiet", dest="quiet",
 313                      help="Suppress no error output",
 314                      action="store_true", default=False)
 315     group.add_option("-s", "--succinct", dest="succinct",
 316                      help="Reduce amount of output",
 317                      action="store_true", default=False)
 318     group.add_option("-v", "--verbose", dest="showOutput",
 319                      help="Show all test output",
 320                      action="store_true", default=False)
 321     group.add_option("", "--no-progress-bar", dest="useProgressBar",
 322                      help="Do not use curses based progress bar",
 323                      action="store_false", default=True)
 324     parser.add_option_group(group)
 325
 326     group = OptionGroup(parser, "Test Execution")
 327     group.add_option("", "--path", dest="path",
 328                      help="Additional paths to add to testing environment",
 329                      action="append", type=str, default=[])
 330     group.add_option("", "--vg", dest="useValgrind",
 331                      help="Run tests under valgrind",
 332                      action="store_true", default=False)
 333     group.add_option("", "--vg-arg", dest="valgrindArgs", metavar="ARG",
 334                      help="Specify an extra argument for valgrind",
 335                      type=str, action="append", default=[])
 336     group.add_option("", "--time-tests", dest="timeTests",
 337                      help="Track elapsed wall time for each test",
 338                      action="store_true", default=False)
 339     group.add_option("", "--no-execute", dest="noExecute",
 340                      help="Don't execute any tests (assume PASS)",
 341                      action="store_true", default=False)
 342     parser.add_option_group(group)
 343
 344     group = OptionGroup(parser, "Test Selection")
 345     group.add_option("", "--max-tests", dest="maxTests", metavar="N",
 346                      help="Maximum number of tests to run",
 347                      action="store", type=int, default=None)
 348     group.add_option("", "--max-time", dest="maxTime", metavar="N",
 349                      help="Maximum time to spend testing (in seconds)",
 350                      action="store", type=float, default=None)
 351     group.add_option("", "--shuffle", dest="shuffle",
 352                      help="Run tests in random order",
 353                      action="store_true", default=False)
 354     parser.add_option_group(group)
 355
 356     group = OptionGroup(parser, "Debug and Experimental Options")
 357     group.add_option("", "--debug", dest="debug",
 358                       help="Enable debugging (for 'lit' development)",
 359                       action="store_true", default=False)
 360     group.add_option("", "--show-suites", dest="showSuites",
 361                       help="Show discovered test suites",
 362                       action="store_true", default=False)
 363     group.add_option("", "--no-tcl-as-sh", dest="useTclAsSh",
 364                       help="Don't run Tcl scripts using 'sh'",
 365                       action="store_false", default=True)
 366     parser.add_option_group(group)
 367
 368     (opts, args) = parser.parse_args()
 369
 370     if not args:
 371         parser.error('No inputs specified')
 372
 373     if opts.numThreads is None:
 374         opts.numThreads = Util.detectCPUs()
 375
 376     inputs = args
 377
 378     # Create the global config object.
 379     litConfig = LitConfig.LitConfig(progname = os.path.basename(sys.argv[0]),
 380                                     path = opts.path,
 381                                     quiet = opts.quiet,
 382                                     useValgrind = opts.useValgrind,
 383                                     valgrindArgs = opts.valgrindArgs,
 384                                     useTclAsSh = opts.useTclAsSh,
 385                                     noExecute = opts.noExecute,
 386                                     debug = opts.debug,
 387                                     isWindows = (platform.system()=='Windows'))
 388
 389     # Load the tests from the inputs.
 390     tests = []
 391     testSuiteCache = {}
 392     localConfigCache = {}
 393     for input in inputs:
 394         prev = len(tests)
 395         tests.extend(getTests(input, litConfig,
 396                               testSuiteCache, localConfigCache))
 397         if prev == len(tests):
 398             litConfig.warning('input %r contained no tests' % input)
 399
 400     # If there were any errors during test discovery, exit now.
 401     if litConfig.numErrors:
 402         print >>sys.stderr, '%d errors, exiting.' % litConfig.numErrors
 403         sys.exit(2)
 404
 405     if opts.showSuites:
 406         suitesAndTests = dict([(ts,[])
 407                                for ts,_ in testSuiteCache.values()])
 408         for t in tests:
 409             suitesAndTests[t.suite].append(t)
 410
 411         print '-- Test Suites --'
 412         suitesAndTests = suitesAndTests.items()
 413         suitesAndTests.sort(key = lambda (ts,_): ts.name)
 414         for ts,tests in suitesAndTests:
 415             print '  %s - %d tests' %(ts.name, len(tests))
 416             print '    Source Root: %s' % ts.source_root
 417             print '    Exec Root  : %s' % ts.exec_root
 418
 419     # Select and order the tests.
 420     numTotalTests = len(tests)
 421     if opts.shuffle:
 422         random.shuffle(tests)
 423     else:
 424         tests.sort(key = lambda t: t.getFullName())
 425     if opts.maxTests is not None:
 426         tests = tests[:opts.maxTests]
 427
 428     extra = ''
 429     if len(tests) != numTotalTests:
 430         extra = ' of %d' % numTotalTests
 431     header = '-- Testing: %d%s tests, %d threads --'%(len(tests),extra,
 432                                                       opts.numThreads)
 433
 434     progressBar = None
 435     if not opts.quiet:
 436         if opts.succinct and opts.useProgressBar:
 437             try:
 438                 tc = ProgressBar.TerminalController()
 439                 progressBar = ProgressBar.ProgressBar(tc, header)
 440             except ValueError:
 441                 print header
 442                 progressBar = ProgressBar.SimpleProgressBar('Testing: ')
 443         else:
 444             print header
 445
 446     # Don't create more threads than tests.
 447     opts.numThreads = min(len(tests), opts.numThreads)
 448
 449     startTime = time.time()
 450     display = TestingProgressDisplay(opts, len(tests), progressBar)
 451     provider = TestProvider(tests, opts.maxTime)
 452     runTests(opts.numThreads, litConfig, provider, display)
 453     display.finish()
 454
 455     if not opts.quiet:
 456         print 'Testing Time: %.2fs'%(time.time() - startTime)
 457
 458     # Update results for any tests which weren't run.
 459     for t in tests:
 460         if t.result is None:
 461             t.setResult(Test.UNRESOLVED, '', 0.0)
 462
 463     # List test results organized by kind.
 464     hasFailures = False
 465     byCode = {}
 466     for t in tests:
 467         if t.result not in byCode:
 468             byCode[t.result] = []
 469         byCode[t.result].append(t)
 470         if t.result.isFailure:
 471             hasFailures = True
 472
 473     # FIXME: Show unresolved and (optionally) unsupported tests.
 474     for title,code in (('Unexpected Passing Tests', Test.XPASS),
 475                        ('Failing Tests', Test.FAIL)):
 476         elts = byCode.get(code)
 477         if not elts:
 478             continue
 479         print '*'*20
 480         print '%s (%d):' % (title, len(elts))
 481         for t in elts:
 482             print '    %s' % t.getFullName()
 483         print
 484
 485     if opts.timeTests:
 486         byTime = list(tests)
 487         byTime.sort(key = lambda t: t.elapsed)
 488         if byTime:
 489             Util.printHistogram([(t.getFullName(), t.elapsed) for t in byTime],
 490                                 title='Tests')
 491
 492     for name,code in (('Expected Passes    ', Test.PASS),
 493                       ('Expected Failures  ', Test.XFAIL),
 494                       ('Unsupported Tests  ', Test.UNSUPPORTED),
 495                       ('Unresolved Tests   ', Test.UNRESOLVED),
 496                       ('Unexpected Passes  ', Test.XPASS),
 497                       ('Unexpected Failures', Test.FAIL),):
 498         if opts.quiet and not code.isFailure:
 499             continue
 500         N = len(byCode.get(code,[]))
 501         if N:
 502             print '  %s: %d' % (name,N)
 503
 504     # If we encountered any additional errors, exit abnormally.
 505     if litConfig.numErrors:
 506         print >>sys.stderr, '\n%d error(s), exiting.' % litConfig.numErrors
 507         sys.exit(2)
 508
 509     # Warn about warnings.
 510     if litConfig.numWarnings:
 511         print >>sys.stderr, '\n%d warning(s) in tests.' % litConfig.numWarnings
 512
 513     if hasFailures:
 514         sys.exit(1)
 515     sys.exit(0)
 516
 517 if __name__=='__main__':
 518     # Bump the GIL check interval, its more important to get any one thread to a
 519     # blocking operation (hopefully exec) than to try and unblock other threads.
 520     import sys
 521     sys.setcheckinterval(1000)
 522     main()