utils/lit/lit.py

   1 #!/usr/bin/env python
   2
   3 """
   4 lit - LLVM Integrated Tester.
   5
   6 See lit.pod for more information.
   7 """
   8
   9 import math, os, platform, random, re, sys, time, threading, traceback
  10
  11 import ProgressBar
  12 import TestRunner
  13 import Util
  14
  15 from TestingConfig import TestingConfig
  16 import LitConfig
  17 import Test
  18
  19 # FIXME: Rename to 'config.lit', 'site.lit', and 'local.lit' ?
  20 kConfigName = 'lit.cfg'
  21 kSiteConfigName = 'lit.site.cfg'
  22 kLocalConfigName = 'lit.local.cfg'
  23
  24 class TestingProgressDisplay:
  25     def __init__(self, opts, numTests, progressBar=None):
  26         self.opts = opts
  27         self.numTests = numTests
  28         self.current = None
  29         self.lock = threading.Lock()
  30         self.progressBar = progressBar
  31         self.completed = 0
  32
  33     def update(self, test):
  34         # Avoid locking overhead in quiet mode
  35         if self.opts.quiet and not test.result.isFailure:
  36             self.completed += 1
  37             return
  38
  39         # Output lock.
  40         self.lock.acquire()
  41         try:
  42             self.handleUpdate(test)
  43         finally:
  44             self.lock.release()
  45
  46     def finish(self):
  47         if self.progressBar:
  48             self.progressBar.clear()
  49         elif self.opts.quiet:
  50             pass
  51         elif self.opts.succinct:
  52             sys.stdout.write('\n')
  53
  54     def handleUpdate(self, test):
  55         self.completed += 1
  56         if self.progressBar:
  57             self.progressBar.update(float(self.completed)/self.numTests,
  58                                     test.getFullName())
  59
  60         if self.opts.succinct and not test.result.isFailure:
  61             return
  62
  63         if self.progressBar:
  64             self.progressBar.clear()
  65
  66         print '%s: %s (%d of %d)' % (test.result.name, test.getFullName(),
  67                                      self.completed, self.numTests)
  68
  69         if test.result.isFailure and self.opts.showOutput:
  70             print "%s TEST '%s' FAILED %s" % ('*'*20, test.getFullName(),
  71                                               '*'*20)
  72             print test.output
  73             print "*" * 20
  74
  75         sys.stdout.flush()
  76
  77 class TestProvider:
  78     def __init__(self, tests, maxTime):
  79         self.maxTime = maxTime
  80         self.iter = iter(tests)
  81         self.lock = threading.Lock()
  82         self.startTime = time.time()
  83
  84     def get(self):
  85         # Check if we have run out of time.
  86         if self.maxTime is not None:
  87             if time.time() - self.startTime > self.maxTime:
  88                 return None
  89
  90         # Otherwise take the next test.
  91         self.lock.acquire()
  92         try:
  93             item = self.iter.next()
  94         except StopIteration:
  95             item = None
  96         self.lock.release()
  97         return item
  98
  99 class Tester(threading.Thread):
 100     def __init__(self, litConfig, provider, display):
 101         threading.Thread.__init__(self)
 102         self.litConfig = litConfig
 103         self.provider = provider
 104         self.display = display
 105
 106     def run(self):
 107         while 1:
 108             item = self.provider.get()
 109             if item is None:
 110                 break
 111             self.runTest(item)
 112
 113     def runTest(self, test):
 114         result = None
 115         startTime = time.time()
 116         try:
 117             result, output = test.config.test_format.execute(test,
 118                                                              self.litConfig)
 119         except KeyboardInterrupt:
 120             # This is a sad hack. Unfortunately subprocess goes
 121             # bonkers with ctrl-c and we start forking merrily.
 122             print '\nCtrl-C detected, goodbye.'
 123             os.kill(0,9)
 124         except:
 125             if self.litConfig.debug:
 126                 raise
 127             result = Test.UNRESOLVED
 128             output = 'Exception during script execution:\n'
 129             output += traceback.format_exc()
 130             output += '\n'
 131         elapsed = time.time() - startTime
 132
 133         test.setResult(result, output, elapsed)
 134         self.display.update(test)
 135
 136 def dirContainsTestSuite(path):
 137     cfgpath = os.path.join(path, kSiteConfigName)
 138     if os.path.exists(cfgpath):
 139         return cfgpath
 140     cfgpath = os.path.join(path, kConfigName)
 141     if os.path.exists(cfgpath):
 142         return cfgpath
 143
 144 def getTestSuite(item, litConfig, cache):
 145     """getTestSuite(item, litConfig, cache) -> (suite, relative_path)
 146
 147     Find the test suite containing @arg item.
 148
 149     @retval (None, ...) - Indicates no test suite contains @arg item.
 150     @retval (suite, relative_path) - The suite that @arg item is in, and its
 151     relative path inside that suite.
 152     """
 153     def search1(path):
 154         # Check for a site config or a lit config.
 155         cfgpath = dirContainsTestSuite(path)
 156
 157         # If we didn't find a config file, keep looking.
 158         if not cfgpath:
 159             parent,base = os.path.split(path)
 160             if parent == path:
 161                 return (None, ())
 162
 163             ts, relative = search(parent)
 164             return (ts, relative + (base,))
 165
 166         # We found a config file, load it.
 167         if litConfig.debug:
 168             litConfig.note('loading suite config %r' % cfgpath)
 169
 170         cfg = TestingConfig.frompath(cfgpath, None, litConfig, mustExist = True)
 171         source_root = os.path.realpath(cfg.test_source_root or path)
 172         exec_root = os.path.realpath(cfg.test_exec_root or path)
 173         return Test.TestSuite(cfg.name, source_root, exec_root, cfg), ()
 174
 175     def search(path):
 176         # Check for an already instantiated test suite.
 177         res = cache.get(path)
 178         if res is None:
 179             cache[path] = res = search1(path)
 180         return res
 181
 182     # Canonicalize the path.
 183     item = os.path.realpath(item)
 184
 185     # Skip files and virtual components.
 186     components = []
 187     while not os.path.isdir(item):
 188         parent,base = os.path.split(item)
 189         if parent == item:
 190             return (None, ())
 191         components.append(base)
 192         item = parent
 193     components.reverse()
 194
 195     ts, relative = search(item)
 196     return ts, tuple(relative + tuple(components))
 197
 198 def getLocalConfig(ts, path_in_suite, litConfig, cache):
 199     def search1(path_in_suite):
 200         # Get the parent config.
 201         if not path_in_suite:
 202             parent = ts.config
 203         else:
 204             parent = search(path_in_suite[:-1])
 205
 206         # Load the local configuration.
 207         source_path = ts.getSourcePath(path_in_suite)
 208         cfgpath = os.path.join(source_path, kLocalConfigName)
 209         if litConfig.debug:
 210             litConfig.note('loading local config %r' % cfgpath)
 211         return TestingConfig.frompath(cfgpath, parent, litConfig,
 212                                     mustExist = False,
 213                                     config = parent.clone(cfgpath))
 214
 215     def search(path_in_suite):
 216         key = (ts, path_in_suite)
 217         res = cache.get(key)
 218         if res is None:
 219             cache[key] = res = search1(path_in_suite)
 220         return res
 221
 222     return search(path_in_suite)
 223
 224 def getTests(path, litConfig, testSuiteCache, localConfigCache):
 225     # Find the test suite for this input and its relative path.
 226     ts,path_in_suite = getTestSuite(path, litConfig, testSuiteCache)
 227     if ts is None:
 228         litConfig.warning('unable to find test suite for %r' % path)
 229         return ()
 230
 231     if litConfig.debug:
 232         litConfig.note('resolved input %r to %r::%r' % (path, ts.name,
 233                                                         path_in_suite))
 234
 235     return getTestsInSuite(ts, path_in_suite, litConfig,
 236                            testSuiteCache, localConfigCache)
 237
 238 def getTestsInSuite(ts, path_in_suite, litConfig,
 239                     testSuiteCache, localConfigCache):
 240     # Check that the source path exists (errors here are reported by the
 241     # caller).
 242     source_path = ts.getSourcePath(path_in_suite)
 243     if not os.path.exists(source_path):
 244         return
 245
 246     # Check if the user named a test directly.
 247     if not os.path.isdir(source_path):
 248         lc = getLocalConfig(ts, path_in_suite[:-1], litConfig, localConfigCache)
 249         yield Test.Test(ts, path_in_suite, lc)
 250         return
 251
 252     # Otherwise we have a directory to search for tests, start by getting the
 253     # local configuration.
 254     lc = getLocalConfig(ts, path_in_suite, litConfig, localConfigCache)
 255     for filename in os.listdir(source_path):
 256         # FIXME: This doesn't belong here?
 257         if filename == 'Output' or filename in lc.excludes:
 258             continue
 259
 260         filepath = os.path.join(source_path, filename)
 261         if os.path.isdir(filepath):
 262             # If this directory contains a test suite, reload it.
 263             if dirContainsTestSuite(filepath):
 264                 for res in getTests(filepath, litConfig,
 265                                     testSuiteCache, localConfigCache):
 266                     yield res
 267             else:
 268                 # Otherwise, continue loading from inside this test suite.
 269                 for res in getTestsInSuite(ts, path_in_suite + (filename,),
 270                                            litConfig, testSuiteCache,
 271                                            localConfigCache):
 272                     yield res
 273         else:
 274             # Otherwise add tests for matching suffixes.
 275             base,ext = os.path.splitext(filename)
 276             if ext in lc.suffixes:
 277                 yield Test.Test(ts, path_in_suite + (filename,), lc)
 278
 279 def runTests(numThreads, litConfig, provider, display):
 280     # If only using one testing thread, don't use threads at all; this lets us
 281     # profile, among other things.
 282     if numThreads == 1:
 283         t = Tester(litConfig, provider, display)
 284         t.run()
 285         return
 286
 287     # Otherwise spin up the testing threads and wait for them to finish.
 288     testers = [Tester(litConfig, provider, display)
 289                for i in range(numThreads)]
 290     for t in testers:
 291         t.start()
 292     try:
 293         for t in testers:
 294             t.join()
 295     except KeyboardInterrupt:
 296         sys.exit(2)
 297
 298 def main():
 299     global options
 300     from optparse import OptionParser, OptionGroup
 301     parser = OptionParser("usage: %prog [options] {file-or-path}")
 302
 303     parser.add_option("-j", "--threads", dest="numThreads", metavar="N",
 304                       help="Number of testing threads",
 305                       type=int, action="store", default=None)
 306
 307     group = OptionGroup(parser, "Output Format")
 308     # FIXME: I find these names very confusing, although I like the
 309     # functionality.
 310     group.add_option("-q", "--quiet", dest="quiet",
 311                      help="Suppress no error output",
 312                      action="store_true", default=False)
 313     group.add_option("-s", "--succinct", dest="succinct",
 314                      help="Reduce amount of output",
 315                      action="store_true", default=False)
 316     group.add_option("-v", "--verbose", dest="showOutput",
 317                      help="Show all test output",
 318                      action="store_true", default=False)
 319     group.add_option("", "--no-progress-bar", dest="useProgressBar",
 320                      help="Do not use curses based progress bar",
 321                      action="store_false", default=True)
 322     parser.add_option_group(group)
 323
 324     group = OptionGroup(parser, "Test Execution")
 325     group.add_option("", "--path", dest="path",
 326                      help="Additional paths to add to testing environment",
 327                      action="append", type=str, default=[])
 328     group.add_option("", "--vg", dest="useValgrind",
 329                      help="Run tests under valgrind",
 330                      action="store_true", default=False)
 331     group.add_option("", "--vg-arg", dest="valgrindArgs", metavar="ARG",
 332                      help="Specify an extra argument for valgrind",
 333                      type=str, action="append", default=[])
 334     group.add_option("", "--time-tests", dest="timeTests",
 335                      help="Track elapsed wall time for each test",
 336                      action="store_true", default=False)
 337     group.add_option("", "--no-execute", dest="noExecute",
 338                      help="Don't execute any tests (assume PASS)",
 339                      action="store_true", default=False)
 340     parser.add_option_group(group)
 341
 342     group = OptionGroup(parser, "Test Selection")
 343     group.add_option("", "--max-tests", dest="maxTests", metavar="N",
 344                      help="Maximum number of tests to run",
 345                      action="store", type=int, default=None)
 346     group.add_option("", "--max-time", dest="maxTime", metavar="N",
 347                      help="Maximum time to spend testing (in seconds)",
 348                      action="store", type=float, default=None)
 349     group.add_option("", "--shuffle", dest="shuffle",
 350                      help="Run tests in random order",
 351                      action="store_true", default=False)
 352     parser.add_option_group(group)
 353
 354     group = OptionGroup(parser, "Debug and Experimental Options")
 355     group.add_option("", "--debug", dest="debug",
 356                       help="Enable debugging (for 'lit' development)",
 357                       action="store_true", default=False)
 358     group.add_option("", "--show-suites", dest="showSuites",
 359                       help="Show discovered test suites",
 360                       action="store_true", default=False)
 361     group.add_option("", "--no-tcl-as-sh", dest="useTclAsSh",
 362                       help="Don't run Tcl scripts using 'sh'",
 363                       action="store_false", default=True)
 364     parser.add_option_group(group)
 365
 366     (opts, args) = parser.parse_args()
 367
 368     if not args:
 369         parser.error('No inputs specified')
 370
 371     if opts.numThreads is None:
 372         opts.numThreads = Util.detectCPUs()
 373
 374     inputs = args
 375
 376     # Create the global config object.
 377     litConfig = LitConfig.LitConfig(progname = os.path.basename(sys.argv[0]),
 378                                     path = opts.path,
 379                                     quiet = opts.quiet,
 380                                     useValgrind = opts.useValgrind,
 381                                     valgrindArgs = opts.valgrindArgs,
 382                                     useTclAsSh = opts.useTclAsSh,
 383                                     noExecute = opts.noExecute,
 384                                     debug = opts.debug,
 385                                     isWindows = (platform.system()=='Windows'))
 386
 387     # Load the tests from the inputs.
 388     tests = []
 389     testSuiteCache = {}
 390     localConfigCache = {}
 391     for input in inputs:
 392         prev = len(tests)
 393         tests.extend(getTests(input, litConfig,
 394                               testSuiteCache, localConfigCache))
 395         if prev == len(tests):
 396             litConfig.warning('input %r contained no tests' % input)
 397
 398     # If there were any errors during test discovery, exit now.
 399     if litConfig.numErrors:
 400         print >>sys.stderr, '%d errors, exiting.' % litConfig.numErrors
 401         sys.exit(2)
 402
 403     if opts.showSuites:
 404         suitesAndTests = dict([(ts,[])
 405                                for ts,_ in testSuiteCache.values()])
 406         for t in tests:
 407             suitesAndTests[t.suite].append(t)
 408
 409         print '-- Test Suites --'
 410         suitesAndTests = suitesAndTests.items()
 411         suitesAndTests.sort(key = lambda (ts,_): ts.name)
 412         for ts,tests in suitesAndTests:
 413             print '  %s - %d tests' %(ts.name, len(tests))
 414             print '    Source Root: %s' % ts.source_root
 415             print '    Exec Root  : %s' % ts.exec_root
 416
 417     # Select and order the tests.
 418     numTotalTests = len(tests)
 419     if opts.shuffle:
 420         random.shuffle(tests)
 421     else:
 422         tests.sort(key = lambda t: t.getFullName())
 423     if opts.maxTests is not None:
 424         tests = tests[:opts.maxTests]
 425
 426     extra = ''
 427     if len(tests) != numTotalTests:
 428         extra = ' of %d' % numTotalTests
 429     header = '-- Testing: %d%s tests, %d threads --'%(len(tests),extra,
 430                                                       opts.numThreads)
 431
 432     progressBar = None
 433     if not opts.quiet:
 434         if opts.succinct and opts.useProgressBar:
 435             try:
 436                 tc = ProgressBar.TerminalController()
 437                 progressBar = ProgressBar.ProgressBar(tc, header)
 438             except ValueError:
 439                 print header
 440                 progressBar = ProgressBar.SimpleProgressBar('Testing: ')
 441         else:
 442             print header
 443
 444     # Don't create more threads than tests.
 445     opts.numThreads = min(len(tests), opts.numThreads)
 446
 447     startTime = time.time()
 448     display = TestingProgressDisplay(opts, len(tests), progressBar)
 449     provider = TestProvider(tests, opts.maxTime)
 450     runTests(opts.numThreads, litConfig, provider, display)
 451     display.finish()
 452
 453     if not opts.quiet:
 454         print 'Testing Time: %.2fs'%(time.time() - startTime)
 455
 456     # Update results for any tests which weren't run.
 457     for t in tests:
 458         if t.result is None:
 459             t.setResult(Test.UNRESOLVED, '', 0.0)
 460
 461     # List test results organized by kind.
 462     hasFailures = False
 463     byCode = {}
 464     for t in tests:
 465         if t.result not in byCode:
 466             byCode[t.result] = []
 467         byCode[t.result].append(t)
 468         if t.result.isFailure:
 469             hasFailures = True
 470
 471     # FIXME: Show unresolved and (optionally) unsupported tests.
 472     for title,code in (('Unexpected Passing Tests', Test.XPASS),
 473                        ('Failing Tests', Test.FAIL)):
 474         elts = byCode.get(code)
 475         if not elts:
 476             continue
 477         print '*'*20
 478         print '%s (%d):' % (title, len(elts))
 479         for t in elts:
 480             print '    %s' % t.getFullName()
 481         print
 482
 483     if opts.timeTests:
 484         byTime = list(tests)
 485         byTime.sort(key = lambda t: t.elapsed)
 486         if byTime:
 487             Util.printHistogram([(t.getFullName(), t.elapsed) for t in byTime],
 488                                 title='Tests')
 489
 490     for name,code in (('Expected Passes    ', Test.PASS),
 491                       ('Expected Failures  ', Test.XFAIL),
 492                       ('Unsupported Tests  ', Test.UNSUPPORTED),
 493                       ('Unresolved Tests   ', Test.UNRESOLVED),
 494                       ('Unexpected Passes  ', Test.XPASS),
 495                       ('Unexpected Failures', Test.FAIL),):
 496         if opts.quiet and not code.isFailure:
 497             continue
 498         N = len(byCode.get(code,[]))
 499         if N:
 500             print '  %s: %d' % (name,N)
 501
 502     # If we encountered any additional errors, exit abnormally.
 503     if litConfig.numErrors:
 504         print >>sys.stderr, '\n%d error(s), exiting.' % litConfig.numErrors
 505         sys.exit(2)
 506
 507     # Warn about warnings.
 508     if litConfig.numWarnings:
 509         print >>sys.stderr, '\n%d warning(s) in tests.' % litConfig.numWarnings
 510
 511     if hasFailures:
 512         sys.exit(1)
 513     sys.exit(0)
 514
 515 if __name__=='__main__':
 516     # Bump the GIL check interval, its more important to get any one thread to a
 517     # blocking operation (hopefully exec) than to try and unblock other threads.
 518     import sys
 519     sys.setcheckinterval(1000)
 520     main()