Lib/test/regrtest.py

   1 #! /usr/bin/env python
   2
   3 """Regression test.
   4
   5 This will find all modules whose name is "test_*" in the test
   6 directory, and run them.  Various command line options provide
   7 additional facilities.
   8
   9 Command line options:
  10
  11 -v: verbose   -- run tests in verbose mode with output to stdout
  12 -q: quiet     -- don't print anything except if a test fails
  13 -g: generate  -- write the output file for a test instead of comparing it
  14 -x: exclude   -- arguments are tests to *exclude*
  15 -s: single    -- run only a single test (see below)
  16 -r: random    -- randomize test execution order
  17 -l: findleaks -- if GC is available detect and print cyclic garbage
  18 --have-resources   -- run tests that require large resources (time/space)
  19
  20 If non-option arguments are present, they are names for tests to run,
  21 unless -x is given, in which case they are names for tests not to run.
  22 If no test names are given, all tests are run.
  23
  24 -v is incompatible with -g and does not compare test output files.
  25
  26 -s means to run only a single test and exit.  This is useful when Purifying
  27 the Python interpreter.  The file /tmp/pynexttest is read to find the next
  28 test to run.  If this file is missing, the first test_*.py file in testdir or
  29 on the command line is used.  (actually tempfile.gettempdir() is used instead
  30 of /tmp).
  31
  32 """
  33
  34 import sys
  35 import string
  36 import os
  37 import getopt
  38 import traceback
  39 import random
  40
  41 import test_support
  42
  43 def main(tests=None, testdir=None, verbose=0, quiet=0, generate=0,
  44          exclude=0, single=0, randomize=0, findleaks=0,
  45          use_large_resources=0):
  46     """Execute a test suite.
  47
  48     This also parses command-line options and modifies its behavior
  49     accordingly.
  50
  51     tests -- a list of strings containing test names (optional)
  52     testdir -- the directory in which to look for tests (optional)
  53
  54     Users other than the Python test suite will certainly want to
  55     specify testdir; if it's omitted, the directory containing the
  56     Python test suite is searched for.
  57
  58     If the tests argument is omitted, the tests listed on the
  59     command-line will be used.  If that's empty, too, then all *.py
  60     files beginning with test_ will be used.
  61
  62     The other seven default arguments (verbose, quiet, generate, exclude,
  63     single, randomize, and findleaks) allow programmers calling main()
  64     directly to set the values that would normally be set by flags on the
  65     command line.
  66
  67     """
  68
  69     try:
  70         opts, args = getopt.getopt(sys.argv[1:], 'vgqxsrl', ['have-resources'])
  71     except getopt.error, msg:
  72         print msg
  73         print __doc__
  74         return 2
  75     for o, a in opts:
  76         if o == '-v': verbose = verbose+1
  77         if o == '-q': quiet = 1; verbose = 0
  78         if o == '-g': generate = 1
  79         if o == '-x': exclude = 1
  80         if o == '-s': single = 1
  81         if o == '-r': randomize = 1
  82         if o == '-l': findleaks = 1
  83         if o == '--have-resources': use_large_resources = 1
  84     if generate and verbose:
  85         print "-g and -v don't go together!"
  86         return 2
  87     good = []
  88     bad = []
  89     skipped = []
  90
  91     if findleaks:
  92         try:
  93             import gc
  94         except ImportError:
  95             print 'cycle garbage collection not available'
  96             findleaks = 0
  97         else:
  98             gc.set_debug(gc.DEBUG_SAVEALL)
  99             found_garbage = []
 100
 101     if single:
 102         from tempfile import gettempdir
 103         filename = os.path.join(gettempdir(), 'pynexttest')
 104         try:
 105             fp = open(filename, 'r')
 106             next = string.strip(fp.read())
 107             tests = [next]
 108             fp.close()
 109         except IOError:
 110             pass
 111     for i in range(len(args)):
 112         # Strip trailing ".py" from arguments
 113         if args[i][-3:] == '.py':
 114             args[i] = args[i][:-3]
 115     stdtests = STDTESTS[:]
 116     nottests = NOTTESTS[:]
 117     if exclude:
 118         for arg in args:
 119             if arg in stdtests:
 120                 stdtests.remove(arg)
 121         nottests[:0] = args
 122         args = []
 123     tests = tests or args or findtests(testdir, stdtests, nottests)
 124     if single:
 125         tests = tests[:1]
 126     if randomize:
 127         random.shuffle(tests)
 128     test_support.verbose = verbose      # Tell tests to be moderately quiet
 129     test_support.use_large_resources = use_large_resources
 130     save_modules = sys.modules.keys()
 131     for test in tests:
 132         if not quiet:
 133             print test
 134         ok = runtest(test, generate, verbose, quiet, testdir)
 135         if ok > 0:
 136             good.append(test)
 137         elif ok == 0:
 138             bad.append(test)
 139         else:
 140             skipped.append(test)
 141         if findleaks:
 142             gc.collect()
 143             if gc.garbage:
 144                 print "garbage:", repr(gc.garbage)
 145                 found_garbage.extend(gc.garbage)
 146                 del gc.garbage[:]
 147         # Unload the newly imported modules (best effort finalization)
 148         for module in sys.modules.keys():
 149             if module not in save_modules and module.startswith("test."):
 150                 test_support.unload(module)
 151     if good and not quiet:
 152         if not bad and not skipped and len(good) > 1:
 153             print "All",
 154         print count(len(good), "test"), "OK."
 155     if bad:
 156         print count(len(bad), "test"), "failed:",
 157         print string.join(bad)
 158     if skipped and not quiet:
 159         print count(len(skipped), "test"), "skipped:",
 160         print string.join(skipped)
 161
 162     if single:
 163         alltests = findtests(testdir, stdtests, nottests)
 164         for i in range(len(alltests)):
 165             if tests[0] == alltests[i]:
 166                 if i == len(alltests) - 1:
 167                     os.unlink(filename)
 168                 else:
 169                     fp = open(filename, 'w')
 170                     fp.write(alltests[i+1] + '\n')
 171                     fp.close()
 172                 break
 173         else:
 174             os.unlink(filename)
 175
 176     return len(bad) > 0
 177
 178 STDTESTS = [
 179     'test_grammar',
 180     'test_opcodes',
 181     'test_operations',
 182     'test_builtin',
 183     'test_exceptions',
 184     'test_types',
 185    ]
 186
 187 NOTTESTS = [
 188     'test_support',
 189     'test_b1',
 190     'test_b2',
 191     ]
 192
 193 def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS):
 194     """Return a list of all applicable test modules."""
 195     if not testdir: testdir = findtestdir()
 196     names = os.listdir(testdir)
 197     tests = []
 198     for name in names:
 199         if name[:5] == "test_" and name[-3:] == ".py":
 200             modname = name[:-3]
 201             if modname not in stdtests and modname not in nottests:
 202                 tests.append(modname)
 203     tests.sort()
 204     return stdtests + tests
 205
 206 def runtest(test, generate, verbose, quiet, testdir = None):
 207     """Run a single test.
 208     test -- the name of the test
 209     generate -- if true, generate output, instead of running the test
 210     and comparing it to a previously created output file
 211     verbose -- if true, print more messages
 212     quiet -- if true, don't print 'skipped' messages (probably redundant)
 213     testdir -- test directory
 214     """
 215     test_support.unload(test)
 216     if not testdir: testdir = findtestdir()
 217     outputdir = os.path.join(testdir, "output")
 218     outputfile = os.path.join(outputdir, test)
 219     try:
 220         if generate:
 221             cfp = open(outputfile, "w")
 222         elif verbose:
 223             cfp = sys.stdout
 224         else:
 225             cfp = Compare(outputfile)
 226     except IOError:
 227         cfp = None
 228         print "Warning: can't open", outputfile
 229     try:
 230         save_stdout = sys.stdout
 231         try:
 232             if cfp:
 233                 sys.stdout = cfp
 234                 print test              # Output file starts with test name
 235             __import__(test, globals(), locals(), [])
 236             if cfp and not (generate or verbose):
 237                 cfp.close()
 238         finally:
 239             sys.stdout = save_stdout
 240     except (ImportError, test_support.TestSkipped), msg:
 241         if not quiet:
 242             print "test", test,
 243             print "skipped -- ", msg
 244         return -1
 245     except KeyboardInterrupt:
 246         raise
 247     except test_support.TestFailed, msg:
 248         print "test", test, "failed --", msg
 249         return 0
 250     except:
 251         type, value = sys.exc_info()[:2]
 252         print "test", test, "crashed --", str(type) + ":", value
 253         if verbose:
 254             traceback.print_exc(file=sys.stdout)
 255         return 0
 256     else:
 257         return 1
 258
 259 def findtestdir():
 260     if __name__ == '__main__':
 261         file = sys.argv[0]
 262     else:
 263         file = __file__
 264     testdir = os.path.dirname(file) or os.curdir
 265     return testdir
 266
 267 def count(n, word):
 268     if n == 1:
 269         return "%d %s" % (n, word)
 270     else:
 271         return "%d %ss" % (n, word)
 272
 273 class Compare:
 274
 275     def __init__(self, filename):
 276         self.fp = open(filename, 'r')
 277
 278     def write(self, data):
 279         expected = self.fp.read(len(data))
 280         if data <> expected:
 281             raise test_support.TestFailed, \
 282                     'Writing: '+`data`+', expected: '+`expected`
 283
 284     def writelines(self, listoflines):
 285         map(self.write, listoflines)
 286
 287     def flush(self):
 288         pass
 289
 290     def close(self):
 291         leftover = self.fp.read()
 292         if leftover:
 293             raise test_support.TestFailed, 'Unread: '+`leftover`
 294         self.fp.close()
 295
 296     def isatty(self):
 297         return 0
 298
 299 if __name__ == '__main__':
 300     sys.exit(main())