Lib/test/regrtest.py

   1 #! /usr/bin/env python
   2
   3 """Regression test.
   4
   5 This will find all modules whose name is "test_*" in the test
   6 directory, and run them.  Various command line options provide
   7 additional facilities.
   8
   9 Command line options:
  10
  11 -v: verbose -- run tests in verbose mode with output to stdout
  12 -q: quiet -- don't print anything except if a test fails
  13 -g: generate -- write the output file for a test instead of comparing it
  14 -x: exclude -- arguments are tests to *exclude*
  15
  16 If non-option arguments are present, they are names for tests to run,
  17 unless -x is given, in which case they are names for tests not to run.
  18 If no test names are given, all tests are run.
  19
  20 -v is incompatible with -g and does not compare test output files.
  21 """
  22
  23 import sys
  24 import string
  25 import os
  26 import getopt
  27 import traceback
  28
  29 import test_support
  30
  31 def main(tests=None, testdir=None):
  32     """Execute a test suite.
  33
  34     This also parses command-line options and modifies its behaviour
  35     accordingly.
  36
  37     tests -- a list of strings containing test names (optional)
  38     testdir -- the directory in which to look for tests (optional)
  39
  40     Users other than the Python test suite will certainly want to
  41     specify testdir; if it's omitted, the directory containing the
  42     Python test suite is searched for.
  43
  44     If the tests argument is omitted, the tests listed on the
  45     command-line will be used.  If that's empty, too, then all *.py
  46     files beginning with test_ will be used.
  47
  48     """
  49
  50     try:
  51         opts, args = getopt.getopt(sys.argv[1:], 'vgqx')
  52     except getopt.error, msg:
  53         print msg
  54         print __doc__
  55         return 2
  56     verbose = 0
  57     quiet = 0
  58     generate = 0
  59     exclude = 0
  60     for o, a in opts:
  61         if o == '-v': verbose = verbose+1
  62         if o == '-q': quiet = 1; verbose = 0
  63         if o == '-g': generate = 1
  64         if o == '-x': exclude = 1
  65     if generate and verbose:
  66         print "-g and -v don't go together!"
  67         return 2
  68     good = []
  69     bad = []
  70     skipped = []
  71     for i in range(len(args)):
  72         # Strip trailing ".py" from arguments
  73         if args[i][-3:] == '.py':
  74             args[i] = args[i][:-3]
  75     stdtests = STDTESTS[:]
  76     nottests = NOTTESTS[:]
  77     if exclude:
  78         for arg in args:
  79             if arg in stdtests:
  80                 stdtests.remove(arg)
  81         nottests[:0] = args
  82         args = []
  83     tests = tests or args or findtests(testdir, stdtests, nottests)
  84     test_support.verbose = verbose      # Tell tests to be moderately quiet
  85     for test in tests:
  86         if not quiet:
  87             print test
  88         ok = runtest(test, generate, verbose, testdir)
  89         if ok > 0:
  90             good.append(test)
  91         elif ok == 0:
  92             bad.append(test)
  93         else:
  94             if not quiet:
  95                 print "test", test,
  96                 print "skipped -- an optional feature could not be imported"
  97             skipped.append(test)
  98     if good and not quiet:
  99         if not bad and not skipped and len(good) > 1:
 100             print "All",
 101         print count(len(good), "test"), "OK."
 102     if bad:
 103         print count(len(bad), "test"), "failed:",
 104         print string.join(bad)
 105     if skipped and not quiet:
 106         print count(len(skipped), "test"), "skipped:",
 107         print string.join(skipped)
 108     return len(bad) > 0
 109
 110 STDTESTS = [
 111     'test_grammar',
 112     'test_opcodes',
 113     'test_operations',
 114     'test_builtin',
 115     'test_exceptions',
 116     'test_types',
 117    ]
 118
 119 NOTTESTS = [
 120     'test_support',
 121     'test_b1',
 122     'test_b2',
 123     ]
 124
 125 def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS):
 126     """Return a list of all applicable test modules."""
 127     if not testdir: testdir = findtestdir()
 128     names = os.listdir(testdir)
 129     tests = []
 130     for name in names:
 131         if name[:5] == "test_" and name[-3:] == ".py":
 132             modname = name[:-3]
 133             if modname not in stdtests and modname not in nottests:
 134                 tests.append(modname)
 135     tests.sort()
 136     return stdtests + tests
 137
 138 def runtest(test, generate, verbose, testdir = None):
 139     """Run a single test.
 140     test -- the name of the test
 141     generate -- if true, generate output, instead of running the test
 142     and comparing it to a previously created output file
 143     verbose -- if true, print more messages
 144     testdir -- test directory
 145     """
 146     test_support.unload(test)
 147     if not testdir: testdir = findtestdir()
 148     outputdir = os.path.join(testdir, "output")
 149     outputfile = os.path.join(outputdir, test)
 150     try:
 151         if generate:
 152             cfp = open(outputfile, "w")
 153         elif verbose:
 154             cfp = sys.stdout
 155         else:
 156             cfp = Compare(outputfile)
 157     except IOError:
 158         cfp = None
 159         print "Warning: can't open", outputfile
 160     try:
 161         save_stdout = sys.stdout
 162         try:
 163             if cfp:
 164                 sys.stdout = cfp
 165                 print test              # Output file starts with test name
 166             __import__(test, globals(), locals(), [])
 167         finally:
 168             sys.stdout = save_stdout
 169     except ImportError, msg:
 170         return -1
 171     except KeyboardInterrupt, v:
 172         raise KeyboardInterrupt, v, sys.exc_info()[2]
 173     except test_support.TestFailed, msg:
 174         print "test", test, "failed --", msg
 175         return 0
 176     except:
 177         type, value = sys.exc_info()[:2]
 178         print "test", test, "crashed --", type, ":", value
 179         if verbose:
 180             traceback.print_exc(file=sys.stdout)
 181         return 0
 182     else:
 183         return 1
 184
 185 def findtestdir():
 186     if __name__ == '__main__':
 187         file = sys.argv[0]
 188     else:
 189         file = __file__
 190     testdir = os.path.dirname(file) or os.curdir
 191     return testdir
 192
 193 def count(n, word):
 194     if n == 1:
 195         return "%d %s" % (n, word)
 196     else:
 197         return "%d %ss" % (n, word)
 198
 199 class Compare:
 200
 201     def __init__(self, filename):
 202         self.fp = open(filename, 'r')
 203
 204     def write(self, data):
 205         expected = self.fp.read(len(data))
 206         if data <> expected:
 207             raise test_support.TestFailed, \
 208                     'Writing: '+`data`+', expected: '+`expected`
 209
 210     def writelines(self, listoflines):
 211         map(self.write, listoflines)
 212
 213     def flush(self):
 214         pass
 215
 216     def close(self):
 217         leftover = self.fp.read()
 218         if leftover:
 219             raise test_support.TestFailed, 'Unread: '+`leftover`
 220         self.fp.close()
 221
 222     def isatty(self):
 223         return 0
 224
 225 if __name__ == '__main__':
 226     sys.exit(main())