Lib/test/regrtest.py

   1 #! /usr/bin/env python
   2
   3 """Regression test.
   4
   5 This will find all modules whose name is "test_*" in the test
   6 directory, and run them.  Various command line options provide
   7 additional facilities.
   8
   9 Command line options:
  10
  11 -v: verbose -- run tests in verbose mode with output to stdout
  12 -q: quiet -- don't print anything except if a test fails
  13 -g: generate -- write the output file for a test instead of comparing it
  14 -x: exclude -- arguments are tests to *exclude*
  15
  16 If non-option arguments are present, they are names for tests to run,
  17 unless -x is given, in which case they are names for tests not to run.
  18 If no test names are given, all tests are run.
  19
  20 -v is incompatible with -g and does not compare test output files.
  21 """
  22
  23 import sys
  24 import string
  25 import os
  26 import getopt
  27 import traceback
  28
  29 import test_support
  30
  31 def main(tests=None, testdir=None):
  32     """Execute a test suite.
  33
  34     This also parses command-line options and modifies its behaviour
  35     accordingly.
  36
  37     tests -- a list of strings containing test names (optional)
  38     testdir -- the directory in which to look for tests (optional)
  39
  40     Users other than the Python test suite will certainly want to
  41     specify testdir; if it's omitted, the directory containing the
  42     Python test suite is searched for.
  43
  44     If the tests argument is omitted, the tests listed on the
  45     command-line will be used.  If that's empty, too, then all *.py
  46     files beginning with test_ will be used.
  47
  48     """
  49
  50     try:
  51         opts, args = getopt.getopt(sys.argv[1:], 'vgqx')
  52     except getopt.error, msg:
  53         print msg
  54         print __doc__
  55         return 2
  56     verbose = 0
  57     quiet = 0
  58     generate = 0
  59     exclude = 0
  60     for o, a in opts:
  61         if o == '-v': verbose = verbose+1
  62         if o == '-q': quiet = 1; verbose = 0
  63         if o == '-g': generate = 1
  64         if o == '-x': exclude = 1
  65     if generate and verbose:
  66         print "-g and -v don't go together!"
  67         return 2
  68     good = []
  69     bad = []
  70     skipped = []
  71     for i in range(len(args)):
  72         # Strip trailing ".py" from arguments
  73         if args[i][-3:] == '.py':
  74             args[i] = args[i][:-3]
  75     if exclude:
  76         nottests[:0] = args
  77         args = []
  78     tests = tests or args or findtests()
  79     test_support.verbose = verbose      # Tell tests to be moderately quiet
  80     for test in tests:
  81         if not quiet:
  82             print test
  83         ok = runtest(test, generate, verbose, testdir)
  84         if ok > 0:
  85             good.append(test)
  86         elif ok == 0:
  87             bad.append(test)
  88         else:
  89             if not quiet:
  90                 print "test", test,
  91                 print "skipped -- an optional feature could not be imported"
  92             skipped.append(test)
  93     if good and not quiet:
  94         if not bad and not skipped and len(good) > 1:
  95             print "All",
  96         print count(len(good), "test"), "OK."
  97     if bad:
  98         print count(len(bad), "test"), "failed:",
  99         print string.join(bad)
 100     if skipped and not quiet:
 101         print count(len(skipped), "test"), "skipped:",
 102         print string.join(skipped)
 103     return len(bad) > 0
 104
 105 STDTESTS = [
 106     'test_grammar',
 107     'test_opcodes',
 108     'test_operations',
 109     'test_builtin',
 110     'test_exceptions',
 111     'test_types',
 112    ]
 113
 114 NOTTESTS = [
 115     'test_support',
 116     'test_b1',
 117     'test_b2',
 118     ]
 119
 120 def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS):
 121     """Return a list of all applicable test modules."""
 122     if not testdir: testdir = findtestdir()
 123     names = os.listdir(testdir)
 124     tests = []
 125     for name in names:
 126         if name[:5] == "test_" and name[-3:] == ".py":
 127             modname = name[:-3]
 128             if modname not in stdtests and modname not in nottests:
 129                 tests.append(modname)
 130     tests.sort()
 131     return stdtests + tests
 132
 133 def runtest(test, generate, verbose, testdir = None):
 134     """Run a single test.
 135     test -- the name of the test
 136     generate -- if true, generate output, instead of running the test
 137     and comparing it to a previously created output file
 138     verbose -- if true, print more messages
 139     testdir -- test directory
 140     """
 141     test_support.unload(test)
 142     if not testdir: testdir = findtestdir()
 143     outputdir = os.path.join(testdir, "output")
 144     outputfile = os.path.join(outputdir, test)
 145     try:
 146         if generate:
 147             cfp = open(outputfile, "w")
 148         elif verbose:
 149             cfp = sys.stdout
 150         else:
 151             cfp = Compare(outputfile)
 152     except IOError:
 153         cfp = None
 154         print "Warning: can't open", outputfile
 155     try:
 156         save_stdout = sys.stdout
 157         try:
 158             if cfp:
 159                 sys.stdout = cfp
 160                 print test              # Output file starts with test name
 161             __import__(test, globals(), locals(), [])
 162         finally:
 163             sys.stdout = save_stdout
 164     except ImportError, msg:
 165         return -1
 166     except KeyboardInterrupt, v:
 167         raise KeyboardInterrupt, v, sys.exc_info()[2]
 168     except test_support.TestFailed, msg:
 169         print "test", test, "failed --", msg
 170         return 0
 171     except:
 172         type, value = sys.exc_info()[:2]
 173         print "test", test, "crashed --", type, ":", value
 174         if verbose:
 175             traceback.print_exc(file=sys.stdout)
 176         return 0
 177     else:
 178         return 1
 179
 180 def findtestdir():
 181     if __name__ == '__main__':
 182         file = sys.argv[0]
 183     else:
 184         file = __file__
 185     testdir = os.path.dirname(file) or os.curdir
 186     return testdir
 187
 188 def count(n, word):
 189     if n == 1:
 190         return "%d %s" % (n, word)
 191     else:
 192         return "%d %ss" % (n, word)
 193
 194 class Compare:
 195
 196     def __init__(self, filename):
 197         self.fp = open(filename, 'r')
 198
 199     def write(self, data):
 200         expected = self.fp.read(len(data))
 201         if data <> expected:
 202             raise test_support.TestFailed, \
 203                     'Writing: '+`data`+', expected: '+`expected`
 204
 205     def writelines(self, listoflines):
 206         map(self.write, listoflines)
 207
 208     def flush(self):
 209         pass
 210
 211     def close(self):
 212         leftover = self.fp.read()
 213         if leftover:
 214             raise test_support.TestFailed, 'Unread: '+`leftover`
 215         self.fp.close()
 216
 217     def isatty(self):
 218         return 0
 219
 220 if __name__ == '__main__':
 221     sys.exit(main())