Lib/test/regrtest.py

   1 #! /usr/bin/env python
   2
   3 """Regression test.
   4
   5 This will find all modules whose name is "test_*" in the test
   6 directory, and run them.  Various command line options provide
   7 additional facilities.
   8
   9 Command line options:
  10
  11 -v: verbose  -- run tests in verbose mode with output to stdout
  12 -q: quiet    -- don't print anything except if a test fails
  13 -g: generate -- write the output file for a test instead of comparing it
  14 -x: exclude  -- arguments are tests to *exclude*
  15 -s: single   -- run only a single test (see below)
  16
  17 If non-option arguments are present, they are names for tests to run,
  18 unless -x is given, in which case they are names for tests not to run.
  19 If no test names are given, all tests are run.
  20
  21 -v is incompatible with -g and does not compare test output files.
  22
  23 -s means to run only a single test and exit.  This is useful when Purifying
  24 the Python interpreter.  The file /tmp/pynexttest is read to find the next
  25 test to run.  If this file is missing, the first test_*.py file in testdir or
  26 on the command line is used.  (actually tempfile.gettempdir() is used instead
  27 of /tmp).
  28
  29 """
  30
  31 import sys
  32 import string
  33 import os
  34 import getopt
  35 import traceback
  36
  37 import test_support
  38
  39 def main(tests=None, testdir=None):
  40     """Execute a test suite.
  41
  42     This also parses command-line options and modifies its behaviour
  43     accordingly.
  44
  45     tests -- a list of strings containing test names (optional)
  46     testdir -- the directory in which to look for tests (optional)
  47
  48     Users other than the Python test suite will certainly want to
  49     specify testdir; if it's omitted, the directory containing the
  50     Python test suite is searched for.
  51
  52     If the tests argument is omitted, the tests listed on the
  53     command-line will be used.  If that's empty, too, then all *.py
  54     files beginning with test_ will be used.
  55
  56     """
  57
  58     try:
  59         opts, args = getopt.getopt(sys.argv[1:], 'vgqxs')
  60     except getopt.error, msg:
  61         print msg
  62         print __doc__
  63         return 2
  64     verbose = 0
  65     quiet = 0
  66     generate = 0
  67     exclude = 0
  68     single = 0
  69     for o, a in opts:
  70         if o == '-v': verbose = verbose+1
  71         if o == '-q': quiet = 1; verbose = 0
  72         if o == '-g': generate = 1
  73         if o == '-x': exclude = 1
  74         if o == '-s': single = 1
  75     if generate and verbose:
  76         print "-g and -v don't go together!"
  77         return 2
  78     good = []
  79     bad = []
  80     skipped = []
  81
  82     if single:
  83         from tempfile import gettempdir
  84         filename = os.path.join(gettempdir(), 'pynexttest')
  85         try:
  86             fp = open(filename, 'r')
  87             next = string.strip(fp.read())
  88             tests = [next]
  89             fp.close()
  90         except IOError:
  91             pass
  92     for i in range(len(args)):
  93         # Strip trailing ".py" from arguments
  94         if args[i][-3:] == '.py':
  95             args[i] = args[i][:-3]
  96     stdtests = STDTESTS[:]
  97     nottests = NOTTESTS[:]
  98     if exclude:
  99         for arg in args:
 100             if arg in stdtests:
 101                 stdtests.remove(arg)
 102         nottests[:0] = args
 103         args = []
 104     tests = tests or args or findtests(testdir, stdtests, nottests)
 105     if single:
 106         tests = tests[:1]
 107     test_support.verbose = verbose      # Tell tests to be moderately quiet
 108     for test in tests:
 109         if not quiet:
 110             print test
 111         ok = runtest(test, generate, verbose, testdir)
 112         if ok > 0:
 113             good.append(test)
 114         elif ok == 0:
 115             bad.append(test)
 116         else:
 117             if not quiet:
 118                 print "test", test,
 119                 print "skipped -- an optional feature could not be imported"
 120             skipped.append(test)
 121     if good and not quiet:
 122         if not bad and not skipped and len(good) > 1:
 123             print "All",
 124         print count(len(good), "test"), "OK."
 125     if bad:
 126         print count(len(bad), "test"), "failed:",
 127         print string.join(bad)
 128     if skipped and not quiet:
 129         print count(len(skipped), "test"), "skipped:",
 130         print string.join(skipped)
 131
 132     if single:
 133         alltests = findtests(testdir, stdtests, nottests)
 134         for i in range(len(alltests)):
 135             if tests[0] == alltests[i]:
 136                 if i == len(alltests) - 1:
 137                     os.unlink(filename)
 138                 else:
 139                     fp = open(filename, 'w')
 140                     fp.write(alltests[i+1] + '\n')
 141                     fp.close()
 142                 break
 143         else:
 144             os.unlink(filename)
 145
 146     return len(bad) > 0
 147
 148 STDTESTS = [
 149     'test_grammar',
 150     'test_opcodes',
 151     'test_operations',
 152     'test_builtin',
 153     'test_exceptions',
 154     'test_types',
 155    ]
 156
 157 NOTTESTS = [
 158     'test_support',
 159     'test_b1',
 160     'test_b2',
 161     ]
 162
 163 def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS):
 164     """Return a list of all applicable test modules."""
 165     if not testdir: testdir = findtestdir()
 166     names = os.listdir(testdir)
 167     tests = []
 168     for name in names:
 169         if name[:5] == "test_" and name[-3:] == ".py":
 170             modname = name[:-3]
 171             if modname not in stdtests and modname not in nottests:
 172                 tests.append(modname)
 173     tests.sort()
 174     return stdtests + tests
 175
 176 def runtest(test, generate, verbose, testdir = None):
 177     """Run a single test.
 178     test -- the name of the test
 179     generate -- if true, generate output, instead of running the test
 180     and comparing it to a previously created output file
 181     verbose -- if true, print more messages
 182     testdir -- test directory
 183     """
 184     test_support.unload(test)
 185     if not testdir: testdir = findtestdir()
 186     outputdir = os.path.join(testdir, "output")
 187     outputfile = os.path.join(outputdir, test)
 188     try:
 189         if generate:
 190             cfp = open(outputfile, "w")
 191         elif verbose:
 192             cfp = sys.stdout
 193         else:
 194             cfp = Compare(outputfile)
 195     except IOError:
 196         cfp = None
 197         print "Warning: can't open", outputfile
 198     try:
 199         save_stdout = sys.stdout
 200         try:
 201             if cfp:
 202                 sys.stdout = cfp
 203                 print test              # Output file starts with test name
 204             __import__(test, globals(), locals(), [])
 205         finally:
 206             sys.stdout = save_stdout
 207     except ImportError, msg:
 208         return -1
 209     except KeyboardInterrupt, v:
 210         raise KeyboardInterrupt, v, sys.exc_info()[2]
 211     except test_support.TestFailed, msg:
 212         print "test", test, "failed --", msg
 213         return 0
 214     except:
 215         type, value = sys.exc_info()[:2]
 216         print "test", test, "crashed --", type, ":", value
 217         if verbose:
 218             traceback.print_exc(file=sys.stdout)
 219         return 0
 220     else:
 221         return 1
 222
 223 def findtestdir():
 224     if __name__ == '__main__':
 225         file = sys.argv[0]
 226     else:
 227         file = __file__
 228     testdir = os.path.dirname(file) or os.curdir
 229     return testdir
 230
 231 def count(n, word):
 232     if n == 1:
 233         return "%d %s" % (n, word)
 234     else:
 235         return "%d %ss" % (n, word)
 236
 237 class Compare:
 238
 239     def __init__(self, filename):
 240         self.fp = open(filename, 'r')
 241
 242     def write(self, data):
 243         expected = self.fp.read(len(data))
 244         if data <> expected:
 245             raise test_support.TestFailed, \
 246                     'Writing: '+`data`+', expected: '+`expected`
 247
 248     def writelines(self, listoflines):
 249         map(self.write, listoflines)
 250
 251     def flush(self):
 252         pass
 253
 254     def close(self):
 255         leftover = self.fp.read()
 256         if leftover:
 257             raise test_support.TestFailed, 'Unread: '+`leftover`
 258         self.fp.close()
 259
 260     def isatty(self):
 261         return 0
 262
 263 if __name__ == '__main__':
 264     sys.exit(main())