Lib/test/regrtest.py

   1 #! /usr/bin/env python
   2
   3 """Regression test.
   4
   5 This will find all modules whose name is "test_*" in the test
   6 directory, and run them.  Various command line options provide
   7 additional facilities.
   8
   9 Command line options:
  10
  11 -v: verbose   -- run tests in verbose mode with output to stdout
  12 -q: quiet     -- don't print anything except if a test fails
  13 -g: generate  -- write the output file for a test instead of comparing it
  14 -x: exclude   -- arguments are tests to *exclude*
  15 -s: single    -- run only a single test (see below)
  16 -r: random    -- randomize test execution order
  17 -f: fromfile  -- read names of tests to run from a file (see below)
  18 -l: findleaks -- if GC is available detect tests that leak memory
  19 -u: use       -- specify which special resource intensive tests to run
  20 -h: help      -- print this text and exit
  21 -t: threshold -- call gc.set_threshold(N)
  22
  23 If non-option arguments are present, they are names for tests to run,
  24 unless -x is given, in which case they are names for tests not to run.
  25 If no test names are given, all tests are run.
  26
  27 -v is incompatible with -g and does not compare test output files.
  28
  29 -s means to run only a single test and exit.  This is useful when
  30 doing memory analysis on the Python interpreter (which tend to consume
  31 too many resources to run the full regression test non-stop).  The
  32 file /tmp/pynexttest is read to find the next test to run.  If this
  33 file is missing, the first test_*.py file in testdir or on the command
  34 line is used.  (actually tempfile.gettempdir() is used instead of
  35 /tmp).
  36
  37 -f reads the names of tests from the file given as f's argument, one
  38 or more test names per line.  Whitespace is ignored.  Blank lines and
  39 lines beginning with '#' are ignored.  This is especially useful for
  40 whittling down failures involving interactions among tests.
  41
  42 -u is used to specify which special resource intensive tests to run,
  43 such as those requiring large file support or network connectivity.
  44 The argument is a comma-separated list of words indicating the
  45 resources to test.  Currently only the following are defined:
  46
  47     all -       Enable all special resources.
  48
  49     curses -    Tests that use curses and will modify the terminal's
  50                 state and output modes.
  51
  52     largefile - It is okay to run some test that may create huge
  53                 files.  These tests can take a long time and may
  54                 consume >2GB of disk space temporarily.
  55
  56     network -   It is okay to run tests that use external network
  57                 resource, e.g. testing SSL support for sockets.
  58 """
  59
  60 import sys
  61 import os
  62 import getopt
  63 import traceback
  64 import random
  65 import StringIO
  66
  67 import test_support
  68
  69
  70 RESOURCE_NAMES = ('curses', 'largefile', 'network')
  71
  72
  73 def usage(code, msg=''):
  74     print __doc__
  75     if msg: print msg
  76     sys.exit(code)
  77
  78
  79 def main(tests=None, testdir=None, verbose=0, quiet=0, generate=0,
  80          exclude=0, single=0, randomize=0, fromfile=None, findleaks=0,
  81          use_resources=None):
  82     """Execute a test suite.
  83
  84     This also parses command-line options and modifies its behavior
  85     accordingly.
  86
  87     tests -- a list of strings containing test names (optional)
  88     testdir -- the directory in which to look for tests (optional)
  89
  90     Users other than the Python test suite will certainly want to
  91     specify testdir; if it's omitted, the directory containing the
  92     Python test suite is searched for.
  93
  94     If the tests argument is omitted, the tests listed on the
  95     command-line will be used.  If that's empty, too, then all *.py
  96     files beginning with test_ will be used.
  97
  98     The other default arguments (verbose, quiet, generate, exclude,
  99     single, randomize, findleaks, and use_resources) allow programmers
 100     calling main() directly to set the values that would normally be
 101     set by flags on the command line.
 102
 103     """
 104
 105     test_support.record_original_stdout(sys.stdout)
 106     try:
 107         opts, args = getopt.getopt(sys.argv[1:], 'hvgqxsrf:lu:t:',
 108                                    ['help', 'verbose', 'quiet', 'generate',
 109                                     'exclude', 'single', 'random', 'fromfile',
 110                                     'findleaks', 'use=', 'threshold='])
 111     except getopt.error, msg:
 112         usage(2, msg)
 113
 114     # Defaults
 115     if use_resources is None:
 116         use_resources = []
 117     for o, a in opts:
 118         if o in ('-h', '--help'):
 119             usage(0)
 120         elif o in ('-v', '--verbose'):
 121             verbose += 1
 122         elif o in ('-q', '--quiet'):
 123             quiet = 1;
 124             verbose = 0
 125         elif o in ('-g', '--generate'):
 126             generate = 1
 127         elif o in ('-x', '--exclude'):
 128             exclude = 1
 129         elif o in ('-s', '--single'):
 130             single = 1
 131         elif o in ('-r', '--randomize'):
 132             randomize = 1
 133         elif o in ('-f', '--fromfile'):
 134             fromfile = a
 135         elif o in ('-l', '--findleaks'):
 136             findleaks = 1
 137         elif o in ('-t', '--threshold'):
 138             import gc
 139             gc.set_threshold(int(a))
 140         elif o in ('-u', '--use'):
 141             u = [x.lower() for x in a.split(',')]
 142             for r in u:
 143                 if r == 'all':
 144                     use_resources = RESOURCE_NAMES
 145                     break
 146                 if r not in RESOURCE_NAMES:
 147                     usage(1, 'Invalid -u/--use option: ' + a)
 148                 if r not in use_resources:
 149                     use_resources.append(r)
 150     if generate and verbose:
 151         usage(2, "-g and -v don't go together!")
 152     if single and fromfile:
 153         usage(2, "-s and -f don't go together!")
 154
 155     good = []
 156     bad = []
 157     skipped = []
 158
 159     if findleaks:
 160         try:
 161             import gc
 162         except ImportError:
 163             print 'No GC available, disabling findleaks.'
 164             findleaks = 0
 165         else:
 166             # Uncomment the line below to report garbage that is not
 167             # freeable by reference counting alone.  By default only
 168             # garbage that is not collectable by the GC is reported.
 169             #gc.set_debug(gc.DEBUG_SAVEALL)
 170             found_garbage = []
 171
 172     if single:
 173         from tempfile import gettempdir
 174         filename = os.path.join(gettempdir(), 'pynexttest')
 175         try:
 176             fp = open(filename, 'r')
 177             next = fp.read().strip()
 178             tests = [next]
 179             fp.close()
 180         except IOError:
 181             pass
 182
 183     if fromfile:
 184         tests = []
 185         fp = open(fromfile)
 186         for line in fp:
 187             guts = line.split() # assuming no test has whitespace in its name
 188             if guts and not guts[0].startswith('#'):
 189                 tests.extend(guts)
 190         fp.close()
 191
 192     # Strip .py extensions.
 193     if args:
 194         args = map(removepy, args)
 195     if tests:
 196         tests = map(removepy, tests)
 197
 198     stdtests = STDTESTS[:]
 199     nottests = NOTTESTS[:]
 200     if exclude:
 201         for arg in args:
 202             if arg in stdtests:
 203                 stdtests.remove(arg)
 204         nottests[:0] = args
 205         args = []
 206     tests = tests or args or findtests(testdir, stdtests, nottests)
 207     if single:
 208         tests = tests[:1]
 209     if randomize:
 210         random.shuffle(tests)
 211     test_support.verbose = verbose      # Tell tests to be moderately quiet
 212     test_support.use_resources = use_resources
 213     save_modules = sys.modules.keys()
 214     for test in tests:
 215         if not quiet:
 216             print test
 217         ok = runtest(test, generate, verbose, quiet, testdir)
 218         if ok > 0:
 219             good.append(test)
 220         elif ok == 0:
 221             bad.append(test)
 222         else:
 223             skipped.append(test)
 224         if findleaks:
 225             gc.collect()
 226             if gc.garbage:
 227                 print "Warning: test created", len(gc.garbage),
 228                 print "uncollectable object(s)."
 229                 # move the uncollectable objects somewhere so we don't see
 230                 # them again
 231                 found_garbage.extend(gc.garbage)
 232                 del gc.garbage[:]
 233         # Unload the newly imported modules (best effort finalization)
 234         for module in sys.modules.keys():
 235             if module not in save_modules and module.startswith("test."):
 236                 test_support.unload(module)
 237
 238     # The lists won't be sorted if running with -r
 239     good.sort()
 240     bad.sort()
 241     skipped.sort()
 242
 243     if good and not quiet:
 244         if not bad and not skipped and len(good) > 1:
 245             print "All",
 246         print count(len(good), "test"), "OK."
 247         if verbose:
 248             print "CAUTION:  stdout isn't compared in verbose mode:  a test"
 249             print "that passes in verbose mode may fail without it."
 250     if bad:
 251         print count(len(bad), "test"), "failed:"
 252         printlist(bad)
 253     if skipped and not quiet:
 254         print count(len(skipped), "test"), "skipped:"
 255         printlist(skipped)
 256
 257         e = _ExpectedSkips()
 258         plat = sys.platform
 259         if e.isvalid():
 260             surprise = _Set(skipped) - e.getexpected()
 261             if surprise:
 262                 print count(len(surprise), "skip"), \
 263                       "unexpected on", plat + ":"
 264                 printlist(surprise)
 265             else:
 266                 print "Those skips are all expected on", plat + "."
 267         else:
 268             print "Ask someone to teach regrtest.py about which tests are"
 269             print "expected to get skipped on", plat + "."
 270
 271     if single:
 272         alltests = findtests(testdir, stdtests, nottests)
 273         for i in range(len(alltests)):
 274             if tests[0] == alltests[i]:
 275                 if i == len(alltests) - 1:
 276                     os.unlink(filename)
 277                 else:
 278                     fp = open(filename, 'w')
 279                     fp.write(alltests[i+1] + '\n')
 280                     fp.close()
 281                 break
 282         else:
 283             os.unlink(filename)
 284
 285     sys.exit(len(bad) > 0)
 286
 287
 288 STDTESTS = [
 289     'test_grammar',
 290     'test_opcodes',
 291     'test_operations',
 292     'test_builtin',
 293     'test_exceptions',
 294     'test_types',
 295    ]
 296
 297 NOTTESTS = [
 298     'test_support',
 299     'test_b1',
 300     'test_b2',
 301     'test_future1',
 302     'test_future2',
 303     'test_future3',
 304     ]
 305
 306 def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS):
 307     """Return a list of all applicable test modules."""
 308     if not testdir: testdir = findtestdir()
 309     names = os.listdir(testdir)
 310     tests = []
 311     for name in names:
 312         if name[:5] == "test_" and name[-3:] == os.extsep+"py":
 313             modname = name[:-3]
 314             if modname not in stdtests and modname not in nottests:
 315                 tests.append(modname)
 316     tests.sort()
 317     return stdtests + tests
 318
 319 def runtest(test, generate, verbose, quiet, testdir = None):
 320     """Run a single test.
 321     test -- the name of the test
 322     generate -- if true, generate output, instead of running the test
 323     and comparing it to a previously created output file
 324     verbose -- if true, print more messages
 325     quiet -- if true, don't print 'skipped' messages (probably redundant)
 326     testdir -- test directory
 327     """
 328     test_support.unload(test)
 329     if not testdir: testdir = findtestdir()
 330     outputdir = os.path.join(testdir, "output")
 331     outputfile = os.path.join(outputdir, test)
 332     if verbose:
 333         cfp = None
 334     else:
 335         cfp = StringIO.StringIO()
 336     try:
 337         save_stdout = sys.stdout
 338         try:
 339             if cfp:
 340                 sys.stdout = cfp
 341                 print test              # Output file starts with test name
 342             the_module = __import__(test, globals(), locals(), [])
 343             # Most tests run to completion simply as a side-effect of
 344             # being imported.  For the benefit of tests that can't run
 345             # that way (like test_threaded_import), explicitly invoke
 346             # their test_main() function (if it exists).
 347             indirect_test = getattr(the_module, "test_main", None)
 348             if indirect_test is not None:
 349                 indirect_test()
 350         finally:
 351             sys.stdout = save_stdout
 352     except (ImportError, test_support.TestSkipped), msg:
 353         if not quiet:
 354             print "test", test, "skipped --", msg
 355         return -1
 356     except KeyboardInterrupt:
 357         raise
 358     except test_support.TestFailed, msg:
 359         print "test", test, "failed --", msg
 360         return 0
 361     except:
 362         type, value = sys.exc_info()[:2]
 363         print "test", test, "crashed --", str(type) + ":", value
 364         if verbose:
 365             traceback.print_exc(file=sys.stdout)
 366         return 0
 367     else:
 368         if not cfp:
 369             return 1
 370         output = cfp.getvalue()
 371         if generate:
 372             if output == test + "\n":
 373                 if os.path.exists(outputfile):
 374                     # Write it since it already exists (and the contents
 375                     # may have changed), but let the user know it isn't
 376                     # needed:
 377                     print "output file", outputfile, \
 378                           "is no longer needed; consider removing it"
 379                 else:
 380                     # We don't need it, so don't create it.
 381                     return 1
 382             fp = open(outputfile, "w")
 383             fp.write(output)
 384             fp.close()
 385             return 1
 386         if os.path.exists(outputfile):
 387             fp = open(outputfile, "r")
 388             expected = fp.read()
 389             fp.close()
 390         else:
 391             expected = test + "\n"
 392         if output == expected:
 393             return 1
 394         print "test", test, "produced unexpected output:"
 395         reportdiff(expected, output)
 396         return 0
 397
 398 def reportdiff(expected, output):
 399     import difflib
 400     print "*" * 70
 401     a = expected.splitlines(1)
 402     b = output.splitlines(1)
 403     sm = difflib.SequenceMatcher(a=a, b=b)
 404     tuples = sm.get_opcodes()
 405
 406     def pair(x0, x1):
 407         # x0:x1 are 0-based slice indices; convert to 1-based line indices.
 408         x0 += 1
 409         if x0 >= x1:
 410             return "line " + str(x0)
 411         else:
 412             return "lines %d-%d" % (x0, x1)
 413
 414     for op, a0, a1, b0, b1 in tuples:
 415         if op == 'equal':
 416             pass
 417
 418         elif op == 'delete':
 419             print "***", pair(a0, a1), "of expected output missing:"
 420             for line in a[a0:a1]:
 421                 print "-", line,
 422
 423         elif op == 'replace':
 424             print "*** mismatch between", pair(a0, a1), "of expected", \
 425                   "output and", pair(b0, b1), "of actual output:"
 426             for line in difflib.ndiff(a[a0:a1], b[b0:b1]):
 427                 print line,
 428
 429         elif op == 'insert':
 430             print "***", pair(b0, b1), "of actual output doesn't appear", \
 431                   "in expected output after line", str(a1)+":"
 432             for line in b[b0:b1]:
 433                 print "+", line,
 434
 435         else:
 436             print "get_opcodes() returned bad tuple?!?!", (op, a0, a1, b0, b1)
 437
 438     print "*" * 70
 439
 440 def findtestdir():
 441     if __name__ == '__main__':
 442         file = sys.argv[0]
 443     else:
 444         file = __file__
 445     testdir = os.path.dirname(file) or os.curdir
 446     return testdir
 447
 448 def removepy(name):
 449     if name.endswith(os.extsep + "py"):
 450         name = name[:-3]
 451     return name
 452
 453 def count(n, word):
 454     if n == 1:
 455         return "%d %s" % (n, word)
 456     else:
 457         return "%d %ss" % (n, word)
 458
 459 def printlist(x, width=70, indent=4):
 460     """Print the elements of a sequence to stdout.
 461
 462     Optional arg width (default 70) is the maximum line length.
 463     Optional arg indent (default 4) is the number of blanks with which to
 464     begin each line.
 465     """
 466
 467     line = ' ' * indent
 468     for one in map(str, x):
 469         w = len(line) + len(one)
 470         if line[-1:] == ' ':
 471             pad = ''
 472         else:
 473             pad = ' '
 474             w += 1
 475         if w > width:
 476             print line
 477             line = ' ' * indent + one
 478         else:
 479             line += pad + one
 480     if len(line) > indent:
 481         print line
 482
 483 class _Set:
 484     def __init__(self, seq=[]):
 485         data = self.data = {}
 486         for x in seq:
 487             data[x] = 1
 488
 489     def __len__(self):
 490         return len(self.data)
 491
 492     def __sub__(self, other):
 493         "Return set of all elements in self not in other."
 494         result = _Set()
 495         data = result.data = self.data.copy()
 496         for x in other.data:
 497             if x in data:
 498                 del data[x]
 499         return result
 500
 501     def __iter__(self):
 502         return iter(self.data)
 503
 504     def tolist(self, sorted=1):
 505         "Return _Set elements as a list."
 506         data = self.data.keys()
 507         if sorted:
 508             data.sort()
 509         return data
 510
 511 # Map sys.platform to a string containing the basenames of tests
 512 # expected to be skipped on that platform.
 513
 514 _expectations = {
 515     'win32':
 516         """
 517         test_al
 518         test_cd
 519         test_cl
 520         test_commands
 521         test_crypt
 522         test_curses
 523         test_dbm
 524         test_dl
 525         test_email_codecs
 526         test_fcntl
 527         test_fork1
 528         test_gdbm
 529         test_gl
 530         test_grp
 531         test_imgfile
 532         test_largefile
 533         test_linuxaudiodev
 534         test_mhlib
 535         test_mpz
 536         test_nis
 537         test_openpty
 538         test_poll
 539         test_pty
 540         test_pwd
 541         test_resource
 542         test_signal
 543         test_socket_ssl
 544         test_socketserver
 545         test_sunaudiodev
 546         test_timing
 547         """,
 548     'linux2':
 549         """
 550         test_al
 551         test_cd
 552         test_cl
 553         test_curses
 554         test_dl
 555         test_email_codecs
 556         test_gl
 557         test_imgfile
 558         test_largefile
 559         test_nis
 560         test_ntpath
 561         test_socket_ssl
 562         test_socketserver
 563         test_sunaudiodev
 564         test_unicode_file
 565         test_winreg
 566         test_winsound
 567         """,
 568    'mac':
 569         """
 570         test_al
 571         test_bsddb
 572         test_cd
 573         test_cl
 574         test_commands
 575         test_crypt
 576         test_curses
 577         test_dbm
 578         test_dl
 579         test_fcntl
 580         test_fork1
 581         test_gl
 582         test_grp
 583         test_imgfile
 584         test_largefile
 585         test_linuxaudiodev
 586         test_locale
 587         test_mmap
 588         test_nis
 589         test_ntpath
 590         test_openpty
 591         test_poll
 592         test_popen2
 593         test_pty
 594         test_pwd
 595         test_signal
 596         test_socket_ssl
 597         test_socketserver
 598         test_sunaudiodev
 599         test_sundry
 600         test_timing
 601         test_unicode_file
 602         test_winreg
 603         test_winsound
 604         """,
 605     'unixware5':
 606         """
 607         test_al
 608         test_bsddb
 609         test_cd
 610         test_cl
 611         test_dl
 612         test_gl
 613         test_imgfile
 614         test_largefile
 615         test_linuxaudiodev
 616         test_minidom
 617         test_nis
 618         test_ntpath
 619         test_openpty
 620         test_pyexpat
 621         test_sax
 622         test_socketserver
 623         test_sunaudiodev
 624         test_sundry
 625         test_unicode_file
 626         test_winreg
 627         test_winsound
 628         """,
 629     'riscos':
 630         """
 631         test_al
 632         test_asynchat
 633         test_bsddb
 634         test_cd
 635         test_cl
 636         test_commands
 637         test_crypt
 638         test_dbm
 639         test_dl
 640         test_fcntl
 641         test_fork1
 642         test_gdbm
 643         test_gl
 644         test_grp
 645         test_imgfile
 646         test_largefile
 647         test_linuxaudiodev
 648         test_locale
 649         test_mmap
 650         test_nis
 651         test_ntpath
 652         test_openpty
 653         test_poll
 654         test_popen2
 655         test_pty
 656         test_pwd
 657         test_socket_ssl
 658         test_socketserver
 659         test_strop
 660         test_sunaudiodev
 661         test_sundry
 662         test_thread
 663         test_threaded_import
 664         test_threadedtempfile
 665         test_threading
 666         test_timing
 667         test_unicode_file
 668         test_winreg
 669         test_winsound
 670         """,
 671     'darwin':
 672         """
 673         test_al
 674         test_cd
 675         test_cl
 676         test_curses
 677         test_dl
 678         test_gdbm
 679         test_gl
 680         test_imgfile
 681         test_largefile
 682         test_linuxaudiodev
 683         test_minidom
 684         test_nis
 685         test_ntpath
 686         test_poll
 687         test_socket_ssl
 688         test_socketserver
 689         test_sunaudiodev
 690         test_unicode_file
 691         test_winreg
 692         test_winsound
 693         """,
 694     'hp-ux11':
 695         """
 696         test_al
 697         test_bsddb
 698         test_cd
 699         test_cl
 700         test_curses
 701         test_dl
 702         test_gdbm
 703         test_gl
 704         test_gzip
 705         test_imgfile
 706         test_largefile
 707         test_linuxaudiodev
 708         test_locale
 709         test_minidom
 710         test_nis
 711         test_ntpath
 712         test_openpty
 713         test_pyexpat
 714         test_sax
 715         test_socket_ssl
 716         test_socketserver
 717         test_sunaudiodev
 718         test_unicode_file
 719         test_winreg
 720         test_winsound
 721         test_zipfile
 722         test_zlib
 723         """,
 724     'atheos':
 725         """
 726         test_al
 727         test_cd
 728         test_cl
 729         test_curses
 730         test_dl
 731         test_email_codecs
 732         test_gdbm
 733         test_gl
 734         test_imgfile
 735         test_largefile
 736         test_linuxaudiodev
 737         test_locale
 738         test_mhlib
 739         test_mmap
 740         test_mpz
 741         test_nis
 742         test_poll
 743         test_popen2
 744         test_resource
 745         test_socket_ssl
 746         test_socketserver
 747         test_sunaudiodev
 748         test_unicode_file
 749         test_winreg
 750         test_winsound
 751         """,
 752 }
 753
 754 class _ExpectedSkips:
 755     def __init__(self):
 756         self.valid = 0
 757         if sys.platform in _expectations:
 758             s = _expectations[sys.platform]
 759             self.expected = _Set(s.split())
 760             self.valid = 1
 761
 762     def isvalid(self):
 763         "Return true iff _ExpectedSkips knows about the current platform."
 764         return self.valid
 765
 766     def getexpected(self):
 767         """Return set of test names we expect to skip on current platform.
 768
 769         self.isvalid() must be true.
 770         """
 771
 772         assert self.isvalid()
 773         return self.expected
 774
 775 if __name__ == '__main__':
 776     main()