build/run_tests.py

   1 #!/usr/bin/env python
   2 #
   3 # run_tests.py - run the tests in the regression test suite.
   4 #
   5
   6 '''usage: python run_tests.py [--url=<base-url>] [--fs-type=<fs-type>]
   7                     [--verbose] [--cleanup] [--enable-sasl] [--parallel]
   8                     [--http-library=<http-library>]
   9                     [--server-minor-version=<version>] <abs_srcdir> <abs_builddir>
  10                     <prog ...>
  11
  12 The optional base-url, fs-type, http-library, server-minor-version,
  13 verbose, parallel, enable-sasl, and cleanup options, and the first two
  14 parameters are passed unchanged to the TestHarness constructor.  All
  15 other parameters are names of test programs.
  16 '''
  17
  18 import os, sys
  19
  20 import getopt
  21 try:
  22   my_getopt = getopt.gnu_getopt
  23 except AttributeError:
  24   my_getopt = getopt.getopt
  25
  26 class TestHarness:
  27   '''Test harness for Subversion tests.
  28   '''
  29
  30   def __init__(self, abs_srcdir, abs_builddir, logfile,
  31                base_url=None, fs_type=None, http_library=None,
  32                server_minor_version=None, verbose=None,
  33                cleanup=None, enable_sasl=None, parallel=None, list_tests=None,
  34                svn_bin=None):
  35     '''Construct a TestHarness instance.
  36
  37     ABS_SRCDIR and ABS_BUILDDIR are the source and build directories.
  38     LOGFILE is the name of the log file.
  39     BASE_URL is the base url for DAV tests.
  40     FS_TYPE is the FS type for repository creation.
  41     HTTP_LIBRARY is the HTTP library for DAV-based communications.
  42     SERVER_MINOR_VERSION is the minor version of the server being tested.
  43     SVN_BIN is the path where the svn binaries are installed.
  44     '''
  45     self.srcdir = abs_srcdir
  46     self.builddir = abs_builddir
  47     self.logfile = logfile
  48     self.base_url = base_url
  49     self.fs_type = fs_type
  50     self.http_library = http_library
  51     self.server_minor_version = server_minor_version
  52     self.verbose = verbose
  53     self.cleanup = cleanup
  54     self.enable_sasl = enable_sasl
  55     self.parallel = parallel
  56     self.list_tests = list_tests
  57     self.svn_bin = svn_bin
  58     self.log = None
  59
  60   def run(self, list):
  61     'Run all test programs given in LIST.'
  62     self._open_log('w')
  63     failed = 0
  64     cnt = 0
  65     for prog in list:
  66       failed = self._run_test(prog, cnt, len(list)) or failed
  67       cnt += 1
  68     self._open_log('r')
  69     log_lines = self.log.readlines()
  70     skipped = filter(lambda x: x[:6] == 'SKIP: ', log_lines)
  71     if skipped:
  72       print 'At least one test was SKIPPED, checking ' + self.logfile
  73       map(sys.stdout.write, skipped)
  74     if failed:
  75       print 'At least one test FAILED, checking ' + self.logfile
  76       map(sys.stdout.write, filter(lambda x: x[:6] in ('FAIL: ', 'XPASS:'),
  77                                    log_lines))
  78     self._close_log()
  79     return failed
  80
  81   def _open_log(self, mode):
  82     'Open the log file with the required MODE.'
  83     self._close_log()
  84     self.log = open(self.logfile, mode)
  85
  86   def _close_log(self):
  87     'Close the log file.'
  88     if not self.log is None:
  89       self.log.close()
  90       self.log = None
  91
  92   def _run_test(self, prog, test_nr, total_tests):
  93     'Run a single test.'
  94
  95     def quote(arg):
  96       if sys.platform == 'win32':
  97         return '"' + arg + '"'
  98       else:
  99         return arg
 100
 101     progdir, progbase = os.path.split(prog)
 102     # Using write here because we don't want even a trailing space
 103     sys.stdout.write('Running all tests in %s [%d/%d]...' % (
 104       progbase, test_nr + 1, total_tests))
 105     print >> self.log, 'START: ' + progbase
 106
 107     if progbase[-3:] == '.py':
 108       progname = sys.executable
 109       cmdline = [quote(progname),
 110                  quote(os.path.join(self.srcdir, prog))]
 111       if self.base_url is not None:
 112         cmdline.append(quote('--url=' + self.base_url))
 113       if self.enable_sasl is not None:
 114         cmdline.append('--enable-sasl')
 115       if self.parallel is not None:
 116         cmdline.append('--parallel')
 117     elif os.access(prog, os.X_OK):
 118       progname = './' + progbase
 119       cmdline = [quote(progname),
 120                  quote('--srcdir=' + os.path.join(self.srcdir, progdir))]
 121     else:
 122       print 'Don\'t know what to do about ' + progbase
 123       sys.exit(1)
 124
 125     if self.verbose is not None:
 126       cmdline.append('--verbose')
 127     if self.cleanup is not None:
 128       cmdline.append('--cleanup')
 129     if self.fs_type is not None:
 130       cmdline.append(quote('--fs-type=' + self.fs_type))
 131     if self.http_library is not None:
 132       cmdline.append(quote('--http-library=' + self.http_library))
 133     if self.server_minor_version is not None:
 134       cmdline.append(quote('--server-minor-version=' + self.server_minor_version))
 135     if self.list_tests is not None:
 136       cmdline.append('--list')
 137     if self.svn_bin is not None:
 138       cmdline.append(quote('--bin=' + self.svn_bin))
 139
 140     old_cwd = os.getcwd()
 141     try:
 142       os.chdir(progdir)
 143       failed = self._run_prog(progname, cmdline)
 144     except:
 145       os.chdir(old_cwd)
 146       raise
 147     else:
 148       os.chdir(old_cwd)
 149
 150     # We always return 1 for failed tests, if some other failure than 1
 151     # probably means the test didn't run at all and probably didn't
 152     # output any failure info.
 153     if failed == 1:
 154       print 'FAILURE'
 155     elif failed:
 156       print >> self.log, 'FAIL:  ' + progbase + ': Unknown test failure see tests.log.\n'
 157       print 'FAILURE'
 158     else:
 159       print 'success'
 160     print >> self.log, 'END: ' + progbase + '\n'
 161     return failed
 162
 163   def _run_prog(self, progname, cmdline):
 164     'Execute COMMAND, redirecting standard output and error to the log file.'
 165     def restore_streams(stdout, stderr):
 166       os.dup2(stdout, 1)
 167       os.dup2(stderr, 2)
 168       os.close(stdout)
 169       os.close(stderr)
 170
 171     sys.stdout.flush()
 172     sys.stderr.flush()
 173     self.log.flush()
 174     old_stdout = os.dup(1)
 175     old_stderr = os.dup(2)
 176     try:
 177       os.dup2(self.log.fileno(), 1)
 178       os.dup2(self.log.fileno(), 2)
 179       rv = os.spawnv(os.P_WAIT, progname, cmdline)
 180     except:
 181       restore_streams(old_stdout, old_stderr)
 182       raise
 183     else:
 184       restore_streams(old_stdout, old_stderr)
 185       return rv
 186
 187
 188 def main():
 189   try:
 190     opts, args = my_getopt(sys.argv[1:], 'u:f:vc',
 191                            ['url=', 'fs-type=', 'verbose', 'cleanup',
 192                             'http-library=', 'server-minor-version=',
 193                             'enable-sasl', 'parallel'])
 194   except getopt.GetoptError:
 195     args = []
 196
 197   if len(args) < 3:
 198     print __doc__
 199     sys.exit(2)
 200
 201   base_url, fs_type, verbose, cleanup, enable_sasl, http_library, \
 202     server_minor_version, parallel = \
 203             None, None, None, None, None, None, None, None
 204   for opt, val in opts:
 205     if opt in ['-u', '--url']:
 206       base_url = val
 207     elif opt in ['-f', '--fs-type']:
 208       fs_type = val
 209     elif opt in ['--http-library']:
 210       http_library = val
 211     elif opt in ['--server-minor-version']:
 212       server_minor_version = val
 213     elif opt in ['-v', '--verbose']:
 214       verbose = 1
 215     elif opt in ['-c', '--cleanup']:
 216       cleanup = 1
 217     elif opt in ['--enable-sasl']:
 218       enable_sasl = 1
 219     elif opt in ['--parallel']:
 220       parallel = 1
 221     else:
 222       raise getopt.GetoptError
 223
 224   th = TestHarness(args[0], args[1],
 225                    os.path.abspath('tests.log'),
 226                    base_url, fs_type, http_library, server_minor_version,
 227                    verbose, cleanup, enable_sasl, parallel)
 228
 229   failed = th.run(args[2:])
 230   if failed:
 231     sys.exit(1)
 232
 233
 234 # Run main if not imported as a module
 235 if __name__ == '__main__':
 236   main()