win-tests.py

   1 """
   2 Driver for running the tests on Windows.
   3
   4 For a list of options, run this script with the --help option.
   5 """
   6
   7 # $HeadURL$
   8 # $LastChangedDate$
   9 # $LastChangedBy$
  10 # $LastChangedRevision$
  11
  12 import os, sys
  13 import filecmp
  14 import shutil
  15 import traceback
  16 import ConfigParser
  17 import string
  18 import random
  19
  20 import getopt
  21 try:
  22     my_getopt = getopt.gnu_getopt
  23 except AttributeError:
  24     my_getopt = getopt.getopt
  25
  26 def _usage_exit():
  27   "print usage, exit the script"
  28
  29   print "Driver for running the tests on Windows."
  30   print "Usage: python win-tests.py [option] [test-path]"
  31   print
  32   print "Valid options:"
  33   print "  -r, --release          : test the Release configuration"
  34   print "  -d, --debug            : test the Debug configuration (default)"
  35   print "  --bin=PATH             : use the svn binaries installed in PATH"
  36   print "  -u URL, --url=URL      : run ra_dav or ra_svn tests against URL;"
  37   print "                           will start svnserve for ra_svn tests"
  38   print "  -v, --verbose          : talk more"
  39   print "  -f, --fs-type=type     : filesystem type to use (fsfs is default)"
  40   print "  -c, --cleanup          : cleanup after running a test"
  41
  42   print "  --svnserve-args=list   : comma-separated list of arguments for"
  43   print "                           svnserve"
  44   print "                           default is '-d,-r,<test-path-root>'"
  45   print "  --asp.net-hack         : use '_svn' instead of '.svn' for the admin"
  46   print "                           dir name"
  47   print "  --httpd-dir            : location where Apache HTTPD is installed"
  48   print "  --httpd-port           : port for Apache HTTPD; random port number"
  49   print "                           will be used, if not specified"
  50   print "  --http-library         : dav library to use, neon (default) or serf"
  51   print "  --list                 : print test doc strings only"
  52   print "  --enable-sasl          : enable Cyrus SASL authentication for"
  53   print "                           svnserve"
  54   print "  -p, --parallel         : run multiple tests in parallel"
  55   print "  --server-minor-version : the minor version of the server being"
  56   print "                           tested"
  57
  58   sys.exit(0)
  59
  60 CMDLINE_TEST_SCRIPT_PATH = 'subversion/tests/cmdline/'
  61 CMDLINE_TEST_SCRIPT_NATIVE_PATH = CMDLINE_TEST_SCRIPT_PATH.replace('/', os.sep)
  62
  63 sys.path.insert(0, os.path.join('build', 'generator'))
  64 sys.path.insert(1, 'build')
  65
  66 import gen_win
  67 version_header = os.path.join('subversion', 'include', 'svn_version.h')
  68 cp = ConfigParser.ConfigParser()
  69 cp.read('gen-make.opts')
  70 gen_obj = gen_win.GeneratorBase('build.conf', version_header,
  71                                 cp.items('options'))
  72 all_tests = gen_obj.test_progs + gen_obj.bdb_test_progs \
  73           + gen_obj.scripts + gen_obj.bdb_scripts
  74 client_tests = filter(lambda x: x.startswith(CMDLINE_TEST_SCRIPT_PATH),
  75                       all_tests)
  76
  77 svn_dlls = []
  78 for section in gen_obj.sections.values():
  79   if section.options.get("msvc-export"):
  80     dll_basename = section.name + "-" + str(gen_obj.version) + ".dll"
  81     svn_dlls.append(os.path.join("subversion", section.name, dll_basename))
  82
  83 opts, args = my_getopt(sys.argv[1:], 'hrdvcpu:f:',
  84                        ['release', 'debug', 'verbose', 'cleanup', 'url=',
  85                         'svnserve-args=', 'fs-type=', 'asp.net-hack',
  86                         'httpd-dir=', 'httpd-port=', 'http-library=', 'help',
  87                         'list', 'enable-sasl', 'bin=', 'parallel'])
  88 if len(args) > 1:
  89   print 'Warning: non-option arguments after the first one will be ignored'
  90
  91 # Interpret the options and set parameters
  92 base_url, fs_type, verbose, cleanup = None, None, None, None
  93 repo_loc = 'local repository.'
  94 objdir = 'Debug'
  95 log = 'tests.log'
  96 run_svnserve = None
  97 svnserve_args = None
  98 run_httpd = None
  99 httpd_port = None
 100 http_library = 'neon'
 101 list_tests = None
 102 enable_sasl = None
 103 svn_bin = None
 104 parallel = None
 105 server_minor_version = None
 106
 107 for opt, val in opts:
 108   if opt in ('-h', '--help'):
 109     _usage_exit()
 110   elif opt in ('-u', '--url'):
 111     base_url = val
 112   elif opt in ('-f', '--fs-type'):
 113     fs_type = val
 114   elif opt in ('-v', '--verbose'):
 115     verbose = 1
 116   elif opt in ('-c', '--cleanup'):
 117     cleanup = 1
 118   elif opt in ['-r', '--release']:
 119     objdir = 'Release'
 120   elif opt in ['-d', '--debug']:
 121     objdir = 'Debug'
 122   elif opt == '--svnserve-args':
 123     svnserve_args = val.split(',')
 124     run_svnserve = 1
 125   elif opt == '--asp.net-hack':
 126     os.environ['SVN_ASP_DOT_NET_HACK'] = opt
 127   elif opt == '--httpd-dir':
 128     abs_httpd_dir = os.path.abspath(val)
 129     run_httpd = 1
 130   elif opt == '--httpd-port':
 131     httpd_port = int(val)
 132   elif opt == '--http-library':
 133     http_library = val
 134   elif opt == '--list':
 135     list_tests = 1
 136   elif opt == '--enable-sasl':
 137     enable_sasl = 1
 138     base_url = "svn://localhost/"
 139   elif opt == '--server-minor-version':
 140     server_minor_version = val
 141   elif opt == '--bin':
 142     svn_bin = val
 143   elif opt in ('-p', '--parallel'):
 144     parallel = 1
 145
 146 # Calculate the source and test directory names
 147 abs_srcdir = os.path.abspath("")
 148 abs_objdir = os.path.join(abs_srcdir, objdir)
 149 if len(args) == 0:
 150   abs_builddir = abs_objdir
 151   create_dirs = 0
 152 else:
 153   abs_builddir = os.path.abspath(args[0])
 154   create_dirs = 1
 155
 156 # Default to fsfs explicitly
 157 if not fs_type:
 158   fs_type = 'fsfs'
 159
 160 # Don't run bdb tests if they want to test fsfs
 161 if fs_type == 'fsfs':
 162   all_tests = gen_obj.test_progs + gen_obj.scripts
 163
 164 if run_httpd:
 165   if not httpd_port:
 166     httpd_port = random.randrange(1024, 30000)
 167   if not base_url:
 168     base_url = 'http://localhost:' + str(httpd_port)
 169
 170 if base_url:
 171   all_tests = client_tests
 172   repo_loc = 'remote repository ' + base_url + '.'
 173   if base_url[:4] == 'http':
 174     log = 'dav-tests.log'
 175   elif base_url[:3] == 'svn':
 176     log = 'svn-tests.log'
 177     run_svnserve = 1
 178   else:
 179     # Don't know this scheme, but who're we to judge whether it's
 180     # correct or not?
 181     log = 'url-tests.log'
 182
 183 # Have to move the executables where the tests expect them to be
 184 copied_execs = []   # Store copied exec files to avoid the final dir scan
 185
 186 def create_target_dir(dirname):
 187   tgt_dir = os.path.join(abs_builddir, dirname)
 188   if not os.path.exists(tgt_dir):
 189     if verbose:
 190       print "mkdir:", tgt_dir
 191     os.makedirs(tgt_dir)
 192
 193 def copy_changed_file(src, tgt):
 194   if not os.path.isfile(src):
 195     print 'Could not find ' + src
 196     sys.exit(1)
 197   if os.path.isdir(tgt):
 198     tgt = os.path.join(tgt, os.path.basename(src))
 199   if os.path.exists(tgt):
 200     assert os.path.isfile(tgt)
 201     if filecmp.cmp(src, tgt):
 202       if verbose:
 203         print "same:", src
 204         print " and:", tgt
 205       return 0
 206   if verbose:
 207     print "copy:", src
 208     print "  to:", tgt
 209   shutil.copy(src, tgt)
 210   return 1
 211
 212 def copy_execs(baton, dirname, names):
 213   copied_execs = baton
 214   for name in names:
 215     ext = os.path.splitext(name)[1]
 216     if ext != ".exe":
 217       continue
 218     src = os.path.join(dirname, name)
 219     tgt = os.path.join(abs_builddir, dirname, name)
 220     create_target_dir(dirname)
 221     if copy_changed_file(src, tgt):
 222       copied_execs.append(tgt)
 223
 224 def locate_libs():
 225   "Move DLLs to a known location and set env vars"
 226
 227   dlls = []
 228
 229   # look for APR 1.x dll's and use those if found
 230   apr_test_path = os.path.join(gen_obj.apr_path, objdir, 'libapr-1.dll')
 231   if os.path.exists(apr_test_path):
 232     suffix = "-1"
 233   else:
 234     suffix = ""
 235   dlls.append(os.path.join(gen_obj.apr_path, objdir,
 236                            'libapr%s.dll' % (suffix)))
 237   dlls.append(os.path.join(gen_obj.apr_util_path, objdir,
 238                              'libaprutil%s.dll' % (suffix)))
 239
 240   if gen_obj.libintl_path is not None:
 241     dlls.append(os.path.join(gen_obj.libintl_path, 'bin', 'intl3_svn.dll'))
 242
 243   if gen_obj.bdb_lib is not None:
 244     partial_path = os.path.join(gen_obj.bdb_path, 'bin', gen_obj.bdb_lib)
 245     if objdir == 'Debug':
 246       dlls.append(partial_path + 'd.dll')
 247     else:
 248       dlls.append(partial_path + '.dll')
 249
 250   if gen_obj.sasl_path is not None:
 251     dlls.append(os.path.join(gen_obj.sasl_path, 'lib', 'libsasl.dll'))
 252
 253   for dll in dlls:
 254     copy_changed_file(dll, abs_objdir)
 255
 256   # Copy the Subversion library DLLs
 257   if not cp.has_option('options', '--disable-shared'):
 258     for svn_dll in svn_dlls:
 259       copy_changed_file(os.path.join(abs_objdir, svn_dll), abs_objdir)
 260
 261   # Copy the Apache modules
 262   if run_httpd and cp.has_option('options', '--with-httpd'):
 263     mod_dav_svn_path = os.path.join(abs_objdir, 'subversion',
 264                                     'mod_dav_svn', 'mod_dav_svn.so')
 265     mod_authz_svn_path = os.path.join(abs_objdir, 'subversion',
 266                                       'mod_authz_svn', 'mod_authz_svn.so')
 267     copy_changed_file(mod_dav_svn_path, abs_objdir)
 268     copy_changed_file(mod_authz_svn_path, abs_objdir)
 269
 270   os.environ['PATH'] = abs_objdir + os.pathsep + os.environ['PATH']
 271
 272 def fix_case(path):
 273     path = os.path.normpath(path)
 274     parts = string.split(path, os.path.sep)
 275     drive = string.upper(parts[0])
 276     parts = parts[1:]
 277     path = drive + os.path.sep
 278     for part in parts:
 279         dirs = os.listdir(path)
 280         for dir in dirs:
 281             if string.lower(dir) == string.lower(part):
 282                 path = os.path.join(path, dir)
 283                 break
 284     return path
 285
 286 class Svnserve:
 287   "Run svnserve for ra_svn tests"
 288   def __init__(self, svnserve_args, objdir, abs_objdir, abs_builddir):
 289     self.args = svnserve_args
 290     self.name = 'svnserve.exe'
 291     self.kind = objdir
 292     self.path = os.path.join(abs_objdir,
 293                              'subversion', 'svnserve', self.name)
 294     self.root = os.path.join(abs_builddir, CMDLINE_TEST_SCRIPT_NATIVE_PATH)
 295     self.proc_handle = None
 296
 297   def __del__(self):
 298     "Stop svnserve when the object is deleted"
 299     self.stop()
 300
 301   def _quote(self, arg):
 302     if ' ' in arg:
 303       return '"' + arg + '"'
 304     else:
 305       return arg
 306
 307   def start(self):
 308     if not self.args:
 309       args = [self.name, '-d', '-r', self.root]
 310     else:
 311       args = [self.name] + self.args
 312     print 'Starting', self.kind, self.name
 313     try:
 314       import win32process
 315       import win32con
 316       args = ' '.join(map(lambda x: self._quote(x), args))
 317       self.proc_handle = (
 318         win32process.CreateProcess(self._quote(self.path), args,
 319                                    None, None, 0,
 320                                    win32con.CREATE_NEW_CONSOLE,
 321                                    None, None, win32process.STARTUPINFO()))[0]
 322     except ImportError:
 323       os.spawnv(os.P_NOWAIT, self.path, args)
 324
 325   def stop(self):
 326     if self.proc_handle is not None:
 327       try:
 328         import win32process
 329         print 'Stopping', self.name
 330         win32process.TerminateProcess(self.proc_handle, 0)
 331         return
 332       except ImportError:
 333         pass
 334     print 'Svnserve.stop not implemented'
 335
 336 class Httpd:
 337   "Run httpd for DAV tests"
 338   def __init__(self, abs_httpd_dir, abs_objdir, abs_builddir, httpd_port):
 339     self.name = 'apache.exe'
 340     self.httpd_port = httpd_port
 341     self.httpd_dir = abs_httpd_dir
 342     self.path = os.path.join(self.httpd_dir, 'bin', self.name)
 343
 344     if not os.path.exists(self.path):
 345       self.name = 'httpd.exe'
 346       self.path = os.path.join(self.httpd_dir, 'bin', self.name)
 347       if not os.path.exists(self.path):
 348         raise RuntimeError, "Could not find a valid httpd binary!"
 349
 350     self.root_dir = os.path.join(CMDLINE_TEST_SCRIPT_NATIVE_PATH, 'httpd')
 351     self.root = os.path.join(abs_builddir, self.root_dir)
 352     self.authz_file = os.path.join(abs_builddir,
 353                                    CMDLINE_TEST_SCRIPT_NATIVE_PATH,
 354                                    'svn-test-work', 'authz')
 355     self.httpd_config = os.path.join(self.root, 'httpd.conf')
 356     self.httpd_users = os.path.join(self.root, 'users')
 357     self.httpd_mime_types = os.path.join(self.root, 'mime.types')
 358     self.abs_builddir = abs_builddir
 359     self.abs_objdir = abs_objdir
 360     self.service_name = 'svn-test-httpd-' + str(httpd_port)
 361     self.httpd_args = [self.name, '-n', self._quote(self.service_name),
 362                        '-f', self._quote(self.httpd_config)]
 363
 364     create_target_dir(self.root_dir)
 365
 366     self._create_users_file()
 367     self._create_mime_types_file()
 368
 369     # Determine version.
 370     if os.path.exists(os.path.join(self.httpd_dir,
 371                                    'modules', 'mod_access_compat.so')):
 372       self.httpd_ver = 2.3
 373     elif os.path.exists(os.path.join(self.httpd_dir,
 374                                      'modules', 'mod_auth_basic.so')):
 375       self.httpd_ver = 2.2
 376     else:
 377       self.httpd_ver = 2.0
 378
 379     # Create httpd config file
 380     fp = open(self.httpd_config, 'w')
 381
 382     # Global Environment
 383     fp.write('ServerRoot   ' + self._quote(self.root) + '\n')
 384     fp.write('DocumentRoot ' + self._quote(self.root) + '\n')
 385     fp.write('ServerName   localhost\n')
 386     fp.write('PidFile      pid\n')
 387     fp.write('ErrorLog     log\n')
 388     fp.write('Listen       ' + str(self.httpd_port) + '\n')
 389
 390     # Write LoadModule for minimal system module
 391     fp.write(self._sys_module('dav_module', 'mod_dav.so'))
 392     if self.httpd_ver >= 2.3:
 393       fp.write(self._sys_module('access_compat_module', 'mod_access_compat.so'))
 394       fp.write(self._sys_module('authz_core_module', 'mod_authz_core.so'))
 395       fp.write(self._sys_module('authz_user_module', 'mod_authz_user.so'))
 396       fp.write(self._sys_module('authn_core_module', 'mod_authn_core.so'))
 397     if self.httpd_ver >= 2.2:
 398       fp.write(self._sys_module('auth_basic_module', 'mod_auth_basic.so'))
 399       fp.write(self._sys_module('authn_file_module', 'mod_authn_file.so'))
 400     else:
 401       fp.write(self._sys_module('auth_module', 'mod_auth.so'))
 402     fp.write(self._sys_module('mime_module', 'mod_mime.so'))
 403     fp.write(self._sys_module('log_config_module', 'mod_log_config.so'))
 404
 405     # Write LoadModule for Subversion modules
 406     fp.write(self._svn_module('dav_svn_module', 'mod_dav_svn.so'))
 407     fp.write(self._svn_module('authz_svn_module', 'mod_authz_svn.so'))
 408
 409     # Define two locations for repositories
 410     fp.write(self._svn_repo('repositories'))
 411     fp.write(self._svn_repo('local_tmp'))
 412
 413     fp.write('TypesConfig     ' + self._quote(self.httpd_mime_types) + '\n')
 414     fp.write('LogLevel        Debug\n')
 415     fp.write('HostNameLookups Off\n')
 416
 417     fp.close()
 418
 419   def __del__(self):
 420     "Stop httpd when the object is deleted"
 421     self.stop()
 422
 423   def _quote(self, arg):
 424     if ' ' in arg:
 425       return '"' + arg + '"'
 426     else:
 427       return arg
 428
 429   def _create_users_file(self):
 430     "Create users file"
 431     htpasswd = os.path.join(self.httpd_dir, 'bin', 'htpasswd.exe')
 432     os.spawnv(os.P_WAIT, htpasswd, ['htpasswd.exe', '-mbc', self.httpd_users,
 433                                     'jrandom', 'rayjandom'])
 434     os.spawnv(os.P_WAIT, htpasswd, ['htpasswd.exe', '-mb',  self.httpd_users,
 435                                     'jconstant', 'rayjandom'])
 436
 437   def _create_mime_types_file(self):
 438     "Create empty mime.types file"
 439     fp = open(self.httpd_mime_types, 'w')
 440     fp.close()
 441
 442   def _sys_module(self, name, path):
 443     full_path = os.path.join(self.httpd_dir, 'modules', path)
 444     return 'LoadModule ' + name + " " + self._quote(full_path) + '\n'
 445
 446   def _svn_module(self, name, path):
 447     full_path = os.path.join(self.abs_objdir, path)
 448     return 'LoadModule ' + name + ' ' + self._quote(full_path) + '\n'
 449
 450   def _svn_repo(self, name):
 451     path = os.path.join(self.abs_builddir,
 452                         CMDLINE_TEST_SCRIPT_NATIVE_PATH,
 453                         'svn-test-work', name)
 454     location = '/svn-test-work/' + name
 455     return \
 456       '<Location ' + location + '>\n' \
 457       '  DAV             svn\n' \
 458       '  SVNParentPath   ' + self._quote(path) + '\n' \
 459       '  AuthzSVNAccessFile ' + self._quote(self.authz_file) + '\n' \
 460       '  AuthType        Basic\n' \
 461       '  AuthName        "Subversion Repository"\n' \
 462       '  AuthUserFile    ' + self._quote(self.httpd_users) + '\n' \
 463       '  Require         valid-user\n' \
 464       '</Location>\n'
 465
 466   def start(self):
 467     "Install and start HTTPD service"
 468     print 'Installing service', self.service_name
 469     os.spawnv(os.P_WAIT, self.path, self.httpd_args + ['-k', 'install'])
 470     print 'Starting service', self.service_name
 471     os.spawnv(os.P_WAIT, self.path, self.httpd_args + ['-k', 'start'])
 472
 473   def stop(self):
 474     "Stop and unintall HTTPD service"
 475     os.spawnv(os.P_WAIT, self.path, self.httpd_args + ['-k', 'stop'])
 476     os.spawnv(os.P_WAIT, self.path, self.httpd_args + ['-k', 'uninstall'])
 477
 478 # Move the binaries to the test directory
 479 locate_libs()
 480 if create_dirs:
 481   old_cwd = os.getcwd()
 482   try:
 483     os.chdir(abs_objdir)
 484     baton = copied_execs
 485     os.path.walk('subversion', copy_execs, baton)
 486   except:
 487     os.chdir(old_cwd)
 488     raise
 489   else:
 490     os.chdir(old_cwd)
 491
 492 # Create the base directory for Python tests
 493 create_target_dir(CMDLINE_TEST_SCRIPT_NATIVE_PATH)
 494
 495 # Ensure the tests directory is correctly cased
 496 abs_builddir = fix_case(abs_builddir)
 497
 498 daemon = None
 499 # Run the tests
 500 if run_svnserve:
 501   daemon = Svnserve(svnserve_args, objdir, abs_objdir, abs_builddir)
 502
 503 if run_httpd:
 504   daemon = Httpd(abs_httpd_dir, abs_objdir, abs_builddir, httpd_port)
 505
 506 # Start service daemon, if any
 507 if daemon:
 508   daemon.start()
 509
 510 print 'Testing', objdir, 'configuration on', repo_loc
 511 sys.path.insert(0, os.path.join(abs_srcdir, 'build'))
 512 import run_tests
 513 th = run_tests.TestHarness(abs_srcdir, abs_builddir,
 514                            os.path.join(abs_builddir, log),
 515                            base_url, fs_type, http_library,
 516                            server_minor_version, 1, cleanup,
 517                            enable_sasl, parallel, list_tests,
 518                            svn_bin)
 519 old_cwd = os.getcwd()
 520 try:
 521   os.chdir(abs_builddir)
 522   failed = th.run(all_tests)
 523 except:
 524   os.chdir(old_cwd)
 525   raise
 526 else:
 527   os.chdir(old_cwd)
 528
 529 # Stop service daemon, if any
 530 if daemon:
 531   del daemon
 532
 533 # Remove the execs again
 534 for tgt in copied_execs:
 535   try:
 536     if os.path.isfile(tgt):
 537       if verbose:
 538         print "kill:", tgt
 539       os.unlink(tgt)
 540   except:
 541     traceback.print_exc(file=sys.stdout)
 542     pass
 543
 544
 545 if failed:
 546   sys.exit(1)