win-tests.py

   1 """
   2 Driver for running the tests on Windows.
   3
   4 For a list of options, run this script with the --help option.
   5 """
   6
   7 # $HeadURL$
   8 # $LastChangedDate$
   9 # $LastChangedBy$
  10 # $LastChangedRevision$
  11
  12 import os, sys
  13 import filecmp
  14 import shutil
  15 import traceback
  16 import ConfigParser
  17 import string
  18 import random
  19
  20 import getopt
  21 try:
  22     my_getopt = getopt.gnu_getopt
  23 except AttributeError:
  24     my_getopt = getopt.getopt
  25
  26 def _usage_exit():
  27   "print usage, exit the script"
  28
  29   print "Driver for running the tests on Windows."
  30   print "Usage: python win-tests.py [option] [test-path]"
  31   print
  32   print "Valid options:"
  33   print "  -r, --release          : test the Release configuration"
  34   print "  -d, --debug            : test the Debug configuration (default)"
  35   print "  --bin=PATH             : use the svn binaries installed in PATH"
  36   print "  -u URL, --url=URL      : run ra_dav or ra_svn tests against URL;"
  37   print "                           will start svnserve for ra_svn tests"
  38   print "  -v, --verbose          : talk more"
  39   print "  -f, --fs-type=type     : filesystem type to use (fsfs is default)"
  40   print "  -c, --cleanup          : cleanup after running a test"
  41
  42   print "  --svnserve-args=list   : comma-separated list of arguments for"
  43   print "                           svnserve"
  44   print "                           default is '-d,-r,<test-path-root>'"
  45   print "  --asp.net-hack         : use '_svn' instead of '.svn' for the admin"
  46   print "                           dir name"
  47   print "  --httpd-dir            : location where Apache HTTPD is installed"
  48   print "  --httpd-port           : port for Apache HTTPD; random port number"
  49   print "                           will be used, if not specified"
  50   print "  --http-library         : dav library to use, neon (default) or serf"
  51   print "  --list                 : print test doc strings only"
  52   print "  --enable-sasl          : enable Cyrus SASL authentication for"
  53   print "                           svnserve"
  54   print "  -p, --parallel         : run multiple tests in parallel"
  55   print "  --server-minor-version : the minor version of the server being"
  56   print "                           tested"
  57
  58   sys.exit(0)
  59
  60 CMDLINE_TEST_SCRIPT_PATH = 'subversion/tests/cmdline/'
  61 CMDLINE_TEST_SCRIPT_NATIVE_PATH = CMDLINE_TEST_SCRIPT_PATH.replace('/', os.sep)
  62
  63 sys.path.insert(0, os.path.join('build', 'generator'))
  64 sys.path.insert(1, 'build')
  65
  66 import gen_win
  67 version_header = os.path.join('subversion', 'include', 'svn_version.h')
  68 cp = ConfigParser.ConfigParser()
  69 cp.read('gen-make.opts')
  70 gen_obj = gen_win.GeneratorBase('build.conf', version_header,
  71                                 cp.items('options'))
  72 all_tests = gen_obj.test_progs + gen_obj.bdb_test_progs \
  73           + gen_obj.scripts + gen_obj.bdb_scripts
  74 client_tests = filter(lambda x: x.startswith(CMDLINE_TEST_SCRIPT_PATH),
  75                       all_tests)
  76
  77 svn_dlls = []
  78 for section in gen_obj.sections.values():
  79   if section.options.get("msvc-export"):
  80     dll_basename = section.name + "-" + str(gen_obj.version) + ".dll"
  81     svn_dlls.append(os.path.join("subversion", section.name, dll_basename))
  82
  83 opts, args = my_getopt(sys.argv[1:], 'hrdvcpu:f:',
  84                        ['release', 'debug', 'verbose', 'cleanup', 'url=',
  85                         'svnserve-args=', 'fs-type=', 'asp.net-hack',
  86                         'httpd-dir=', 'httpd-port=', 'http-library=', 'help',
  87                         'list', 'enable-sasl', 'bin=', 'parallel'])
  88 if len(args) > 1:
  89   print 'Warning: non-option arguments after the first one will be ignored'
  90
  91 # Interpret the options and set parameters
  92 base_url, fs_type, verbose, cleanup = None, None, None, None
  93 repo_loc = 'local repository.'
  94 objdir = 'Debug'
  95 log = 'tests.log'
  96 run_svnserve = None
  97 svnserve_args = None
  98 run_httpd = None
  99 httpd_port = None
 100 http_library = 'neon'
 101 list_tests = None
 102 enable_sasl = None
 103 svn_bin = None
 104 parallel = None
 105 server_minor_version = None
 106
 107 for opt, val in opts:
 108   if opt in ('-h', '--help'):
 109     _usage_exit()
 110   elif opt in ('-u', '--url'):
 111     base_url = val
 112   elif opt in ('-f', '--fs-type'):
 113     fs_type = val
 114   elif opt in ('-v', '--verbose'):
 115     verbose = 1
 116   elif opt in ('-c', '--cleanup'):
 117     cleanup = 1
 118   elif opt in ['-r', '--release']:
 119     objdir = 'Release'
 120   elif opt in ['-d', '--debug']:
 121     objdir = 'Debug'
 122   elif opt == '--svnserve-args':
 123     svnserve_args = val.split(',')
 124     run_svnserve = 1
 125   elif opt == '--asp.net-hack':
 126     os.environ['SVN_ASP_DOT_NET_HACK'] = opt
 127   elif opt == '--httpd-dir':
 128     abs_httpd_dir = os.path.abspath(val)
 129     run_httpd = 1
 130   elif opt == '--httpd-port':
 131     httpd_port = int(val)
 132   elif opt == '--http-library':
 133     http_library = val
 134   elif opt == '--list':
 135     list_tests = 1
 136   elif opt == '--enable-sasl':
 137     enable_sasl = 1
 138     base_url = "svn://localhost/"
 139   elif opt == '--server-minor-version':
 140     server_minor_version = val
 141   elif opt == '--bin':
 142     svn_bin = val
 143   elif opt in ('-p', '--parallel'):
 144     parallel = 1
 145
 146 # Calculate the source and test directory names
 147 abs_srcdir = os.path.abspath("")
 148 abs_objdir = os.path.join(abs_srcdir, objdir)
 149 if len(args) == 0:
 150   abs_builddir = abs_objdir
 151   create_dirs = 0
 152 else:
 153   abs_builddir = os.path.abspath(args[0])
 154   create_dirs = 1
 155
 156 # Default to fsfs explicitly
 157 if not fs_type:
 158   fs_type = 'fsfs'
 159
 160 # Don't run bdb tests if they want to test fsfs
 161 if fs_type == 'fsfs':
 162   all_tests = gen_obj.test_progs + gen_obj.scripts
 163
 164 if run_httpd:
 165   if not httpd_port:
 166     httpd_port = random.randrange(1024, 30000)
 167   if not base_url:
 168     base_url = 'http://localhost:' + str(httpd_port)
 169
 170 if base_url:
 171   all_tests = client_tests
 172   repo_loc = 'remote repository ' + base_url + '.'
 173   if base_url[:4] == 'http':
 174     log = 'dav-tests.log'
 175   elif base_url[:3] == 'svn':
 176     log = 'svn-tests.log'
 177     run_svnserve = 1
 178   else:
 179     # Don't know this scheme, but who're we to judge whether it's
 180     # correct or not?
 181     log = 'url-tests.log'
 182
 183 # Have to move the executables where the tests expect them to be
 184 copied_execs = []   # Store copied exec files to avoid the final dir scan
 185
 186 def create_target_dir(dirname):
 187   tgt_dir = os.path.join(abs_builddir, dirname)
 188   if not os.path.exists(tgt_dir):
 189     if verbose:
 190       print "mkdir:", tgt_dir
 191     os.makedirs(tgt_dir)
 192
 193 def copy_changed_file(src, tgt):
 194   if not os.path.isfile(src):
 195     print 'Could not find ' + src
 196     sys.exit(1)
 197   if os.path.isdir(tgt):
 198     tgt = os.path.join(tgt, os.path.basename(src))
 199   if os.path.exists(tgt):
 200     assert os.path.isfile(tgt)
 201     if filecmp.cmp(src, tgt):
 202       if verbose:
 203         print "same:", src
 204         print " and:", tgt
 205       return 0
 206   if verbose:
 207     print "copy:", src
 208     print "  to:", tgt
 209   shutil.copy(src, tgt)
 210   return 1
 211
 212 def copy_execs(baton, dirname, names):
 213   copied_execs = baton
 214   for name in names:
 215     ext = os.path.splitext(name)[1]
 216     if ext != ".exe":
 217       continue
 218     src = os.path.join(dirname, name)
 219     tgt = os.path.join(abs_builddir, dirname, name)
 220     create_target_dir(dirname)
 221     if copy_changed_file(src, tgt):
 222       copied_execs.append(tgt)
 223
 224 def locate_libs():
 225   "Move DLLs to a known location and set env vars"
 226
 227   dlls = []
 228
 229   # look for APR 1.x dll's and use those if found
 230   apr_test_path = os.path.join(gen_obj.apr_path, objdir, 'libapr-1.dll')
 231   if os.path.exists(apr_test_path):
 232     suffix = "-1"
 233   else:
 234     suffix = ""
 235   dlls.append(os.path.join(gen_obj.apr_path, objdir,
 236                            'libapr%s.dll' % (suffix)))
 237   dlls.append(os.path.join(gen_obj.apr_util_path, objdir,
 238                              'libaprutil%s.dll' % (suffix)))
 239   dlls.append(os.path.join(gen_obj.apr_iconv_path, objdir,
 240                            'libapriconv%s.dll' % (suffix)))
 241
 242   if gen_obj.libintl_path is not None:
 243     dlls.append(os.path.join(gen_obj.libintl_path, 'bin', 'intl3_svn.dll'))
 244
 245   dlls.append(os.path.join(gen_obj.sqlite_path, 'bin', 'sqlite3.dll'))
 246
 247   if gen_obj.bdb_lib is not None:
 248     partial_path = os.path.join(gen_obj.bdb_path, 'bin', gen_obj.bdb_lib)
 249     if objdir == 'Debug':
 250       dlls.append(partial_path + 'd.dll')
 251     else:
 252       dlls.append(partial_path + '.dll')
 253
 254   if gen_obj.sasl_path is not None:
 255     dlls.append(os.path.join(gen_obj.sasl_path, 'lib', 'libsasl.dll'))
 256
 257   for dll in dlls:
 258     copy_changed_file(dll, abs_objdir)
 259
 260   # Copy the Subversion library DLLs
 261   if not cp.has_option('options', '--disable-shared'):
 262     for svn_dll in svn_dlls:
 263       copy_changed_file(os.path.join(abs_objdir, svn_dll), abs_objdir)
 264
 265   # Copy the Apache modules
 266   if run_httpd and cp.has_option('options', '--with-httpd'):
 267     mod_dav_svn_path = os.path.join(abs_objdir, 'subversion',
 268                                     'mod_dav_svn', 'mod_dav_svn.so')
 269     mod_authz_svn_path = os.path.join(abs_objdir, 'subversion',
 270                                       'mod_authz_svn', 'mod_authz_svn.so')
 271     copy_changed_file(mod_dav_svn_path, abs_objdir)
 272     copy_changed_file(mod_authz_svn_path, abs_objdir)
 273
 274   apriconv_so_path = os.path.join(gen_obj.apr_iconv_path, objdir, 'iconv')
 275   os.environ['APR_ICONV_PATH'] = os.path.abspath(apriconv_so_path)
 276   os.environ['PATH'] = abs_objdir + os.pathsep + os.environ['PATH']
 277
 278 def fix_case(path):
 279     path = os.path.normpath(path)
 280     parts = string.split(path, os.path.sep)
 281     drive = string.upper(parts[0])
 282     parts = parts[1:]
 283     path = drive + os.path.sep
 284     for part in parts:
 285         dirs = os.listdir(path)
 286         for dir in dirs:
 287             if string.lower(dir) == string.lower(part):
 288                 path = os.path.join(path, dir)
 289                 break
 290     return path
 291
 292 class Svnserve:
 293   "Run svnserve for ra_svn tests"
 294   def __init__(self, svnserve_args, objdir, abs_objdir, abs_builddir):
 295     self.args = svnserve_args
 296     self.name = 'svnserve.exe'
 297     self.kind = objdir
 298     self.path = os.path.join(abs_objdir,
 299                              'subversion', 'svnserve', self.name)
 300     self.root = os.path.join(abs_builddir, CMDLINE_TEST_SCRIPT_NATIVE_PATH)
 301     self.proc_handle = None
 302
 303   def __del__(self):
 304     "Stop svnserve when the object is deleted"
 305     self.stop()
 306
 307   def _quote(self, arg):
 308     if ' ' in arg:
 309       return '"' + arg + '"'
 310     else:
 311       return arg
 312
 313   def start(self):
 314     if not self.args:
 315       args = [self.name, '-d', '-r', self.root]
 316     else:
 317       args = [self.name] + self.args
 318     print 'Starting', self.kind, self.name
 319     try:
 320       import win32process
 321       import win32con
 322       args = ' '.join(map(lambda x: self._quote(x), args))
 323       self.proc_handle = (
 324         win32process.CreateProcess(self._quote(self.path), args,
 325                                    None, None, 0,
 326                                    win32con.CREATE_NEW_CONSOLE,
 327                                    None, None, win32process.STARTUPINFO()))[0]
 328     except ImportError:
 329       os.spawnv(os.P_NOWAIT, self.path, args)
 330
 331   def stop(self):
 332     if self.proc_handle is not None:
 333       try:
 334         import win32process
 335         print 'Stopping', self.name
 336         win32process.TerminateProcess(self.proc_handle, 0)
 337         return
 338       except ImportError:
 339         pass
 340     print 'Svnserve.stop not implemented'
 341
 342 class Httpd:
 343   "Run httpd for DAV tests"
 344   def __init__(self, abs_httpd_dir, abs_objdir, abs_builddir, httpd_port):
 345     self.name = 'apache.exe'
 346     self.httpd_port = httpd_port
 347     self.httpd_dir = abs_httpd_dir
 348     self.path = os.path.join(self.httpd_dir, 'bin', self.name)
 349
 350     if not os.path.exists(self.path):
 351       self.name = 'httpd.exe'
 352       self.path = os.path.join(self.httpd_dir, 'bin', self.name)
 353       if not os.path.exists(self.path):
 354         raise RuntimeError, "Could not find a valid httpd binary!"
 355
 356     self.root_dir = os.path.join(CMDLINE_TEST_SCRIPT_NATIVE_PATH, 'httpd')
 357     self.root = os.path.join(abs_builddir, self.root_dir)
 358     self.authz_file = os.path.join(abs_builddir,
 359                                    CMDLINE_TEST_SCRIPT_NATIVE_PATH,
 360                                    'svn-test-work', 'authz')
 361     self.httpd_config = os.path.join(self.root, 'httpd.conf')
 362     self.httpd_users = os.path.join(self.root, 'users')
 363     self.httpd_mime_types = os.path.join(self.root, 'mime.types')
 364     self.abs_builddir = abs_builddir
 365     self.abs_objdir = abs_objdir
 366     self.service_name = 'svn-test-httpd-' + str(httpd_port)
 367     self.httpd_args = [self.name, '-n', self._quote(self.service_name),
 368                        '-f', self._quote(self.httpd_config)]
 369
 370     create_target_dir(self.root_dir)
 371
 372     self._create_users_file()
 373     self._create_mime_types_file()
 374
 375     # Determine version.
 376     if os.path.exists(os.path.join(self.httpd_dir,
 377                                    'modules', 'mod_access_compat.so')):
 378       self.httpd_ver = 2.3
 379     elif os.path.exists(os.path.join(self.httpd_dir,
 380                                      'modules', 'mod_auth_basic.so')):
 381       self.httpd_ver = 2.2
 382     else:
 383       self.httpd_ver = 2.0
 384
 385     # Create httpd config file
 386     fp = open(self.httpd_config, 'w')
 387
 388     # Global Environment
 389     fp.write('ServerRoot   ' + self._quote(self.root) + '\n')
 390     fp.write('DocumentRoot ' + self._quote(self.root) + '\n')
 391     fp.write('ServerName   localhost\n')
 392     fp.write('PidFile      pid\n')
 393     fp.write('ErrorLog     log\n')
 394     fp.write('Listen       ' + str(self.httpd_port) + '\n')
 395
 396     # Write LoadModule for minimal system module
 397     fp.write(self._sys_module('dav_module', 'mod_dav.so'))
 398     if self.httpd_ver >= 2.3:
 399       fp.write(self._sys_module('access_compat_module', 'mod_access_compat.so'))
 400       fp.write(self._sys_module('authz_core_module', 'mod_authz_core.so'))
 401       fp.write(self._sys_module('authz_user_module', 'mod_authz_user.so'))
 402       fp.write(self._sys_module('authn_core_module', 'mod_authn_core.so'))
 403     if self.httpd_ver >= 2.2:
 404       fp.write(self._sys_module('auth_basic_module', 'mod_auth_basic.so'))
 405       fp.write(self._sys_module('authn_file_module', 'mod_authn_file.so'))
 406     else:
 407       fp.write(self._sys_module('auth_module', 'mod_auth.so'))
 408     fp.write(self._sys_module('mime_module', 'mod_mime.so'))
 409     fp.write(self._sys_module('log_config_module', 'mod_log_config.so'))
 410
 411     # Write LoadModule for Subversion modules
 412     fp.write(self._svn_module('dav_svn_module', 'mod_dav_svn.so'))
 413     fp.write(self._svn_module('authz_svn_module', 'mod_authz_svn.so'))
 414
 415     # Define two locations for repositories
 416     fp.write(self._svn_repo('repositories'))
 417     fp.write(self._svn_repo('local_tmp'))
 418
 419     fp.write('TypesConfig     ' + self._quote(self.httpd_mime_types) + '\n')
 420     fp.write('LogLevel        Debug\n')
 421     fp.write('HostNameLookups Off\n')
 422
 423     fp.close()
 424
 425   def __del__(self):
 426     "Stop httpd when the object is deleted"
 427     self.stop()
 428
 429   def _quote(self, arg):
 430     if ' ' in arg:
 431       return '"' + arg + '"'
 432     else:
 433       return arg
 434
 435   def _create_users_file(self):
 436     "Create users file"
 437     htpasswd = os.path.join(self.httpd_dir, 'bin', 'htpasswd.exe')
 438     os.spawnv(os.P_WAIT, htpasswd, ['htpasswd.exe', '-mbc', self.httpd_users,
 439                                     'jrandom', 'rayjandom'])
 440     os.spawnv(os.P_WAIT, htpasswd, ['htpasswd.exe', '-mb',  self.httpd_users,
 441                                     'jconstant', 'rayjandom'])
 442
 443   def _create_mime_types_file(self):
 444     "Create empty mime.types file"
 445     fp = open(self.httpd_mime_types, 'w')
 446     fp.close()
 447
 448   def _sys_module(self, name, path):
 449     full_path = os.path.join(self.httpd_dir, 'modules', path)
 450     return 'LoadModule ' + name + " " + self._quote(full_path) + '\n'
 451
 452   def _svn_module(self, name, path):
 453     full_path = os.path.join(self.abs_objdir, path)
 454     return 'LoadModule ' + name + ' ' + self._quote(full_path) + '\n'
 455
 456   def _svn_repo(self, name):
 457     path = os.path.join(self.abs_builddir,
 458                         CMDLINE_TEST_SCRIPT_NATIVE_PATH,
 459                         'svn-test-work', name)
 460     location = '/svn-test-work/' + name
 461     return \
 462       '<Location ' + location + '>\n' \
 463       '  DAV             svn\n' \
 464       '  SVNParentPath   ' + self._quote(path) + '\n' \
 465       '  AuthzSVNAccessFile ' + self._quote(self.authz_file) + '\n' \
 466       '  AuthType        Basic\n' \
 467       '  AuthName        "Subversion Repository"\n' \
 468       '  AuthUserFile    ' + self._quote(self.httpd_users) + '\n' \
 469       '  Require         valid-user\n' \
 470       '</Location>\n'
 471
 472   def start(self):
 473     "Install and start HTTPD service"
 474     print 'Installing service', self.service_name
 475     os.spawnv(os.P_WAIT, self.path, self.httpd_args + ['-k', 'install'])
 476     print 'Starting service', self.service_name
 477     os.spawnv(os.P_WAIT, self.path, self.httpd_args + ['-k', 'start'])
 478
 479   def stop(self):
 480     "Stop and unintall HTTPD service"
 481     os.spawnv(os.P_WAIT, self.path, self.httpd_args + ['-k', 'stop'])
 482     os.spawnv(os.P_WAIT, self.path, self.httpd_args + ['-k', 'uninstall'])
 483
 484 # Move the binaries to the test directory
 485 locate_libs()
 486 if create_dirs:
 487   old_cwd = os.getcwd()
 488   try:
 489     os.chdir(abs_objdir)
 490     baton = copied_execs
 491     os.path.walk('subversion', copy_execs, baton)
 492   except:
 493     os.chdir(old_cwd)
 494     raise
 495   else:
 496     os.chdir(old_cwd)
 497
 498 # Create the base directory for Python tests
 499 create_target_dir(CMDLINE_TEST_SCRIPT_NATIVE_PATH)
 500
 501 # Ensure the tests directory is correctly cased
 502 abs_builddir = fix_case(abs_builddir)
 503
 504 daemon = None
 505 # Run the tests
 506 if run_svnserve:
 507   daemon = Svnserve(svnserve_args, objdir, abs_objdir, abs_builddir)
 508
 509 if run_httpd:
 510   daemon = Httpd(abs_httpd_dir, abs_objdir, abs_builddir, httpd_port)
 511
 512 # Start service daemon, if any
 513 if daemon:
 514   daemon.start()
 515
 516 print 'Testing', objdir, 'configuration on', repo_loc
 517 sys.path.insert(0, os.path.join(abs_srcdir, 'build'))
 518 import run_tests
 519 th = run_tests.TestHarness(abs_srcdir, abs_builddir,
 520                            os.path.join(abs_builddir, log),
 521                            base_url, fs_type, http_library,
 522                            server_minor_version, 1, cleanup,
 523                            enable_sasl, parallel, list_tests,
 524                            svn_bin)
 525 old_cwd = os.getcwd()
 526 try:
 527   os.chdir(abs_builddir)
 528   failed = th.run(all_tests)
 529 except:
 530   os.chdir(old_cwd)
 531   raise
 532 else:
 533   os.chdir(old_cwd)
 534
 535 # Stop service daemon, if any
 536 if daemon:
 537   del daemon
 538
 539 # Remove the execs again
 540 for tgt in copied_execs:
 541   try:
 542     if os.path.isfile(tgt):
 543       if verbose:
 544         print "kill:", tgt
 545       os.unlink(tgt)
 546   except:
 547     traceback.print_exc(file=sys.stdout)
 548     pass
 549
 550
 551 if failed:
 552   sys.exit(1)