contrib/client-side/svn_export_empty_files.py

   1 #!/usr/bin/env python
   2
   3 # Copyright (c) 2005 Sony Pictures Imageworks Inc.  All rights reserved.
   4 #
   5 # This software/script is free software; you may redistribute it
   6 # and/or modify it under the terms of Version 2 or later of the GNU
   7 # General Public License ("GPL") as published by the Free Software
   8 # Foundation.
   9 #
  10 # This software/script is distributed "AS IS," WITHOUT ANY EXPRESS OR
  11 # IMPLIED WARRANTIES OR REPRESENTATIONS OF ANY KIND WHATSOEVER,
  12 # including without any implied warranty of MERCHANTABILITY or FITNESS
  13 # FOR A PARTICULAR PURPOSE.  See the GNU GPL (Version 2 or later) for
  14 # details and license obligations.
  15
  16 """
  17 Script to "export" from a Subversion repository a clean directory tree
  18 of empty files instead of the content contained in those files in the
  19 repository.  The directory tree will also omit the .svn directories.
  20
  21 The export is done from the repository specified by URL at HEAD into
  22 PATH.  If PATH is omitted, the last components of the URL is used for
  23 the local directory name.  If the --delete command line option is
  24 given, then files and directories in PATH that do not exist in the
  25 Subversion repository are deleted.
  26
  27 As Subversion does not have any built-in tools to help locate files
  28 and directories, in extremely large repositories it can be hard to
  29 find what you are looking for.  This script was written to create a
  30 smaller non-working working copy that can be crawled with find or
  31 find's locate utility to make it easier to find files.
  32
  33 $HeadURL$
  34 $LastChangedRevision$
  35 $LastChangedDate$
  36 $LastChangedBy$
  37 """
  38
  39 import getopt
  40 try:
  41     my_getopt = getopt.gnu_getopt
  42 except AttributeError:
  43     my_getopt = getopt.getopt
  44 import os
  45 import sys
  46
  47 import svn.client
  48 import svn.core
  49
  50 class context:
  51     """A container for holding process context."""
  52
  53 def recursive_delete(dirname):
  54     """Recursively delete the given directory name."""
  55
  56     for filename in os.listdir(dirname):
  57         file_or_dir = os.path.join(dirname, filename)
  58         if os.path.isdir(file_or_dir) and not os.path.islink(file_or_dir):
  59             recursive_delete(file_or_dir)
  60         else:
  61             os.unlink(file_or_dir)
  62     os.rmdir(dirname)
  63
  64 def check_url_for_export(ctx, url, revision, client_ctx):
  65     """Given a URL to a Subversion repository, check that the URL is
  66     in the repository and that it refers to a directory and not a
  67     non-directory."""
  68
  69     # Try to do a listing on the URL to see if the repository can be
  70     # contacted.  Do not catch failures here, as they imply that there
  71     # is something wrong with the given URL.
  72     try:
  73         if ctx.verbose:
  74             print "Trying to list '%s'" % url
  75         svn.client.ls(url, revision, 0, client_ctx)
  76
  77         # Given a URL, the ls command does not tell you if
  78         # you have a directory or a non-directory, so try doing a
  79         # listing on the parent URL.  If the listing on the parent URL
  80         # fails, then assume that the given URL was the top of the
  81         # repository and hence a directory.
  82         try:
  83             last_slash_index = url.rindex('/')
  84         except ValueError:
  85             print "Cannot find a / in the URL '%s'" % url
  86             return False
  87
  88         parent_url = url[:last_slash_index]
  89         path_name = url[last_slash_index+1:]
  90
  91         try:
  92             if ctx.verbose:
  93                 print "Trying to list '%s'" % parent_url
  94             remote_ls = svn.client.ls(parent_url,
  95                                       revision,
  96                                       0,
  97                                       client_ctx)
  98         except svn.core.SubversionException:
  99             if ctx.verbose:
 100                 print "Listing of '%s' failed, assuming URL is top of repos" \
 101                       % parent_url
 102             return True
 103
 104         try:
 105             path_info = remote_ls[path_name]
 106         except ValueError:
 107             print "Able to ls '%s' but '%s' not in ls of '%s'" \
 108                   % (url, path_name, parent_url)
 109             return False
 110
 111         if svn.core.svn_node_dir != path_info.kind:
 112             if ctx.verbose:
 113                 print "The URL '%s' is not a directory" % url
 114             return False
 115         else:
 116             if ctx.verbose:
 117                 print "The URL '%s' is a directory" % url
 118             return True
 119     finally:
 120         pass
 121
 122 LOCAL_PATH_DIR = 'Directory'
 123 LOCAL_PATH_NON_DIR = 'Non-directory'
 124 LOCAL_PATH_NONE = 'Nonexistent'
 125 def get_local_path_kind(pathname):
 126     """Determine if there is a path in the filesystem and if the path
 127     is a directory or non-directory."""
 128
 129     try:
 130         os.stat(pathname)
 131         if os.path.isdir(pathname):
 132             status = LOCAL_PATH_DIR
 133         else:
 134             status = LOCAL_PATH_NON_DIR
 135     except OSError:
 136         status = LOCAL_PATH_NONE
 137
 138     return status
 139
 140 def synchronize_dir(ctx, url, dir_name, revision, client_ctx):
 141     """Synchronize a directory given by a URL to a Subversion
 142     repository with a local directory located by the dir_name
 143     argument."""
 144
 145     status = True
 146
 147     # Determine if there is a path in the filesystem and if the path
 148     # is a directory or non-directory.
 149     local_path_kind = get_local_path_kind(dir_name)
 150
 151     # If the path on the local filesystem is not a directory, then
 152     # delete it if deletes are enabled, otherwise return.
 153     if LOCAL_PATH_NON_DIR == local_path_kind:
 154         msg = ("'%s' which is a local non-directory but remotely a " +
 155                "directory") % dir_name
 156         if ctx.delete_local_paths:
 157             print "Removing", msg
 158             os.unlink(dir_name)
 159             local_path_kind = LOCAL_PATH_NONE
 160         else:
 161             print "Need to remove", msg
 162             ctx.delete_needed = True
 163             return False
 164
 165     if LOCAL_PATH_NONE == local_path_kind:
 166         print "Creating directory '%s'" % dir_name
 167         os.mkdir(dir_name)
 168
 169     remote_ls = svn.client.ls(url,
 170                               revision,
 171                               0,
 172                               client_ctx)
 173
 174     if ctx.verbose:
 175         print "Syncing '%s' to '%s'" % (url, dir_name)
 176
 177     remote_pathnames = remote_ls.keys()
 178     remote_pathnames.sort()
 179
 180     local_pathnames = os.listdir(dir_name)
 181
 182     for remote_pathname in remote_pathnames:
 183         # For each name in the remote list, remove it from the local
 184         # list so that the remaining names may be deleted.
 185         try:
 186             local_pathnames.remove(remote_pathname)
 187         except ValueError:
 188             pass
 189
 190         full_remote_pathname = os.path.join(dir_name, remote_pathname)
 191
 192         if remote_pathname in ctx.ignore_names or \
 193                full_remote_pathname in ctx.ignore_paths:
 194             print "Skipping '%s'" % full_remote_pathname
 195             continue
 196
 197         # Get the remote path kind.
 198         remote_path_kind = remote_ls[remote_pathname].kind
 199
 200         # If the remote path is a directory, then recursively handle
 201         # that here.
 202         if svn.core.svn_node_dir == remote_path_kind:
 203             s = synchronize_dir(ctx,
 204                                 os.path.join(url, remote_pathname),
 205                                 full_remote_pathname,
 206                                 revision,
 207                                 client_ctx)
 208             status &= s
 209
 210         else:
 211             # Determine if there is a path in the filesystem and if
 212             # the path is a directory or non-directory.
 213             local_path_kind = get_local_path_kind(full_remote_pathname)
 214
 215             # If the path exists on the local filesystem but its kind
 216             # does not match the kind in the Subversion repository,
 217             # then either remove it if the local paths should be
 218             # deleted or continue to the next path if deletes should
 219             # not be done.
 220             if LOCAL_PATH_DIR == local_path_kind:
 221                 msg = ("'%s' which is a local directory but remotely a " +
 222                        "non-directory") % full_remote_pathname
 223                 if ctx.delete_local_paths:
 224                     print "Removing", msg
 225                     recursive_delete(full_remote_pathname)
 226                     local_path_kind = LOCAL_PATH_NONE
 227                 else:
 228                     print "Need to remove", msg
 229                     ctx.delete_needed = True
 230                     continue
 231
 232             if LOCAL_PATH_NONE == local_path_kind:
 233                 print "Creating file '%s'" % full_remote_pathname
 234                 f = file(full_remote_pathname, 'w')
 235                 f.close()
 236
 237     # Any remaining local paths should be removed.
 238     local_pathnames.sort()
 239     for local_pathname in local_pathnames:
 240         full_local_pathname = os.path.join(dir_name, local_pathname)
 241         if os.path.isdir(full_local_pathname):
 242             if ctx.delete_local_paths:
 243                 print "Removing directory '%s'" % full_local_pathname
 244                 recursive_delete(full_local_pathname)
 245             else:
 246                 print "Need to remove directory '%s'" % full_local_pathname
 247                 ctx.delete_needed = True
 248         else:
 249             if ctx.delete_local_paths:
 250                 print "Removing file '%s'" % full_local_pathname
 251                 os.unlink(full_local_pathname)
 252             else:
 253                 print "Need to remove file '%s'" % full_local_pathname
 254                 ctx.delete_needed = True
 255
 256     return status
 257
 258 def main(ctx, url, export_pathname):
 259     # Create a client context to run all Subversion client commands
 260     # with.
 261     client_ctx = svn.client.create_context()
 262
 263     # Give the client context baton a suite of authentication
 264     # providers.
 265     providers = [
 266         svn.client.get_simple_provider(),
 267         svn.client.get_ssl_client_cert_file_provider(),
 268         svn.client.get_ssl_client_cert_pw_file_provider(),
 269         svn.client.get_ssl_server_trust_file_provider(),
 270         svn.client.get_username_provider(),
 271         ]
 272     client_ctx.auth_baton = svn.core.svn_auth_open(providers)
 273
 274     # Load the configuration information from the configuration files.
 275     client_ctx.config = svn.core.svn_config_get_config(None)
 276
 277     # Use the HEAD revision to check out.
 278     head_revision = svn.core.svn_opt_revision_t()
 279     head_revision.kind = svn.core.svn_opt_revision_head
 280
 281     # Check that the URL refers to a directory in the repository and
 282     # not non-directory (file, special, etc).
 283     status = check_url_for_export(ctx, url, head_revision, client_ctx)
 284     if not status:
 285         return 1
 286
 287     # Synchronize the current working directory with the given URL and
 288     # descend recursively into the repository.
 289     status = synchronize_dir(ctx,
 290                              url,
 291                              export_pathname,
 292                              head_revision,
 293                              client_ctx)
 294
 295     if ctx.delete_needed:
 296         print "There are files and directories in the local filesystem"
 297         print "that do not exist in the Subversion repository that were"
 298         print "not deleted.  ",
 299         if ctx.delete_needed:
 300             print "Please pass the --delete command line option"
 301             print "to have this script delete those files and directories."
 302         else:
 303             print ""
 304
 305     if status:
 306         return 0
 307     else:
 308         return 1
 309
 310 def usage(verbose_usage):
 311     message1 = \
 312 """usage: %s [options] URL [PATH]
 313 Options include
 314     --delete           delete files and directories that don't exist in repos
 315     -h (--help)        show this message
 316     -n (--name) arg    add arg to the list of file or dir names to ignore
 317     -p (--path) arg    add arg to the list of file or dir paths to ignore
 318     -v (--verbose)     be verbose in output"""
 319
 320     message2 = \
 321 """Script to "export" from a Subversion repository a clean directory tree
 322 of empty files instead of the content contained in those files in the
 323 repository.  The directory tree will also omit the .svn directories.
 324
 325 The export is done from the repository specified by URL at HEAD into
 326 PATH.  If PATH is omitted, the last components of the URL is used for
 327 the local directory name.  If the --delete command line option is
 328 given, then files and directories in PATH that do not exist in the
 329 Subversion repository are deleted.
 330
 331 As Subversion does have any built-in tools to help locate files and
 332 directories, in extremely large repositories it can be hard to find
 333 what you are looking for.  This script was written to create a smaller
 334 non-working working copy that can be crawled with find or find's
 335 locate utility to make it easier to find files."""
 336
 337     print >>sys.stderr, message1 % sys.argv[0]
 338     if verbose_usage:
 339         print >>sys.stderr, message2
 340     sys.exit(1)
 341
 342 if __name__ == '__main__':
 343     ctx = context()
 344
 345     # Context storing command line options settings.
 346     ctx.delete_local_paths = False
 347     ctx.ignore_names = []
 348     ctx.ignore_paths = []
 349     ctx.verbose = False
 350
 351     # Context storing state from running the sync.
 352     ctx.delete_needed = False
 353
 354     try:
 355         opts, args = my_getopt(sys.argv[1:],
 356                                'hn:p:v',
 357                                ['delete',
 358                                 'help',
 359                                 'name=',
 360                                 'path=',
 361                                 'verbose'
 362                                 ])
 363     except getopt.GetoptError:
 364         usage(False)
 365     if len(args) < 1 or len(args) > 2:
 366         print >>sys.stderr, "Incorrect number of arguments"
 367         usage(False)
 368
 369     for o, a in opts:
 370         if o in ('--delete',):
 371             ctx.delete_local_paths = True
 372             continue
 373         if o in ('-h', '--help'):
 374             usage(True)
 375             continue
 376         if o in ('-n', '--name'):
 377             ctx.ignore_names += [a]
 378             continue
 379         if o in ('-p', '--path'):
 380             ctx.ignore_paths += [a]
 381             continue
 382         if o in ('-v', '--verbose'):
 383             ctx.verbose = True
 384             continue
 385
 386     # Get the URL to export and remove any trailing /'s from it.
 387     url = args[0]
 388     args = args[1:]
 389     while url[-1] == '/':
 390         url = url[:-1]
 391
 392     # Get the local path to export into.
 393     if args:
 394         export_pathname = args[0]
 395         args = args[1:]
 396     else:
 397         try:
 398             last_slash_index = url.rindex('/')
 399         except ValueError:
 400             print >>sys.stderr, "Cannot find a / in the URL '%s'" % url
 401             usage(False)
 402         export_pathname = url[last_slash_index+1:]
 403
 404     sys.exit(main(ctx, url, export_pathname))