contrib/client-side/svn-merge-vendor.py

   1 #!/usr/bin/env python
   2 # -*-mode: python; coding: utf-8 -*-
   3 #
   4 # Inspired from svn-import.py by astrand@cendio.se (ref :
   5 # http://svn.haxx.se/users/archive-2006-10/0857.shtml)
   6 #
   7 # svn-merge-vendor.py (v1.0.1) - Import a new release, such as a vendor drop.
   8 #
   9 # The "Vendor branches" chapter of "Version Control with Subversion"
  10 # describes how to do a new vendor drop with:
  11 #
  12 # >The goal here is to make our current directory contain only the
  13 # >libcomplex 1.1 code, and to ensure that all that code is under version
  14 # >control. Oh, and we want to do this with as little version control
  15 # >history disturbance as possible.
  16 #
  17 # This utility tries to take you to this goal - automatically. Files
  18 # new in this release is added to version control, and files removed
  19 # in this new release are removed from version control.  It will
  20 # detect the moved files by looking in the svn log to find the
  21 # "copied-from" path !
  22 #
  23 # Compared to svn_load_dirs.pl, this utility:
  24 #
  25 # * DETECTS THE MOVED FILES !!
  26 # * Does not hard-code commit messages
  27 # * Allows you to fine-tune the import before commit, which
  28 #   allows you to turn adds+deletes into moves.
  29 #
  30 # TODO :
  31 #   * support --username and --password
  32 #
  33 # This tool is provided under GPL license.  Please read
  34 # http://www.gnu.org/licenses/gpl.html for the original text.
  35 #
  36 # $HeadURL$
  37 # $LastChangedRevision$
  38 # $LastChangedDate$
  39 # $LastChangedBy$
  40
  41 import os
  42 import re
  43 import tempfile
  44 import atexit
  45 import subprocess
  46 import shutil
  47 import sys
  48 import getopt
  49 import logging
  50 import string
  51 from StringIO import StringIO
  52 # lxml module can be found here : http://codespeak.net/lxml/
  53 from lxml import etree
  54 import types
  55
  56 prog_name = os.path.basename(sys.argv[0])
  57 orig_svn_subroot = None
  58 base_copied_paths = []
  59 r_from = None
  60 r_to = None
  61 log_tree = None
  62 entries_to_treat = []
  63 entries_to_delete = []
  64 added_paths = []
  65 logger = None
  66
  67 def del_temp_tree(tmpdir):
  68     """Delete tree, standring in the root"""
  69     global logger
  70     logger.info("Deleting tmpdir "+tmpdir)
  71     os.chdir("/")
  72     try:
  73         shutil.rmtree(tmpdir)
  74     except OSError:
  75         print logger.warn("Couldn't delete tmpdir %s. Don't forget to remove it manually." % (tmpdir))
  76
  77
  78 def checkout(url, revision=None):
  79     """Checks out the given URL at the given revision, using HEAD if not defined. Returns the working copy directory"""
  80     global logger
  81     # Create a temp dir to hold our working copy
  82     wc_dir = tempfile.mkdtemp(prefix=prog_name)
  83     atexit.register(del_temp_tree, wc_dir)
  84
  85     if (revision):
  86         url += "@"+revision
  87
  88     # Check out
  89     logger.info("Checking out "+url+" to "+wc_dir)
  90     returncode = call_cmd(["svn", "checkout", url, wc_dir])
  91
  92     if (returncode == 1):
  93         return None
  94     else:
  95         return wc_dir
  96
  97 def merge(wc_dir, revision_from, revision_to):
  98     """Merges repo_url from revision revision_from to revision revision_to into wc_dir"""
  99     global logger
 100     logger.info("Merging between revisions %s and %s into %s" % (revision_from, revision_to, wc_dir))
 101     os.chdir(wc_dir)
 102     return call_cmd(["svn", "merge", "-r", revision_from+":"+revision_to, wc_dir])
 103
 104 def treat_status(wc_dir_orig, wc_dir):
 105     """Copies modification from official vendor branch to wc"""
 106     global logger
 107     logger.info("Copying modification from official vendor branch %s to wc %s" % (wc_dir_orig, wc_dir))
 108     os.chdir(wc_dir_orig)
 109     status_tree = call_cmd_xml_tree_out(["svn", "status", "--xml"])
 110     global entries_to_treat, entries_to_delete
 111     entries_to_treat = status_tree.xpath("/status/target/entry")
 112     entries_to_delete = []
 113
 114     while len(entries_to_treat) > 0:
 115         entry = entries_to_treat.pop(0)
 116         entry_type = get_entry_type(entry)
 117         file = get_entry_path(entry)
 118         if entry_type == 'added':
 119             if is_entry_copied(entry):
 120                 check_exit(copy(wc_dir_orig, wc_dir, file), "Error during copy")
 121             else:
 122                 check_exit(add(wc_dir_orig, wc_dir, file), "Error during add")
 123         elif entry_type == 'deleted':
 124             entries_to_delete.append(entry)
 125         elif entry_type == 'modified' or entry_type == 'replaced':
 126             check_exit(update(wc_dir_orig, wc_dir, file), "Error during update")
 127         elif entry_type == 'normal':
 128             logger.info("File %s has a 'normal' state (unchanged). Ignoring." % (file))
 129         else:
 130             logger.error("Status not understood : '%s' not supported (file : %s)" % (entry_type, file))
 131
 132     # We then treat the left deletions
 133     for entry in entries_to_delete:
 134         check_exit(delete(wc_dir_orig, wc_dir, get_entry_path(entry)), "Error during delete")
 135
 136     return 0
 137
 138 def get_entry_type(entry):
 139     return get_xml_text_content(entry, "wc-status/@item")
 140
 141 def get_entry_path(entry):
 142     return get_xml_text_content(entry, "@path")
 143
 144 def is_entry_copied(entry):
 145     return get_xml_text_content(entry, "wc-status/@copied") == 'true'
 146
 147 def copy(wc_dir_orig, wc_dir, file):
 148     global logger
 149     logger.info("A+ %s" % (file))
 150
 151     # Retreiving the original URL
 152     os.chdir(wc_dir_orig)
 153     info_tree = call_cmd_xml_tree_out(["svn", "info", "--xml", os.path.join(wc_dir_orig, file)])
 154     url = get_xml_text_content(info_tree, "/info/entry/url")
 155
 156     # Detecting original svn root
 157     global orig_svn_subroot
 158     if not orig_svn_subroot:
 159         orig_svn_root = get_xml_text_content(info_tree, "/info/entry/repository/root")
 160         #print >>sys.stderr, "url : %s" % (url)
 161         sub_url = url.split(orig_svn_root)[-1]
 162         sub_url = os.path.normpath(sub_url)
 163         #print >>sys.stderr, "sub_url : %s" % (sub_url)
 164         if sub_url.startswith(os.path.sep):
 165             sub_url = sub_url[1:]
 166
 167         orig_svn_subroot = '/'+sub_url.split(file)[0].replace(os.path.sep, '/')
 168         #print >>sys.stderr, "orig_svn_subroot : %s" % (orig_svn_subroot)
 169
 170     global log_tree
 171     if not log_tree:
 172         # Detecting original file copy path
 173         os.chdir(wc_dir_orig)
 174         orig_svn_root_subroot = get_xml_text_content(info_tree, "/info/entry/repository/root") + orig_svn_subroot
 175         real_from = str(int(r_from)+1)
 176         logger.info("Retreiving log of the original trunk %s between revisions %s and %s ..." % (orig_svn_root_subroot, real_from, r_to))
 177         log_tree = call_cmd_xml_tree_out(["svn", "log", "--xml", "-v", "-r", "%s:%s" % (real_from, r_to), orig_svn_root_subroot])
 178
 179     # Detecting the path of the original moved or copied file
 180     orig_url_file = orig_svn_subroot+file.replace(os.path.sep, '/')
 181     orig_url_file_old = None
 182     #print >>sys.stderr, "  orig_url_file : %s" % (orig_url_file)
 183     while orig_url_file:
 184         orig_url_file_old = orig_url_file
 185         orig_url_file = get_xml_text_content(log_tree, "//path[(@action='R' or @action='A') and text()='%s']/@copyfrom-path" % (orig_url_file))
 186         logger.debug("orig_url_file : %s" % (orig_url_file))
 187     orig_url_file = orig_url_file_old
 188
 189     # Getting the relative url for the original url file
 190     if orig_url_file:
 191         orig_file = convert_relative_url_to_path(orig_url_file)
 192     else:
 193         orig_file = None
 194     global base_copied_paths, added_paths
 195     # If there is no "moved origin" for that file, or the origin doesn't exist in the working directory, or the origin is the same as the given file, or the origin is an added file
 196     if not orig_url_file or (orig_file and (not os.path.exists(os.path.join(wc_dir, orig_file)) or orig_file == file or orig_file in added_paths)):
 197         # Check if the file is within a recently copied path
 198         for path in base_copied_paths:
 199             if file.startswith(path):
 200                 logger.warn("The path %s to add is a sub-path of recently copied %s. Ignoring the A+." % (file, path))
 201                 return 0
 202         # Simple add the file
 203         logger.warn("Log paths for the file %s don't correspond with any file in the wc. Will do a simple A." % (file))
 204         return add(wc_dir_orig, wc_dir, file)
 205
 206     # We catch the relative URL for the original file
 207     orig_file = convert_relative_url_to_path(orig_url_file)
 208
 209     # Detect if it's a move
 210     cmd = 'copy'
 211     global entries_to_treat, entries_to_delete
 212     if search_and_remove_delete_entry(entries_to_treat, orig_file) or search_and_remove_delete_entry(entries_to_delete, orig_file):
 213         # It's a move, removing the delete, and treating it as a move
 214         cmd = 'move'
 215
 216     logger.info("%s from %s" % (cmd, orig_url_file))
 217     returncode = call_cmd(["svn", cmd, os.path.join(wc_dir, orig_file), os.path.join(wc_dir, file)])
 218     if returncode == 0:
 219         if os.path.isdir(os.path.join(wc_dir, orig_file)):
 220             base_copied_paths.append(file)
 221         else:
 222             # Copy the last version of the file from the original repository
 223             shutil.copy(os.path.join(wc_dir_orig, file), os.path.join(wc_dir, file))
 224     return returncode
 225
 226 def search_and_remove_delete_entry(entries, orig_file):
 227     for entry in entries:
 228         if get_entry_type(entry) == 'deleted' and get_entry_path(entry) == orig_file:
 229             entries.remove(entry)
 230             return True
 231     return False
 232
 233 def convert_relative_url_to_path(url):
 234     global orig_svn_subroot
 235     return os.path.normpath(url.split(orig_svn_subroot)[-1])
 236
 237 def new_added_path(returncode, file):
 238     if not is_returncode_bad(returncode):
 239         global added_paths
 240         added_paths.append(file)
 241
 242 def add(wc_dir_orig, wc_dir, file):
 243     global logger
 244     logger.info("A  %s" % (file))
 245     if os.path.exists(os.path.join(wc_dir, file)):
 246         logger.warn("Target file %s already exists. Will do a simple M" % (file))
 247         return update(wc_dir_orig, wc_dir, file)
 248     os.chdir(wc_dir)
 249     if os.path.isdir(os.path.join(wc_dir_orig, file)):
 250         returncode = call_cmd(["svn", "mkdir", file])
 251         new_added_path(returncode, file)
 252         return returncode
 253     else:
 254         shutil.copy(os.path.join(wc_dir_orig, file), os.path.join(wc_dir, file))
 255         returncode = call_cmd(["svn", "add", file])
 256         new_added_path(returncode, file)
 257         return returncode
 258
 259 def delete(wc_dir_orig, wc_dir, file):
 260     global logger
 261     logger.info("D  %s" % (file))
 262     os.chdir(wc_dir)
 263     if not os.path.exists(file):
 264         logger.warn("File %s doesn't exist. Ignoring D." % (file))
 265         return 0
 266     return call_cmd(["svn", "delete", file])
 267
 268 def update(wc_dir_orig, wc_dir, file):
 269     global logger
 270     logger.info("M  %s" % (file))
 271     if os.path.isdir(os.path.join(wc_dir_orig, file)):
 272         logger.warn("%s is a directory. Ignoring M." % (file))
 273         return 0
 274     shutil.copy(os.path.join(wc_dir_orig, file), os.path.join(wc_dir, file))
 275     return 0
 276
 277 def fine_tune(wc_dir):
 278     """Gives the user a chance to fine-tune"""
 279     alert(["If you want to fine-tune import, do so in working copy located at : %s" % (wc_dir),
 280         "When done, press Enter to commit, or Ctrl-C to abort."])
 281
 282 def alert(messages):
 283     """Wait the user to <ENTER> or abort the program"""
 284     for message in messages:
 285         print >> sys.stderr, message
 286     try:
 287         return sys.stdin.readline()
 288     except KeyboardInterrupt:
 289         sys.exit(0)
 290
 291 def commit(wc_dir, message):
 292     """Commits the wc_dir"""
 293     os.chdir(wc_dir)
 294     cmd = ["svn", "commit"]
 295     if (message):
 296         cmd += ["-m", message]
 297     return call_cmd(cmd)
 298
 299 def tag_wc(repo_url, current, tag, message):
 300     """Tags the wc_dir"""
 301     cmd = ["svn", "copy"]
 302     if (message):
 303         cmd += ["-m", message]
 304     return call_cmd(cmd + [repo_url+"/"+current, repo_url+"/"+tag])
 305
 306 def call_cmd(cmd):
 307     global logger
 308     logger.debug(string.join(cmd, ' '))
 309     return subprocess.call(cmd, stdout=DEVNULL, stderr=sys.stderr)#subprocess.STDOUT)
 310
 311 def call_cmd_out(cmd):
 312     global logger
 313     logger.debug(string.join(cmd, ' '))
 314     return subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=sys.stderr).stdout
 315
 316 def call_cmd_str_out(cmd):
 317     out = call_cmd_out(cmd)
 318     str_out = ""
 319     for line in out.readlines():
 320         str_out += line
 321     out.close()
 322     return str_out
 323
 324 def call_cmd_xml_tree_out(cmd):
 325     return etree.parse(StringIO(call_cmd_str_out(cmd)))
 326
 327 def get_xml_text_content(xml_doc, xpath):
 328     result_nodes = xml_doc.xpath(xpath)
 329     if result_nodes:
 330         if type(result_nodes[0]) == types.StringType:
 331             return result_nodes[0]
 332         else:
 333             return result_nodes[0].text
 334     else:
 335         return None
 336
 337 def usage(error = None):
 338     """Print usage message and exit"""
 339     print >>sys.stderr, """%s: Merges the difference between two revisions of the original repository of the vendor, to the vendor branch
 340 usage: %s [options] REPO_URL CURRENT_PATH ORIGINAL_REPO_URL -r N:M
 341
 342   - REPO_URL : repository URL for the vendor branch (i.e: http://svn.example.com/repos/vendor/libcomplex)
 343   - CURRENT_PATH : relative path to the current folder (i.e: current)
 344   - ORIGINAL_REPO_URL : original base repository URL
 345   - N:M : from revision N to revision M
 346
 347   This command executes these steps:
 348
 349   1. Check out directory specified by ORIGINAL_REPO_URL@N in a temporary directory.(1)
 350   2. Merges changes to revision M.(1)
 351   3. Check out directory specified by REPO_URL in a second temporary directory.(2)
 352   4. Treat the merge by "svn status" on the working copy of ORIGINAL_REPO_URL. If the history is kept ('+' when svn st), do a move instead of a delete / add.
 353   5. Allow user to fine-tune import.
 354   6. Commit.
 355   7. Optionally tag new release.
 356   8. Delete the temporary directories.
 357
 358   (1) : if -c wasn't passed
 359   (2) : if -w wasn't passed
 360
 361 Valid options:
 362   -r [--revision] N:M      : specify revisions N to M
 363   -h [--help]              : show this usage
 364   -t [--tag] arg           : copy new release to directory ARG, relative to REPO_URL,
 365                              using automatic commit message. Example:
 366                              -t ../0.42
 367   --non-interactive        : do no interactive prompting, do not allow manual fine-tune
 368   -m [--message] arg       : specify commit message ARG
 369   -v [--verbose]           : verbose mode
 370   -c [--merged-vendor] arg : working copy path of the original already merged vendor trunk (skips the steps 1. and 2.)
 371   -w [--current-wc] arg    : working copy path of the current checked out trunk of the vendor branch (skips the step 3.)
 372     """ % ((prog_name,) * 2)
 373
 374     if error:
 375         print >>sys.stder, "", "Current error : "+error
 376
 377     sys.exit(1)
 378
 379 def main():
 380     tag = None
 381     message = None
 382     interactive = 1
 383     revision_to_parse = None
 384     merged_vendor = None
 385     wc_dir = None
 386
 387     # Initializing logger
 388     global logger
 389     logger = logging.getLogger('svn-merge-vendor')
 390     hdlr = logging.StreamHandler(sys.stderr)
 391     formatter = logging.Formatter('%(levelname)-8s %(message)s')
 392     hdlr.setFormatter(formatter)
 393     logger.addHandler(hdlr)
 394     logger.setLevel(logging.INFO)
 395
 396     try:
 397         opts, args = getopt.gnu_getopt(sys.argv[1:], "ht:m:vr:c:w:",
 398                                        ["help", "tag", "message", "non-interactive", "verbose", "revision", "merged-vendor", "current-wc"])
 399     except getopt.GetoptError:
 400         # print help information and exit:
 401         usage()
 402
 403     for o, a in opts:
 404         if o in ("-h", "--help"):
 405             usage()
 406         if o in ("-t", "--tag"):
 407             tag = a
 408         if o in ("-m", "--message"):
 409             message = a
 410         if o in ("--non-interactive"):
 411             interactive = 0
 412         if o in ("-v", "--verbose"):
 413             logger.setLevel(logging.DEBUG)
 414         if o in ("-r", "--revision"):
 415             revision_to_parse = a
 416         if o in ("-c", "--merged-vendor"):
 417             merged_vendor = a
 418         if o in ("-w", "--current-wc"):
 419             wc_dir = a
 420
 421     if len(args) != 3:
 422         usage()
 423
 424     repo_url, current_path, orig_repo_url = args[0:3]
 425
 426     if (not revision_to_parse):
 427         usage("the revision numbers are mendatory")
 428     global r_from, r_to
 429     r_from, r_to = re.match("(\d+):(\d+)", revision_to_parse).groups()
 430
 431     if not r_from or not r_to:
 432         usage("the revision numbers are mendatory")
 433
 434     try:
 435         r_from_int = int(r_from)
 436         r_to_int = int(r_to)
 437     except ValueError:
 438         usage("the revision parameter is not a number")
 439
 440     if r_from_int >= r_to_int:
 441         usage("the 'from revision' must be inferior to the 'to revision'")
 442
 443     if not merged_vendor:
 444         if orig_repo_url.startswith("http://"):
 445             wc_dir_orig = checkout(orig_repo_url, r_from)
 446             check_exit(wc_dir_orig, "Error during checkout")
 447
 448             check_exit(merge(wc_dir_orig, r_from, r_to), "Error during merge")
 449         else:
 450             usage("ORIGINAL_REPO_URL must start with 'http://'")
 451     else:
 452         wc_dir_orig = merged_vendor
 453
 454     if not wc_dir:
 455         wc_dir = checkout(repo_url+"/"+current_path)
 456         check_exit(wc_dir, "Error during checkout")
 457
 458     check_exit(treat_status(wc_dir_orig, wc_dir), "Error during resolving")
 459
 460     if (interactive):
 461         fine_tune(wc_dir)
 462
 463     if not message:
 464         message = "New vendor version, upgrading from revision %s to revision %s" % (r_from, r_to)
 465         alert(["No message was specified to commit, the program will use that default one : '%s'" % (message),
 466             "Press Enter to commit, or Ctrl-C to abort."])
 467
 468     check_exit(commit(wc_dir, message), "Error during commit")
 469
 470     if tag:
 471         if not message:
 472             message = "Tag %s, when upgrading the vendor branch from revision %s to revision %s" % (tag, r_from, r_to)
 473             alert(["No message was specified to tag, the program will use that default one : '%s'" % (message),
 474                 "Press Enter to tag, or Ctrl-C to abort."])
 475         check_exit(tag_wc(repo_url, current_path, tag, message), "Error during tag")
 476
 477     logger.info("Vendor branch merged, passed from %s to %s !" % (r_from, r_to))
 478
 479 def is_returncode_bad(returncode):
 480     return returncode is None or returncode == 1
 481
 482 def check_exit(returncode, message):
 483     global logger
 484     if is_returncode_bad(returncode):
 485         logger.error(message)
 486         sys.exit(1)
 487
 488 if __name__ == "__main__":
 489     if (os.name == "nt"):
 490         DEVNULL = open("nul:", "w")
 491     else:
 492         DEVNULL = open("/dev/null", "w")
 493     main()