plugins/svn.py

   1
   2 from yap.yap import YapCore, YapError
   3 from yap.util import get_output, takes_options, run_command, run_safely, short_help
   4
   5 import os
   6 import tempfile
   7 import glob
   8 import pickle
   9 import re
  10 import bisect
  11 import struct
  12
  13 class RepoBlob(object):
  14     def __init__(self, keys):
  15         self.keys = keys
  16
  17         self.uuid = None
  18         self.branches = None
  19         self.tags = None
  20         self.metadata = {}
  21
  22     def add_metadata(self, branch):
  23         assert branch not in self.metadata
  24         gitdir = get_output("git rev-parse --git-dir")
  25         assert gitdir
  26         revmap = os.path.join(gitdir[0], "svn", "svn", branch, ".rev_map*")
  27         revmap = glob.glob(revmap)
  28         if not revmap:
  29             return
  30         uuid = revmap[0].split('.')[-1]
  31         if self.uuid is None:
  32             self.uuid = uuid
  33         assert self.uuid == uuid
  34         rev = get_output("git rev-parse refs/remotes/svn/%s" % branch)
  35         data = file(revmap[0]).read()
  36         self.metadata[branch] = rev[0], data
  37
  38 # Helper class for dealing with SVN metadata
  39 class SVNRevMap(object):
  40     RECORD_SZ = 24
  41     def __init__(self, filename):
  42         self.fd = file(filename, "rb")
  43         size = os.stat(filename)[6]
  44         self.nrecords = size / self.RECORD_SZ
  45
  46     def __getitem__(self, index):
  47         if index >= self.nrecords:
  48             raise IndexError
  49         return self.get_record(index)[0]
  50
  51     def __len__(self):
  52         return self.nrecords
  53
  54     def get_record(self, index):
  55         self.fd.seek(index * self.RECORD_SZ)
  56         record = self.fd.read(self.RECORD_SZ)
  57         return self.parse_record(record)
  58
  59     def parse_record(self, record):
  60         record = struct.unpack("!I20B", record)
  61         rev = record[0]
  62         hash = map(lambda x: "%02x" % x, record[1:])
  63         hash = ''.join(hash)
  64         return rev, hash
  65
  66 class SvnPlugin(YapCore):
  67     "Allow yap to interoperate with Subversion repositories"
  68
  69     revpat = re.compile('^r(\d+)$')
  70
  71     def __init__(self, *args, **flags):
  72         super(SvnPlugin, self).__init__(*args, **flags)
  73         self._svn_next_rev = None
  74
  75     def _get_root(self, url):
  76         root = get_output("svn info %s 2>/dev/null | gawk '/Repository Root:/{print $3}'" % url)
  77         if not root:
  78             raise YapError("Not an SVN repo: %s" % url)
  79         return root[0]
  80
  81     def _configure_repo(self, url, fetch=None):
  82         root = self._get_root(url)
  83         os.system("git config svn-remote.svn.url %s" % root)
  84         if fetch is None:
  85             trunk = url.replace(root, '').strip('/')
  86         else:
  87             trunk = fetch.split(':')[0]
  88         os.system("git config svn-remote.svn.fetch %s:refs/remotes/svn/trunk"
  89                 % trunk)
  90
  91         branches = trunk.replace('trunk', 'branches')
  92         if branches != trunk:
  93             os.system("git config svn-remote.svn.branches %s/*:refs/remotes/svn/*" % branches)
  94         tags = trunk.replace('trunk', 'tags')
  95         if tags != trunk:
  96             os.system("git config svn-remote.svn.tags %s/*:refs/remotes/svn/tags/*" % tags)
  97         self.cmd_repo("svn", url)
  98         os.system("git config yap.svn.enabled 1")
  99
 100     def _create_tagged_blob(self):
 101         keys = dict()
 102         for i in get_output("git config --list | grep ^svn-remote."):
 103             k, v = i.split('=')
 104             keys[k] = v
 105         blob = RepoBlob(keys)
 106         for b in get_output("git for-each-ref --format='%(refname)' 'refs/remotes/svn/*'"):
 107             b = b.replace('refs/remotes/svn/', '')
 108             blob.add_metadata(b)
 109
 110         fd_w, fd_r = os.popen2("git hash-object -w --stdin")
 111         pickle.dump(blob, fd_w)
 112         fd_w.close()
 113         hash = fd_r.readline().strip()
 114         run_safely("git tag -f yap-svn %s" % hash)
 115
 116     def _cleanup_branches(self):
 117         for b in get_output("git for-each-ref --format='%(refname)' 'refs/remotes/svn/*@*'"):
 118             head = b.replace('refs/remotes/svn/', '')
 119             path = os.path.join(".git", "svn", "svn", head)
 120             files = os.listdir(path)
 121             for f in files:
 122                 os.unlink(os.path.join(path, f))
 123             os.rmdir(path)
 124
 125             ref = get_output("git rev-parse %s" % b)
 126             if ref:
 127                 run_safely("git update-ref -d %s %s" % (b, ref[0]))
 128
 129     def _clone_svn(self, url, directory=None, **flags):
 130         url = url.rstrip('/')
 131         if directory is None:
 132             directory = url.rsplit('/')[-1]
 133             directory = directory.replace('.git', '')
 134
 135         try:
 136             os.mkdir(directory)
 137         except OSError:
 138             raise YapError("Directory exists: %s" % directory)
 139         os.chdir(directory)
 140
 141         self.cmd_init()
 142         self._configure_repo(url)
 143         os.system("git svn fetch -r %s:HEAD" % flags.get('-r', '1'))
 144
 145         self._cleanup_branches()
 146         self._create_tagged_blob()
 147
 148     def _push_svn(self, branch, **flags):
 149         if '-d' in flags:
 150             raise YapError("Deleting svn branches not supported")
 151         print "Verifying branch is up-to-date"
 152         run_safely("git svn fetch svn")
 153
 154         branch = branch.replace('refs/heads/', '')
 155         rev = get_output("git rev-parse --verify refs/remotes/svn/%s" % branch)
 156
 157         # Create the branch if requested
 158         if not rev:
 159             if '-c' not in flags:
 160                 raise YapError("No matching branch on the repo.  Use -c to create a new branch there.")
 161             src  = get_output("git svn info | gawk '/URL:/{print $2}'")[0]
 162             brev = get_output("git svn info | gawk '/Revision:/{print $2}'")[0]
 163             root = get_output("git config svn-remote.svn.url")[0]
 164             branch_path = get_output("git config svn-remote.svn.branches")[0].split(':')[0]
 165             branch_path = branch_path.rstrip('/*')
 166             dst = '/'.join((root, branch_path, branch))
 167
 168             # Create the branch in svn
 169             run_safely("svn cp -r%s %s %s -m 'create branch %s'"
 170                     % (brev, src, dst, branch))
 171             run_safely("git svn fetch svn")
 172             rev = get_output("git rev-parse refs/remotes/svn/%s 2>/dev/null" % branch)
 173             base = get_output("git svn find-rev r%s" % brev)
 174
 175             # Apply our commits to the new branch
 176             try:
 177                 fd, tmpfile = tempfile.mkstemp("yap")
 178                 os.close(fd)
 179                 os.system("git format-patch -k --stdout '%s' > %s"
 180                         % (base[0], tmpfile))
 181                 start = get_output("git rev-parse HEAD")
 182                 self.cmd_point("refs/remotes/svn/%s"
 183                         % branch, **{'-f': True})
 184
 185                 stat = os.stat(tmpfile)
 186                 size = stat[6]
 187                 if size > 0:
 188                     rc = run_command("git am -3 %s" % tmpfile)
 189                     if (rc):
 190                         self.cmd_point(start[0], **{'-f': True})
 191                         raise YapError("Failed to port changes to new svn branch")
 192             finally:
 193                 os.unlink(tmpfile)
 194
 195         base = get_output("git merge-base HEAD %s" % rev[0])
 196         if base[0] != rev[0]:
 197             raise YapError("Branch not up-to-date.  Update first.")
 198         current = get_output("git symbolic-ref HEAD")
 199         if not current:
 200             raise YapError("Not on a branch!")
 201         current = current[0].replace('refs/heads/', '')
 202         self._confirm_push(current, branch, "svn")
 203         if run_command("git update-index --refresh"):
 204             raise YapError("Can't push with uncommitted changes")
 205
 206         master = get_output("git rev-parse --verify refs/heads/master 2>/dev/null")
 207         os.system("git svn dcommit")
 208         run_safely("git svn rebase")
 209         if not master:
 210             master = get_output("git rev-parse --verify refs/heads/master 2>/dev/null")
 211             if master:
 212                 run_safely("git update-ref -d refs/heads/master %s" % master[0])
 213
 214     def _lock_svn(self):
 215         repo = get_output('git rev-parse --git-dir')[0]
 216         dir = os.path.join(repo, 'yap')
 217         fd, tmplock = tempfile.mkstemp("yap", dir=dir)
 218         try:
 219             os.close(fd)
 220
 221             lockfile = os.path.join(dir, 'svn-lock')
 222             try:
 223                 os.link(tmplock, lockfile)
 224             except OSError:
 225                 raise YapError("A subversion operation is already in progress")
 226         finally:
 227             os.unlink(tmplock)
 228
 229     def _unlock_svn(self):
 230         repo = get_output('git rev-parse --git-dir')[0]
 231         dir = os.path.join(repo, 'yap')
 232         lockfile = os.path.join(dir, 'svn-lock')
 233
 234         try:
 235             os.unlink(lockfile)
 236         except OSError:
 237             pass
 238
 239     def _fetch_svn(self):
 240         self._lock_svn()
 241         os.system("git svn fetch svn")
 242         self._unlock_svn()
 243         self._create_tagged_blob()
 244         self._cleanup_branches()
 245
 246     def _enabled(self):
 247         enabled = get_output("git config yap.svn.enabled")
 248         return bool(enabled)
 249
 250     def _applicable(self, args):
 251         if not self._enabled():
 252             return False
 253
 254         if args and args[0] == 'svn':
 255             return True
 256
 257         if not args:
 258             current = get_output("git symbolic-ref HEAD")
 259             if not current:
 260                 raise YapError("Not on a branch!")
 261
 262             current = current[0].replace('refs/heads/', '')
 263             remote, merge = self._get_tracking(current)
 264             if remote == "svn":
 265                 return True
 266
 267         return False
 268
 269     # Ensure users don't accidentally kill our "svn" repo
 270     def cmd_repo(self, *args, **flags):
 271         if self._enabled():
 272             if '-d' in flags and args and args[0] == "svn":
 273                 raise YapError("Refusing to delete special svn repository")
 274         super(SvnPlugin, self).cmd_repo(*args, **flags)
 275
 276     @takes_options("r:")
 277     def cmd_clone(self, *args, **flags):
 278         handled = True
 279         if not args:
 280             handled = False
 281         if (handled and not args[0].startswith("http")
 282                     and not args[0].startswith("svn")
 283                     and not args[0].startswith("file://")):
 284             handled = False
 285         if handled and run_command("svn info %s" % args[0]):
 286             handled = False
 287
 288         if handled:
 289             self._clone_svn(*args, **flags)
 290         else:
 291             super(SvnPlugin, self).cmd_clone(*args, **flags)
 292
 293         if self._enabled():
 294             # nothing to do
 295             return
 296
 297         run_safely("git fetch origin --tags")
 298         hash = get_output("git rev-parse --verify refs/tags/yap-svn 2>/dev/null")
 299         if not hash:
 300             return
 301
 302         fd = os.popen("git cat-file blob %s" % hash[0])
 303         blob = pickle.load(fd)
 304         for k, v in blob.keys.items():
 305             run_safely("git config %s %s" % (k, v))
 306
 307         self.cmd_repo("svn", blob.keys['svn-remote.svn.url'])
 308         os.system("git config yap.svn.enabled 1")
 309         run_safely("git fetch origin 'refs/remotes/svn/*:refs/remotes/svn/*'")
 310
 311         for b in blob.metadata.keys():
 312             branch = os.path.join(".git", "svn", "svn", b)
 313             os.makedirs(branch)
 314             fd = file(os.path.join(branch, ".rev_map.%s" % blob.uuid), "w")
 315
 316             rev, metadata = blob.metadata[b]
 317             fd.write(metadata)
 318             run_command("git update-ref refs/remotes/svn/%s %s" % (b, rev))
 319
 320     def cmd_fetch(self, *args, **flags):
 321         if self._applicable(args):
 322             self._fetch_svn()
 323             return
 324
 325         super(SvnPlugin, self).cmd_fetch(*args, **flags)
 326
 327     def cmd_push(self, *args, **flags):
 328         if self._applicable(args):
 329             if len (args) >= 2:
 330                 merge = args[1]
 331             else:
 332                 current = get_output("git symbolic-ref HEAD")
 333                 if not current:
 334                     raise YapError("Not on a branch!")
 335
 336                 current = current[0].replace('refs/heads/', '')
 337                 remote, merge = self._get_tracking(current)
 338                 if remote != "svn":
 339                     raise YapError("Need a branch name")
 340             self._push_svn(merge, **flags)
 341             return
 342         super(SvnPlugin, self).cmd_push(*args, **flags)
 343
 344     @short_help("change options for the svn plugin")
 345     def cmd_svn(self, subcmd):
 346         "enable"
 347
 348         if subcmd not in ["enable"]:
 349             raise TypeError
 350
 351         if "svn" in [x[0] for x in self._list_remotes()]:
 352             raise YapError("A remote named 'svn' already exists")
 353
 354
 355         if not run_command("git config svn-remote.svn.branches"):
 356             raise YapError("Cannot currently enable in a repository with svn branches")
 357
 358         url = get_output("git config svn-remote.svn.url")
 359         if not url:
 360             raise YapError("Not a git-svn repository?")
 361         fetch = get_output("git config svn-remote.svn.fetch")
 362         assert fetch
 363         lhs, rhs = fetch[0].split(':')
 364
 365
 366         rev = get_output("git rev-parse %s" % rhs)
 367         assert rev
 368         run_safely("git update-ref refs/remotes/svn/trunk %s" % rev[0])
 369
 370         url = '/'.join((url[0], lhs))
 371         self._configure_repo(url)
 372         run_safely("git update-ref -d %s %s" % (rhs, rev[0]))
 373
 374     # We are intentionally overriding yap utility functions
 375     def _filter_log(self, commit):
 376         commit = super(SvnPlugin, self)._filter_log(commit)
 377
 378         new = []
 379         for line in commit:
 380             if line.strip().startswith("git-svn-id:"):
 381                 while not new[-1].strip():
 382                     new = new[:-1]
 383
 384                 urlrev = line.strip().split(' ')[1]
 385                 url, rev = urlrev.split('@')
 386                 hash = commit[0].split(' ')[1].strip()
 387                 if self._svn_next_rev != hash:
 388                     h2 = self._resolve_svn_rev(int(rev))
 389                     if h2 != hash:
 390                         continue
 391
 392                 next_hash = get_output("git rev-parse --verify %s^" % hash)
 393                 if next_hash:
 394                     self._svn_next_rev = next_hash[0]
 395                 else:
 396                     self._svn_next_rev = None
 397                 root = get_output("git config svn-remote.svn.url")
 398                 assert root
 399                 url = url.replace(root[0], '')
 400                 new.insert(1, "Subversion: r%s %s\n" % (rev, url))
 401
 402                 continue
 403             new.append(line)
 404         return new
 405
 406     def _resolve_svn_rev(self, revnum):
 407         rev = None
 408         gitdir = get_output("git rev-parse --git-dir")
 409         assert gitdir
 410
 411         # Work with whateven svn remote is configured
 412         remotes = get_output("git config --get-regexp 'svn-remote.*.fetch'")
 413         assert remotes
 414
 415         revmaps = []
 416         for remote in remotes:
 417             remote = remote.split(' ')
 418             remote = remote[1].split(':')
 419             remote = remote[1].split('/')
 420             remote = remote[2]
 421             path = os.path.join(gitdir[0], "svn", remote,
 422                     "*", ".rev_map*")
 423             revmaps += glob.glob(path)
 424
 425         for f in revmaps:
 426             rm = SVNRevMap(f)
 427             idx = bisect.bisect_left(rm, revnum)
 428             if idx >= len(rm) or rm[idx] != revnum:
 429                 continue
 430
 431             revnum, rev = rm.get_record(idx)
 432             if hash == "0" * 40:
 433                 continue
 434             break
 435
 436         if not rev:
 437             rev = get_output("git svn find-rev r%d 2>/dev/null" % revnum)
 438         if not rev:
 439             rev = None
 440         return rev
 441
 442     def _resolve_rev(self, *args, **flags):
 443         rev = None
 444         if args:
 445             m = self.revpat.match(args[0])
 446             if m is not None:
 447                 revnum = int(m.group(1))
 448                 rev = self._resolve_svn_rev(revnum)
 449
 450         if not rev:
 451             rev = super(SvnPlugin, self)._resolve_rev(*args, **flags)
 452         return rev