utils/git-svn/git-llvm

   1 #!/usr/bin/env python
   2 #
   3 # ======- git-llvm - LLVM Git Help Integration ---------*- python -*--========#
   4 #
   5 # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   6 # See https://llvm.org/LICENSE.txt for license information.
   7 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   8 #
   9 # ==------------------------------------------------------------------------==#
  10
  11 """
  12 git-llvm integration
  13 ====================
  14
  15 This file provides integration for git.
  16 """
  17
  18 from __future__ import print_function
  19 import argparse
  20 import collections
  21 import contextlib
  22 import errno
  23 import os
  24 import re
  25 import subprocess
  26 import sys
  27 import tempfile
  28 import time
  29 assert sys.version_info >= (2, 7)
  30
  31 try:
  32     dict.iteritems
  33 except AttributeError:
  34     # Python 3
  35     def iteritems(d):
  36         return iter(d.items())
  37 else:
  38     # Python 2
  39     def iteritems(d):
  40         return d.iteritems()
  41
  42 # It's *almost* a straightforward mapping from the monorepo to svn...
  43 GIT_TO_SVN_DIR = {
  44     d: (d + '/trunk')
  45     for d in [
  46         'clang-tools-extra',
  47         'compiler-rt',
  48         'debuginfo-tests',
  49         'dragonegg',
  50         'klee',
  51         'libclc',
  52         'libcxx',
  53         'libcxxabi',
  54         'libunwind',
  55         'lld',
  56         'lldb',
  57         'llgo',
  58         'llvm',
  59         'openmp',
  60         'parallel-libs',
  61         'polly',
  62         'pstl',
  63     ]
  64 }
  65 GIT_TO_SVN_DIR.update({'clang': 'cfe/trunk'})
  66 GIT_TO_SVN_DIR.update({'': 'monorepo-root/trunk'})
  67
  68 VERBOSE = False
  69 QUIET = False
  70 dev_null_fd = None
  71
  72
  73 def eprint(*args, **kwargs):
  74     print(*args, file=sys.stderr, **kwargs)
  75
  76
  77 def log(*args, **kwargs):
  78     if QUIET:
  79         return
  80     print(*args, **kwargs)
  81
  82
  83 def log_verbose(*args, **kwargs):
  84     if not VERBOSE:
  85         return
  86     print(*args, **kwargs)
  87
  88
  89 def die(msg):
  90     eprint(msg)
  91     sys.exit(1)
  92
  93
  94 def split_first_path_component(d):
  95     # Assuming we have a git path, it'll use slashes even on windows...I hope.
  96     if '/' in d:
  97         return d.split('/', 1)
  98     else:
  99         return (d, None)
 100
 101
 102 def get_dev_null():
 103     """Lazily create a /dev/null fd for use in shell()"""
 104     global dev_null_fd
 105     if dev_null_fd is None:
 106         dev_null_fd = open(os.devnull, 'w')
 107     return dev_null_fd
 108
 109
 110 def shell(cmd, strip=True, cwd=None, stdin=None, die_on_failure=True,
 111           ignore_errors=False, text=True):
 112     log_verbose('Running in %s: %s' % (cwd, ' '.join(cmd)))
 113
 114     err_pipe = subprocess.PIPE
 115     if ignore_errors:
 116         # Silence errors if requested.
 117         err_pipe = get_dev_null()
 118
 119     start = time.time()
 120     p = subprocess.Popen(cmd, cwd=cwd, stdout=subprocess.PIPE, stderr=err_pipe,
 121                          stdin=subprocess.PIPE,
 122                          universal_newlines=text)
 123     stdout, stderr = p.communicate(input=stdin)
 124     elapsed = time.time() - start
 125
 126     log_verbose('Command took %0.1fs' % elapsed)
 127
 128     if p.returncode == 0 or ignore_errors:
 129         if stderr and not ignore_errors:
 130             eprint('`%s` printed to stderr:' % ' '.join(cmd))
 131             eprint(stderr.rstrip())
 132         if strip:
 133             if text:
 134                 stdout = stdout.rstrip('\r\n')
 135             else:
 136                 stdout = stdout.rstrip(b'\r\n')
 137         if VERBOSE:
 138             for l in stdout.splitlines():
 139                 log_verbose("STDOUT: %s" % l)
 140         return stdout
 141     err_msg = '`%s` returned %s' % (' '.join(cmd), p.returncode)
 142     eprint(err_msg)
 143     if stderr:
 144         eprint(stderr.rstrip())
 145     if die_on_failure:
 146         sys.exit(2)
 147     raise RuntimeError(err_msg)
 148
 149
 150 def git(*cmd, **kwargs):
 151     return shell(['git'] + list(cmd), **kwargs)
 152
 153
 154 def svn(cwd, *cmd, **kwargs):
 155     return shell(['svn'] + list(cmd), cwd=cwd, **kwargs)
 156
 157 def program_exists(cmd):
 158     if sys.platform == 'win32' and not cmd.endswith('.exe'):
 159         cmd += '.exe'
 160     for path in os.environ["PATH"].split(os.pathsep):
 161         if os.access(os.path.join(path, cmd), os.X_OK):
 162             return True
 163     return False
 164
 165 def get_default_rev_range():
 166     # Get the branch tracked by the current branch, as set by
 167     # git branch --set-upstream-to  See http://serverfault.com/a/352236/38694.
 168     cur_branch = git('rev-parse', '--symbolic-full-name', 'HEAD')
 169     upstream_branch = git('for-each-ref', '--format=%(upstream:short)',
 170                           cur_branch)
 171     if not upstream_branch:
 172         upstream_branch = 'origin/master'
 173
 174     # Get the newest common ancestor between HEAD and our upstream branch.
 175     upstream_rev = git('merge-base', 'HEAD', upstream_branch)
 176     return '%s..' % upstream_rev
 177
 178
 179 def get_revs_to_push(rev_range):
 180     if not rev_range:
 181         rev_range = get_default_rev_range()
 182     # Use git show rather than some plumbing command to figure out which revs
 183     # are in rev_range because it handles single revs (HEAD^) and ranges
 184     # (foo..bar) like we want.
 185     revs = git('show', '--reverse', '--quiet',
 186                '--pretty=%h', rev_range).splitlines()
 187     if not revs:
 188         die('Nothing to push: No revs in range %s.' % rev_range)
 189     return revs
 190
 191
 192 def clean_svn(svn_repo):
 193     svn(svn_repo, 'revert', '-R', '.')
 194
 195     # Unfortunately it appears there's no svn equivalent for git clean, so we
 196     # have to do it ourselves.
 197     for line in svn(svn_repo, 'status', '--no-ignore').split('\n'):
 198         if not line.startswith('?'):
 199             continue
 200         filename = line[1:].strip()
 201         os.remove(os.path.join(svn_repo, filename))
 202
 203
 204 def svn_init(svn_root):
 205     if not os.path.exists(svn_root):
 206         log('Creating svn staging directory: (%s)' % (svn_root))
 207         os.makedirs(svn_root)
 208         svn(svn_root, 'checkout', '--depth=empty',
 209             'https://llvm.org/svn/llvm-project/', '.')
 210         log("svn staging area ready in '%s'" % svn_root)
 211     if not os.path.isdir(svn_root):
 212         die("Can't initialize svn staging dir (%s)" % svn_root)
 213
 214
 215 def fix_eol_style_native(rev, svn_sr_path, files):
 216     """Fix line endings before applying patches with Unix endings
 217
 218     SVN on Windows will check out files with CRLF for files with the
 219     svn:eol-style property set to "native". This breaks `git apply`, which
 220     typically works with Unix-line ending patches. Work around the problem here
 221     by doing a dos2unix up front for files with svn:eol-style set to "native".
 222     SVN will not commit a mass line ending re-doing because it detects the line
 223     ending format for files with this property.
 224     """
 225     # Skip files that don't exist in SVN yet.
 226     files = [f for f in files if os.path.exists(os.path.join(svn_sr_path, f))]
 227     # Use ignore_errors because 'svn propget' prints errors if the file doesn't
 228     # have the named property. There doesn't seem to be a way to suppress that.
 229     eol_props = svn(svn_sr_path, 'propget', 'svn:eol-style', *files,
 230                     ignore_errors=True)
 231     crlf_files = []
 232     if len(files) == 1:
 233         # No need to split propget output on ' - ' when we have one file.
 234         if eol_props.strip() in ['native', 'CRLF']:
 235             crlf_files = files
 236     else:
 237         for eol_prop in eol_props.split('\n'):
 238             # Remove spare CR.
 239             eol_prop = eol_prop.strip('\r')
 240             if not eol_prop:
 241                 continue
 242             prop_parts = eol_prop.rsplit(' - ', 1)
 243             if len(prop_parts) != 2:
 244                 eprint("unable to parse svn propget line:")
 245                 eprint(eol_prop)
 246                 continue
 247             (f, eol_style) = prop_parts
 248             if eol_style == 'native':
 249                 crlf_files.append(f)
 250     if crlf_files:
 251         # Reformat all files with native SVN line endings to Unix format. SVN
 252         # knows files with native line endings are text files. It will commit
 253         # just the diff, and not a mass line ending change.
 254         shell(['dos2unix'] + crlf_files, ignore_errors=True, cwd=svn_sr_path)
 255
 256 def split_subrepo(f):
 257     # Given a path, splits it into (subproject, rest-of-path). If the path is
 258     # not in a subproject, returns ('', full-path).
 259
 260     subproject, remainder = split_first_path_component(f)
 261
 262     if subproject in GIT_TO_SVN_DIR:
 263         return subproject, remainder
 264     else:
 265         return '', f
 266
 267 def get_all_parent_dirs(name):
 268     parts = []
 269     head, tail = os.path.split(name)
 270     while head:
 271         parts.append(head)
 272         head, tail = os.path.split(head)
 273     return parts
 274
 275 def svn_push_one_rev(svn_repo, rev, dry_run):
 276     files = git('diff-tree', '--no-commit-id', '--name-only', '-r',
 277                 rev).split('\n')
 278     if not files:
 279         raise RuntimeError('Empty diff for rev %s?' % rev)
 280
 281     # Split files by subrepo
 282     subrepo_files = collections.defaultdict(list)
 283     for f in files:
 284         subrepo, remainder = split_subrepo(f)
 285         subrepo_files[subrepo].append(remainder)
 286
 287     status = svn(svn_repo, 'status', '--no-ignore')
 288     if status:
 289         die("Can't push git rev %s because svn status is not empty:\n%s" %
 290             (rev, status))
 291
 292     svn_dirs_to_update = set()
 293     for sr, files in iteritems(subrepo_files):
 294         svn_sr_path = GIT_TO_SVN_DIR[sr]
 295         for f in files:
 296             svn_dirs_to_update.add(
 297                 os.path.dirname(os.path.join(svn_sr_path, f)))
 298
 299     # We also need to svn update any parent directories which are not yet present
 300     parent_dirs = set()
 301     for dir in svn_dirs_to_update:
 302         parent_dirs.update(get_all_parent_dirs(dir))
 303     parent_dirs = set(dir for dir in parent_dirs
 304                       if not os.path.exists(os.path.join(svn_repo, dir)))
 305     svn_dirs_to_update.update(parent_dirs)
 306
 307     # Sort by length to ensure that the parent directories are passed to svn
 308     # before child directories.
 309     sorted_dirs_to_update = sorted(svn_dirs_to_update, key=len)
 310
 311     # SVN update only in the affected directories.
 312     svn(svn_repo, 'update', '--depth=files', *sorted_dirs_to_update)
 313
 314     for sr, files in iteritems(subrepo_files):
 315         svn_sr_path = os.path.join(svn_repo, GIT_TO_SVN_DIR[sr])
 316         if os.name == 'nt':
 317             fix_eol_style_native(rev, svn_sr_path, files)
 318         # We use text=False (and pass '--binary') so that we can get an exact
 319         # diff that can be passed as-is to 'git apply' without any line ending,
 320         # encoding, or other mangling.
 321         diff = git('show', '--binary', rev, '--',
 322                    *(os.path.join(sr, f) for f in files),
 323                    strip=False, text=False)
 324         # git is the only thing that can handle its own patches...
 325         if sr == '':
 326             prefix_strip = '-p1'
 327         else:
 328             prefix_strip = '-p2'
 329         try:
 330             shell(['git', 'apply', prefix_strip, '-'], cwd=svn_sr_path,
 331                   stdin=diff, die_on_failure=False, text=False)
 332         except RuntimeError as e:
 333             eprint("Patch doesn't apply: maybe you should try `git pull -r` "
 334                    "first?")
 335             sys.exit(2)
 336
 337     status_lines = svn(svn_repo, 'status', '--no-ignore').split('\n')
 338
 339     for l in (l for l in status_lines if (l.startswith('?') or
 340                                           l.startswith('I'))):
 341         svn(svn_repo, 'add', '--no-ignore', l[1:].strip())
 342     for l in (l for l in status_lines if l.startswith('!')):
 343         svn(svn_repo, 'remove', l[1:].strip())
 344
 345     # Now we're ready to commit.
 346     commit_msg = git('show', '--pretty=%B', '--quiet', rev)
 347     if not dry_run:
 348         log(svn(svn_repo, 'commit', '-m', commit_msg, '--force-interactive'))
 349         log('Committed %s to svn.' % rev)
 350     else:
 351         log("Would have committed %s to svn, if this weren't a dry run." % rev)
 352
 353
 354 def cmd_push(args):
 355     '''Push changes back to SVN: this is extracted from Justin Lebar's script
 356     available here: https://github.com/jlebar/llvm-repo-tools/
 357
 358     Note: a current limitation is that git does not track file rename, so they
 359     will show up in SVN as delete+add.
 360     '''
 361     # Get the git root
 362     git_root = git('rev-parse', '--show-toplevel')
 363     if not os.path.isdir(git_root):
 364         die("Can't find git root dir")
 365
 366     # Push from the root of the git repo
 367     os.chdir(git_root)
 368
 369     # We need a staging area for SVN, let's hide it in the .git directory.
 370     dot_git_dir = git('rev-parse', '--git-common-dir')
 371     svn_root = os.path.join(dot_git_dir, 'llvm-upstream-svn')
 372     svn_init(svn_root)
 373
 374     rev_range = args.rev_range
 375     dry_run = args.dry_run
 376     revs = get_revs_to_push(rev_range)
 377     log('Pushing %d commit%s:\n%s' %
 378         (len(revs), 's' if len(revs) != 1
 379          else '', '\n'.join('  ' + git('show', '--oneline', '--quiet', c)
 380                             for c in revs)))
 381     for r in revs:
 382         clean_svn(svn_root)
 383         svn_push_one_rev(svn_root, r, dry_run)
 384
 385
 386 if __name__ == '__main__':
 387     if not program_exists('svn'):
 388         die('error: git-llvm needs svn command, but svn is not installed.')
 389
 390     argv = sys.argv[1:]
 391     p = argparse.ArgumentParser(
 392         prog='git llvm', formatter_class=argparse.RawDescriptionHelpFormatter,
 393         description=__doc__)
 394     subcommands = p.add_subparsers(title='subcommands',
 395                                    description='valid subcommands',
 396                                    help='additional help')
 397     verbosity_group = p.add_mutually_exclusive_group()
 398     verbosity_group.add_argument('-q', '--quiet', action='store_true',
 399                                  help='print less information')
 400     verbosity_group.add_argument('-v', '--verbose', action='store_true',
 401                                  help='print more information')
 402
 403     parser_push = subcommands.add_parser(
 404         'push', description=cmd_push.__doc__,
 405         help='push changes back to the LLVM SVN repository')
 406     parser_push.add_argument(
 407         '-n',
 408         '--dry-run',
 409         dest='dry_run',
 410         action='store_true',
 411         help='Do everything other than commit to svn.  Leaves junk in the svn '
 412         'repo, so probably will not work well if you try to commit more '
 413         'than one rev.')
 414     parser_push.add_argument(
 415         'rev_range',
 416         metavar='GIT_REVS',
 417         type=str,
 418         nargs='?',
 419         help="revs to push (default: everything not in the branch's "
 420         'upstream, or not in origin/master if the branch lacks '
 421         'an explicit upstream)')
 422     parser_push.set_defaults(func=cmd_push)
 423     args = p.parse_args(argv)
 424     VERBOSE = args.verbose
 425     QUIET = args.quiet
 426
 427     # Dispatch to the right subcommand
 428     args.func(args)