utils/git-svn/git-llvm

   1 #!/usr/bin/env python
   2 #
   3 # ======- git-llvm - LLVM Git Help Integration ---------*- python -*--========#
   4 #
   5 # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   6 # See https://llvm.org/LICENSE.txt for license information.
   7 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   8 #
   9 # ==------------------------------------------------------------------------==#
  10
  11 """
  12 git-llvm integration
  13 ====================
  14
  15 This file provides integration for git.
  16 """
  17
  18 from __future__ import print_function
  19 import argparse
  20 import collections
  21 import contextlib
  22 import errno
  23 import os
  24 import re
  25 import shutil
  26 import subprocess
  27 import sys
  28 import tempfile
  29 import time
  30 assert sys.version_info >= (2, 7)
  31
  32 try:
  33     dict.iteritems
  34 except AttributeError:
  35     # Python 3
  36     def iteritems(d):
  37         return iter(d.items())
  38 else:
  39     # Python 2
  40     def iteritems(d):
  41         return d.iteritems()
  42
  43 try:
  44     # Python 3
  45     from shlex import quote
  46 except ImportError:
  47     # Python 2
  48     from pipes import quote
  49
  50 # It's *almost* a straightforward mapping from the monorepo to svn...
  51 GIT_TO_SVN_DIR = {
  52     d: (d + '/trunk')
  53     for d in [
  54         'clang-tools-extra',
  55         'compiler-rt',
  56         'debuginfo-tests',
  57         'dragonegg',
  58         'klee',
  59         'libclc',
  60         'libcxx',
  61         'libcxxabi',
  62         'libunwind',
  63         'lld',
  64         'lldb',
  65         'llgo',
  66         'llvm',
  67         'openmp',
  68         'parallel-libs',
  69         'polly',
  70         'pstl',
  71     ]
  72 }
  73 GIT_TO_SVN_DIR.update({'clang': 'cfe/trunk'})
  74 GIT_TO_SVN_DIR.update({'': 'monorepo-root/trunk'})
  75
  76 VERBOSE = False
  77 QUIET = False
  78 dev_null_fd = None
  79
  80
  81 def eprint(*args, **kwargs):
  82     print(*args, file=sys.stderr, **kwargs)
  83
  84
  85 def log(*args, **kwargs):
  86     if QUIET:
  87         return
  88     print(*args, **kwargs)
  89
  90
  91 def log_verbose(*args, **kwargs):
  92     if not VERBOSE:
  93         return
  94     print(*args, **kwargs)
  95
  96
  97 def die(msg):
  98     eprint(msg)
  99     sys.exit(1)
 100
 101
 102 def split_first_path_component(d):
 103     # Assuming we have a git path, it'll use slashes even on windows...I hope.
 104     if '/' in d:
 105         return d.split('/', 1)
 106     else:
 107         return (d, None)
 108
 109
 110 def get_dev_null():
 111     """Lazily create a /dev/null fd for use in shell()"""
 112     global dev_null_fd
 113     if dev_null_fd is None:
 114         dev_null_fd = open(os.devnull, 'w')
 115     return dev_null_fd
 116
 117
 118 def shell(cmd, strip=True, cwd=None, stdin=None, die_on_failure=True,
 119           ignore_errors=False, text=True):
 120     # Escape args when logging for easy repro.
 121     quoted_cmd = [quote(arg) for arg in cmd]
 122     log_verbose('Running in %s: %s' % (cwd, ' '.join(quoted_cmd)))
 123
 124     err_pipe = subprocess.PIPE
 125     if ignore_errors:
 126         # Silence errors if requested.
 127         err_pipe = get_dev_null()
 128
 129     start = time.time()
 130     p = subprocess.Popen(cmd, cwd=cwd, stdout=subprocess.PIPE, stderr=err_pipe,
 131                          stdin=subprocess.PIPE,
 132                          universal_newlines=text)
 133     stdout, stderr = p.communicate(input=stdin)
 134     elapsed = time.time() - start
 135
 136     log_verbose('Command took %0.1fs' % elapsed)
 137
 138     if p.returncode == 0 or ignore_errors:
 139         if stderr and not ignore_errors:
 140             eprint('`%s` printed to stderr:' % ' '.join(quoted_cmd))
 141             eprint(stderr.rstrip())
 142         if strip:
 143             if text:
 144                 stdout = stdout.rstrip('\r\n')
 145             else:
 146                 stdout = stdout.rstrip(b'\r\n')
 147         if VERBOSE:
 148             for l in stdout.splitlines():
 149                 log_verbose("STDOUT: %s" % l)
 150         return stdout
 151     err_msg = '`%s` returned %s' % (' '.join(quoted_cmd), p.returncode)
 152     eprint(err_msg)
 153     if stderr:
 154         eprint(stderr.rstrip())
 155     if die_on_failure:
 156         sys.exit(2)
 157     raise RuntimeError(err_msg)
 158
 159
 160 def git(*cmd, **kwargs):
 161     return shell(['git'] + list(cmd), **kwargs)
 162
 163
 164 def svn(cwd, *cmd, **kwargs):
 165     return shell(['svn'] + list(cmd), cwd=cwd, **kwargs)
 166
 167 def program_exists(cmd):
 168     if sys.platform == 'win32' and not cmd.endswith('.exe'):
 169         cmd += '.exe'
 170     for path in os.environ["PATH"].split(os.pathsep):
 171         if os.access(os.path.join(path, cmd), os.X_OK):
 172             return True
 173     return False
 174
 175 def get_default_rev_range():
 176     # Get the branch tracked by the current branch, as set by
 177     # git branch --set-upstream-to  See http://serverfault.com/a/352236/38694.
 178     cur_branch = git('rev-parse', '--symbolic-full-name', 'HEAD')
 179     upstream_branch = git('for-each-ref', '--format=%(upstream:short)',
 180                           cur_branch)
 181     if not upstream_branch:
 182         upstream_branch = 'origin/master'
 183
 184     # Get the newest common ancestor between HEAD and our upstream branch.
 185     upstream_rev = git('merge-base', 'HEAD', upstream_branch)
 186     return '%s..' % upstream_rev
 187
 188
 189 def get_revs_to_push(rev_range):
 190     if not rev_range:
 191         rev_range = get_default_rev_range()
 192     # Use git show rather than some plumbing command to figure out which revs
 193     # are in rev_range because it handles single revs (HEAD^) and ranges
 194     # (foo..bar) like we want.
 195     revs = git('show', '--reverse', '--quiet',
 196                '--pretty=%h', rev_range).splitlines()
 197     if not revs:
 198         die('Nothing to push: No revs in range %s.' % rev_range)
 199     return revs
 200
 201
 202 def clean_svn(svn_repo):
 203     svn(svn_repo, 'revert', '-R', '.')
 204
 205     # Unfortunately it appears there's no svn equivalent for git clean, so we
 206     # have to do it ourselves.
 207     for line in svn(svn_repo, 'status', '--no-ignore').split('\n'):
 208         if not line.startswith('?'):
 209             continue
 210         filename = line[1:].strip()
 211         filepath = os.path.abspath(os.path.join(svn_repo, filename))
 212         abs_svn_repo = os.path.abspath(svn_repo)
 213         # Safety check that the directory we are about to delete is
 214         # actually within our svn staging dir.
 215         if not filepath.startswith(abs_svn_repo):
 216             die("Path to clean (%s) is not in svn staging dir (%s)"
 217                 % (filepath, abs_svn_repo))
 218
 219         if os.path.isdir(filepath):
 220             shutil.rmtree(filepath)
 221         else:
 222             os.remove(filepath)
 223
 224
 225 def svn_init(svn_root):
 226     if not os.path.exists(svn_root):
 227         log('Creating svn staging directory: (%s)' % (svn_root))
 228         os.makedirs(svn_root)
 229         svn(svn_root, 'checkout', '--depth=empty',
 230             'https://llvm.org/svn/llvm-project/', '.')
 231         log("svn staging area ready in '%s'" % svn_root)
 232     if not os.path.isdir(svn_root):
 233         die("Can't initialize svn staging dir (%s)" % svn_root)
 234
 235
 236 def fix_eol_style_native(rev, svn_sr_path, files):
 237     """Fix line endings before applying patches with Unix endings
 238
 239     SVN on Windows will check out files with CRLF for files with the
 240     svn:eol-style property set to "native". This breaks `git apply`, which
 241     typically works with Unix-line ending patches. Work around the problem here
 242     by doing a dos2unix up front for files with svn:eol-style set to "native".
 243     SVN will not commit a mass line ending re-doing because it detects the line
 244     ending format for files with this property.
 245     """
 246     # Skip files that don't exist in SVN yet.
 247     files = [f for f in files if os.path.exists(os.path.join(svn_sr_path, f))]
 248     # Use ignore_errors because 'svn propget' prints errors if the file doesn't
 249     # have the named property. There doesn't seem to be a way to suppress that.
 250     eol_props = svn(svn_sr_path, 'propget', 'svn:eol-style', *files,
 251                     ignore_errors=True)
 252     crlf_files = []
 253     if len(files) == 1:
 254         # No need to split propget output on ' - ' when we have one file.
 255         if eol_props.strip() in ['native', 'CRLF']:
 256             crlf_files = files
 257     else:
 258         for eol_prop in eol_props.split('\n'):
 259             # Remove spare CR.
 260             eol_prop = eol_prop.strip('\r')
 261             if not eol_prop:
 262                 continue
 263             prop_parts = eol_prop.rsplit(' - ', 1)
 264             if len(prop_parts) != 2:
 265                 eprint("unable to parse svn propget line:")
 266                 eprint(eol_prop)
 267                 continue
 268             (f, eol_style) = prop_parts
 269             if eol_style == 'native':
 270                 crlf_files.append(f)
 271     if crlf_files:
 272         # Reformat all files with native SVN line endings to Unix format. SVN
 273         # knows files with native line endings are text files. It will commit
 274         # just the diff, and not a mass line ending change.
 275         shell(['dos2unix'] + crlf_files, ignore_errors=True, cwd=svn_sr_path)
 276
 277 def split_subrepo(f):
 278     # Given a path, splits it into (subproject, rest-of-path). If the path is
 279     # not in a subproject, returns ('', full-path).
 280
 281     subproject, remainder = split_first_path_component(f)
 282
 283     if subproject in GIT_TO_SVN_DIR:
 284         return subproject, remainder
 285     else:
 286         return '', f
 287
 288 def get_all_parent_dirs(name):
 289     parts = []
 290     head, tail = os.path.split(name)
 291     while head:
 292         parts.append(head)
 293         head, tail = os.path.split(head)
 294     return parts
 295
 296 def svn_push_one_rev(svn_repo, rev, dry_run):
 297     files = git('diff-tree', '--no-commit-id', '--name-only', '-r',
 298                 rev).split('\n')
 299     if not files:
 300         raise RuntimeError('Empty diff for rev %s?' % rev)
 301
 302     # Split files by subrepo
 303     subrepo_files = collections.defaultdict(list)
 304     for f in files:
 305         subrepo, remainder = split_subrepo(f)
 306         subrepo_files[subrepo].append(remainder)
 307
 308     status = svn(svn_repo, 'status', '--no-ignore')
 309     if status:
 310         die("Can't push git rev %s because svn status is not empty:\n%s" %
 311             (rev, status))
 312
 313     svn_dirs_to_update = set()
 314     for sr, files in iteritems(subrepo_files):
 315         svn_sr_path = GIT_TO_SVN_DIR[sr]
 316         for f in files:
 317             svn_dirs_to_update.add(
 318                 os.path.dirname(os.path.join(svn_sr_path, f)))
 319
 320     # We also need to svn update any parent directories which are not yet present
 321     parent_dirs = set()
 322     for dir in svn_dirs_to_update:
 323         parent_dirs.update(get_all_parent_dirs(dir))
 324     parent_dirs = set(dir for dir in parent_dirs
 325                       if not os.path.exists(os.path.join(svn_repo, dir)))
 326     svn_dirs_to_update.update(parent_dirs)
 327
 328     # Sort by length to ensure that the parent directories are passed to svn
 329     # before child directories.
 330     sorted_dirs_to_update = sorted(svn_dirs_to_update, key=len)
 331
 332     # SVN update only in the affected directories.
 333     svn(svn_repo, 'update', '--depth=files', *sorted_dirs_to_update)
 334
 335     for sr, files in iteritems(subrepo_files):
 336         svn_sr_path = os.path.join(svn_repo, GIT_TO_SVN_DIR[sr])
 337         if os.name == 'nt':
 338             fix_eol_style_native(rev, svn_sr_path, files)
 339         # We use text=False (and pass '--binary') so that we can get an exact
 340         # diff that can be passed as-is to 'git apply' without any line ending,
 341         # encoding, or other mangling.
 342         diff = git('show', '--binary', rev, '--',
 343                    *(os.path.join(sr, f) for f in files),
 344                    strip=False, text=False)
 345         # git is the only thing that can handle its own patches...
 346         if sr == '':
 347             prefix_strip = '-p1'
 348         else:
 349             prefix_strip = '-p2'
 350         try:
 351             shell(['git', 'apply', prefix_strip, '-'], cwd=svn_sr_path,
 352                   stdin=diff, die_on_failure=False, text=False)
 353         except RuntimeError as e:
 354             eprint("Patch doesn't apply: maybe you should try `git pull -r` "
 355                    "first?")
 356             sys.exit(2)
 357
 358     status_lines = svn(svn_repo, 'status', '--no-ignore').split('\n')
 359
 360     for l in (l for l in status_lines if (l.startswith('?') or
 361                                           l.startswith('I'))):
 362         svn(svn_repo, 'add', '--no-ignore', l[1:].strip())
 363     for l in (l for l in status_lines if l.startswith('!')):
 364         svn(svn_repo, 'remove', l[1:].strip())
 365
 366     # Now we're ready to commit.
 367     commit_msg = git('show', '--pretty=%B', '--quiet', rev)
 368     if not dry_run:
 369         commit_args = ['commit', '-m', commit_msg]
 370         if '--force-interactive' in svn(svn_repo, 'commit', '--help'):
 371             commit_args.append('--force-interactive')
 372         log(svn(svn_repo, *commit_args))
 373         log('Committed %s to svn.' % rev)
 374     else:
 375         log("Would have committed %s to svn, if this weren't a dry run." % rev)
 376
 377
 378 def cmd_push(args):
 379     '''Push changes back to SVN: this is extracted from Justin Lebar's script
 380     available here: https://github.com/jlebar/llvm-repo-tools/
 381
 382     Note: a current limitation is that git does not track file rename, so they
 383     will show up in SVN as delete+add.
 384     '''
 385     # Get the git root
 386     git_root = git('rev-parse', '--show-toplevel')
 387     if not os.path.isdir(git_root):
 388         die("Can't find git root dir")
 389
 390     # Push from the root of the git repo
 391     os.chdir(git_root)
 392
 393     # We need a staging area for SVN, let's hide it in the .git directory.
 394     dot_git_dir = git('rev-parse', '--git-common-dir')
 395     # Not all versions of git support --git-common-dir and just print the
 396     # unknown command back. If this happens, fall back to --git-dir
 397     if dot_git_dir == '--git-common-dir':
 398       dot_git_dir = git('rev-parse', '--git-dir')
 399
 400     svn_root = os.path.join(dot_git_dir, 'llvm-upstream-svn')
 401     svn_init(svn_root)
 402
 403     rev_range = args.rev_range
 404     dry_run = args.dry_run
 405     revs = get_revs_to_push(rev_range)
 406     log('Pushing %d commit%s:\n%s' %
 407         (len(revs), 's' if len(revs) != 1
 408          else '', '\n'.join('  ' + git('show', '--oneline', '--quiet', c)
 409                             for c in revs)))
 410     for r in revs:
 411         clean_svn(svn_root)
 412         svn_push_one_rev(svn_root, r, dry_run)
 413
 414
 415 def lookup_llvm_svn_id(git_commit_hash):
 416     # Use --format=%b to get the raw commit message, without any extra
 417     # whitespace.
 418     commit_msg = git('log', '-1', '--format=%b', git_commit_hash,
 419                      ignore_errors=True)
 420     if len(commit_msg) == 0:
 421       die("Can't find git commit " + git_commit_hash)
 422     # If a commit has multiple "llvm-svn:" lines (e.g. if the commit is
 423     # reverting/quoting a previous commit), choose the last one, which should
 424     # be the authoritative one.
 425     svn_match_iter = re.finditer('^llvm-svn: (\d{5,7})$', commit_msg,
 426                                  re.MULTILINE)
 427     svn_match = None
 428     for m in svn_match_iter:
 429       svn_match = m.group(1)
 430     if svn_match:
 431       return int(svn_match)
 432     die("Can't find svn revision in git commit " + git_commit_hash)
 433
 434
 435 def cmd_svn_lookup(args):
 436     '''Find the SVN revision id for a given git commit hash.
 437
 438     This is identified by 'llvm-svn: NNNNNN' in the git commit message.'''
 439     # Get the git root
 440     git_root = git('rev-parse', '--show-toplevel')
 441     if not os.path.isdir(git_root):
 442         die("Can't find git root dir")
 443
 444     # Run commands from the root
 445     os.chdir(git_root)
 446
 447     log('r' + str(lookup_llvm_svn_id(args.git_commit_hash)))
 448
 449
 450 def git_hash_by_svn_rev(svn_rev):
 451     '''Find the git hash for a given svn revision.
 452
 453     This check is paranoid: 'llvm-svn: NNNNNN' could exist on its own line
 454     somewhere else in the commit message. Look in the full log message to see
 455     if it's actually on the last line.
 456
 457     Since this check is expensive (we're searching every single commit), limit
 458     to the past 10k commits (about 5 months).
 459     '''
 460     possible_hashes = git(
 461         'log', '--format=%H', '--grep', '^llvm-svn: %d$' % svn_rev,
 462         'HEAD~10000...HEAD').split('\n')
 463     matching_hashes = [h for h in possible_hashes
 464                        if lookup_llvm_svn_id(h) == svn_rev]
 465     if len(matching_hashes) > 1:
 466       die("svn revision r%d has ambiguous commits: %s" % (
 467           svn_rev, ', '.join(matching_hashes)))
 468     elif len(matching_hashes) < 1:
 469       die("svn revision r%d matches no commits" % svn_rev)
 470     return matching_hashes[0]
 471
 472 def cmd_revert(args):
 473     '''Revert a commit by either SVN id (rNNNNNN) or git hash. This also
 474     populates the git commit message with both the SVN revision and git hash of
 475     the change being reverted.'''
 476
 477     # Get the git root
 478     git_root = git('rev-parse', '--show-toplevel')
 479     if not os.path.isdir(git_root):
 480         die("Can't find git root dir")
 481
 482     # Run commands from the root
 483     os.chdir(git_root)
 484
 485     # Check for a client branch first.
 486     open_files = git('status', '-uno', '-s', '--porcelain')
 487     if len(open_files) > 0:
 488       die("Found open files. Please stash and then revert.\n" + open_files)
 489
 490     # If the revision looks like rNNNNNN, use that. Otherwise, look for it in
 491     # the git commit.
 492     svn_match = re.match('^r(\d{5,7})$', args.revision)
 493     if svn_match:
 494       # If the revision looks like rNNNNNN, use that as the svn revision, and
 495       # grep through git commits to find which one corresponds to that svn
 496       # revision.
 497       svn_rev = int(svn_match.group(1))
 498       git_hash = git_hash_by_svn_rev(svn_rev)
 499     else:
 500       # Otherwise, this looks like a git hash, so we just need to grab the svn
 501       # revision from the end of the commit message.
 502       # Get the actual git hash in case the revision is something like "HEAD~1"
 503       git_hash = git('rev-parse', '--verify', args.revision + '^{commit}')
 504       svn_rev = lookup_llvm_svn_id(git_hash)
 505
 506     msg = git('log', '-1', '--format=%s', git_hash)
 507
 508     log_verbose('Ready to revert r%d (%s): "%s"' % (svn_rev, git_hash, msg))
 509
 510     revert_args = ['revert', '--no-commit', git_hash]
 511     # TODO: Running --edit doesn't seem to work, with errors that stdin is not
 512     # a tty.
 513     commit_args = [
 514         'commit', '-m', 'Revert ' + msg,
 515         '-m', 'This reverts r%d (git commit %s)' % (svn_rev, git_hash)]
 516     if args.dry_run:
 517       log("Would have run the following commands, if this weren't a dry run:\n"
 518           '1) git %s\n2) git %s' % (
 519               ' '.join(quote(arg) for arg in revert_args),
 520               ' '.join(quote(arg) for arg in commit_args)))
 521       return
 522
 523     git(*revert_args)
 524     commit_log = git(*commit_args)
 525
 526     log('Created revert of r%d: %s' % (svn_rev, commit_log))
 527     log("Run 'git llvm push -n' to inspect your changes and "
 528         "run 'git llvm push' when ready")
 529
 530 if __name__ == '__main__':
 531     if not program_exists('svn'):
 532         die('error: git-llvm needs svn command, but svn is not installed.')
 533
 534     argv = sys.argv[1:]
 535     p = argparse.ArgumentParser(
 536         prog='git llvm', formatter_class=argparse.RawDescriptionHelpFormatter,
 537         description=__doc__)
 538     subcommands = p.add_subparsers(title='subcommands',
 539                                    description='valid subcommands',
 540                                    help='additional help')
 541     verbosity_group = p.add_mutually_exclusive_group()
 542     verbosity_group.add_argument('-q', '--quiet', action='store_true',
 543                                  help='print less information')
 544     verbosity_group.add_argument('-v', '--verbose', action='store_true',
 545                                  help='print more information')
 546
 547     parser_push = subcommands.add_parser(
 548         'push', description=cmd_push.__doc__,
 549         help='push changes back to the LLVM SVN repository')
 550     parser_push.add_argument(
 551         '-n',
 552         '--dry-run',
 553         dest='dry_run',
 554         action='store_true',
 555         help='Do everything other than commit to svn.  Leaves junk in the svn '
 556         'repo, so probably will not work well if you try to commit more '
 557         'than one rev.')
 558     parser_push.add_argument(
 559         'rev_range',
 560         metavar='GIT_REVS',
 561         type=str,
 562         nargs='?',
 563         help="revs to push (default: everything not in the branch's "
 564         'upstream, or not in origin/master if the branch lacks '
 565         'an explicit upstream)')
 566     parser_push.set_defaults(func=cmd_push)
 567
 568     parser_revert = subcommands.add_parser(
 569         'revert', description=cmd_revert.__doc__,
 570         help='Revert a commit locally.')
 571     parser_revert.add_argument(
 572         'revision',
 573         help='Revision to revert. Can either be an SVN revision number '
 574         "(rNNNNNN) or a git commit hash (anything that doesn't look "
 575         'like an SVN revision number).')
 576     parser_revert.add_argument(
 577         '-n',
 578         '--dry-run',
 579         dest='dry_run',
 580         action='store_true',
 581         help='Do everything other than perform a revert. Prints the git '
 582         'revert command it would have run.')
 583     parser_revert.set_defaults(func=cmd_revert)
 584
 585     parser_svn_lookup = subcommands.add_parser(
 586         'svn-lookup', description=cmd_svn_lookup.__doc__,
 587         help='Find the llvm-svn revision for a given commit.')
 588     parser_svn_lookup.add_argument(
 589         'git_commit_hash',
 590         help='git_commit_hash for which we will look up the svn revision id.')
 591     parser_svn_lookup.set_defaults(func=cmd_svn_lookup)
 592
 593     args = p.parse_args(argv)
 594     VERBOSE = args.verbose
 595     QUIET = args.quiet
 596
 597     # Dispatch to the right subcommand
 598     args.func(args)