3 # ======- git-llvm - LLVM Git Help Integration ---------*- python -*--========#
5 # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
6 # See https://llvm.org/LICENSE.txt for license information.
7 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
9 # ==------------------------------------------------------------------------==#
15 This file provides integration for git.
18 from __future__
import print_function
27 assert sys
.version_info
>= (2, 7)
31 except AttributeError:
34 return iter(d
.items())
42 from shlex
import quote
45 from pipes
import quote
47 # It's *almost* a straightforward mapping from the monorepo to svn...
48 LLVM_MONOREPO_SVN_MAPPING
= {
70 LLVM_MONOREPO_SVN_MAPPING
.update({'clang': 'cfe/trunk'})
71 LLVM_MONOREPO_SVN_MAPPING
.update({'': 'monorepo-root/trunk'})
73 SPLIT_REPO_NAMES
= {'llvm-' + d
: d
+ '/trunk'
74 for d
in ['www', 'zorg', 'test-suite', 'lnt']}
81 def eprint(*args
, **kwargs
):
82 print(*args
, file=sys
.stderr
, **kwargs
)
85 def log(*args
, **kwargs
):
88 print(*args
, **kwargs
)
91 def log_verbose(*args
, **kwargs
):
94 print(*args
, **kwargs
)
102 def ask_confirm(prompt
):
103 # Python 2/3 compatibility
105 read_input
= raw_input
110 query
= read_input('%s (y/N): ' % (prompt
))
111 if query
.lower() not in ['y','n', '']:
112 print('Expect y or n!')
114 return query
.lower() == 'y'
117 def split_first_path_component(d
):
118 # Assuming we have a git path, it'll use slashes even on windows...I hope.
120 return d
.split('/', 1)
126 """Lazily create a /dev/null fd for use in shell()"""
128 if dev_null_fd
is None:
129 dev_null_fd
= open(os
.devnull
, 'w')
133 def shell(cmd
, strip
=True, cwd
=None, stdin
=None, die_on_failure
=True,
134 ignore_errors
=False, text
=True):
135 # Escape args when logging for easy repro.
136 quoted_cmd
= [quote(arg
) for arg
in cmd
]
137 log_verbose('Running in %s: %s' % (cwd
, ' '.join(quoted_cmd
)))
139 err_pipe
= subprocess
.PIPE
141 # Silence errors if requested.
142 err_pipe
= get_dev_null()
145 p
= subprocess
.Popen(cmd
, cwd
=cwd
, stdout
=subprocess
.PIPE
, stderr
=err_pipe
,
146 stdin
=subprocess
.PIPE
,
147 universal_newlines
=text
)
148 stdout
, stderr
= p
.communicate(input=stdin
)
149 elapsed
= time
.time() - start
151 log_verbose('Command took %0.1fs' % elapsed
)
153 if p
.returncode
== 0 or ignore_errors
:
154 if stderr
and not ignore_errors
:
155 eprint('`%s` printed to stderr:' % ' '.join(quoted_cmd
))
156 eprint(stderr
.rstrip())
159 stdout
= stdout
.rstrip('\r\n')
161 stdout
= stdout
.rstrip(b
'\r\n')
163 for l
in stdout
.splitlines():
164 log_verbose("STDOUT: %s" % l
)
166 err_msg
= '`%s` returned %s' % (' '.join(quoted_cmd
), p
.returncode
)
169 eprint(stderr
.rstrip())
172 raise RuntimeError(err_msg
)
175 def git(*cmd
, **kwargs
):
176 return shell(['git'] + list(cmd
), **kwargs
)
179 def svn(cwd
, *cmd
, **kwargs
):
180 return shell(['svn'] + list(cmd
), cwd
=cwd
, **kwargs
)
183 def program_exists(cmd
):
184 if sys
.platform
== 'win32' and not cmd
.endswith('.exe'):
186 for path
in os
.environ
["PATH"].split(os
.pathsep
):
187 if os
.access(os
.path
.join(path
, cmd
), os
.X_OK
):
192 def get_default_rev_range():
193 # Get the branch tracked by the current branch, as set by
194 # git branch --set-upstream-to See http://serverfault.com/a/352236/38694.
195 cur_branch
= git('rev-parse', '--symbolic-full-name', 'HEAD')
196 upstream_branch
= git('for-each-ref', '--format=%(upstream:short)',
198 if not upstream_branch
:
199 upstream_branch
= 'origin/master'
201 # Get the newest common ancestor between HEAD and our upstream branch.
202 upstream_rev
= git('merge-base', 'HEAD', upstream_branch
)
203 return '%s..' % upstream_rev
206 def get_revs_to_push(rev_range
):
208 rev_range
= get_default_rev_range()
209 # Use git show rather than some plumbing command to figure out which revs
210 # are in rev_range because it handles single revs (HEAD^) and ranges
211 # (foo..bar) like we want.
212 revs
= git('show', '--reverse', '--quiet',
213 '--pretty=%h', rev_range
).splitlines()
215 die('Nothing to push: No revs in range %s.' % rev_range
)
219 def clean_svn(svn_repo
):
220 svn(svn_repo
, 'revert', '-R', '.')
222 # Unfortunately it appears there's no svn equivalent for git clean, so we
223 # have to do it ourselves.
224 for line
in svn(svn_repo
, 'status', '--no-ignore').split('\n'):
225 if not line
.startswith('?'):
227 filename
= line
[1:].strip()
228 filepath
= os
.path
.abspath(os
.path
.join(svn_repo
, filename
))
229 abs_svn_repo
= os
.path
.abspath(svn_repo
)
230 # Safety check that the directory we are about to delete is
231 # actually within our svn staging dir.
232 if not filepath
.startswith(abs_svn_repo
):
233 die("Path to clean (%s) is not in svn staging dir (%s)"
234 % (filepath
, abs_svn_repo
))
236 if os
.path
.isdir(filepath
):
237 shutil
.rmtree(filepath
)
242 def svn_init(svn_root
):
243 if not os
.path
.exists(svn_root
):
244 log('Creating svn staging directory: (%s)' % (svn_root
))
245 os
.makedirs(svn_root
)
246 svn(svn_root
, 'checkout', '--depth=empty',
247 'https://llvm.org/svn/llvm-project/', '.')
248 log("svn staging area ready in '%s'" % svn_root
)
249 if not os
.path
.isdir(svn_root
):
250 die("Can't initialize svn staging dir (%s)" % svn_root
)
253 def fix_eol_style_native(rev
, svn_sr_path
, files
):
254 """Fix line endings before applying patches with Unix endings
256 SVN on Windows will check out files with CRLF for files with the
257 svn:eol-style property set to "native". This breaks `git apply`, which
258 typically works with Unix-line ending patches. Work around the problem here
259 by doing a dos2unix up front for files with svn:eol-style set to "native".
260 SVN will not commit a mass line ending re-doing because it detects the line
261 ending format for files with this property.
263 # Skip files that don't exist in SVN yet.
264 files
= [f
for f
in files
if os
.path
.exists(os
.path
.join(svn_sr_path
, f
))]
265 # Use ignore_errors because 'svn propget' prints errors if the file doesn't
266 # have the named property. There doesn't seem to be a way to suppress that.
267 eol_props
= svn(svn_sr_path
, 'propget', 'svn:eol-style', *files
,
271 # No need to split propget output on ' - ' when we have one file.
272 if eol_props
.strip() in ['native', 'CRLF']:
275 for eol_prop
in eol_props
.split('\n'):
277 eol_prop
= eol_prop
.strip('\r')
280 prop_parts
= eol_prop
.rsplit(' - ', 1)
281 if len(prop_parts
) != 2:
282 eprint("unable to parse svn propget line:")
285 (f
, eol_style
) = prop_parts
286 if eol_style
== 'native':
289 # Reformat all files with native SVN line endings to Unix format. SVN
290 # knows files with native line endings are text files. It will commit
291 # just the diff, and not a mass line ending change.
292 shell(['dos2unix'] + crlf_files
, ignore_errors
=True, cwd
=svn_sr_path
)
295 def split_subrepo(f
, git_to_svn_mapping
):
296 # Given a path, splits it into (subproject, rest-of-path). If the path is
297 # not in a subproject, returns ('', full-path).
299 subproject
, remainder
= split_first_path_component(f
)
301 if subproject
in git_to_svn_mapping
:
302 return subproject
, remainder
307 def get_all_parent_dirs(name
):
309 head
, tail
= os
.path
.split(name
)
312 head
, tail
= os
.path
.split(head
)
316 def svn_push_one_rev(svn_repo
, rev
, git_to_svn_mapping
, dry_run
):
320 files_status
= [split_status(x
) for x
in
321 git('diff-tree', '--no-commit-id', '--name-status',
322 '--no-renames', '-r', rev
).split('\n')]
324 raise RuntimeError('Empty diff for rev %s?' % rev
)
326 # Split files by subrepo
327 subrepo_files
= collections
.defaultdict(list)
328 for f
, st
in files_status
:
329 subrepo
, remainder
= split_subrepo(f
, git_to_svn_mapping
)
330 subrepo_files
[subrepo
].append((remainder
, st
))
332 status
= svn(svn_repo
, 'status', '--no-ignore')
334 die("Can't push git rev %s because status in svn staging dir (%s) is "
335 "not empty:\n%s" % (rev
, svn_repo
, status
))
337 svn_dirs_to_update
= set()
338 for sr
, files_status
in iteritems(subrepo_files
):
339 svn_sr_path
= git_to_svn_mapping
[sr
]
340 for f
, _
in files_status
:
341 svn_dirs_to_update
.add(
342 os
.path
.dirname(os
.path
.join(svn_sr_path
, f
)))
344 # We also need to svn update any parent directories which are not yet
347 for dir in svn_dirs_to_update
:
348 parent_dirs
.update(get_all_parent_dirs(dir))
349 parent_dirs
= set(dir for dir in parent_dirs
350 if not os
.path
.exists(os
.path
.join(svn_repo
, dir)))
351 svn_dirs_to_update
.update(parent_dirs
)
353 # Sort by length to ensure that the parent directories are passed to svn
354 # before child directories.
355 sorted_dirs_to_update
= sorted(svn_dirs_to_update
, key
=len)
357 # SVN update only in the affected directories.
358 svn(svn_repo
, 'update', '--depth=files', *sorted_dirs_to_update
)
360 for sr
, files_status
in iteritems(subrepo_files
):
361 svn_sr_path
= os
.path
.join(svn_repo
, git_to_svn_mapping
[sr
])
363 fix_eol_style_native(rev
, svn_sr_path
,
364 [f
for f
, _
in files_status
])
366 # We use text=False (and pass '--binary') so that we can get an exact
367 # diff that can be passed as-is to 'git apply' without any line ending,
368 # encoding, or other mangling.
369 diff
= git('show', '--binary', rev
, '--',
370 *(os
.path
.join(sr
, f
) for f
, _
in files_status
),
371 strip
=False, text
=False)
372 # git is the only thing that can handle its own patches...
378 shell(['git', 'apply', prefix_strip
, '-'], cwd
=svn_sr_path
,
379 stdin
=diff
, die_on_failure
=False, text
=False)
380 except RuntimeError as e
:
381 eprint("Patch doesn't apply: maybe you should try `git pull -r` "
385 # Handle removed files and directories. We need to be careful not to
386 # remove directories just because they _look_ empty in the svn tree, as
387 # we might be missing sibling directories in the working copy. So, only
388 # remove parent directories if they're empty on both the git and svn
390 maybe_dirs_to_remove
= set()
391 for f
, st
in files_status
:
393 maybe_dirs_to_remove
.update(get_all_parent_dirs(f
))
394 svn(svn_sr_path
, 'remove', f
)
395 elif not (st
== 'A' or st
== 'M' or st
== 'T'):
396 # Add is handled below, and nothing needs to be done for Modify.
397 # (FIXME: Type-change between symlink and file might need some
398 # special handling, but let's ignore that for now.)
399 die("Unexpected git status for %r: %r" % (f
, st
))
401 maybe_dirs_to_remove
= sorted(maybe_dirs_to_remove
, key
=len)
402 for f
in maybe_dirs_to_remove
:
403 if(not os
.path
.exists(os
.path
.join(svn_sr_path
, f
)) and
404 git('ls-tree', '-d', rev
, os
.path
.join(sr
, f
)) == ''):
405 svn(svn_sr_path
, 'remove', f
)
407 status_lines
= svn(svn_repo
, 'status', '--no-ignore').split('\n')
409 for l
in status_lines
:
411 if l
.startswith('?') or l
.startswith('I'):
412 svn(svn_repo
, 'add', '--no-ignore', f
)
414 # Now we're ready to commit.
415 commit_msg
= git('show', '--pretty=%B', '--quiet', rev
)
417 commit_args
= ['commit', '-m', commit_msg
]
418 if '--force-interactive' in svn(svn_repo
, 'commit', '--help'):
419 commit_args
.append('--force-interactive')
420 log(svn(svn_repo
, *commit_args
))
421 log('Committed %s to svn.' % rev
)
423 log("Would have committed %s to svn, if this weren't a dry run." % rev
)
427 '''Push changes back to SVN: this is extracted from Justin Lebar's script
428 available here: https://github.com/jlebar/llvm-repo-tools/
430 Note: a current limitation is that git does not track file rename, so they
431 will show up in SVN as delete+add.
434 git_root
= git('rev-parse', '--show-toplevel')
435 if not os
.path
.isdir(git_root
):
436 die("Can't find git root dir")
438 # Push from the root of the git repo
441 # Get the remote URL, and check if it's one of the standalone repos.
442 git_remote_url
= git('ls-remote', '--get-url', 'origin')
443 git_remote_url
= git_remote_url
.rstrip('.git').rstrip('/')
444 git_remote_repo_name
= git_remote_url
.rsplit('/', 1)[-1]
445 split_repo_path
= SPLIT_REPO_NAMES
.get(git_remote_repo_name
)
447 git_to_svn_mapping
= {'': split_repo_path
}
449 # Default to the monorepo mapping
450 git_to_svn_mapping
= LLVM_MONOREPO_SVN_MAPPING
452 # We need a staging area for SVN, let's hide it in the .git directory.
453 dot_git_dir
= git('rev-parse', '--git-common-dir')
454 # Not all versions of git support --git-common-dir and just print the
455 # unknown command back. If this happens, fall back to --git-dir
456 if dot_git_dir
== '--git-common-dir':
457 dot_git_dir
= git('rev-parse', '--git-dir')
459 svn_root
= os
.path
.join(dot_git_dir
, 'llvm-upstream-svn')
462 rev_range
= args
.rev_range
463 dry_run
= args
.dry_run
464 revs
= get_revs_to_push(rev_range
)
465 log('%sPushing %d %s commit%s:\n%s' %
466 ('[DryRun] ' if dry_run
else '', len(revs
),
467 'split-repo (%s)' % split_repo_path
468 if split_repo_path
else 'monorepo',
469 's' if len(revs
) != 1 else '',
470 '\n'.join(' ' + git('show', '--oneline', '--quiet', c
)
473 # Ask confirmation if multiple commits are about to be pushed
475 if not ask_confirm("Are you sure you want to create %d commits?" % len(revs
)):
480 svn_push_one_rev(svn_root
, r
, git_to_svn_mapping
, dry_run
)
483 def lookup_llvm_svn_id(git_commit_hash
):
484 # Use --format=%b to get the raw commit message, without any extra
486 commit_msg
= git('log', '-1', '--format=%b', git_commit_hash
,
488 if len(commit_msg
) == 0:
489 die("Can't find git commit " + git_commit_hash
)
490 # If a commit has multiple "llvm-svn:" lines (e.g. if the commit is
491 # reverting/quoting a previous commit), choose the last one, which should
492 # be the authoritative one.
493 svn_match_iter
= re
.finditer('^llvm-svn: (\d{5,7})$', commit_msg
,
496 for m
in svn_match_iter
:
497 svn_match
= m
.group(1)
499 return int(svn_match
)
500 die("Can't find svn revision in git commit " + git_commit_hash
)
503 def cmd_svn_lookup(args
):
504 '''Find the SVN revision id for a given git commit hash.
506 This is identified by 'llvm-svn: NNNNNN' in the git commit message.'''
508 git_root
= git('rev-parse', '--show-toplevel')
509 if not os
.path
.isdir(git_root
):
510 die("Can't find git root dir")
512 # Run commands from the root
515 log('r' + str(lookup_llvm_svn_id(args
.git_commit_hash
)))
518 def git_hash_by_svn_rev(svn_rev
):
519 '''Find the git hash for a given svn revision.
521 This check is paranoid: 'llvm-svn: NNNNNN' could exist on its own line
522 somewhere else in the commit message. Look in the full log message to see
523 if it's actually on the last line.
525 Since this check is expensive (we're searching every single commit), limit
526 to the past 10k commits (about 5 months).
528 possible_hashes
= git(
529 'log', '--format=%H', '--grep', '^llvm-svn: %d$' % svn_rev
,
530 'HEAD~10000...HEAD').split('\n')
531 matching_hashes
= [h
for h
in possible_hashes
532 if lookup_llvm_svn_id(h
) == svn_rev
]
533 if len(matching_hashes
) > 1:
534 die("svn revision r%d has ambiguous commits: %s" % (
535 svn_rev
, ', '.join(matching_hashes
)))
536 elif len(matching_hashes
) < 1:
537 die("svn revision r%d matches no commits" % svn_rev
)
538 return matching_hashes
[0]
541 def cmd_revert(args
):
542 '''Revert a commit by either SVN id (rNNNNNN) or git hash. This also
543 populates the git commit message with both the SVN revision and git hash of
544 the change being reverted.'''
547 git_root
= git('rev-parse', '--show-toplevel')
548 if not os
.path
.isdir(git_root
):
549 die("Can't find git root dir")
551 # Run commands from the root
554 # Check for a client branch first.
555 open_files
= git('status', '-uno', '-s', '--porcelain')
556 if len(open_files
) > 0:
557 die("Found open files. Please stash and then revert.\n" + open_files
)
559 # If the revision looks like rNNNNNN, use that. Otherwise, look for it in
561 svn_match
= re
.match('^r(\d{5,7})$', args
.revision
)
563 # If the revision looks like rNNNNNN, use that as the svn revision, and
564 # grep through git commits to find which one corresponds to that svn
566 svn_rev
= int(svn_match
.group(1))
567 git_hash
= git_hash_by_svn_rev(svn_rev
)
569 # Otherwise, this looks like a git hash, so we just need to grab the
570 # svn revision from the end of the commit message. Get the actual git
571 # hash in case the revision is something like "HEAD~1"
572 git_hash
= git('rev-parse', '--verify', args
.revision
+ '^{commit}')
573 svn_rev
= lookup_llvm_svn_id(git_hash
)
575 msg
= git('log', '-1', '--format=%s', git_hash
)
577 log_verbose('Ready to revert r%d (%s): "%s"' % (svn_rev
, git_hash
, msg
))
579 revert_args
= ['revert', '--no-commit', git_hash
]
580 # TODO: Running --edit doesn't seem to work, with errors that stdin is not
583 'commit', '-m', 'Revert ' + msg
,
584 '-m', 'This reverts r%d (git commit %s)' % (svn_rev
, git_hash
)]
586 log("Would have run the following commands, if this weren't a"
588 '1) git %s\n2) git %s' % (
589 ' '.join(quote(arg
) for arg
in revert_args
),
590 ' '.join(quote(arg
) for arg
in commit_args
)))
594 commit_log
= git(*commit_args
)
596 log('Created revert of r%d: %s' % (svn_rev
, commit_log
))
597 log("Run 'git llvm push -n' to inspect your changes and "
598 "run 'git llvm push' when ready")
601 if __name__
== '__main__':
602 if not program_exists('svn'):
603 die('error: git-llvm needs svn command, but svn is not installed.')
606 p
= argparse
.ArgumentParser(
607 prog
='git llvm', formatter_class
=argparse
.RawDescriptionHelpFormatter
,
609 subcommands
= p
.add_subparsers(title
='subcommands',
610 description
='valid subcommands',
611 help='additional help')
612 verbosity_group
= p
.add_mutually_exclusive_group()
613 verbosity_group
.add_argument('-q', '--quiet', action
='store_true',
614 help='print less information')
615 verbosity_group
.add_argument('-v', '--verbose', action
='store_true',
616 help='print more information')
618 parser_push
= subcommands
.add_parser(
619 'push', description
=cmd_push
.__doc
__,
620 help='push changes back to the LLVM SVN repository')
621 parser_push
.add_argument(
626 help='Do everything other than commit to svn. Leaves junk in the svn '
627 'repo, so probably will not work well if you try to commit more '
629 parser_push
.add_argument(
634 help="revs to push (default: everything not in the branch's "
635 'upstream, or not in origin/master if the branch lacks '
636 'an explicit upstream)')
637 parser_push
.set_defaults(func
=cmd_push
)
639 parser_revert
= subcommands
.add_parser(
640 'revert', description
=cmd_revert
.__doc
__,
641 help='Revert a commit locally.')
642 parser_revert
.add_argument(
644 help='Revision to revert. Can either be an SVN revision number '
645 "(rNNNNNN) or a git commit hash (anything that doesn't look "
646 'like an SVN revision number).')
647 parser_revert
.add_argument(
652 help='Do everything other than perform a revert. Prints the git '
653 'revert command it would have run.')
654 parser_revert
.set_defaults(func
=cmd_revert
)
656 parser_svn_lookup
= subcommands
.add_parser(
657 'svn-lookup', description
=cmd_svn_lookup
.__doc
__,
658 help='Find the llvm-svn revision for a given commit.')
659 parser_svn_lookup
.add_argument(
661 help='git_commit_hash for which we will look up the svn revision id.')
662 parser_svn_lookup
.set_defaults(func
=cmd_svn_lookup
)
664 args
= p
.parse_args(argv
)
665 VERBOSE
= args
.verbose
668 # Python3 workaround, for when not arguments are provided.
669 # See https://bugs.python.org/issue16308
672 except AttributeError:
673 # No arguments or subcommands were given.
677 # Dispatch to the right subcommand