3 # ======- git-llvm - LLVM Git Help Integration ---------*- python -*--========#
5 # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
6 # See https://llvm.org/LICENSE.txt for license information.
7 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
9 # ==------------------------------------------------------------------------==#
15 This file provides integration for git.
18 from __future__
import print_function
27 assert sys
.version_info
>= (2, 7)
31 except AttributeError:
34 return iter(d
.items())
42 from shlex
import quote
45 from pipes
import quote
47 # It's *almost* a straightforward mapping from the monorepo to svn...
48 LLVM_MONOREPO_SVN_MAPPING
= {
70 LLVM_MONOREPO_SVN_MAPPING
.update({'clang': 'cfe/trunk'})
71 LLVM_MONOREPO_SVN_MAPPING
.update({'': 'monorepo-root/trunk'})
73 SPLIT_REPO_NAMES
= {'llvm-' + d
: d
+ '/trunk'
74 for d
in ['www', 'zorg', 'test-suite', 'lnt']}
81 def eprint(*args
, **kwargs
):
82 print(*args
, file=sys
.stderr
, **kwargs
)
85 def log(*args
, **kwargs
):
88 print(*args
, **kwargs
)
91 def log_verbose(*args
, **kwargs
):
94 print(*args
, **kwargs
)
102 def ask_confirm(prompt
):
103 # Python 2/3 compatibility
105 read_input
= raw_input
110 query
= read_input('%s (y/N): ' % (prompt
))
111 if query
.lower() not in ['y','n', '']:
112 print('Expect y or n!')
114 return query
.lower() == 'y'
117 def split_first_path_component(d
):
118 # Assuming we have a git path, it'll use slashes even on windows...I hope.
120 return d
.split('/', 1)
126 """Lazily create a /dev/null fd for use in shell()"""
128 if dev_null_fd
is None:
129 dev_null_fd
= open(os
.devnull
, 'w')
133 def shell(cmd
, strip
=True, cwd
=None, stdin
=None, die_on_failure
=True,
134 ignore_errors
=False, text
=True):
135 # Escape args when logging for easy repro.
136 quoted_cmd
= [quote(arg
) for arg
in cmd
]
137 log_verbose('Running in %s: %s' % (cwd
, ' '.join(quoted_cmd
)))
139 err_pipe
= subprocess
.PIPE
141 # Silence errors if requested.
142 err_pipe
= get_dev_null()
145 p
= subprocess
.Popen(cmd
, cwd
=cwd
, stdout
=subprocess
.PIPE
, stderr
=err_pipe
,
146 stdin
=subprocess
.PIPE
,
147 universal_newlines
=text
)
148 stdout
, stderr
= p
.communicate(input=stdin
)
149 elapsed
= time
.time() - start
151 log_verbose('Command took %0.1fs' % elapsed
)
153 if p
.returncode
== 0 or ignore_errors
:
154 if stderr
and not ignore_errors
:
155 eprint('`%s` printed to stderr:' % ' '.join(quoted_cmd
))
156 eprint(stderr
.rstrip())
159 stdout
= stdout
.rstrip('\r\n')
161 stdout
= stdout
.rstrip(b
'\r\n')
163 for l
in stdout
.splitlines():
164 log_verbose("STDOUT: %s" % l
)
166 err_msg
= '`%s` returned %s' % (' '.join(quoted_cmd
), p
.returncode
)
169 eprint(stderr
.rstrip())
172 raise RuntimeError(err_msg
)
175 def git(*cmd
, **kwargs
):
176 return shell(['git'] + list(cmd
), **kwargs
)
179 def svn(cwd
, *cmd
, **kwargs
):
180 return shell(['svn'] + list(cmd
), cwd
=cwd
, **kwargs
)
183 def program_exists(cmd
):
184 if sys
.platform
== 'win32' and not cmd
.endswith('.exe'):
186 for path
in os
.environ
["PATH"].split(os
.pathsep
):
187 if os
.access(os
.path
.join(path
, cmd
), os
.X_OK
):
192 def get_default_rev_range():
193 # Get the branch tracked by the current branch, as set by
194 # git branch --set-upstream-to See http://serverfault.com/a/352236/38694.
195 cur_branch
= git('rev-parse', '--symbolic-full-name', 'HEAD')
196 upstream_branch
= git('for-each-ref', '--format=%(upstream:short)',
198 if not upstream_branch
:
199 upstream_branch
= 'origin/master'
201 # Get the newest common ancestor between HEAD and our upstream branch.
202 upstream_rev
= git('merge-base', 'HEAD', upstream_branch
)
203 return '%s..' % upstream_rev
206 def get_revs_to_push(rev_range
):
208 rev_range
= get_default_rev_range()
209 # Use git show rather than some plumbing command to figure out which revs
210 # are in rev_range because it handles single revs (HEAD^) and ranges
211 # (foo..bar) like we want.
212 return git('show', '--reverse', '--quiet',
213 '--pretty=%h', rev_range
).splitlines()
216 def clean_svn(svn_repo
):
217 svn(svn_repo
, 'revert', '-R', '.')
219 # Unfortunately it appears there's no svn equivalent for git clean, so we
220 # have to do it ourselves.
221 for line
in svn(svn_repo
, 'status', '--no-ignore').split('\n'):
222 if not line
.startswith('?'):
224 filename
= line
[1:].strip()
225 filepath
= os
.path
.abspath(os
.path
.join(svn_repo
, filename
))
226 abs_svn_repo
= os
.path
.abspath(svn_repo
)
227 # Safety check that the directory we are about to delete is
228 # actually within our svn staging dir.
229 if not filepath
.startswith(abs_svn_repo
):
230 die("Path to clean (%s) is not in svn staging dir (%s)"
231 % (filepath
, abs_svn_repo
))
233 if os
.path
.isdir(filepath
):
234 shutil
.rmtree(filepath
)
239 def svn_init(svn_root
):
240 if not os
.path
.exists(svn_root
):
241 log('Creating svn staging directory: (%s)' % (svn_root
))
242 os
.makedirs(svn_root
)
243 svn(svn_root
, 'checkout', '--depth=empty',
244 'https://llvm.org/svn/llvm-project/', '.')
245 log("svn staging area ready in '%s'" % svn_root
)
246 if not os
.path
.isdir(svn_root
):
247 die("Can't initialize svn staging dir (%s)" % svn_root
)
250 def fix_eol_style_native(rev
, svn_sr_path
, files
):
251 """Fix line endings before applying patches with Unix endings
253 SVN on Windows will check out files with CRLF for files with the
254 svn:eol-style property set to "native". This breaks `git apply`, which
255 typically works with Unix-line ending patches. Work around the problem here
256 by doing a dos2unix up front for files with svn:eol-style set to "native".
257 SVN will not commit a mass line ending re-doing because it detects the line
258 ending format for files with this property.
260 # Skip files that don't exist in SVN yet.
261 files
= [f
for f
in files
if os
.path
.exists(os
.path
.join(svn_sr_path
, f
))]
262 # Use ignore_errors because 'svn propget' prints errors if the file doesn't
263 # have the named property. There doesn't seem to be a way to suppress that.
264 eol_props
= svn(svn_sr_path
, 'propget', 'svn:eol-style', *files
,
268 # No need to split propget output on ' - ' when we have one file.
269 if eol_props
.strip() in ['native', 'CRLF']:
272 for eol_prop
in eol_props
.split('\n'):
274 eol_prop
= eol_prop
.strip('\r')
277 prop_parts
= eol_prop
.rsplit(' - ', 1)
278 if len(prop_parts
) != 2:
279 eprint("unable to parse svn propget line:")
282 (f
, eol_style
) = prop_parts
283 if eol_style
== 'native':
286 # Reformat all files with native SVN line endings to Unix format. SVN
287 # knows files with native line endings are text files. It will commit
288 # just the diff, and not a mass line ending change.
289 shell(['dos2unix'] + crlf_files
, ignore_errors
=True, cwd
=svn_sr_path
)
292 def split_subrepo(f
, git_to_svn_mapping
):
293 # Given a path, splits it into (subproject, rest-of-path). If the path is
294 # not in a subproject, returns ('', full-path).
296 subproject
, remainder
= split_first_path_component(f
)
298 if subproject
in git_to_svn_mapping
:
299 return subproject
, remainder
304 def get_all_parent_dirs(name
):
306 head
, tail
= os
.path
.split(name
)
309 head
, tail
= os
.path
.split(head
)
313 def svn_push_one_rev(svn_repo
, rev
, git_to_svn_mapping
, dry_run
):
317 files_status
= [split_status(x
) for x
in
318 git('diff-tree', '--no-commit-id', '--name-status',
319 '--no-renames', '-r', rev
).split('\n')]
321 raise RuntimeError('Empty diff for rev %s?' % rev
)
323 # Split files by subrepo
324 subrepo_files
= collections
.defaultdict(list)
325 for f
, st
in files_status
:
326 subrepo
, remainder
= split_subrepo(f
, git_to_svn_mapping
)
327 subrepo_files
[subrepo
].append((remainder
, st
))
329 status
= svn(svn_repo
, 'status', '--no-ignore')
331 die("Can't push git rev %s because status in svn staging dir (%s) is "
332 "not empty:\n%s" % (rev
, svn_repo
, status
))
334 svn_dirs_to_update
= set()
335 for sr
, files_status
in iteritems(subrepo_files
):
336 svn_sr_path
= git_to_svn_mapping
[sr
]
337 for f
, _
in files_status
:
338 svn_dirs_to_update
.add(
339 os
.path
.dirname(os
.path
.join(svn_sr_path
, f
)))
341 # We also need to svn update any parent directories which are not yet
344 for dir in svn_dirs_to_update
:
345 parent_dirs
.update(get_all_parent_dirs(dir))
346 parent_dirs
= set(dir for dir in parent_dirs
347 if not os
.path
.exists(os
.path
.join(svn_repo
, dir)))
348 svn_dirs_to_update
.update(parent_dirs
)
350 # Sort by length to ensure that the parent directories are passed to svn
351 # before child directories.
352 sorted_dirs_to_update
= sorted(svn_dirs_to_update
, key
=len)
354 # SVN update only in the affected directories.
355 svn(svn_repo
, 'update', '--depth=files', *sorted_dirs_to_update
)
357 for sr
, files_status
in iteritems(subrepo_files
):
358 svn_sr_path
= os
.path
.join(svn_repo
, git_to_svn_mapping
[sr
])
360 fix_eol_style_native(rev
, svn_sr_path
,
361 [f
for f
, _
in files_status
])
363 # We use text=False (and pass '--binary') so that we can get an exact
364 # diff that can be passed as-is to 'git apply' without any line ending,
365 # encoding, or other mangling.
366 diff
= git('show', '--binary', rev
, '--',
367 *(os
.path
.join(sr
, f
) for f
, _
in files_status
),
368 strip
=False, text
=False)
369 # git is the only thing that can handle its own patches...
375 shell(['git', 'apply', prefix_strip
, '-'], cwd
=svn_sr_path
,
376 stdin
=diff
, die_on_failure
=False, text
=False)
377 except RuntimeError as e
:
378 eprint("Patch doesn't apply: maybe you should try `git pull -r` "
382 # Handle removed files and directories. We need to be careful not to
383 # remove directories just because they _look_ empty in the svn tree, as
384 # we might be missing sibling directories in the working copy. So, only
385 # remove parent directories if they're empty on both the git and svn
387 maybe_dirs_to_remove
= set()
388 for f
, st
in files_status
:
390 maybe_dirs_to_remove
.update(get_all_parent_dirs(f
))
391 svn(svn_sr_path
, 'remove', f
)
392 elif not (st
== 'A' or st
== 'M' or st
== 'T'):
393 # Add is handled below, and nothing needs to be done for Modify.
394 # (FIXME: Type-change between symlink and file might need some
395 # special handling, but let's ignore that for now.)
396 die("Unexpected git status for %r: %r" % (f
, st
))
398 maybe_dirs_to_remove
= sorted(maybe_dirs_to_remove
, key
=len)
399 for f
in maybe_dirs_to_remove
:
400 if(not os
.path
.exists(os
.path
.join(svn_sr_path
, f
)) and
401 git('ls-tree', '-d', rev
, os
.path
.join(sr
, f
)) == ''):
402 svn(svn_sr_path
, 'remove', f
)
404 status_lines
= svn(svn_repo
, 'status', '--no-ignore').split('\n')
406 for l
in status_lines
:
408 if l
.startswith('?') or l
.startswith('I'):
409 svn(svn_repo
, 'add', '--no-ignore', f
)
411 # Now we're ready to commit.
412 commit_msg
= git('show', '--pretty=%B', '--quiet', rev
)
414 commit_args
= ['commit', '-m', commit_msg
]
415 if '--force-interactive' in svn(svn_repo
, 'commit', '--help'):
416 commit_args
.append('--force-interactive')
417 log(svn(svn_repo
, *commit_args
))
418 log('Committed %s to svn.' % rev
)
420 log("Would have committed %s to svn, if this weren't a dry run." % rev
)
424 '''Push changes back to SVN: this is extracted from Justin Lebar's script
425 available here: https://github.com/jlebar/llvm-repo-tools/
427 Note: a current limitation is that git does not track file rename, so they
428 will show up in SVN as delete+add.
431 git_root
= git('rev-parse', '--show-toplevel')
432 if not os
.path
.isdir(git_root
):
433 die("Can't find git root dir")
435 # Push from the root of the git repo
438 # Get the remote URL, and check if it's one of the standalone repos.
439 git_remote_url
= git('ls-remote', '--get-url', 'origin')
440 git_remote_url
= git_remote_url
.rstrip('.git').rstrip('/')
441 git_remote_repo_name
= git_remote_url
.rsplit('/', 1)[-1]
442 split_repo_path
= SPLIT_REPO_NAMES
.get(git_remote_repo_name
)
444 git_to_svn_mapping
= {'': split_repo_path
}
446 # Default to the monorepo mapping
447 git_to_svn_mapping
= LLVM_MONOREPO_SVN_MAPPING
449 # We need a staging area for SVN, let's hide it in the .git directory.
450 dot_git_dir
= git('rev-parse', '--git-common-dir')
451 # Not all versions of git support --git-common-dir and just print the
452 # unknown command back. If this happens, fall back to --git-dir
453 if dot_git_dir
== '--git-common-dir':
454 dot_git_dir
= git('rev-parse', '--git-dir')
456 svn_root
= os
.path
.join(dot_git_dir
, 'llvm-upstream-svn')
459 rev_range
= args
.rev_range
460 dry_run
= args
.dry_run
461 revs
= get_revs_to_push(rev_range
)
463 if not args
.force
and not revs
:
464 die('Nothing to push: No revs in range %s.' % rev_range
)
466 log('%sPushing %d %s commit%s:\n%s' %
467 ('[DryRun] ' if dry_run
else '', len(revs
),
468 'split-repo (%s)' % split_repo_path
469 if split_repo_path
else 'monorepo',
470 's' if len(revs
) != 1 else '',
471 '\n'.join(' ' + git('show', '--oneline', '--quiet', c
)
474 # Ask confirmation if multiple commits are about to be pushed
475 if not args
.force
and len(revs
) > 1:
476 if not ask_confirm("Are you sure you want to create %d commits?" % len(revs
)):
481 svn_push_one_rev(svn_root
, r
, git_to_svn_mapping
, dry_run
)
484 def lookup_llvm_svn_id(git_commit_hash
):
485 # Use --format=%b to get the raw commit message, without any extra
487 commit_msg
= git('log', '-1', '--format=%b', git_commit_hash
,
489 if len(commit_msg
) == 0:
490 die("Can't find git commit " + git_commit_hash
)
491 # If a commit has multiple "llvm-svn:" lines (e.g. if the commit is
492 # reverting/quoting a previous commit), choose the last one, which should
493 # be the authoritative one.
494 svn_match_iter
= re
.finditer('^llvm-svn: (\d{5,7})$', commit_msg
,
497 for m
in svn_match_iter
:
498 svn_match
= m
.group(1)
500 return int(svn_match
)
501 die("Can't find svn revision in git commit " + git_commit_hash
)
504 def cmd_svn_lookup(args
):
505 '''Find the SVN revision id for a given git commit hash.
507 This is identified by 'llvm-svn: NNNNNN' in the git commit message.'''
509 git_root
= git('rev-parse', '--show-toplevel')
510 if not os
.path
.isdir(git_root
):
511 die("Can't find git root dir")
513 # Run commands from the root
516 log('r' + str(lookup_llvm_svn_id(args
.git_commit_hash
)))
519 def git_hash_by_svn_rev(svn_rev
):
520 '''Find the git hash for a given svn revision.
522 This check is paranoid: 'llvm-svn: NNNNNN' could exist on its own line
523 somewhere else in the commit message. Look in the full log message to see
524 if it's actually on the last line.
526 Since this check is expensive (we're searching every single commit), limit
527 to the past 10k commits (about 5 months).
529 possible_hashes
= git(
530 'log', '--format=%H', '--grep', '^llvm-svn: %d$' % svn_rev
,
531 'HEAD~10000...HEAD').split('\n')
532 matching_hashes
= [h
for h
in possible_hashes
533 if lookup_llvm_svn_id(h
) == svn_rev
]
534 if len(matching_hashes
) > 1:
535 die("svn revision r%d has ambiguous commits: %s" % (
536 svn_rev
, ', '.join(matching_hashes
)))
537 elif len(matching_hashes
) < 1:
538 die("svn revision r%d matches no commits" % svn_rev
)
539 return matching_hashes
[0]
542 def cmd_revert(args
):
543 '''Revert a commit by either SVN id (rNNNNNN) or git hash. This also
544 populates the git commit message with both the SVN revision and git hash of
545 the change being reverted.'''
548 git_root
= git('rev-parse', '--show-toplevel')
549 if not os
.path
.isdir(git_root
):
550 die("Can't find git root dir")
552 # Run commands from the root
555 # Check for a client branch first.
556 open_files
= git('status', '-uno', '-s', '--porcelain')
557 if len(open_files
) > 0:
558 die("Found open files. Please stash and then revert.\n" + open_files
)
560 # If the revision looks like rNNNNNN (or with a callsign, e.g. rLLDNNNNNN),
561 # use that. Otherwise, look for it in the git commit.
562 svn_match
= re
.match('^r[A-Z]*(\d{5,7})$', args
.revision
)
564 # If the revision looks like rNNNNNN, use that as the svn revision, and
565 # grep through git commits to find which one corresponds to that svn
567 svn_rev
= int(svn_match
.group(1))
568 git_hash
= git_hash_by_svn_rev(svn_rev
)
570 # Otherwise, this looks like a git hash, so we just need to grab the
571 # svn revision from the end of the commit message. Get the actual git
572 # hash in case the revision is something like "HEAD~1"
573 git_hash
= git('rev-parse', '--verify', args
.revision
+ '^{commit}')
574 svn_rev
= lookup_llvm_svn_id(git_hash
)
576 msg
= git('log', '-1', '--format=%s', git_hash
)
578 log_verbose('Ready to revert r%d (%s): "%s"' % (svn_rev
, git_hash
, msg
))
580 revert_args
= ['revert', '--no-commit', git_hash
]
581 # TODO: Running --edit doesn't seem to work, with errors that stdin is not
584 'commit', '-m', 'Revert ' + msg
,
585 '-m', 'This reverts r%d (git commit %s)' % (svn_rev
, git_hash
)]
587 log("Would have run the following commands, if this weren't a"
589 '1) git %s\n2) git %s' % (
590 ' '.join(quote(arg
) for arg
in revert_args
),
591 ' '.join(quote(arg
) for arg
in commit_args
)))
595 commit_log
= git(*commit_args
)
597 log('Created revert of r%d: %s' % (svn_rev
, commit_log
))
598 log("Run 'git llvm push -n' to inspect your changes and "
599 "run 'git llvm push' when ready")
602 if __name__
== '__main__':
603 if not program_exists('svn'):
604 die('error: git-llvm needs svn command, but svn is not installed.')
607 p
= argparse
.ArgumentParser(
608 prog
='git llvm', formatter_class
=argparse
.RawDescriptionHelpFormatter
,
610 subcommands
= p
.add_subparsers(title
='subcommands',
611 description
='valid subcommands',
612 help='additional help')
613 verbosity_group
= p
.add_mutually_exclusive_group()
614 verbosity_group
.add_argument('-q', '--quiet', action
='store_true',
615 help='print less information')
616 verbosity_group
.add_argument('-v', '--verbose', action
='store_true',
617 help='print more information')
619 parser_push
= subcommands
.add_parser(
620 'push', description
=cmd_push
.__doc
__,
621 help='push changes back to the LLVM SVN repository')
622 parser_push
.add_argument(
627 help='Do everything other than commit to svn. Leaves junk in the svn '
628 'repo, so probably will not work well if you try to commit more '
630 parser_push
.add_argument(
634 help='Do not ask for confirmation when pushing multiple commits.')
635 parser_push
.add_argument(
640 help="revs to push (default: everything not in the branch's "
641 'upstream, or not in origin/master if the branch lacks '
642 'an explicit upstream)')
643 parser_push
.set_defaults(func
=cmd_push
)
645 parser_revert
= subcommands
.add_parser(
646 'revert', description
=cmd_revert
.__doc
__,
647 help='Revert a commit locally.')
648 parser_revert
.add_argument(
650 help='Revision to revert. Can either be an SVN revision number '
651 "(rNNNNNN) or a git commit hash (anything that doesn't look "
652 'like an SVN revision number).')
653 parser_revert
.add_argument(
658 help='Do everything other than perform a revert. Prints the git '
659 'revert command it would have run.')
660 parser_revert
.set_defaults(func
=cmd_revert
)
662 parser_svn_lookup
= subcommands
.add_parser(
663 'svn-lookup', description
=cmd_svn_lookup
.__doc
__,
664 help='Find the llvm-svn revision for a given commit.')
665 parser_svn_lookup
.add_argument(
667 help='git_commit_hash for which we will look up the svn revision id.')
668 parser_svn_lookup
.set_defaults(func
=cmd_svn_lookup
)
670 args
= p
.parse_args(argv
)
671 VERBOSE
= args
.verbose
674 # Python3 workaround, for when not arguments are provided.
675 # See https://bugs.python.org/issue16308
678 except AttributeError:
679 # No arguments or subcommands were given.
683 # Dispatch to the right subcommand