3 # ======- git-llvm - LLVM Git Help Integration ---------*- python -*--========#
5 # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
6 # See https://llvm.org/LICENSE.txt for license information.
7 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
9 # ==------------------------------------------------------------------------==#
15 This file provides integration for git.
18 from __future__
import print_function
27 assert sys
.version_info
>= (2, 7)
31 except AttributeError:
34 return iter(d
.items())
42 from shlex
import quote
45 from pipes
import quote
47 # It's *almost* a straightforward mapping from the monorepo to svn...
48 LLVM_MONOREPO_SVN_MAPPING
= {
70 LLVM_MONOREPO_SVN_MAPPING
.update({'clang': 'cfe/trunk'})
71 LLVM_MONOREPO_SVN_MAPPING
.update({'': 'monorepo-root/trunk'})
73 SPLIT_REPO_NAMES
= {'llvm-' + d
: d
+ '/trunk'
74 for d
in ['www', 'zorg', 'test-suite', 'lnt']}
81 def eprint(*args
, **kwargs
):
82 print(*args
, file=sys
.stderr
, **kwargs
)
85 def log(*args
, **kwargs
):
88 print(*args
, **kwargs
)
91 def log_verbose(*args
, **kwargs
):
94 print(*args
, **kwargs
)
102 def ask_confirm(prompt
):
103 # Python 2/3 compatibility
105 read_input
= raw_input
110 query
= read_input('%s (y/N): ' % (prompt
))
111 if query
.lower() not in ['y','n', '']:
112 print('Expect y or n!')
114 return query
.lower() == 'y'
117 def split_first_path_component(d
):
118 # Assuming we have a git path, it'll use slashes even on windows...I hope.
120 return d
.split('/', 1)
126 """Lazily create a /dev/null fd for use in shell()"""
128 if dev_null_fd
is None:
129 dev_null_fd
= open(os
.devnull
, 'w')
133 def shell(cmd
, strip
=True, cwd
=None, stdin
=None, die_on_failure
=True,
134 ignore_errors
=False, text
=True):
135 # Escape args when logging for easy repro.
136 quoted_cmd
= [quote(arg
) for arg
in cmd
]
137 log_verbose('Running in %s: %s' % (cwd
, ' '.join(quoted_cmd
)))
139 err_pipe
= subprocess
.PIPE
141 # Silence errors if requested.
142 err_pipe
= get_dev_null()
145 p
= subprocess
.Popen(cmd
, cwd
=cwd
, stdout
=subprocess
.PIPE
, stderr
=err_pipe
,
146 stdin
=subprocess
.PIPE
,
147 universal_newlines
=text
)
148 stdout
, stderr
= p
.communicate(input=stdin
)
149 elapsed
= time
.time() - start
151 log_verbose('Command took %0.1fs' % elapsed
)
153 if p
.returncode
== 0 or ignore_errors
:
154 if stderr
and not ignore_errors
:
155 eprint('`%s` printed to stderr:' % ' '.join(quoted_cmd
))
156 eprint(stderr
.rstrip())
159 stdout
= stdout
.rstrip('\r\n')
161 stdout
= stdout
.rstrip(b
'\r\n')
163 for l
in stdout
.splitlines():
164 log_verbose("STDOUT: %s" % l
)
166 err_msg
= '`%s` returned %s' % (' '.join(quoted_cmd
), p
.returncode
)
169 eprint(stderr
.rstrip())
172 raise RuntimeError(err_msg
)
175 def git(*cmd
, **kwargs
):
176 return shell(['git'] + list(cmd
), **kwargs
)
179 def svn(cwd
, *cmd
, **kwargs
):
180 return shell(['svn'] + list(cmd
), cwd
=cwd
, **kwargs
)
183 def program_exists(cmd
):
184 if sys
.platform
== 'win32' and not cmd
.endswith('.exe'):
186 for path
in os
.environ
["PATH"].split(os
.pathsep
):
187 if os
.access(os
.path
.join(path
, cmd
), os
.X_OK
):
192 def get_default_rev_range():
193 # Get the newest common ancestor between HEAD and our upstream branch.
194 upstream_rev
= git('merge-base', 'HEAD', '@{upstream}')
195 return '%s..' % upstream_rev
198 def get_revs_to_push(rev_range
):
200 rev_range
= get_default_rev_range()
201 # Use git show rather than some plumbing command to figure out which revs
202 # are in rev_range because it handles single revs (HEAD^) and ranges
203 # (foo..bar) like we want.
204 return git('show', '--reverse', '--quiet',
205 '--pretty=%h', rev_range
).splitlines()
208 def clean_svn(svn_repo
):
209 svn(svn_repo
, 'revert', '-R', '.')
211 # Unfortunately it appears there's no svn equivalent for git clean, so we
212 # have to do it ourselves.
213 for line
in svn(svn_repo
, 'status', '--no-ignore').split('\n'):
214 if not line
.startswith('?'):
216 filename
= line
[1:].strip()
217 filepath
= os
.path
.abspath(os
.path
.join(svn_repo
, filename
))
218 abs_svn_repo
= os
.path
.abspath(svn_repo
)
219 # Safety check that the directory we are about to delete is
220 # actually within our svn staging dir.
221 if not filepath
.startswith(abs_svn_repo
):
222 die("Path to clean (%s) is not in svn staging dir (%s)"
223 % (filepath
, abs_svn_repo
))
225 if os
.path
.isdir(filepath
):
226 shutil
.rmtree(filepath
)
231 def svn_init(svn_root
):
232 if not os
.path
.exists(svn_root
):
233 log('Creating svn staging directory: (%s)' % (svn_root
))
234 os
.makedirs(svn_root
)
235 svn(svn_root
, 'checkout', '--depth=empty',
236 'https://llvm.org/svn/llvm-project/', '.')
237 log("svn staging area ready in '%s'" % svn_root
)
238 if not os
.path
.isdir(svn_root
):
239 die("Can't initialize svn staging dir (%s)" % svn_root
)
242 def fix_eol_style_native(rev
, svn_sr_path
, files
):
243 """Fix line endings before applying patches with Unix endings
245 SVN on Windows will check out files with CRLF for files with the
246 svn:eol-style property set to "native". This breaks `git apply`, which
247 typically works with Unix-line ending patches. Work around the problem here
248 by doing a dos2unix up front for files with svn:eol-style set to "native".
249 SVN will not commit a mass line ending re-doing because it detects the line
250 ending format for files with this property.
252 # Skip files that don't exist in SVN yet.
253 files
= [f
for f
in files
if os
.path
.exists(os
.path
.join(svn_sr_path
, f
))]
254 # Use ignore_errors because 'svn propget' prints errors if the file doesn't
255 # have the named property. There doesn't seem to be a way to suppress that.
256 eol_props
= svn(svn_sr_path
, 'propget', 'svn:eol-style', *files
,
260 # No need to split propget output on ' - ' when we have one file.
261 if eol_props
.strip() in ['native', 'CRLF']:
264 for eol_prop
in eol_props
.split('\n'):
266 eol_prop
= eol_prop
.strip('\r')
269 prop_parts
= eol_prop
.rsplit(' - ', 1)
270 if len(prop_parts
) != 2:
271 eprint("unable to parse svn propget line:")
274 (f
, eol_style
) = prop_parts
275 if eol_style
== 'native':
278 # Reformat all files with native SVN line endings to Unix format. SVN
279 # knows files with native line endings are text files. It will commit
280 # just the diff, and not a mass line ending change.
281 shell(['dos2unix'] + crlf_files
, ignore_errors
=True, cwd
=svn_sr_path
)
284 def split_subrepo(f
, git_to_svn_mapping
):
285 # Given a path, splits it into (subproject, rest-of-path). If the path is
286 # not in a subproject, returns ('', full-path).
288 subproject
, remainder
= split_first_path_component(f
)
290 if subproject
in git_to_svn_mapping
:
291 return subproject
, remainder
296 def get_all_parent_dirs(name
):
298 head
, tail
= os
.path
.split(name
)
301 head
, tail
= os
.path
.split(head
)
305 def svn_push_one_rev(svn_repo
, rev
, git_to_svn_mapping
, dry_run
):
309 files_status
= [split_status(x
) for x
in
310 git('diff-tree', '--no-commit-id', '--name-status',
311 '--no-renames', '-r', rev
).split('\n')]
313 raise RuntimeError('Empty diff for rev %s?' % rev
)
315 # Split files by subrepo
316 subrepo_files
= collections
.defaultdict(list)
317 for f
, st
in files_status
:
318 subrepo
, remainder
= split_subrepo(f
, git_to_svn_mapping
)
319 subrepo_files
[subrepo
].append((remainder
, st
))
321 status
= svn(svn_repo
, 'status', '--no-ignore')
323 die("Can't push git rev %s because status in svn staging dir (%s) is "
324 "not empty:\n%s" % (rev
, svn_repo
, status
))
326 svn_dirs_to_update
= set()
327 for sr
, files_status
in iteritems(subrepo_files
):
328 svn_sr_path
= git_to_svn_mapping
[sr
]
329 for f
, _
in files_status
:
330 svn_dirs_to_update
.add(
331 os
.path
.dirname(os
.path
.join(svn_sr_path
, f
)))
333 # We also need to svn update any parent directories which are not yet
336 for dir in svn_dirs_to_update
:
337 parent_dirs
.update(get_all_parent_dirs(dir))
338 parent_dirs
= set(dir for dir in parent_dirs
339 if not os
.path
.exists(os
.path
.join(svn_repo
, dir)))
340 svn_dirs_to_update
.update(parent_dirs
)
342 # Sort by length to ensure that the parent directories are passed to svn
343 # before child directories.
344 sorted_dirs_to_update
= sorted(svn_dirs_to_update
, key
=len)
346 # SVN update only in the affected directories.
347 svn(svn_repo
, 'update', '--depth=files', *sorted_dirs_to_update
)
349 for sr
, files_status
in iteritems(subrepo_files
):
350 svn_sr_path
= os
.path
.join(svn_repo
, git_to_svn_mapping
[sr
])
352 fix_eol_style_native(rev
, svn_sr_path
,
353 [f
for f
, _
in files_status
])
355 # We use text=False (and pass '--binary') so that we can get an exact
356 # diff that can be passed as-is to 'git apply' without any line ending,
357 # encoding, or other mangling.
358 diff
= git('show', '--binary', rev
, '--',
359 *(os
.path
.join(sr
, f
) for f
, _
in files_status
),
360 strip
=False, text
=False)
361 # git is the only thing that can handle its own patches...
367 shell(['git', 'apply', prefix_strip
, '-'], cwd
=svn_sr_path
,
368 stdin
=diff
, die_on_failure
=False, text
=False)
369 except RuntimeError as e
:
370 eprint("Patch doesn't apply: maybe you should try `git pull -r` "
374 # Handle removed files and directories. We need to be careful not to
375 # remove directories just because they _look_ empty in the svn tree, as
376 # we might be missing sibling directories in the working copy. So, only
377 # remove parent directories if they're empty on both the git and svn
379 maybe_dirs_to_remove
= set()
380 for f
, st
in files_status
:
382 maybe_dirs_to_remove
.update(get_all_parent_dirs(f
))
383 svn(svn_sr_path
, 'remove', f
)
384 elif not (st
== 'A' or st
== 'M' or st
== 'T'):
385 # Add is handled below, and nothing needs to be done for Modify.
386 # (FIXME: Type-change between symlink and file might need some
387 # special handling, but let's ignore that for now.)
388 die("Unexpected git status for %r: %r" % (f
, st
))
390 maybe_dirs_to_remove
= sorted(maybe_dirs_to_remove
, key
=len)
391 for f
in maybe_dirs_to_remove
:
392 if(not os
.path
.exists(os
.path
.join(svn_sr_path
, f
)) and
393 git('ls-tree', '-d', rev
, os
.path
.join(sr
, f
)) == ''):
394 svn(svn_sr_path
, 'remove', f
)
396 status_lines
= svn(svn_repo
, 'status', '--no-ignore').split('\n')
398 for l
in status_lines
:
400 if l
.startswith('?') or l
.startswith('I'):
401 svn(svn_repo
, 'add', '--no-ignore', f
)
403 # Now we're ready to commit.
404 commit_msg
= git('show', '--pretty=%B', '--quiet', rev
)
406 commit_args
= ['commit', '-m', commit_msg
]
407 if '--force-interactive' in svn(svn_repo
, 'commit', '--help'):
408 commit_args
.append('--force-interactive')
409 log(svn(svn_repo
, *commit_args
))
410 log('Committed %s to svn.' % rev
)
412 log("Would have committed %s to svn, if this weren't a dry run." % rev
)
416 '''Push changes back to SVN: this is extracted from Justin Lebar's script
417 available here: https://github.com/jlebar/llvm-repo-tools/
419 Note: a current limitation is that git does not track file rename, so they
420 will show up in SVN as delete+add.
423 git_root
= git('rev-parse', '--show-toplevel')
424 if not os
.path
.isdir(git_root
):
425 die("Can't find git root dir")
427 # Push from the root of the git repo
430 # Get the remote URL, and check if it's one of the standalone repos.
431 git_remote_url
= git('ls-remote', '--get-url', 'origin')
432 git_remote_url
= git_remote_url
.rstrip('.git').rstrip('/')
433 git_remote_repo_name
= git_remote_url
.rsplit('/', 1)[-1]
434 split_repo_path
= SPLIT_REPO_NAMES
.get(git_remote_repo_name
)
436 git_to_svn_mapping
= {'': split_repo_path
}
438 # Default to the monorepo mapping
439 git_to_svn_mapping
= LLVM_MONOREPO_SVN_MAPPING
441 # We need a staging area for SVN, let's hide it in the .git directory.
442 dot_git_dir
= git('rev-parse', '--git-common-dir')
443 # Not all versions of git support --git-common-dir and just print the
444 # unknown command back. If this happens, fall back to --git-dir
445 if dot_git_dir
== '--git-common-dir':
446 dot_git_dir
= git('rev-parse', '--git-dir')
448 svn_root
= os
.path
.join(dot_git_dir
, 'llvm-upstream-svn')
451 rev_range
= args
.rev_range
452 dry_run
= args
.dry_run
453 revs
= get_revs_to_push(rev_range
)
455 if not args
.force
and not revs
:
456 die('Nothing to push: No revs in range %s.' % rev_range
)
458 log('%sPushing %d %s commit%s:\n%s' %
459 ('[DryRun] ' if dry_run
else '', len(revs
),
460 'split-repo (%s)' % split_repo_path
461 if split_repo_path
else 'monorepo',
462 's' if len(revs
) != 1 else '',
463 '\n'.join(' ' + git('show', '--oneline', '--quiet', c
)
466 # Ask confirmation if multiple commits are about to be pushed
467 if not args
.force
and len(revs
) > 1:
468 if not ask_confirm("Are you sure you want to create %d commits?" % len(revs
)):
473 svn_push_one_rev(svn_root
, r
, git_to_svn_mapping
, dry_run
)
476 def lookup_llvm_svn_id(git_commit_hash
):
477 # Use --format=%b to get the raw commit message, without any extra
479 commit_msg
= git('log', '-1', '--format=%b', git_commit_hash
,
481 if len(commit_msg
) == 0:
482 die("Can't find git commit " + git_commit_hash
)
483 # If a commit has multiple "llvm-svn:" lines (e.g. if the commit is
484 # reverting/quoting a previous commit), choose the last one, which should
485 # be the authoritative one.
486 svn_match_iter
= re
.finditer('^llvm-svn: (\d{5,7})$', commit_msg
,
489 for m
in svn_match_iter
:
490 svn_match
= m
.group(1)
492 return int(svn_match
)
493 die("Can't find svn revision in git commit " + git_commit_hash
)
496 def cmd_svn_lookup(args
):
497 '''Find the SVN revision id for a given git commit hash.
499 This is identified by 'llvm-svn: NNNNNN' in the git commit message.'''
501 git_root
= git('rev-parse', '--show-toplevel')
502 if not os
.path
.isdir(git_root
):
503 die("Can't find git root dir")
505 # Run commands from the root
508 log('r' + str(lookup_llvm_svn_id(args
.git_commit_hash
)))
511 def git_hash_by_svn_rev(svn_rev
):
512 '''Find the git hash for a given svn revision.
514 This check is paranoid: 'llvm-svn: NNNNNN' could exist on its own line
515 somewhere else in the commit message. Look in the full log message to see
516 if it's actually on the last line.
518 Since this check is expensive (we're searching every single commit), limit
519 to the past 10k commits (about 5 months).
521 possible_hashes
= git(
522 'log', '--format=%H', '--grep', '^llvm-svn: %d$' % svn_rev
,
523 'HEAD~10000...HEAD').split('\n')
524 matching_hashes
= [h
for h
in possible_hashes
525 if lookup_llvm_svn_id(h
) == svn_rev
]
526 if len(matching_hashes
) > 1:
527 die("svn revision r%d has ambiguous commits: %s" % (
528 svn_rev
, ', '.join(matching_hashes
)))
529 elif len(matching_hashes
) < 1:
530 die("svn revision r%d matches no commits" % svn_rev
)
531 return matching_hashes
[0]
534 def cmd_revert(args
):
535 '''Revert a commit by either SVN id (rNNNNNN) or git hash. This also
536 populates the git commit message with both the SVN revision and git hash of
537 the change being reverted.'''
540 git_root
= git('rev-parse', '--show-toplevel')
541 if not os
.path
.isdir(git_root
):
542 die("Can't find git root dir")
544 # Run commands from the root
547 # Check for a client branch first.
548 open_files
= git('status', '-uno', '-s', '--porcelain')
549 if len(open_files
) > 0:
550 die("Found open files. Please stash and then revert.\n" + open_files
)
552 # If the revision looks like rNNNNNN (or with a callsign, e.g. rLLDNNNNNN),
553 # use that. Otherwise, look for it in the git commit.
554 svn_match
= re
.match('^r[A-Z]*(\d{5,7})$', args
.revision
)
556 # If the revision looks like rNNNNNN, use that as the svn revision, and
557 # grep through git commits to find which one corresponds to that svn
559 svn_rev
= int(svn_match
.group(1))
560 git_hash
= git_hash_by_svn_rev(svn_rev
)
562 # Otherwise, this looks like a git hash, so we just need to grab the
563 # svn revision from the end of the commit message. Get the actual git
564 # hash in case the revision is something like "HEAD~1"
565 git_hash
= git('rev-parse', '--verify', args
.revision
+ '^{commit}')
566 svn_rev
= lookup_llvm_svn_id(git_hash
)
568 msg
= git('log', '-1', '--format=%s', git_hash
)
570 log_verbose('Ready to revert r%d (%s): "%s"' % (svn_rev
, git_hash
, msg
))
572 revert_args
= ['revert', '--no-commit', git_hash
]
573 # TODO: Running --edit doesn't seem to work, with errors that stdin is not
576 'commit', '-m', 'Revert ' + msg
,
577 '-m', 'This reverts r%d (git commit %s)' % (svn_rev
, git_hash
)]
579 log("Would have run the following commands, if this weren't a"
581 '1) git %s\n2) git %s' % (
582 ' '.join(quote(arg
) for arg
in revert_args
),
583 ' '.join(quote(arg
) for arg
in commit_args
)))
587 commit_log
= git(*commit_args
)
589 log('Created revert of r%d: %s' % (svn_rev
, commit_log
))
590 log("Run 'git llvm push -n' to inspect your changes and "
591 "run 'git llvm push' when ready")
594 if __name__
== '__main__':
595 if not program_exists('svn'):
596 die('error: git-llvm needs svn command, but svn is not installed.')
599 p
= argparse
.ArgumentParser(
600 prog
='git llvm', formatter_class
=argparse
.RawDescriptionHelpFormatter
,
602 subcommands
= p
.add_subparsers(title
='subcommands',
603 description
='valid subcommands',
604 help='additional help')
605 verbosity_group
= p
.add_mutually_exclusive_group()
606 verbosity_group
.add_argument('-q', '--quiet', action
='store_true',
607 help='print less information')
608 verbosity_group
.add_argument('-v', '--verbose', action
='store_true',
609 help='print more information')
611 parser_push
= subcommands
.add_parser(
612 'push', description
=cmd_push
.__doc
__,
613 help='push changes back to the LLVM SVN repository')
614 parser_push
.add_argument(
619 help='Do everything other than commit to svn. Leaves junk in the svn '
620 'repo, so probably will not work well if you try to commit more '
622 parser_push
.add_argument(
626 help='Do not ask for confirmation when pushing multiple commits.')
627 parser_push
.add_argument(
632 help="revs to push (default: everything not in the upstream branch).")
633 parser_push
.set_defaults(func
=cmd_push
)
635 parser_revert
= subcommands
.add_parser(
636 'revert', description
=cmd_revert
.__doc
__,
637 help='Revert a commit locally.')
638 parser_revert
.add_argument(
640 help='Revision to revert. Can either be an SVN revision number '
641 "(rNNNNNN) or a git commit hash (anything that doesn't look "
642 'like an SVN revision number).')
643 parser_revert
.add_argument(
648 help='Do everything other than perform a revert. Prints the git '
649 'revert command it would have run.')
650 parser_revert
.set_defaults(func
=cmd_revert
)
652 parser_svn_lookup
= subcommands
.add_parser(
653 'svn-lookup', description
=cmd_svn_lookup
.__doc
__,
654 help='Find the llvm-svn revision for a given commit.')
655 parser_svn_lookup
.add_argument(
657 help='git_commit_hash for which we will look up the svn revision id.')
658 parser_svn_lookup
.set_defaults(func
=cmd_svn_lookup
)
660 args
= p
.parse_args(argv
)
661 VERBOSE
= args
.verbose
664 # Python3 workaround, for when not arguments are provided.
665 # See https://bugs.python.org/issue16308
668 except AttributeError:
669 # No arguments or subcommands were given.
673 # Dispatch to the right subcommand