3 # ======- git-llvm - LLVM Git Help Integration ---------*- python -*--========#
5 # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
6 # See https://llvm.org/LICENSE.txt for license information.
7 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
9 # ==------------------------------------------------------------------------==#
15 This file provides integration for git.
18 from __future__
import print_function
27 assert sys
.version_info
>= (2, 7)
31 except AttributeError:
34 return iter(d
.items())
42 from shlex
import quote
45 from pipes
import quote
47 # It's *almost* a straightforward mapping from the monorepo to svn...
48 LLVM_MONOREPO_SVN_MAPPING
= {
71 LLVM_MONOREPO_SVN_MAPPING
.update({'clang': 'cfe/trunk'})
72 LLVM_MONOREPO_SVN_MAPPING
.update({'': 'monorepo-root/trunk'})
74 SPLIT_REPO_NAMES
= {'llvm-' + d
: d
+ '/trunk'
75 for d
in ['www', 'zorg', 'test-suite', 'lnt']}
82 def eprint(*args
, **kwargs
):
83 print(*args
, file=sys
.stderr
, **kwargs
)
86 def log(*args
, **kwargs
):
89 print(*args
, **kwargs
)
92 def log_verbose(*args
, **kwargs
):
95 print(*args
, **kwargs
)
103 def ask_confirm(prompt
):
104 # Python 2/3 compatibility
106 read_input
= raw_input
111 query
= read_input('%s (y/N): ' % (prompt
))
112 if query
.lower() not in ['y','n', '']:
113 print('Expect y or n!')
115 return query
.lower() == 'y'
118 def split_first_path_component(d
):
119 # Assuming we have a git path, it'll use slashes even on windows...I hope.
121 return d
.split('/', 1)
127 """Lazily create a /dev/null fd for use in shell()"""
129 if dev_null_fd
is None:
130 dev_null_fd
= open(os
.devnull
, 'w')
134 def shell(cmd
, strip
=True, cwd
=None, stdin
=None, die_on_failure
=True,
135 ignore_errors
=False, text
=True):
136 # Escape args when logging for easy repro.
137 quoted_cmd
= [quote(arg
) for arg
in cmd
]
138 log_verbose('Running in %s: %s' % (cwd
, ' '.join(quoted_cmd
)))
140 err_pipe
= subprocess
.PIPE
142 # Silence errors if requested.
143 err_pipe
= get_dev_null()
146 p
= subprocess
.Popen(cmd
, cwd
=cwd
, stdout
=subprocess
.PIPE
, stderr
=err_pipe
,
147 stdin
=subprocess
.PIPE
,
148 universal_newlines
=text
)
149 stdout
, stderr
= p
.communicate(input=stdin
)
150 elapsed
= time
.time() - start
152 log_verbose('Command took %0.1fs' % elapsed
)
154 if p
.returncode
== 0 or ignore_errors
:
155 if stderr
and not ignore_errors
:
156 eprint('`%s` printed to stderr:' % ' '.join(quoted_cmd
))
157 eprint(stderr
.rstrip())
160 stdout
= stdout
.rstrip('\r\n')
162 stdout
= stdout
.rstrip(b
'\r\n')
164 for l
in stdout
.splitlines():
165 log_verbose("STDOUT: %s" % l
)
167 err_msg
= '`%s` returned %s' % (' '.join(quoted_cmd
), p
.returncode
)
170 eprint(stderr
.rstrip())
173 raise RuntimeError(err_msg
)
176 def git(*cmd
, **kwargs
):
177 return shell(['git'] + list(cmd
), **kwargs
)
180 def svn(cwd
, *cmd
, **kwargs
):
181 return shell(['svn'] + list(cmd
), cwd
=cwd
, **kwargs
)
184 def program_exists(cmd
):
185 if sys
.platform
== 'win32' and not cmd
.endswith('.exe'):
187 for path
in os
.environ
["PATH"].split(os
.pathsep
):
188 if os
.access(os
.path
.join(path
, cmd
), os
.X_OK
):
193 def get_default_rev_range():
194 # Get the newest common ancestor between HEAD and our upstream branch.
195 upstream_rev
= git('merge-base', 'HEAD', '@{upstream}', ignore_errors
=True)
197 eprint("Warning: git-llvm assumes that origin/master is the upstream "
198 "branch but git does not.")
199 eprint("To make this warning go away: git branch -u origin/master")
200 eprint("To avoid this warning when creating branches: "
201 "git checkout -b MyBranchName origin/master")
202 upstream_rev
= git('merge-base', 'HEAD', 'origin/master')
204 return '%s..' % upstream_rev
207 def get_revs_to_push(rev_range
):
209 rev_range
= get_default_rev_range()
210 # Use git show rather than some plumbing command to figure out which revs
211 # are in rev_range because it handles single revs (HEAD^) and ranges
212 # (foo..bar) like we want.
213 return git('show', '--reverse', '--quiet',
214 '--pretty=%h', rev_range
).splitlines()
217 def clean_svn(svn_repo
):
218 svn(svn_repo
, 'revert', '-R', '.')
220 # Unfortunately it appears there's no svn equivalent for git clean, so we
221 # have to do it ourselves.
222 for line
in svn(svn_repo
, 'status', '--no-ignore').split('\n'):
223 if not line
.startswith('?'):
225 filename
= line
[1:].strip()
226 filepath
= os
.path
.abspath(os
.path
.join(svn_repo
, filename
))
227 abs_svn_repo
= os
.path
.abspath(svn_repo
)
228 # Safety check that the directory we are about to delete is
229 # actually within our svn staging dir.
230 if not filepath
.startswith(abs_svn_repo
):
231 die("Path to clean (%s) is not in svn staging dir (%s)"
232 % (filepath
, abs_svn_repo
))
234 if os
.path
.isdir(filepath
):
235 shutil
.rmtree(filepath
)
240 def svn_init(svn_root
):
241 if not os
.path
.exists(svn_root
):
242 log('Creating svn staging directory: (%s)' % (svn_root
))
243 os
.makedirs(svn_root
)
244 svn(svn_root
, 'checkout', '--depth=empty',
245 'https://llvm.org/svn/llvm-project/', '.')
246 log("svn staging area ready in '%s'" % svn_root
)
247 if not os
.path
.isdir(svn_root
):
248 die("Can't initialize svn staging dir (%s)" % svn_root
)
251 def fix_eol_style_native(rev
, svn_sr_path
, files
):
252 """Fix line endings before applying patches with Unix endings
254 SVN on Windows will check out files with CRLF for files with the
255 svn:eol-style property set to "native". This breaks `git apply`, which
256 typically works with Unix-line ending patches. Work around the problem here
257 by doing a dos2unix up front for files with svn:eol-style set to "native".
258 SVN will not commit a mass line ending re-doing because it detects the line
259 ending format for files with this property.
261 # Skip files that don't exist in SVN yet.
262 files
= [f
for f
in files
if os
.path
.exists(os
.path
.join(svn_sr_path
, f
))]
263 # Use ignore_errors because 'svn propget' prints errors if the file doesn't
264 # have the named property. There doesn't seem to be a way to suppress that.
265 eol_props
= svn(svn_sr_path
, 'propget', 'svn:eol-style', *files
,
269 # No need to split propget output on ' - ' when we have one file.
270 if eol_props
.strip() in ['native', 'CRLF']:
273 for eol_prop
in eol_props
.split('\n'):
275 eol_prop
= eol_prop
.strip('\r')
278 prop_parts
= eol_prop
.rsplit(' - ', 1)
279 if len(prop_parts
) != 2:
280 eprint("unable to parse svn propget line:")
283 (f
, eol_style
) = prop_parts
284 if eol_style
== 'native':
287 # Reformat all files with native SVN line endings to Unix format. SVN
288 # knows files with native line endings are text files. It will commit
289 # just the diff, and not a mass line ending change.
290 shell(['dos2unix'] + crlf_files
, ignore_errors
=True, cwd
=svn_sr_path
)
293 def split_subrepo(f
, git_to_svn_mapping
):
294 # Given a path, splits it into (subproject, rest-of-path). If the path is
295 # not in a subproject, returns ('', full-path).
297 subproject
, remainder
= split_first_path_component(f
)
299 if subproject
in git_to_svn_mapping
:
300 return subproject
, remainder
305 def get_all_parent_dirs(name
):
307 head
, tail
= os
.path
.split(name
)
310 head
, tail
= os
.path
.split(head
)
314 def svn_push_one_rev(svn_repo
, rev
, git_to_svn_mapping
, dry_run
):
318 files_status
= [split_status(x
) for x
in
319 git('diff-tree', '--no-commit-id', '--name-status',
320 '--no-renames', '-r', rev
).split('\n')]
322 raise RuntimeError('Empty diff for rev %s?' % rev
)
324 # Split files by subrepo
325 subrepo_files
= collections
.defaultdict(list)
326 for f
, st
in files_status
:
327 subrepo
, remainder
= split_subrepo(f
, git_to_svn_mapping
)
328 subrepo_files
[subrepo
].append((remainder
, st
))
330 status
= svn(svn_repo
, 'status', '--no-ignore')
332 die("Can't push git rev %s because status in svn staging dir (%s) is "
333 "not empty:\n%s" % (rev
, svn_repo
, status
))
335 svn_dirs_to_update
= set()
336 for sr
, files_status
in iteritems(subrepo_files
):
337 svn_sr_path
= git_to_svn_mapping
[sr
]
338 for f
, _
in files_status
:
339 svn_dirs_to_update
.add(
340 os
.path
.dirname(os
.path
.join(svn_sr_path
, f
)))
342 # We also need to svn update any parent directories which are not yet
345 for dir in svn_dirs_to_update
:
346 parent_dirs
.update(get_all_parent_dirs(dir))
347 parent_dirs
= set(dir for dir in parent_dirs
348 if not os
.path
.exists(os
.path
.join(svn_repo
, dir)))
349 svn_dirs_to_update
.update(parent_dirs
)
351 # Sort by length to ensure that the parent directories are passed to svn
352 # before child directories.
353 sorted_dirs_to_update
= sorted(svn_dirs_to_update
, key
=len)
355 # SVN update only in the affected directories.
356 svn(svn_repo
, 'update', '--depth=files', *sorted_dirs_to_update
)
358 for sr
, files_status
in iteritems(subrepo_files
):
359 svn_sr_path
= os
.path
.join(svn_repo
, git_to_svn_mapping
[sr
])
361 fix_eol_style_native(rev
, svn_sr_path
,
362 [f
for f
, _
in files_status
])
364 # We use text=False (and pass '--binary') so that we can get an exact
365 # diff that can be passed as-is to 'git apply' without any line ending,
366 # encoding, or other mangling.
367 diff
= git('show', '--binary', rev
, '--',
368 *(os
.path
.join(sr
, f
) for f
, _
in files_status
),
369 strip
=False, text
=False)
370 # git is the only thing that can handle its own patches...
376 shell(['git', 'apply', prefix_strip
, '-'], cwd
=svn_sr_path
,
377 stdin
=diff
, die_on_failure
=False, text
=False)
378 except RuntimeError as e
:
379 eprint("Patch doesn't apply: maybe you should try `git pull -r` "
383 # Handle removed files and directories. We need to be careful not to
384 # remove directories just because they _look_ empty in the svn tree, as
385 # we might be missing sibling directories in the working copy. So, only
386 # remove parent directories if they're empty on both the git and svn
388 maybe_dirs_to_remove
= set()
389 for f
, st
in files_status
:
391 maybe_dirs_to_remove
.update(get_all_parent_dirs(f
))
392 svn(svn_sr_path
, 'remove', f
)
393 elif not (st
== 'A' or st
== 'M' or st
== 'T'):
394 # Add is handled below, and nothing needs to be done for Modify.
395 # (FIXME: Type-change between symlink and file might need some
396 # special handling, but let's ignore that for now.)
397 die("Unexpected git status for %r: %r" % (f
, st
))
399 maybe_dirs_to_remove
= sorted(maybe_dirs_to_remove
, key
=len)
400 for f
in maybe_dirs_to_remove
:
401 if(not os
.path
.exists(os
.path
.join(svn_sr_path
, f
)) and
402 git('ls-tree', '-d', rev
, os
.path
.join(sr
, f
)) == ''):
403 svn(svn_sr_path
, 'remove', f
)
405 status_lines
= svn(svn_repo
, 'status', '--no-ignore').split('\n')
407 for l
in status_lines
:
409 if l
.startswith('?') or l
.startswith('I'):
410 svn(svn_repo
, 'add', '--no-ignore', f
)
412 # Now we're ready to commit.
413 commit_msg
= git('show', '--pretty=%B', '--quiet', rev
)
415 commit_args
= ['commit', '-m', commit_msg
]
416 if '--force-interactive' in svn(svn_repo
, 'commit', '--help'):
417 commit_args
.append('--force-interactive')
418 log(svn(svn_repo
, *commit_args
))
419 log('Committed %s to svn.' % rev
)
421 log("Would have committed %s to svn, if this weren't a dry run." % rev
)
425 '''Push changes back to SVN: this is extracted from Justin Lebar's script
426 available here: https://github.com/jlebar/llvm-repo-tools/
428 Note: a current limitation is that git does not track file rename, so they
429 will show up in SVN as delete+add.
432 git_root
= git('rev-parse', '--show-toplevel')
433 if not os
.path
.isdir(git_root
):
434 die("Can't find git root dir")
436 # Push from the root of the git repo
439 # Get the remote URL, and check if it's one of the standalone repos.
440 git_remote_url
= git('ls-remote', '--get-url', 'origin')
441 git_remote_url
= git_remote_url
.rstrip('.git').rstrip('/')
442 git_remote_repo_name
= git_remote_url
.rsplit('/', 1)[-1]
443 split_repo_path
= SPLIT_REPO_NAMES
.get(git_remote_repo_name
)
445 git_to_svn_mapping
= {'': split_repo_path
}
447 # Default to the monorepo mapping
448 git_to_svn_mapping
= LLVM_MONOREPO_SVN_MAPPING
450 # We need a staging area for SVN, let's hide it in the .git directory.
451 dot_git_dir
= git('rev-parse', '--git-common-dir')
452 # Not all versions of git support --git-common-dir and just print the
453 # unknown command back. If this happens, fall back to --git-dir
454 if dot_git_dir
== '--git-common-dir':
455 dot_git_dir
= git('rev-parse', '--git-dir')
457 svn_root
= os
.path
.join(dot_git_dir
, 'llvm-upstream-svn')
460 rev_range
= args
.rev_range
461 dry_run
= args
.dry_run
462 revs
= get_revs_to_push(rev_range
)
464 if not args
.force
and not revs
:
465 die('Nothing to push: No revs in range %s.' % rev_range
)
467 log('%sPushing %d %s commit%s:\n%s' %
468 ('[DryRun] ' if dry_run
else '', len(revs
),
469 'split-repo (%s)' % split_repo_path
470 if split_repo_path
else 'monorepo',
471 's' if len(revs
) != 1 else '',
472 '\n'.join(' ' + git('show', '--oneline', '--quiet', c
)
475 # Ask confirmation if multiple commits are about to be pushed
476 if not args
.force
and len(revs
) > 1:
477 if not ask_confirm("Are you sure you want to create %d commits?" % len(revs
)):
482 svn_push_one_rev(svn_root
, r
, git_to_svn_mapping
, dry_run
)
485 def lookup_llvm_svn_id(git_commit_hash
):
486 # Use --format=%b to get the raw commit message, without any extra
488 commit_msg
= git('log', '-1', '--format=%b', git_commit_hash
,
490 if len(commit_msg
) == 0:
491 die("Can't find git commit " + git_commit_hash
)
492 # If a commit has multiple "llvm-svn:" lines (e.g. if the commit is
493 # reverting/quoting a previous commit), choose the last one, which should
494 # be the authoritative one.
495 svn_match_iter
= re
.finditer('^llvm-svn: (\d{5,7})$', commit_msg
,
498 for m
in svn_match_iter
:
499 svn_match
= m
.group(1)
501 return int(svn_match
)
502 die("Can't find svn revision in git commit " + git_commit_hash
)
505 def cmd_svn_lookup(args
):
506 '''Find the SVN revision id for a given git commit hash.
508 This is identified by 'llvm-svn: NNNNNN' in the git commit message.'''
510 git_root
= git('rev-parse', '--show-toplevel')
511 if not os
.path
.isdir(git_root
):
512 die("Can't find git root dir")
514 # Run commands from the root
517 log('r' + str(lookup_llvm_svn_id(args
.git_commit_hash
)))
520 def git_hash_by_svn_rev(svn_rev
):
521 '''Find the git hash for a given svn revision.
523 This check is paranoid: 'llvm-svn: NNNNNN' could exist on its own line
524 somewhere else in the commit message. Look in the full log message to see
525 if it's actually on the last line.
527 Since this check is expensive (we're searching every single commit), limit
528 to the past 10k commits (about 5 months).
530 possible_hashes
= git(
531 'log', '--format=%H', '--grep', '^llvm-svn: %d$' % svn_rev
,
532 'HEAD~10000...HEAD').split('\n')
533 matching_hashes
= [h
for h
in possible_hashes
534 if lookup_llvm_svn_id(h
) == svn_rev
]
535 if len(matching_hashes
) > 1:
536 die("svn revision r%d has ambiguous commits: %s" % (
537 svn_rev
, ', '.join(matching_hashes
)))
538 elif len(matching_hashes
) < 1:
539 die("svn revision r%d matches no commits" % svn_rev
)
540 return matching_hashes
[0]
543 def cmd_revert(args
):
544 '''Revert a commit by either SVN id (rNNNNNN) or git hash. This also
545 populates the git commit message with both the SVN revision and git hash of
546 the change being reverted.'''
549 git_root
= git('rev-parse', '--show-toplevel')
550 if not os
.path
.isdir(git_root
):
551 die("Can't find git root dir")
553 # Run commands from the root
556 # Check for a client branch first.
557 open_files
= git('status', '-uno', '-s', '--porcelain')
558 if len(open_files
) > 0:
559 die("Found open files. Please stash and then revert.\n" + open_files
)
561 # If the revision looks like rNNNNNN (or with a callsign, e.g. rLLDNNNNNN),
562 # use that. Otherwise, look for it in the git commit.
563 svn_match
= re
.match('^r[A-Z]*(\d{5,7})$', args
.revision
)
565 # If the revision looks like rNNNNNN, use that as the svn revision, and
566 # grep through git commits to find which one corresponds to that svn
568 svn_rev
= int(svn_match
.group(1))
569 git_hash
= git_hash_by_svn_rev(svn_rev
)
571 # Otherwise, this looks like a git hash, so we just need to grab the
572 # svn revision from the end of the commit message. Get the actual git
573 # hash in case the revision is something like "HEAD~1"
574 git_hash
= git('rev-parse', '--verify', args
.revision
+ '^{commit}')
575 svn_rev
= lookup_llvm_svn_id(git_hash
)
577 msg
= git('log', '-1', '--format=%s', git_hash
)
579 log_verbose('Ready to revert r%d (%s): "%s"' % (svn_rev
, git_hash
, msg
))
581 revert_args
= ['revert', '--no-commit', git_hash
]
582 # TODO: Running --edit doesn't seem to work, with errors that stdin is not
585 'commit', '-m', 'Revert ' + msg
,
586 '-m', 'This reverts r%d (git commit %s)' % (svn_rev
, git_hash
)]
588 log("Would have run the following commands, if this weren't a"
590 '1) git %s\n2) git %s' % (
591 ' '.join(quote(arg
) for arg
in revert_args
),
592 ' '.join(quote(arg
) for arg
in commit_args
)))
596 commit_log
= git(*commit_args
)
598 log('Created revert of r%d: %s' % (svn_rev
, commit_log
))
599 log("Run 'git llvm push -n' to inspect your changes and "
600 "run 'git llvm push' when ready")
603 if __name__
== '__main__':
604 if not program_exists('svn'):
605 die('error: git-llvm needs svn command, but svn is not installed.')
608 p
= argparse
.ArgumentParser(
609 prog
='git llvm', formatter_class
=argparse
.RawDescriptionHelpFormatter
,
611 subcommands
= p
.add_subparsers(title
='subcommands',
612 description
='valid subcommands',
613 help='additional help')
614 verbosity_group
= p
.add_mutually_exclusive_group()
615 verbosity_group
.add_argument('-q', '--quiet', action
='store_true',
616 help='print less information')
617 verbosity_group
.add_argument('-v', '--verbose', action
='store_true',
618 help='print more information')
620 parser_push
= subcommands
.add_parser(
621 'push', description
=cmd_push
.__doc
__,
622 help='push changes back to the LLVM SVN repository')
623 parser_push
.add_argument(
628 help='Do everything other than commit to svn. Leaves junk in the svn '
629 'repo, so probably will not work well if you try to commit more '
631 parser_push
.add_argument(
635 help='Do not ask for confirmation when pushing multiple commits.')
636 parser_push
.add_argument(
641 help="revs to push (default: everything not in the branch's "
642 'upstream, or not in origin/master if the branch lacks '
643 'an explicit upstream)')
644 parser_push
.set_defaults(func
=cmd_push
)
646 parser_revert
= subcommands
.add_parser(
647 'revert', description
=cmd_revert
.__doc
__,
648 help='Revert a commit locally.')
649 parser_revert
.add_argument(
651 help='Revision to revert. Can either be an SVN revision number '
652 "(rNNNNNN) or a git commit hash (anything that doesn't look "
653 'like an SVN revision number).')
654 parser_revert
.add_argument(
659 help='Do everything other than perform a revert. Prints the git '
660 'revert command it would have run.')
661 parser_revert
.set_defaults(func
=cmd_revert
)
663 parser_svn_lookup
= subcommands
.add_parser(
664 'svn-lookup', description
=cmd_svn_lookup
.__doc
__,
665 help='Find the llvm-svn revision for a given commit.')
666 parser_svn_lookup
.add_argument(
668 help='git_commit_hash for which we will look up the svn revision id.')
669 parser_svn_lookup
.set_defaults(func
=cmd_svn_lookup
)
671 args
= p
.parse_args(argv
)
672 VERBOSE
= args
.verbose
675 # Python3 workaround, for when not arguments are provided.
676 # See https://bugs.python.org/issue16308
679 except AttributeError:
680 # No arguments or subcommands were given.
684 # Dispatch to the right subcommand