3 # ======- git-llvm - LLVM Git Help Integration ---------*- python -*--========#
5 # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
6 # See https://llvm.org/LICENSE.txt for license information.
7 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
9 # ==------------------------------------------------------------------------==#
15 This file provides integration for git.
18 from __future__
import print_function
27 assert sys
.version_info
>= (2, 7)
31 except AttributeError:
34 return iter(d
.items())
42 from shlex
import quote
45 from pipes
import quote
47 # It's *almost* a straightforward mapping from the monorepo to svn...
48 LLVM_MONOREPO_SVN_MAPPING
= {
70 LLVM_MONOREPO_SVN_MAPPING
.update({'clang': 'cfe/trunk'})
71 LLVM_MONOREPO_SVN_MAPPING
.update({'': 'monorepo-root/trunk'})
73 SPLIT_REPO_NAMES
= {'llvm-' + d
: d
+ '/trunk'
74 for d
in ['www', 'zorg', 'test-suite', 'lnt']}
81 def eprint(*args
, **kwargs
):
82 print(*args
, file=sys
.stderr
, **kwargs
)
85 def log(*args
, **kwargs
):
88 print(*args
, **kwargs
)
91 def log_verbose(*args
, **kwargs
):
94 print(*args
, **kwargs
)
102 def split_first_path_component(d
):
103 # Assuming we have a git path, it'll use slashes even on windows...I hope.
105 return d
.split('/', 1)
111 """Lazily create a /dev/null fd for use in shell()"""
113 if dev_null_fd
is None:
114 dev_null_fd
= open(os
.devnull
, 'w')
118 def shell(cmd
, strip
=True, cwd
=None, stdin
=None, die_on_failure
=True,
119 ignore_errors
=False, text
=True):
120 # Escape args when logging for easy repro.
121 quoted_cmd
= [quote(arg
) for arg
in cmd
]
122 log_verbose('Running in %s: %s' % (cwd
, ' '.join(quoted_cmd
)))
124 err_pipe
= subprocess
.PIPE
126 # Silence errors if requested.
127 err_pipe
= get_dev_null()
130 p
= subprocess
.Popen(cmd
, cwd
=cwd
, stdout
=subprocess
.PIPE
, stderr
=err_pipe
,
131 stdin
=subprocess
.PIPE
,
132 universal_newlines
=text
)
133 stdout
, stderr
= p
.communicate(input=stdin
)
134 elapsed
= time
.time() - start
136 log_verbose('Command took %0.1fs' % elapsed
)
138 if p
.returncode
== 0 or ignore_errors
:
139 if stderr
and not ignore_errors
:
140 eprint('`%s` printed to stderr:' % ' '.join(quoted_cmd
))
141 eprint(stderr
.rstrip())
144 stdout
= stdout
.rstrip('\r\n')
146 stdout
= stdout
.rstrip(b
'\r\n')
148 for l
in stdout
.splitlines():
149 log_verbose("STDOUT: %s" % l
)
151 err_msg
= '`%s` returned %s' % (' '.join(quoted_cmd
), p
.returncode
)
154 eprint(stderr
.rstrip())
157 raise RuntimeError(err_msg
)
160 def git(*cmd
, **kwargs
):
161 return shell(['git'] + list(cmd
), **kwargs
)
164 def svn(cwd
, *cmd
, **kwargs
):
165 return shell(['svn'] + list(cmd
), cwd
=cwd
, **kwargs
)
168 def program_exists(cmd
):
169 if sys
.platform
== 'win32' and not cmd
.endswith('.exe'):
171 for path
in os
.environ
["PATH"].split(os
.pathsep
):
172 if os
.access(os
.path
.join(path
, cmd
), os
.X_OK
):
177 def get_default_rev_range():
178 # Get the branch tracked by the current branch, as set by
179 # git branch --set-upstream-to See http://serverfault.com/a/352236/38694.
180 cur_branch
= git('rev-parse', '--symbolic-full-name', 'HEAD')
181 upstream_branch
= git('for-each-ref', '--format=%(upstream:short)',
183 if not upstream_branch
:
184 upstream_branch
= 'origin/master'
186 # Get the newest common ancestor between HEAD and our upstream branch.
187 upstream_rev
= git('merge-base', 'HEAD', upstream_branch
)
188 return '%s..' % upstream_rev
191 def get_revs_to_push(rev_range
):
193 rev_range
= get_default_rev_range()
194 # Use git show rather than some plumbing command to figure out which revs
195 # are in rev_range because it handles single revs (HEAD^) and ranges
196 # (foo..bar) like we want.
197 revs
= git('show', '--reverse', '--quiet',
198 '--pretty=%h', rev_range
).splitlines()
200 die('Nothing to push: No revs in range %s.' % rev_range
)
204 def clean_svn(svn_repo
):
205 svn(svn_repo
, 'revert', '-R', '.')
207 # Unfortunately it appears there's no svn equivalent for git clean, so we
208 # have to do it ourselves.
209 for line
in svn(svn_repo
, 'status', '--no-ignore').split('\n'):
210 if not line
.startswith('?'):
212 filename
= line
[1:].strip()
213 filepath
= os
.path
.abspath(os
.path
.join(svn_repo
, filename
))
214 abs_svn_repo
= os
.path
.abspath(svn_repo
)
215 # Safety check that the directory we are about to delete is
216 # actually within our svn staging dir.
217 if not filepath
.startswith(abs_svn_repo
):
218 die("Path to clean (%s) is not in svn staging dir (%s)"
219 % (filepath
, abs_svn_repo
))
221 if os
.path
.isdir(filepath
):
222 shutil
.rmtree(filepath
)
227 def svn_init(svn_root
):
228 if not os
.path
.exists(svn_root
):
229 log('Creating svn staging directory: (%s)' % (svn_root
))
230 os
.makedirs(svn_root
)
231 svn(svn_root
, 'checkout', '--depth=empty',
232 'https://llvm.org/svn/llvm-project/', '.')
233 log("svn staging area ready in '%s'" % svn_root
)
234 if not os
.path
.isdir(svn_root
):
235 die("Can't initialize svn staging dir (%s)" % svn_root
)
238 def fix_eol_style_native(rev
, svn_sr_path
, files
):
239 """Fix line endings before applying patches with Unix endings
241 SVN on Windows will check out files with CRLF for files with the
242 svn:eol-style property set to "native". This breaks `git apply`, which
243 typically works with Unix-line ending patches. Work around the problem here
244 by doing a dos2unix up front for files with svn:eol-style set to "native".
245 SVN will not commit a mass line ending re-doing because it detects the line
246 ending format for files with this property.
248 # Skip files that don't exist in SVN yet.
249 files
= [f
for f
in files
if os
.path
.exists(os
.path
.join(svn_sr_path
, f
))]
250 # Use ignore_errors because 'svn propget' prints errors if the file doesn't
251 # have the named property. There doesn't seem to be a way to suppress that.
252 eol_props
= svn(svn_sr_path
, 'propget', 'svn:eol-style', *files
,
256 # No need to split propget output on ' - ' when we have one file.
257 if eol_props
.strip() in ['native', 'CRLF']:
260 for eol_prop
in eol_props
.split('\n'):
262 eol_prop
= eol_prop
.strip('\r')
265 prop_parts
= eol_prop
.rsplit(' - ', 1)
266 if len(prop_parts
) != 2:
267 eprint("unable to parse svn propget line:")
270 (f
, eol_style
) = prop_parts
271 if eol_style
== 'native':
274 # Reformat all files with native SVN line endings to Unix format. SVN
275 # knows files with native line endings are text files. It will commit
276 # just the diff, and not a mass line ending change.
277 shell(['dos2unix'] + crlf_files
, ignore_errors
=True, cwd
=svn_sr_path
)
280 def split_subrepo(f
, git_to_svn_mapping
):
281 # Given a path, splits it into (subproject, rest-of-path). If the path is
282 # not in a subproject, returns ('', full-path).
284 subproject
, remainder
= split_first_path_component(f
)
286 if subproject
in git_to_svn_mapping
:
287 return subproject
, remainder
292 def get_all_parent_dirs(name
):
294 head
, tail
= os
.path
.split(name
)
297 head
, tail
= os
.path
.split(head
)
301 def svn_push_one_rev(svn_repo
, rev
, git_to_svn_mapping
, dry_run
):
302 files
= git('diff-tree', '--no-commit-id', '--name-only', '-r',
305 raise RuntimeError('Empty diff for rev %s?' % rev
)
307 # Split files by subrepo
308 subrepo_files
= collections
.defaultdict(list)
310 subrepo
, remainder
= split_subrepo(f
, git_to_svn_mapping
)
311 subrepo_files
[subrepo
].append(remainder
)
313 status
= svn(svn_repo
, 'status', '--no-ignore')
315 die("Can't push git rev %s because status in svn staging dir (%s) is "
316 "not empty:\n%s" % (rev
, svn_repo
, status
))
318 svn_dirs_to_update
= set()
319 for sr
, files
in iteritems(subrepo_files
):
320 svn_sr_path
= git_to_svn_mapping
[sr
]
322 svn_dirs_to_update
.add(
323 os
.path
.dirname(os
.path
.join(svn_sr_path
, f
)))
325 # We also need to svn update any parent directories which are not yet
328 for dir in svn_dirs_to_update
:
329 parent_dirs
.update(get_all_parent_dirs(dir))
330 parent_dirs
= set(dir for dir in parent_dirs
331 if not os
.path
.exists(os
.path
.join(svn_repo
, dir)))
332 svn_dirs_to_update
.update(parent_dirs
)
334 # Sort by length to ensure that the parent directories are passed to svn
335 # before child directories.
336 sorted_dirs_to_update
= sorted(svn_dirs_to_update
, key
=len)
338 # SVN update only in the affected directories.
339 svn(svn_repo
, 'update', '--depth=files', *sorted_dirs_to_update
)
341 for sr
, files
in iteritems(subrepo_files
):
342 svn_sr_path
= os
.path
.join(svn_repo
, git_to_svn_mapping
[sr
])
344 fix_eol_style_native(rev
, svn_sr_path
, files
)
345 # We use text=False (and pass '--binary') so that we can get an exact
346 # diff that can be passed as-is to 'git apply' without any line ending,
347 # encoding, or other mangling.
348 diff
= git('show', '--binary', rev
, '--',
349 *(os
.path
.join(sr
, f
) for f
in files
),
350 strip
=False, text
=False)
351 # git is the only thing that can handle its own patches...
357 shell(['git', 'apply', prefix_strip
, '-'], cwd
=svn_sr_path
,
358 stdin
=diff
, die_on_failure
=False, text
=False)
359 except RuntimeError as e
:
360 eprint("Patch doesn't apply: maybe you should try `git pull -r` "
364 status_lines
= svn(svn_repo
, 'status', '--no-ignore').split('\n')
366 for l
in (l
for l
in status_lines
if (l
.startswith('?') or
368 svn(svn_repo
, 'add', '--no-ignore', l
[1:].strip())
369 for l
in (l
for l
in status_lines
if l
.startswith('!')):
370 svn(svn_repo
, 'remove', l
[1:].strip())
372 # Now we're ready to commit.
373 commit_msg
= git('show', '--pretty=%B', '--quiet', rev
)
375 commit_args
= ['commit', '-m', commit_msg
]
376 if '--force-interactive' in svn(svn_repo
, 'commit', '--help'):
377 commit_args
.append('--force-interactive')
378 log(svn(svn_repo
, *commit_args
))
379 log('Committed %s to svn.' % rev
)
381 log("Would have committed %s to svn, if this weren't a dry run." % rev
)
385 '''Push changes back to SVN: this is extracted from Justin Lebar's script
386 available here: https://github.com/jlebar/llvm-repo-tools/
388 Note: a current limitation is that git does not track file rename, so they
389 will show up in SVN as delete+add.
392 git_root
= git('rev-parse', '--show-toplevel')
393 if not os
.path
.isdir(git_root
):
394 die("Can't find git root dir")
396 # Push from the root of the git repo
399 # Get the remote URL, and check if it's one of the standalone repos.
400 git_remote_url
= git('remote', 'get-url', 'origin')
401 git_remote_url
= git_remote_url
.rstrip('.git').rstrip('/')
402 git_remote_repo_name
= git_remote_url
.rsplit('/', 1)[-1]
403 split_repo_path
= SPLIT_REPO_NAMES
.get(git_remote_repo_name
)
405 git_to_svn_mapping
= {'': split_repo_path
}
407 # Default to the monorepo mapping
408 git_to_svn_mapping
= LLVM_MONOREPO_SVN_MAPPING
410 # We need a staging area for SVN, let's hide it in the .git directory.
411 dot_git_dir
= git('rev-parse', '--git-common-dir')
412 # Not all versions of git support --git-common-dir and just print the
413 # unknown command back. If this happens, fall back to --git-dir
414 if dot_git_dir
== '--git-common-dir':
415 dot_git_dir
= git('rev-parse', '--git-dir')
417 svn_root
= os
.path
.join(dot_git_dir
, 'llvm-upstream-svn')
420 rev_range
= args
.rev_range
421 dry_run
= args
.dry_run
422 revs
= get_revs_to_push(rev_range
)
423 log('%sPushing %d %s commit%s:\n%s' %
424 ('[DryRun] ' if dry_run
else '', len(revs
),
425 'split-repo (%s)' % split_repo_path
426 if split_repo_path
else 'monorepo',
427 's' if len(revs
) != 1 else '',
428 '\n'.join(' ' + git('show', '--oneline', '--quiet', c
)
432 svn_push_one_rev(svn_root
, r
, git_to_svn_mapping
, dry_run
)
435 def lookup_llvm_svn_id(git_commit_hash
):
436 # Use --format=%b to get the raw commit message, without any extra
438 commit_msg
= git('log', '-1', '--format=%b', git_commit_hash
,
440 if len(commit_msg
) == 0:
441 die("Can't find git commit " + git_commit_hash
)
442 # If a commit has multiple "llvm-svn:" lines (e.g. if the commit is
443 # reverting/quoting a previous commit), choose the last one, which should
444 # be the authoritative one.
445 svn_match_iter
= re
.finditer('^llvm-svn: (\d{5,7})$', commit_msg
,
448 for m
in svn_match_iter
:
449 svn_match
= m
.group(1)
451 return int(svn_match
)
452 die("Can't find svn revision in git commit " + git_commit_hash
)
455 def cmd_svn_lookup(args
):
456 '''Find the SVN revision id for a given git commit hash.
458 This is identified by 'llvm-svn: NNNNNN' in the git commit message.'''
460 git_root
= git('rev-parse', '--show-toplevel')
461 if not os
.path
.isdir(git_root
):
462 die("Can't find git root dir")
464 # Run commands from the root
467 log('r' + str(lookup_llvm_svn_id(args
.git_commit_hash
)))
470 def git_hash_by_svn_rev(svn_rev
):
471 '''Find the git hash for a given svn revision.
473 This check is paranoid: 'llvm-svn: NNNNNN' could exist on its own line
474 somewhere else in the commit message. Look in the full log message to see
475 if it's actually on the last line.
477 Since this check is expensive (we're searching every single commit), limit
478 to the past 10k commits (about 5 months).
480 possible_hashes
= git(
481 'log', '--format=%H', '--grep', '^llvm-svn: %d$' % svn_rev
,
482 'HEAD~10000...HEAD').split('\n')
483 matching_hashes
= [h
for h
in possible_hashes
484 if lookup_llvm_svn_id(h
) == svn_rev
]
485 if len(matching_hashes
) > 1:
486 die("svn revision r%d has ambiguous commits: %s" % (
487 svn_rev
, ', '.join(matching_hashes
)))
488 elif len(matching_hashes
) < 1:
489 die("svn revision r%d matches no commits" % svn_rev
)
490 return matching_hashes
[0]
493 def cmd_revert(args
):
494 '''Revert a commit by either SVN id (rNNNNNN) or git hash. This also
495 populates the git commit message with both the SVN revision and git hash of
496 the change being reverted.'''
499 git_root
= git('rev-parse', '--show-toplevel')
500 if not os
.path
.isdir(git_root
):
501 die("Can't find git root dir")
503 # Run commands from the root
506 # Check for a client branch first.
507 open_files
= git('status', '-uno', '-s', '--porcelain')
508 if len(open_files
) > 0:
509 die("Found open files. Please stash and then revert.\n" + open_files
)
511 # If the revision looks like rNNNNNN, use that. Otherwise, look for it in
513 svn_match
= re
.match('^r(\d{5,7})$', args
.revision
)
515 # If the revision looks like rNNNNNN, use that as the svn revision, and
516 # grep through git commits to find which one corresponds to that svn
518 svn_rev
= int(svn_match
.group(1))
519 git_hash
= git_hash_by_svn_rev(svn_rev
)
521 # Otherwise, this looks like a git hash, so we just need to grab the
522 # svn revision from the end of the commit message. Get the actual git
523 # hash in case the revision is something like "HEAD~1"
524 git_hash
= git('rev-parse', '--verify', args
.revision
+ '^{commit}')
525 svn_rev
= lookup_llvm_svn_id(git_hash
)
527 msg
= git('log', '-1', '--format=%s', git_hash
)
529 log_verbose('Ready to revert r%d (%s): "%s"' % (svn_rev
, git_hash
, msg
))
531 revert_args
= ['revert', '--no-commit', git_hash
]
532 # TODO: Running --edit doesn't seem to work, with errors that stdin is not
535 'commit', '-m', 'Revert ' + msg
,
536 '-m', 'This reverts r%d (git commit %s)' % (svn_rev
, git_hash
)]
538 log("Would have run the following commands, if this weren't a"
540 '1) git %s\n2) git %s' % (
541 ' '.join(quote(arg
) for arg
in revert_args
),
542 ' '.join(quote(arg
) for arg
in commit_args
)))
546 commit_log
= git(*commit_args
)
548 log('Created revert of r%d: %s' % (svn_rev
, commit_log
))
549 log("Run 'git llvm push -n' to inspect your changes and "
550 "run 'git llvm push' when ready")
553 if __name__
== '__main__':
554 if not program_exists('svn'):
555 die('error: git-llvm needs svn command, but svn is not installed.')
558 p
= argparse
.ArgumentParser(
559 prog
='git llvm', formatter_class
=argparse
.RawDescriptionHelpFormatter
,
561 subcommands
= p
.add_subparsers(title
='subcommands',
562 description
='valid subcommands',
563 help='additional help')
564 verbosity_group
= p
.add_mutually_exclusive_group()
565 verbosity_group
.add_argument('-q', '--quiet', action
='store_true',
566 help='print less information')
567 verbosity_group
.add_argument('-v', '--verbose', action
='store_true',
568 help='print more information')
570 parser_push
= subcommands
.add_parser(
571 'push', description
=cmd_push
.__doc
__,
572 help='push changes back to the LLVM SVN repository')
573 parser_push
.add_argument(
578 help='Do everything other than commit to svn. Leaves junk in the svn '
579 'repo, so probably will not work well if you try to commit more '
581 parser_push
.add_argument(
586 help="revs to push (default: everything not in the branch's "
587 'upstream, or not in origin/master if the branch lacks '
588 'an explicit upstream)')
589 parser_push
.set_defaults(func
=cmd_push
)
591 parser_revert
= subcommands
.add_parser(
592 'revert', description
=cmd_revert
.__doc
__,
593 help='Revert a commit locally.')
594 parser_revert
.add_argument(
596 help='Revision to revert. Can either be an SVN revision number '
597 "(rNNNNNN) or a git commit hash (anything that doesn't look "
598 'like an SVN revision number).')
599 parser_revert
.add_argument(
604 help='Do everything other than perform a revert. Prints the git '
605 'revert command it would have run.')
606 parser_revert
.set_defaults(func
=cmd_revert
)
608 parser_svn_lookup
= subcommands
.add_parser(
609 'svn-lookup', description
=cmd_svn_lookup
.__doc
__,
610 help='Find the llvm-svn revision for a given commit.')
611 parser_svn_lookup
.add_argument(
613 help='git_commit_hash for which we will look up the svn revision id.')
614 parser_svn_lookup
.set_defaults(func
=cmd_svn_lookup
)
616 args
= p
.parse_args(argv
)
617 VERBOSE
= args
.verbose
620 # Dispatch to the right subcommand