[ARM] Add or update a number of costmodel tests. NFC
[llvm-complete.git] / utils / git-svn / git-llvm
blob43ec4fb06ef1259a658d411a11b0529492ad537c
1 #!/usr/bin/env python
3 # ======- git-llvm - LLVM Git Help Integration ---------*- python -*--========#
5 # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
6 # See https://llvm.org/LICENSE.txt for license information.
7 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
9 # ==------------------------------------------------------------------------==#
11 """
12 git-llvm integration
13 ====================
15 This file provides integration for git.
16 """
18 from __future__ import print_function
19 import argparse
20 import collections
21 import os
22 import re
23 import shutil
24 import subprocess
25 import sys
26 import time
27 assert sys.version_info >= (2, 7)
29 try:
30 dict.iteritems
31 except AttributeError:
32 # Python 3
33 def iteritems(d):
34 return iter(d.items())
35 else:
36 # Python 2
37 def iteritems(d):
38 return d.iteritems()
40 try:
41 # Python 3
42 from shlex import quote
43 except ImportError:
44 # Python 2
45 from pipes import quote
47 # It's *almost* a straightforward mapping from the monorepo to svn...
48 LLVM_MONOREPO_SVN_MAPPING = {
49 d: (d + '/trunk')
50 for d in [
51 'clang-tools-extra',
52 'compiler-rt',
53 'debuginfo-tests',
54 'dragonegg',
55 'klee',
56 'libclc',
57 'libcxx',
58 'libcxxabi',
59 'libunwind',
60 'lld',
61 'lldb',
62 'llgo',
63 'llvm',
64 'openmp',
65 'parallel-libs',
66 'polly',
67 'pstl',
70 LLVM_MONOREPO_SVN_MAPPING.update({'clang': 'cfe/trunk'})
71 LLVM_MONOREPO_SVN_MAPPING.update({'': 'monorepo-root/trunk'})
73 SPLIT_REPO_NAMES = {'llvm-' + d: d + '/trunk'
74 for d in ['www', 'zorg', 'test-suite', 'lnt']}
76 VERBOSE = False
77 QUIET = False
78 dev_null_fd = None
81 def eprint(*args, **kwargs):
82 print(*args, file=sys.stderr, **kwargs)
85 def log(*args, **kwargs):
86 if QUIET:
87 return
88 print(*args, **kwargs)
91 def log_verbose(*args, **kwargs):
92 if not VERBOSE:
93 return
94 print(*args, **kwargs)
97 def die(msg):
98 eprint(msg)
99 sys.exit(1)
102 def ask_confirm(prompt):
103 # Python 2/3 compatibility
104 try:
105 read_input = raw_input
106 except NameError:
107 read_input = input
109 while True:
110 query = read_input('%s (y/N): ' % (prompt))
111 if query.lower() not in ['y','n', '']:
112 print('Expect y or n!')
113 continue
114 return query.lower() == 'y'
117 def split_first_path_component(d):
118 # Assuming we have a git path, it'll use slashes even on windows...I hope.
119 if '/' in d:
120 return d.split('/', 1)
121 else:
122 return (d, None)
125 def get_dev_null():
126 """Lazily create a /dev/null fd for use in shell()"""
127 global dev_null_fd
128 if dev_null_fd is None:
129 dev_null_fd = open(os.devnull, 'w')
130 return dev_null_fd
133 def shell(cmd, strip=True, cwd=None, stdin=None, die_on_failure=True,
134 ignore_errors=False, text=True):
135 # Escape args when logging for easy repro.
136 quoted_cmd = [quote(arg) for arg in cmd]
137 log_verbose('Running in %s: %s' % (cwd, ' '.join(quoted_cmd)))
139 err_pipe = subprocess.PIPE
140 if ignore_errors:
141 # Silence errors if requested.
142 err_pipe = get_dev_null()
144 start = time.time()
145 p = subprocess.Popen(cmd, cwd=cwd, stdout=subprocess.PIPE, stderr=err_pipe,
146 stdin=subprocess.PIPE,
147 universal_newlines=text)
148 stdout, stderr = p.communicate(input=stdin)
149 elapsed = time.time() - start
151 log_verbose('Command took %0.1fs' % elapsed)
153 if p.returncode == 0 or ignore_errors:
154 if stderr and not ignore_errors:
155 eprint('`%s` printed to stderr:' % ' '.join(quoted_cmd))
156 eprint(stderr.rstrip())
157 if strip:
158 if text:
159 stdout = stdout.rstrip('\r\n')
160 else:
161 stdout = stdout.rstrip(b'\r\n')
162 if VERBOSE:
163 for l in stdout.splitlines():
164 log_verbose("STDOUT: %s" % l)
165 return stdout
166 err_msg = '`%s` returned %s' % (' '.join(quoted_cmd), p.returncode)
167 eprint(err_msg)
168 if stderr:
169 eprint(stderr.rstrip())
170 if die_on_failure:
171 sys.exit(2)
172 raise RuntimeError(err_msg)
175 def git(*cmd, **kwargs):
176 return shell(['git'] + list(cmd), **kwargs)
179 def svn(cwd, *cmd, **kwargs):
180 return shell(['svn'] + list(cmd), cwd=cwd, **kwargs)
183 def program_exists(cmd):
184 if sys.platform == 'win32' and not cmd.endswith('.exe'):
185 cmd += '.exe'
186 for path in os.environ["PATH"].split(os.pathsep):
187 if os.access(os.path.join(path, cmd), os.X_OK):
188 return True
189 return False
192 def get_default_rev_range():
193 # Get the branch tracked by the current branch, as set by
194 # git branch --set-upstream-to See http://serverfault.com/a/352236/38694.
195 cur_branch = git('rev-parse', '--symbolic-full-name', 'HEAD')
196 upstream_branch = git('for-each-ref', '--format=%(upstream:short)',
197 cur_branch)
198 if not upstream_branch:
199 upstream_branch = 'origin/master'
201 # Get the newest common ancestor between HEAD and our upstream branch.
202 upstream_rev = git('merge-base', 'HEAD', upstream_branch)
203 return '%s..' % upstream_rev
206 def get_revs_to_push(rev_range):
207 if not rev_range:
208 rev_range = get_default_rev_range()
209 # Use git show rather than some plumbing command to figure out which revs
210 # are in rev_range because it handles single revs (HEAD^) and ranges
211 # (foo..bar) like we want.
212 revs = git('show', '--reverse', '--quiet',
213 '--pretty=%h', rev_range).splitlines()
214 if not revs:
215 die('Nothing to push: No revs in range %s.' % rev_range)
216 return revs
219 def clean_svn(svn_repo):
220 svn(svn_repo, 'revert', '-R', '.')
222 # Unfortunately it appears there's no svn equivalent for git clean, so we
223 # have to do it ourselves.
224 for line in svn(svn_repo, 'status', '--no-ignore').split('\n'):
225 if not line.startswith('?'):
226 continue
227 filename = line[1:].strip()
228 filepath = os.path.abspath(os.path.join(svn_repo, filename))
229 abs_svn_repo = os.path.abspath(svn_repo)
230 # Safety check that the directory we are about to delete is
231 # actually within our svn staging dir.
232 if not filepath.startswith(abs_svn_repo):
233 die("Path to clean (%s) is not in svn staging dir (%s)"
234 % (filepath, abs_svn_repo))
236 if os.path.isdir(filepath):
237 shutil.rmtree(filepath)
238 else:
239 os.remove(filepath)
242 def svn_init(svn_root):
243 if not os.path.exists(svn_root):
244 log('Creating svn staging directory: (%s)' % (svn_root))
245 os.makedirs(svn_root)
246 svn(svn_root, 'checkout', '--depth=empty',
247 'https://llvm.org/svn/llvm-project/', '.')
248 log("svn staging area ready in '%s'" % svn_root)
249 if not os.path.isdir(svn_root):
250 die("Can't initialize svn staging dir (%s)" % svn_root)
253 def fix_eol_style_native(rev, svn_sr_path, files):
254 """Fix line endings before applying patches with Unix endings
256 SVN on Windows will check out files with CRLF for files with the
257 svn:eol-style property set to "native". This breaks `git apply`, which
258 typically works with Unix-line ending patches. Work around the problem here
259 by doing a dos2unix up front for files with svn:eol-style set to "native".
260 SVN will not commit a mass line ending re-doing because it detects the line
261 ending format for files with this property.
263 # Skip files that don't exist in SVN yet.
264 files = [f for f in files if os.path.exists(os.path.join(svn_sr_path, f))]
265 # Use ignore_errors because 'svn propget' prints errors if the file doesn't
266 # have the named property. There doesn't seem to be a way to suppress that.
267 eol_props = svn(svn_sr_path, 'propget', 'svn:eol-style', *files,
268 ignore_errors=True)
269 crlf_files = []
270 if len(files) == 1:
271 # No need to split propget output on ' - ' when we have one file.
272 if eol_props.strip() in ['native', 'CRLF']:
273 crlf_files = files
274 else:
275 for eol_prop in eol_props.split('\n'):
276 # Remove spare CR.
277 eol_prop = eol_prop.strip('\r')
278 if not eol_prop:
279 continue
280 prop_parts = eol_prop.rsplit(' - ', 1)
281 if len(prop_parts) != 2:
282 eprint("unable to parse svn propget line:")
283 eprint(eol_prop)
284 continue
285 (f, eol_style) = prop_parts
286 if eol_style == 'native':
287 crlf_files.append(f)
288 if crlf_files:
289 # Reformat all files with native SVN line endings to Unix format. SVN
290 # knows files with native line endings are text files. It will commit
291 # just the diff, and not a mass line ending change.
292 shell(['dos2unix'] + crlf_files, ignore_errors=True, cwd=svn_sr_path)
295 def split_subrepo(f, git_to_svn_mapping):
296 # Given a path, splits it into (subproject, rest-of-path). If the path is
297 # not in a subproject, returns ('', full-path).
299 subproject, remainder = split_first_path_component(f)
301 if subproject in git_to_svn_mapping:
302 return subproject, remainder
303 else:
304 return '', f
307 def get_all_parent_dirs(name):
308 parts = []
309 head, tail = os.path.split(name)
310 while head:
311 parts.append(head)
312 head, tail = os.path.split(head)
313 return parts
316 def svn_push_one_rev(svn_repo, rev, git_to_svn_mapping, dry_run):
317 def split_status(x):
318 x = x.split('\t')
319 return x[1], x[0]
320 files_status = [split_status(x) for x in
321 git('diff-tree', '--no-commit-id', '--name-status',
322 '--no-renames', '-r', rev).split('\n')]
323 if not files_status:
324 raise RuntimeError('Empty diff for rev %s?' % rev)
326 # Split files by subrepo
327 subrepo_files = collections.defaultdict(list)
328 for f, st in files_status:
329 subrepo, remainder = split_subrepo(f, git_to_svn_mapping)
330 subrepo_files[subrepo].append((remainder, st))
332 status = svn(svn_repo, 'status', '--no-ignore')
333 if status:
334 die("Can't push git rev %s because status in svn staging dir (%s) is "
335 "not empty:\n%s" % (rev, svn_repo, status))
337 svn_dirs_to_update = set()
338 for sr, files_status in iteritems(subrepo_files):
339 svn_sr_path = git_to_svn_mapping[sr]
340 for f, _ in files_status:
341 svn_dirs_to_update.add(
342 os.path.dirname(os.path.join(svn_sr_path, f)))
344 # We also need to svn update any parent directories which are not yet
345 # present
346 parent_dirs = set()
347 for dir in svn_dirs_to_update:
348 parent_dirs.update(get_all_parent_dirs(dir))
349 parent_dirs = set(dir for dir in parent_dirs
350 if not os.path.exists(os.path.join(svn_repo, dir)))
351 svn_dirs_to_update.update(parent_dirs)
353 # Sort by length to ensure that the parent directories are passed to svn
354 # before child directories.
355 sorted_dirs_to_update = sorted(svn_dirs_to_update, key=len)
357 # SVN update only in the affected directories.
358 svn(svn_repo, 'update', '--depth=files', *sorted_dirs_to_update)
360 for sr, files_status in iteritems(subrepo_files):
361 svn_sr_path = os.path.join(svn_repo, git_to_svn_mapping[sr])
362 if os.name == 'nt':
363 fix_eol_style_native(rev, svn_sr_path,
364 [f for f, _ in files_status])
366 # We use text=False (and pass '--binary') so that we can get an exact
367 # diff that can be passed as-is to 'git apply' without any line ending,
368 # encoding, or other mangling.
369 diff = git('show', '--binary', rev, '--',
370 *(os.path.join(sr, f) for f, _ in files_status),
371 strip=False, text=False)
372 # git is the only thing that can handle its own patches...
373 if sr == '':
374 prefix_strip = '-p1'
375 else:
376 prefix_strip = '-p2'
377 try:
378 shell(['git', 'apply', prefix_strip, '-'], cwd=svn_sr_path,
379 stdin=diff, die_on_failure=False, text=False)
380 except RuntimeError as e:
381 eprint("Patch doesn't apply: maybe you should try `git pull -r` "
382 "first?")
383 sys.exit(2)
385 # Handle removed files and directories. We need to be careful not to
386 # remove directories just because they _look_ empty in the svn tree, as
387 # we might be missing sibling directories in the working copy. So, only
388 # remove parent directories if they're empty on both the git and svn
389 # sides.
390 maybe_dirs_to_remove = set()
391 for f, st in files_status:
392 if st == 'D':
393 maybe_dirs_to_remove.update(get_all_parent_dirs(f))
394 svn(svn_sr_path, 'remove', f)
395 elif not (st == 'A' or st == 'M' or st == 'T'):
396 # Add is handled below, and nothing needs to be done for Modify.
397 # (FIXME: Type-change between symlink and file might need some
398 # special handling, but let's ignore that for now.)
399 die("Unexpected git status for %r: %r" % (f, st))
401 maybe_dirs_to_remove = sorted(maybe_dirs_to_remove, key=len)
402 for f in maybe_dirs_to_remove:
403 if(not os.path.exists(os.path.join(svn_sr_path, f)) and
404 git('ls-tree', '-d', rev, os.path.join(sr, f)) == ''):
405 svn(svn_sr_path, 'remove', f)
407 status_lines = svn(svn_repo, 'status', '--no-ignore').split('\n')
409 for l in status_lines:
410 f = l[1:].strip()
411 if l.startswith('?') or l.startswith('I'):
412 svn(svn_repo, 'add', '--no-ignore', f)
414 # Now we're ready to commit.
415 commit_msg = git('show', '--pretty=%B', '--quiet', rev)
416 if not dry_run:
417 commit_args = ['commit', '-m', commit_msg]
418 if '--force-interactive' in svn(svn_repo, 'commit', '--help'):
419 commit_args.append('--force-interactive')
420 log(svn(svn_repo, *commit_args))
421 log('Committed %s to svn.' % rev)
422 else:
423 log("Would have committed %s to svn, if this weren't a dry run." % rev)
426 def cmd_push(args):
427 '''Push changes back to SVN: this is extracted from Justin Lebar's script
428 available here: https://github.com/jlebar/llvm-repo-tools/
430 Note: a current limitation is that git does not track file rename, so they
431 will show up in SVN as delete+add.
433 # Get the git root
434 git_root = git('rev-parse', '--show-toplevel')
435 if not os.path.isdir(git_root):
436 die("Can't find git root dir")
438 # Push from the root of the git repo
439 os.chdir(git_root)
441 # Get the remote URL, and check if it's one of the standalone repos.
442 git_remote_url = git('ls-remote', '--get-url', 'origin')
443 git_remote_url = git_remote_url.rstrip('.git').rstrip('/')
444 git_remote_repo_name = git_remote_url.rsplit('/', 1)[-1]
445 split_repo_path = SPLIT_REPO_NAMES.get(git_remote_repo_name)
446 if split_repo_path:
447 git_to_svn_mapping = {'': split_repo_path}
448 else:
449 # Default to the monorepo mapping
450 git_to_svn_mapping = LLVM_MONOREPO_SVN_MAPPING
452 # We need a staging area for SVN, let's hide it in the .git directory.
453 dot_git_dir = git('rev-parse', '--git-common-dir')
454 # Not all versions of git support --git-common-dir and just print the
455 # unknown command back. If this happens, fall back to --git-dir
456 if dot_git_dir == '--git-common-dir':
457 dot_git_dir = git('rev-parse', '--git-dir')
459 svn_root = os.path.join(dot_git_dir, 'llvm-upstream-svn')
460 svn_init(svn_root)
462 rev_range = args.rev_range
463 dry_run = args.dry_run
464 revs = get_revs_to_push(rev_range)
465 log('%sPushing %d %s commit%s:\n%s' %
466 ('[DryRun] ' if dry_run else '', len(revs),
467 'split-repo (%s)' % split_repo_path
468 if split_repo_path else 'monorepo',
469 's' if len(revs) != 1 else '',
470 '\n'.join(' ' + git('show', '--oneline', '--quiet', c)
471 for c in revs)))
473 # Ask confirmation if multiple commits are about to be pushed
474 if len(revs) != 1:
475 if not ask_confirm("Are you sure you want to create %d commits?" % len(revs)):
476 die("Aborting")
478 for r in revs:
479 clean_svn(svn_root)
480 svn_push_one_rev(svn_root, r, git_to_svn_mapping, dry_run)
483 def lookup_llvm_svn_id(git_commit_hash):
484 # Use --format=%b to get the raw commit message, without any extra
485 # whitespace.
486 commit_msg = git('log', '-1', '--format=%b', git_commit_hash,
487 ignore_errors=True)
488 if len(commit_msg) == 0:
489 die("Can't find git commit " + git_commit_hash)
490 # If a commit has multiple "llvm-svn:" lines (e.g. if the commit is
491 # reverting/quoting a previous commit), choose the last one, which should
492 # be the authoritative one.
493 svn_match_iter = re.finditer('^llvm-svn: (\d{5,7})$', commit_msg,
494 re.MULTILINE)
495 svn_match = None
496 for m in svn_match_iter:
497 svn_match = m.group(1)
498 if svn_match:
499 return int(svn_match)
500 die("Can't find svn revision in git commit " + git_commit_hash)
503 def cmd_svn_lookup(args):
504 '''Find the SVN revision id for a given git commit hash.
506 This is identified by 'llvm-svn: NNNNNN' in the git commit message.'''
507 # Get the git root
508 git_root = git('rev-parse', '--show-toplevel')
509 if not os.path.isdir(git_root):
510 die("Can't find git root dir")
512 # Run commands from the root
513 os.chdir(git_root)
515 log('r' + str(lookup_llvm_svn_id(args.git_commit_hash)))
518 def git_hash_by_svn_rev(svn_rev):
519 '''Find the git hash for a given svn revision.
521 This check is paranoid: 'llvm-svn: NNNNNN' could exist on its own line
522 somewhere else in the commit message. Look in the full log message to see
523 if it's actually on the last line.
525 Since this check is expensive (we're searching every single commit), limit
526 to the past 10k commits (about 5 months).
528 possible_hashes = git(
529 'log', '--format=%H', '--grep', '^llvm-svn: %d$' % svn_rev,
530 'HEAD~10000...HEAD').split('\n')
531 matching_hashes = [h for h in possible_hashes
532 if lookup_llvm_svn_id(h) == svn_rev]
533 if len(matching_hashes) > 1:
534 die("svn revision r%d has ambiguous commits: %s" % (
535 svn_rev, ', '.join(matching_hashes)))
536 elif len(matching_hashes) < 1:
537 die("svn revision r%d matches no commits" % svn_rev)
538 return matching_hashes[0]
541 def cmd_revert(args):
542 '''Revert a commit by either SVN id (rNNNNNN) or git hash. This also
543 populates the git commit message with both the SVN revision and git hash of
544 the change being reverted.'''
546 # Get the git root
547 git_root = git('rev-parse', '--show-toplevel')
548 if not os.path.isdir(git_root):
549 die("Can't find git root dir")
551 # Run commands from the root
552 os.chdir(git_root)
554 # Check for a client branch first.
555 open_files = git('status', '-uno', '-s', '--porcelain')
556 if len(open_files) > 0:
557 die("Found open files. Please stash and then revert.\n" + open_files)
559 # If the revision looks like rNNNNNN, use that. Otherwise, look for it in
560 # the git commit.
561 svn_match = re.match('^r(\d{5,7})$', args.revision)
562 if svn_match:
563 # If the revision looks like rNNNNNN, use that as the svn revision, and
564 # grep through git commits to find which one corresponds to that svn
565 # revision.
566 svn_rev = int(svn_match.group(1))
567 git_hash = git_hash_by_svn_rev(svn_rev)
568 else:
569 # Otherwise, this looks like a git hash, so we just need to grab the
570 # svn revision from the end of the commit message. Get the actual git
571 # hash in case the revision is something like "HEAD~1"
572 git_hash = git('rev-parse', '--verify', args.revision + '^{commit}')
573 svn_rev = lookup_llvm_svn_id(git_hash)
575 msg = git('log', '-1', '--format=%s', git_hash)
577 log_verbose('Ready to revert r%d (%s): "%s"' % (svn_rev, git_hash, msg))
579 revert_args = ['revert', '--no-commit', git_hash]
580 # TODO: Running --edit doesn't seem to work, with errors that stdin is not
581 # a tty.
582 commit_args = [
583 'commit', '-m', 'Revert ' + msg,
584 '-m', 'This reverts r%d (git commit %s)' % (svn_rev, git_hash)]
585 if args.dry_run:
586 log("Would have run the following commands, if this weren't a"
587 "dry run:\n"
588 '1) git %s\n2) git %s' % (
589 ' '.join(quote(arg) for arg in revert_args),
590 ' '.join(quote(arg) for arg in commit_args)))
591 return
593 git(*revert_args)
594 commit_log = git(*commit_args)
596 log('Created revert of r%d: %s' % (svn_rev, commit_log))
597 log("Run 'git llvm push -n' to inspect your changes and "
598 "run 'git llvm push' when ready")
601 if __name__ == '__main__':
602 if not program_exists('svn'):
603 die('error: git-llvm needs svn command, but svn is not installed.')
605 argv = sys.argv[1:]
606 p = argparse.ArgumentParser(
607 prog='git llvm', formatter_class=argparse.RawDescriptionHelpFormatter,
608 description=__doc__)
609 subcommands = p.add_subparsers(title='subcommands',
610 description='valid subcommands',
611 help='additional help')
612 verbosity_group = p.add_mutually_exclusive_group()
613 verbosity_group.add_argument('-q', '--quiet', action='store_true',
614 help='print less information')
615 verbosity_group.add_argument('-v', '--verbose', action='store_true',
616 help='print more information')
618 parser_push = subcommands.add_parser(
619 'push', description=cmd_push.__doc__,
620 help='push changes back to the LLVM SVN repository')
621 parser_push.add_argument(
622 '-n',
623 '--dry-run',
624 dest='dry_run',
625 action='store_true',
626 help='Do everything other than commit to svn. Leaves junk in the svn '
627 'repo, so probably will not work well if you try to commit more '
628 'than one rev.')
629 parser_push.add_argument(
630 'rev_range',
631 metavar='GIT_REVS',
632 type=str,
633 nargs='?',
634 help="revs to push (default: everything not in the branch's "
635 'upstream, or not in origin/master if the branch lacks '
636 'an explicit upstream)')
637 parser_push.set_defaults(func=cmd_push)
639 parser_revert = subcommands.add_parser(
640 'revert', description=cmd_revert.__doc__,
641 help='Revert a commit locally.')
642 parser_revert.add_argument(
643 'revision',
644 help='Revision to revert. Can either be an SVN revision number '
645 "(rNNNNNN) or a git commit hash (anything that doesn't look "
646 'like an SVN revision number).')
647 parser_revert.add_argument(
648 '-n',
649 '--dry-run',
650 dest='dry_run',
651 action='store_true',
652 help='Do everything other than perform a revert. Prints the git '
653 'revert command it would have run.')
654 parser_revert.set_defaults(func=cmd_revert)
656 parser_svn_lookup = subcommands.add_parser(
657 'svn-lookup', description=cmd_svn_lookup.__doc__,
658 help='Find the llvm-svn revision for a given commit.')
659 parser_svn_lookup.add_argument(
660 'git_commit_hash',
661 help='git_commit_hash for which we will look up the svn revision id.')
662 parser_svn_lookup.set_defaults(func=cmd_svn_lookup)
664 args = p.parse_args(argv)
665 VERBOSE = args.verbose
666 QUIET = args.quiet
668 # Python3 workaround, for when not arguments are provided.
669 # See https://bugs.python.org/issue16308
670 try:
671 func = args.func
672 except AttributeError:
673 # No arguments or subcommands were given.
674 parser.print_help()
675 parser.exit()
677 # Dispatch to the right subcommand
678 args.func(args)