[ARM] Prevent generating NEON stack accesses under MVE.
[llvm-complete.git] / utils / git-svn / git-llvm
blobe03479a6df1a4f791abb1cbb265f1a460037eade
1 #!/usr/bin/env python
3 # ======- git-llvm - LLVM Git Help Integration ---------*- python -*--========#
5 # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
6 # See https://llvm.org/LICENSE.txt for license information.
7 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
9 # ==------------------------------------------------------------------------==#
11 """
12 git-llvm integration
13 ====================
15 This file provides integration for git.
16 """
18 from __future__ import print_function
19 import argparse
20 import collections
21 import os
22 import re
23 import shutil
24 import subprocess
25 import sys
26 import time
27 assert sys.version_info >= (2, 7)
29 try:
30 dict.iteritems
31 except AttributeError:
32 # Python 3
33 def iteritems(d):
34 return iter(d.items())
35 else:
36 # Python 2
37 def iteritems(d):
38 return d.iteritems()
40 try:
41 # Python 3
42 from shlex import quote
43 except ImportError:
44 # Python 2
45 from pipes import quote
47 # It's *almost* a straightforward mapping from the monorepo to svn...
48 LLVM_MONOREPO_SVN_MAPPING = {
49 d: (d + '/trunk')
50 for d in [
51 'clang-tools-extra',
52 'compiler-rt',
53 'debuginfo-tests',
54 'dragonegg',
55 'klee',
56 'libclc',
57 'libcxx',
58 'libcxxabi',
59 'libunwind',
60 'lld',
61 'lldb',
62 'llgo',
63 'llvm',
64 'openmp',
65 'parallel-libs',
66 'polly',
67 'pstl',
70 LLVM_MONOREPO_SVN_MAPPING.update({'clang': 'cfe/trunk'})
71 LLVM_MONOREPO_SVN_MAPPING.update({'': 'monorepo-root/trunk'})
73 SPLIT_REPO_NAMES = {'llvm-' + d: d + '/trunk'
74 for d in ['www', 'zorg', 'test-suite', 'lnt']}
76 VERBOSE = False
77 QUIET = False
78 dev_null_fd = None
81 def eprint(*args, **kwargs):
82 print(*args, file=sys.stderr, **kwargs)
85 def log(*args, **kwargs):
86 if QUIET:
87 return
88 print(*args, **kwargs)
91 def log_verbose(*args, **kwargs):
92 if not VERBOSE:
93 return
94 print(*args, **kwargs)
97 def die(msg):
98 eprint(msg)
99 sys.exit(1)
102 def ask_confirm(prompt):
103 # Python 2/3 compatibility
104 try:
105 read_input = raw_input
106 except NameError:
107 read_input = input
109 while True:
110 query = read_input('%s (y/N): ' % (prompt))
111 if query.lower() not in ['y','n', '']:
112 print('Expect y or n!')
113 continue
114 return query.lower() == 'y'
117 def split_first_path_component(d):
118 # Assuming we have a git path, it'll use slashes even on windows...I hope.
119 if '/' in d:
120 return d.split('/', 1)
121 else:
122 return (d, None)
125 def get_dev_null():
126 """Lazily create a /dev/null fd for use in shell()"""
127 global dev_null_fd
128 if dev_null_fd is None:
129 dev_null_fd = open(os.devnull, 'w')
130 return dev_null_fd
133 def shell(cmd, strip=True, cwd=None, stdin=None, die_on_failure=True,
134 ignore_errors=False, text=True):
135 # Escape args when logging for easy repro.
136 quoted_cmd = [quote(arg) for arg in cmd]
137 log_verbose('Running in %s: %s' % (cwd, ' '.join(quoted_cmd)))
139 err_pipe = subprocess.PIPE
140 if ignore_errors:
141 # Silence errors if requested.
142 err_pipe = get_dev_null()
144 start = time.time()
145 p = subprocess.Popen(cmd, cwd=cwd, stdout=subprocess.PIPE, stderr=err_pipe,
146 stdin=subprocess.PIPE,
147 universal_newlines=text)
148 stdout, stderr = p.communicate(input=stdin)
149 elapsed = time.time() - start
151 log_verbose('Command took %0.1fs' % elapsed)
153 if p.returncode == 0 or ignore_errors:
154 if stderr and not ignore_errors:
155 eprint('`%s` printed to stderr:' % ' '.join(quoted_cmd))
156 eprint(stderr.rstrip())
157 if strip:
158 if text:
159 stdout = stdout.rstrip('\r\n')
160 else:
161 stdout = stdout.rstrip(b'\r\n')
162 if VERBOSE:
163 for l in stdout.splitlines():
164 log_verbose("STDOUT: %s" % l)
165 return stdout
166 err_msg = '`%s` returned %s' % (' '.join(quoted_cmd), p.returncode)
167 eprint(err_msg)
168 if stderr:
169 eprint(stderr.rstrip())
170 if die_on_failure:
171 sys.exit(2)
172 raise RuntimeError(err_msg)
175 def git(*cmd, **kwargs):
176 return shell(['git'] + list(cmd), **kwargs)
179 def svn(cwd, *cmd, **kwargs):
180 return shell(['svn'] + list(cmd), cwd=cwd, **kwargs)
183 def program_exists(cmd):
184 if sys.platform == 'win32' and not cmd.endswith('.exe'):
185 cmd += '.exe'
186 for path in os.environ["PATH"].split(os.pathsep):
187 if os.access(os.path.join(path, cmd), os.X_OK):
188 return True
189 return False
192 def get_default_rev_range():
193 # Get the newest common ancestor between HEAD and our upstream branch.
194 upstream_rev = git('merge-base', 'HEAD', '@{upstream}')
195 return '%s..' % upstream_rev
198 def get_revs_to_push(rev_range):
199 if not rev_range:
200 rev_range = get_default_rev_range()
201 # Use git show rather than some plumbing command to figure out which revs
202 # are in rev_range because it handles single revs (HEAD^) and ranges
203 # (foo..bar) like we want.
204 return git('show', '--reverse', '--quiet',
205 '--pretty=%h', rev_range).splitlines()
208 def clean_svn(svn_repo):
209 svn(svn_repo, 'revert', '-R', '.')
211 # Unfortunately it appears there's no svn equivalent for git clean, so we
212 # have to do it ourselves.
213 for line in svn(svn_repo, 'status', '--no-ignore').split('\n'):
214 if not line.startswith('?'):
215 continue
216 filename = line[1:].strip()
217 filepath = os.path.abspath(os.path.join(svn_repo, filename))
218 abs_svn_repo = os.path.abspath(svn_repo)
219 # Safety check that the directory we are about to delete is
220 # actually within our svn staging dir.
221 if not filepath.startswith(abs_svn_repo):
222 die("Path to clean (%s) is not in svn staging dir (%s)"
223 % (filepath, abs_svn_repo))
225 if os.path.isdir(filepath):
226 shutil.rmtree(filepath)
227 else:
228 os.remove(filepath)
231 def svn_init(svn_root):
232 if not os.path.exists(svn_root):
233 log('Creating svn staging directory: (%s)' % (svn_root))
234 os.makedirs(svn_root)
235 svn(svn_root, 'checkout', '--depth=empty',
236 'https://llvm.org/svn/llvm-project/', '.')
237 log("svn staging area ready in '%s'" % svn_root)
238 if not os.path.isdir(svn_root):
239 die("Can't initialize svn staging dir (%s)" % svn_root)
242 def fix_eol_style_native(rev, svn_sr_path, files):
243 """Fix line endings before applying patches with Unix endings
245 SVN on Windows will check out files with CRLF for files with the
246 svn:eol-style property set to "native". This breaks `git apply`, which
247 typically works with Unix-line ending patches. Work around the problem here
248 by doing a dos2unix up front for files with svn:eol-style set to "native".
249 SVN will not commit a mass line ending re-doing because it detects the line
250 ending format for files with this property.
252 # Skip files that don't exist in SVN yet.
253 files = [f for f in files if os.path.exists(os.path.join(svn_sr_path, f))]
254 # Use ignore_errors because 'svn propget' prints errors if the file doesn't
255 # have the named property. There doesn't seem to be a way to suppress that.
256 eol_props = svn(svn_sr_path, 'propget', 'svn:eol-style', *files,
257 ignore_errors=True)
258 crlf_files = []
259 if len(files) == 1:
260 # No need to split propget output on ' - ' when we have one file.
261 if eol_props.strip() in ['native', 'CRLF']:
262 crlf_files = files
263 else:
264 for eol_prop in eol_props.split('\n'):
265 # Remove spare CR.
266 eol_prop = eol_prop.strip('\r')
267 if not eol_prop:
268 continue
269 prop_parts = eol_prop.rsplit(' - ', 1)
270 if len(prop_parts) != 2:
271 eprint("unable to parse svn propget line:")
272 eprint(eol_prop)
273 continue
274 (f, eol_style) = prop_parts
275 if eol_style == 'native':
276 crlf_files.append(f)
277 if crlf_files:
278 # Reformat all files with native SVN line endings to Unix format. SVN
279 # knows files with native line endings are text files. It will commit
280 # just the diff, and not a mass line ending change.
281 shell(['dos2unix'] + crlf_files, ignore_errors=True, cwd=svn_sr_path)
284 def split_subrepo(f, git_to_svn_mapping):
285 # Given a path, splits it into (subproject, rest-of-path). If the path is
286 # not in a subproject, returns ('', full-path).
288 subproject, remainder = split_first_path_component(f)
290 if subproject in git_to_svn_mapping:
291 return subproject, remainder
292 else:
293 return '', f
296 def get_all_parent_dirs(name):
297 parts = []
298 head, tail = os.path.split(name)
299 while head:
300 parts.append(head)
301 head, tail = os.path.split(head)
302 return parts
305 def svn_push_one_rev(svn_repo, rev, git_to_svn_mapping, dry_run):
306 def split_status(x):
307 x = x.split('\t')
308 return x[1], x[0]
309 files_status = [split_status(x) for x in
310 git('diff-tree', '--no-commit-id', '--name-status',
311 '--no-renames', '-r', rev).split('\n')]
312 if not files_status:
313 raise RuntimeError('Empty diff for rev %s?' % rev)
315 # Split files by subrepo
316 subrepo_files = collections.defaultdict(list)
317 for f, st in files_status:
318 subrepo, remainder = split_subrepo(f, git_to_svn_mapping)
319 subrepo_files[subrepo].append((remainder, st))
321 status = svn(svn_repo, 'status', '--no-ignore')
322 if status:
323 die("Can't push git rev %s because status in svn staging dir (%s) is "
324 "not empty:\n%s" % (rev, svn_repo, status))
326 svn_dirs_to_update = set()
327 for sr, files_status in iteritems(subrepo_files):
328 svn_sr_path = git_to_svn_mapping[sr]
329 for f, _ in files_status:
330 svn_dirs_to_update.add(
331 os.path.dirname(os.path.join(svn_sr_path, f)))
333 # We also need to svn update any parent directories which are not yet
334 # present
335 parent_dirs = set()
336 for dir in svn_dirs_to_update:
337 parent_dirs.update(get_all_parent_dirs(dir))
338 parent_dirs = set(dir for dir in parent_dirs
339 if not os.path.exists(os.path.join(svn_repo, dir)))
340 svn_dirs_to_update.update(parent_dirs)
342 # Sort by length to ensure that the parent directories are passed to svn
343 # before child directories.
344 sorted_dirs_to_update = sorted(svn_dirs_to_update, key=len)
346 # SVN update only in the affected directories.
347 svn(svn_repo, 'update', '--depth=files', *sorted_dirs_to_update)
349 for sr, files_status in iteritems(subrepo_files):
350 svn_sr_path = os.path.join(svn_repo, git_to_svn_mapping[sr])
351 if os.name == 'nt':
352 fix_eol_style_native(rev, svn_sr_path,
353 [f for f, _ in files_status])
355 # We use text=False (and pass '--binary') so that we can get an exact
356 # diff that can be passed as-is to 'git apply' without any line ending,
357 # encoding, or other mangling.
358 diff = git('show', '--binary', rev, '--',
359 *(os.path.join(sr, f) for f, _ in files_status),
360 strip=False, text=False)
361 # git is the only thing that can handle its own patches...
362 if sr == '':
363 prefix_strip = '-p1'
364 else:
365 prefix_strip = '-p2'
366 try:
367 shell(['git', 'apply', prefix_strip, '-'], cwd=svn_sr_path,
368 stdin=diff, die_on_failure=False, text=False)
369 except RuntimeError as e:
370 eprint("Patch doesn't apply: maybe you should try `git pull -r` "
371 "first?")
372 sys.exit(2)
374 # Handle removed files and directories. We need to be careful not to
375 # remove directories just because they _look_ empty in the svn tree, as
376 # we might be missing sibling directories in the working copy. So, only
377 # remove parent directories if they're empty on both the git and svn
378 # sides.
379 maybe_dirs_to_remove = set()
380 for f, st in files_status:
381 if st == 'D':
382 maybe_dirs_to_remove.update(get_all_parent_dirs(f))
383 svn(svn_sr_path, 'remove', f)
384 elif not (st == 'A' or st == 'M' or st == 'T'):
385 # Add is handled below, and nothing needs to be done for Modify.
386 # (FIXME: Type-change between symlink and file might need some
387 # special handling, but let's ignore that for now.)
388 die("Unexpected git status for %r: %r" % (f, st))
390 maybe_dirs_to_remove = sorted(maybe_dirs_to_remove, key=len)
391 for f in maybe_dirs_to_remove:
392 if(not os.path.exists(os.path.join(svn_sr_path, f)) and
393 git('ls-tree', '-d', rev, os.path.join(sr, f)) == ''):
394 svn(svn_sr_path, 'remove', f)
396 status_lines = svn(svn_repo, 'status', '--no-ignore').split('\n')
398 for l in status_lines:
399 f = l[1:].strip()
400 if l.startswith('?') or l.startswith('I'):
401 svn(svn_repo, 'add', '--no-ignore', f)
403 # Now we're ready to commit.
404 commit_msg = git('show', '--pretty=%B', '--quiet', rev)
405 if not dry_run:
406 commit_args = ['commit', '-m', commit_msg]
407 if '--force-interactive' in svn(svn_repo, 'commit', '--help'):
408 commit_args.append('--force-interactive')
409 log(svn(svn_repo, *commit_args))
410 log('Committed %s to svn.' % rev)
411 else:
412 log("Would have committed %s to svn, if this weren't a dry run." % rev)
415 def cmd_push(args):
416 '''Push changes back to SVN: this is extracted from Justin Lebar's script
417 available here: https://github.com/jlebar/llvm-repo-tools/
419 Note: a current limitation is that git does not track file rename, so they
420 will show up in SVN as delete+add.
422 # Get the git root
423 git_root = git('rev-parse', '--show-toplevel')
424 if not os.path.isdir(git_root):
425 die("Can't find git root dir")
427 # Push from the root of the git repo
428 os.chdir(git_root)
430 # Get the remote URL, and check if it's one of the standalone repos.
431 git_remote_url = git('ls-remote', '--get-url', 'origin')
432 git_remote_url = git_remote_url.rstrip('.git').rstrip('/')
433 git_remote_repo_name = git_remote_url.rsplit('/', 1)[-1]
434 split_repo_path = SPLIT_REPO_NAMES.get(git_remote_repo_name)
435 if split_repo_path:
436 git_to_svn_mapping = {'': split_repo_path}
437 else:
438 # Default to the monorepo mapping
439 git_to_svn_mapping = LLVM_MONOREPO_SVN_MAPPING
441 # We need a staging area for SVN, let's hide it in the .git directory.
442 dot_git_dir = git('rev-parse', '--git-common-dir')
443 # Not all versions of git support --git-common-dir and just print the
444 # unknown command back. If this happens, fall back to --git-dir
445 if dot_git_dir == '--git-common-dir':
446 dot_git_dir = git('rev-parse', '--git-dir')
448 svn_root = os.path.join(dot_git_dir, 'llvm-upstream-svn')
449 svn_init(svn_root)
451 rev_range = args.rev_range
452 dry_run = args.dry_run
453 revs = get_revs_to_push(rev_range)
455 if not args.force and not revs:
456 die('Nothing to push: No revs in range %s.' % rev_range)
458 log('%sPushing %d %s commit%s:\n%s' %
459 ('[DryRun] ' if dry_run else '', len(revs),
460 'split-repo (%s)' % split_repo_path
461 if split_repo_path else 'monorepo',
462 's' if len(revs) != 1 else '',
463 '\n'.join(' ' + git('show', '--oneline', '--quiet', c)
464 for c in revs)))
466 # Ask confirmation if multiple commits are about to be pushed
467 if not args.force and len(revs) > 1:
468 if not ask_confirm("Are you sure you want to create %d commits?" % len(revs)):
469 die("Aborting")
471 for r in revs:
472 clean_svn(svn_root)
473 svn_push_one_rev(svn_root, r, git_to_svn_mapping, dry_run)
476 def lookup_llvm_svn_id(git_commit_hash):
477 # Use --format=%b to get the raw commit message, without any extra
478 # whitespace.
479 commit_msg = git('log', '-1', '--format=%b', git_commit_hash,
480 ignore_errors=True)
481 if len(commit_msg) == 0:
482 die("Can't find git commit " + git_commit_hash)
483 # If a commit has multiple "llvm-svn:" lines (e.g. if the commit is
484 # reverting/quoting a previous commit), choose the last one, which should
485 # be the authoritative one.
486 svn_match_iter = re.finditer('^llvm-svn: (\d{5,7})$', commit_msg,
487 re.MULTILINE)
488 svn_match = None
489 for m in svn_match_iter:
490 svn_match = m.group(1)
491 if svn_match:
492 return int(svn_match)
493 die("Can't find svn revision in git commit " + git_commit_hash)
496 def cmd_svn_lookup(args):
497 '''Find the SVN revision id for a given git commit hash.
499 This is identified by 'llvm-svn: NNNNNN' in the git commit message.'''
500 # Get the git root
501 git_root = git('rev-parse', '--show-toplevel')
502 if not os.path.isdir(git_root):
503 die("Can't find git root dir")
505 # Run commands from the root
506 os.chdir(git_root)
508 log('r' + str(lookup_llvm_svn_id(args.git_commit_hash)))
511 def git_hash_by_svn_rev(svn_rev):
512 '''Find the git hash for a given svn revision.
514 This check is paranoid: 'llvm-svn: NNNNNN' could exist on its own line
515 somewhere else in the commit message. Look in the full log message to see
516 if it's actually on the last line.
518 Since this check is expensive (we're searching every single commit), limit
519 to the past 10k commits (about 5 months).
521 possible_hashes = git(
522 'log', '--format=%H', '--grep', '^llvm-svn: %d$' % svn_rev,
523 'HEAD~10000...HEAD').split('\n')
524 matching_hashes = [h for h in possible_hashes
525 if lookup_llvm_svn_id(h) == svn_rev]
526 if len(matching_hashes) > 1:
527 die("svn revision r%d has ambiguous commits: %s" % (
528 svn_rev, ', '.join(matching_hashes)))
529 elif len(matching_hashes) < 1:
530 die("svn revision r%d matches no commits" % svn_rev)
531 return matching_hashes[0]
534 def cmd_revert(args):
535 '''Revert a commit by either SVN id (rNNNNNN) or git hash. This also
536 populates the git commit message with both the SVN revision and git hash of
537 the change being reverted.'''
539 # Get the git root
540 git_root = git('rev-parse', '--show-toplevel')
541 if not os.path.isdir(git_root):
542 die("Can't find git root dir")
544 # Run commands from the root
545 os.chdir(git_root)
547 # Check for a client branch first.
548 open_files = git('status', '-uno', '-s', '--porcelain')
549 if len(open_files) > 0:
550 die("Found open files. Please stash and then revert.\n" + open_files)
552 # If the revision looks like rNNNNNN (or with a callsign, e.g. rLLDNNNNNN),
553 # use that. Otherwise, look for it in the git commit.
554 svn_match = re.match('^r[A-Z]*(\d{5,7})$', args.revision)
555 if svn_match:
556 # If the revision looks like rNNNNNN, use that as the svn revision, and
557 # grep through git commits to find which one corresponds to that svn
558 # revision.
559 svn_rev = int(svn_match.group(1))
560 git_hash = git_hash_by_svn_rev(svn_rev)
561 else:
562 # Otherwise, this looks like a git hash, so we just need to grab the
563 # svn revision from the end of the commit message. Get the actual git
564 # hash in case the revision is something like "HEAD~1"
565 git_hash = git('rev-parse', '--verify', args.revision + '^{commit}')
566 svn_rev = lookup_llvm_svn_id(git_hash)
568 msg = git('log', '-1', '--format=%s', git_hash)
570 log_verbose('Ready to revert r%d (%s): "%s"' % (svn_rev, git_hash, msg))
572 revert_args = ['revert', '--no-commit', git_hash]
573 # TODO: Running --edit doesn't seem to work, with errors that stdin is not
574 # a tty.
575 commit_args = [
576 'commit', '-m', 'Revert ' + msg,
577 '-m', 'This reverts r%d (git commit %s)' % (svn_rev, git_hash)]
578 if args.dry_run:
579 log("Would have run the following commands, if this weren't a"
580 "dry run:\n"
581 '1) git %s\n2) git %s' % (
582 ' '.join(quote(arg) for arg in revert_args),
583 ' '.join(quote(arg) for arg in commit_args)))
584 return
586 git(*revert_args)
587 commit_log = git(*commit_args)
589 log('Created revert of r%d: %s' % (svn_rev, commit_log))
590 log("Run 'git llvm push -n' to inspect your changes and "
591 "run 'git llvm push' when ready")
594 if __name__ == '__main__':
595 if not program_exists('svn'):
596 die('error: git-llvm needs svn command, but svn is not installed.')
598 argv = sys.argv[1:]
599 p = argparse.ArgumentParser(
600 prog='git llvm', formatter_class=argparse.RawDescriptionHelpFormatter,
601 description=__doc__)
602 subcommands = p.add_subparsers(title='subcommands',
603 description='valid subcommands',
604 help='additional help')
605 verbosity_group = p.add_mutually_exclusive_group()
606 verbosity_group.add_argument('-q', '--quiet', action='store_true',
607 help='print less information')
608 verbosity_group.add_argument('-v', '--verbose', action='store_true',
609 help='print more information')
611 parser_push = subcommands.add_parser(
612 'push', description=cmd_push.__doc__,
613 help='push changes back to the LLVM SVN repository')
614 parser_push.add_argument(
615 '-n',
616 '--dry-run',
617 dest='dry_run',
618 action='store_true',
619 help='Do everything other than commit to svn. Leaves junk in the svn '
620 'repo, so probably will not work well if you try to commit more '
621 'than one rev.')
622 parser_push.add_argument(
623 '-f',
624 '--force',
625 action='store_true',
626 help='Do not ask for confirmation when pushing multiple commits.')
627 parser_push.add_argument(
628 'rev_range',
629 metavar='GIT_REVS',
630 type=str,
631 nargs='?',
632 help="revs to push (default: everything not in the upstream branch).")
633 parser_push.set_defaults(func=cmd_push)
635 parser_revert = subcommands.add_parser(
636 'revert', description=cmd_revert.__doc__,
637 help='Revert a commit locally.')
638 parser_revert.add_argument(
639 'revision',
640 help='Revision to revert. Can either be an SVN revision number '
641 "(rNNNNNN) or a git commit hash (anything that doesn't look "
642 'like an SVN revision number).')
643 parser_revert.add_argument(
644 '-n',
645 '--dry-run',
646 dest='dry_run',
647 action='store_true',
648 help='Do everything other than perform a revert. Prints the git '
649 'revert command it would have run.')
650 parser_revert.set_defaults(func=cmd_revert)
652 parser_svn_lookup = subcommands.add_parser(
653 'svn-lookup', description=cmd_svn_lookup.__doc__,
654 help='Find the llvm-svn revision for a given commit.')
655 parser_svn_lookup.add_argument(
656 'git_commit_hash',
657 help='git_commit_hash for which we will look up the svn revision id.')
658 parser_svn_lookup.set_defaults(func=cmd_svn_lookup)
660 args = p.parse_args(argv)
661 VERBOSE = args.verbose
662 QUIET = args.quiet
664 # Python3 workaround, for when not arguments are provided.
665 # See https://bugs.python.org/issue16308
666 try:
667 func = args.func
668 except AttributeError:
669 # No arguments or subcommands were given.
670 parser.print_help()
671 parser.exit()
673 # Dispatch to the right subcommand
674 args.func(args)