[ARM] Basic And/Or/Xor handling for MVE predicates
[llvm-complete.git] / utils / git-svn / git-llvm
blob763bcbec9bb7ef2eb6a95ea90b8c42b2f952b98b
1 #!/usr/bin/env python
3 # ======- git-llvm - LLVM Git Help Integration ---------*- python -*--========#
5 # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
6 # See https://llvm.org/LICENSE.txt for license information.
7 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
9 # ==------------------------------------------------------------------------==#
11 """
12 git-llvm integration
13 ====================
15 This file provides integration for git.
16 """
18 from __future__ import print_function
19 import argparse
20 import collections
21 import os
22 import re
23 import shutil
24 import subprocess
25 import sys
26 import time
27 assert sys.version_info >= (2, 7)
29 try:
30 dict.iteritems
31 except AttributeError:
32 # Python 3
33 def iteritems(d):
34 return iter(d.items())
35 else:
36 # Python 2
37 def iteritems(d):
38 return d.iteritems()
40 try:
41 # Python 3
42 from shlex import quote
43 except ImportError:
44 # Python 2
45 from pipes import quote
47 # It's *almost* a straightforward mapping from the monorepo to svn...
48 LLVM_MONOREPO_SVN_MAPPING = {
49 d: (d + '/trunk')
50 for d in [
51 'clang-tools-extra',
52 'compiler-rt',
53 'debuginfo-tests',
54 'dragonegg',
55 'klee',
56 'libclc',
57 'libcxx',
58 'libcxxabi',
59 'libunwind',
60 'lld',
61 'lldb',
62 'llgo',
63 'llvm',
64 'openmp',
65 'parallel-libs',
66 'polly',
67 'pstl',
70 LLVM_MONOREPO_SVN_MAPPING.update({'clang': 'cfe/trunk'})
71 LLVM_MONOREPO_SVN_MAPPING.update({'': 'monorepo-root/trunk'})
73 SPLIT_REPO_NAMES = {'llvm-' + d: d + '/trunk'
74 for d in ['www', 'zorg', 'test-suite', 'lnt']}
76 VERBOSE = False
77 QUIET = False
78 dev_null_fd = None
81 def eprint(*args, **kwargs):
82 print(*args, file=sys.stderr, **kwargs)
85 def log(*args, **kwargs):
86 if QUIET:
87 return
88 print(*args, **kwargs)
91 def log_verbose(*args, **kwargs):
92 if not VERBOSE:
93 return
94 print(*args, **kwargs)
97 def die(msg):
98 eprint(msg)
99 sys.exit(1)
102 def split_first_path_component(d):
103 # Assuming we have a git path, it'll use slashes even on windows...I hope.
104 if '/' in d:
105 return d.split('/', 1)
106 else:
107 return (d, None)
110 def get_dev_null():
111 """Lazily create a /dev/null fd for use in shell()"""
112 global dev_null_fd
113 if dev_null_fd is None:
114 dev_null_fd = open(os.devnull, 'w')
115 return dev_null_fd
118 def shell(cmd, strip=True, cwd=None, stdin=None, die_on_failure=True,
119 ignore_errors=False, text=True):
120 # Escape args when logging for easy repro.
121 quoted_cmd = [quote(arg) for arg in cmd]
122 log_verbose('Running in %s: %s' % (cwd, ' '.join(quoted_cmd)))
124 err_pipe = subprocess.PIPE
125 if ignore_errors:
126 # Silence errors if requested.
127 err_pipe = get_dev_null()
129 start = time.time()
130 p = subprocess.Popen(cmd, cwd=cwd, stdout=subprocess.PIPE, stderr=err_pipe,
131 stdin=subprocess.PIPE,
132 universal_newlines=text)
133 stdout, stderr = p.communicate(input=stdin)
134 elapsed = time.time() - start
136 log_verbose('Command took %0.1fs' % elapsed)
138 if p.returncode == 0 or ignore_errors:
139 if stderr and not ignore_errors:
140 eprint('`%s` printed to stderr:' % ' '.join(quoted_cmd))
141 eprint(stderr.rstrip())
142 if strip:
143 if text:
144 stdout = stdout.rstrip('\r\n')
145 else:
146 stdout = stdout.rstrip(b'\r\n')
147 if VERBOSE:
148 for l in stdout.splitlines():
149 log_verbose("STDOUT: %s" % l)
150 return stdout
151 err_msg = '`%s` returned %s' % (' '.join(quoted_cmd), p.returncode)
152 eprint(err_msg)
153 if stderr:
154 eprint(stderr.rstrip())
155 if die_on_failure:
156 sys.exit(2)
157 raise RuntimeError(err_msg)
160 def git(*cmd, **kwargs):
161 return shell(['git'] + list(cmd), **kwargs)
164 def svn(cwd, *cmd, **kwargs):
165 return shell(['svn'] + list(cmd), cwd=cwd, **kwargs)
168 def program_exists(cmd):
169 if sys.platform == 'win32' and not cmd.endswith('.exe'):
170 cmd += '.exe'
171 for path in os.environ["PATH"].split(os.pathsep):
172 if os.access(os.path.join(path, cmd), os.X_OK):
173 return True
174 return False
177 def get_default_rev_range():
178 # Get the branch tracked by the current branch, as set by
179 # git branch --set-upstream-to See http://serverfault.com/a/352236/38694.
180 cur_branch = git('rev-parse', '--symbolic-full-name', 'HEAD')
181 upstream_branch = git('for-each-ref', '--format=%(upstream:short)',
182 cur_branch)
183 if not upstream_branch:
184 upstream_branch = 'origin/master'
186 # Get the newest common ancestor between HEAD and our upstream branch.
187 upstream_rev = git('merge-base', 'HEAD', upstream_branch)
188 return '%s..' % upstream_rev
191 def get_revs_to_push(rev_range):
192 if not rev_range:
193 rev_range = get_default_rev_range()
194 # Use git show rather than some plumbing command to figure out which revs
195 # are in rev_range because it handles single revs (HEAD^) and ranges
196 # (foo..bar) like we want.
197 revs = git('show', '--reverse', '--quiet',
198 '--pretty=%h', rev_range).splitlines()
199 if not revs:
200 die('Nothing to push: No revs in range %s.' % rev_range)
201 return revs
204 def clean_svn(svn_repo):
205 svn(svn_repo, 'revert', '-R', '.')
207 # Unfortunately it appears there's no svn equivalent for git clean, so we
208 # have to do it ourselves.
209 for line in svn(svn_repo, 'status', '--no-ignore').split('\n'):
210 if not line.startswith('?'):
211 continue
212 filename = line[1:].strip()
213 filepath = os.path.abspath(os.path.join(svn_repo, filename))
214 abs_svn_repo = os.path.abspath(svn_repo)
215 # Safety check that the directory we are about to delete is
216 # actually within our svn staging dir.
217 if not filepath.startswith(abs_svn_repo):
218 die("Path to clean (%s) is not in svn staging dir (%s)"
219 % (filepath, abs_svn_repo))
221 if os.path.isdir(filepath):
222 shutil.rmtree(filepath)
223 else:
224 os.remove(filepath)
227 def svn_init(svn_root):
228 if not os.path.exists(svn_root):
229 log('Creating svn staging directory: (%s)' % (svn_root))
230 os.makedirs(svn_root)
231 svn(svn_root, 'checkout', '--depth=empty',
232 'https://llvm.org/svn/llvm-project/', '.')
233 log("svn staging area ready in '%s'" % svn_root)
234 if not os.path.isdir(svn_root):
235 die("Can't initialize svn staging dir (%s)" % svn_root)
238 def fix_eol_style_native(rev, svn_sr_path, files):
239 """Fix line endings before applying patches with Unix endings
241 SVN on Windows will check out files with CRLF for files with the
242 svn:eol-style property set to "native". This breaks `git apply`, which
243 typically works with Unix-line ending patches. Work around the problem here
244 by doing a dos2unix up front for files with svn:eol-style set to "native".
245 SVN will not commit a mass line ending re-doing because it detects the line
246 ending format for files with this property.
248 # Skip files that don't exist in SVN yet.
249 files = [f for f in files if os.path.exists(os.path.join(svn_sr_path, f))]
250 # Use ignore_errors because 'svn propget' prints errors if the file doesn't
251 # have the named property. There doesn't seem to be a way to suppress that.
252 eol_props = svn(svn_sr_path, 'propget', 'svn:eol-style', *files,
253 ignore_errors=True)
254 crlf_files = []
255 if len(files) == 1:
256 # No need to split propget output on ' - ' when we have one file.
257 if eol_props.strip() in ['native', 'CRLF']:
258 crlf_files = files
259 else:
260 for eol_prop in eol_props.split('\n'):
261 # Remove spare CR.
262 eol_prop = eol_prop.strip('\r')
263 if not eol_prop:
264 continue
265 prop_parts = eol_prop.rsplit(' - ', 1)
266 if len(prop_parts) != 2:
267 eprint("unable to parse svn propget line:")
268 eprint(eol_prop)
269 continue
270 (f, eol_style) = prop_parts
271 if eol_style == 'native':
272 crlf_files.append(f)
273 if crlf_files:
274 # Reformat all files with native SVN line endings to Unix format. SVN
275 # knows files with native line endings are text files. It will commit
276 # just the diff, and not a mass line ending change.
277 shell(['dos2unix'] + crlf_files, ignore_errors=True, cwd=svn_sr_path)
280 def split_subrepo(f, git_to_svn_mapping):
281 # Given a path, splits it into (subproject, rest-of-path). If the path is
282 # not in a subproject, returns ('', full-path).
284 subproject, remainder = split_first_path_component(f)
286 if subproject in git_to_svn_mapping:
287 return subproject, remainder
288 else:
289 return '', f
292 def get_all_parent_dirs(name):
293 parts = []
294 head, tail = os.path.split(name)
295 while head:
296 parts.append(head)
297 head, tail = os.path.split(head)
298 return parts
301 def svn_push_one_rev(svn_repo, rev, git_to_svn_mapping, dry_run):
302 files = git('diff-tree', '--no-commit-id', '--name-only', '-r',
303 rev).split('\n')
304 if not files:
305 raise RuntimeError('Empty diff for rev %s?' % rev)
307 # Split files by subrepo
308 subrepo_files = collections.defaultdict(list)
309 for f in files:
310 subrepo, remainder = split_subrepo(f, git_to_svn_mapping)
311 subrepo_files[subrepo].append(remainder)
313 status = svn(svn_repo, 'status', '--no-ignore')
314 if status:
315 die("Can't push git rev %s because status in svn staging dir (%s) is "
316 "not empty:\n%s" % (rev, svn_repo, status))
318 svn_dirs_to_update = set()
319 for sr, files in iteritems(subrepo_files):
320 svn_sr_path = git_to_svn_mapping[sr]
321 for f in files:
322 svn_dirs_to_update.add(
323 os.path.dirname(os.path.join(svn_sr_path, f)))
325 # We also need to svn update any parent directories which are not yet
326 # present
327 parent_dirs = set()
328 for dir in svn_dirs_to_update:
329 parent_dirs.update(get_all_parent_dirs(dir))
330 parent_dirs = set(dir for dir in parent_dirs
331 if not os.path.exists(os.path.join(svn_repo, dir)))
332 svn_dirs_to_update.update(parent_dirs)
334 # Sort by length to ensure that the parent directories are passed to svn
335 # before child directories.
336 sorted_dirs_to_update = sorted(svn_dirs_to_update, key=len)
338 # SVN update only in the affected directories.
339 svn(svn_repo, 'update', '--depth=files', *sorted_dirs_to_update)
341 for sr, files in iteritems(subrepo_files):
342 svn_sr_path = os.path.join(svn_repo, git_to_svn_mapping[sr])
343 if os.name == 'nt':
344 fix_eol_style_native(rev, svn_sr_path, files)
345 # We use text=False (and pass '--binary') so that we can get an exact
346 # diff that can be passed as-is to 'git apply' without any line ending,
347 # encoding, or other mangling.
348 diff = git('show', '--binary', rev, '--',
349 *(os.path.join(sr, f) for f in files),
350 strip=False, text=False)
351 # git is the only thing that can handle its own patches...
352 if sr == '':
353 prefix_strip = '-p1'
354 else:
355 prefix_strip = '-p2'
356 try:
357 shell(['git', 'apply', prefix_strip, '-'], cwd=svn_sr_path,
358 stdin=diff, die_on_failure=False, text=False)
359 except RuntimeError as e:
360 eprint("Patch doesn't apply: maybe you should try `git pull -r` "
361 "first?")
362 sys.exit(2)
364 status_lines = svn(svn_repo, 'status', '--no-ignore').split('\n')
366 for l in (l for l in status_lines if (l.startswith('?') or
367 l.startswith('I'))):
368 svn(svn_repo, 'add', '--no-ignore', l[1:].strip())
369 for l in (l for l in status_lines if l.startswith('!')):
370 svn(svn_repo, 'remove', l[1:].strip())
372 # Now we're ready to commit.
373 commit_msg = git('show', '--pretty=%B', '--quiet', rev)
374 if not dry_run:
375 commit_args = ['commit', '-m', commit_msg]
376 if '--force-interactive' in svn(svn_repo, 'commit', '--help'):
377 commit_args.append('--force-interactive')
378 log(svn(svn_repo, *commit_args))
379 log('Committed %s to svn.' % rev)
380 else:
381 log("Would have committed %s to svn, if this weren't a dry run." % rev)
384 def cmd_push(args):
385 '''Push changes back to SVN: this is extracted from Justin Lebar's script
386 available here: https://github.com/jlebar/llvm-repo-tools/
388 Note: a current limitation is that git does not track file rename, so they
389 will show up in SVN as delete+add.
391 # Get the git root
392 git_root = git('rev-parse', '--show-toplevel')
393 if not os.path.isdir(git_root):
394 die("Can't find git root dir")
396 # Push from the root of the git repo
397 os.chdir(git_root)
399 # Get the remote URL, and check if it's one of the standalone repos.
400 git_remote_url = git('remote', 'get-url', 'origin')
401 git_remote_url = git_remote_url.rstrip('.git').rstrip('/')
402 git_remote_repo_name = git_remote_url.rsplit('/', 1)[-1]
403 split_repo_path = SPLIT_REPO_NAMES.get(git_remote_repo_name)
404 if split_repo_path:
405 git_to_svn_mapping = {'': split_repo_path}
406 else:
407 # Default to the monorepo mapping
408 git_to_svn_mapping = LLVM_MONOREPO_SVN_MAPPING
410 # We need a staging area for SVN, let's hide it in the .git directory.
411 dot_git_dir = git('rev-parse', '--git-common-dir')
412 # Not all versions of git support --git-common-dir and just print the
413 # unknown command back. If this happens, fall back to --git-dir
414 if dot_git_dir == '--git-common-dir':
415 dot_git_dir = git('rev-parse', '--git-dir')
417 svn_root = os.path.join(dot_git_dir, 'llvm-upstream-svn')
418 svn_init(svn_root)
420 rev_range = args.rev_range
421 dry_run = args.dry_run
422 revs = get_revs_to_push(rev_range)
423 log('%sPushing %d %s commit%s:\n%s' %
424 ('[DryRun] ' if dry_run else '', len(revs),
425 'split-repo (%s)' % split_repo_path
426 if split_repo_path else 'monorepo',
427 's' if len(revs) != 1 else '',
428 '\n'.join(' ' + git('show', '--oneline', '--quiet', c)
429 for c in revs)))
430 for r in revs:
431 clean_svn(svn_root)
432 svn_push_one_rev(svn_root, r, git_to_svn_mapping, dry_run)
435 def lookup_llvm_svn_id(git_commit_hash):
436 # Use --format=%b to get the raw commit message, without any extra
437 # whitespace.
438 commit_msg = git('log', '-1', '--format=%b', git_commit_hash,
439 ignore_errors=True)
440 if len(commit_msg) == 0:
441 die("Can't find git commit " + git_commit_hash)
442 # If a commit has multiple "llvm-svn:" lines (e.g. if the commit is
443 # reverting/quoting a previous commit), choose the last one, which should
444 # be the authoritative one.
445 svn_match_iter = re.finditer('^llvm-svn: (\d{5,7})$', commit_msg,
446 re.MULTILINE)
447 svn_match = None
448 for m in svn_match_iter:
449 svn_match = m.group(1)
450 if svn_match:
451 return int(svn_match)
452 die("Can't find svn revision in git commit " + git_commit_hash)
455 def cmd_svn_lookup(args):
456 '''Find the SVN revision id for a given git commit hash.
458 This is identified by 'llvm-svn: NNNNNN' in the git commit message.'''
459 # Get the git root
460 git_root = git('rev-parse', '--show-toplevel')
461 if not os.path.isdir(git_root):
462 die("Can't find git root dir")
464 # Run commands from the root
465 os.chdir(git_root)
467 log('r' + str(lookup_llvm_svn_id(args.git_commit_hash)))
470 def git_hash_by_svn_rev(svn_rev):
471 '''Find the git hash for a given svn revision.
473 This check is paranoid: 'llvm-svn: NNNNNN' could exist on its own line
474 somewhere else in the commit message. Look in the full log message to see
475 if it's actually on the last line.
477 Since this check is expensive (we're searching every single commit), limit
478 to the past 10k commits (about 5 months).
480 possible_hashes = git(
481 'log', '--format=%H', '--grep', '^llvm-svn: %d$' % svn_rev,
482 'HEAD~10000...HEAD').split('\n')
483 matching_hashes = [h for h in possible_hashes
484 if lookup_llvm_svn_id(h) == svn_rev]
485 if len(matching_hashes) > 1:
486 die("svn revision r%d has ambiguous commits: %s" % (
487 svn_rev, ', '.join(matching_hashes)))
488 elif len(matching_hashes) < 1:
489 die("svn revision r%d matches no commits" % svn_rev)
490 return matching_hashes[0]
493 def cmd_revert(args):
494 '''Revert a commit by either SVN id (rNNNNNN) or git hash. This also
495 populates the git commit message with both the SVN revision and git hash of
496 the change being reverted.'''
498 # Get the git root
499 git_root = git('rev-parse', '--show-toplevel')
500 if not os.path.isdir(git_root):
501 die("Can't find git root dir")
503 # Run commands from the root
504 os.chdir(git_root)
506 # Check for a client branch first.
507 open_files = git('status', '-uno', '-s', '--porcelain')
508 if len(open_files) > 0:
509 die("Found open files. Please stash and then revert.\n" + open_files)
511 # If the revision looks like rNNNNNN, use that. Otherwise, look for it in
512 # the git commit.
513 svn_match = re.match('^r(\d{5,7})$', args.revision)
514 if svn_match:
515 # If the revision looks like rNNNNNN, use that as the svn revision, and
516 # grep through git commits to find which one corresponds to that svn
517 # revision.
518 svn_rev = int(svn_match.group(1))
519 git_hash = git_hash_by_svn_rev(svn_rev)
520 else:
521 # Otherwise, this looks like a git hash, so we just need to grab the
522 # svn revision from the end of the commit message. Get the actual git
523 # hash in case the revision is something like "HEAD~1"
524 git_hash = git('rev-parse', '--verify', args.revision + '^{commit}')
525 svn_rev = lookup_llvm_svn_id(git_hash)
527 msg = git('log', '-1', '--format=%s', git_hash)
529 log_verbose('Ready to revert r%d (%s): "%s"' % (svn_rev, git_hash, msg))
531 revert_args = ['revert', '--no-commit', git_hash]
532 # TODO: Running --edit doesn't seem to work, with errors that stdin is not
533 # a tty.
534 commit_args = [
535 'commit', '-m', 'Revert ' + msg,
536 '-m', 'This reverts r%d (git commit %s)' % (svn_rev, git_hash)]
537 if args.dry_run:
538 log("Would have run the following commands, if this weren't a"
539 "dry run:\n"
540 '1) git %s\n2) git %s' % (
541 ' '.join(quote(arg) for arg in revert_args),
542 ' '.join(quote(arg) for arg in commit_args)))
543 return
545 git(*revert_args)
546 commit_log = git(*commit_args)
548 log('Created revert of r%d: %s' % (svn_rev, commit_log))
549 log("Run 'git llvm push -n' to inspect your changes and "
550 "run 'git llvm push' when ready")
553 if __name__ == '__main__':
554 if not program_exists('svn'):
555 die('error: git-llvm needs svn command, but svn is not installed.')
557 argv = sys.argv[1:]
558 p = argparse.ArgumentParser(
559 prog='git llvm', formatter_class=argparse.RawDescriptionHelpFormatter,
560 description=__doc__)
561 subcommands = p.add_subparsers(title='subcommands',
562 description='valid subcommands',
563 help='additional help')
564 verbosity_group = p.add_mutually_exclusive_group()
565 verbosity_group.add_argument('-q', '--quiet', action='store_true',
566 help='print less information')
567 verbosity_group.add_argument('-v', '--verbose', action='store_true',
568 help='print more information')
570 parser_push = subcommands.add_parser(
571 'push', description=cmd_push.__doc__,
572 help='push changes back to the LLVM SVN repository')
573 parser_push.add_argument(
574 '-n',
575 '--dry-run',
576 dest='dry_run',
577 action='store_true',
578 help='Do everything other than commit to svn. Leaves junk in the svn '
579 'repo, so probably will not work well if you try to commit more '
580 'than one rev.')
581 parser_push.add_argument(
582 'rev_range',
583 metavar='GIT_REVS',
584 type=str,
585 nargs='?',
586 help="revs to push (default: everything not in the branch's "
587 'upstream, or not in origin/master if the branch lacks '
588 'an explicit upstream)')
589 parser_push.set_defaults(func=cmd_push)
591 parser_revert = subcommands.add_parser(
592 'revert', description=cmd_revert.__doc__,
593 help='Revert a commit locally.')
594 parser_revert.add_argument(
595 'revision',
596 help='Revision to revert. Can either be an SVN revision number '
597 "(rNNNNNN) or a git commit hash (anything that doesn't look "
598 'like an SVN revision number).')
599 parser_revert.add_argument(
600 '-n',
601 '--dry-run',
602 dest='dry_run',
603 action='store_true',
604 help='Do everything other than perform a revert. Prints the git '
605 'revert command it would have run.')
606 parser_revert.set_defaults(func=cmd_revert)
608 parser_svn_lookup = subcommands.add_parser(
609 'svn-lookup', description=cmd_svn_lookup.__doc__,
610 help='Find the llvm-svn revision for a given commit.')
611 parser_svn_lookup.add_argument(
612 'git_commit_hash',
613 help='git_commit_hash for which we will look up the svn revision id.')
614 parser_svn_lookup.set_defaults(func=cmd_svn_lookup)
616 args = p.parse_args(argv)
617 VERBOSE = args.verbose
618 QUIET = args.quiet
620 # Dispatch to the right subcommand
621 args.func(args)