utils/gn/build/sync_source_lists_from_cmake.py

   1 #!/usr/bin/env python
   2
   3 """Helps to keep BUILD.gn files in sync with the corresponding CMakeLists.txt.
   4
   5 For each BUILD.gn file in the tree, checks if the list of cpp files in
   6 it is identical to the list of cpp files in the corresponding CMakeLists.txt
   7 file, and prints the difference if not.
   8
   9 Also checks that each CMakeLists.txt file below unittests/ folders that define
  10 binaries have corresponding BUILD.gn files.
  11
  12 If --write is passed, tries to write modified .gn files and adds one git
  13 commit for each cmake commit this merges. If an error is reported, the state
  14 of HEAD is unspecified; run `git reset --hard origin/master` if this happens.
  15 """
  16
  17 from __future__ import print_function
  18
  19 from collections import defaultdict
  20 import os
  21 import re
  22 import subprocess
  23 import sys
  24
  25
  26 def patch_gn_file(gn_file, add, remove):
  27     with open(gn_file) as f:
  28       gn_contents = f.read()
  29
  30     srcs_tok = 'sources = ['
  31     tokloc = gn_contents.find(srcs_tok)
  32
  33     if tokloc == -1: raise ValueError(gn_file + ': Failed to find source list')
  34     if gn_contents.find(srcs_tok, tokloc + 1) != -1:
  35         raise ValueError(gn_file + ': Multiple source lists')
  36     if gn_file.find('# NOSORT', 0, tokloc) != -1:
  37         raise ValueError(gn_file + ': Found # NOSORT, needs manual merge')
  38
  39     tokloc += len(srcs_tok)
  40     for a in add:
  41         gn_contents = (gn_contents[:tokloc] + ('"%s",' % a) +
  42                        gn_contents[tokloc:])
  43     for r in remove:
  44         gn_contents = gn_contents.replace('"%s",' % r, '')
  45     with open(gn_file, 'w') as f:
  46       f.write(gn_contents)
  47
  48     # Run `gn format`.
  49     gn = os.path.join(os.path.dirname(__file__), '..', 'gn.py')
  50     subprocess.check_call([sys.executable, gn, 'format', '-q', gn_file])
  51
  52
  53 def sync_source_lists(write):
  54     # Use shell=True on Windows in case git is a bat file.
  55     def git(args): subprocess.check_call(['git'] + args, shell=os.name == 'nt')
  56     def git_out(args):
  57         return subprocess.check_output(['git'] + args, shell=os.name == 'nt')
  58     gn_files = git_out(['ls-files', '*BUILD.gn']).splitlines()
  59
  60     # Matches e.g. |   "foo.cpp",|, captures |foo| in group 1.
  61     gn_cpp_re = re.compile(r'^\s*"([^"]+\.(?:cpp|c|h|S))",$', re.MULTILINE)
  62     # Matches e.g. |   foo.cpp|, captures |foo| in group 1.
  63     cmake_cpp_re = re.compile(r'^\s*([A-Za-z_0-9./-]+\.(?:cpp|c|h|S))$',
  64                               re.MULTILINE)
  65
  66     changes_by_rev = defaultdict(lambda: defaultdict(lambda: defaultdict(list)))
  67
  68     def find_gitrev(touched_line, in_file):
  69         return git_out(
  70             ['log', '--format=%h', '-1', '-S' + touched_line, in_file]).rstrip()
  71     def svnrev_from_gitrev(gitrev):
  72         git_llvm = os.path.join(
  73             os.path.dirname(__file__), '..', '..', 'git-svn', 'git-llvm')
  74         return int(subprocess.check_output(
  75             [sys.executable, git_llvm, 'svn-lookup', gitrev],
  76             ).rstrip().lstrip('r'))
  77
  78     # Collect changes to gn files, grouped by revision.
  79     for gn_file in gn_files:
  80         # The CMakeLists.txt for llvm/utils/gn/secondary/foo/BUILD.gn is
  81         # at foo/CMakeLists.txt.
  82         strip_prefix = 'llvm/utils/gn/secondary/'
  83         if not gn_file.startswith(strip_prefix):
  84             continue
  85         cmake_file = os.path.join(
  86                 os.path.dirname(gn_file[len(strip_prefix):]), 'CMakeLists.txt')
  87         if not os.path.exists(cmake_file):
  88             continue
  89
  90         def get_sources(source_re, text):
  91             return set([m.group(1) for m in source_re.finditer(text)])
  92         gn_cpp = get_sources(gn_cpp_re, open(gn_file).read())
  93         cmake_cpp = get_sources(cmake_cpp_re, open(cmake_file).read())
  94
  95         if gn_cpp == cmake_cpp:
  96             continue
  97
  98         def by_rev(files, key):
  99             for f in files:
 100                 svnrev = svnrev_from_gitrev(find_gitrev(f, cmake_file))
 101                 changes_by_rev[svnrev][gn_file][key].append(f)
 102         by_rev(sorted(cmake_cpp - gn_cpp), 'add')
 103         by_rev(sorted(gn_cpp - cmake_cpp), 'remove')
 104
 105     # Output necessary changes grouped by revision.
 106     for svnrev in sorted(changes_by_rev):
 107         print('gn build: Merge r{0} -- https://reviews.llvm.org/rL{0}'
 108             .format(svnrev))
 109         for gn_file, data in sorted(changes_by_rev[svnrev].items()):
 110             add = data.get('add', [])
 111             remove = data.get('remove', [])
 112             if write:
 113                 patch_gn_file(gn_file, add, remove)
 114                 git(['add', gn_file])
 115             else:
 116                 print('  ' + gn_file)
 117                 if add:
 118                     print('   add:\n' + '\n'.join('    "%s",' % a for a in add))
 119                 if remove:
 120                     print('   remove:\n    ' + '\n    '.join(remove))
 121                 print()
 122         if write:
 123             git(['commit', '-m', 'gn build: Merge r%d' % svnrev])
 124         else:
 125             print()
 126
 127     return bool(changes_by_rev) and not write
 128
 129
 130 def sync_unittests():
 131     # Matches e.g. |add_llvm_unittest_with_input_files|.
 132     unittest_re = re.compile(r'^add_\S+_unittest', re.MULTILINE)
 133
 134     checked = [ 'clang', 'clang-tools-extra', 'lld', 'llvm' ]
 135     changed = False
 136     for c in checked:
 137         for root, _, _ in os.walk(os.path.join(c, 'unittests')):
 138             cmake_file = os.path.join(root, 'CMakeLists.txt')
 139             if not os.path.exists(cmake_file):
 140                 continue
 141             if not unittest_re.search(open(cmake_file).read()):
 142                 continue  # Skip CMake files that just add subdirectories.
 143             gn_file = os.path.join('llvm/utils/gn/secondary', root, 'BUILD.gn')
 144             if not os.path.exists(gn_file):
 145                 changed = True
 146                 print('missing GN file %s for unittest CMake file %s' %
 147                       (gn_file, cmake_file))
 148     return changed
 149
 150
 151 def main():
 152     src = sync_source_lists(len(sys.argv) > 1 and sys.argv[1] == '--write')
 153     tests = sync_unittests()
 154     if src or tests:
 155         sys.exit(1)
 156
 157
 158 if __name__ == '__main__':
 159     main()