tools/gn/bin/gyp_flag_compare.py

   1 #!/usr/bin/env python
   2
   3 # Copyright 2014 The Chromium Authors. All rights reserved.
   4 # Use of this source code is governed by a BSD-style license that can be
   5 # found in the LICENSE file.
   6
   7 """Given the output of -t commands from a ninja build for a gyp and GN generated
   8 build, report on differences between the command lines."""
   9
  10
  11 import os
  12 import shlex
  13 import subprocess
  14 import sys
  15
  16
  17 # Must be in src/.
  18 os.chdir(os.path.join(os.path.dirname(__file__), '..', '..', '..'))
  19
  20
  21 g_total_differences = 0
  22
  23
  24 def FindAndRemoveArgWithValue(command_line, argname):
  25   """Given a command line as a list, remove and return the value of an option
  26   that takes a value as a separate entry.
  27
  28   Modifies |command_line| in place.
  29   """
  30   if argname not in command_line:
  31     return ''
  32   location = command_line.index(argname)
  33   value = command_line[location + 1]
  34   command_line[location:location + 2] = []
  35   return value
  36
  37
  38 def MergeSpacedArgs(command_line, argname):
  39   """Combine all arguments |argname| with their values, separated by a space."""
  40   i = 0
  41   result = []
  42   while i < len(command_line):
  43     arg = command_line[i]
  44     if arg == argname:
  45       result.append(arg + ' ' + command_line[i + 1])
  46       i += 1
  47     else:
  48       result.append(arg)
  49     i += 1
  50   return result
  51
  52
  53 def NormalizeSymbolArguments(command_line):
  54   """Normalize -g arguments.
  55
  56   If there's no -g args, it's equivalent to -g0. -g2 is equivalent to -g.
  57   Modifies |command_line| in place.
  58   """
  59   # Strip -g0 if there's no symbols.
  60   have_some_symbols = False
  61   for x in command_line:
  62     if x.startswith('-g') and x != '-g0':
  63       have_some_symbols = True
  64   if not have_some_symbols and '-g0' in command_line:
  65     command_line.remove('-g0')
  66
  67   # Rename -g2 to -g.
  68   if '-g2' in command_line:
  69     command_line[command_line.index('-g2')] = '-g'
  70
  71
  72 def GetFlags(lines, build_dir):
  73   """Turn a list of command lines into a semi-structured dict."""
  74   is_win = sys.platform == 'win32'
  75   flags_by_output = {}
  76   for line in lines:
  77     command_line = shlex.split(line.strip(), posix=not is_win)[1:]
  78
  79     output_name = FindAndRemoveArgWithValue(command_line, '-o')
  80     dep_name = FindAndRemoveArgWithValue(command_line, '-MF')
  81
  82     NormalizeSymbolArguments(command_line)
  83
  84     command_line = MergeSpacedArgs(command_line, '-Xclang')
  85
  86     cc_file = [x for x in command_line if x.endswith('.cc') or
  87                                           x.endswith('.c') or
  88                                           x.endswith('.cpp')]
  89     if len(cc_file) != 1:
  90       print 'Skipping %s' % command_line
  91       continue
  92     assert len(cc_file) == 1
  93
  94     if is_win:
  95       rsp_file = [x for x in command_line if x.endswith('.rsp')]
  96       assert len(rsp_file) <= 1
  97       if rsp_file:
  98         rsp_file = os.path.join(build_dir, rsp_file[0][1:])
  99         with open(rsp_file, "r") as open_rsp_file:
 100           command_line = shlex.split(open_rsp_file, posix=False)
 101
 102     defines = [x for x in command_line if x.startswith('-D')]
 103     include_dirs = [x for x in command_line if x.startswith('-I')]
 104     dash_f = [x for x in command_line if x.startswith('-f')]
 105     warnings = \
 106         [x for x in command_line if x.startswith('/wd' if is_win else '-W')]
 107     others = [x for x in command_line if x not in defines and \
 108                                          x not in include_dirs and \
 109                                          x not in dash_f and \
 110                                          x not in warnings and \
 111                                          x not in cc_file]
 112
 113     for index, value in enumerate(include_dirs):
 114       if value == '-Igen':
 115         continue
 116       path = value[2:]
 117       if not os.path.isabs(path):
 118         path = os.path.join(build_dir, path)
 119       include_dirs[index] = '-I' + os.path.normpath(path)
 120
 121     # GYP supports paths above the source root like <(DEPTH)/../foo while such
 122     # paths are unsupported by gn. But gn allows to use system-absolute paths
 123     # instead (paths that start with single '/'). Normalize all paths.
 124     cc_file = [os.path.normpath(os.path.join(build_dir, cc_file[0]))]
 125
 126     # Filter for libFindBadConstructs.so having a relative path in one and
 127     # absolute path in the other.
 128     others_filtered = []
 129     for x in others:
 130       if x.startswith('-Xclang ') and x.endswith('libFindBadConstructs.so'):
 131         others_filtered.append(
 132             '-Xclang ' +
 133             os.path.join(os.getcwd(),
 134                          os.path.normpath(
 135                              os.path.join('out/gn_flags', x.split(' ', 1)[1]))))
 136       elif x.startswith('-B'):
 137         others_filtered.append(
 138             '-B' +
 139             os.path.join(os.getcwd(),
 140                          os.path.normpath(os.path.join('out/gn_flags', x[2:]))))
 141       else:
 142         others_filtered.append(x)
 143     others = others_filtered
 144
 145     flags_by_output[cc_file[0]] = {
 146       'output': output_name,
 147       'depname': dep_name,
 148       'defines': sorted(defines),
 149       'include_dirs': sorted(include_dirs),  # TODO(scottmg): This is wrong.
 150       'dash_f': sorted(dash_f),
 151       'warnings': sorted(warnings),
 152       'other': sorted(others),
 153     }
 154   return flags_by_output
 155
 156
 157 def CompareLists(gyp, gn, name, dont_care_gyp=None, dont_care_gn=None):
 158   """Return a report of any differences between gyp and gn lists, ignoring
 159   anything in |dont_care_{gyp|gn}| respectively."""
 160   global g_total_differences
 161   if not dont_care_gyp:
 162     dont_care_gyp = []
 163   if not dont_care_gn:
 164     dont_care_gn = []
 165   output = ''
 166   if gyp[name] != gn[name]:
 167     gyp_set = set(gyp[name])
 168     gn_set = set(gn[name])
 169     missing_in_gyp = gyp_set - gn_set
 170     missing_in_gn = gn_set - gyp_set
 171     missing_in_gyp -= set(dont_care_gyp)
 172     missing_in_gn -= set(dont_care_gn)
 173     if missing_in_gyp or missing_in_gn:
 174       output += '  %s differ:\n' % name
 175     if missing_in_gyp:
 176       output += '    In gyp, but not in GN:\n      %s' % '\n      '.join(
 177           sorted(missing_in_gyp)) + '\n'
 178       g_total_differences += len(missing_in_gyp)
 179     if missing_in_gn:
 180       output += '    In GN, but not in gyp:\n      %s' % '\n      '.join(
 181           sorted(missing_in_gn)) + '\n\n'
 182       g_total_differences += len(missing_in_gn)
 183   return output
 184
 185
 186 def Run(command_line):
 187   """Run |command_line| as a subprocess and return stdout. Raises on error."""
 188   return subprocess.check_output(command_line, shell=True)
 189
 190
 191 def main():
 192   if len(sys.argv) != 2 and len(sys.argv) != 3:
 193     print 'usage: %s gyp_target gn_target' % __file__
 194     print '   or: %s target' % __file__
 195     return 1
 196
 197   if len(sys.argv) == 2:
 198     sys.argv.append(sys.argv[1])
 199
 200   gn_out_dir = 'out/gn_flags'
 201   print >> sys.stderr, 'Regenerating in %s...' % gn_out_dir
 202   # Currently only Release, non-component.
 203   Run('gn gen %s --args="is_debug=false is_component_build=false"' % gn_out_dir)
 204   gn = Run('ninja -C %s -t commands %s' % (gn_out_dir, sys.argv[2]))
 205   if sys.platform == 'win32':
 206     # On Windows flags are stored in .rsp files which are created during build.
 207     print >> sys.stderr, 'Building in %s...' % gn_out_dir
 208     Run('ninja -C %s -d keeprsp %s' % (gn_out_dir, sys.argv[2]))
 209
 210   os.environ.pop('GYP_DEFINES', None)
 211   # Remove environment variables required by gn but conflicting with GYP.
 212   # Relevant if Windows toolchain isn't provided by depot_tools.
 213   os.environ.pop('GYP_MSVS_OVERRIDE_PATH', None)
 214   os.environ.pop('WINDOWSSDKDIR', None)
 215
 216   gyp_out_dir = 'out_gyp_flags/Release'
 217   print >> sys.stderr, 'Regenerating in %s...' % gyp_out_dir
 218   Run('python build/gyp_chromium -Goutput_dir=out_gyp_flags -Gconfig=Release')
 219   gyp = Run('ninja -C %s -t commands %s' % (gyp_out_dir, sys.argv[1]))
 220   if sys.platform == 'win32':
 221     # On Windows flags are stored in .rsp files which are created during build.
 222     print >> sys.stderr, 'Building in %s...' % gyp_out_dir
 223     Run('ninja -C %s -d keeprsp %s' % (gyp_out_dir, sys.argv[2]))
 224
 225   all_gyp_flags = GetFlags(gyp.splitlines(),
 226                            os.path.join(os.getcwd(), gyp_out_dir))
 227   all_gn_flags = GetFlags(gn.splitlines(),
 228                           os.path.join(os.getcwd(), gn_out_dir))
 229   gyp_files = set(all_gyp_flags.keys())
 230   gn_files = set(all_gn_flags.keys())
 231   different_source_list = gyp_files != gn_files
 232   if different_source_list:
 233     print 'Different set of sources files:'
 234     print '  In gyp, not in GN:\n    %s' % '\n    '.join(
 235         sorted(gyp_files - gn_files))
 236     print '  In GN, not in gyp:\n    %s' % '\n    '.join(
 237         sorted(gn_files - gyp_files))
 238     print '\nNote that flags will only be compared for files in both sets.\n'
 239   file_list = gyp_files & gn_files
 240   files_with_given_differences = {}
 241   for filename in sorted(file_list):
 242     gyp_flags = all_gyp_flags[filename]
 243     gn_flags = all_gn_flags[filename]
 244     differences = CompareLists(gyp_flags, gn_flags, 'dash_f')
 245     differences += CompareLists(gyp_flags, gn_flags, 'defines')
 246     differences += CompareLists(gyp_flags, gn_flags, 'include_dirs')
 247     differences += CompareLists(gyp_flags, gn_flags, 'warnings',
 248         # More conservative warnings in GN we consider to be OK.
 249         dont_care_gyp=[
 250           '/wd4091',  # 'keyword' : ignored on left of 'type' when no variable
 251                       # is declared.
 252           '/wd4456',  # Declaration hides previous local declaration.
 253           '/wd4457',  # Declaration hides function parameter.
 254           '/wd4458',  # Declaration hides class member.
 255           '/wd4459',  # Declaration hides global declaration.
 256           '/wd4702',  # Unreachable code.
 257           '/wd4800',  # Forcing value to bool 'true' or 'false'.
 258           '/wd4838',  # Conversion from 'type' to 'type' requires a narrowing
 259                       # conversion.
 260         ] if sys.platform == 'win32' else None,
 261         dont_care_gn=[
 262           '-Wendif-labels',
 263           '-Wextra',
 264           '-Wsign-compare',
 265         ] if not sys.platform == 'win32' else None)
 266     differences += CompareLists(gyp_flags, gn_flags, 'other')
 267     if differences:
 268       files_with_given_differences.setdefault(differences, []).append(filename)
 269
 270   for diff, files in files_with_given_differences.iteritems():
 271     print '\n'.join(sorted(files))
 272     print diff
 273
 274   print 'Total differences:', g_total_differences
 275   # TODO(scottmg): Return failure on difference once we're closer to identical.
 276   return 0
 277
 278
 279 if __name__ == '__main__':
 280   sys.exit(main())