ppapi/generate_ppapi_size_checks.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """This script should be run manually on occasion to make sure all PPAPI types
   7 have appropriate size checking.
   8 """
   9
  10 import optparse
  11 import os
  12 import subprocess
  13 import sys
  14
  15
  16 # The string that the PrintNamesAndSizes plugin uses to indicate a type is
  17 # expected to have architecture-dependent size.
  18 ARCH_DEPENDENT_STRING = "ArchDependentSize"
  19
  20
  21 COPYRIGHT_STRING_C = (
  22 """/* Copyright (c) %s The Chromium Authors. All rights reserved.
  23  * Use of this source code is governed by a BSD-style license that can be
  24  * found in the LICENSE file.
  25  *
  26  * This file has compile assertions for the sizes of types that are dependent
  27  * on the architecture for which they are compiled (i.e., 32-bit vs 64-bit).
  28  */
  29
  30 """) % datetime.date.today().year
  31
  32
  33 class SourceLocation(object):
  34   """A class representing the source location of a definiton."""
  35
  36   def __init__(self, filename="", start_line=-1, end_line=-1):
  37     self.filename = os.path.normpath(filename)
  38     self.start_line = start_line
  39     self.end_line = end_line
  40
  41
  42 class TypeInfo(object):
  43   """A class representing information about a C++ type.  It contains the
  44   following fields:
  45    - kind:  The Clang TypeClassName (Record, Enum, Typedef, Union, etc)
  46    - name:  The unmangled string name of the type.
  47    - size:  The size in bytes of the type.
  48    - arch_dependent:  True if the type may have architecture dependent size
  49                       according to PrintNamesAndSizes.  False otherwise.  Types
  50                       which are considered architecture-dependent from 32-bit
  51                       to 64-bit are pointers, longs, unsigned longs, and any
  52                       type that contains an architecture-dependent type.
  53    - source_location:  A SourceLocation describing where the type is defined.
  54    - target:  The target Clang was compiling when it found the type definition.
  55               This is used only for diagnostic output.
  56    - parsed_line:  The line which Clang output which was used to create this
  57                    TypeInfo (as the info_string parameter to __init__).  This is
  58                    used only for diagnostic output.
  59   """
  60
  61   def __init__(self, info_string, target):
  62     """Create a TypeInfo from a given info_string.  Also store the name of the
  63     target for which the TypeInfo was first created just so we can print useful
  64     error information.
  65     info_string is a comma-delimited string of the following form:
  66     kind,name,size,arch_dependent,source_file,start_line,end_line
  67     Where:
  68    - kind:  The Clang TypeClassName (Record, Enum, Typedef, Union, etc)
  69    - name:  The unmangled string name of the type.
  70    - size:  The size in bytes of the type.
  71    - arch_dependent:  'ArchDependentSize' if the type has architecture-dependent
  72                       size, NotArchDependentSize otherwise.
  73    - source_file:  The source file in which the type is defined.
  74    - first_line:  The first line of the definition (counting from 0).
  75    - last_line:  The last line of the definition (counting from 0).
  76    This should match the output of the PrintNamesAndSizes plugin.
  77    """
  78     [self.kind, self.name, self.size, arch_dependent_string, source_file,
  79         start_line, end_line] = info_string.split(',')
  80     self.target = target
  81     self.parsed_line = info_string
  82     # Note that Clang counts line numbers from 1, but we want to count from 0.
  83     self.source_location = SourceLocation(source_file,
  84                                           int(start_line)-1,
  85                                           int(end_line)-1)
  86     self.arch_dependent = (arch_dependent_string == ARCH_DEPENDENT_STRING)
  87
  88
  89 class FilePatch(object):
  90   """A class representing a set of line-by-line changes to a particular file.
  91   None of the changes are applied until Apply is called.  All line numbers are
  92   counted from 0.
  93   """
  94
  95   def __init__(self, filename):
  96     self.filename = filename
  97     self.linenums_to_delete = set()
  98     # A dictionary from line number to an array of strings to be inserted at
  99     # that line number.
 100     self.lines_to_add = {}
 101
 102   def Delete(self, start_line, end_line):
 103     """Make the patch delete the lines starting with |start_line| up to but not
 104     including |end_line|.
 105     """
 106     self.linenums_to_delete |= set(range(start_line, end_line))
 107
 108   def Add(self, text, line_number):
 109     """Add the given text before the text on the given line number."""
 110     if line_number in self.lines_to_add:
 111       self.lines_to_add[line_number].append(text)
 112     else:
 113       self.lines_to_add[line_number] = [text]
 114
 115   def Apply(self):
 116     """Apply the patch by writing it to self.filename."""
 117     # Read the lines of the existing file in to a list.
 118     sourcefile = open(self.filename, "r")
 119     file_lines = sourcefile.readlines()
 120     sourcefile.close()
 121     # Now apply the patch.  Our strategy is to keep the array at the same size,
 122     # and just edit strings in the file_lines list as necessary.  When we delete
 123     # lines, we just blank the line and keep it in the list.  When we add lines,
 124     # we just prepend the added source code to the start of the existing line at
 125     # that line number.  This way, all the line numbers we cached from calls to
 126     # Add and Delete remain valid list indices, and we don't have to worry about
 127     # maintaining any offsets.  Each element of file_lines at the end may
 128     # contain any number of lines (0 or more) delimited by carriage returns.
 129     for linenum_to_delete in self.linenums_to_delete:
 130       file_lines[linenum_to_delete] = "";
 131     for linenum, sourcelines in self.lines_to_add.items():
 132       # Sort the lines we're adding so we get relatively consistent results.
 133       sourcelines.sort()
 134       # Prepend the new lines.  When we output
 135       file_lines[linenum] = "".join(sourcelines) + file_lines[linenum]
 136     newsource = open(self.filename, "w")
 137     for line in file_lines:
 138       newsource.write(line)
 139     newsource.close()
 140
 141
 142 def CheckAndInsert(typeinfo, typeinfo_map):
 143   """Check if a TypeInfo exists already in the given map with the same name.  If
 144   so, make sure the size is consistent.
 145   - If the name exists but the sizes do not match, print a message and
 146     exit with non-zero exit code.
 147   - If the name exists and the sizes match, do nothing.
 148   - If the name does not exist, insert the typeinfo in to the map.
 149
 150   """
 151   # If the type is unnamed, ignore it.
 152   if typeinfo.name == "":
 153     return
 154   # If the size is 0, ignore it.
 155   elif int(typeinfo.size) == 0:
 156     return
 157   # If the type is not defined under ppapi, ignore it.
 158   elif typeinfo.source_location.filename.find("ppapi") == -1:
 159     return
 160   # If the type is defined under GLES2, ignore it.
 161   elif typeinfo.source_location.filename.find("GLES2") > -1:
 162     return
 163   # If the type is an interface (by convention, starts with PPP_ or PPB_),
 164   # ignore it.
 165   elif (typeinfo.name[:4] == "PPP_") or (typeinfo.name[:4] == "PPB_"):
 166     return
 167   elif typeinfo.name in typeinfo_map:
 168     if typeinfo.size != typeinfo_map[typeinfo.name].size:
 169       print "Error: '" + typeinfo.name + "' is", \
 170           typeinfo_map[typeinfo.name].size, \
 171           "bytes on target '" + typeinfo_map[typeinfo.name].target + \
 172           "', but", typeinfo.size, "on target '" + typeinfo.target + "'"
 173       print typeinfo_map[typeinfo.name].parsed_line
 174       print typeinfo.parsed_line
 175       sys.exit(1)
 176     else:
 177       # It's already in the map and the sizes match.
 178       pass
 179   else:
 180     typeinfo_map[typeinfo.name] = typeinfo
 181
 182
 183 def ProcessTarget(clang_command, target, types):
 184   """Run clang using the given clang_command for the given target string.  Parse
 185   the output to create TypeInfos for each discovered type.  Insert each type in
 186   to the 'types' dictionary.  If the type already exists in the types
 187   dictionary, make sure that the size matches what's already in the map.  If
 188   not, exit with an error message.
 189   """
 190   p = subprocess.Popen(clang_command + " -triple " + target,
 191                        shell=True,
 192                        stdout=subprocess.PIPE)
 193   lines = p.communicate()[0].split()
 194   for line in lines:
 195     typeinfo = TypeInfo(line, target)
 196     CheckAndInsert(typeinfo, types)
 197
 198
 199 def ToAssertionCode(typeinfo):
 200   """Convert the TypeInfo to an appropriate C compile assertion.
 201   If it's a struct (Record in Clang terminology), we want a line like this:
 202     PP_COMPILE_ASSERT_STRUCT_SIZE_IN_BYTES(<name>, <size>);\n
 203   Enums:
 204     PP_COMPILE_ASSERT_ENUM_SIZE_IN_BYTES(<name>, <size>);\n
 205   Typedefs:
 206     PP_COMPILE_ASSERT_SIZE_IN_BYTES(<name>, <size>);\n
 207
 208   """
 209   line = "PP_COMPILE_ASSERT_"
 210   if typeinfo.kind == "Enum":
 211     line += "ENUM_"
 212   elif typeinfo.kind == "Record":
 213     line += "STRUCT_"
 214   line += "SIZE_IN_BYTES("
 215   line += typeinfo.name
 216   line += ", "
 217   line += typeinfo.size
 218   line += ");\n"
 219   return line
 220
 221
 222 def IsMacroDefinedName(typename):
 223   """Return true iff the given typename came from a PPAPI compile assertion."""
 224   return typename.find("PP_Dummy_Struct_For_") == 0
 225
 226
 227 def WriteArchSpecificCode(types, root, filename):
 228   """Write a header file that contains a compile-time assertion for the size of
 229      each of the given typeinfos, in to a file named filename rooted at root.
 230   """
 231   assertion_lines = [ToAssertionCode(typeinfo) for typeinfo in types]
 232   assertion_lines.sort()
 233   outfile = open(os.path.join(root, filename), "w")
 234   header_guard = "PPAPI_TESTS_" + filename.upper().replace(".", "_") + "_"
 235   outfile.write(COPYRIGHT_STRING_C)
 236   outfile.write('#ifndef ' + header_guard + '\n')
 237   outfile.write('#define ' + header_guard + '\n\n')
 238   outfile.write('#include "ppapi/tests/test_struct_sizes.c"\n\n')
 239   for line in assertion_lines:
 240     outfile.write(line)
 241   outfile.write('\n#endif  /* ' + header_guard + ' */\n')
 242
 243
 244 def main(argv):
 245   # See README file for example command-line invocation.  This script runs the
 246   # PrintNamesAndSizes Clang plugin with 'test_struct_sizes.c' as input, which
 247   # should include all C headers and all existing size checks.  It runs the
 248   # plugin multiple times;  once for each of a set of targets, some 32-bit and
 249   # some 64-bit.  It verifies that wherever possible, types have a consistent
 250   # size on both platforms.  Types that can't easily have consistent size (e.g.
 251   # ones that contain a pointer) are checked to make sure they are consistent
 252   # for all 32-bit platforms and consistent on all 64-bit platforms, but the
 253   # sizes on 32 vs 64 are allowed to differ.
 254   #
 255   # Then, if all the types have consistent size as expected, compile assertions
 256   # are added to the source code.  Types whose size is independent of
 257   # architectureacross have their compile assertions placed immediately after
 258   # their definition in the C API header.  Types whose size differs on 32-bit
 259   # vs 64-bit have a compile assertion placed in each of:
 260   # ppapi/tests/arch_dependent_sizes_32.h and
 261   # ppapi/tests/arch_dependent_sizes_64.h.
 262   #
 263   # Note that you should always check the results of the tool to make sure
 264   # they are sane.
 265   parser = optparse.OptionParser()
 266   parser.add_option(
 267       '-c', '--clang-path', dest='clang_path',
 268       default=(''),
 269       help='the path to the clang binary (default is to get it from your path)')
 270   parser.add_option(
 271       '-p', '--plugin', dest='plugin',
 272       default='tests/clang/libPrintNamesAndSizes.so',
 273       help='The path to the PrintNamesAndSizes plugin library.')
 274   parser.add_option(
 275       '--targets32', dest='targets32',
 276       default='i386-pc-linux,arm-pc-linux,i386-pc-win32',
 277       help='Which 32-bit target triples to provide to clang.')
 278   parser.add_option(
 279       '--targets64', dest='targets64',
 280       default='x86_64-pc-linux,x86_64-pc-win',
 281       help='Which 32-bit target triples to provide to clang.')
 282   parser.add_option(
 283       '-r', '--ppapi-root', dest='ppapi_root',
 284       default='.',
 285       help='The root directory of ppapi.')
 286   options, args = parser.parse_args(argv)
 287   if args:
 288     parser.print_help()
 289     print 'ERROR: invalid argument'
 290     sys.exit(1)
 291
 292   clang_executable = os.path.join(options.clang_path, 'clang')
 293   clang_command = clang_executable + " -cc1" \
 294       + " -load " + options.plugin \
 295       + " -plugin PrintNamesAndSizes" \
 296       + " -I" + os.path.join(options.ppapi_root, "..") \
 297       + " " \
 298       + os.path.join(options.ppapi_root, "tests", "test_struct_sizes.c")
 299
 300   # Dictionaries mapping type names to TypeInfo objects.
 301   # Types that have size dependent on architecture, for 32-bit
 302   types32 = {}
 303   # Types that have size dependent on architecture, for 64-bit
 304   types64 = {}
 305   # Note that types32 and types64 should contain the same types, but with
 306   # different sizes.
 307
 308   # Types whose size should be consistent regardless of architecture.
 309   types_independent = {}
 310
 311   # Now run clang for each target.  Along the way, make sure architecture-
 312   # dependent types are consistent sizes on all 32-bit platforms and consistent
 313   # on all 64-bit platforms.
 314   targets32 = options.targets32.split(',');
 315   for target in targets32:
 316     # For each 32-bit target, run the PrintNamesAndSizes Clang plugin to get
 317     # information about all types in the translation unit, and add a TypeInfo
 318     # for each of them to types32.  If any size mismatches are found,
 319     # ProcessTarget will spit out an error and exit.
 320     ProcessTarget(clang_command, target, types32)
 321   targets64 = options.targets64.split(',');
 322   for target in targets64:
 323     # Do the same as above for each 64-bit target;  put all types in types64.
 324     ProcessTarget(clang_command, target, types64)
 325
 326   # Now for each dictionary, find types whose size are consistent regardless of
 327   # architecture, and move those in to types_independent.  Anywhere sizes
 328   # differ, make sure they are expected to be architecture-dependent based on
 329   # their structure.  If we find types which could easily be consistent but
 330   # aren't, spit out an error and exit.
 331   types_independent = {}
 332   for typename, typeinfo32 in types32.items():
 333     if (typename in types64):
 334       typeinfo64 = types64[typename]
 335       if (typeinfo64.size == typeinfo32.size):
 336         # The types are the same size, so we can treat it as arch-independent.
 337         types_independent[typename] = typeinfo32
 338         del types32[typename]
 339         del types64[typename]
 340       elif (typeinfo32.arch_dependent or typeinfo64.arch_dependent):
 341         # The type is defined in such a way that it would be difficult to make
 342         # its size consistent.  E.g., it has pointers.  We'll leave it in the
 343         # arch-dependent maps so that we can put arch-dependent size checks in
 344         # test code.
 345         pass
 346       else:
 347         # The sizes don't match, but there's no reason they couldn't.  It's
 348         # probably due to an alignment mismatch between Win32/NaCl vs Linux32/
 349         # Mac32.
 350         print "Error: '" + typename + "' is", typeinfo32.size, \
 351             "bytes on target '" + typeinfo32.target + \
 352             "', but", typeinfo64.size, "on target '" + typeinfo64.target + "'"
 353         print typeinfo32.parsed_line
 354         print typeinfo64.parsed_line
 355         sys.exit(1)
 356     else:
 357       print "WARNING:  Type '", typename, "' was defined for target '",
 358       print typeinfo32.target, ", but not for any 64-bit targets."
 359
 360   # Now we have all the information we need to generate our static assertions.
 361   # Types that have consistent size across architectures will have the static
 362   # assertion placed immediately after their definition.  Types whose size
 363   # depends on 32-bit vs 64-bit architecture will have checks placed in
 364   # tests/arch_dependent_sizes_32/64.h.
 365
 366   # This dictionary maps file names to FilePatch objects.  We will add items
 367   # to it as needed.  Each FilePatch represents a set of changes to make to the
 368   # associated file (additions and deletions).
 369   file_patches = {}
 370
 371   # Find locations of existing macros, and just delete them all.  Note that
 372   # normally, only things in 'types_independent' need to be deleted, as arch-
 373   # dependent checks exist in tests/arch_dependent_sizes_32/64.h, which are
 374   # always completely over-ridden.  However, it's possible that a type that used
 375   # to be arch-independent has changed to now be arch-dependent (e.g., because
 376   # a pointer was added), and we want to delete the old check in that case.
 377   for name, typeinfo in \
 378       types_independent.items() + types32.items() + types64.items():
 379     if IsMacroDefinedName(name):
 380       sourcefile = typeinfo.source_location.filename
 381       if sourcefile not in file_patches:
 382         file_patches[sourcefile] = FilePatch(sourcefile)
 383       file_patches[sourcefile].Delete(typeinfo.source_location.start_line,
 384                                       typeinfo.source_location.end_line+1)
 385
 386   # Add a compile-time assertion for each type whose size is independent of
 387   # architecture.  These assertions go immediately after the class definition.
 388   for name, typeinfo in types_independent.items():
 389     # Ignore dummy types that were defined by macros and also ignore types that
 390     # are 0 bytes (i.e., typedefs to void).
 391     if not IsMacroDefinedName(name) and typeinfo.size > 0:
 392       sourcefile = typeinfo.source_location.filename
 393       if sourcefile not in file_patches:
 394         file_patches[sourcefile] = FilePatch(sourcefile)
 395       # Add the assertion code just after the definition of the type.
 396       # E.g.:
 397       # struct Foo {
 398       #   int32_t x;
 399       # };
 400       # PP_COMPILE_ASSERT_STRUCT_SIZE_IN_BYTES(Foo, 4); <---Add this line
 401       file_patches[sourcefile].Add(ToAssertionCode(typeinfo),
 402                                    typeinfo.source_location.end_line+1)
 403
 404   # Apply our patches.  This actually edits the files containing the definitions
 405   # for the types in types_independent.
 406   for filename, patch in file_patches.items():
 407     patch.Apply()
 408
 409   # Write out a file of checks for 32-bit architectures and a separate file for
 410   # 64-bit architectures.  These only have checks for types that are
 411   # architecture-dependent.
 412   c_source_root = os.path.join(options.ppapi_root, "tests")
 413   WriteArchSpecificCode(types32.values(),
 414                         c_source_root,
 415                         "arch_dependent_sizes_32.h")
 416   WriteArchSpecificCode(types64.values(),
 417                         c_source_root,
 418                         "arch_dependent_sizes_64.h")
 419
 420   return 0
 421
 422
 423 if __name__ == '__main__':
 424     sys.exit(main(sys.argv[1:]))