tools/json_schema_compiler/idl_schema.py

   1 #! /usr/bin/env python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 import itertools
   7 import json
   8 import os.path
   9 import pprint
  10 import re
  11 import sys
  12
  13 from json_parse import OrderedDict
  14
  15 # This file is a peer to json_schema.py. Each of these files understands a
  16 # certain format describing APIs (either JSON or IDL), reads files written
  17 # in that format into memory, and emits them as a Python array of objects
  18 # corresponding to those APIs, where the objects are formatted in a way that
  19 # the JSON schema compiler understands. compiler.py drives both idl_schema.py
  20 # and json_schema.py.
  21
  22 # idl_parser expects to be able to import certain files in its directory,
  23 # so let's set things up the way it wants.
  24 _idl_generators_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
  25                                     os.pardir, os.pardir, 'ppapi', 'generators')
  26 if _idl_generators_path in sys.path:
  27   import idl_parser
  28 else:
  29   sys.path.insert(0, _idl_generators_path)
  30   try:
  31     import idl_parser
  32   finally:
  33     sys.path.pop(0)
  34
  35 def ProcessComment(comment):
  36   '''
  37   Convert a comment into a parent comment and a list of parameter comments.
  38
  39   Function comments are of the form:
  40     Function documentation. May contain HTML and multiple lines.
  41
  42     |arg1_name|: Description of arg1. Use <var>argument</var> to refer
  43     to other arguments.
  44     |arg2_name|: Description of arg2...
  45
  46   Newlines are removed, and leading and trailing whitespace is stripped.
  47
  48   Args:
  49     comment: The string from a Comment node.
  50
  51   Returns: A tuple that looks like:
  52     (
  53       "The processed comment, minus all |parameter| mentions.",
  54       {
  55         'parameter_name_1': "The comment that followed |parameter_name_1|:",
  56         ...
  57       }
  58     )
  59   '''
  60   def add_paragraphs(content):
  61     paragraphs = content.split('\n\n')
  62     if len(paragraphs) < 2:
  63       return content
  64     return '<p>' + '</p><p>'.join(p.strip() for p in paragraphs) + '</p>'
  65
  66   # Find all the parameter comments of the form '|name|: comment'.
  67   parameter_starts = list(re.finditer(r' *\|([^|]*)\| *: *', comment))
  68
  69   # Get the parent comment (everything before the first parameter comment.
  70   first_parameter_location = (parameter_starts[0].start()
  71                               if parameter_starts else len(comment))
  72   parent_comment = (add_paragraphs(comment[:first_parameter_location].strip())
  73                     .replace('\n', ''))
  74
  75   params = OrderedDict()
  76   for (cur_param, next_param) in itertools.izip_longest(parameter_starts,
  77                                                         parameter_starts[1:]):
  78     param_name = cur_param.group(1)
  79
  80     # A parameter's comment goes from the end of its introduction to the
  81     # beginning of the next parameter's introduction.
  82     param_comment_start = cur_param.end()
  83     param_comment_end = next_param.start() if next_param else len(comment)
  84     params[param_name] = (
  85         add_paragraphs(comment[param_comment_start:param_comment_end].strip())
  86         .replace('\n', ''))
  87
  88   return (parent_comment, params)
  89
  90
  91 class Callspec(object):
  92   '''
  93   Given a Callspec node representing an IDL function declaration, converts into
  94   a tuple:
  95       (name, list of function parameters, return type)
  96   '''
  97   def __init__(self, callspec_node, comment):
  98     self.node = callspec_node
  99     self.comment = comment
 100
 101   def process(self, callbacks):
 102     parameters = []
 103     return_type = None
 104     if self.node.GetProperty('TYPEREF') not in ('void', None):
 105       return_type = Typeref(self.node.GetProperty('TYPEREF'),
 106                             self.node.parent,
 107                             {'name': self.node.GetName()}).process(callbacks)
 108       # The IDL parser doesn't allow specifying return types as optional.
 109       # Instead we infer any object return values to be optional.
 110       # TODO(asargent): fix the IDL parser to support optional return types.
 111       if return_type.get('type') == 'object' or '$ref' in return_type:
 112         return_type['optional'] = True
 113     for node in self.node.GetChildren():
 114       parameter = Param(node).process(callbacks)
 115       if parameter['name'] in self.comment:
 116         parameter['description'] = self.comment[parameter['name']]
 117       parameters.append(parameter)
 118     return (self.node.GetName(), parameters, return_type)
 119
 120
 121 class Param(object):
 122   '''
 123   Given a Param node representing a function parameter, converts into a Python
 124   dictionary that the JSON schema compiler expects to see.
 125   '''
 126   def __init__(self, param_node):
 127     self.node = param_node
 128
 129   def process(self, callbacks):
 130     return Typeref(self.node.GetProperty('TYPEREF'),
 131                    self.node,
 132                    {'name': self.node.GetName()}).process(callbacks)
 133
 134
 135 class Dictionary(object):
 136   '''
 137   Given an IDL Dictionary node, converts into a Python dictionary that the JSON
 138   schema compiler expects to see.
 139   '''
 140   def __init__(self, dictionary_node):
 141     self.node = dictionary_node
 142
 143   def process(self, callbacks):
 144     properties = OrderedDict()
 145     for node in self.node.GetChildren():
 146       if node.cls == 'Member':
 147         k, v = Member(node).process(callbacks)
 148         properties[k] = v
 149     result = {'id': self.node.GetName(),
 150               'properties': properties,
 151               'type': 'object'}
 152     if self.node.GetProperty('nodoc'):
 153       result['nodoc'] = True
 154     elif self.node.GetProperty('inline_doc'):
 155       result['inline_doc'] = True
 156     elif self.node.GetProperty('noinline_doc'):
 157       result['noinline_doc'] = True
 158     return result
 159
 160
 161
 162 class Member(object):
 163   '''
 164   Given an IDL dictionary or interface member, converts into a name/value pair
 165   where the value is a Python dictionary that the JSON schema compiler expects
 166   to see.
 167   '''
 168   def __init__(self, member_node):
 169     self.node = member_node
 170
 171   def process(self, callbacks, functions_are_properties=False):
 172     properties = OrderedDict()
 173     name = self.node.GetName()
 174     if self.node.GetProperty('deprecated'):
 175       properties['deprecated'] = self.node.GetProperty('deprecated')
 176     if self.node.GetProperty('allowAmbiguousOptionalArguments'):
 177       properties['allowAmbiguousOptionalArguments'] = True
 178     for property_name in ('OPTIONAL', 'nodoc', 'nocompile', 'nodart'):
 179       if self.node.GetProperty(property_name):
 180         properties[property_name.lower()] = True
 181     for option_name, sanitizer in [
 182         ('maxListeners', int),
 183         ('supportsFilters', lambda s: s == 'true'),
 184         ('supportsListeners', lambda s: s == 'true'),
 185         ('supportsRules', lambda s: s == 'true')]:
 186       if self.node.GetProperty(option_name):
 187         if 'options' not in properties:
 188           properties['options'] = {}
 189         properties['options'][option_name] = sanitizer(self.node.GetProperty(
 190           option_name))
 191     type_override = None
 192     parameter_comments = OrderedDict()
 193     for node in self.node.GetChildren():
 194       if node.cls == 'Comment':
 195         (parent_comment, parameter_comments) = ProcessComment(node.GetName())
 196         properties['description'] = parent_comment
 197       elif node.cls == 'Callspec':
 198         name, parameters, return_type = (Callspec(node, parameter_comments)
 199                                          .process(callbacks))
 200         if functions_are_properties:
 201           # If functions are treated as properties (which will happen if the
 202           # interface is named Properties) then this isn't a function, it's a
 203           # property which is encoded as a function with no arguments. The
 204           # property type is the return type. This is an egregious hack in lieu
 205           # of the IDL parser supporting 'const'.
 206           assert parameters == [], (
 207                  'Property "%s" must be no-argument functions '
 208                  'with a non-void return type' % name)
 209           assert return_type is not None, (
 210                  'Property "%s" must be no-argument functions '
 211                  'with a non-void return type' % name)
 212           assert 'type' in return_type, (
 213                  'Property return type "%s" from "%s" must specify a '
 214                  'fundamental IDL type.' % (pprint.pformat(return_type), name))
 215           type_override = return_type['type']
 216         else:
 217           type_override = 'function'
 218           properties['parameters'] = parameters
 219           if return_type is not None:
 220             properties['returns'] = return_type
 221     properties['name'] = name
 222     if type_override is not None:
 223       properties['type'] = type_override
 224     else:
 225       properties = Typeref(self.node.GetProperty('TYPEREF'),
 226                            self.node, properties).process(callbacks)
 227     value = self.node.GetProperty('value')
 228     if value is not None:
 229       # IDL always returns values as strings, so cast to their real type.
 230       properties['value'] = self.cast_from_json_type(properties['type'], value)
 231     enum_values = self.node.GetProperty('legalValues')
 232     if enum_values:
 233       # IDL always returns enum values as strings, so cast to their real type.
 234       properties['enum'] = [self.cast_from_json_type(properties['type'], enum)
 235                             for enum in enum_values]
 236     return name, properties
 237
 238   def cast_from_json_type(self, json_type, string_value):
 239     '''Casts from string |string_value| to a real Python type based on a JSON
 240     Schema type |json_type|. For example, a string value of '42' and a JSON
 241     Schema type 'integer' will cast to int('42') ==> 42.
 242     '''
 243     if json_type == 'integer':
 244       return int(string_value)
 245     if json_type == 'number':
 246       return float(string_value)
 247     # Add more as necessary.
 248     assert json_type == 'string', (
 249            'No rule exists to cast JSON Schema type "%s" to its equivalent '
 250            'Python type for value "%s". You must add a new rule here.' %
 251            (json_type, string_value))
 252     return string_value
 253
 254
 255 class Typeref(object):
 256   '''
 257   Given a TYPEREF property representing the type of dictionary member or
 258   function parameter, converts into a Python dictionary that the JSON schema
 259   compiler expects to see.
 260   '''
 261   def __init__(self, typeref, parent, additional_properties):
 262     self.typeref = typeref
 263     self.parent = parent
 264     self.additional_properties = additional_properties
 265
 266   def process(self, callbacks):
 267     properties = self.additional_properties
 268     result = properties
 269
 270     if self.parent.GetPropertyLocal('OPTIONAL'):
 271       properties['optional'] = True
 272
 273     # The IDL parser denotes array types by adding a child 'Array' node onto
 274     # the Param node in the Callspec.
 275     for sibling in self.parent.GetChildren():
 276       if sibling.cls == 'Array' and sibling.GetName() == self.parent.GetName():
 277         properties['type'] = 'array'
 278         properties['items'] = OrderedDict()
 279         properties = properties['items']
 280         break
 281
 282     if self.typeref == 'DOMString':
 283       properties['type'] = 'string'
 284     elif self.typeref == 'boolean':
 285       properties['type'] = 'boolean'
 286     elif self.typeref == 'double':
 287       properties['type'] = 'number'
 288     elif self.typeref == 'long':
 289       properties['type'] = 'integer'
 290     elif self.typeref == 'any':
 291       properties['type'] = 'any'
 292     elif self.typeref == 'object':
 293       properties['type'] = 'object'
 294       if 'additionalProperties' not in properties:
 295         properties['additionalProperties'] = OrderedDict()
 296       properties['additionalProperties']['type'] = 'any'
 297       instance_of = self.parent.GetProperty('instanceOf')
 298       if instance_of:
 299         properties['isInstanceOf'] = instance_of
 300     elif self.typeref == 'ArrayBuffer':
 301       properties['type'] = 'binary'
 302       properties['isInstanceOf'] = 'ArrayBuffer'
 303     elif self.typeref == 'FileEntry':
 304       properties['type'] = 'object'
 305       properties['isInstanceOf'] = 'FileEntry'
 306       if 'additionalProperties' not in properties:
 307         properties['additionalProperties'] = OrderedDict()
 308       properties['additionalProperties']['type'] = 'any'
 309     elif self.parent.GetPropertyLocal('Union'):
 310       properties['choices'] = [Typeref(node.GetProperty('TYPEREF'),
 311                                        node,
 312                                        OrderedDict()).process(callbacks)
 313                                for node in self.parent.GetChildren()
 314                                if node.cls == 'Option']
 315     elif self.typeref is None:
 316       properties['type'] = 'function'
 317     else:
 318       if self.typeref in callbacks:
 319         # Do not override name and description if they are already specified.
 320         name = properties.get('name', None)
 321         description = properties.get('description', None)
 322         properties.update(callbacks[self.typeref])
 323         if description is not None:
 324           properties['description'] = description
 325         if name is not None:
 326           properties['name'] = name
 327       else:
 328         properties['$ref'] = self.typeref
 329     return result
 330
 331
 332 class Enum(object):
 333   '''
 334   Given an IDL Enum node, converts into a Python dictionary that the JSON
 335   schema compiler expects to see.
 336   '''
 337   def __init__(self, enum_node):
 338     self.node = enum_node
 339     self.description = ''
 340
 341   def process(self):
 342     enum = []
 343     for node in self.node.GetChildren():
 344       if node.cls == 'EnumItem':
 345         enum_value = {'name': node.GetName()}
 346         for child in node.GetChildren():
 347           if child.cls == 'Comment':
 348             enum_value['description'] = ProcessComment(child.GetName())[0]
 349           else:
 350             raise ValueError('Did not process %s %s' % (child.cls, child))
 351         enum.append(enum_value)
 352       elif node.cls == 'Comment':
 353         self.description = ProcessComment(node.GetName())[0]
 354       else:
 355         sys.exit('Did not process %s %s' % (node.cls, node))
 356     result = {'id' : self.node.GetName(),
 357               'description': self.description,
 358               'type': 'string',
 359               'enum': enum}
 360     for property_name in (
 361         'inline_doc', 'noinline_doc', 'nodoc', 'cpp_enum_prefix_override',):
 362       if self.node.GetProperty(property_name):
 363         result[property_name] = self.node.GetProperty(property_name)
 364     if self.node.GetProperty('deprecated'):
 365       result['deprecated'] = self.node.GetProperty('deprecated')
 366     return result
 367
 368
 369 class Namespace(object):
 370   '''
 371   Given an IDLNode representing an IDL namespace, converts into a Python
 372   dictionary that the JSON schema compiler expects to see.
 373   '''
 374
 375   def __init__(self,
 376                namespace_node,
 377                description,
 378                nodoc=False,
 379                internal=False,
 380                platforms=None,
 381                compiler_options=None,
 382                deprecated=None,
 383                documentation_options=None):
 384     self.namespace = namespace_node
 385     self.nodoc = nodoc
 386     self.internal = internal
 387     self.platforms = platforms
 388     self.compiler_options = compiler_options
 389     self.events = []
 390     self.functions = []
 391     self.properties = OrderedDict()
 392     self.types = []
 393     self.callbacks = OrderedDict()
 394     self.description = description
 395     self.deprecated = deprecated
 396     self.documentation_options = documentation_options
 397
 398   def process(self):
 399     for node in self.namespace.GetChildren():
 400       if node.cls == 'Dictionary':
 401         self.types.append(Dictionary(node).process(self.callbacks))
 402       elif node.cls == 'Callback':
 403         k, v = Member(node).process(self.callbacks)
 404         self.callbacks[k] = v
 405       elif node.cls == 'Interface' and node.GetName() == 'Functions':
 406         self.functions = self.process_interface(node)
 407       elif node.cls == 'Interface' and node.GetName() == 'Events':
 408         self.events = self.process_interface(node)
 409       elif node.cls == 'Interface' and node.GetName() == 'Properties':
 410         properties_as_list = self.process_interface(
 411             node, functions_are_properties=True)
 412         for prop in properties_as_list:
 413           # Properties are given as key-value pairs, but IDL will parse
 414           # it as a list. Convert back to key-value pairs.
 415           prop_name = prop.pop('name')
 416           assert not self.properties.has_key(prop_name), (
 417                  'Property "%s" cannot be specified more than once.' %
 418                  prop_name)
 419           self.properties[prop_name] = prop
 420       elif node.cls == 'Enum':
 421         self.types.append(Enum(node).process())
 422       else:
 423         sys.exit('Did not process %s %s' % (node.cls, node))
 424     compiler_options = self.compiler_options or {}
 425     documentation_options = self.documentation_options or {}
 426     return {'namespace': self.namespace.GetName(),
 427             'description': self.description,
 428             'nodoc': self.nodoc,
 429             'types': self.types,
 430             'functions': self.functions,
 431             'properties': self.properties,
 432             'internal': self.internal,
 433             'events': self.events,
 434             'platforms': self.platforms,
 435             'compiler_options': compiler_options,
 436             'deprecated': self.deprecated,
 437             'documentation_options': documentation_options}
 438
 439   def process_interface(self, node, functions_are_properties=False):
 440     members = []
 441     for member in node.GetChildren():
 442       if member.cls == 'Member':
 443         _, properties = Member(member).process(
 444             self.callbacks,
 445             functions_are_properties=functions_are_properties)
 446         members.append(properties)
 447     return members
 448
 449
 450 class IDLSchema(object):
 451   '''
 452   Given a list of IDLNodes and IDLAttributes, converts into a Python list
 453   of api_defs that the JSON schema compiler expects to see.
 454   '''
 455
 456   def __init__(self, idl):
 457     self.idl = idl
 458
 459   def process(self):
 460     namespaces = []
 461     nodoc = False
 462     internal = False
 463     description = None
 464     platforms = None
 465     compiler_options = {}
 466     deprecated = None
 467     documentation_options = {}
 468     for node in self.idl:
 469       if node.cls == 'Namespace':
 470         if not description:
 471           # TODO(kalman): Go back to throwing an error here.
 472           print('%s must have a namespace-level comment. This will '
 473                            'appear on the API summary page.' % node.GetName())
 474           description = ''
 475         namespace = Namespace(node, description, nodoc, internal,
 476                               platforms=platforms,
 477                               compiler_options=compiler_options or None,
 478                               deprecated=deprecated,
 479                               documentation_options=documentation_options)
 480         namespaces.append(namespace.process())
 481         nodoc = False
 482         internal = False
 483         platforms = None
 484         compiler_options = None
 485       elif node.cls == 'Copyright':
 486         continue
 487       elif node.cls == 'Comment':
 488         description = node.GetName()
 489       elif node.cls == 'ExtAttribute':
 490         if node.name == 'nodoc':
 491           nodoc = bool(node.value)
 492         elif node.name == 'internal':
 493           internal = bool(node.value)
 494         elif node.name == 'platforms':
 495           platforms = list(node.value)
 496         elif node.name == 'implemented_in':
 497           compiler_options['implemented_in'] = node.value
 498         elif node.name == 'camel_case_enum_to_string':
 499           compiler_options['camel_case_enum_to_string'] = node.value
 500         elif node.name == 'deprecated':
 501           deprecated = str(node.value)
 502         elif node.name == 'documentation_title':
 503           documentation_options['title'] = node.value
 504         elif node.name == 'documentation_namespace':
 505           documentation_options['namespace'] = node.value
 506         elif node.name == 'documented_in':
 507           documentation_options['documented_in'] = node.value
 508         else:
 509           continue
 510       else:
 511         sys.exit('Did not process %s %s' % (node.cls, node))
 512     return namespaces
 513
 514
 515 def Load(filename):
 516   '''
 517   Given the filename of an IDL file, parses it and returns an equivalent
 518   Python dictionary in a format that the JSON schema compiler expects to see.
 519   '''
 520
 521   f = open(filename, 'r')
 522   contents = f.read()
 523   f.close()
 524
 525   return Process(contents, filename)
 526
 527
 528 def Process(contents, filename):
 529   '''
 530   Processes the contents of a file and returns an equivalent Python dictionary
 531   in a format that the JSON schema compiler expects to see. (Separate from
 532   Load primarily for testing purposes.)
 533   '''
 534
 535   idl = idl_parser.IDLParser().ParseData(contents, filename)
 536   idl_schema = IDLSchema(idl)
 537   return idl_schema.process()
 538
 539
 540 def Main():
 541   '''
 542   Dump a json serialization of parse result for the IDL files whose names
 543   were passed in on the command line.
 544   '''
 545   if len(sys.argv) > 1:
 546     for filename in sys.argv[1:]:
 547       schema = Load(filename)
 548       print json.dumps(schema, indent=2)
 549   else:
 550     contents = sys.stdin.read()
 551     idl = idl_parser.IDLParser().ParseData(contents, '<stdin>')
 552     schema = IDLSchema(idl).process()
 553     print json.dumps(schema, indent=2)
 554
 555
 556 if __name__ == '__main__':
 557   Main()