tools/json_schema_compiler/idl_schema.py

   1 #! /usr/bin/env python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 import itertools
   7 import json
   8 import os.path
   9 import re
  10 import sys
  11
  12 from json_parse import OrderedDict
  13
  14 # This file is a peer to json_schema.py. Each of these files understands a
  15 # certain format describing APIs (either JSON or IDL), reads files written
  16 # in that format into memory, and emits them as a Python array of objects
  17 # corresponding to those APIs, where the objects are formatted in a way that
  18 # the JSON schema compiler understands. compiler.py drives both idl_schema.py
  19 # and json_schema.py.
  20
  21 # idl_parser expects to be able to import certain files in its directory,
  22 # so let's set things up the way it wants.
  23 _idl_generators_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
  24                                     os.pardir, os.pardir, 'ppapi', 'generators')
  25 if _idl_generators_path in sys.path:
  26   import idl_parser
  27 else:
  28   sys.path.insert(0, _idl_generators_path)
  29   try:
  30     import idl_parser
  31   finally:
  32     sys.path.pop(0)
  33
  34 def ProcessComment(comment):
  35   '''
  36   Convert a comment into a parent comment and a list of parameter comments.
  37
  38   Function comments are of the form:
  39     Function documentation. May contain HTML and multiple lines.
  40
  41     |arg1_name|: Description of arg1. Use <var>argument</var> to refer
  42     to other arguments.
  43     |arg2_name|: Description of arg2...
  44
  45   Newlines are removed, and leading and trailing whitespace is stripped.
  46
  47   Args:
  48     comment: The string from a Comment node.
  49
  50   Returns: A tuple that looks like:
  51     (
  52       "The processed comment, minus all |parameter| mentions.",
  53       {
  54         'parameter_name_1': "The comment that followed |parameter_name_1|:",
  55         ...
  56       }
  57     )
  58   '''
  59   def add_paragraphs(content):
  60     paragraphs = content.split('\n\n')
  61     if len(paragraphs) < 2:
  62       return content
  63     return '<p>' + '</p><p>'.join(p.strip() for p in paragraphs) + '</p>'
  64
  65   # Find all the parameter comments of the form '|name|: comment'.
  66   parameter_starts = list(re.finditer(r' *\|([^|]*)\| *: *', comment))
  67
  68   # Get the parent comment (everything before the first parameter comment.
  69   first_parameter_location = (parameter_starts[0].start()
  70                               if parameter_starts else len(comment))
  71   parent_comment = (add_paragraphs(comment[:first_parameter_location].strip())
  72                     .replace('\n', ''))
  73
  74   params = OrderedDict()
  75   for (cur_param, next_param) in itertools.izip_longest(parameter_starts,
  76                                                         parameter_starts[1:]):
  77     param_name = cur_param.group(1)
  78
  79     # A parameter's comment goes from the end of its introduction to the
  80     # beginning of the next parameter's introduction.
  81     param_comment_start = cur_param.end()
  82     param_comment_end = next_param.start() if next_param else len(comment)
  83     params[param_name] = (
  84         add_paragraphs(comment[param_comment_start:param_comment_end].strip())
  85         .replace('\n', ''))
  86
  87   return (parent_comment, params)
  88
  89
  90 class Callspec(object):
  91   '''
  92   Given a Callspec node representing an IDL function declaration, converts into
  93   a tuple:
  94       (name, list of function parameters, return type)
  95   '''
  96   def __init__(self, callspec_node, comment):
  97     self.node = callspec_node
  98     self.comment = comment
  99
 100   def process(self, callbacks):
 101     parameters = []
 102     return_type = None
 103     if self.node.GetProperty('TYPEREF') not in ('void', None):
 104       return_type = Typeref(self.node.GetProperty('TYPEREF'),
 105                             self.node.parent,
 106                             {'name': self.node.GetName()}).process(callbacks)
 107       # The IDL parser doesn't allow specifying return types as optional.
 108       # Instead we infer any object return values to be optional.
 109       # TODO(asargent): fix the IDL parser to support optional return types.
 110       if return_type.get('type') == 'object' or '$ref' in return_type:
 111         return_type['optional'] = True
 112     for node in self.node.GetChildren():
 113       parameter = Param(node).process(callbacks)
 114       if parameter['name'] in self.comment:
 115         parameter['description'] = self.comment[parameter['name']]
 116       parameters.append(parameter)
 117     return (self.node.GetName(), parameters, return_type)
 118
 119
 120 class Param(object):
 121   '''
 122   Given a Param node representing a function parameter, converts into a Python
 123   dictionary that the JSON schema compiler expects to see.
 124   '''
 125   def __init__(self, param_node):
 126     self.node = param_node
 127
 128   def process(self, callbacks):
 129     return Typeref(self.node.GetProperty('TYPEREF'),
 130                    self.node,
 131                    {'name': self.node.GetName()}).process(callbacks)
 132
 133
 134 class Dictionary(object):
 135   '''
 136   Given an IDL Dictionary node, converts into a Python dictionary that the JSON
 137   schema compiler expects to see.
 138   '''
 139   def __init__(self, dictionary_node):
 140     self.node = dictionary_node
 141
 142   def process(self, callbacks):
 143     properties = OrderedDict()
 144     for node in self.node.GetChildren():
 145       if node.cls == 'Member':
 146         k, v = Member(node).process(callbacks)
 147         properties[k] = v
 148     result = {'id': self.node.GetName(),
 149               'properties': properties,
 150               'type': 'object'}
 151     if self.node.GetProperty('nodoc'):
 152       result['nodoc'] = True
 153     elif self.node.GetProperty('inline_doc'):
 154       result['inline_doc'] = True
 155     elif self.node.GetProperty('noinline_doc'):
 156       result['noinline_doc'] = True
 157     return result
 158
 159
 160
 161 class Member(object):
 162   '''
 163   Given an IDL dictionary or interface member, converts into a name/value pair
 164   where the value is a Python dictionary that the JSON schema compiler expects
 165   to see.
 166   '''
 167   def __init__(self, member_node):
 168     self.node = member_node
 169
 170   def process(self, callbacks):
 171     properties = OrderedDict()
 172     name = self.node.GetName()
 173     if self.node.GetProperty('deprecated'):
 174       properties['deprecated'] = self.node.GetProperty('deprecated')
 175     if self.node.GetProperty('allowAmbiguousOptionalArguments'):
 176       properties['allowAmbiguousOptionalArguments'] = True
 177     for property_name in ('OPTIONAL', 'nodoc', 'nocompile', 'nodart'):
 178       if self.node.GetProperty(property_name):
 179         properties[property_name.lower()] = True
 180     for option_name, sanitizer in [
 181         ('maxListeners', int),
 182         ('supportsFilters', lambda s: s == 'true'),
 183         ('supportsListeners', lambda s: s == 'true'),
 184         ('supportsRules', lambda s: s == 'true')]:
 185       if self.node.GetProperty(option_name):
 186         if 'options' not in properties:
 187           properties['options'] = {}
 188         properties['options'][option_name] = sanitizer(self.node.GetProperty(
 189           option_name))
 190     is_function = False
 191     parameter_comments = OrderedDict()
 192     for node in self.node.GetChildren():
 193       if node.cls == 'Comment':
 194         (parent_comment, parameter_comments) = ProcessComment(node.GetName())
 195         properties['description'] = parent_comment
 196       elif node.cls == 'Callspec':
 197         is_function = True
 198         name, parameters, return_type = (Callspec(node, parameter_comments)
 199                                          .process(callbacks))
 200         properties['parameters'] = parameters
 201         if return_type is not None:
 202           properties['returns'] = return_type
 203     properties['name'] = name
 204     if is_function:
 205       properties['type'] = 'function'
 206     else:
 207       properties = Typeref(self.node.GetProperty('TYPEREF'),
 208                            self.node, properties).process(callbacks)
 209     enum_values = self.node.GetProperty('legalValues')
 210     if enum_values:
 211       if properties['type'] == 'integer':
 212         enum_values = map(int, enum_values)
 213       elif properties['type'] == 'double':
 214         enum_values = map(float, enum_values)
 215       properties['enum'] = enum_values
 216     return name, properties
 217
 218
 219 class Typeref(object):
 220   '''
 221   Given a TYPEREF property representing the type of dictionary member or
 222   function parameter, converts into a Python dictionary that the JSON schema
 223   compiler expects to see.
 224   '''
 225   def __init__(self, typeref, parent, additional_properties):
 226     self.typeref = typeref
 227     self.parent = parent
 228     self.additional_properties = additional_properties
 229
 230   def process(self, callbacks):
 231     properties = self.additional_properties
 232     result = properties
 233
 234     if self.parent.GetPropertyLocal('OPTIONAL'):
 235       properties['optional'] = True
 236
 237     # The IDL parser denotes array types by adding a child 'Array' node onto
 238     # the Param node in the Callspec.
 239     for sibling in self.parent.GetChildren():
 240       if sibling.cls == 'Array' and sibling.GetName() == self.parent.GetName():
 241         properties['type'] = 'array'
 242         properties['items'] = OrderedDict()
 243         properties = properties['items']
 244         break
 245
 246     if self.typeref == 'DOMString':
 247       properties['type'] = 'string'
 248     elif self.typeref == 'boolean':
 249       properties['type'] = 'boolean'
 250     elif self.typeref == 'double':
 251       properties['type'] = 'number'
 252     elif self.typeref == 'long':
 253       properties['type'] = 'integer'
 254     elif self.typeref == 'any':
 255       properties['type'] = 'any'
 256     elif self.typeref == 'object':
 257       properties['type'] = 'object'
 258       if 'additionalProperties' not in properties:
 259         properties['additionalProperties'] = OrderedDict()
 260       properties['additionalProperties']['type'] = 'any'
 261       instance_of = self.parent.GetProperty('instanceOf')
 262       if instance_of:
 263         properties['isInstanceOf'] = instance_of
 264     elif self.typeref == 'ArrayBuffer':
 265       properties['type'] = 'binary'
 266       properties['isInstanceOf'] = 'ArrayBuffer'
 267     elif self.typeref == 'FileEntry':
 268       properties['type'] = 'object'
 269       properties['isInstanceOf'] = 'FileEntry'
 270       if 'additionalProperties' not in properties:
 271         properties['additionalProperties'] = OrderedDict()
 272       properties['additionalProperties']['type'] = 'any'
 273     elif self.parent.GetPropertyLocal('Union'):
 274       choices = []
 275       properties['choices'] = [Typeref(node.GetProperty('TYPEREF'),
 276                                        node,
 277                                        OrderedDict()).process(callbacks)
 278                                for node in self.parent.GetChildren()
 279                                if node.cls == 'Option']
 280     elif self.typeref is None:
 281       properties['type'] = 'function'
 282     else:
 283       if self.typeref in callbacks:
 284         # Do not override name and description if they are already specified.
 285         name = properties.get('name', None)
 286         description = properties.get('description', None)
 287         properties.update(callbacks[self.typeref])
 288         if description is not None:
 289           properties['description'] = description
 290         if name is not None:
 291           properties['name'] = name
 292       else:
 293         properties['$ref'] = self.typeref
 294     return result
 295
 296
 297 class Enum(object):
 298   '''
 299   Given an IDL Enum node, converts into a Python dictionary that the JSON
 300   schema compiler expects to see.
 301   '''
 302   def __init__(self, enum_node):
 303     self.node = enum_node
 304     self.description = ''
 305
 306   def process(self, callbacks):
 307     enum = []
 308     for node in self.node.GetChildren():
 309       if node.cls == 'EnumItem':
 310         enum_value = {'name': node.GetName()}
 311         for child in node.GetChildren():
 312           if child.cls == 'Comment':
 313             enum_value['description'] = ProcessComment(child.GetName())[0]
 314           else:
 315             raise ValueError('Did not process %s %s' % (child.cls, child))
 316         enum.append(enum_value)
 317       elif node.cls == 'Comment':
 318         self.description = ProcessComment(node.GetName())[0]
 319       else:
 320         sys.exit('Did not process %s %s' % (node.cls, node))
 321     result = {'id' : self.node.GetName(),
 322               'description': self.description,
 323               'type': 'string',
 324               'enum': enum}
 325     for property_name in (
 326         'inline_doc', 'noinline_doc', 'nodoc', 'cpp_enum_prefix_override',):
 327       if self.node.GetProperty(property_name):
 328         result[property_name] = self.node.GetProperty(property_name)
 329     if self.node.GetProperty('deprecated'):
 330         result[deprecated] = self.node.GetProperty('deprecated')
 331     return result
 332
 333
 334 class Namespace(object):
 335   '''
 336   Given an IDLNode representing an IDL namespace, converts into a Python
 337   dictionary that the JSON schema compiler expects to see.
 338   '''
 339
 340   def __init__(self,
 341                namespace_node,
 342                description,
 343                nodoc=False,
 344                internal=False,
 345                platforms=None,
 346                compiler_options=None,
 347                deprecated=None):
 348     self.namespace = namespace_node
 349     self.nodoc = nodoc
 350     self.internal = internal
 351     self.platforms = platforms
 352     self.compiler_options = compiler_options
 353     self.events = []
 354     self.functions = []
 355     self.types = []
 356     self.callbacks = OrderedDict()
 357     self.description = description
 358     self.deprecated = deprecated
 359
 360   def process(self):
 361     for node in self.namespace.GetChildren():
 362       if node.cls == 'Dictionary':
 363         self.types.append(Dictionary(node).process(self.callbacks))
 364       elif node.cls == 'Callback':
 365         k, v = Member(node).process(self.callbacks)
 366         self.callbacks[k] = v
 367       elif node.cls == 'Interface' and node.GetName() == 'Functions':
 368         self.functions = self.process_interface(node)
 369       elif node.cls == 'Interface' and node.GetName() == 'Events':
 370         self.events = self.process_interface(node)
 371       elif node.cls == 'Enum':
 372         self.types.append(Enum(node).process(self.callbacks))
 373       else:
 374         sys.exit('Did not process %s %s' % (node.cls, node))
 375     if self.compiler_options is not None:
 376       compiler_options = self.compiler_options
 377     else:
 378       compiler_options = {}
 379     return {'namespace': self.namespace.GetName(),
 380             'description': self.description,
 381             'nodoc': self.nodoc,
 382             'types': self.types,
 383             'functions': self.functions,
 384             'internal': self.internal,
 385             'events': self.events,
 386             'platforms': self.platforms,
 387             'compiler_options': compiler_options,
 388             'deprecated': self.deprecated}
 389
 390   def process_interface(self, node):
 391     members = []
 392     for member in node.GetChildren():
 393       if member.cls == 'Member':
 394         name, properties = Member(member).process(self.callbacks)
 395         members.append(properties)
 396     return members
 397
 398
 399 class IDLSchema(object):
 400   '''
 401   Given a list of IDLNodes and IDLAttributes, converts into a Python list
 402   of api_defs that the JSON schema compiler expects to see.
 403   '''
 404
 405   def __init__(self, idl):
 406     self.idl = idl
 407
 408   def process(self):
 409     namespaces = []
 410     nodoc = False
 411     internal = False
 412     description = None
 413     platforms = None
 414     compiler_options = {}
 415     deprecated = None
 416     for node in self.idl:
 417       if node.cls == 'Namespace':
 418         if not description:
 419           # TODO(kalman): Go back to throwing an error here.
 420           print('%s must have a namespace-level comment. This will '
 421                            'appear on the API summary page.' % node.GetName())
 422           description = ''
 423         namespace = Namespace(node, description, nodoc, internal,
 424                               platforms=platforms,
 425                               compiler_options=compiler_options or None,
 426                               deprecated=deprecated)
 427         namespaces.append(namespace.process())
 428         nodoc = False
 429         internal = False
 430         platforms = None
 431         compiler_options = None
 432       elif node.cls == 'Copyright':
 433         continue
 434       elif node.cls == 'Comment':
 435         description = node.GetName()
 436       elif node.cls == 'ExtAttribute':
 437         if node.name == 'nodoc':
 438           nodoc = bool(node.value)
 439         elif node.name == 'internal':
 440           internal = bool(node.value)
 441         elif node.name == 'platforms':
 442           platforms = list(node.value)
 443         elif node.name == 'implemented_in':
 444           compiler_options['implemented_in'] = node.value
 445         elif node.name == 'camel_case_enum_to_string':
 446           compiler_options['camel_case_enum_to_string'] = node.value
 447         elif node.name == 'deprecated':
 448           deprecated = str(node.value)
 449         else:
 450           continue
 451       else:
 452         sys.exit('Did not process %s %s' % (node.cls, node))
 453     return namespaces
 454
 455
 456 def Load(filename):
 457   '''
 458   Given the filename of an IDL file, parses it and returns an equivalent
 459   Python dictionary in a format that the JSON schema compiler expects to see.
 460   '''
 461
 462   f = open(filename, 'r')
 463   contents = f.read()
 464   f.close()
 465
 466   idl = idl_parser.IDLParser().ParseData(contents, filename)
 467   idl_schema = IDLSchema(idl)
 468   return idl_schema.process()
 469
 470
 471 def Main():
 472   '''
 473   Dump a json serialization of parse result for the IDL files whose names
 474   were passed in on the command line.
 475   '''
 476   if len(sys.argv) > 1:
 477     for filename in sys.argv[1:]:
 478       schema = Load(filename)
 479       print json.dumps(schema, indent=2)
 480   else:
 481     contents = sys.stdin.read()
 482     idl = idl_parser.IDLParser().ParseData(contents, '<stdin>')
 483     schema = IDLSchema(idl).process()
 484     print json.dumps(schema, indent=2)
 485
 486
 487 if __name__ == '__main__':
 488   Main()