ppapi/generators/idl_parser.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """ Parser for PPAPI IDL """
   7
   8 #
   9 # IDL Parser
  10 #
  11 # The parser is uses the PLY yacc library to build a set of parsing rules based
  12 # on WebIDL.
  13 #
  14 # WebIDL, and WebIDL regular expressions can be found at:
  15 #   http://dev.w3.org/2006/webapi/WebIDL/
  16 # PLY can be found at:
  17 #   http://www.dabeaz.com/ply/
  18 #
  19 # The parser generates a tree by recursively matching sets of items against
  20 # defined patterns.  When a match is made, that set of items is reduced
  21 # to a new item.   The new item can provide a match for parent patterns.
  22 # In this way an AST is built (reduced) depth first.
  23
  24
  25 import getopt
  26 import glob
  27 import os.path
  28 import re
  29 import sys
  30 import time
  31
  32 from idl_ast import IDLAst
  33 from idl_log import ErrOut, InfoOut, WarnOut
  34 from idl_lexer import IDLLexer
  35 from idl_node import IDLAttribute, IDLFile, IDLNode
  36 from idl_option import GetOption, Option, ParseOptions
  37 from idl_lint import Lint
  38
  39 from ply import lex
  40 from ply import yacc
  41
  42 Option('build_debug', 'Debug tree building.')
  43 Option('parse_debug', 'Debug parse reduction steps.')
  44 Option('token_debug', 'Debug token generation.')
  45 Option('dump_tree', 'Dump the tree.')
  46 Option('srcroot', 'Working directory.', default=os.path.join('..', 'api'))
  47 Option('include_private', 'Include private IDL directory in default API paths.')
  48
  49 #
  50 # ERROR_REMAP
  51 #
  52 # Maps the standard error formula into a more friendly error message.
  53 #
  54 ERROR_REMAP = {
  55   'Unexpected ")" after "(".' : 'Empty argument list.',
  56   'Unexpected ")" after ",".' : 'Missing argument.',
  57   'Unexpected "}" after ",".' : 'Trailing comma in block.',
  58   'Unexpected "}" after "{".' : 'Unexpected empty block.',
  59   'Unexpected comment after "}".' : 'Unexpected trailing comment.',
  60   'Unexpected "{" after keyword "enum".' : 'Enum missing name.',
  61   'Unexpected "{" after keyword "struct".' : 'Struct missing name.',
  62   'Unexpected "{" after keyword "interface".' : 'Interface missing name.',
  63 }
  64
  65 # DumpReduction
  66 #
  67 # Prints out the set of items which matched a particular pattern and the
  68 # new item or set it was reduced to.
  69 def DumpReduction(cls, p):
  70   if p[0] is None:
  71     InfoOut.Log("OBJ: %s(%d) - None\n" % (cls, len(p)))
  72     InfoOut.Log("  [%s]\n" % [str(x) for x in p[1:]])
  73   else:
  74     out = ""
  75     for index in range(len(p) - 1):
  76       out += " >%s< " % str(p[index + 1])
  77     InfoOut.Log("OBJ: %s(%d) - %s : %s\n"  % (cls, len(p), str(p[0]), out))
  78
  79
  80 # CopyToList
  81 #
  82 # Takes an input item, list, or None, and returns a new list of that set.
  83 def CopyToList(item):
  84   # If the item is 'Empty' make it an empty list
  85   if not item: item = []
  86
  87   # If the item is not a list
  88   if type(item) is not type([]): item = [item]
  89
  90   # Make a copy we can modify
  91   return list(item)
  92
  93
  94
  95 # ListFromConcat
  96 #
  97 # Generate a new List by joining of two sets of inputs which can be an
  98 # individual item, a list of items, or None.
  99 def ListFromConcat(*items):
 100   itemsout = []
 101   for item in items:
 102     itemlist = CopyToList(item)
 103     itemsout.extend(itemlist)
 104
 105   return itemsout
 106
 107
 108 # TokenTypeName
 109 #
 110 # Generate a string which has the type and value of the token.
 111 def TokenTypeName(t):
 112   if t.type == 'SYMBOL':  return 'symbol %s' % t.value
 113   if t.type in ['HEX', 'INT', 'OCT', 'FLOAT']:
 114     return 'value %s' % t.value
 115   if t.type == 'STRING' : return 'string "%s"' % t.value
 116   if t.type == 'COMMENT' : return 'comment'
 117   if t.type == t.value: return '"%s"' % t.value
 118   return 'keyword "%s"' % t.value
 119
 120
 121 #
 122 # IDL Parser
 123 #
 124 # The Parser inherits the from the Lexer to provide PLY with the tokenizing
 125 # definitions.  Parsing patterns are encoded as function where p_<name> is
 126 # is called any time a patern matching the function documentation is found.
 127 # Paterns are expressed in the form of:
 128 # """ <new item> : <item> ....
 129 #                | <item> ...."""
 130 #
 131 # Where new item is the result of a match against one or more sets of items
 132 # separated by the "|".
 133 #
 134 # The function is called with an object 'p' where p[0] is the output object
 135 # and p[n] is the set of inputs for positive values of 'n'.  Len(p) can be
 136 # used to distinguish between multiple item sets in the pattern.
 137 #
 138 # For more details on parsing refer to the PLY documentation at
 139 #    http://www.dabeaz.com/ply/
 140 #
 141 #
 142 # The parser uses the following conventions:
 143 #   a <type>_block defines a block of <type> definitions in the form of:
 144 #       [comment] [ext_attr_block] <type> <name> '{' <type>_list '}' ';'
 145 #   A block is reduced by returning an object of <type> with a name of <name>
 146 #   which in turn has <type>_list as children.
 147 #
 148 #   A [comment] is a optional C style comment block enclosed in /* ... */ which
 149 #   is appended to the adjacent node as a child.
 150 #
 151 #   A [ext_attr_block] is an optional list of Extended Attributes which is
 152 #   appended to the adjacent node as a child.
 153 #
 154 #   a <type>_list defines a list of <type> items which will be passed as a
 155 #   list of children to the parent pattern.  A list is in the form of:
 156 #       [comment] [ext_attr_block] <...DEF...> ';' <type>_list | (empty)
 157 # or
 158 #       [comment] [ext_attr_block] <...DEF...> <type>_cont
 159 #
 160 #   In the first form, the list is reduced recursively, where the right side
 161 #   <type>_list is first reduced then joined with pattern currently being
 162 #   matched.  The list is terminated with the (empty) pattern is matched.
 163 #
 164 #   In the second form the list is reduced recursively, where the right side
 165 #   <type>_cont is first reduced then joined with the pattern currently being
 166 #   matched.  The type_<cont> is in the form of:
 167 #       ',' <type>_list | (empty)
 168 #   The <type>_cont form is used to consume the ',' which only occurs when
 169 #   there is more than one object in the list.  The <type>_cont also provides
 170 #   the terminating (empty) definition.
 171 #
 172
 173
 174 class IDLParser(IDLLexer):
 175 # TOP
 176 #
 177 # This pattern defines the top of the parse tree.  The parse tree is in the
 178 # the form of:
 179 #
 180 # top
 181 #   *modifiers
 182 #     *comments
 183 #     *ext_attr_block
 184 #       ext_attr_list
 185 #          attr_arg_list
 186 #   *integer, value
 187 #   *param_list
 188 #   *typeref
 189 #
 190 #   top_list
 191 #     describe_block
 192 #       describe_list
 193 #     enum_block
 194 #       enum_item
 195 #     interface_block
 196 #       member
 197 #     label_block
 198 #       label_item
 199 #     struct_block
 200 #       member
 201 #     typedef_decl
 202 #       typedef_data
 203 #       typedef_func
 204 #
 205 # (* sub matches found at multiple levels and are not truly children of top)
 206 #
 207 # We force all input files to start with two comments.  The first comment is a
 208 # Copyright notice followed by a set of file wide Extended Attributes, followed
 209 # by the file comment and finally by file level patterns.
 210 #
 211   # Find the Copyright, File comment, and optional file wide attributes.  We
 212   # use a match with COMMENT instead of comments to force the token to be
 213   # present.  The extended attributes and the top_list become siblings which
 214   # in turn are children of the file object created from the results of top.
 215   def p_top(self, p):
 216     """top : COMMENT COMMENT ext_attr_block top_list"""
 217
 218     Copyright = self.BuildComment('Copyright', p, 1)
 219     Filedoc = self.BuildComment('Comment', p, 2)
 220
 221     p[0] = ListFromConcat(Copyright, Filedoc, p[3], p[4])
 222     if self.parse_debug: DumpReduction('top', p)
 223
 224   def p_top_short(self, p):
 225     """top : COMMENT ext_attr_block top_list"""
 226     Copyright = self.BuildComment('Copyright', p, 1)
 227     Filedoc = IDLNode('Comment', self.lexobj.filename, p.lineno(2)-1,
 228         p.lexpos(2)-1, [self.BuildAttribute('NAME', ''),
 229           self.BuildAttribute('FORM', 'cc')])
 230     p[0] = ListFromConcat(Copyright, Filedoc, p[2], p[3])
 231     if self.parse_debug: DumpReduction('top', p)
 232
 233   # Build a list of top level items.
 234   def p_top_list(self, p):
 235     """top_list : callback_decl top_list
 236                 | describe_block top_list
 237                 | dictionary_block top_list
 238                 | enum_block top_list
 239                 | inline top_list
 240                 | interface_block top_list
 241                 | label_block top_list
 242                 | namespace top_list
 243                 | struct_block top_list
 244                 | typedef_decl top_list
 245                 | bad_decl top_list
 246                 | """
 247     if len(p) > 2:
 248       p[0] = ListFromConcat(p[1], p[2])
 249     if self.parse_debug: DumpReduction('top_list', p)
 250
 251   # Recover from error and continue parsing at the next top match.
 252   def p_top_error(self, p):
 253     """top_list : error top_list"""
 254     p[0] = p[2]
 255
 256   # Recover from error and continue parsing at the next top match.
 257   def p_bad_decl(self, p):
 258     """bad_decl : modifiers SYMBOL error '}' ';'"""
 259     p[0] = []
 260
 261 #
 262 # Modifier List
 263 #
 264 #
 265   def p_modifiers(self, p):
 266     """modifiers : comments ext_attr_block"""
 267     p[0] = ListFromConcat(p[1], p[2])
 268     if self.parse_debug: DumpReduction('modifiers', p)
 269
 270 #
 271 # Scoped name is a name with an optional scope.
 272 #
 273 # Used for types and namespace names. eg. foo_bar.hello_world, or
 274 # foo_bar.hello_world.SomeType.
 275 #
 276   def p_scoped_name(self, p):
 277     """scoped_name : SYMBOL scoped_name_rest"""
 278     p[0] = ''.join(p[1:])
 279     if self.parse_debug: DumpReduction('scoped_name', p)
 280
 281   def p_scoped_name_rest(self, p):
 282     """scoped_name_rest : '.' scoped_name
 283                         |"""
 284     p[0] = ''.join(p[1:])
 285     if self.parse_debug: DumpReduction('scoped_name_rest', p)
 286
 287 #
 288 # Type reference
 289 #
 290 #
 291   def p_typeref(self, p):
 292     """typeref : scoped_name"""
 293     p[0] = p[1]
 294     if self.parse_debug: DumpReduction('typeref', p)
 295
 296
 297 #
 298 # Comments
 299 #
 300 # Comments are optional list of C style comment objects.  Comments are returned
 301 # as a list or None.
 302 #
 303   def p_comments(self, p):
 304     """comments : COMMENT comments
 305                 | """
 306     if len(p) > 1:
 307       child = self.BuildComment('Comment', p, 1)
 308       p[0] = ListFromConcat(child, p[2])
 309       if self.parse_debug: DumpReduction('comments', p)
 310     else:
 311       if self.parse_debug: DumpReduction('no comments', p)
 312
 313
 314 #
 315 # Namespace
 316 #
 317 # A namespace provides a named scope to an enclosed top_list.
 318 #
 319   def p_namespace(self, p):
 320     """namespace : modifiers NAMESPACE namespace_name '{' top_list '}' ';'"""
 321     children = ListFromConcat(p[1], p[5])
 322     p[0] = self.BuildNamed('Namespace', p, 3, children)
 323
 324   # We allow namespace names of the form foo.bar.baz.
 325   def p_namespace_name(self, p):
 326     """namespace_name : scoped_name"""
 327     p[0] = p[1]
 328
 329
 330 #
 331 # Dictionary
 332 #
 333 # A dictionary is a named list of optional and required members.
 334 #
 335   def p_dictionary_block(self, p):
 336     """dictionary_block : modifiers DICTIONARY SYMBOL '{' struct_list '}' ';'"""
 337     p[0] = self.BuildNamed('Dictionary', p, 3, ListFromConcat(p[1], p[5]))
 338
 339   def p_dictionary_errorA(self, p):
 340     """dictionary_block : modifiers DICTIONARY error ';'"""
 341     p[0] = []
 342
 343   def p_dictionary_errorB(self, p):
 344     """dictionary_block : modifiers DICTIONARY error '{' struct_list '}' ';'"""
 345     p[0] = []
 346
 347 #
 348 # Callback
 349 #
 350 # A callback is essentially a single function declaration (outside of an
 351 # Interface).
 352 #
 353   def p_callback_decl(self, p):
 354     """callback_decl : modifiers CALLBACK SYMBOL '=' SYMBOL param_list ';'"""
 355     children = ListFromConcat(p[1], p[6])
 356     p[0] = self.BuildNamed('Callback', p, 3, children)
 357
 358
 359 #
 360 # Inline
 361 #
 362 # Inline blocks define option code to be emitted based on language tag,
 363 # in the form of:
 364 # #inline <LANGUAGE>
 365 # <CODE>
 366 # #endinl
 367 #
 368   def p_inline(self, p):
 369     """inline : modifiers INLINE"""
 370     words = p[2].split()
 371     name = self.BuildAttribute('NAME', words[1])
 372     lines = p[2].split('\n')
 373     value = self.BuildAttribute('VALUE', '\n'.join(lines[1:-1]) + '\n')
 374     children = ListFromConcat(name, value, p[1])
 375     p[0] = self.BuildProduction('Inline', p, 2, children)
 376     if self.parse_debug: DumpReduction('inline', p)
 377
 378 # Extended Attributes
 379 #
 380 # Extended Attributes denote properties which will be applied to a node in the
 381 # AST.  A list of extended attributes are denoted by a brackets '[' ... ']'
 382 # enclosing a comma separated list of extended attributes in the form of:
 383 #
 384 #  Name
 385 #  Name=HEX | INT | OCT | FLOAT
 386 #  Name="STRING"
 387 #  Name=Function(arg ...)
 388 #  TODO(bradnelson) -Not currently supported:
 389 #  ** Name(arg ...) ...
 390 #  ** Name=Scope::Value
 391 #
 392 # Extended Attributes are returned as a list or None.
 393
 394   def p_ext_attr_block(self, p):
 395     """ext_attr_block : '[' ext_attr_list ']'
 396                   | """
 397     if len(p) > 1:
 398       p[0] = p[2]
 399       if self.parse_debug: DumpReduction('ext_attr_block', p)
 400     else:
 401       if self.parse_debug: DumpReduction('no ext_attr_block', p)
 402
 403   def p_ext_attr_list(self, p):
 404     """ext_attr_list : SYMBOL '=' SYMBOL ext_attr_cont
 405                      | SYMBOL '=' value ext_attr_cont
 406                      | SYMBOL '=' SYMBOL param_list ext_attr_cont
 407                      | SYMBOL ext_attr_cont"""
 408     # If there are 4 tokens plus a return slot, this must be in the form
 409     # SYMBOL = SYMBOL|value ext_attr_cont
 410     if len(p) == 5:
 411       p[0] = ListFromConcat(self.BuildAttribute(p[1], p[3]), p[4])
 412     # If there are 5 tokens plus a return slot, this must be in the form
 413     # SYMBOL = SYMBOL (param_list) ext_attr_cont
 414     elif len(p) == 6:
 415       member = self.BuildNamed('Member', p, 3, [p[4]])
 416       p[0] = ListFromConcat(self.BuildAttribute(p[1], member), p[5])
 417     # Otherwise, this must be: SYMBOL ext_attr_cont
 418     else:
 419       p[0] = ListFromConcat(self.BuildAttribute(p[1], 'True'), p[2])
 420     if self.parse_debug: DumpReduction('ext_attribute_list', p)
 421
 422   def p_ext_attr_list_values(self, p):
 423     """ext_attr_list : SYMBOL '=' '(' values ')' ext_attr_cont
 424                      | SYMBOL '=' '(' symbols ')' ext_attr_cont"""
 425     p[0] = ListFromConcat(self.BuildAttribute(p[1], p[4]), p[6])
 426
 427   def p_values(self, p):
 428     """values : value values_cont"""
 429     p[0] = ListFromConcat(p[1], p[2])
 430
 431   def p_symbols(self, p):
 432     """symbols : SYMBOL symbols_cont"""
 433     p[0] = ListFromConcat(p[1], p[2])
 434
 435   def p_symbols_cont(self, p):
 436     """symbols_cont : ',' SYMBOL symbols_cont
 437                     | """
 438     if len(p) > 1: p[0] = ListFromConcat(p[2], p[3])
 439
 440   def p_values_cont(self, p):
 441     """values_cont : ',' value values_cont
 442                    | """
 443     if len(p) > 1: p[0] = ListFromConcat(p[2], p[3])
 444
 445   def p_ext_attr_cont(self, p):
 446     """ext_attr_cont : ',' ext_attr_list
 447                      |"""
 448     if len(p) > 1: p[0] = p[2]
 449     if self.parse_debug: DumpReduction('ext_attribute_cont', p)
 450
 451   def p_ext_attr_func(self, p):
 452     """ext_attr_list : SYMBOL '(' attr_arg_list ')' ext_attr_cont"""
 453     p[0] = ListFromConcat(self.BuildAttribute(p[1] + '()', p[3]), p[5])
 454     if self.parse_debug: DumpReduction('attr_arg_func', p)
 455
 456   def p_ext_attr_arg_list(self, p):
 457     """attr_arg_list : SYMBOL attr_arg_cont
 458                      | value attr_arg_cont"""
 459     p[0] = ListFromConcat(p[1], p[2])
 460
 461   def p_attr_arg_cont(self, p):
 462     """attr_arg_cont : ',' attr_arg_list
 463                      | """
 464     if self.parse_debug: DumpReduction('attr_arg_cont', p)
 465     if len(p) > 1: p[0] = p[2]
 466
 467   def p_attr_arg_error(self, p):
 468     """attr_arg_cont : error attr_arg_cont"""
 469     p[0] = p[2]
 470     if self.parse_debug: DumpReduction('attr_arg_error', p)
 471
 472
 473 #
 474 # Describe
 475 #
 476 # A describe block is defined at the top level.  It provides a mechanism for
 477 # attributing a group of ext_attr to a describe_list.  Members of the
 478 # describe list are language specific 'Type' declarations
 479 #
 480   def p_describe_block(self, p):
 481     """describe_block : modifiers DESCRIBE '{' describe_list '}' ';'"""
 482     children = ListFromConcat(p[1], p[4])
 483     p[0] = self.BuildProduction('Describe', p, 2, children)
 484     if self.parse_debug: DumpReduction('describe_block', p)
 485
 486   # Recover from describe error and continue parsing at the next top match.
 487   def p_describe_error(self, p):
 488     """describe_list : error describe_list"""
 489     p[0] = []
 490
 491   def p_describe_list(self, p):
 492     """describe_list : modifiers SYMBOL ';' describe_list
 493                      | modifiers ENUM ';' describe_list
 494                      | modifiers STRUCT ';' describe_list
 495                      | modifiers TYPEDEF ';' describe_list
 496                      | """
 497     if len(p) > 1:
 498       Type = self.BuildNamed('Type', p, 2, p[1])
 499       p[0] = ListFromConcat(Type, p[4])
 500
 501 #
 502 # Constant Values (integer, value)
 503 #
 504 # Constant values can be found at various levels.  A Constant value is returns
 505 # as the string value after validated against a FLOAT, HEX, INT, OCT or
 506 # STRING pattern as appropriate.
 507 #
 508   def p_value(self, p):
 509     """value : FLOAT
 510              | HEX
 511              | INT
 512              | OCT
 513              | STRING"""
 514     p[0] = p[1]
 515     if self.parse_debug: DumpReduction('value', p)
 516
 517   def p_value_lshift(self, p):
 518     """value : integer LSHIFT INT"""
 519     p[0] = "%s << %s" % (p[1], p[3])
 520     if self.parse_debug: DumpReduction('value', p)
 521
 522 # Integers are numbers which may not be floats used in cases like array sizes.
 523   def p_integer(self, p):
 524     """integer : HEX
 525                | INT
 526                | OCT"""
 527     p[0] = p[1]
 528     if self.parse_debug: DumpReduction('integer', p)
 529
 530 #
 531 # Expression
 532 #
 533 # A simple arithmetic expression.
 534 #
 535   precedence = (
 536     ('left','|','&','^'),
 537     ('left','LSHIFT','RSHIFT'),
 538     ('left','+','-'),
 539     ('left','*','/'),
 540     ('right','UMINUS','~'),
 541     )
 542
 543   def p_expression_binop(self, p):
 544     """expression : expression LSHIFT expression
 545                   | expression RSHIFT expression
 546                   | expression '|' expression
 547                   | expression '&' expression
 548                   | expression '^' expression
 549                   | expression '+' expression
 550                   | expression '-' expression
 551                   | expression '*' expression
 552                   | expression '/' expression"""
 553     p[0] = "%s %s %s" % (str(p[1]), str(p[2]), str(p[3]))
 554     if self.parse_debug: DumpReduction('expression_binop', p)
 555
 556   def p_expression_unop(self, p):
 557     """expression : '-' expression %prec UMINUS
 558                   | '~' expression %prec '~'"""
 559     p[0] = "%s%s" % (str(p[1]), str(p[2]))
 560     if self.parse_debug: DumpReduction('expression_unop', p)
 561
 562   def p_expression_term(self, p):
 563     """expression : '(' expression ')'"""
 564     p[0] = "%s%s%s" % (str(p[1]), str(p[2]), str(p[3]))
 565     if self.parse_debug: DumpReduction('expression_term', p)
 566
 567   def p_expression_symbol(self, p):
 568     """expression : SYMBOL"""
 569     p[0] = p[1]
 570     if self.parse_debug: DumpReduction('expression_symbol', p)
 571
 572   def p_expression_integer(self, p):
 573     """expression : integer"""
 574     p[0] = p[1]
 575     if self.parse_debug: DumpReduction('expression_integer', p)
 576
 577 #
 578 # Array List
 579 #
 580 # Defined a list of array sizes (if any).
 581 #
 582   def p_arrays(self, p):
 583     """arrays : '[' ']' arrays
 584               | '[' integer ']' arrays
 585               | """
 586     # If there are 3 tokens plus a return slot it is an unsized array
 587     if len(p) == 4:
 588       array = self.BuildProduction('Array', p, 1)
 589       p[0] = ListFromConcat(array, p[3])
 590     # If there are 4 tokens plus a return slot it is a fixed array
 591     elif len(p) == 5:
 592       count = self.BuildAttribute('FIXED', p[2])
 593       array = self.BuildProduction('Array', p, 2, [count])
 594       p[0] = ListFromConcat(array, p[4])
 595     # If there is only a return slot, do not fill it for this terminator.
 596     elif len(p) == 1: return
 597     if self.parse_debug: DumpReduction('arrays', p)
 598
 599
 600 # An identifier is a legal value for a parameter or attribute name. Lots of
 601 # existing IDL files use "callback" as a parameter/attribute name, so we allow
 602 # a SYMBOL or the CALLBACK keyword.
 603   def p_identifier(self, p):
 604     """identifier : SYMBOL
 605                   | CALLBACK"""
 606     p[0] = p[1]
 607     # Save the line number of the underlying token (otherwise it gets
 608     # discarded), since we use it in the productions with an identifier in
 609     # them.
 610     p.set_lineno(0, p.lineno(1))
 611
 612
 613 #
 614 # Union
 615 #
 616 # A union allows multiple choices of types for a parameter or member.
 617 #
 618
 619   def p_union_option(self, p):
 620     """union_option : modifiers SYMBOL arrays"""
 621     typeref = self.BuildAttribute('TYPEREF', p[2])
 622     children = ListFromConcat(p[1], typeref, p[3])
 623     p[0] = self.BuildProduction('Option', p, 2, children)
 624
 625   def p_union_list(self, p):
 626     """union_list : union_option OR union_list
 627                   | union_option"""
 628     if len(p) > 2:
 629       p[0] = ListFromConcat(p[1], p[3])
 630     else:
 631       p[0] = p[1]
 632
 633 #
 634 # Parameter List
 635 #
 636 # A parameter list is a collection of arguments which are passed to a
 637 # function.
 638 #
 639   def p_param_list(self, p):
 640     """param_list : '(' param_item param_cont ')'
 641                   | '(' ')' """
 642     if len(p) > 3:
 643       args = ListFromConcat(p[2], p[3])
 644     else:
 645       args = []
 646     p[0] = self.BuildProduction('Callspec', p, 1, args)
 647     if self.parse_debug: DumpReduction('param_list', p)
 648
 649   def p_param_item(self, p):
 650     """param_item : modifiers optional typeref arrays identifier"""
 651     typeref = self.BuildAttribute('TYPEREF', p[3])
 652     children = ListFromConcat(p[1], p[2], typeref, p[4])
 653     p[0] = self.BuildNamed('Param', p, 5, children)
 654     if self.parse_debug: DumpReduction('param_item', p)
 655
 656   def p_param_item_union(self, p):
 657     """param_item : modifiers optional '(' union_list ')' identifier"""
 658     union = self.BuildAttribute('Union', True)
 659     children = ListFromConcat(p[1], p[2], p[4], union)
 660     p[0] = self.BuildNamed('Param', p, 6, children)
 661     if self.parse_debug: DumpReduction('param_item', p)
 662
 663   def p_optional(self, p):
 664     """optional : OPTIONAL
 665                 | """
 666     if len(p) == 2:
 667       p[0] = self.BuildAttribute('OPTIONAL', True)
 668
 669
 670   def p_param_cont(self, p):
 671     """param_cont : ',' param_item param_cont
 672                   | """
 673     if len(p) > 1:
 674       p[0] = ListFromConcat(p[2], p[3])
 675       if self.parse_debug: DumpReduction('param_cont', p)
 676
 677   def p_param_error(self, p):
 678     """param_cont : error param_cont"""
 679     p[0] = p[2]
 680
 681
 682 #
 683 # Typedef
 684 #
 685 # A typedef creates a new referencable type.  The typedef can specify an array
 686 # definition as well as a function declaration.
 687 #
 688   def p_typedef_data(self, p):
 689     """typedef_decl : modifiers TYPEDEF SYMBOL SYMBOL ';' """
 690     typeref = self.BuildAttribute('TYPEREF', p[3])
 691     children = ListFromConcat(p[1], typeref)
 692     p[0] = self.BuildNamed('Typedef', p, 4, children)
 693     if self.parse_debug: DumpReduction('typedef_data', p)
 694
 695   def p_typedef_array(self, p):
 696     """typedef_decl : modifiers TYPEDEF SYMBOL arrays SYMBOL ';' """
 697     typeref = self.BuildAttribute('TYPEREF', p[3])
 698     children = ListFromConcat(p[1], typeref, p[4])
 699     p[0] = self.BuildNamed('Typedef', p, 5, children)
 700     if self.parse_debug: DumpReduction('typedef_array', p)
 701
 702   def p_typedef_func(self, p):
 703     """typedef_decl : modifiers TYPEDEF SYMBOL SYMBOL param_list ';' """
 704     typeref = self.BuildAttribute('TYPEREF', p[3])
 705     children = ListFromConcat(p[1], typeref, p[5])
 706     p[0] = self.BuildNamed('Typedef', p, 4, children)
 707     if self.parse_debug: DumpReduction('typedef_func', p)
 708
 709 #
 710 # Enumeration
 711 #
 712 # An enumeration is a set of named integer constants.  An enumeration
 713 # is valid type which can be referenced in other definitions.
 714 #
 715   def p_enum_block(self, p):
 716     """enum_block : modifiers ENUM SYMBOL '{' enum_list '}' ';'"""
 717     p[0] = self.BuildNamed('Enum', p, 3, ListFromConcat(p[1], p[5]))
 718     if self.parse_debug: DumpReduction('enum_block', p)
 719
 720   # Recover from enum error and continue parsing at the next top match.
 721   def p_enum_errorA(self, p):
 722     """enum_block : modifiers ENUM error '{' enum_list '}' ';'"""
 723     p[0] = []
 724
 725   def p_enum_errorB(self, p):
 726     """enum_block : modifiers ENUM error ';'"""
 727     p[0] = []
 728
 729   def p_enum_list(self, p):
 730     """enum_list : modifiers SYMBOL '=' expression enum_cont
 731                  | modifiers SYMBOL enum_cont"""
 732     if len(p) > 4:
 733       val  = self.BuildAttribute('VALUE', p[4])
 734       enum = self.BuildNamed('EnumItem', p, 2, ListFromConcat(val, p[1]))
 735       p[0] = ListFromConcat(enum, p[5])
 736     else:
 737       enum = self.BuildNamed('EnumItem', p, 2, p[1])
 738       p[0] = ListFromConcat(enum, p[3])
 739     if self.parse_debug: DumpReduction('enum_list', p)
 740
 741   def p_enum_cont(self, p):
 742     """enum_cont : ',' enum_list
 743                  |"""
 744     if len(p) > 1: p[0] = p[2]
 745     if self.parse_debug: DumpReduction('enum_cont', p)
 746
 747   def p_enum_cont_error(self, p):
 748     """enum_cont : error enum_cont"""
 749     p[0] = p[2]
 750     if self.parse_debug: DumpReduction('enum_error', p)
 751
 752
 753 #
 754 # Label
 755 #
 756 # A label is a special kind of enumeration which allows us to go from a
 757 # set of labels
 758 #
 759   def p_label_block(self, p):
 760     """label_block : modifiers LABEL SYMBOL '{' label_list '}' ';'"""
 761     p[0] = self.BuildNamed('Label', p, 3, ListFromConcat(p[1], p[5]))
 762     if self.parse_debug: DumpReduction('label_block', p)
 763
 764   def p_label_list(self, p):
 765     """label_list : modifiers SYMBOL '=' FLOAT label_cont"""
 766     val  = self.BuildAttribute('VALUE', p[4])
 767     label = self.BuildNamed('LabelItem', p, 2, ListFromConcat(val, p[1]))
 768     p[0] = ListFromConcat(label, p[5])
 769     if self.parse_debug: DumpReduction('label_list', p)
 770
 771   def p_label_cont(self, p):
 772     """label_cont : ',' label_list
 773                  |"""
 774     if len(p) > 1: p[0] = p[2]
 775     if self.parse_debug: DumpReduction('label_cont', p)
 776
 777   def p_label_cont_error(self, p):
 778     """label_cont : error label_cont"""
 779     p[0] = p[2]
 780     if self.parse_debug: DumpReduction('label_error', p)
 781
 782
 783 #
 784 # Members
 785 #
 786 # A member attribute or function of a struct or interface.
 787 #
 788   def p_member_attribute(self, p):
 789     """member_attribute : modifiers typeref arrays questionmark identifier"""
 790     typeref = self.BuildAttribute('TYPEREF', p[2])
 791     children = ListFromConcat(p[1], typeref, p[3], p[4])
 792     p[0] = self.BuildNamed('Member', p, 5, children)
 793     if self.parse_debug: DumpReduction('attribute', p)
 794
 795   def p_member_attribute_union(self, p):
 796     """member_attribute : modifiers '(' union_list ')' questionmark identifier"""
 797     union = self.BuildAttribute('Union', True)
 798     children = ListFromConcat(p[1], p[3], p[5], union)
 799     p[0] = self.BuildNamed('Member', p, 6, children)
 800     if self.parse_debug: DumpReduction('attribute', p)
 801
 802   def p_member_function(self, p):
 803     """member_function : modifiers static typeref arrays SYMBOL param_list"""
 804     typeref = self.BuildAttribute('TYPEREF', p[3])
 805     children = ListFromConcat(p[1], p[2], typeref, p[4], p[6])
 806     p[0] = self.BuildNamed('Member', p, 5, children)
 807     if self.parse_debug: DumpReduction('function', p)
 808
 809   def p_static(self, p):
 810     """static : STATIC
 811               | """
 812     if len(p) == 2:
 813       p[0] = self.BuildAttribute('STATIC', True)
 814
 815   def p_questionmark(self, p):
 816     """questionmark : '?'
 817                     | """
 818     if len(p) == 2:
 819       p[0] = self.BuildAttribute('OPTIONAL', True)
 820
 821 #
 822 # Interface
 823 #
 824 # An interface is a named collection of functions.
 825 #
 826   def p_interface_block(self, p):
 827     """interface_block : modifiers INTERFACE SYMBOL '{' interface_list '}' ';'"""
 828     p[0] = self.BuildNamed('Interface', p, 3, ListFromConcat(p[1], p[5]))
 829     if self.parse_debug: DumpReduction('interface_block', p)
 830
 831   def p_interface_error(self, p):
 832     """interface_block : modifiers INTERFACE error '{' interface_list '}' ';'"""
 833     p[0] = []
 834
 835   def p_interface_list(self, p):
 836     """interface_list : member_function ';' interface_list
 837                       | """
 838     if len(p) > 1 :
 839       p[0] = ListFromConcat(p[1], p[3])
 840       if self.parse_debug: DumpReduction('interface_list', p)
 841
 842
 843 #
 844 # Struct
 845 #
 846 # A struct is a named collection of members which in turn reference other
 847 # types.  The struct is a referencable type.
 848 #
 849   def p_struct_block(self, p):
 850     """struct_block : modifiers STRUCT SYMBOL '{' struct_list '}' ';'"""
 851     children = ListFromConcat(p[1], p[5])
 852     p[0] = self.BuildNamed('Struct', p, 3, children)
 853     if self.parse_debug: DumpReduction('struct_block', p)
 854
 855   # Recover from struct error and continue parsing at the next top match.
 856   def p_struct_error(self, p):
 857     """enum_block : modifiers STRUCT error '{' struct_list '}' ';'"""
 858     p[0] = []
 859
 860   def p_struct_list(self, p):
 861     """struct_list : member_attribute ';' struct_list
 862                    | member_function ';' struct_list
 863                    |"""
 864     if len(p) > 1: p[0] = ListFromConcat(p[1], p[3])
 865
 866
 867 #
 868 # Parser Errors
 869 #
 870 # p_error is called whenever the parser can not find a pattern match for
 871 # a set of items from the current state.  The p_error function defined here
 872 # is triggered logging an error, and parsing recover happens as the
 873 # p_<type>_error functions defined above are called.  This allows the parser
 874 # to continue so as to capture more than one error per file.
 875 #
 876   def p_error(self, t):
 877     filename = self.lexobj.filename
 878     self.parse_errors += 1
 879     if t:
 880       lineno = t.lineno
 881       pos = t.lexpos
 882       prev = self.yaccobj.symstack[-1]
 883       if type(prev) == lex.LexToken:
 884         msg = "Unexpected %s after %s." % (
 885             TokenTypeName(t), TokenTypeName(prev))
 886       else:
 887         msg = "Unexpected %s." % (t.value)
 888     else:
 889       lineno = self.last.lineno
 890       pos = self.last.lexpos
 891       msg = "Unexpected end of file after %s." % TokenTypeName(self.last)
 892       self.yaccobj.restart()
 893
 894     # Attempt to remap the error to a friendlier form
 895     if msg in ERROR_REMAP:
 896       msg = ERROR_REMAP[msg]
 897
 898     # Log the error
 899     ErrOut.LogLine(filename, lineno, pos, msg)
 900
 901   def Warn(self, node, msg):
 902     WarnOut.LogLine(node.filename, node.lineno, node.pos, msg)
 903     self.parse_warnings += 1
 904
 905   def __init__(self):
 906     IDLLexer.__init__(self)
 907     self.yaccobj = yacc.yacc(module=self, tabmodule=None, debug=False,
 908                              optimize=0, write_tables=0)
 909
 910     self.build_debug = GetOption('build_debug')
 911     self.parse_debug = GetOption('parse_debug')
 912     self.token_debug = GetOption('token_debug')
 913     self.verbose = GetOption('verbose')
 914     self.parse_errors = 0
 915
 916 #
 917 # Tokenizer
 918 #
 919 # The token function returns the next token provided by IDLLexer for matching
 920 # against the leaf paterns.
 921 #
 922   def token(self):
 923     tok = self.lexobj.token()
 924     if tok:
 925       self.last = tok
 926       if self.token_debug:
 927         InfoOut.Log("TOKEN %s(%s)" % (tok.type, tok.value))
 928     return tok
 929
 930 #
 931 # BuildProduction
 932 #
 933 # Production is the set of items sent to a grammar rule resulting in a new
 934 # item being returned.
 935 #
 936 # p - Is the Yacc production object containing the stack of items
 937 # index - Index into the production of the name for the item being produced.
 938 # cls - The type of item being producted
 939 # childlist - The children of the new item
 940   def BuildProduction(self, cls, p, index, childlist=None):
 941     if not childlist: childlist = []
 942     filename = self.lexobj.filename
 943     lineno = p.lineno(index)
 944     pos = p.lexpos(index)
 945     out = IDLNode(cls, filename, lineno, pos, childlist)
 946     if self.build_debug:
 947       InfoOut.Log("Building %s" % out)
 948     return out
 949
 950   def BuildNamed(self, cls, p, index, childlist=None):
 951     if not childlist: childlist = []
 952     childlist.append(self.BuildAttribute('NAME', p[index]))
 953     return self.BuildProduction(cls, p, index, childlist)
 954
 955   def BuildComment(self, cls, p, index):
 956     name = p[index]
 957
 958     # Remove comment markers
 959     lines = []
 960     if name[:2] == '//':
 961       # For C++ style, remove any leading whitespace and the '//' marker from
 962       # each line.
 963       form = 'cc'
 964       for line in name.split('\n'):
 965         start = line.find('//')
 966         lines.append(line[start+2:])
 967     else:
 968       # For C style, remove ending '*/''
 969       form = 'c'
 970       for line in name[:-2].split('\n'):
 971         # Remove characters until start marker for this line '*' if found
 972         # otherwise it should be blank.
 973         offs = line.find('*')
 974         if offs >= 0:
 975           line = line[offs + 1:].rstrip()
 976         else:
 977           line = ''
 978         lines.append(line)
 979     name = '\n'.join(lines)
 980
 981     childlist = [self.BuildAttribute('NAME', name),
 982                  self.BuildAttribute('FORM', form)]
 983     return self.BuildProduction(cls, p, index, childlist)
 984
 985 #
 986 # BuildAttribute
 987 #
 988 # An ExtendedAttribute is a special production that results in a property
 989 # which is applied to the adjacent item.  Attributes have no children and
 990 # instead represent key/value pairs.
 991 #
 992   def BuildAttribute(self, key, val):
 993     return IDLAttribute(key, val)
 994
 995
 996 #
 997 # ParseData
 998 #
 999 # Attempts to parse the current data loaded in the lexer.
1000 #
1001   def ParseData(self, data, filename='<Internal>'):
1002     self.SetData(filename, data)
1003     try:
1004       self.parse_errors = 0
1005       self.parse_warnings = 0
1006       return self.yaccobj.parse(lexer=self)
1007
1008     except lex.LexError as le:
1009       ErrOut.Log(str(le))
1010       return []
1011
1012 #
1013 # ParseFile
1014 #
1015 # Loads a new file into the lexer and attemps to parse it.
1016 #
1017   def ParseFile(self, filename):
1018     date = time.ctime(os.path.getmtime(filename))
1019     data = open(filename).read()
1020     if self.verbose:
1021       InfoOut.Log("Parsing %s" % filename)
1022     try:
1023       out = self.ParseData(data, filename)
1024
1025       # If we have a src root specified, remove it from the path
1026       srcroot = GetOption('srcroot')
1027       if srcroot and filename.find(srcroot) == 0:
1028         filename = filename[len(srcroot) + 1:]
1029       filenode = IDLFile(filename, out, self.parse_errors + self.lex_errors)
1030       filenode.SetProperty('DATETIME', date)
1031       return filenode
1032
1033     except Exception as e:
1034       ErrOut.LogLine(filename, self.last.lineno, self.last.lexpos,
1035                      'Internal parsing error - %s.' % str(e))
1036       raise
1037
1038
1039
1040 #
1041 # Flatten Tree
1042 #
1043 # Flattens the tree of IDLNodes for use in testing.
1044 #
1045 def FlattenTree(node):
1046   add_self = False
1047   out = []
1048   for child in node.GetChildren():
1049     if child.IsA('Comment'):
1050       add_self = True
1051     else:
1052       out.extend(FlattenTree(child))
1053
1054   if add_self:
1055     out = [str(node)] + out
1056   return out
1057
1058
1059 def TestErrors(filename, filenode):
1060   nodelist = filenode.GetChildren()
1061
1062   lexer = IDLLexer()
1063   data = open(filename).read()
1064   lexer.SetData(filename, data)
1065
1066   pass_comments = []
1067   fail_comments = []
1068   while True:
1069     tok = lexer.lexobj.token()
1070     if tok == None: break
1071     if tok.type == 'COMMENT':
1072       args = tok.value[3:-3].split()
1073       if args[0] == 'OK':
1074         pass_comments.append((tok.lineno, ' '.join(args[1:])))
1075       else:
1076         if args[0] == 'FAIL':
1077           fail_comments.append((tok.lineno, ' '.join(args[1:])))
1078   obj_list = []
1079   for node in nodelist:
1080     obj_list.extend(FlattenTree(node))
1081
1082   errors = 0
1083
1084   #
1085   # Check for expected successes
1086   #
1087   obj_cnt = len(obj_list)
1088   pass_cnt = len(pass_comments)
1089   if obj_cnt != pass_cnt:
1090     InfoOut.Log("Mismatched pass (%d) vs. nodes built (%d)."
1091         % (pass_cnt, obj_cnt))
1092     InfoOut.Log("PASS: %s" % [x[1] for x in pass_comments])
1093     InfoOut.Log("OBJS: %s" % obj_list)
1094     errors += 1
1095     if pass_cnt > obj_cnt: pass_cnt = obj_cnt
1096
1097   for i in range(pass_cnt):
1098     line, comment = pass_comments[i]
1099     if obj_list[i] != comment:
1100       ErrOut.LogLine(filename, line, None, "OBJ %s : EXPECTED %s\n" %
1101                      (obj_list[i], comment))
1102       errors += 1
1103
1104   #
1105   # Check for expected errors
1106   #
1107   err_list = ErrOut.DrainLog()
1108   err_cnt = len(err_list)
1109   fail_cnt = len(fail_comments)
1110   if err_cnt != fail_cnt:
1111     InfoOut.Log("Mismatched fail (%d) vs. errors seen (%d)."
1112         % (fail_cnt, err_cnt))
1113     InfoOut.Log("FAIL: %s" % [x[1] for x in fail_comments])
1114     InfoOut.Log("ERRS: %s" % err_list)
1115     errors += 1
1116     if fail_cnt > err_cnt:  fail_cnt = err_cnt
1117
1118   for i in range(fail_cnt):
1119     line, comment = fail_comments[i]
1120     err = err_list[i].strip()
1121
1122     if err_list[i] != comment:
1123       ErrOut.Log("%s(%d) Error\n\tERROR : %s\n\tEXPECT: %s" % (
1124         filename, line, err_list[i], comment))
1125       errors += 1
1126
1127   # Clear the error list for the next run
1128   err_list = []
1129   return errors
1130
1131
1132 def TestFile(parser, filename):
1133   # Capture errors instead of reporting them so we can compare them
1134   # with the expected errors.
1135   ErrOut.SetConsole(False)
1136   ErrOut.SetCapture(True)
1137
1138   filenode = parser.ParseFile(filename)
1139
1140   # Renable output
1141   ErrOut.SetConsole(True)
1142   ErrOut.SetCapture(False)
1143
1144   # Compare captured errors
1145   return TestErrors(filename, filenode)
1146
1147
1148 def TestErrorFiles(filter):
1149   idldir = os.path.split(sys.argv[0])[0]
1150   idldir = os.path.join(idldir, 'test_parser', '*.idl')
1151   filenames = glob.glob(idldir)
1152   parser = IDLParser()
1153   total_errs = 0
1154   for filename in filenames:
1155     if filter and filename not in filter: continue
1156     errs = TestFile(parser, filename)
1157     if errs:
1158       ErrOut.Log("%s test failed with %d error(s)." % (filename, errs))
1159       total_errs += errs
1160
1161   if total_errs:
1162     ErrOut.Log("Failed parsing test.")
1163   else:
1164     InfoOut.Log("Passed parsing test.")
1165   return total_errs
1166
1167
1168 def TestNamespaceFiles(filter):
1169   idldir = os.path.split(sys.argv[0])[0]
1170   idldir = os.path.join(idldir, 'test_namespace', '*.idl')
1171   filenames = glob.glob(idldir)
1172   testnames = []
1173
1174   for filename in filenames:
1175     if filter and filename not in filter: continue
1176     testnames.append(filename)
1177
1178   # If we have no files to test, then skip this test
1179   if not testnames:
1180     InfoOut.Log('No files to test for namespace.')
1181     return 0
1182
1183   InfoOut.SetConsole(False)
1184   ast = ParseFiles(testnames)
1185   InfoOut.SetConsole(True)
1186
1187   errs = ast.GetProperty('ERRORS')
1188   if errs:
1189     ErrOut.Log("Failed namespace test.")
1190   else:
1191     InfoOut.Log("Passed namespace test.")
1192   return errs
1193
1194
1195
1196 def FindVersionError(releases, node):
1197   err_cnt = 0
1198   if node.IsA('Interface', 'Struct'):
1199     comment_list = []
1200     comment = node.GetOneOf('Comment')
1201     if comment and comment.GetName()[:4] == 'REL:':
1202       comment_list = comment.GetName()[5:].strip().split(' ')
1203
1204     first_list = [node.first_release[rel] for rel in releases]
1205     first_list = sorted(set(first_list))
1206     if first_list != comment_list:
1207       node.Error("Mismatch in releases: %s vs %s." % (
1208           comment_list, first_list))
1209       err_cnt += 1
1210
1211   for child in node.GetChildren():
1212     err_cnt += FindVersionError(releases, child)
1213   return err_cnt
1214
1215
1216 def TestVersionFiles(filter):
1217   idldir = os.path.split(sys.argv[0])[0]
1218   idldir = os.path.join(idldir, 'test_version', '*.idl')
1219   filenames = glob.glob(idldir)
1220   testnames = []
1221
1222   for filename in filenames:
1223     if filter and filename not in filter: continue
1224     testnames.append(filename)
1225
1226   # If we have no files to test, then skip this test
1227   if not testnames:
1228     InfoOut.Log('No files to test for version.')
1229     return 0
1230
1231   ast = ParseFiles(testnames)
1232   errs = FindVersionError(ast.releases, ast)
1233   errs += ast.errors
1234
1235   if errs:
1236     ErrOut.Log("Failed version test.")
1237   else:
1238     InfoOut.Log("Passed version test.")
1239   return errs
1240
1241
1242 default_dirs = ['.', 'trusted', 'dev', 'private']
1243 def ParseFiles(filenames):
1244   parser = IDLParser()
1245   filenodes = []
1246
1247   if not filenames:
1248     filenames = []
1249     srcroot = GetOption('srcroot')
1250     dirs = default_dirs
1251     if GetOption('include_private'):
1252       dirs += ['private']
1253     for dirname in dirs:
1254       srcdir = os.path.join(srcroot, dirname, '*.idl')
1255       srcdir = os.path.normpath(srcdir)
1256       filenames += sorted(glob.glob(srcdir))
1257
1258   if not filenames:
1259     ErrOut.Log('No sources provided.')
1260
1261   for filename in filenames:
1262     filenode = parser.ParseFile(filename)
1263     filenodes.append(filenode)
1264
1265   ast = IDLAst(filenodes)
1266   if GetOption('dump_tree'): ast.Dump(0)
1267
1268   Lint(ast)
1269   return ast
1270
1271
1272 def Main(args):
1273   filenames = ParseOptions(args)
1274
1275   # If testing...
1276   if GetOption('test'):
1277     errs = TestErrorFiles(filenames)
1278     errs = TestNamespaceFiles(filenames)
1279     errs = TestVersionFiles(filenames)
1280     if errs:
1281       ErrOut.Log("Parser failed with %d errors." % errs)
1282       return  -1
1283     return 0
1284
1285   # Otherwise, build the AST
1286   ast = ParseFiles(filenames)
1287   errs = ast.GetProperty('ERRORS')
1288   if errs:
1289     ErrOut.Log('Found %d error(s).' % errs);
1290   InfoOut.Log("%d files processed." % len(filenames))
1291   return errs
1292
1293
1294 if __name__ == '__main__':
1295   sys.exit(Main(sys.argv[1:]))
1296