ppapi/generators/idl_parser.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """ Parser for PPAPI IDL """
   7
   8 #
   9 # IDL Parser
  10 #
  11 # The parser is uses the PLY yacc library to build a set of parsing rules based
  12 # on WebIDL.
  13 #
  14 # WebIDL, and WebIDL regular expressions can be found at:
  15 #   http://dev.w3.org/2006/webapi/WebIDL/
  16 # PLY can be found at:
  17 #   http://www.dabeaz.com/ply/
  18 #
  19 # The parser generates a tree by recursively matching sets of items against
  20 # defined patterns.  When a match is made, that set of items is reduced
  21 # to a new item.   The new item can provide a match for parent patterns.
  22 # In this way an AST is built (reduced) depth first.
  23
  24
  25 import getopt
  26 import glob
  27 import os.path
  28 import re
  29 import sys
  30 import time
  31
  32 from idl_ast import IDLAst
  33 from idl_log import ErrOut, InfoOut, WarnOut
  34 from idl_lexer import IDLLexer
  35 from idl_node import IDLAttribute, IDLFile, IDLNode
  36 from idl_option import GetOption, Option, ParseOptions
  37 from idl_lint import Lint
  38 from idl_visitor import IDLVisitor
  39
  40 from ply import lex
  41 from ply import yacc
  42
  43 Option('build_debug', 'Debug tree building.')
  44 Option('parse_debug', 'Debug parse reduction steps.')
  45 Option('token_debug', 'Debug token generation.')
  46 Option('dump_tree', 'Dump the tree.')
  47 Option('srcroot', 'Working directory.', default=os.path.join('..', 'api'))
  48 Option('include_private', 'Include private IDL directory in default API paths.')
  49
  50 #
  51 # ERROR_REMAP
  52 #
  53 # Maps the standard error formula into a more friendly error message.
  54 #
  55 ERROR_REMAP = {
  56   'Unexpected ")" after "(".' : 'Empty argument list.',
  57   'Unexpected ")" after ",".' : 'Missing argument.',
  58   'Unexpected "}" after ",".' : 'Trailing comma in block.',
  59   'Unexpected "}" after "{".' : 'Unexpected empty block.',
  60   'Unexpected comment after "}".' : 'Unexpected trailing comment.',
  61   'Unexpected "{" after keyword "enum".' : 'Enum missing name.',
  62   'Unexpected "{" after keyword "struct".' : 'Struct missing name.',
  63   'Unexpected "{" after keyword "interface".' : 'Interface missing name.',
  64 }
  65
  66 # DumpReduction
  67 #
  68 # Prints out the set of items which matched a particular pattern and the
  69 # new item or set it was reduced to.
  70 def DumpReduction(cls, p):
  71   if p[0] is None:
  72     InfoOut.Log("OBJ: %s(%d) - None\n" % (cls, len(p)))
  73     InfoOut.Log("  [%s]\n" % [str(x) for x in p[1:]])
  74   else:
  75     out = ""
  76     for index in range(len(p) - 1):
  77       out += " >%s< " % str(p[index + 1])
  78     InfoOut.Log("OBJ: %s(%d) - %s : %s\n"  % (cls, len(p), str(p[0]), out))
  79
  80
  81 # CopyToList
  82 #
  83 # Takes an input item, list, or None, and returns a new list of that set.
  84 def CopyToList(item):
  85   # If the item is 'Empty' make it an empty list
  86   if not item: item = []
  87
  88   # If the item is not a list
  89   if type(item) is not type([]): item = [item]
  90
  91   # Make a copy we can modify
  92   return list(item)
  93
  94
  95
  96 # ListFromConcat
  97 #
  98 # Generate a new List by joining of two sets of inputs which can be an
  99 # individual item, a list of items, or None.
 100 def ListFromConcat(*items):
 101   itemsout = []
 102   for item in items:
 103     itemlist = CopyToList(item)
 104     itemsout.extend(itemlist)
 105
 106   return itemsout
 107
 108
 109 # TokenTypeName
 110 #
 111 # Generate a string which has the type and value of the token.
 112 def TokenTypeName(t):
 113   if t.type == 'SYMBOL':  return 'symbol %s' % t.value
 114   if t.type in ['HEX', 'INT', 'OCT', 'FLOAT']:
 115     return 'value %s' % t.value
 116   if t.type == 'STRING' : return 'string "%s"' % t.value
 117   if t.type == 'COMMENT' : return 'comment'
 118   if t.type == t.value: return '"%s"' % t.value
 119   return 'keyword "%s"' % t.value
 120
 121
 122 #
 123 # IDL Parser
 124 #
 125 # The Parser inherits the from the Lexer to provide PLY with the tokenizing
 126 # definitions.  Parsing patterns are encoded as function where p_<name> is
 127 # is called any time a patern matching the function documentation is found.
 128 # Paterns are expressed in the form of:
 129 # """ <new item> : <item> ....
 130 #                | <item> ...."""
 131 #
 132 # Where new item is the result of a match against one or more sets of items
 133 # separated by the "|".
 134 #
 135 # The function is called with an object 'p' where p[0] is the output object
 136 # and p[n] is the set of inputs for positive values of 'n'.  Len(p) can be
 137 # used to distinguish between multiple item sets in the pattern.
 138 #
 139 # For more details on parsing refer to the PLY documentation at
 140 #    http://www.dabeaz.com/ply/
 141 #
 142 #
 143 # The parser uses the following conventions:
 144 #   a <type>_block defines a block of <type> definitions in the form of:
 145 #       [comment] [ext_attr_block] <type> <name> '{' <type>_list '}' ';'
 146 #   A block is reduced by returning an object of <type> with a name of <name>
 147 #   which in turn has <type>_list as children.
 148 #
 149 #   A [comment] is a optional C style comment block enclosed in /* ... */ which
 150 #   is appended to the adjacent node as a child.
 151 #
 152 #   A [ext_attr_block] is an optional list of Extended Attributes which is
 153 #   appended to the adjacent node as a child.
 154 #
 155 #   a <type>_list defines a list of <type> items which will be passed as a
 156 #   list of children to the parent pattern.  A list is in the form of:
 157 #       [comment] [ext_attr_block] <...DEF...> ';' <type>_list | (empty)
 158 # or
 159 #       [comment] [ext_attr_block] <...DEF...> <type>_cont
 160 #
 161 #   In the first form, the list is reduced recursively, where the right side
 162 #   <type>_list is first reduced then joined with pattern currently being
 163 #   matched.  The list is terminated with the (empty) pattern is matched.
 164 #
 165 #   In the second form the list is reduced recursively, where the right side
 166 #   <type>_cont is first reduced then joined with the pattern currently being
 167 #   matched.  The type_<cont> is in the form of:
 168 #       ',' <type>_list | (empty)
 169 #   The <type>_cont form is used to consume the ',' which only occurs when
 170 #   there is more than one object in the list.  The <type>_cont also provides
 171 #   the terminating (empty) definition.
 172 #
 173
 174
 175 class IDLParser(IDLLexer):
 176 # TOP
 177 #
 178 # This pattern defines the top of the parse tree.  The parse tree is in the
 179 # the form of:
 180 #
 181 # top
 182 #   *modifiers
 183 #     *comments
 184 #     *ext_attr_block
 185 #       ext_attr_list
 186 #          attr_arg_list
 187 #   *integer, value
 188 #   *param_list
 189 #   *typeref
 190 #
 191 #   top_list
 192 #     describe_block
 193 #       describe_list
 194 #     enum_block
 195 #       enum_item
 196 #     interface_block
 197 #       member
 198 #     label_block
 199 #       label_item
 200 #     struct_block
 201 #       member
 202 #     typedef_decl
 203 #       typedef_data
 204 #       typedef_func
 205 #
 206 # (* sub matches found at multiple levels and are not truly children of top)
 207 #
 208 # We force all input files to start with two comments.  The first comment is a
 209 # Copyright notice followed by a set of file wide Extended Attributes, followed
 210 # by the file comment and finally by file level patterns.
 211 #
 212   # Find the Copyright, File comment, and optional file wide attributes.  We
 213   # use a match with COMMENT instead of comments to force the token to be
 214   # present.  The extended attributes and the top_list become siblings which
 215   # in turn are children of the file object created from the results of top.
 216   def p_top(self, p):
 217     """top : COMMENT COMMENT ext_attr_block top_list"""
 218
 219     Copyright = self.BuildComment('Copyright', p, 1)
 220     Filedoc = self.BuildComment('Comment', p, 2)
 221
 222     p[0] = ListFromConcat(Copyright, Filedoc, p[3], p[4])
 223     if self.parse_debug: DumpReduction('top', p)
 224
 225   def p_top_short(self, p):
 226     """top : COMMENT ext_attr_block top_list"""
 227     Copyright = self.BuildComment('Copyright', p, 1)
 228     Filedoc = IDLNode('Comment', self.lexobj.filename, p.lineno(2)-1,
 229         p.lexpos(2)-1, [self.BuildAttribute('NAME', ''),
 230           self.BuildAttribute('FORM', 'cc')])
 231     p[0] = ListFromConcat(Copyright, Filedoc, p[2], p[3])
 232     if self.parse_debug: DumpReduction('top', p)
 233
 234   # Build a list of top level items.
 235   def p_top_list(self, p):
 236     """top_list : callback_decl top_list
 237                 | describe_block top_list
 238                 | dictionary_block top_list
 239                 | enum_block top_list
 240                 | inline top_list
 241                 | interface_block top_list
 242                 | label_block top_list
 243                 | namespace top_list
 244                 | struct_block top_list
 245                 | typedef_decl top_list
 246                 | bad_decl top_list
 247                 | """
 248     if len(p) > 2:
 249       p[0] = ListFromConcat(p[1], p[2])
 250     if self.parse_debug: DumpReduction('top_list', p)
 251
 252   # Recover from error and continue parsing at the next top match.
 253   def p_top_error(self, p):
 254     """top_list : error top_list"""
 255     p[0] = p[2]
 256
 257   # Recover from error and continue parsing at the next top match.
 258   def p_bad_decl(self, p):
 259     """bad_decl : modifiers SYMBOL error '}' ';'"""
 260     p[0] = []
 261
 262 #
 263 # Modifier List
 264 #
 265 #
 266   def p_modifiers(self, p):
 267     """modifiers : comments ext_attr_block"""
 268     p[0] = ListFromConcat(p[1], p[2])
 269     if self.parse_debug: DumpReduction('modifiers', p)
 270
 271 #
 272 # Comments
 273 #
 274 # Comments are optional list of C style comment objects.  Comments are returned
 275 # as a list or None.
 276 #
 277   def p_comments(self, p):
 278     """comments : COMMENT comments
 279                 | """
 280     if len(p) > 1:
 281       child = self.BuildComment('Comment', p, 1)
 282       p[0] = ListFromConcat(child, p[2])
 283       if self.parse_debug: DumpReduction('comments', p)
 284     else:
 285       if self.parse_debug: DumpReduction('no comments', p)
 286
 287
 288 #
 289 # Namespace
 290 #
 291 # A namespace provides a named scope to an enclosed top_list.
 292 #
 293   def p_namespace(self, p):
 294     """namespace : modifiers NAMESPACE namespace_name '{' top_list '}' ';'"""
 295     children = ListFromConcat(p[1], p[5])
 296     p[0] = self.BuildNamed('Namespace', p, 3, children)
 297
 298   # We allow namespace names of the form foo.bar.baz.
 299   def p_namespace_name(self, p):
 300     """namespace_name : SYMBOL
 301                       | SYMBOL '.' namespace_name"""
 302     p[0] = "".join(p[1:])
 303
 304
 305 #
 306 # Dictionary
 307 #
 308 # A dictionary is a named list of optional and required members.
 309 #
 310   def p_dictionary_block(self, p):
 311     """dictionary_block : modifiers DICTIONARY SYMBOL '{' struct_list '}' ';'"""
 312     p[0] = self.BuildNamed('Dictionary', p, 3, ListFromConcat(p[1], p[5]))
 313
 314 #
 315 # Callback
 316 #
 317 # A callback is essentially a single function declaration (outside of an
 318 # Interface).
 319 #
 320   def p_callback_decl(self, p):
 321     """callback_decl : modifiers CALLBACK SYMBOL '=' SYMBOL param_list ';'"""
 322     children = ListFromConcat(p[1], p[6])
 323     p[0] = self.BuildNamed('Callback', p, 3, children)
 324
 325
 326 #
 327 # Inline
 328 #
 329 # Inline blocks define option code to be emitted based on language tag,
 330 # in the form of:
 331 # #inline <LANGUAGE>
 332 # <CODE>
 333 # #endinl
 334 #
 335   def p_inline(self, p):
 336     """inline : modifiers INLINE"""
 337     words = p[2].split()
 338     name = self.BuildAttribute('NAME', words[1])
 339     lines = p[2].split('\n')
 340     value = self.BuildAttribute('VALUE', '\n'.join(lines[1:-1]) + '\n')
 341     children = ListFromConcat(name, value, p[1])
 342     p[0] = self.BuildProduction('Inline', p, 2, children)
 343     if self.parse_debug: DumpReduction('inline', p)
 344
 345 # Extended Attributes
 346 #
 347 # Extended Attributes denote properties which will be applied to a node in the
 348 # AST.  A list of extended attributes are denoted by a brackets '[' ... ']'
 349 # enclosing a comma separated list of extended attributes in the form of:
 350 #
 351 #  Name
 352 #  Name=HEX | INT | OCT | FLOAT
 353 #  Name="STRING"
 354 #  Name=Function(arg ...)
 355 #  TODO(noelallen) -Not currently supported:
 356 #  ** Name(arg ...) ...
 357 #  ** Name=Scope::Value
 358 #
 359 # Extended Attributes are returned as a list or None.
 360
 361   def p_ext_attr_block(self, p):
 362     """ext_attr_block : '[' ext_attr_list ']'
 363                   | """
 364     if len(p) > 1:
 365       p[0] = p[2]
 366       if self.parse_debug: DumpReduction('ext_attr_block', p)
 367     else:
 368       if self.parse_debug: DumpReduction('no ext_attr_block', p)
 369
 370   def p_ext_attr_list(self, p):
 371     """ext_attr_list : SYMBOL '=' SYMBOL ext_attr_cont
 372                      | SYMBOL '=' value ext_attr_cont
 373                      | SYMBOL '=' SYMBOL param_list ext_attr_cont
 374                      | SYMBOL ext_attr_cont"""
 375     # If there are 4 tokens plus a return slot, this must be in the form
 376     # SYMBOL = SYMBOL|value ext_attr_cont
 377     if len(p) == 5:
 378       p[0] = ListFromConcat(self.BuildAttribute(p[1], p[3]), p[4])
 379     # If there are 5 tokens plus a return slot, this must be in the form
 380     # SYMBOL = SYMBOL (param_list) ext_attr_cont
 381     elif len(p) == 6:
 382       member = self.BuildNamed('Member', p, 3, [p[4]])
 383       p[0] = ListFromConcat(self.BuildAttribute(p[1], member), p[5])
 384     # Otherwise, this must be: SYMBOL ext_attr_cont
 385     else:
 386       p[0] = ListFromConcat(self.BuildAttribute(p[1], 'True'), p[2])
 387     if self.parse_debug: DumpReduction('ext_attribute_list', p)
 388
 389   def p_ext_attr_list_values(self, p):
 390     """ext_attr_list : SYMBOL '=' '(' values ')' ext_attr_cont
 391                      | SYMBOL '=' '(' symbols ')' ext_attr_cont"""
 392     p[0] = ListFromConcat(self.BuildAttribute(p[1], p[4]), p[6])
 393
 394   def p_values(self, p):
 395     """values : value values_cont"""
 396     p[0] = ListFromConcat(p[1], p[2])
 397
 398   def p_symbols(self, p):
 399     """symbols : SYMBOL symbols_cont"""
 400     p[0] = ListFromConcat(p[1], p[2])
 401
 402   def p_symbols_cont(self, p):
 403     """symbols_cont : ',' SYMBOL symbols_cont
 404                     | """
 405     if len(p) > 1: p[0] = ListFromConcat(p[2], p[3])
 406
 407   def p_values_cont(self, p):
 408     """values_cont : ',' value values_cont
 409                    | """
 410     if len(p) > 1: p[0] = ListFromConcat(p[2], p[3])
 411
 412   def p_ext_attr_cont(self, p):
 413     """ext_attr_cont : ',' ext_attr_list
 414                      |"""
 415     if len(p) > 1: p[0] = p[2]
 416     if self.parse_debug: DumpReduction('ext_attribute_cont', p)
 417
 418   def p_ext_attr_func(self, p):
 419     """ext_attr_list : SYMBOL '(' attr_arg_list ')' ext_attr_cont"""
 420     p[0] = ListFromConcat(self.BuildAttribute(p[1] + '()', p[3]), p[5])
 421     if self.parse_debug: DumpReduction('attr_arg_func', p)
 422
 423   def p_ext_attr_arg_list(self, p):
 424     """attr_arg_list : SYMBOL attr_arg_cont
 425                      | value attr_arg_cont"""
 426     p[0] = ListFromConcat(p[1], p[2])
 427
 428   def p_attr_arg_cont(self, p):
 429     """attr_arg_cont : ',' attr_arg_list
 430                      | """
 431     if self.parse_debug: DumpReduction('attr_arg_cont', p)
 432     if len(p) > 1: p[0] = p[2]
 433
 434   def p_attr_arg_error(self, p):
 435     """attr_arg_cont : error attr_arg_cont"""
 436     p[0] = p[2]
 437     if self.parse_debug: DumpReduction('attr_arg_error', p)
 438
 439
 440 #
 441 # Describe
 442 #
 443 # A describe block is defined at the top level.  It provides a mechanism for
 444 # attributing a group of ext_attr to a describe_list.  Members of the
 445 # describe list are language specific 'Type' declarations
 446 #
 447   def p_describe_block(self, p):
 448     """describe_block : modifiers DESCRIBE '{' describe_list '}' ';'"""
 449     children = ListFromConcat(p[1], p[4])
 450     p[0] = self.BuildProduction('Describe', p, 2, children)
 451     if self.parse_debug: DumpReduction('describe_block', p)
 452
 453   # Recover from describe error and continue parsing at the next top match.
 454   def p_describe_error(self, p):
 455     """describe_list : error describe_list"""
 456     p[0] = []
 457
 458   def p_describe_list(self, p):
 459     """describe_list : modifiers SYMBOL ';' describe_list
 460                      | modifiers ENUM ';' describe_list
 461                      | modifiers STRUCT ';' describe_list
 462                      | modifiers TYPEDEF ';' describe_list
 463                      | """
 464     if len(p) > 1:
 465       Type = self.BuildNamed('Type', p, 2, p[1])
 466       p[0] = ListFromConcat(Type, p[4])
 467
 468 #
 469 # Constant Values (integer, value)
 470 #
 471 # Constant values can be found at various levels.  A Constant value is returns
 472 # as the string value after validated against a FLOAT, HEX, INT, OCT or
 473 # STRING pattern as appropriate.
 474 #
 475   def p_value(self, p):
 476     """value : FLOAT
 477              | HEX
 478              | INT
 479              | OCT
 480              | STRING"""
 481     p[0] = p[1]
 482     if self.parse_debug: DumpReduction('value', p)
 483
 484   def p_value_lshift(self, p):
 485     """value : integer LSHIFT INT"""
 486     p[0] = "%s << %s" % (p[1], p[3])
 487     if self.parse_debug: DumpReduction('value', p)
 488
 489 # Integers are numbers which may not be floats used in cases like array sizes.
 490   def p_integer(self, p):
 491     """integer : HEX
 492                | INT
 493                | OCT"""
 494     p[0] = p[1]
 495     if self.parse_debug: DumpReduction('integer', p)
 496
 497 #
 498 # Expression
 499 #
 500 # A simple arithmetic expression.
 501 #
 502   precedence = (
 503     ('left','|','&','^'),
 504     ('left','LSHIFT','RSHIFT'),
 505     ('left','+','-'),
 506     ('left','*','/'),
 507     ('right','UMINUS','~'),
 508     )
 509
 510   def p_expression_binop(self, p):
 511     """expression : expression LSHIFT expression
 512                   | expression RSHIFT expression
 513                   | expression '|' expression
 514                   | expression '&' expression
 515                   | expression '^' expression
 516                   | expression '+' expression
 517                   | expression '-' expression
 518                   | expression '*' expression
 519                   | expression '/' expression"""
 520     p[0] = "%s %s %s" % (str(p[1]), str(p[2]), str(p[3]))
 521     if self.parse_debug: DumpReduction('expression_binop', p)
 522
 523   def p_expression_unop(self, p):
 524     """expression : '-' expression %prec UMINUS
 525                   | '~' expression %prec '~'"""
 526     p[0] = "%s%s" % (str(p[1]), str(p[2]))
 527     if self.parse_debug: DumpReduction('expression_unop', p)
 528
 529   def p_expression_term(self, p):
 530     "expression : '(' expression ')'"
 531     p[0] = "%s%s%s" % (str(p[1]), str(p[2]), str(p[3]))
 532     if self.parse_debug: DumpReduction('expression_term', p)
 533
 534   def p_expression_symbol(self, p):
 535     "expression : SYMBOL"
 536     p[0] = p[1]
 537     if self.parse_debug: DumpReduction('expression_symbol', p)
 538
 539   def p_expression_integer(self, p):
 540     "expression : integer"
 541     p[0] = p[1]
 542     if self.parse_debug: DumpReduction('expression_integer', p)
 543
 544 #
 545 # Array List
 546 #
 547 # Defined a list of array sizes (if any).
 548 #
 549   def p_arrays(self, p):
 550     """arrays : '[' ']' arrays
 551               | '[' integer ']' arrays
 552               | """
 553     # If there are 3 tokens plus a return slot it is an unsized array
 554     if len(p) == 4:
 555       array = self.BuildProduction('Array', p, 1)
 556       p[0] = ListFromConcat(array, p[3])
 557     # If there are 4 tokens plus a return slot it is a fixed array
 558     elif len(p) == 5:
 559       count = self.BuildAttribute('FIXED', p[2])
 560       array = self.BuildProduction('Array', p, 2, [count])
 561       p[0] = ListFromConcat(array, p[4])
 562     # If there is only a return slot, do not fill it for this terminator.
 563     elif len(p) == 1: return
 564     if self.parse_debug: DumpReduction('arrays', p)
 565
 566
 567 # An identifier is a legal value for a parameter or attribute name. Lots of
 568 # existing IDL files use "callback" as a parameter/attribute name, so we allow
 569 # a SYMBOL or the CALLBACK keyword.
 570   def p_identifier(self, p):
 571     """identifier : SYMBOL
 572                   | CALLBACK"""
 573     p[0] = p[1]
 574     # Save the line number of the underlying token (otherwise it gets
 575     # discarded), since we use it in the productions with an identifier in
 576     # them.
 577     p.set_lineno(0, p.lineno(1))
 578
 579 #
 580 # Parameter List
 581 #
 582 # A parameter list is a collection of arguments which are passed to a
 583 # function.
 584 #
 585   def p_param_list(self, p):
 586     """param_list : '(' param_item param_cont ')'
 587                   | '(' ')' """
 588     if len(p) > 3:
 589       args = ListFromConcat(p[2], p[3])
 590     else:
 591       args = []
 592     p[0] = self.BuildProduction('Callspec', p, 1, args)
 593     if self.parse_debug: DumpReduction('param_list', p)
 594
 595   def p_param_item(self, p):
 596     """param_item : modifiers optional SYMBOL arrays identifier"""
 597     typeref = self.BuildAttribute('TYPEREF', p[3])
 598     children = ListFromConcat(p[1], p[2], typeref, p[4])
 599     p[0] = self.BuildNamed('Param', p, 5, children)
 600     if self.parse_debug: DumpReduction('param_item', p)
 601
 602   def p_optional(self, p):
 603     """optional : OPTIONAL
 604                 | """
 605     if len(p) == 2:
 606       p[0] = self.BuildAttribute('OPTIONAL', True)
 607
 608
 609   def p_param_cont(self, p):
 610     """param_cont : ',' param_item param_cont
 611                   | """
 612     if len(p) > 1:
 613       p[0] = ListFromConcat(p[2], p[3])
 614       if self.parse_debug: DumpReduction('param_cont', p)
 615
 616   def p_param_error(self, p):
 617     """param_cont : error param_cont"""
 618     p[0] = p[2]
 619
 620
 621 #
 622 # Typedef
 623 #
 624 # A typedef creates a new referencable type.  The typedef can specify an array
 625 # definition as well as a function declaration.
 626 #
 627   def p_typedef_data(self, p):
 628     """typedef_decl : modifiers TYPEDEF SYMBOL SYMBOL ';' """
 629     typeref = self.BuildAttribute('TYPEREF', p[3])
 630     children = ListFromConcat(p[1], typeref)
 631     p[0] = self.BuildNamed('Typedef', p, 4, children)
 632     if self.parse_debug: DumpReduction('typedef_data', p)
 633
 634   def p_typedef_array(self, p):
 635     """typedef_decl : modifiers TYPEDEF SYMBOL arrays SYMBOL ';' """
 636     typeref = self.BuildAttribute('TYPEREF', p[3])
 637     children = ListFromConcat(p[1], typeref, p[4])
 638     p[0] = self.BuildNamed('Typedef', p, 5, children)
 639     if self.parse_debug: DumpReduction('typedef_array', p)
 640
 641   def p_typedef_func(self, p):
 642     """typedef_decl : modifiers TYPEDEF SYMBOL SYMBOL param_list ';' """
 643     typeref = self.BuildAttribute('TYPEREF', p[3])
 644     children = ListFromConcat(p[1], typeref, p[5])
 645     p[0] = self.BuildNamed('Typedef', p, 4, children)
 646     if self.parse_debug: DumpReduction('typedef_func', p)
 647
 648 #
 649 # Enumeration
 650 #
 651 # An enumeration is a set of named integer constants.  An enumeration
 652 # is valid type which can be referenced in other definitions.
 653 #
 654   def p_enum_block(self, p):
 655     """enum_block : modifiers ENUM SYMBOL '{' enum_list '}' ';'"""
 656     p[0] = self.BuildNamed('Enum', p, 3, ListFromConcat(p[1], p[5]))
 657     if self.parse_debug: DumpReduction('enum_block', p)
 658
 659   # Recover from enum error and continue parsing at the next top match.
 660   def p_enum_errorA(self, p):
 661     """enum_block : modifiers ENUM error '{' enum_list '}' ';'"""
 662     p[0] = []
 663
 664   def p_enum_errorB(self, p):
 665     """enum_block : modifiers ENUM error ';'"""
 666     p[0] = []
 667
 668   def p_enum_list(self, p):
 669     """enum_list : modifiers SYMBOL '=' expression enum_cont
 670                  | modifiers SYMBOL enum_cont"""
 671     if len(p) > 4:
 672       val  = self.BuildAttribute('VALUE', p[4])
 673       enum = self.BuildNamed('EnumItem', p, 2, ListFromConcat(val, p[1]))
 674       p[0] = ListFromConcat(enum, p[5])
 675     else:
 676       enum = self.BuildNamed('EnumItem', p, 2, p[1])
 677       p[0] = ListFromConcat(enum, p[3])
 678     if self.parse_debug: DumpReduction('enum_list', p)
 679
 680   def p_enum_cont(self, p):
 681     """enum_cont : ',' enum_list
 682                  |"""
 683     if len(p) > 1: p[0] = p[2]
 684     if self.parse_debug: DumpReduction('enum_cont', p)
 685
 686   def p_enum_cont_error(self, p):
 687     """enum_cont : error enum_cont"""
 688     p[0] = p[2]
 689     if self.parse_debug: DumpReduction('enum_error', p)
 690
 691
 692 #
 693 # Label
 694 #
 695 # A label is a special kind of enumeration which allows us to go from a
 696 # set of labels
 697 #
 698   def p_label_block(self, p):
 699     """label_block : modifiers LABEL SYMBOL '{' label_list '}' ';'"""
 700     p[0] = self.BuildNamed('Label', p, 3, ListFromConcat(p[1], p[5]))
 701     if self.parse_debug: DumpReduction('label_block', p)
 702
 703   def p_label_list(self, p):
 704     """label_list : modifiers SYMBOL '=' FLOAT label_cont"""
 705     val  = self.BuildAttribute('VALUE', p[4])
 706     label = self.BuildNamed('LabelItem', p, 2, ListFromConcat(val, p[1]))
 707     p[0] = ListFromConcat(label, p[5])
 708     if self.parse_debug: DumpReduction('label_list', p)
 709
 710   def p_label_cont(self, p):
 711     """label_cont : ',' label_list
 712                  |"""
 713     if len(p) > 1: p[0] = p[2]
 714     if self.parse_debug: DumpReduction('label_cont', p)
 715
 716   def p_label_cont_error(self, p):
 717     """label_cont : error label_cont"""
 718     p[0] = p[2]
 719     if self.parse_debug: DumpReduction('label_error', p)
 720
 721
 722 #
 723 # Members
 724 #
 725 # A member attribute or function of a struct or interface.
 726 #
 727   def p_member_attribute(self, p):
 728     """member_attribute : modifiers SYMBOL arrays questionmark identifier"""
 729     typeref = self.BuildAttribute('TYPEREF', p[2])
 730     children = ListFromConcat(p[1], typeref, p[3], p[4])
 731     p[0] = self.BuildNamed('Member', p, 5, children)
 732     if self.parse_debug: DumpReduction('attribute', p)
 733
 734   def p_member_function(self, p):
 735     """member_function : modifiers static SYMBOL SYMBOL param_list"""
 736     typeref = self.BuildAttribute('TYPEREF', p[3])
 737     children = ListFromConcat(p[1], p[2], typeref, p[5])
 738     p[0] = self.BuildNamed('Member', p, 4, children)
 739     if self.parse_debug: DumpReduction('function', p)
 740
 741   def p_static(self, p):
 742     """static : STATIC
 743               | """
 744     if len(p) == 2:
 745       p[0] = self.BuildAttribute('STATIC', True)
 746
 747   def p_questionmark(self, p):
 748     """questionmark : '?'
 749                     | """
 750     if len(p) == 2:
 751       p[0] = self.BuildAttribute('OPTIONAL', True)
 752
 753 #
 754 # Interface
 755 #
 756 # An interface is a named collection of functions.
 757 #
 758   def p_interface_block(self, p):
 759     """interface_block : modifiers INTERFACE SYMBOL '{' interface_list '}' ';'"""
 760     p[0] = self.BuildNamed('Interface', p, 3, ListFromConcat(p[1], p[5]))
 761     if self.parse_debug: DumpReduction('interface_block', p)
 762
 763   def p_interface_error(self, p):
 764     """interface_block : modifiers INTERFACE error '{' interface_list '}' ';'"""
 765     p[0] = []
 766
 767   def p_interface_list(self, p):
 768     """interface_list : member_function ';' interface_list
 769                       | """
 770     if len(p) > 1 :
 771       p[0] = ListFromConcat(p[1], p[3])
 772       if self.parse_debug: DumpReduction('interface_list', p)
 773
 774
 775 #
 776 # Struct
 777 #
 778 # A struct is a named collection of members which in turn reference other
 779 # types.  The struct is a referencable type.
 780 #
 781   def p_struct_block(self, p):
 782     """struct_block : modifiers STRUCT SYMBOL '{' struct_list '}' ';'"""
 783     children = ListFromConcat(p[1], p[5])
 784     p[0] = self.BuildNamed('Struct', p, 3, children)
 785     if self.parse_debug: DumpReduction('struct_block', p)
 786
 787   # Recover from struct error and continue parsing at the next top match.
 788   def p_struct_error(self, p):
 789     """enum_block : modifiers STRUCT error '{' struct_list '}' ';'"""
 790     p[0] = []
 791
 792   def p_struct_list(self, p):
 793     """struct_list : member_attribute ';' struct_list
 794                    | member_function ';' struct_list
 795                    |"""
 796     if len(p) > 1: p[0] = ListFromConcat(p[1], p[3])
 797
 798
 799 #
 800 # Parser Errors
 801 #
 802 # p_error is called whenever the parser can not find a pattern match for
 803 # a set of items from the current state.  The p_error function defined here
 804 # is triggered logging an error, and parsing recover happens as the
 805 # p_<type>_error functions defined above are called.  This allows the parser
 806 # to continue so as to capture more than one error per file.
 807 #
 808   def p_error(self, t):
 809     filename = self.lexobj.filename
 810     self.parse_errors += 1
 811     if t:
 812       lineno = t.lineno
 813       pos = t.lexpos
 814       prev = self.yaccobj.symstack[-1]
 815       if type(prev) == lex.LexToken:
 816         msg = "Unexpected %s after %s." % (
 817             TokenTypeName(t), TokenTypeName(prev))
 818       else:
 819         msg = "Unexpected %s." % (t.value)
 820     else:
 821       lineno = self.last.lineno
 822       pos = self.last.lexpos
 823       msg = "Unexpected end of file after %s." % TokenTypeName(self.last)
 824       self.yaccobj.restart()
 825
 826     # Attempt to remap the error to a friendlier form
 827     if msg in ERROR_REMAP:
 828       msg = ERROR_REMAP[msg]
 829
 830     # Log the error
 831     ErrOut.LogLine(filename, lineno, pos, msg)
 832
 833   def Warn(self, node, msg):
 834     WarnOut.LogLine(node.filename, node.lineno, node.pos, msg)
 835     self.parse_warnings += 1
 836
 837   def __init__(self):
 838     IDLLexer.__init__(self)
 839     self.yaccobj = yacc.yacc(module=self, tabmodule=None, debug=False,
 840                              optimize=0, write_tables=0)
 841
 842     self.build_debug = GetOption('build_debug')
 843     self.parse_debug = GetOption('parse_debug')
 844     self.token_debug = GetOption('token_debug')
 845     self.verbose = GetOption('verbose')
 846     self.parse_errors = 0
 847
 848 #
 849 # Tokenizer
 850 #
 851 # The token function returns the next token provided by IDLLexer for matching
 852 # against the leaf paterns.
 853 #
 854   def token(self):
 855     tok = self.lexobj.token()
 856     if tok:
 857       self.last = tok
 858       if self.token_debug:
 859         InfoOut.Log("TOKEN %s(%s)" % (tok.type, tok.value))
 860     return tok
 861
 862 #
 863 # BuildProduction
 864 #
 865 # Production is the set of items sent to a grammar rule resulting in a new
 866 # item being returned.
 867 #
 868 # p - Is the Yacc production object containing the stack of items
 869 # index - Index into the production of the name for the item being produced.
 870 # cls - The type of item being producted
 871 # childlist - The children of the new item
 872   def BuildProduction(self, cls, p, index, childlist=None):
 873     if not childlist: childlist = []
 874     filename = self.lexobj.filename
 875     lineno = p.lineno(index)
 876     pos = p.lexpos(index)
 877     out = IDLNode(cls, filename, lineno, pos, childlist)
 878     if self.build_debug:
 879       InfoOut.Log("Building %s" % out)
 880     return out
 881
 882   def BuildNamed(self, cls, p, index, childlist=None):
 883     if not childlist: childlist = []
 884     childlist.append(self.BuildAttribute('NAME', p[index]))
 885     return self.BuildProduction(cls, p, index, childlist)
 886
 887   def BuildComment(self, cls, p, index):
 888     name = p[index]
 889
 890     # Remove comment markers
 891     lines = []
 892     if name[:2] == '//':
 893       # For C++ style, remove any leading whitespace and the '//' marker from
 894       # each line.
 895       form = 'cc'
 896       for line in name.split('\n'):
 897         start = line.find('//')
 898         lines.append(line[start+2:])
 899     else:
 900       # For C style, remove ending '*/''
 901       form = 'c'
 902       for line in name[:-2].split('\n'):
 903         # Remove characters until start marker for this line '*' if found
 904         # otherwise it should be blank.
 905         offs = line.find('*')
 906         if offs >= 0:
 907           line = line[offs + 1:].rstrip()
 908         else:
 909           line = ''
 910         lines.append(line)
 911     name = '\n'.join(lines)
 912
 913     childlist = [self.BuildAttribute('NAME', name),
 914                  self.BuildAttribute('FORM', form)]
 915     return self.BuildProduction(cls, p, index, childlist)
 916
 917 #
 918 # BuildAttribute
 919 #
 920 # An ExtendedAttribute is a special production that results in a property
 921 # which is applied to the adjacent item.  Attributes have no children and
 922 # instead represent key/value pairs.
 923 #
 924   def BuildAttribute(self, key, val):
 925     return IDLAttribute(key, val)
 926
 927
 928 #
 929 # ParseData
 930 #
 931 # Attempts to parse the current data loaded in the lexer.
 932 #
 933   def ParseData(self, data, filename='<Internal>'):
 934     self.SetData(filename, data)
 935     try:
 936       self.parse_errors = 0
 937       self.parse_warnings = 0
 938       return self.yaccobj.parse(lexer=self)
 939
 940     except lex.LexError as le:
 941       ErrOut.Log(str(le))
 942       return []
 943
 944 #
 945 # ParseFile
 946 #
 947 # Loads a new file into the lexer and attemps to parse it.
 948 #
 949   def ParseFile(self, filename):
 950     date = time.ctime(os.path.getmtime(filename))
 951     data = open(filename).read()
 952     if self.verbose:
 953       InfoOut.Log("Parsing %s" % filename)
 954     try:
 955       out = self.ParseData(data, filename)
 956
 957       # If we have a src root specified, remove it from the path
 958       srcroot = GetOption('srcroot')
 959       if srcroot and filename.find(srcroot) == 0:
 960         filename = filename[len(srcroot) + 1:]
 961       filenode = IDLFile(filename, out, self.parse_errors + self.lex_errors)
 962       filenode.SetProperty('DATETIME', date)
 963       return filenode
 964
 965     except Exception as e:
 966       ErrOut.LogLine(filename, self.last.lineno, self.last.lexpos,
 967                      'Internal parsing error - %s.' % str(e))
 968       raise
 969
 970
 971
 972 #
 973 # Flatten Tree
 974 #
 975 # Flattens the tree of IDLNodes for use in testing.
 976 #
 977 def FlattenTree(node):
 978   add_self = False
 979   out = []
 980   for child in node.children:
 981     if child.IsA('Comment'):
 982       add_self = True
 983     else:
 984       out.extend(FlattenTree(child))
 985
 986   if add_self:
 987     out = [str(node)] + out
 988   return out
 989
 990
 991 def TestErrors(filename, filenode):
 992   nodelist = filenode.GetChildren()
 993
 994   lexer = IDLLexer()
 995   data = open(filename).read()
 996   lexer.SetData(filename, data)
 997
 998   pass_comments = []
 999   fail_comments = []
1000   while True:
1001     tok = lexer.lexobj.token()
1002     if tok == None: break
1003     if tok.type == 'COMMENT':
1004       args = tok.value[3:-3].split()
1005       if args[0] == 'OK':
1006         pass_comments.append((tok.lineno, ' '.join(args[1:])))
1007       else:
1008         if args[0] == 'FAIL':
1009           fail_comments.append((tok.lineno, ' '.join(args[1:])))
1010   obj_list = []
1011   for node in nodelist:
1012     obj_list.extend(FlattenTree(node))
1013
1014   errors = 0
1015
1016   #
1017   # Check for expected successes
1018   #
1019   obj_cnt = len(obj_list)
1020   pass_cnt = len(pass_comments)
1021   if obj_cnt != pass_cnt:
1022     InfoOut.Log("Mismatched pass (%d) vs. nodes built (%d)."
1023         % (pass_cnt, obj_cnt))
1024     InfoOut.Log("PASS: %s" % [x[1] for x in pass_comments])
1025     InfoOut.Log("OBJS: %s" % obj_list)
1026     errors += 1
1027     if pass_cnt > obj_cnt: pass_cnt = obj_cnt
1028
1029   for i in range(pass_cnt):
1030     line, comment = pass_comments[i]
1031     if obj_list[i] != comment:
1032       ErrOut.LogLine(filename, line, None, "OBJ %s : EXPECTED %s\n" %
1033                      (obj_list[i], comment))
1034       errors += 1
1035
1036   #
1037   # Check for expected errors
1038   #
1039   err_list = ErrOut.DrainLog()
1040   err_cnt = len(err_list)
1041   fail_cnt = len(fail_comments)
1042   if err_cnt != fail_cnt:
1043     InfoOut.Log("Mismatched fail (%d) vs. errors seen (%d)."
1044         % (fail_cnt, err_cnt))
1045     InfoOut.Log("FAIL: %s" % [x[1] for x in fail_comments])
1046     InfoOut.Log("ERRS: %s" % err_list)
1047     errors += 1
1048     if fail_cnt > err_cnt:  fail_cnt = err_cnt
1049
1050   for i in range(fail_cnt):
1051     line, comment = fail_comments[i]
1052     err = err_list[i].strip()
1053
1054     if err_list[i] != comment:
1055       ErrOut.Log("%s(%d) Error\n\tERROR : %s\n\tEXPECT: %s" % (
1056         filename, line, err_list[i], comment))
1057       errors += 1
1058
1059   # Clear the error list for the next run
1060   err_list = []
1061   return errors
1062
1063
1064 def TestFile(parser, filename):
1065   # Capture errors instead of reporting them so we can compare them
1066   # with the expected errors.
1067   ErrOut.SetConsole(False)
1068   ErrOut.SetCapture(True)
1069
1070   filenode = parser.ParseFile(filename)
1071
1072   # Renable output
1073   ErrOut.SetConsole(True)
1074   ErrOut.SetCapture(False)
1075
1076   # Compare captured errors
1077   return TestErrors(filename, filenode)
1078
1079
1080 def TestErrorFiles(filter):
1081   idldir = os.path.split(sys.argv[0])[0]
1082   idldir = os.path.join(idldir, 'test_parser', '*.idl')
1083   filenames = glob.glob(idldir)
1084   parser = IDLParser()
1085   total_errs = 0
1086   for filename in filenames:
1087     if filter and filename not in filter: continue
1088     errs = TestFile(parser, filename)
1089     if errs:
1090       ErrOut.Log("%s test failed with %d error(s)." % (filename, errs))
1091       total_errs += errs
1092
1093   if total_errs:
1094     ErrOut.Log("Failed parsing test.")
1095   else:
1096     InfoOut.Log("Passed parsing test.")
1097   return total_errs
1098
1099
1100 def TestNamespaceFiles(filter):
1101   idldir = os.path.split(sys.argv[0])[0]
1102   idldir = os.path.join(idldir, 'test_namespace', '*.idl')
1103   filenames = glob.glob(idldir)
1104   testnames = []
1105
1106   for filename in filenames:
1107     if filter and filename not in filter: continue
1108     testnames.append(filename)
1109
1110   # If we have no files to test, then skip this test
1111   if not testnames:
1112     InfoOut.Log('No files to test for namespace.')
1113     return 0
1114
1115   InfoOut.SetConsole(False)
1116   ast = ParseFiles(testnames)
1117   InfoOut.SetConsole(True)
1118
1119   errs = ast.GetProperty('ERRORS')
1120   if errs:
1121     ErrOut.Log("Failed namespace test.")
1122   else:
1123     InfoOut.Log("Passed namespace test.")
1124   return errs
1125
1126
1127
1128 def FindVersionError(releases, node):
1129   err_cnt = 0
1130   if node.IsA('Interface', 'Struct'):
1131     comment_list = []
1132     comment = node.GetOneOf('Comment')
1133     if comment and comment.GetName()[:4] == 'REL:':
1134       comment_list = comment.GetName()[5:].strip().split(' ')
1135
1136     first_list = [node.first_release[rel] for rel in releases]
1137     first_list = sorted(set(first_list))
1138     if first_list != comment_list:
1139       node.Error("Mismatch in releases: %s vs %s." % (
1140           comment_list, first_list))
1141       err_cnt += 1
1142
1143   for child in node.GetChildren():
1144     err_cnt += FindVersionError(releases, child)
1145   return err_cnt
1146
1147
1148 def TestVersionFiles(filter):
1149   idldir = os.path.split(sys.argv[0])[0]
1150   idldir = os.path.join(idldir, 'test_version', '*.idl')
1151   filenames = glob.glob(idldir)
1152   testnames = []
1153
1154   for filename in filenames:
1155     if filter and filename not in filter: continue
1156     testnames.append(filename)
1157
1158   # If we have no files to test, then skip this test
1159   if not testnames:
1160     InfoOut.Log('No files to test for version.')
1161     return 0
1162
1163   ast = ParseFiles(testnames)
1164   errs = FindVersionError(ast.releases, ast)
1165
1166   if errs:
1167     ErrOut.Log("Failed version test.")
1168   else:
1169     InfoOut.Log("Passed version test.")
1170   return errs
1171
1172
1173 default_dirs = ['.', 'trusted', 'dev', 'private', 'extensions',
1174                 'extensions/dev']
1175 def ParseFiles(filenames):
1176   parser = IDLParser()
1177   filenodes = []
1178
1179   if not filenames:
1180     filenames = []
1181     srcroot = GetOption('srcroot')
1182     dirs = default_dirs
1183     if GetOption('include_private'):
1184       dirs += ['private']
1185     for dirname in dirs:
1186       srcdir = os.path.join(srcroot, dirname, '*.idl')
1187       srcdir = os.path.normpath(srcdir)
1188       filenames += sorted(glob.glob(srcdir))
1189
1190   if not filenames:
1191     ErrOut.Log('No sources provided.')
1192
1193   for filename in filenames:
1194     filenode = parser.ParseFile(filename)
1195     filenodes.append(filenode)
1196
1197   ast = IDLAst(filenodes)
1198   if GetOption('dump_tree'): ast.Dump(0)
1199
1200   Lint(ast)
1201   return ast
1202
1203
1204 def Main(args):
1205   filenames = ParseOptions(args)
1206
1207   # If testing...
1208   if GetOption('test'):
1209     errs = TestErrorFiles(filenames)
1210     errs = TestNamespaceFiles(filenames)
1211     errs = TestVersionFiles(filenames)
1212     if errs:
1213       ErrOut.Log("Parser failed with %d errors." % errs)
1214       return  -1
1215     return 0
1216
1217   # Otherwise, build the AST
1218   ast = ParseFiles(filenames)
1219   errs = ast.GetProperty('ERRORS')
1220   if errs:
1221     ErrOut.Log('Found %d error(s).' % errs);
1222   InfoOut.Log("%d files processed." % len(filenames))
1223   return errs
1224
1225
1226 if __name__ == '__main__':
1227   sys.exit(Main(sys.argv[1:]))
1228