tools/idl_parser/idl_parser.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """ Parser for PPAPI IDL """
   7
   8 #
   9 # IDL Parser
  10 #
  11 # The parser is uses the PLY yacc library to build a set of parsing rules based
  12 # on WebIDL.
  13 #
  14 # WebIDL, and WebIDL grammar can be found at:
  15 #   http://heycam.github.io/webidl/
  16 # PLY can be found at:
  17 #   http://www.dabeaz.com/ply/
  18 #
  19 # The parser generates a tree by recursively matching sets of items against
  20 # defined patterns.  When a match is made, that set of items is reduced
  21 # to a new item.   The new item can provide a match for parent patterns.
  22 # In this way an AST is built (reduced) depth first.
  23 #
  24
  25 #
  26 # Disable check for line length and Member as Function due to how grammar rules
  27 # are defined with PLY
  28 #
  29 # pylint: disable=R0201
  30 # pylint: disable=C0301
  31
  32 import os.path
  33 import sys
  34 import time
  35
  36 from idl_lexer import IDLLexer
  37 from idl_node import IDLAttribute, IDLNode
  38
  39 #
  40 # Try to load the ply module, if not, then assume it is in the third_party
  41 # directory.
  42 #
  43 try:
  44   # Disable lint check which fails to find the ply module.
  45   # pylint: disable=F0401
  46   from ply import lex
  47   from ply import yacc
  48 except ImportError:
  49   module_path, module_name = os.path.split(__file__)
  50   third_party = os.path.join(module_path, os.par, os.par, 'third_party')
  51   sys.path.append(third_party)
  52   # pylint: disable=F0401
  53   from ply import lex
  54   from ply import yacc
  55
  56 #
  57 # ERROR_REMAP
  58 #
  59 # Maps the standard error formula into a more friendly error message.
  60 #
  61 ERROR_REMAP = {
  62   'Unexpected ")" after "(".' : 'Empty argument list.',
  63   'Unexpected ")" after ",".' : 'Missing argument.',
  64   'Unexpected "}" after ",".' : 'Trailing comma in block.',
  65   'Unexpected "}" after "{".' : 'Unexpected empty block.',
  66   'Unexpected comment after "}".' : 'Unexpected trailing comment.',
  67   'Unexpected "{" after keyword "enum".' : 'Enum missing name.',
  68   'Unexpected "{" after keyword "struct".' : 'Struct missing name.',
  69   'Unexpected "{" after keyword "interface".' : 'Interface missing name.',
  70 }
  71
  72
  73 def Boolean(val):
  74   """Convert to strict boolean type."""
  75   if val:
  76     return True
  77   return False
  78
  79
  80 def ListFromConcat(*items):
  81   """Generate list by concatenating inputs"""
  82   itemsout = []
  83   for item in items:
  84     if item is None:
  85       continue
  86     if type(item) is not type([]):
  87       itemsout.append(item)
  88     else:
  89       itemsout.extend(item)
  90
  91   return itemsout
  92
  93 def ExpandProduction(p):
  94   if type(p) == list:
  95     return '[' + ', '.join([ExpandProduction(x) for x in p]) + ']'
  96   if type(p) == IDLNode:
  97     return 'Node:' + str(p)
  98   if type(p) == IDLAttribute:
  99     return 'Attr:' + str(p)
 100   if type(p) == str:
 101     return 'str:' + p
 102   return '%s:%s' % (p.__class__.__name__, str(p))
 103
 104 # TokenTypeName
 105 #
 106 # Generate a string which has the type and value of the token.
 107 #
 108 def TokenTypeName(t):
 109   if t.type == 'SYMBOL':
 110     return 'symbol %s' % t.value
 111   if t.type in ['HEX', 'INT', 'OCT', 'FLOAT']:
 112     return 'value %s' % t.value
 113   if t.type == 'string' :
 114     return 'string "%s"' % t.value
 115   if t.type == 'COMMENT' :
 116     return 'comment'
 117   if t.type == t.value:
 118     return '"%s"' % t.value
 119   if t.type == ',':
 120     return 'Comma'
 121   if t.type == 'identifier':
 122     return 'identifier "%s"' % t.value
 123   return 'keyword "%s"' % t.value
 124
 125
 126 #
 127 # IDL Parser
 128 #
 129 # The Parser inherits the from the Lexer to provide PLY with the tokenizing
 130 # definitions.  Parsing patterns are encoded as functions where p_<name> is
 131 # is called any time a patern matching the function documentation is found.
 132 # Paterns are expressed in the form of:
 133 # """ <new item> : <item> ....
 134 #                | <item> ...."""
 135 #
 136 # Where new item is the result of a match against one or more sets of items
 137 # separated by the "|".
 138 #
 139 # The function is called with an object 'p' where p[0] is the output object
 140 # and p[n] is the set of inputs for positive values of 'n'.  Len(p) can be
 141 # used to distinguish between multiple item sets in the pattern.
 142 #
 143 # For more details on parsing refer to the PLY documentation at
 144 #    http://www.dabeaz.com/ply/
 145 #
 146 # The parser is based on the WebIDL standard.  See:
 147 #    http://heycam.github.io/webidl/#idl-grammar
 148 #
 149 # The various productions are annotated so that the WHOLE number greater than
 150 # zero in the comment denotes the matching WebIDL grammar definition.
 151 #
 152 # Productions with a fractional component in the comment denote additions to
 153 # the WebIDL spec, such as comments.
 154 #
 155
 156
 157 class IDLParser(object):
 158 #
 159 # We force all input files to start with two comments.  The first comment is a
 160 # Copyright notice followed by a file comment and finally by file level
 161 # productions.
 162 #
 163   # [0] Insert a TOP definition for Copyright and Comments
 164   def p_Top(self, p):
 165     """Top : COMMENT COMMENT Definitions"""
 166     Copyright = self.BuildComment('Copyright', p, 1)
 167     Filedoc = self.BuildComment('Comment', p, 2)
 168     p[0] = ListFromConcat(Copyright, Filedoc, p[3])
 169
 170   # [0.1] Add support for Multiple COMMENTS
 171   def p_Comments(self, p):
 172     """Comments : CommentsRest"""
 173     if len(p) > 1:
 174       p[0] = p[1]
 175
 176   # [0.2] Produce a COMMENT and aggregate sibling comments
 177   def p_CommentsRest(self, p):
 178     """CommentsRest : COMMENT CommentsRest
 179                     | """
 180     if len(p) > 1:
 181       p[0] = ListFromConcat(self.BuildComment('Comment', p, 1), p[2])
 182
 183
 184 #
 185 #The parser is based on the WebIDL standard.  See:
 186 # http://heycam.github.io/webidl/#idl-grammar
 187 #
 188   # [1]
 189   def p_Definitions(self, p):
 190     """Definitions : ExtendedAttributeList Definition Definitions
 191            | """
 192     if len(p) > 1:
 193       p[2].AddChildren(p[1])
 194       p[0] = ListFromConcat(p[2], p[3])
 195
 196   # [2]
 197   def p_Definition(self, p):
 198     """Definition : CallbackOrInterface
 199                   | Partial
 200                   | Dictionary
 201                   | Exception
 202                   | Enum
 203                   | Typedef
 204                   | ImplementsStatement"""
 205     p[0] = p[1]
 206
 207   # [2.1] Error recovery for definition
 208   def p_DefinitionError(self, p):
 209     """Definition : error ';'"""
 210     p[0] = self.BuildError(p, 'Definition')
 211
 212   # [3]
 213   def p_CallbackOrInterface(self, p):
 214     """CallbackOrInterface : CALLBACK CallbackRestOrInterface
 215                            | Interface"""
 216     if len(p) > 2:
 217       p[0] = p[2]
 218     else:
 219       p[0] = p[1]
 220
 221   # [4]
 222   def p_CallbackRestOrInterface(self, p):
 223     """CallbackRestOrInterface : CallbackRest
 224                                | Interface"""
 225     p[0] = p[1]
 226
 227   # [5]
 228   def p_Interface(self, p):
 229     """Interface : INTERFACE identifier Inheritance '{' InterfaceMembers '}' ';'"""
 230     p[0] = self.BuildNamed('Interface', p, 2, ListFromConcat(p[3], p[5]))
 231
 232   # [6]
 233   def p_Partial(self, p):
 234     """Partial : PARTIAL PartialDefinition"""
 235     p[2].AddChildren(self.BuildTrue('Partial'))
 236     p[0] = p[2]
 237
 238   # [6.1] Error recovery for Partial
 239   def p_PartialError(self, p):
 240     """Partial : PARTIAL error"""
 241     p[0] = self.BuildError(p, 'Partial')
 242
 243   # [7]
 244   def p_PartialDefinition(self, p):
 245     """PartialDefinition : PartialDictionary
 246                          | PartialInterface"""
 247     p[0] = p[1]
 248
 249   # [8]
 250   def p_PartialInterface(self, p):
 251     """PartialInterface : INTERFACE identifier '{' InterfaceMembers '}' ';'"""
 252     p[0] = self.BuildNamed('Interface', p, 2, p[4])
 253
 254   # [9]
 255   def p_InterfaceMembers(self, p):
 256     """InterfaceMembers : ExtendedAttributeList InterfaceMember InterfaceMembers
 257                         |"""
 258     if len(p) > 1:
 259       p[2].AddChildren(p[1])
 260       p[0] = ListFromConcat(p[2], p[3])
 261
 262   # [10]
 263   def p_InterfaceMember(self, p):
 264     """InterfaceMember : Const
 265                        | AttributeOrOperationOrIterator"""
 266     p[0] = p[1]
 267
 268   # [11]
 269   def p_Dictionary(self, p):
 270     """Dictionary : DICTIONARY identifier Inheritance '{' DictionaryMembers '}' ';'"""
 271     p[0] = self.BuildNamed('Dictionary', p, 2, ListFromConcat(p[3], p[5]))
 272
 273   # [11.1] Error recovery for regular Dictionary
 274   def p_DictionaryError(self, p):
 275     """Dictionary : DICTIONARY error ';'"""
 276     p[0] = self.BuildError(p, 'Dictionary')
 277
 278   # [12]
 279   def p_DictionaryMembers(self, p):
 280     """DictionaryMembers : ExtendedAttributeList DictionaryMember DictionaryMembers
 281                          |"""
 282     if len(p) > 1:
 283       p[2].AddChildren(p[1])
 284       p[0] = ListFromConcat(p[2], p[3])
 285
 286   # [13]
 287   def p_DictionaryMember(self, p):
 288     """DictionaryMember : Type identifier Default ';'"""
 289     p[0] = self.BuildNamed('Key', p, 2, ListFromConcat(p[1], p[3]))
 290
 291   # [14]
 292   def p_PartialDictionary(self, p):
 293     """PartialDictionary : DICTIONARY identifier '{' DictionaryMembers '}' ';'"""
 294     partial = self.BuildTrue('Partial')
 295     p[0] = self.BuildNamed('Dictionary', p, 2, ListFromConcat(p[4], partial))
 296
 297   # [14.1] Error recovery for Partial Dictionary
 298   def p_PartialDictionaryError(self, p):
 299     """PartialDictionary : DICTIONARY error ';'"""
 300     p[0] = self.BuildError(p, 'PartialDictionary')
 301
 302   # [15]
 303   def p_Default(self, p):
 304     """Default : '=' DefaultValue
 305                |"""
 306     if len(p) > 1:
 307       p[0] = self.BuildProduction('Default', p, 2, p[2])
 308
 309   # [16]
 310   def p_DefaultValue(self, p):
 311     """DefaultValue : ConstValue
 312                     | string"""
 313     if type(p[1]) == str:
 314       p[0] = ListFromConcat(self.BuildAttribute('TYPE', 'DOMString'),
 315                             self.BuildAttribute('NAME', p[1]))
 316     else:
 317       p[0] = p[1]
 318
 319   # [17]
 320   def p_Exception(self, p):
 321     """Exception : EXCEPTION identifier Inheritance '{' ExceptionMembers '}' ';'"""
 322     p[0] = self.BuildNamed('Exception', p, 2, ListFromConcat(p[3], p[5]))
 323
 324   # [18]
 325   def p_ExceptionMembers(self, p):
 326     """ExceptionMembers : ExtendedAttributeList ExceptionMember ExceptionMembers
 327                         |"""
 328     if len(p) > 1:
 329       p[2].AddChildren(p[1])
 330       p[0] = ListFromConcat(p[2], p[3])
 331
 332   # [18.1] Error recovery for ExceptionMembers
 333   def p_ExceptionMembersError(self, p):
 334     """ExceptionMembers : error"""
 335     p[0] = self.BuildError(p, 'ExceptionMembers')
 336
 337   # [19]
 338   def p_Inheritance(self, p):
 339     """Inheritance : ':' identifier
 340                    |"""
 341     if len(p) > 1:
 342       p[0] = self.BuildNamed('Inherit', p, 2)
 343
 344   # [20]
 345   def p_Enum(self, p):
 346     """Enum : ENUM identifier '{' EnumValueList '}' ';'"""
 347     p[0] = self.BuildNamed('Enum', p, 2, p[4])
 348
 349   # [20.1] Error recovery for Enums
 350   def p_EnumError(self, p):
 351     """Enum : ENUM error ';'"""
 352     p[0] = self.BuildError(p, 'Enum')
 353
 354   # [21]
 355   def p_EnumValueList(self, p):
 356     """EnumValueList : ExtendedAttributeList string EnumValueListComma"""
 357     enum = self.BuildNamed('EnumItem', p, 2, p[1])
 358     p[0] = ListFromConcat(enum, p[3])
 359
 360   # [22]
 361   def p_EnumValueListComma(self, p):
 362     """EnumValueListComma : ',' EnumValueListString
 363                           |"""
 364     if len(p) > 1:
 365       p[0] = p[2]
 366
 367   # [23]
 368   def p_EnumValueListString(self, p):
 369     """EnumValueListString : ExtendedAttributeList string EnumValueListComma
 370                            |"""
 371     if len(p) > 1:
 372       enum = self.BuildNamed('EnumItem', p, 2, p[1])
 373       p[0] = ListFromConcat(enum, p[3])
 374
 375   # [24]
 376   def p_CallbackRest(self, p):
 377     """CallbackRest : identifier '=' ReturnType '(' ArgumentList ')' ';'"""
 378     arguments = self.BuildProduction('Arguments', p, 4, p[5])
 379     p[0] = self.BuildNamed('Callback', p, 1, ListFromConcat(p[3], arguments))
 380
 381   # [25]
 382   def p_Typedef(self, p):
 383     """Typedef : TYPEDEF ExtendedAttributeListNoComments Type identifier ';'"""
 384     p[0] = self.BuildNamed('Typedef', p, 4, ListFromConcat(p[2], p[3]))
 385
 386   # [25.1] Error recovery for Typedefs
 387   def p_TypedefError(self, p):
 388     """Typedef : TYPEDEF error ';'"""
 389     p[0] = self.BuildError(p, 'Typedef')
 390
 391   # [26]
 392   def p_ImplementsStatement(self, p):
 393     """ImplementsStatement : identifier IMPLEMENTS identifier ';'"""
 394     name = self.BuildAttribute('REFERENCE', p[3])
 395     p[0] = self.BuildNamed('Implements', p, 1, name)
 396
 397   # [27]
 398   def p_Const(self,  p):
 399     """Const : CONST ConstType identifier '=' ConstValue ';'"""
 400     value = self.BuildProduction('Value', p, 5, p[5])
 401     p[0] = self.BuildNamed('Const', p, 3, ListFromConcat(p[2], value))
 402
 403   # [28]
 404   def p_ConstValue(self, p):
 405     """ConstValue : BooleanLiteral
 406                   | FloatLiteral
 407                   | integer
 408                   | null"""
 409     if type(p[1]) == str:
 410       p[0] = ListFromConcat(self.BuildAttribute('TYPE', 'integer'),
 411                             self.BuildAttribute('NAME', p[1]))
 412     else:
 413       p[0] = p[1]
 414
 415   # [28.1] Add definition for NULL
 416   def p_null(self, p):
 417     """null : NULL"""
 418     p[0] = ListFromConcat(self.BuildAttribute('TYPE', 'NULL'),
 419                           self.BuildAttribute('NAME', 'NULL'))
 420
 421   # [29]
 422   def p_BooleanLiteral(self, p):
 423     """BooleanLiteral : TRUE
 424                       | FALSE"""
 425     value = self.BuildAttribute('VALUE', Boolean(p[1] == 'true'))
 426     p[0] = ListFromConcat(self.BuildAttribute('TYPE', 'boolean'), value)
 427
 428   # [30]
 429   def p_FloatLiteral(self, p):
 430     """FloatLiteral : float
 431                     | '-' INFINITY
 432                     | INFINITY
 433                     | NAN """
 434     if len(p) > 2:
 435       val = '-Infinity'
 436     else:
 437       val = p[1]
 438     p[0] = ListFromConcat(self.BuildAttribute('TYPE', 'float'),
 439                           self.BuildAttribute('VALUE', val))
 440
 441   # [31] Removed unsupported: Serializer
 442   def p_AttributeOrOperationOrIterator(self, p):
 443     """AttributeOrOperationOrIterator : Stringifier
 444                                       | StaticMember
 445                                       | Attribute
 446                                       | OperationOrIterator"""
 447     p[0] = p[1]
 448
 449   # [32-37] NOT IMPLEMENTED (Serializer)
 450
 451   # [38]
 452   def p_Stringifier(self, p):
 453     """Stringifier : STRINGIFIER StringifierRest"""
 454     p[0] = self.BuildProduction('Stringifier', p, 1, p[2])
 455
 456   # [39]
 457   def p_StringifierRest(self, p):
 458     """StringifierRest : AttributeRest
 459                        | ReturnType OperationRest
 460                        | ';'"""
 461     if len(p) == 3:
 462       p[2].AddChildren(p[1])
 463       p[0] = p[2]
 464     elif p[1] != ';':
 465       p[0] = p[1]
 466
 467   # [40]
 468   def p_StaticMember(self, p):
 469     """StaticMember : STATIC StaticMemberRest"""
 470     p[2].AddChildren(self.BuildTrue('STATIC'))
 471     p[0] = p[2]
 472
 473   # [41]
 474   def p_StaticMemberRest(self, p):
 475     """StaticMemberRest : AttributeRest
 476                         | ReturnType OperationRest"""
 477     if len(p) == 2:
 478       p[0] = p[1]
 479     else:
 480       p[2].AddChildren(p[1])
 481       p[0] = p[2]
 482
 483   # [42]
 484   def p_Attribute(self, p):
 485     """Attribute : Inherit AttributeRest"""
 486     p[2].AddChildren(ListFromConcat(p[1]))
 487     p[0] = p[2]
 488
 489   # [43]
 490   def p_AttributeRest(self, p):
 491     """AttributeRest : ReadOnly ATTRIBUTE Type identifier ';'"""
 492     p[0] = self.BuildNamed('Attribute', p, 4,
 493                            ListFromConcat(p[1], p[3]))
 494
 495   # [44]
 496   def p_Inherit(self, p):
 497     """Inherit : INHERIT
 498                |"""
 499     if len(p) > 1:
 500       p[0] = self.BuildTrue('INHERIT')
 501
 502   # [45]
 503   def p_ReadOnly(self, p):
 504     """ReadOnly : READONLY
 505                 |"""
 506     if len(p) > 1:
 507       p[0] = self.BuildTrue('READONLY')
 508
 509   # [46]
 510   def p_OperationOrIterator(self, p):
 511     """OperationOrIterator : ReturnType OperationOrIteratorRest
 512                            | SpecialOperation"""
 513     if len(p) == 3:
 514       p[2].AddChildren(p[1])
 515       p[0] = p[2]
 516     else:
 517       p[0] = p[1]
 518
 519   # [47]
 520   def p_SpecialOperation(self, p):
 521     """SpecialOperation : Special Specials ReturnType OperationRest"""
 522     p[4].AddChildren(ListFromConcat(p[1], p[2], p[3]))
 523     p[0] = p[4]
 524
 525   # [48]
 526   def p_Specials(self, p):
 527     """Specials : Special Specials
 528                 | """
 529     if len(p) > 1:
 530       p[0] = ListFromConcat(p[1], p[2])
 531
 532   # [49]
 533   def p_Special(self, p):
 534     """Special : GETTER
 535                | SETTER
 536                | CREATOR
 537                | DELETER
 538                | LEGACYCALLER"""
 539     p[0] = self.BuildTrue(p[1].upper())
 540
 541   # [50] Removed unsupported: IteratorRest
 542   def p_OperationOrIteratorRest(self, p):
 543     """OperationOrIteratorRest : OperationRest"""
 544     p[0] = p[1]
 545
 546   # [51-53] NOT IMPLEMENTED (IteratorRest)
 547
 548   # [54]
 549   def p_OperationRest(self, p):
 550     """OperationRest : OptionalIdentifier '(' ArgumentList ')' ';'"""
 551     arguments = self.BuildProduction('Arguments', p, 2, p[3])
 552     p[0] = self.BuildNamed('Operation', p, 1, arguments)
 553
 554   # [55]
 555   def p_OptionalIdentifier(self, p):
 556     """OptionalIdentifier : identifier
 557                           |"""
 558     if len(p) > 1:
 559       p[0] = p[1]
 560     else:
 561       p[0] = '_unnamed_'
 562
 563   # [56]
 564   def p_ArgumentList(self, p):
 565     """ArgumentList : Argument Arguments
 566                     |"""
 567     if len(p) > 1:
 568       p[0] = ListFromConcat(p[1], p[2])
 569
 570   # [56.1] ArgumentList error recovery
 571   def p_ArgumentListError(self, p):
 572     """ArgumentList : error """
 573     p[0] = self.BuildError(p, 'ArgumentList')
 574
 575   # [57]
 576   def p_Arguments(self, p):
 577     """Arguments : ',' Argument Arguments
 578                  |"""
 579     if len(p) > 1:
 580       p[0] = ListFromConcat(p[2], p[3])
 581
 582   # [58]
 583   def p_Argument(self, p):
 584     """Argument : ExtendedAttributeList OptionalOrRequiredArgument"""
 585     p[2].AddChildren(p[1])
 586     p[0] = p[2]
 587
 588   # [59]
 589   def p_OptionalOrRequiredArgument(self, p):
 590     """OptionalOrRequiredArgument : OPTIONAL Type ArgumentName Default
 591                                   | Type Ellipsis ArgumentName"""
 592     if len(p) > 4:
 593       arg = self.BuildNamed('Argument', p, 3, ListFromConcat(p[2], p[4]))
 594       arg.AddChildren(self.BuildTrue('OPTIONAL'))
 595     else:
 596       arg = self.BuildNamed('Argument', p, 3, ListFromConcat(p[1], p[2]))
 597     p[0] = arg
 598
 599   # [60]
 600   def p_ArgumentName(self, p):
 601     """ArgumentName : ArgumentNameKeyword
 602                     | identifier"""
 603     p[0] = p[1]
 604
 605   # [61]
 606   def p_Ellipsis(self, p):
 607     """Ellipsis : ELLIPSIS
 608                 |"""
 609     if len(p) > 1:
 610       p[0] = self.BuildNamed('Argument', p, 1)
 611       p[0].AddChildren(self.BuildTrue('ELLIPSIS'))
 612
 613   # [62]
 614   def p_ExceptionMember(self, p):
 615     """ExceptionMember : Const
 616                        | ExceptionField"""
 617     p[0] = p[1]
 618
 619   # [63]
 620   def p_ExceptionField(self, p):
 621     """ExceptionField : Type identifier ';'"""
 622     p[0] = self.BuildNamed('ExceptionField', p, 2, p[1])
 623
 624   # [63.1] Error recovery for ExceptionMembers
 625   def p_ExceptionFieldError(self, p):
 626     """ExceptionField : error"""
 627     p[0] = self.BuildError(p, 'ExceptionField')
 628
 629   # [64] No comment version for mid statement attributes.
 630   def p_ExtendedAttributeListNoComments(self, p):
 631     """ExtendedAttributeListNoComments : '[' ExtendedAttribute ExtendedAttributes ']'
 632                                        | """
 633     if len(p) > 2:
 634       items = ListFromConcat(p[2], p[3])
 635       p[0] = self.BuildProduction('ExtAttributes', p, 1, items)
 636
 637   # [64.1] Add optional comment field for start of statements.
 638   def p_ExtendedAttributeList(self, p):
 639     """ExtendedAttributeList : Comments '[' ExtendedAttribute ExtendedAttributes ']'
 640                              | Comments """
 641     if len(p) > 2:
 642       items = ListFromConcat(p[3], p[4])
 643       attribs = self.BuildProduction('ExtAttributes', p, 2, items)
 644       p[0] = ListFromConcat(p[1], attribs)
 645     else:
 646       p[0] = p[1]
 647
 648   # [65]
 649   def p_ExtendedAttributes(self, p):
 650     """ExtendedAttributes : ',' ExtendedAttribute ExtendedAttributes
 651                           |"""
 652     if len(p) > 1:
 653       p[0] = ListFromConcat(p[2], p[3])
 654
 655   # We only support:
 656   #    [ identifier ]
 657   #    [ identifier ( ArgumentList ) ]
 658   #    [ identifier = identifier ]
 659   #    [ identifier = ( IdentifierList ) ]
 660   #    [ identifier = identifier ( ArgumentList ) ]
 661   # [66] map directly to [91-93, 95]
 662   # [67-69, 71] are unsupported
 663   def p_ExtendedAttribute(self, p):
 664     """ExtendedAttribute : ExtendedAttributeNoArgs
 665                          | ExtendedAttributeArgList
 666                          | ExtendedAttributeIdent
 667                          | ExtendedAttributeIdentList
 668                          | ExtendedAttributeNamedArgList"""
 669     p[0] = p[1]
 670
 671   # [70]
 672   def p_ArgumentNameKeyword(self, p):
 673     """ArgumentNameKeyword : ATTRIBUTE
 674                            | CALLBACK
 675                            | CONST
 676                            | CREATOR
 677                            | DELETER
 678                            | DICTIONARY
 679                            | ENUM
 680                            | EXCEPTION
 681                            | GETTER
 682                            | IMPLEMENTS
 683                            | INHERIT
 684                            | LEGACYCALLER
 685                            | PARTIAL
 686                            | SERIALIZER
 687                            | SETTER
 688                            | STATIC
 689                            | STRINGIFIER
 690                            | TYPEDEF
 691                            | UNRESTRICTED"""
 692     p[0] = p[1]
 693
 694   # [72]
 695   def p_Type(self, p):
 696     """Type : SingleType
 697             | UnionType TypeSuffix"""
 698     if len(p) == 2:
 699       p[0] = self.BuildProduction('Type', p, 1, p[1])
 700     else:
 701       p[0] = self.BuildProduction('Type', p, 1, ListFromConcat(p[1], p[2]))
 702
 703   # [73]
 704   def p_SingleType(self, p):
 705     """SingleType : NonAnyType
 706                   | ANY TypeSuffixStartingWithArray"""
 707     if len(p) == 2:
 708       p[0] = p[1]
 709     else:
 710       p[0] = ListFromConcat(self.BuildProduction('Any', p, 1), p[2])
 711
 712   # [74]
 713   def p_UnionType(self, p):
 714     """UnionType : '(' UnionMemberType OR UnionMemberType UnionMemberTypes ')'"""
 715
 716   # [75]
 717   def p_UnionMemberType(self, p):
 718     """UnionMemberType : NonAnyType
 719                        | UnionType TypeSuffix
 720                        | ANY '[' ']' TypeSuffix"""
 721   # [76]
 722   def p_UnionMemberTypes(self, p):
 723     """UnionMemberTypes : OR UnionMemberType UnionMemberTypes
 724                         |"""
 725
 726   # [77] Moved BYTESTRING, DOMSTRING, OBJECT, DATE, REGEXP to PrimitiveType
 727   # Moving all built-in types into PrimitiveType makes it easier to
 728   # differentiate between them and 'identifier', since p[1] would be a string in
 729   # both cases.
 730   def p_NonAnyType(self, p):
 731     """NonAnyType : PrimitiveType TypeSuffix
 732                   | identifier TypeSuffix
 733                   | SEQUENCE '<' Type '>' Null"""
 734     if len(p) == 3:
 735       if type(p[1]) == str:
 736         typeref = self.BuildNamed('Typeref', p, 1)
 737       else:
 738         typeref = p[1]
 739       p[0] = ListFromConcat(typeref, p[2])
 740
 741     if len(p) == 6:
 742       p[0] = self.BuildProduction('Sequence', p, 1, ListFromConcat(p[3], p[5]))
 743
 744
 745   # [78]
 746   def p_ConstType(self,  p):
 747     """ConstType : PrimitiveType Null
 748                  | identifier Null"""
 749     if type(p[1]) == str:
 750       p[0] = self.BuildNamed('Typeref', p, 1, p[2])
 751     else:
 752       p[1].AddChildren(p[2])
 753       p[0] = p[1]
 754
 755
 756   # [79] Added BYTESTRING, DOMSTRING, OBJECT, DATE, REGEXP
 757   def p_PrimitiveType(self, p):
 758     """PrimitiveType : UnsignedIntegerType
 759                      | UnrestrictedFloatType
 760                      | BOOLEAN
 761                      | BYTE
 762                      | OCTET
 763                      | BYTESTRING
 764                      | DOMSTRING
 765                      | OBJECT
 766                      | DATE
 767                      | REGEXP"""
 768     if type(p[1]) == str:
 769       p[0] = self.BuildNamed('PrimitiveType', p, 1)
 770     else:
 771       p[0] = p[1]
 772
 773
 774   # [80]
 775   def p_UnrestrictedFloatType(self, p):
 776     """UnrestrictedFloatType : UNRESTRICTED FloatType
 777                              | FloatType"""
 778     if len(p) == 2:
 779       typeref = self.BuildNamed('PrimitiveType', p, 1)
 780     else:
 781       typeref = self.BuildNamed('PrimitiveType', p, 2)
 782       typeref.AddChildren(self.BuildTrue('UNRESTRICTED'))
 783     p[0] = typeref
 784
 785
 786   # [81]
 787   def p_FloatType(self, p):
 788     """FloatType : FLOAT
 789                  | DOUBLE"""
 790     p[0] = p[1]
 791
 792   # [82]
 793   def p_UnsignedIntegerType(self, p):
 794     """UnsignedIntegerType : UNSIGNED IntegerType
 795                            | IntegerType"""
 796     if len(p) == 2:
 797       p[0] = p[1]
 798     else:
 799       p[0] = 'unsigned ' + p[2]
 800
 801   # [83]
 802   def p_IntegerType(self, p):
 803     """IntegerType : SHORT
 804                    | LONG OptionalLong"""
 805     if len(p) == 2:
 806       p[0] = p[1]
 807     else:
 808       p[0] = p[1] + p[2]
 809
 810   # [84]
 811   def p_OptionalLong(self, p):
 812     """OptionalLong : LONG
 813                     | """
 814     if len(p) > 1:
 815       p[0] = ' ' + p[1]
 816     else:
 817       p[0] = ''
 818
 819
 820   # [85] Add support for sized array
 821   def p_TypeSuffix(self, p):
 822     """TypeSuffix : '[' integer ']' TypeSuffix
 823                   | '[' ']' TypeSuffix
 824                   | '?' TypeSuffixStartingWithArray
 825                   | """
 826     if len(p) == 5:
 827       p[0] = self.BuildNamed('Array', p, 2, p[4])
 828
 829     if len(p) == 4:
 830       p[0] = self.BuildProduction('Array', p, 1, p[3])
 831
 832     if len(p) == 3:
 833       p[0] = ListFromConcat(self.BuildTrue('NULLABLE'), p[2])
 834
 835
 836   # [86]
 837   def p_TypeSuffixStartingWithArray(self, p):
 838     """TypeSuffixStartingWithArray : '[' ']' TypeSuffix
 839                                    | """
 840     if len(p) > 1:
 841       p[0] = self.BuildProduction('Array', p, 0, p[3])
 842
 843   # [87]
 844   def p_Null(self, p):
 845     """Null : '?'
 846             |"""
 847     if len(p) > 1:
 848       p[0] = self.BuildTrue('NULLABLE')
 849
 850   # [88]
 851   def p_ReturnType(self, p):
 852     """ReturnType : Type
 853                   | VOID"""
 854     if p[1] == 'void':
 855       p[0] = self.BuildProduction('Type', p, 1)
 856       p[0].AddChildren(self.BuildNamed('PrimitiveType', p, 1))
 857     else:
 858       p[0] = p[1]
 859
 860   # [89]
 861   def p_IdentifierList(self, p):
 862     """IdentifierList : identifier Identifiers"""
 863     p[0] = ListFromConcat(p[1], p[2])
 864
 865   # [90]
 866   def p_Identifiers(self, p):
 867     """Identifiers : ',' identifier Identifiers
 868                    |"""
 869     if len(p) > 1:
 870       p[0] = ListFromConcat(p[2], p[3])
 871
 872   # [91]
 873   def p_ExtendedAttributeNoArgs(self, p):
 874     """ExtendedAttributeNoArgs : identifier"""
 875     p[0] = self.BuildNamed('ExtAttribute', p, 1)
 876
 877   # [92]
 878   def p_ExtendedAttributeArgList(self, p):
 879     """ExtendedAttributeArgList : identifier '(' ArgumentList ')'"""
 880     arguments = self.BuildProduction('Arguments', p, 2, p[3])
 881     p[0] = self.BuildNamed('ExtAttribute', p, 1, arguments)
 882
 883   # [93]
 884   def p_ExtendedAttributeIdent(self, p):
 885     """ExtendedAttributeIdent : identifier '=' identifier"""
 886     value = self.BuildAttribute('VALUE', p[3])
 887     p[0] = self.BuildNamed('ExtAttribute', p, 1, value)
 888
 889   # [94]
 890   def p_ExtendedAttributeIdentList(self, p):
 891     """ExtendedAttributeIdentList : identifier '=' '(' IdentifierList ')'"""
 892     value = self.BuildAttribute('VALUE', p[4])
 893     p[0] = self.BuildNamed('ExtAttribute', p, 1, value)
 894
 895   # [95]
 896   def p_ExtendedAttributeNamedArgList(self, p):
 897     """ExtendedAttributeNamedArgList : identifier '=' identifier '(' ArgumentList ')'"""
 898     args = self.BuildProduction('Arguments', p, 4, p[5])
 899     value = self.BuildNamed('Call', p, 3, args)
 900     p[0] = self.BuildNamed('ExtAttribute', p, 1, value)
 901
 902   # [96] NOT IMPLEMENTED (ExtendedAttributeTypePair)
 903
 904 #
 905 # Parser Errors
 906 #
 907 # p_error is called whenever the parser can not find a pattern match for
 908 # a set of items from the current state.  The p_error function defined here
 909 # is triggered logging an error, and parsing recovery happens as the
 910 # p_<type>_error functions defined above are called.  This allows the parser
 911 # to continue so as to capture more than one error per file.
 912 #
 913   def p_error(self, t):
 914     if t:
 915       lineno = t.lineno
 916       pos = t.lexpos
 917       prev = self.yaccobj.symstack[-1]
 918       if type(prev) == lex.LexToken:
 919         msg = "Unexpected %s after %s." % (
 920             TokenTypeName(t), TokenTypeName(prev))
 921       else:
 922         msg = "Unexpected %s." % (t.value)
 923     else:
 924       last = self.LastToken()
 925       lineno = last.lineno
 926       pos = last.lexpos
 927       msg = "Unexpected end of file after %s." % TokenTypeName(last)
 928       self.yaccobj.restart()
 929
 930     # Attempt to remap the error to a friendlier form
 931     if msg in ERROR_REMAP:
 932       msg = ERROR_REMAP[msg]
 933
 934     self._last_error_msg = msg
 935     self._last_error_lineno = lineno
 936     self._last_error_pos = pos
 937
 938   def Warn(self, node, msg):
 939     sys.stdout.write(node.GetLogLine(msg))
 940     self.parse_warnings += 1
 941
 942   def LastToken(self):
 943     return self.lexer.last
 944
 945   def __init__(self, lexer, verbose=False, debug=False, mute_error=False):
 946     self.lexer = lexer
 947     self.tokens = lexer.KnownTokens()
 948     self.yaccobj = yacc.yacc(module=self, tabmodule=None, debug=debug,
 949                              optimize=0, write_tables=0)
 950     self.parse_debug = debug
 951     self.verbose = verbose
 952     self.mute_error = mute_error
 953     self._parse_errors = 0
 954     self._parse_warnings = 0
 955     self._last_error_msg = None
 956     self._last_error_lineno = 0
 957     self._last_error_pos = 0
 958
 959
 960 #
 961 # BuildProduction
 962 #
 963 # Production is the set of items sent to a grammar rule resulting in a new
 964 # item being returned.
 965 #
 966 # p - Is the Yacc production object containing the stack of items
 967 # index - Index into the production of the name for the item being produced.
 968 # cls - The type of item being producted
 969 # childlist - The children of the new item
 970   def BuildProduction(self, cls, p, index, childlist=None):
 971     try:
 972       if not childlist:
 973         childlist = []
 974
 975       filename = self.lexer.Lexer().filename
 976       lineno = p.lineno(index)
 977       pos = p.lexpos(index)
 978       out = IDLNode(cls, filename, lineno, pos, childlist)
 979       return out
 980     except:
 981       print 'Exception while parsing:'
 982       for num, item in enumerate(p):
 983         print '  [%d] %s' % (num, ExpandProduction(item))
 984       if self.LastToken():
 985         print 'Last token: %s' % str(self.LastToken())
 986       raise
 987
 988   def BuildNamed(self, cls, p, index, childlist=None):
 989     childlist = ListFromConcat(childlist)
 990     childlist.append(self.BuildAttribute('NAME', p[index]))
 991     return self.BuildProduction(cls, p, index, childlist)
 992
 993   def BuildComment(self, cls, p, index):
 994     name = p[index]
 995
 996     # Remove comment markers
 997     lines = []
 998     if name[:2] == '//':
 999       # For C++ style, remove any leading whitespace and the '//' marker from
1000       # each line.
1001       form = 'cc'
1002       for line in name.split('\n'):
1003         start = line.find('//')
1004         lines.append(line[start+2:])
1005     else:
1006       # For C style, remove ending '*/''
1007       form = 'c'
1008       for line in name[:-2].split('\n'):
1009         # Remove characters until start marker for this line '*' if found
1010         # otherwise it should be blank.
1011         offs = line.find('*')
1012         if offs >= 0:
1013           line = line[offs + 1:].rstrip()
1014         else:
1015           line = ''
1016         lines.append(line)
1017     name = '\n'.join(lines)
1018     childlist = [self.BuildAttribute('NAME', name),
1019                  self.BuildAttribute('FORM', form)]
1020     return self.BuildProduction(cls, p, index, childlist)
1021
1022 #
1023 # BuildError
1024 #
1025 # Build and Errror node as part of the recovery process.
1026 #
1027 #
1028   def BuildError(self, p, prod):
1029     self._parse_errors += 1
1030     name = self.BuildAttribute('NAME', self._last_error_msg)
1031     line = self.BuildAttribute('LINE', self._last_error_lineno)
1032     pos = self.BuildAttribute('POS', self._last_error_pos)
1033     prod = self.BuildAttribute('PROD', prod)
1034
1035     node = self.BuildProduction('Error', p, 1,
1036                                 ListFromConcat(name, line, pos, prod))
1037     if not self.mute_error:
1038       node.Error(self._last_error_msg)
1039
1040     return node
1041
1042 #
1043 # BuildAttribute
1044 #
1045 # An ExtendedAttribute is a special production that results in a property
1046 # which is applied to the adjacent item.  Attributes have no children and
1047 # instead represent key/value pairs.
1048 #
1049   def BuildAttribute(self, key, val):
1050     return IDLAttribute(key, val)
1051
1052   def BuildFalse(self, key):
1053     return IDLAttribute(key, Boolean(False))
1054
1055   def BuildTrue(self, key):
1056     return IDLAttribute(key, Boolean(True))
1057
1058   def GetErrors(self):
1059     # Access lexer errors, despite being private
1060     # pylint: disable=W0212
1061     return self._parse_errors + self.lexer._lex_errors
1062
1063 #
1064 # ParseData
1065 #
1066 # Attempts to parse the current data loaded in the lexer.
1067 #
1068   def ParseText(self, filename, data):
1069     self._parse_errors = 0
1070     self._parse_warnings = 0
1071     self._last_error_msg = None
1072     self._last_error_lineno = 0
1073     self._last_error_pos = 0
1074
1075     try:
1076       self.lexer.Tokenize(data, filename)
1077       nodes = self.yaccobj.parse(lexer=self.lexer) or []
1078       name = self.BuildAttribute('NAME', filename)
1079       return IDLNode('File', filename, 0, 0, nodes + [name])
1080
1081     except lex.LexError as lexError:
1082       sys.stderr.write('Error in token: %s\n' % str(lexError))
1083     return None
1084
1085
1086
1087 def ParseFile(parser, filename):
1088   """Parse a file and return a File type of node."""
1089   with open(filename) as fileobject:
1090     try:
1091       out = parser.ParseText(filename, fileobject.read())
1092       out.SetProperty('DATETIME', time.ctime(os.path.getmtime(filename)))
1093       out.SetProperty('ERRORS', parser.GetErrors())
1094       return out
1095
1096     except Exception as e:
1097       last = parser.LastToken()
1098       sys.stderr.write('%s(%d) : Internal parsing error\n\t%s.\n' % (
1099                        filename, last.lineno, str(e)))
1100
1101
1102 def main(argv):
1103   nodes = []
1104   parser = IDLParser(IDLLexer())
1105   errors = 0
1106   for filename in argv:
1107     filenode = ParseFile(parser, filename)
1108     if (filenode):
1109       errors += filenode.GetProperty('ERRORS')
1110       nodes.append(filenode)
1111
1112   ast = IDLNode('AST', '__AST__', 0, 0, nodes)
1113
1114   print '\n'.join(ast.Tree(accept_props=['PROD']))
1115   if errors:
1116     print '\nFound %d errors.\n' % errors
1117
1118   return errors
1119
1120
1121 if __name__ == '__main__':
1122   sys.exit(main(sys.argv[1:]))