tools/idl_parser/idl_parser.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """ Parser for PPAPI IDL """
   7
   8 #
   9 # IDL Parser
  10 #
  11 # The parser is uses the PLY yacc library to build a set of parsing rules based
  12 # on WebIDL.
  13 #
  14 # WebIDL, and WebIDL grammar can be found at:
  15 #   http://heycam.github.io/webidl/
  16 # PLY can be found at:
  17 #   http://www.dabeaz.com/ply/
  18 #
  19 # The parser generates a tree by recursively matching sets of items against
  20 # defined patterns.  When a match is made, that set of items is reduced
  21 # to a new item.   The new item can provide a match for parent patterns.
  22 # In this way an AST is built (reduced) depth first.
  23 #
  24
  25 #
  26 # Disable check for line length and Member as Function due to how grammar rules
  27 # are defined with PLY
  28 #
  29 # pylint: disable=R0201
  30 # pylint: disable=C0301
  31
  32 import os.path
  33 import sys
  34 import time
  35
  36 from idl_lexer import IDLLexer
  37 from idl_node import IDLAttribute, IDLNode
  38
  39 #
  40 # Try to load the ply module, if not, then assume it is in the third_party
  41 # directory.
  42 #
  43 try:
  44   # Disable lint check which fails to find the ply module.
  45   # pylint: disable=F0401
  46   from ply import lex
  47   from ply import yacc
  48 except ImportError:
  49   module_path, module_name = os.path.split(__file__)
  50   third_party = os.path.join(module_path, os.par, os.par, 'third_party')
  51   sys.path.append(third_party)
  52   # pylint: disable=F0401
  53   from ply import lex
  54   from ply import yacc
  55
  56 #
  57 # ERROR_REMAP
  58 #
  59 # Maps the standard error formula into a more friendly error message.
  60 #
  61 ERROR_REMAP = {
  62   'Unexpected ")" after "(".' : 'Empty argument list.',
  63   'Unexpected ")" after ",".' : 'Missing argument.',
  64   'Unexpected "}" after ",".' : 'Trailing comma in block.',
  65   'Unexpected "}" after "{".' : 'Unexpected empty block.',
  66   'Unexpected comment after "}".' : 'Unexpected trailing comment.',
  67   'Unexpected "{" after keyword "enum".' : 'Enum missing name.',
  68   'Unexpected "{" after keyword "struct".' : 'Struct missing name.',
  69   'Unexpected "{" after keyword "interface".' : 'Interface missing name.',
  70 }
  71
  72
  73 def Boolean(val):
  74   """Convert to strict boolean type."""
  75   if val:
  76     return True
  77   return False
  78
  79
  80 def ListFromConcat(*items):
  81   """Generate list by concatenating inputs"""
  82   itemsout = []
  83   for item in items:
  84     if item is None:
  85       continue
  86     if type(item) is not type([]):
  87       itemsout.append(item)
  88     else:
  89       itemsout.extend(item)
  90
  91   return itemsout
  92
  93 def ExpandProduction(p):
  94   if type(p) == list:
  95     return '[' + ', '.join([ExpandProduction(x) for x in p]) + ']'
  96   if type(p) == IDLNode:
  97     return 'Node:' + str(p)
  98   if type(p) == IDLAttribute:
  99     return 'Attr:' + str(p)
 100   if type(p) == str:
 101     return 'str:' + p
 102   return '%s:%s' % (p.__class__.__name__, str(p))
 103
 104 # TokenTypeName
 105 #
 106 # Generate a string which has the type and value of the token.
 107 #
 108 def TokenTypeName(t):
 109   if t.type == 'SYMBOL':
 110     return 'symbol %s' % t.value
 111   if t.type in ['HEX', 'INT', 'OCT', 'FLOAT']:
 112     return 'value %s' % t.value
 113   if t.type == 'string' :
 114     return 'string "%s"' % t.value
 115   if t.type == 'COMMENT' :
 116     return 'comment'
 117   if t.type == t.value:
 118     return '"%s"' % t.value
 119   if t.type == ',':
 120     return 'Comma'
 121   if t.type == 'identifier':
 122     return 'identifier "%s"' % t.value
 123   return 'keyword "%s"' % t.value
 124
 125
 126 #
 127 # IDL Parser
 128 #
 129 # The Parser inherits the from the Lexer to provide PLY with the tokenizing
 130 # definitions.  Parsing patterns are encoded as functions where p_<name> is
 131 # is called any time a patern matching the function documentation is found.
 132 # Paterns are expressed in the form of:
 133 # """ <new item> : <item> ....
 134 #                | <item> ...."""
 135 #
 136 # Where new item is the result of a match against one or more sets of items
 137 # separated by the "|".
 138 #
 139 # The function is called with an object 'p' where p[0] is the output object
 140 # and p[n] is the set of inputs for positive values of 'n'.  Len(p) can be
 141 # used to distinguish between multiple item sets in the pattern.
 142 #
 143 # For more details on parsing refer to the PLY documentation at
 144 #    http://www.dabeaz.com/ply/
 145 #
 146 # The parser is based on the WebIDL standard.  See:
 147 #    http://heycam.github.io/webidl/#idl-grammar
 148 #
 149 # The various productions are annotated so that the WHOLE number greater than
 150 # zero in the comment denotes the matching WebIDL grammar definition.
 151 #
 152 # Productions with a fractional component in the comment denote additions to
 153 # the WebIDL spec, such as comments.
 154 #
 155
 156
 157 class IDLParser(object):
 158 #
 159 # We force all input files to start with two comments.  The first comment is a
 160 # Copyright notice followed by a file comment and finally by file level
 161 # productions.
 162 #
 163   # [0] Insert a TOP definition for Copyright and Comments
 164   def p_Top(self, p):
 165     """Top : COMMENT COMMENT Definitions"""
 166     Copyright = self.BuildComment('Copyright', p, 1)
 167     Filedoc = self.BuildComment('Comment', p, 2)
 168     p[0] = ListFromConcat(Copyright, Filedoc, p[3])
 169
 170   # [0.1] Add support for Multiple COMMENTS
 171   def p_Comments(self, p):
 172     """Comments : CommentsRest"""
 173     if len(p) > 1:
 174       p[0] = p[1]
 175
 176   # [0.2] Produce a COMMENT and aggregate sibling comments
 177   def p_CommentsRest(self, p):
 178     """CommentsRest : COMMENT CommentsRest
 179                     | """
 180     if len(p) > 1:
 181       p[0] = ListFromConcat(self.BuildComment('Comment', p, 1), p[2])
 182
 183
 184 #
 185 #The parser is based on the WebIDL standard.  See:
 186 # http://heycam.github.io/webidl/#idl-grammar
 187 #
 188   # [1]
 189   def p_Definitions(self, p):
 190     """Definitions : ExtendedAttributeList Definition Definitions
 191            | """
 192     if len(p) > 1:
 193       p[2].AddChildren(p[1])
 194       p[0] = ListFromConcat(p[2], p[3])
 195
 196   # [2]
 197   def p_Definition(self, p):
 198     """Definition : CallbackOrInterface
 199                   | Partial
 200                   | Dictionary
 201                   | Exception
 202                   | Enum
 203                   | Typedef
 204                   | ImplementsStatement"""
 205     p[0] = p[1]
 206
 207   # [2.1] Error recovery for definition
 208   def p_DefinitionError(self, p):
 209     """Definition : error ';'"""
 210     p[0] = self.BuildError(p, 'Definition')
 211
 212   # [3]
 213   def p_CallbackOrInterface(self, p):
 214     """CallbackOrInterface : CALLBACK CallbackRestOrInterface
 215                            | Interface"""
 216     if len(p) > 2:
 217       p[0] = p[2]
 218     else:
 219       p[0] = p[1]
 220
 221   # [4]
 222   def p_CallbackRestOrInterface(self, p):
 223     """CallbackRestOrInterface : CallbackRest
 224                                | Interface"""
 225     p[0] = p[1]
 226
 227   # [5]
 228   def p_Interface(self, p):
 229     """Interface : INTERFACE identifier Inheritance '{' InterfaceMembers '}' ';'"""
 230     p[0] = self.BuildNamed('Interface', p, 2, ListFromConcat(p[3], p[5]))
 231
 232   # [5.1] Error recovery for interface.
 233   def p_InterfaceError(self, p):
 234     """Interface : INTERFACE identifier Inheritance '{' error"""
 235     p[0] = self.BuildError(p, 'Interface')
 236
 237   # [6]
 238   def p_Partial(self, p):
 239     """Partial : PARTIAL PartialDefinition"""
 240     p[2].AddChildren(self.BuildTrue('Partial'))
 241     p[0] = p[2]
 242
 243   # [6.1] Error recovery for Partial
 244   def p_PartialError(self, p):
 245     """Partial : PARTIAL error"""
 246     p[0] = self.BuildError(p, 'Partial')
 247
 248   # [7]
 249   def p_PartialDefinition(self, p):
 250     """PartialDefinition : PartialDictionary
 251                          | PartialInterface"""
 252     p[0] = p[1]
 253
 254   # [8]
 255   def p_PartialInterface(self, p):
 256     """PartialInterface : INTERFACE identifier '{' InterfaceMembers '}' ';'"""
 257     p[0] = self.BuildNamed('Interface', p, 2, p[4])
 258
 259   # [9]
 260   def p_InterfaceMembers(self, p):
 261     """InterfaceMembers : ExtendedAttributeList InterfaceMember InterfaceMembers
 262                         |"""
 263     if len(p) > 1:
 264       p[2].AddChildren(p[1])
 265       p[0] = ListFromConcat(p[2], p[3])
 266
 267   # [9.1] Error recovery for InterfaceMembers
 268   def p_InterfaceMembersError(self, p):
 269     """InterfaceMembers : error"""
 270     p[0] = self.BuildError(p, 'InterfaceMembers')
 271
 272   # [10] Removed unsupported: Serializer
 273   def p_InterfaceMember(self, p):
 274     """InterfaceMember : Const
 275                        | Operation
 276                        | Serializer
 277                        | Stringifier
 278                        | StaticMember
 279                        | Iterable
 280                        | ReadonlyMember
 281                        | ReadWriteAttribute
 282                        | ReadWriteMaplike
 283                        | ReadWriteSetlike"""
 284     p[0] = p[1]
 285
 286   # [11]
 287   def p_Dictionary(self, p):
 288     """Dictionary : DICTIONARY identifier Inheritance '{' DictionaryMembers '}' ';'"""
 289     p[0] = self.BuildNamed('Dictionary', p, 2, ListFromConcat(p[3], p[5]))
 290
 291   # [11.1] Error recovery for regular Dictionary
 292   def p_DictionaryError(self, p):
 293     """Dictionary : DICTIONARY error ';'"""
 294     p[0] = self.BuildError(p, 'Dictionary')
 295
 296   # [11.2] Error recovery for regular Dictionary
 297   # (for errors inside dictionary definition)
 298   def p_DictionaryError2(self, p):
 299     """Dictionary : DICTIONARY identifier Inheritance '{' error"""
 300     p[0] = self.BuildError(p, 'Dictionary')
 301
 302   # [12]
 303   def p_DictionaryMembers(self, p):
 304     """DictionaryMembers : ExtendedAttributeList DictionaryMember DictionaryMembers
 305                          |"""
 306     if len(p) > 1:
 307       p[2].AddChildren(p[1])
 308       p[0] = ListFromConcat(p[2], p[3])
 309
 310   # [13]
 311   def p_DictionaryMember(self, p):
 312     """DictionaryMember : Required Type identifier Default ';'"""
 313     p[0] = self.BuildNamed('Key', p, 3, ListFromConcat(p[1], p[2], p[4]))
 314
 315   # [14]
 316   def p_Required(self, p):
 317     """Required : REQUIRED
 318                 |"""
 319     if len(p) > 1:
 320       p[0] = self.BuildTrue('REQUIRED')
 321
 322   # [15]
 323   def p_PartialDictionary(self, p):
 324     """PartialDictionary : DICTIONARY identifier '{' DictionaryMembers '}' ';'"""
 325     partial = self.BuildTrue('Partial')
 326     p[0] = self.BuildNamed('Dictionary', p, 2, ListFromConcat(p[4], partial))
 327
 328   # [15.1] Error recovery for Partial Dictionary
 329   def p_PartialDictionaryError(self, p):
 330     """PartialDictionary : DICTIONARY error ';'"""
 331     p[0] = self.BuildError(p, 'PartialDictionary')
 332
 333   # [16]
 334   def p_Default(self, p):
 335     """Default : '=' DefaultValue
 336                |"""
 337     if len(p) > 1:
 338       p[0] = self.BuildProduction('Default', p, 2, p[2])
 339
 340   # [17]
 341   def p_DefaultValue(self, p):
 342     """DefaultValue : ConstValue
 343                     | string
 344                     | '[' ']'"""
 345     if len(p) == 3:
 346       p[0] = ListFromConcat(self.BuildAttribute('TYPE', 'sequence'),
 347                             self.BuildAttribute('VALUE', '[]'))
 348     elif type(p[1]) == str:
 349       p[0] = ListFromConcat(self.BuildAttribute('TYPE', 'DOMString'),
 350                             self.BuildAttribute('NAME', p[1]))
 351     else:
 352       p[0] = p[1]
 353
 354   # [] - Not specified
 355   def p_Exception(self, p):
 356     """Exception : EXCEPTION identifier Inheritance '{' ExceptionMembers '}' ';'"""
 357     p[0] = self.BuildNamed('Exception', p, 2, ListFromConcat(p[3], p[5]))
 358
 359   # [] - Not specified
 360   def p_ExceptionMembers(self, p):
 361     """ExceptionMembers : ExtendedAttributeList ExceptionMember ExceptionMembers
 362                         |"""
 363     if len(p) > 1:
 364       p[2].AddChildren(p[1])
 365       p[0] = ListFromConcat(p[2], p[3])
 366
 367   # [.1] Error recovery for ExceptionMembers - Not specified
 368   def p_ExceptionMembersError(self, p):
 369     """ExceptionMembers : error"""
 370     p[0] = self.BuildError(p, 'ExceptionMembers')
 371
 372   # [18]
 373   def p_Inheritance(self, p):
 374     """Inheritance : ':' identifier
 375                    |"""
 376     if len(p) > 1:
 377       p[0] = self.BuildNamed('Inherit', p, 2)
 378
 379   # [19]
 380   def p_Enum(self, p):
 381     """Enum : ENUM identifier '{' EnumValueList '}' ';'"""
 382     p[0] = self.BuildNamed('Enum', p, 2, p[4])
 383
 384   # [19.1] Error recovery for Enums
 385   def p_EnumError(self, p):
 386     """Enum : ENUM error ';'"""
 387     p[0] = self.BuildError(p, 'Enum')
 388
 389   # [20]
 390   def p_EnumValueList(self, p):
 391     """EnumValueList : ExtendedAttributeList string EnumValueListComma"""
 392     enum = self.BuildNamed('EnumItem', p, 2, p[1])
 393     p[0] = ListFromConcat(enum, p[3])
 394
 395   # [21]
 396   def p_EnumValueListComma(self, p):
 397     """EnumValueListComma : ',' EnumValueListString
 398                           |"""
 399     if len(p) > 1:
 400       p[0] = p[2]
 401
 402   # [22]
 403   def p_EnumValueListString(self, p):
 404     """EnumValueListString : ExtendedAttributeList string EnumValueListComma
 405                            |"""
 406     if len(p) > 1:
 407       enum = self.BuildNamed('EnumItem', p, 2, p[1])
 408       p[0] = ListFromConcat(enum, p[3])
 409
 410   # [23]
 411   def p_CallbackRest(self, p):
 412     """CallbackRest : identifier '=' ReturnType '(' ArgumentList ')' ';'"""
 413     arguments = self.BuildProduction('Arguments', p, 4, p[5])
 414     p[0] = self.BuildNamed('Callback', p, 1, ListFromConcat(p[3], arguments))
 415
 416   # [24]
 417   def p_Typedef(self, p):
 418     """Typedef : TYPEDEF ExtendedAttributeListNoComments Type identifier ';'"""
 419     p[0] = self.BuildNamed('Typedef', p, 4, ListFromConcat(p[2], p[3]))
 420
 421   # [24.1] Error recovery for Typedefs
 422   def p_TypedefError(self, p):
 423     """Typedef : TYPEDEF error ';'"""
 424     p[0] = self.BuildError(p, 'Typedef')
 425
 426   # [25]
 427   def p_ImplementsStatement(self, p):
 428     """ImplementsStatement : identifier IMPLEMENTS identifier ';'"""
 429     name = self.BuildAttribute('REFERENCE', p[3])
 430     p[0] = self.BuildNamed('Implements', p, 1, name)
 431
 432   # [26]
 433   def p_Const(self,  p):
 434     """Const : CONST ConstType identifier '=' ConstValue ';'"""
 435     value = self.BuildProduction('Value', p, 5, p[5])
 436     p[0] = self.BuildNamed('Const', p, 3, ListFromConcat(p[2], value))
 437
 438   # [27]
 439   def p_ConstValue(self, p):
 440     """ConstValue : BooleanLiteral
 441                   | FloatLiteral
 442                   | integer
 443                   | null"""
 444     if type(p[1]) == str:
 445       p[0] = ListFromConcat(self.BuildAttribute('TYPE', 'integer'),
 446                             self.BuildAttribute('NAME', p[1]))
 447     else:
 448       p[0] = p[1]
 449
 450   # [27.1] Add definition for NULL
 451   def p_null(self, p):
 452     """null : NULL"""
 453     p[0] = ListFromConcat(self.BuildAttribute('TYPE', 'NULL'),
 454                           self.BuildAttribute('NAME', 'NULL'))
 455
 456   # [28]
 457   def p_BooleanLiteral(self, p):
 458     """BooleanLiteral : TRUE
 459                       | FALSE"""
 460     value = self.BuildAttribute('VALUE', Boolean(p[1] == 'true'))
 461     p[0] = ListFromConcat(self.BuildAttribute('TYPE', 'boolean'), value)
 462
 463   # [29]
 464   def p_FloatLiteral(self, p):
 465     """FloatLiteral : float
 466                     | '-' INFINITY
 467                     | INFINITY
 468                     | NAN """
 469     if len(p) > 2:
 470       val = '-Infinity'
 471     else:
 472       val = p[1]
 473     p[0] = ListFromConcat(self.BuildAttribute('TYPE', 'float'),
 474                           self.BuildAttribute('VALUE', val))
 475
 476   # [30]
 477   def p_Serializer(self, p):
 478     """Serializer : SERIALIZER SerializerRest"""
 479     p[0] = self.BuildProduction('Serializer', p, 1, p[2])
 480
 481   # [31]
 482   # TODO(jl): This adds ReturnType and ';', missing from the spec's grammar.
 483   # https://www.w3.org/Bugs/Public/show_bug.cgi?id=20361
 484   def p_SerializerRest(self, p):
 485     """SerializerRest : ReturnType OperationRest
 486                       | '=' SerializationPattern ';'
 487                       | ';'"""
 488     if len(p) == 3:
 489       p[2].AddChildren(p[1])
 490       p[0] = p[2]
 491     elif len(p) == 4:
 492       p[0] = p[2]
 493
 494   # [32]
 495   def p_SerializationPattern(self, p):
 496     """SerializationPattern : '{' SerializationPatternMap '}'
 497                             | '[' SerializationPatternList ']'
 498                             | identifier"""
 499     if len(p) > 2:
 500       p[0] = p[2]
 501     else:
 502       p[0] = self.BuildAttribute('ATTRIBUTE', p[1])
 503
 504   # [33]
 505   # TODO(jl): This adds the "ATTRIBUTE" and "INHERIT ',' ATTRIBUTE" variants,
 506   # missing from the spec's grammar.
 507   # https://www.w3.org/Bugs/Public/show_bug.cgi?id=20361
 508   def p_SerializationPatternMap(self, p):
 509     """SerializationPatternMap : GETTER
 510                                | ATTRIBUTE
 511                                | INHERIT ',' ATTRIBUTE
 512                                | INHERIT Identifiers
 513                                | identifier Identifiers
 514                                |"""
 515     p[0] = self.BuildProduction('Map', p, 0)
 516     if len(p) == 4:
 517       p[0].AddChildren(self.BuildTrue('INHERIT'))
 518       p[0].AddChildren(self.BuildTrue('ATTRIBUTE'))
 519     elif len(p) > 1:
 520       if p[1] == 'getter':
 521         p[0].AddChildren(self.BuildTrue('GETTER'))
 522       elif p[1] == 'attribute':
 523         p[0].AddChildren(self.BuildTrue('ATTRIBUTE'))
 524       else:
 525         if p[1] == 'inherit':
 526           p[0].AddChildren(self.BuildTrue('INHERIT'))
 527           attributes = p[2]
 528         else:
 529           attributes = ListFromConcat(p[1], p[2])
 530         p[0].AddChildren(self.BuildAttribute('ATTRIBUTES', attributes))
 531
 532   # [34]
 533   def p_SerializationPatternList(self, p):
 534     """SerializationPatternList : GETTER
 535                                 | identifier Identifiers
 536                                 |"""
 537     p[0] = self.BuildProduction('List', p, 0)
 538     if len(p) > 1:
 539       if p[1] == 'getter':
 540         p[0].AddChildren(self.BuildTrue('GETTER'))
 541       else:
 542         attributes = ListFromConcat(p[1], p[2])
 543         p[0].AddChildren(self.BuildAttribute('ATTRIBUTES', attributes))
 544
 545   # [35]
 546   def p_Stringifier(self, p):
 547     """Stringifier : STRINGIFIER StringifierRest"""
 548     p[0] = self.BuildProduction('Stringifier', p, 1, p[2])
 549
 550   # [36]
 551   def p_StringifierRest(self, p):
 552     """StringifierRest : AttributeRest
 553                        | ReturnType OperationRest
 554                        | ';'"""
 555     if len(p) == 3:
 556       p[2].AddChildren(p[1])
 557       p[0] = p[2]
 558     elif p[1] != ';':
 559       p[0] = p[1]
 560
 561   # [37]
 562   def p_StaticMember(self, p):
 563     """StaticMember : STATIC StaticMemberRest"""
 564     p[2].AddChildren(self.BuildTrue('STATIC'))
 565     p[0] = p[2]
 566
 567   # [38]
 568   def p_StaticMemberRest(self, p):
 569     """StaticMemberRest : ReadOnly AttributeRest
 570                         | ReturnType OperationRest"""
 571     if len(p) == 2:
 572       p[0] = p[1]
 573     else:
 574       p[2].AddChildren(p[1])
 575       p[0] = p[2]
 576
 577   # [39]
 578   def p_ReadonlyMember(self, p):
 579     """ReadonlyMember : READONLY ReadonlyMemberRest"""
 580     p[2].AddChildren(self.BuildTrue('READONLY'))
 581     p[0] = p[2]
 582
 583   # [40]
 584   def p_ReadonlyMemberRest(self, p):
 585     """ReadonlyMemberRest : AttributeRest
 586                           | MaplikeRest
 587                           | SetlikeRest"""
 588     p[0] = p[1]
 589
 590   # [41]
 591   def p_ReadWriteAttribute(self, p):
 592     """ReadWriteAttribute : INHERIT ReadOnly AttributeRest
 593                           | AttributeRest"""
 594     if len(p) > 2:
 595       inherit = self.BuildTrue('INHERIT')
 596       p[3].AddChildren(ListFromConcat(inherit, p[2]))
 597       p[0] = p[3]
 598     else:
 599       p[0] = p[1]
 600
 601   # [42]
 602   def p_AttributeRest(self, p):
 603     """AttributeRest : ATTRIBUTE Type AttributeName ';'"""
 604     p[0] = self.BuildNamed('Attribute', p, 3, p[2])
 605
 606   # [43]
 607   def p_AttributeName(self, p):
 608     """AttributeName : AttributeNameKeyword
 609                      | identifier"""
 610     p[0] = p[1]
 611
 612   # [44]
 613   def p_AttributeNameKeyword(self, p):
 614     """AttributeNameKeyword : REQUIRED"""
 615     p[0] = p[1]
 616
 617   # [45] Unreferenced in the specification
 618
 619   # [46]
 620   def p_ReadOnly(self, p):
 621     """ReadOnly : READONLY
 622                 |"""
 623     if len(p) > 1:
 624       p[0] = self.BuildTrue('READONLY')
 625
 626   # [47]
 627   def p_Operation(self, p):
 628     """Operation : ReturnType OperationRest
 629                  | SpecialOperation"""
 630     if len(p) == 3:
 631       p[2].AddChildren(p[1])
 632       p[0] = p[2]
 633     else:
 634       p[0] = p[1]
 635
 636   # [48]
 637   def p_SpecialOperation(self, p):
 638     """SpecialOperation : Special Specials ReturnType OperationRest"""
 639     p[4].AddChildren(ListFromConcat(p[1], p[2], p[3]))
 640     p[0] = p[4]
 641
 642   # [49]
 643   def p_Specials(self, p):
 644     """Specials : Special Specials
 645                 | """
 646     if len(p) > 1:
 647       p[0] = ListFromConcat(p[1], p[2])
 648
 649   # [50]
 650   def p_Special(self, p):
 651     """Special : GETTER
 652                | SETTER
 653                | CREATOR
 654                | DELETER
 655                | LEGACYCALLER"""
 656     p[0] = self.BuildTrue(p[1].upper())
 657
 658   # [51]
 659   def p_OperationRest(self, p):
 660     """OperationRest : OptionalIdentifier '(' ArgumentList ')' ';'"""
 661     arguments = self.BuildProduction('Arguments', p, 2, p[3])
 662     p[0] = self.BuildNamed('Operation', p, 1, arguments)
 663
 664   # [52]
 665   def p_OptionalIdentifier(self, p):
 666     """OptionalIdentifier : identifier
 667                           |"""
 668     if len(p) > 1:
 669       p[0] = p[1]
 670     else:
 671       p[0] = '_unnamed_'
 672
 673   # [53]
 674   def p_ArgumentList(self, p):
 675     """ArgumentList : Argument Arguments
 676                     |"""
 677     if len(p) > 1:
 678       p[0] = ListFromConcat(p[1], p[2])
 679
 680   # [53.1] ArgumentList error recovery
 681   def p_ArgumentListError(self, p):
 682     """ArgumentList : error """
 683     p[0] = self.BuildError(p, 'ArgumentList')
 684
 685   # [54]
 686   def p_Arguments(self, p):
 687     """Arguments : ',' Argument Arguments
 688                  |"""
 689     if len(p) > 1:
 690       p[0] = ListFromConcat(p[2], p[3])
 691
 692   # [54.1] Arguments error recovery
 693   def p_ArgumentsError(self, p):
 694     """Arguments : ',' error"""
 695     p[0] = self.BuildError(p, 'Arguments')
 696
 697   # [55]
 698   def p_Argument(self, p):
 699     """Argument : ExtendedAttributeList OptionalOrRequiredArgument"""
 700     p[2].AddChildren(p[1])
 701     p[0] = p[2]
 702
 703   # [56]
 704   def p_OptionalOrRequiredArgument(self, p):
 705     """OptionalOrRequiredArgument : OPTIONAL Type ArgumentName Default
 706                                   | Type Ellipsis ArgumentName"""
 707     if len(p) > 4:
 708       arg = self.BuildNamed('Argument', p, 3, ListFromConcat(p[2], p[4]))
 709       arg.AddChildren(self.BuildTrue('OPTIONAL'))
 710     else:
 711       arg = self.BuildNamed('Argument', p, 3, ListFromConcat(p[1], p[2]))
 712     p[0] = arg
 713
 714   # [57]
 715   def p_ArgumentName(self, p):
 716     """ArgumentName : ArgumentNameKeyword
 717                     | identifier"""
 718     p[0] = p[1]
 719
 720   # [58]
 721   def p_Ellipsis(self, p):
 722     """Ellipsis : ELLIPSIS
 723                 |"""
 724     if len(p) > 1:
 725       p[0] = self.BuildNamed('Argument', p, 1)
 726       p[0].AddChildren(self.BuildTrue('ELLIPSIS'))
 727
 728   # [] Unspecified
 729   def p_ExceptionMember(self, p):
 730     """ExceptionMember : Const
 731                        | ExceptionField"""
 732     p[0] = p[1]
 733
 734   # [] Unspecified
 735   def p_ExceptionField(self, p):
 736     """ExceptionField : Type identifier ';'"""
 737     p[0] = self.BuildNamed('ExceptionField', p, 2, p[1])
 738
 739   # [] Error recovery for ExceptionMembers - Unspecified
 740   def p_ExceptionFieldError(self, p):
 741     """ExceptionField : error"""
 742     p[0] = self.BuildError(p, 'ExceptionField')
 743
 744   # [59]
 745   def p_Iterable(self, p):
 746     """Iterable : ITERABLE '<' Type OptionalType '>' ';'
 747                 | LEGACYITERABLE '<' Type '>' ';'"""
 748     if len(p) > 6:
 749       childlist = ListFromConcat(p[3], p[4])
 750       p[0] = self.BuildProduction('Iterable', p, 2, childlist)
 751     else:
 752       p[0] = self.BuildProduction('LegacyIterable', p, 2, p[3])
 753
 754   # [60]
 755   def p_OptionalType(self, p):
 756     """OptionalType : ',' Type
 757                     |"""
 758     if len(p) > 1:
 759       p[0] = p[2]
 760
 761   # [61]
 762   def p_ReadWriteMaplike(self, p):
 763     """ReadWriteMaplike : MaplikeRest"""
 764     p[0] = p[1]
 765
 766   # [62]
 767   def p_ReadWriteSetlike(self, p):
 768     """ReadWriteSetlike : SetlikeRest"""
 769     p[0] = p[1]
 770
 771   # [63]
 772   def p_MaplikeRest(self, p):
 773     """MaplikeRest : MAPLIKE '<' Type ',' Type '>' ';'"""
 774     childlist = ListFromConcat(p[3], p[5])
 775     p[0] = self.BuildProduction('Maplike', p, 2, childlist)
 776
 777   # [64]
 778   def p_SetlikeRest(self, p):
 779     """SetlikeRest : SETLIKE '<' Type '>' ';'"""
 780     p[0] = self.BuildProduction('Setlike', p, 2, p[3])
 781
 782   # [65] No comment version for mid statement attributes.
 783   def p_ExtendedAttributeListNoComments(self, p):
 784     """ExtendedAttributeListNoComments : '[' ExtendedAttribute ExtendedAttributes ']'
 785                                        | """
 786     if len(p) > 2:
 787       items = ListFromConcat(p[2], p[3])
 788       p[0] = self.BuildProduction('ExtAttributes', p, 1, items)
 789
 790   # [65.1] Add optional comment field for start of statements.
 791   def p_ExtendedAttributeList(self, p):
 792     """ExtendedAttributeList : Comments '[' ExtendedAttribute ExtendedAttributes ']'
 793                              | Comments """
 794     if len(p) > 2:
 795       items = ListFromConcat(p[3], p[4])
 796       attribs = self.BuildProduction('ExtAttributes', p, 2, items)
 797       p[0] = ListFromConcat(p[1], attribs)
 798     else:
 799       p[0] = p[1]
 800
 801   # [66]
 802   def p_ExtendedAttributes(self, p):
 803     """ExtendedAttributes : ',' ExtendedAttribute ExtendedAttributes
 804                           |"""
 805     if len(p) > 1:
 806       p[0] = ListFromConcat(p[2], p[3])
 807
 808   # We only support:
 809   #    [ identifier ]
 810   #    [ identifier ( ArgumentList ) ]
 811   #    [ identifier = identifier ]
 812   #    [ identifier = ( IdentifierList ) ]
 813   #    [ identifier = identifier ( ArgumentList ) ]
 814   # [66] map directly to [91-93, 95]
 815   # [67-69, 71] are unsupported
 816   def p_ExtendedAttribute(self, p):
 817     """ExtendedAttribute : ExtendedAttributeNoArgs
 818                          | ExtendedAttributeArgList
 819                          | ExtendedAttributeIdent
 820                          | ExtendedAttributeIdentList
 821                          | ExtendedAttributeNamedArgList"""
 822     p[0] = p[1]
 823
 824   # [71]
 825   def p_ArgumentNameKeyword(self, p):
 826     """ArgumentNameKeyword : ATTRIBUTE
 827                            | CALLBACK
 828                            | CONST
 829                            | CREATOR
 830                            | DELETER
 831                            | DICTIONARY
 832                            | ENUM
 833                            | EXCEPTION
 834                            | GETTER
 835                            | IMPLEMENTS
 836                            | INHERIT
 837                            | LEGACYCALLER
 838                            | PARTIAL
 839                            | SERIALIZER
 840                            | SETTER
 841                            | STATIC
 842                            | STRINGIFIER
 843                            | TYPEDEF
 844                            | UNRESTRICTED"""
 845     p[0] = p[1]
 846
 847   # [72] NOT IMPLEMENTED (OtherOrComma)
 848
 849   # [73]
 850   def p_Type(self, p):
 851     """Type : SingleType
 852             | UnionType TypeSuffix"""
 853     if len(p) == 2:
 854       p[0] = self.BuildProduction('Type', p, 1, p[1])
 855     else:
 856       p[0] = self.BuildProduction('Type', p, 1, ListFromConcat(p[1], p[2]))
 857
 858   # [74]
 859   def p_SingleType(self, p):
 860     """SingleType : NonAnyType
 861                   | ANY TypeSuffixStartingWithArray"""
 862     if len(p) == 2:
 863       p[0] = p[1]
 864     else:
 865       p[0] = ListFromConcat(self.BuildProduction('Any', p, 1), p[2])
 866
 867   # [75]
 868   def p_UnionType(self, p):
 869     """UnionType : '(' UnionMemberType OR UnionMemberType UnionMemberTypes ')'"""
 870
 871   # [76]
 872   def p_UnionMemberType(self, p):
 873     """UnionMemberType : NonAnyType
 874                        | UnionType TypeSuffix
 875                        | ANY '[' ']' TypeSuffix"""
 876   # [77]
 877   def p_UnionMemberTypes(self, p):
 878     """UnionMemberTypes : OR UnionMemberType UnionMemberTypes
 879                         |"""
 880
 881   # [78] Moved BYTESTRING, DOMSTRING, OBJECT, DATE, REGEXP to PrimitiveType
 882   # Moving all built-in types into PrimitiveType makes it easier to
 883   # differentiate between them and 'identifier', since p[1] would be a string in
 884   # both cases.
 885   def p_NonAnyType(self, p):
 886     """NonAnyType : PrimitiveType TypeSuffix
 887                   | PromiseType Null
 888                   | identifier TypeSuffix
 889                   | SEQUENCE '<' Type '>' Null"""
 890     if len(p) == 3:
 891       if type(p[1]) == str:
 892         typeref = self.BuildNamed('Typeref', p, 1)
 893       else:
 894         typeref = p[1]
 895       p[0] = ListFromConcat(typeref, p[2])
 896
 897     if len(p) == 6:
 898       p[0] = self.BuildProduction('Sequence', p, 1, ListFromConcat(p[3], p[5]))
 899
 900   # [79] NOT IMPLEMENTED (BufferRelatedType)
 901
 902   # [80]
 903   def p_ConstType(self,  p):
 904     """ConstType : PrimitiveType Null
 905                  | identifier Null"""
 906     if type(p[1]) == str:
 907       p[0] = self.BuildNamed('Typeref', p, 1, p[2])
 908     else:
 909       p[1].AddChildren(p[2])
 910       p[0] = p[1]
 911
 912
 913   # [81] Added BYTESTRING, DOMSTRING, OBJECT, DATE, REGEXP
 914   def p_PrimitiveType(self, p):
 915     """PrimitiveType : UnsignedIntegerType
 916                      | UnrestrictedFloatType
 917                      | BOOLEAN
 918                      | BYTE
 919                      | OCTET
 920                      | BYTESTRING
 921                      | DOMSTRING
 922                      | OBJECT
 923                      | DATE
 924                      | REGEXP"""
 925     if type(p[1]) == str:
 926       p[0] = self.BuildNamed('PrimitiveType', p, 1)
 927     else:
 928       p[0] = p[1]
 929
 930
 931   # [82]
 932   def p_UnrestrictedFloatType(self, p):
 933     """UnrestrictedFloatType : UNRESTRICTED FloatType
 934                              | FloatType"""
 935     if len(p) == 2:
 936       typeref = self.BuildNamed('PrimitiveType', p, 1)
 937     else:
 938       typeref = self.BuildNamed('PrimitiveType', p, 2)
 939       typeref.AddChildren(self.BuildTrue('UNRESTRICTED'))
 940     p[0] = typeref
 941
 942
 943   # [83]
 944   def p_FloatType(self, p):
 945     """FloatType : FLOAT
 946                  | DOUBLE"""
 947     p[0] = p[1]
 948
 949   # [84]
 950   def p_UnsignedIntegerType(self, p):
 951     """UnsignedIntegerType : UNSIGNED IntegerType
 952                            | IntegerType"""
 953     if len(p) == 2:
 954       p[0] = p[1]
 955     else:
 956       p[0] = 'unsigned ' + p[2]
 957
 958   # [85]
 959   def p_IntegerType(self, p):
 960     """IntegerType : SHORT
 961                    | LONG OptionalLong"""
 962     if len(p) == 2:
 963       p[0] = p[1]
 964     else:
 965       p[0] = p[1] + p[2]
 966
 967   # [86]
 968   def p_OptionalLong(self, p):
 969     """OptionalLong : LONG
 970                     | """
 971     if len(p) > 1:
 972       p[0] = ' ' + p[1]
 973     else:
 974       p[0] = ''
 975
 976   # [87] Add unqualified Promise
 977   def p_PromiseType(self, p):
 978     """PromiseType : PROMISE '<' ReturnType '>'
 979                    | PROMISE"""
 980     if len(p) == 2:
 981       # Promise without resolution type is not specified in the Web IDL spec.
 982       # As it is used in some specs and in the blink implementation,
 983       # we allow that here.
 984       resolution_type = self.BuildProduction('Type', p, 1,
 985                                              self.BuildProduction('Any', p, 1))
 986       p[0] = self.BuildNamed('Promise', p, 1, resolution_type)
 987     else:
 988       p[0] = self.BuildNamed('Promise', p, 1, p[3])
 989
 990   # [88] Add support for sized array
 991   def p_TypeSuffix(self, p):
 992     """TypeSuffix : '[' integer ']' TypeSuffix
 993                   | '[' ']' TypeSuffix
 994                   | '?' TypeSuffixStartingWithArray
 995                   | """
 996     if len(p) == 5:
 997       p[0] = self.BuildNamed('Array', p, 2, p[4])
 998
 999     if len(p) == 4:
1000       p[0] = self.BuildProduction('Array', p, 1, p[3])
1001
1002     if len(p) == 3:
1003       p[0] = ListFromConcat(self.BuildTrue('NULLABLE'), p[2])
1004
1005
1006   # [89]
1007   def p_TypeSuffixStartingWithArray(self, p):
1008     """TypeSuffixStartingWithArray : '[' ']' TypeSuffix
1009                                    | """
1010     if len(p) > 1:
1011       p[0] = self.BuildProduction('Array', p, 0, p[3])
1012
1013   # [90]
1014   def p_Null(self, p):
1015     """Null : '?'
1016             |"""
1017     if len(p) > 1:
1018       p[0] = self.BuildTrue('NULLABLE')
1019
1020   # [91]
1021   def p_ReturnType(self, p):
1022     """ReturnType : Type
1023                   | VOID"""
1024     if p[1] == 'void':
1025       p[0] = self.BuildProduction('Type', p, 1)
1026       p[0].AddChildren(self.BuildNamed('PrimitiveType', p, 1))
1027     else:
1028       p[0] = p[1]
1029
1030   # [92]
1031   def p_IdentifierList(self, p):
1032     """IdentifierList : identifier Identifiers"""
1033     p[0] = ListFromConcat(p[1], p[2])
1034
1035   # [93]
1036   def p_Identifiers(self, p):
1037     """Identifiers : ',' identifier Identifiers
1038                    |"""
1039     if len(p) > 1:
1040       p[0] = ListFromConcat(p[2], p[3])
1041
1042   # [94]
1043   def p_ExtendedAttributeNoArgs(self, p):
1044     """ExtendedAttributeNoArgs : identifier"""
1045     p[0] = self.BuildNamed('ExtAttribute', p, 1)
1046
1047   # [95]
1048   def p_ExtendedAttributeArgList(self, p):
1049     """ExtendedAttributeArgList : identifier '(' ArgumentList ')'"""
1050     arguments = self.BuildProduction('Arguments', p, 2, p[3])
1051     p[0] = self.BuildNamed('ExtAttribute', p, 1, arguments)
1052
1053   # [96]
1054   def p_ExtendedAttributeIdent(self, p):
1055     """ExtendedAttributeIdent : identifier '=' identifier"""
1056     value = self.BuildAttribute('VALUE', p[3])
1057     p[0] = self.BuildNamed('ExtAttribute', p, 1, value)
1058
1059   # [97]
1060   def p_ExtendedAttributeIdentList(self, p):
1061     """ExtendedAttributeIdentList : identifier '=' '(' IdentifierList ')'"""
1062     value = self.BuildAttribute('VALUE', p[4])
1063     p[0] = self.BuildNamed('ExtAttribute', p, 1, value)
1064
1065   # [98]
1066   def p_ExtendedAttributeNamedArgList(self, p):
1067     """ExtendedAttributeNamedArgList : identifier '=' identifier '(' ArgumentList ')'"""
1068     args = self.BuildProduction('Arguments', p, 4, p[5])
1069     value = self.BuildNamed('Call', p, 3, args)
1070     p[0] = self.BuildNamed('ExtAttribute', p, 1, value)
1071
1072 #
1073 # Parser Errors
1074 #
1075 # p_error is called whenever the parser can not find a pattern match for
1076 # a set of items from the current state.  The p_error function defined here
1077 # is triggered logging an error, and parsing recovery happens as the
1078 # p_<type>_error functions defined above are called.  This allows the parser
1079 # to continue so as to capture more than one error per file.
1080 #
1081   def p_error(self, t):
1082     if t:
1083       lineno = t.lineno
1084       pos = t.lexpos
1085       prev = self.yaccobj.symstack[-1]
1086       if type(prev) == lex.LexToken:
1087         msg = "Unexpected %s after %s." % (
1088             TokenTypeName(t), TokenTypeName(prev))
1089       else:
1090         msg = "Unexpected %s." % (t.value)
1091     else:
1092       last = self.LastToken()
1093       lineno = last.lineno
1094       pos = last.lexpos
1095       msg = "Unexpected end of file after %s." % TokenTypeName(last)
1096       self.yaccobj.restart()
1097
1098     # Attempt to remap the error to a friendlier form
1099     if msg in ERROR_REMAP:
1100       msg = ERROR_REMAP[msg]
1101
1102     self._last_error_msg = msg
1103     self._last_error_lineno = lineno
1104     self._last_error_pos = pos
1105
1106   def Warn(self, node, msg):
1107     sys.stdout.write(node.GetLogLine(msg))
1108     self.parse_warnings += 1
1109
1110   def LastToken(self):
1111     return self.lexer.last
1112
1113   def __init__(self, lexer, verbose=False, debug=False, mute_error=False):
1114     self.lexer = lexer
1115     self.tokens = lexer.KnownTokens()
1116     self.yaccobj = yacc.yacc(module=self, tabmodule=None, debug=debug,
1117                              optimize=0, write_tables=0)
1118     self.parse_debug = debug
1119     self.verbose = verbose
1120     self.mute_error = mute_error
1121     self._parse_errors = 0
1122     self._parse_warnings = 0
1123     self._last_error_msg = None
1124     self._last_error_lineno = 0
1125     self._last_error_pos = 0
1126
1127
1128 #
1129 # BuildProduction
1130 #
1131 # Production is the set of items sent to a grammar rule resulting in a new
1132 # item being returned.
1133 #
1134 # p - Is the Yacc production object containing the stack of items
1135 # index - Index into the production of the name for the item being produced.
1136 # cls - The type of item being producted
1137 # childlist - The children of the new item
1138   def BuildProduction(self, cls, p, index, childlist=None):
1139     try:
1140       if not childlist:
1141         childlist = []
1142
1143       filename = self.lexer.Lexer().filename
1144       lineno = p.lineno(index)
1145       pos = p.lexpos(index)
1146       out = IDLNode(cls, filename, lineno, pos, childlist)
1147       return out
1148     except:
1149       print 'Exception while parsing:'
1150       for num, item in enumerate(p):
1151         print '  [%d] %s' % (num, ExpandProduction(item))
1152       if self.LastToken():
1153         print 'Last token: %s' % str(self.LastToken())
1154       raise
1155
1156   def BuildNamed(self, cls, p, index, childlist=None):
1157     childlist = ListFromConcat(childlist)
1158     childlist.append(self.BuildAttribute('NAME', p[index]))
1159     return self.BuildProduction(cls, p, index, childlist)
1160
1161   def BuildComment(self, cls, p, index):
1162     name = p[index]
1163
1164     # Remove comment markers
1165     lines = []
1166     if name[:2] == '//':
1167       # For C++ style, remove any leading whitespace and the '//' marker from
1168       # each line.
1169       form = 'cc'
1170       for line in name.split('\n'):
1171         start = line.find('//')
1172         lines.append(line[start+2:])
1173     else:
1174       # For C style, remove ending '*/''
1175       form = 'c'
1176       for line in name[:-2].split('\n'):
1177         # Remove characters until start marker for this line '*' if found
1178         # otherwise it should be blank.
1179         offs = line.find('*')
1180         if offs >= 0:
1181           line = line[offs + 1:].rstrip()
1182         else:
1183           line = ''
1184         lines.append(line)
1185     name = '\n'.join(lines)
1186     childlist = [self.BuildAttribute('NAME', name),
1187                  self.BuildAttribute('FORM', form)]
1188     return self.BuildProduction(cls, p, index, childlist)
1189
1190 #
1191 # BuildError
1192 #
1193 # Build and Errror node as part of the recovery process.
1194 #
1195 #
1196   def BuildError(self, p, prod):
1197     self._parse_errors += 1
1198     name = self.BuildAttribute('NAME', self._last_error_msg)
1199     line = self.BuildAttribute('LINE', self._last_error_lineno)
1200     pos = self.BuildAttribute('POS', self._last_error_pos)
1201     prod = self.BuildAttribute('PROD', prod)
1202
1203     node = self.BuildProduction('Error', p, 1,
1204                                 ListFromConcat(name, line, pos, prod))
1205     if not self.mute_error:
1206       node.Error(self._last_error_msg)
1207
1208     return node
1209
1210 #
1211 # BuildAttribute
1212 #
1213 # An ExtendedAttribute is a special production that results in a property
1214 # which is applied to the adjacent item.  Attributes have no children and
1215 # instead represent key/value pairs.
1216 #
1217   def BuildAttribute(self, key, val):
1218     return IDLAttribute(key, val)
1219
1220   def BuildFalse(self, key):
1221     return IDLAttribute(key, Boolean(False))
1222
1223   def BuildTrue(self, key):
1224     return IDLAttribute(key, Boolean(True))
1225
1226   def GetErrors(self):
1227     # Access lexer errors, despite being private
1228     # pylint: disable=W0212
1229     return self._parse_errors + self.lexer._lex_errors
1230
1231 #
1232 # ParseData
1233 #
1234 # Attempts to parse the current data loaded in the lexer.
1235 #
1236   def ParseText(self, filename, data):
1237     self._parse_errors = 0
1238     self._parse_warnings = 0
1239     self._last_error_msg = None
1240     self._last_error_lineno = 0
1241     self._last_error_pos = 0
1242
1243     try:
1244       self.lexer.Tokenize(data, filename)
1245       nodes = self.yaccobj.parse(lexer=self.lexer) or []
1246       name = self.BuildAttribute('NAME', filename)
1247       return IDLNode('File', filename, 0, 0, nodes + [name])
1248
1249     except lex.LexError as lexError:
1250       sys.stderr.write('Error in token: %s\n' % str(lexError))
1251     return None
1252
1253
1254
1255 def ParseFile(parser, filename):
1256   """Parse a file and return a File type of node."""
1257   with open(filename) as fileobject:
1258     try:
1259       out = parser.ParseText(filename, fileobject.read())
1260       out.SetProperty('DATETIME', time.ctime(os.path.getmtime(filename)))
1261       out.SetProperty('ERRORS', parser.GetErrors())
1262       return out
1263
1264     except Exception as e:
1265       last = parser.LastToken()
1266       sys.stderr.write('%s(%d) : Internal parsing error\n\t%s.\n' % (
1267                        filename, last.lineno, str(e)))
1268
1269
1270 def main(argv):
1271   nodes = []
1272   parser = IDLParser(IDLLexer())
1273   errors = 0
1274   for filename in argv:
1275     filenode = ParseFile(parser, filename)
1276     if (filenode):
1277       errors += filenode.GetProperty('ERRORS')
1278       nodes.append(filenode)
1279
1280   ast = IDLNode('AST', '__AST__', 0, 0, nodes)
1281
1282   print '\n'.join(ast.Tree(accept_props=['PROD']))
1283   if errors:
1284     print '\nFound %d errors.\n' % errors
1285
1286   return errors
1287
1288
1289 if __name__ == '__main__':
1290   sys.exit(main(sys.argv[1:]))