tools/idl_parser/idl_ppapi_lexer.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """ Lexer for PPAPI IDL
   7
   8 The lexer uses the PLY library to build a tokenizer which understands both
   9 WebIDL and Pepper tokens.
  10
  11 WebIDL, and WebIDL regular expressions can be found at:
  12    http://www.w3.org/TR/2012/CR-WebIDL-20120419/
  13 PLY can be found at:
  14    http://www.dabeaz.com/ply/
  15 """
  16
  17 from idl_lexer import IDLLexer
  18
  19
  20 #
  21 # IDL PPAPI Lexer
  22 #
  23 class IDLPPAPILexer(IDLLexer):
  24   # Token definitions
  25   #
  26   # These need to be methods for lexer construction, despite not using self.
  27   # pylint: disable=R0201
  28
  29   # Special multi-character operators
  30   def t_LSHIFT(self, t):
  31     r'<<'
  32     return t
  33
  34   def t_RSHIFT(self, t):
  35     r'>>'
  36     return t
  37
  38   def t_INLINE(self, t):
  39     r'\#inline (.|\n)*?\#endinl.*'
  40     self.AddLines(t.value.count('\n'))
  41     return t
  42
  43   # Return a "preprocessor" inline block
  44   def __init__(self):
  45     IDLLexer.__init__(self)
  46     self._AddTokens(['INLINE', 'LSHIFT', 'RSHIFT'])
  47     self._AddKeywords(['label', 'struct'])
  48
  49     # Add number types
  50     self._AddKeywords(['char', 'int8_t', 'int16_t', 'int32_t', 'int64_t'])
  51     self._AddKeywords(['uint8_t', 'uint16_t', 'uint32_t', 'uint64_t'])
  52     self._AddKeywords(['double_t', 'float_t'])
  53
  54     # Add handle types
  55     self._AddKeywords(['handle_t', 'PP_FileHandle'])
  56
  57     # Add pointer types (void*, char*, const char*, const void*)
  58     self._AddKeywords(['mem_t', 'str_t', 'cstr_t', 'interface_t'])
  59
  60     # Remove JS types
  61     self._DelKeywords(['boolean', 'byte', 'Date', 'DOMString', 'double',
  62                        'float', 'long', 'object', 'octet', 'short', 'unsigned'])
  63
  64
  65 # If run by itself, attempt to build the lexer
  66 if __name__ == '__main__':
  67   lexer = IDLPPAPILexer()
  68   lexer.Tokenize(open('test_parser/inline_ppapi.idl').read())
  69   for tok in lexer.GetTokens():
  70     print '\n' + str(tok)