tools/idl_parser/idl_lexer.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """ Lexer for PPAPI IDL
   7
   8 The lexer uses the PLY library to build a tokenizer which understands both
   9 WebIDL and Pepper tokens.
  10
  11 WebIDL, and WebIDL regular expressions can be found at:
  12    http://www.w3.org/TR/2012/CR-WebIDL-20120419/
  13 PLY can be found at:
  14    http://www.dabeaz.com/ply/
  15 """
  16
  17 import os.path
  18 import sys
  19
  20 #
  21 # Try to load the ply module, if not, then assume it is in the third_party
  22 # directory.
  23 #
  24 try:
  25   # Disable lint check which fails to find the ply module.
  26   # pylint: disable=F0401
  27   from ply import lex
  28 except ImportError:
  29   module_path, module_name = os.path.split(__file__)
  30   third_party = os.path.join(module_path, '..', '..', 'third_party')
  31   sys.path.append(third_party)
  32   # pylint: disable=F0401
  33   from ply import lex
  34
  35 #
  36 # IDL Lexer
  37 #
  38 class IDLLexer(object):
  39   # 'literals' is a value expected by lex which specifies a list of valid
  40   # literal tokens, meaning the token type and token value are identical.
  41   literals = r'"*.(){}[],;:=+-/~|&^?<>'
  42
  43   # 't_ignore' contains ignored characters (spaces and tabs)
  44   t_ignore = ' \t'
  45
  46   # 'tokens' is a value required by lex which specifies the complete list
  47   # of valid token types.
  48   tokens = [
  49     # Data types
  50       'float',
  51       'integer',
  52       'string',
  53
  54     # Symbol and keywords types
  55       'COMMENT',
  56       'identifier',
  57
  58     # MultiChar operators
  59       'ELLIPSIS',
  60   ]
  61
  62   # 'keywords' is a map of string to token type.  All tokens matching
  63   # KEYWORD_OR_SYMBOL are matched against keywords dictionary, to determine
  64   # if the token is actually a keyword.
  65   keywords = {
  66     'any' : 'ANY',
  67     'attribute' : 'ATTRIBUTE',
  68     'boolean' : 'BOOLEAN',
  69     'byte' : 'BYTE',
  70     'ByteString' : 'BYTESTRING',
  71     'callback' : 'CALLBACK',
  72     'const' : 'CONST',
  73     'creator' : 'CREATOR',
  74     'Date' : 'DATE',
  75     'deleter' : 'DELETER',
  76     'dictionary' : 'DICTIONARY',
  77     'DOMString' : 'DOMSTRING',
  78     'double' : 'DOUBLE',
  79     'enum'  : 'ENUM',
  80     'exception' : 'EXCEPTION',
  81     'false' : 'FALSE',
  82     'float' : 'FLOAT',
  83     'FrozenArray' : 'FROZENARRAY',
  84     'getter': 'GETTER',
  85     'implements' : 'IMPLEMENTS',
  86     'Infinity' : 'INFINITY',
  87     'inherit' : 'INHERIT',
  88     'interface' : 'INTERFACE',
  89     'iterable': 'ITERABLE',
  90     'legacycaller' : 'LEGACYCALLER',
  91     'legacyiterable' : 'LEGACYITERABLE',
  92     'long' : 'LONG',
  93     'maplike': 'MAPLIKE',
  94     'Nan' : 'NAN',
  95     'null' : 'NULL',
  96     'object' : 'OBJECT',
  97     'octet' : 'OCTET',
  98     'optional' : 'OPTIONAL',
  99     'or' : 'OR',
 100     'partial' : 'PARTIAL',
 101     'Promise' : 'PROMISE',
 102     'readonly' : 'READONLY',
 103     'RegExp' : 'REGEXP',
 104     'required' : 'REQUIRED',
 105     'sequence' : 'SEQUENCE',
 106     'serializer' : 'SERIALIZER',
 107     'setlike' : 'SETLIKE',
 108     'setter': 'SETTER',
 109     'short' : 'SHORT',
 110     'static' : 'STATIC',
 111     'stringifier' : 'STRINGIFIER',
 112     'typedef' : 'TYPEDEF',
 113     'true' : 'TRUE',
 114     'unsigned' : 'UNSIGNED',
 115     'unrestricted' : 'UNRESTRICTED',
 116     'void' : 'VOID'
 117   }
 118
 119   # Token definitions
 120   #
 121   # Lex assumes any value or function in the form of 't_<TYPE>' represents a
 122   # regular expression where a match will emit a token of type <TYPE>.  In the
 123   # case of a function, the function is called when a match is made. These
 124   # definitions come from WebIDL.
 125   #
 126   # These need to be methods for lexer construction, despite not using self.
 127   # pylint: disable=R0201
 128   def t_ELLIPSIS(self, t):
 129     r'\.\.\.'
 130     return t
 131
 132   # Regex needs to be in the docstring
 133   # pylint: disable=C0301
 134   def t_float(self, t):
 135     r'-?(([0-9]+\.[0-9]*|[0-9]*\.[0-9]+)([Ee][+-]?[0-9]+)?|[0-9]+[Ee][+-]?[0-9]+)'
 136     return t
 137
 138   def t_integer(self, t):
 139     r'-?([1-9][0-9]*|0[Xx][0-9A-Fa-f]+|0[0-7]*)'
 140     return t
 141
 142
 143   # A line ending '\n', we use this to increment the line number
 144   def t_LINE_END(self, t):
 145     r'\n+'
 146     self.AddLines(len(t.value))
 147
 148   # We do not process escapes in the IDL strings.  Strings are exclusively
 149   # used for attributes and enums, and not used as typical 'C' constants.
 150   def t_string(self, t):
 151     r'"[^"]*"'
 152     t.value = t.value[1:-1]
 153     self.AddLines(t.value.count('\n'))
 154     return t
 155
 156   # A C or C++ style comment:  /* xxx */ or //
 157   def t_COMMENT(self, t):
 158     r'(/\*(.|\n)*?\*/)|(//.*(\n[ \t]*//.*)*)'
 159     self.AddLines(t.value.count('\n'))
 160     return t
 161
 162   # A symbol or keyword.
 163   def t_KEYWORD_OR_SYMBOL(self, t):
 164     r'_?[A-Za-z][A-Za-z_0-9]*'
 165
 166     # All non-keywords are assumed to be symbols
 167     t.type = self.keywords.get(t.value, 'identifier')
 168
 169     # We strip leading underscores so that you can specify symbols with the same
 170     # value as a keywords (E.g. a dictionary named 'interface').
 171     if t.value[0] == '_':
 172       t.value = t.value[1:]
 173     return t
 174
 175   def t_ANY_error(self, t):
 176     msg = 'Unrecognized input'
 177     line = self.Lexer().lineno
 178
 179     # If that line has not been accounted for, then we must have hit
 180     # EoF, so compute the beginning of the line that caused the problem.
 181     if line >= len(self.index):
 182       # Find the offset in the line of the first word causing the issue
 183       word = t.value.split()[0]
 184       offs = self.lines[line - 1].find(word)
 185       # Add the computed line's starting position
 186       self.index.append(self.Lexer().lexpos - offs)
 187       msg = 'Unexpected EoF reached after'
 188
 189     pos = self.Lexer().lexpos - self.index[line]
 190     out = self.ErrorMessage(line, pos, msg)
 191     sys.stderr.write(out + '\n')
 192     self._lex_errors += 1
 193
 194
 195   def AddLines(self, count):
 196     # Set the lexer position for the beginning of the next line.  In the case
 197     # of multiple lines, tokens can not exist on any of the lines except the
 198     # last one, so the recorded value for previous lines are unused.  We still
 199     # fill the array however, to make sure the line count is correct.
 200     self.Lexer().lineno += count
 201     for _ in range(count):
 202       self.index.append(self.Lexer().lexpos)
 203
 204   def FileLineMsg(self, line, msg):
 205     # Generate a message containing the file and line number of a token.
 206     filename = self.Lexer().filename
 207     if filename:
 208       return "%s(%d) : %s" % (filename, line + 1, msg)
 209     return "<BuiltIn> : %s" % msg
 210
 211   def SourceLine(self, line, pos):
 212     # Create a source line marker
 213     caret = ' ' * pos + '^'
 214     # We decrement the line number since the array is 0 based while the
 215     # line numbers are 1 based.
 216     return "%s\n%s" % (self.lines[line - 1], caret)
 217
 218   def ErrorMessage(self, line, pos, msg):
 219     return "\n%s\n%s" % (
 220         self.FileLineMsg(line, msg),
 221         self.SourceLine(line, pos))
 222
 223 #
 224 # Tokenizer
 225 #
 226 # The token function returns the next token provided by IDLLexer for matching
 227 # against the leaf paterns.
 228 #
 229   def token(self):
 230     tok = self.Lexer().token()
 231     if tok:
 232       self.last = tok
 233     return tok
 234
 235
 236   def GetTokens(self):
 237     outlist = []
 238     while True:
 239       t = self.Lexer().token()
 240       if not t:
 241         break
 242       outlist.append(t)
 243     return outlist
 244
 245   def Tokenize(self, data, filename='__no_file__'):
 246     lexer = self.Lexer()
 247     lexer.lineno = 1
 248     lexer.filename = filename
 249     lexer.input(data)
 250     self.lines = data.split('\n')
 251
 252   def KnownTokens(self):
 253     return self.tokens
 254
 255   def Lexer(self):
 256     if not self._lexobj:
 257       self._lexobj = lex.lex(object=self, lextab=None, optimize=0)
 258     return self._lexobj
 259
 260   def _AddToken(self, token):
 261     if token in self.tokens:
 262       raise RuntimeError('Same token: ' + token)
 263     self.tokens.append(token)
 264
 265   def _AddTokens(self, tokens):
 266     for token in tokens:
 267       self._AddToken(token)
 268
 269   def _AddKeywords(self, keywords):
 270     for key in keywords:
 271       value = key.upper()
 272       self._AddToken(value)
 273       self.keywords[key] = value
 274
 275   def _DelKeywords(self, keywords):
 276     for key in keywords:
 277       self.tokens.remove(key.upper())
 278       del self.keywords[key]
 279
 280   def __init__(self):
 281     self.index = [0]
 282     self._lex_errors = 0
 283     self.linex = []
 284     self.filename = None
 285     self.keywords = {}
 286     self.tokens = []
 287     self._AddTokens(IDLLexer.tokens)
 288     self._AddKeywords(IDLLexer.keywords)
 289     self._lexobj = None
 290     self.last = None
 291     self.lines = None
 292
 293 # If run by itself, attempt to build the lexer
 294 if __name__ == '__main__':
 295   lexer_object = IDLLexer()