lexers/python.lua

   1 -- Copyright 2006-2015 Mitchell mitchell.att.foicica.com. See LICENSE.
   2 -- Python LPeg lexer.
   3
   4 local l = require('lexer')
   5 local token, word_match = l.token, l.word_match
   6 local P, R, S = lpeg.P, lpeg.R, lpeg.S
   7
   8 local M = {_NAME = 'python'}
   9
  10 -- Whitespace.
  11 local ws = token(l.WHITESPACE, l.space^1)
  12
  13 -- Comments.
  14 local comment = token(l.COMMENT, '#' * l.nonnewline_esc^0)
  15
  16 -- Strings.
  17 local sq_str = P('u')^-1 * l.delimited_range("'", true)
  18 local dq_str = P('U')^-1 * l.delimited_range('"', true)
  19 local triple_sq_str = "'''" * (l.any - "'''")^0 * P("'''")^-1
  20 local triple_dq_str = '"""' * (l.any - '"""')^0 * P('"""')^-1
  21 -- TODO: raw_strs cannot end in single \.
  22 local raw_sq_str = P('u')^-1 * 'r' * l.delimited_range("'", false, true)
  23 local raw_dq_str = P('U')^-1 * 'R' * l.delimited_range('"', false, true)
  24 local string = token(l.STRING, triple_sq_str + triple_dq_str + sq_str + dq_str +
  25                                raw_sq_str + raw_dq_str)
  26
  27 -- Numbers.
  28 local dec = l.digit^1 * S('Ll')^-1
  29 local bin = '0b' * S('01')^1 * ('_' * S('01')^1)^0
  30 local oct = '0' * R('07')^1 * S('Ll')^-1
  31 local integer = S('+-')^-1 * (bin + l.hex_num + oct + dec)
  32 local number = token(l.NUMBER, l.float + integer)
  33
  34 -- Keywords.
  35 local keyword = token(l.KEYWORD, word_match{
  36   'and', 'as', 'assert', 'break', 'class', 'continue', 'def', 'del', 'elif',
  37   'else', 'except', 'exec', 'finally', 'for', 'from', 'global', 'if', 'import',
  38   'in', 'is', 'lambda', 'nonlocal', 'not', 'or', 'pass', 'print', 'raise',
  39   'return', 'try', 'while', 'with', 'yield',
  40   -- Descriptors/attr access.
  41   '__get__', '__set__', '__delete__', '__slots__',
  42   -- Class.
  43   '__new__', '__init__', '__del__', '__repr__', '__str__', '__cmp__',
  44   '__index__', '__lt__', '__le__', '__gt__', '__ge__', '__eq__', '__ne__',
  45   '__hash__', '__nonzero__', '__getattr__', '__getattribute__', '__setattr__',
  46   '__delattr__', '__call__',
  47   -- Operator.
  48   '__add__', '__sub__', '__mul__', '__div__', '__floordiv__', '__mod__',
  49   '__divmod__', '__pow__', '__and__', '__xor__', '__or__', '__lshift__',
  50   '__rshift__', '__nonzero__', '__neg__', '__pos__', '__abs__', '__invert__',
  51   '__iadd__', '__isub__', '__imul__', '__idiv__', '__ifloordiv__', '__imod__',
  52   '__ipow__', '__iand__', '__ixor__', '__ior__', '__ilshift__', '__irshift__',
  53   -- Conversions.
  54   '__int__', '__long__', '__float__', '__complex__', '__oct__', '__hex__',
  55   '__coerce__',
  56   -- Containers.
  57   '__len__', '__getitem__', '__missing__', '__setitem__', '__delitem__',
  58   '__contains__', '__iter__', '__getslice__', '__setslice__', '__delslice__',
  59   -- Module and class attribs.
  60   '__doc__', '__name__', '__dict__', '__file__', '__path__', '__module__',
  61   '__bases__', '__class__', '__self__',
  62   -- Stdlib/sys.
  63   '__builtin__', '__future__', '__main__', '__import__', '__stdin__',
  64   '__stdout__', '__stderr__',
  65   -- Other.
  66   '__debug__', '__doc__', '__import__', '__name__'
  67 })
  68
  69 -- Functions.
  70 local func = token(l.FUNCTION, word_match{
  71   'abs', 'all', 'any', 'apply', 'basestring', 'bool', 'buffer', 'callable',
  72   'chr', 'classmethod', 'cmp', 'coerce', 'compile', 'complex', 'copyright',
  73   'credits', 'delattr', 'dict', 'dir', 'divmod', 'enumerate', 'eval',
  74   'execfile', 'exit', 'file', 'filter', 'float', 'frozenset', 'getattr',
  75   'globals', 'hasattr', 'hash', 'help', 'hex', 'id', 'input', 'int', 'intern',
  76   'isinstance', 'issubclass', 'iter', 'len', 'license', 'list', 'locals',
  77   'long', 'map', 'max', 'min', 'object', 'oct', 'open', 'ord', 'pow',
  78   'property', 'quit', 'range', 'raw_input', 'reduce', 'reload', 'repr',
  79   'reversed', 'round', 'set', 'setattr', 'slice', 'sorted', 'staticmethod',
  80   'str', 'sum', 'super', 'tuple', 'type', 'unichr', 'unicode', 'vars', 'xrange',
  81   'zip'
  82 })
  83
  84 -- Constants.
  85 local constant = token(l.CONSTANT, word_match{
  86   'ArithmeticError', 'AssertionError', 'AttributeError', 'BaseException',
  87   'DeprecationWarning', 'EOFError', 'Ellipsis', 'EnvironmentError', 'Exception',
  88   'False', 'FloatingPointError', 'FutureWarning', 'GeneratorExit', 'IOError',
  89   'ImportError', 'ImportWarning', 'IndentationError', 'IndexError', 'KeyError',
  90   'KeyboardInterrupt', 'LookupError', 'MemoryError', 'NameError', 'None',
  91   'NotImplemented', 'NotImplementedError', 'OSError', 'OverflowError',
  92   'PendingDeprecationWarning', 'ReferenceError', 'RuntimeError',
  93   'RuntimeWarning', 'StandardError', 'StopIteration', 'SyntaxError',
  94   'SyntaxWarning', 'SystemError', 'SystemExit', 'TabError', 'True', 'TypeError',
  95   'UnboundLocalError', 'UnicodeDecodeError', 'UnicodeEncodeError',
  96   'UnicodeError', 'UnicodeTranslateError', 'UnicodeWarning', 'UserWarning',
  97   'ValueError', 'Warning', 'ZeroDivisionError'
  98 })
  99
 100 -- Self.
 101 local self = token('self', P('self'))
 102
 103 -- Identifiers.
 104 local identifier = token(l.IDENTIFIER, l.word)
 105
 106 -- Operators.
 107 local operator = token(l.OPERATOR, S('!%^&*()[]{}-=+/|:;.,?<>~`'))
 108
 109 -- Decorators.
 110 local decorator = token('decorator', l.starts_line('@') * l.nonnewline^0)
 111
 112 M._rules = {
 113   {'whitespace', ws},
 114   {'keyword', keyword},
 115   {'function', func},
 116   {'constant', constant},
 117   {'self', self},
 118   {'identifier', identifier},
 119   {'comment', comment},
 120   {'string', string},
 121   {'number', number},
 122   {'decorator', decorator},
 123   {'operator', operator},
 124 }
 125
 126
 127 M._tokenstyles = {
 128   self = l.STYLE_TYPE,
 129   decorator = l.STYLE_PREPROCESSOR
 130 }
 131
 132 M._FOLDBYINDENTATION = true
 133
 134 return M