Lib/token.py

   1 # Tokens (from "token.h")
   2
   3 ENDMARKER = 0
   4 NAME = 1
   5 NUMBER = 2
   6 STRING = 3
   7 NEWLINE = 4
   8 INDENT = 5
   9 DEDENT = 6
  10 LPAR = 7
  11 RPAR = 8
  12 LSQB = 9
  13 RSQB = 10
  14 COLON = 11
  15 COMMA = 12
  16 SEMI = 13
  17 PLUS = 14
  18 MINUS = 15
  19 STAR = 16
  20 SLASH = 17
  21 VBAR = 18
  22 AMPER = 19
  23 LESS = 20
  24 GREATER = 21
  25 EQUAL = 22
  26 DOT = 23
  27 PERCENT = 24
  28 BACKQUOTE = 25
  29 LBRACE = 26
  30 RBRACE = 27
  31 EQEQUAL = 28
  32 NOTEQUAL = 29
  33 LESSEQUAL = 30
  34 GREATEREQUAL = 31
  35 TILDE = 32
  36 CIRCUMFLEX = 33
  37 LEFTSHIFT = 34
  38 RIGHTSHIFT = 35
  39 OP = 36
  40 ERRORTOKEN = 37
  41
  42 names = dir()
  43 tok_name = {}
  44 for name in names:
  45         number = eval(name)
  46         tok_name[number] = name
  47
  48 N_TOKENS = 38                           # Number of tokens including ERRORTOKEN
  49
  50 NT_OFFSET = 256                         # Start of non-terminal symbols