Lib/test/tokenize_tests.py

   1 # Tests for the 'tokenize' module.
   2 # Large bits stolen from test_grammar.py.
   3
   4 # Comments
   5 "#"
   6 #'
   7 #"
   8 #\
   9        #
  10     # abc
  11 '''#
  12 #'''
  13
  14 x = 1  #
  15
  16 # Balancing continuation
  17
  18 a = (3, 4,
  19   5, 6)
  20 y = [3, 4,
  21   5]
  22 z = {'a':5,
  23   'b':6}
  24 x = (len(`y`) + 5*x - a[
  25    3 ]
  26    - x + len({
  27    }
  28     )
  29   )
  30
  31 # Backslash means line continuation:
  32 x = 1 \
  33 + 1
  34
  35 # Backslash does not means continuation in comments :\
  36 x = 0
  37
  38 # Ordinary integers
  39 0xff <> 255
  40 0377 <> 255
  41 2147483647   != 017777777777
  42 -2147483647-1 != 020000000000
  43 037777777777 != -1
  44 0xffffffff != -1
  45
  46 # Long integers
  47 x = 0L
  48 x = 0l
  49 x = 0xffffffffffffffffL
  50 x = 0xffffffffffffffffl
  51 x = 077777777777777777L
  52 x = 077777777777777777l
  53 x = 123456789012345678901234567890L
  54 x = 123456789012345678901234567890l
  55
  56 # Floating-point numbers
  57 x = 3.14
  58 x = 314.
  59 x = 0.314
  60 # XXX x = 000.314
  61 x = .314
  62 x = 3e14
  63 x = 3E14
  64 x = 3e-14
  65 x = 3e+14
  66 x = 3.e14
  67 x = .3e14
  68 x = 3.1e4
  69
  70 # String literals
  71 x = ''; y = "";
  72 x = '\''; y = "'";
  73 x = '"'; y = "\"";
  74 x = "doesn't \"shrink\" does it"
  75 y = 'doesn\'t "shrink" does it'
  76 x = "does \"shrink\" doesn't it"
  77 y = 'does "shrink" doesn\'t it'
  78 x = """
  79 The "quick"
  80 brown fox
  81 jumps over
  82 the 'lazy' dog.
  83 """
  84 y = '\nThe "quick"\nbrown fox\njumps over\nthe \'lazy\' dog.\n'
  85 y = '''
  86 The "quick"
  87 brown fox
  88 jumps over
  89 the 'lazy' dog.
  90 ''';
  91 y = "\n\
  92 The \"quick\"\n\
  93 brown fox\n\
  94 jumps over\n\
  95 the 'lazy' dog.\n\
  96 ";
  97 y = '\n\
  98 The \"quick\"\n\
  99 brown fox\n\
 100 jumps over\n\
 101 the \'lazy\' dog.\n\
 102 ';
 103 x = r'\\' + R'\\'
 104 x = r'\'' + ''
 105 y = r'''
 106 foo bar \\
 107 baz''' + R'''
 108 foo'''
 109 y = r"""foo
 110 bar \\ baz
 111 """ + R'''spam
 112 '''
 113
 114 # Indentation
 115 if 1:
 116     x = 2
 117 if 1:
 118         x = 2
 119 if 1:
 120     while 0:
 121      if 0:
 122            x = 2
 123      x = 2
 124 if 0:
 125   if 2:
 126    while 0:
 127         if 1:
 128           x = 2
 129
 130 # Operators
 131
 132 def d22(a, b, c=1, d=2): pass
 133 def d01v(a=1, *rest, **rest): pass
 134
 135 (x, y) <> ({'a':1}, {'b':2})
 136
 137 # comparison
 138 if 1 < 1 > 1 == 1 >= 1 <= 1 <> 1 != 1 in 1 not in 1 is 1 is not 1: pass
 139
 140 # binary
 141 x = 1 & 1
 142 x = 1 ^ 1
 143 x = 1 | 1
 144
 145 # shift
 146 x = 1 << 1 >> 1
 147
 148 # additive
 149 x = 1 - 1 + 1 - 1 + 1
 150
 151 # multiplicative
 152 x = 1 / 1 * 1 % 1
 153
 154 # unary
 155 x = ~1 ^ 1 & 1 | 1 & 1 ^ -1
 156 x = -1*1/1 + 1*1 - ---1*1
 157
 158 # selector
 159 import sys, time
 160 x = sys.modules['time'].time()
 161