2 1,0-1,35: COMMENT "# Tests for the 'tokenize' module.\012"
3 2,0-2,43: COMMENT '# Large bits stolen from test_grammar.py. \012'
5 4,0-4,11: COMMENT '# Comments\012'
7 5,3-5,4: NEWLINE '\012'
8 6,0-6,3: COMMENT "#'\012"
9 7,0-7,3: COMMENT '#"\012'
10 8,0-8,3: COMMENT '#\\\012'
11 9,7-9,9: COMMENT '#\012'
12 10,4-10,10: COMMENT '# abc\012'
13 11,0-12,4: STRING "'''#\012#'''"
14 12,4-12,5: NEWLINE '\012'
19 14,7-14,8: COMMENT '#'
20 14,8-14,9: NEWLINE '\012'
22 16,0-16,25: COMMENT '# Balancing continuation\012'
31 18,10-18,11: NL '\012'
36 19,7-19,8: NEWLINE '\012'
44 20,10-20,11: NL '\012'
47 21,4-21,5: NEWLINE '\012'
51 22,5-22,8: STRING "'a'"
53 22,9-22,10: NUMBER '5'
55 22,11-22,12: NL '\012'
56 23,2-23,5: STRING "'b'"
60 23,8-23,9: NEWLINE '\012'
71 24,16-24,17: NUMBER '5'
77 24,24-24,25: NL '\012'
84 26,9-26,12: NAME 'len'
87 26,14-26,15: NL '\012'
93 29,3-29,4: NEWLINE '\012'
95 31,0-31,37: COMMENT '# Backslash means line continuation:\012'
100 33,2-33,3: NUMBER '1'
101 33,3-33,4: NEWLINE '\012'
103 35,0-35,55: COMMENT '# Backslash does not means continuation in comments :\\\012'
106 36,4-36,5: NUMBER '0'
107 36,5-36,6: NEWLINE '\012'
109 38,0-38,20: COMMENT '# Ordinary integers\012'
110 39,0-39,4: NUMBER '0xff'
112 39,8-39,11: NUMBER '255'
113 39,11-39,12: NEWLINE '\012'
114 40,0-40,4: NUMBER '0377'
116 40,8-40,11: NUMBER '255'
117 40,11-40,12: NEWLINE '\012'
118 41,0-41,10: NUMBER '2147483647'
120 41,16-41,28: NUMBER '017777777777'
121 41,28-41,29: NEWLINE '\012'
123 42,1-42,11: NUMBER '2147483647'
125 42,12-42,13: NUMBER '1'
127 42,17-42,29: NUMBER '020000000000'
128 42,29-42,30: NEWLINE '\012'
129 43,0-43,12: NUMBER '037777777777'
132 43,17-43,18: NUMBER '1'
133 43,18-43,19: NEWLINE '\012'
134 44,0-44,10: NUMBER '0xffffffff'
137 44,15-44,16: NUMBER '1'
138 44,16-44,17: NEWLINE '\012'
140 46,0-46,16: COMMENT '# Long integers\012'
143 47,4-47,6: NUMBER '0L'
144 47,6-47,7: NEWLINE '\012'
147 48,4-48,6: NUMBER '0l'
148 48,6-48,7: NEWLINE '\012'
151 49,4-49,23: NUMBER '0xffffffffffffffffL'
152 49,23-49,24: NEWLINE '\012'
155 50,4-50,23: NUMBER '0xffffffffffffffffl'
156 50,23-50,24: NEWLINE '\012'
159 51,4-51,23: NUMBER '077777777777777777L'
160 51,23-51,24: NEWLINE '\012'
163 52,4-52,23: NUMBER '077777777777777777l'
164 52,23-52,24: NEWLINE '\012'
167 53,4-53,35: NUMBER '123456789012345678901234567890L'
168 53,35-53,36: NEWLINE '\012'
171 54,4-54,35: NUMBER '123456789012345678901234567890l'
172 54,35-54,36: NEWLINE '\012'
174 56,0-56,25: COMMENT '# Floating-point numbers\012'
177 57,4-57,8: NUMBER '3.14'
178 57,8-57,9: NEWLINE '\012'
181 58,4-58,8: NUMBER '314.'
182 58,8-58,9: NEWLINE '\012'
185 59,4-59,9: NUMBER '0.314'
186 59,9-59,10: NEWLINE '\012'
187 60,0-60,18: COMMENT '# XXX x = 000.314\012'
190 61,4-61,8: NUMBER '.314'
191 61,8-61,9: NEWLINE '\012'
194 62,4-62,8: NUMBER '3e14'
195 62,8-62,9: NEWLINE '\012'
198 63,4-63,8: NUMBER '3E14'
199 63,8-63,9: NEWLINE '\012'
202 64,4-64,9: NUMBER '3e-14'
203 64,9-64,10: NEWLINE '\012'
206 65,4-65,9: NUMBER '3e+14'
207 65,9-65,10: NEWLINE '\012'
210 66,4-66,9: NUMBER '3.e14'
211 66,9-66,10: NEWLINE '\012'
214 67,4-67,9: NUMBER '.3e14'
215 67,9-67,10: NEWLINE '\012'
218 68,4-68,9: NUMBER '3.1e4'
219 68,9-68,10: NEWLINE '\012'
221 70,0-70,18: COMMENT '# String literals\012'
224 71,4-71,6: STRING "''"
228 71,12-71,14: STRING '""'
230 71,15-71,16: NEWLINE '\012'
233 72,4-72,8: STRING "'\\''"
235 72,10-72,11: NAME 'y'
237 72,14-72,17: STRING '"\'"'
239 72,18-72,19: NEWLINE '\012'
242 73,4-73,7: STRING '\'"\''
246 73,13-73,17: STRING '"\\""'
248 73,18-73,19: NEWLINE '\012'
251 74,4-74,32: STRING '"doesn\'t \\"shrink\\" does it"'
252 74,32-74,33: NEWLINE '\012'
255 75,4-75,31: STRING '\'doesn\\\'t "shrink" does it\''
256 75,31-75,32: NEWLINE '\012'
259 76,4-76,32: STRING '"does \\"shrink\\" doesn\'t it"'
260 76,32-76,33: NEWLINE '\012'
263 77,4-77,31: STRING '\'does "shrink" doesn\\\'t it\''
264 77,31-77,32: NEWLINE '\012'
267 78,4-83,3: STRING '"""\012The "quick"\012brown fox\012jumps over\012the \'lazy\' dog.\012"""'
268 83,3-83,4: NEWLINE '\012'
271 84,4-84,63: STRING '\'\\nThe "quick"\\nbrown fox\\njumps over\\nthe \\\'lazy\\\' dog.\\n\''
272 84,63-84,64: NEWLINE '\012'
275 85,4-90,3: STRING '\'\'\'\012The "quick"\012brown fox\012jumps over\012the \'lazy\' dog.\012\'\'\''
277 90,4-90,5: NEWLINE '\012'
280 91,4-96,1: STRING '"\\n\\\012The \\"quick\\"\\n\\\012brown fox\\n\\\012jumps over\\n\\\012the \'lazy\' dog.\\n\\\012"'
282 96,2-96,3: NEWLINE '\012'
285 97,4-102,1: STRING '\'\\n\\\012The \\"quick\\"\\n\\\012brown fox\\n\\\012jumps over\\n\\\012the \\\'lazy\\\' dog.\\n\\\012\''
287 102,2-102,3: NEWLINE '\012'
288 103,0-103,1: NAME 'x'
290 103,4-103,9: STRING "r'\\\\'"
291 103,10-103,11: OP '+'
292 103,12-103,17: STRING "R'\\\\'"
293 103,17-103,18: NEWLINE '\012'
294 104,0-104,1: NAME 'x'
296 104,4-104,9: STRING "r'\\''"
297 104,10-104,11: OP '+'
298 104,12-104,14: STRING "''"
299 104,14-104,15: NEWLINE '\012'
300 105,0-105,1: NAME 'y'
302 105,4-107,6: STRING "r'''\012foo bar \\\\\012baz'''"
304 107,9-108,6: STRING "R'''\012foo'''"
305 108,6-108,7: NEWLINE '\012'
306 109,0-109,1: NAME 'y'
308 109,4-111,3: STRING 'r"""foo\012bar \\\\ baz\012"""'
310 111,6-112,3: STRING "R'''spam\012'''"
311 112,3-112,4: NEWLINE '\012'
312 113,0-113,1: NL '\012'
313 114,0-114,14: COMMENT '# Indentation\012'
314 115,0-115,2: NAME 'if'
315 115,3-115,4: NUMBER '1'
317 115,5-115,6: NEWLINE '\012'
318 116,0-116,4: INDENT ' '
319 116,4-116,5: NAME 'x'
321 116,8-116,9: NUMBER '2'
322 116,9-116,10: NEWLINE '\012'
323 117,0-117,0: DEDENT ''
324 117,0-117,2: NAME 'if'
325 117,3-117,4: NUMBER '1'
327 117,5-117,6: NEWLINE '\012'
328 118,0-118,8: INDENT ' '
329 118,8-118,9: NAME 'x'
330 118,10-118,11: OP '='
331 118,12-118,13: NUMBER '2'
332 118,13-118,14: NEWLINE '\012'
333 119,0-119,0: DEDENT ''
334 119,0-119,2: NAME 'if'
335 119,3-119,4: NUMBER '1'
337 119,5-119,6: NEWLINE '\012'
338 120,0-120,4: INDENT ' '
339 120,4-120,9: NAME 'while'
340 120,10-120,11: NUMBER '0'
341 120,11-120,12: OP ':'
342 120,12-120,13: NEWLINE '\012'
343 121,0-121,5: INDENT ' '
344 121,5-121,7: NAME 'if'
345 121,8-121,9: NUMBER '0'
347 121,10-121,11: NEWLINE '\012'
348 122,0-122,11: INDENT ' '
349 122,11-122,12: NAME 'x'
350 122,13-122,14: OP '='
351 122,15-122,16: NUMBER '2'
352 122,16-122,17: NEWLINE '\012'
353 123,5-123,5: DEDENT ''
354 123,5-123,6: NAME 'x'
356 123,9-123,10: NUMBER '2'
357 123,10-123,11: NEWLINE '\012'
358 124,0-124,0: DEDENT ''
359 124,0-124,0: DEDENT ''
360 124,0-124,2: NAME 'if'
361 124,3-124,4: NUMBER '0'
363 124,5-124,6: NEWLINE '\012'
364 125,0-125,2: INDENT ' '
365 125,2-125,4: NAME 'if'
366 125,5-125,6: NUMBER '2'
368 125,7-125,8: NEWLINE '\012'
369 126,0-126,3: INDENT ' '
370 126,3-126,8: NAME 'while'
371 126,9-126,10: NUMBER '0'
372 126,10-126,11: OP ':'
373 126,11-126,12: NEWLINE '\012'
374 127,0-127,8: INDENT ' '
375 127,8-127,10: NAME 'if'
376 127,11-127,12: NUMBER '1'
377 127,12-127,13: OP ':'
378 127,13-127,14: NEWLINE '\012'
379 128,0-128,10: INDENT ' '
380 128,10-128,11: NAME 'x'
381 128,12-128,13: OP '='
382 128,14-128,15: NUMBER '2'
383 128,15-128,16: NEWLINE '\012'
384 129,0-129,1: NL '\012'
385 130,0-130,12: COMMENT '# Operators\012'
386 131,0-131,1: NL '\012'
387 132,0-132,0: DEDENT ''
388 132,0-132,0: DEDENT ''
389 132,0-132,0: DEDENT ''
390 132,0-132,0: DEDENT ''
391 132,0-132,3: NAME 'def'
392 132,4-132,7: NAME 'd22'
394 132,8-132,9: NAME 'a'
396 132,11-132,12: NAME 'b'
397 132,12-132,13: OP ','
398 132,14-132,15: NAME 'c'
399 132,15-132,16: OP '='
400 132,16-132,17: NUMBER '1'
401 132,17-132,18: OP ','
402 132,19-132,20: NAME 'd'
403 132,20-132,21: OP '='
404 132,21-132,22: NUMBER '2'
405 132,22-132,23: OP ')'
406 132,23-132,24: OP ':'
407 132,25-132,29: NAME 'pass'
408 132,29-132,30: NEWLINE '\012'
409 133,0-133,3: NAME 'def'
410 133,4-133,8: NAME 'd01v'
412 133,9-133,10: NAME 'a'
413 133,10-133,11: OP '='
414 133,11-133,12: NUMBER '1'
415 133,12-133,13: OP ','
416 133,14-133,15: OP '*'
417 133,15-133,19: NAME 'rest'
418 133,19-133,20: OP ','
419 133,21-133,23: OP '**'
420 133,23-133,27: NAME 'rest'
421 133,27-133,28: OP ')'
422 133,28-133,29: OP ':'
423 133,30-133,34: NAME 'pass'
424 133,34-133,35: NEWLINE '\012'
425 134,0-134,1: NL '\012'
427 135,1-135,2: NAME 'x'
429 135,4-135,5: NAME 'y'
432 135,10-135,11: OP '('
433 135,11-135,12: OP '{'
434 135,12-135,15: STRING "'a'"
435 135,15-135,16: OP ':'
436 135,16-135,17: NUMBER '1'
437 135,17-135,18: OP '}'
438 135,18-135,19: OP ','
439 135,20-135,21: OP '{'
440 135,21-135,24: STRING "'b'"
441 135,24-135,25: OP ':'
442 135,25-135,26: NUMBER '2'
443 135,26-135,27: OP '}'
444 135,27-135,28: OP ')'
445 135,28-135,29: NEWLINE '\012'
446 136,0-136,1: NL '\012'
447 137,0-137,13: COMMENT '# comparison\012'
448 138,0-138,2: NAME 'if'
449 138,3-138,4: NUMBER '1'
451 138,7-138,8: NUMBER '1'
453 138,11-138,12: NUMBER '1'
454 138,13-138,15: OP '=='
455 138,16-138,17: NUMBER '1'
456 138,18-138,20: OP '>='
457 138,21-138,22: NUMBER '1'
458 138,23-138,25: OP '<='
459 138,26-138,27: NUMBER '1'
460 138,28-138,30: OP '<>'
461 138,31-138,32: NUMBER '1'
462 138,33-138,35: OP '!='
463 138,36-138,37: NUMBER '1'
464 138,38-138,40: NAME 'in'
465 138,41-138,42: NUMBER '1'
466 138,43-138,46: NAME 'not'
467 138,47-138,49: NAME 'in'
468 138,50-138,51: NUMBER '1'
469 138,52-138,54: NAME 'is'
470 138,55-138,56: NUMBER '1'
471 138,57-138,59: NAME 'is'
472 138,60-138,63: NAME 'not'
473 138,64-138,65: NUMBER '1'
474 138,65-138,66: OP ':'
475 138,67-138,71: NAME 'pass'
476 138,71-138,72: NEWLINE '\012'
477 139,0-139,1: NL '\012'
478 140,0-140,9: COMMENT '# binary\012'
479 141,0-141,1: NAME 'x'
481 141,4-141,5: NUMBER '1'
483 141,8-141,9: NUMBER '1'
484 141,9-141,10: NEWLINE '\012'
485 142,0-142,1: NAME 'x'
487 142,4-142,5: NUMBER '1'
489 142,8-142,9: NUMBER '1'
490 142,9-142,10: NEWLINE '\012'
491 143,0-143,1: NAME 'x'
493 143,4-143,5: NUMBER '1'
495 143,8-143,9: NUMBER '1'
496 143,9-143,10: NEWLINE '\012'
497 144,0-144,1: NL '\012'
498 145,0-145,8: COMMENT '# shift\012'
499 146,0-146,1: NAME 'x'
501 146,4-146,5: NUMBER '1'
503 146,9-146,10: NUMBER '1'
504 146,11-146,13: OP '>>'
505 146,14-146,15: NUMBER '1'
506 146,15-146,16: NEWLINE '\012'
507 147,0-147,1: NL '\012'
508 148,0-148,11: COMMENT '# additive\012'
509 149,0-149,1: NAME 'x'
511 149,4-149,5: NUMBER '1'
513 149,8-149,9: NUMBER '1'
514 149,10-149,11: OP '+'
515 149,12-149,13: NUMBER '1'
516 149,14-149,15: OP '-'
517 149,16-149,17: NUMBER '1'
518 149,18-149,19: OP '+'
519 149,20-149,21: NUMBER '1'
520 149,21-149,22: NEWLINE '\012'
521 150,0-150,1: NL '\012'
522 151,0-151,17: COMMENT '# multiplicative\012'
523 152,0-152,1: NAME 'x'
525 152,4-152,5: NUMBER '1'
527 152,8-152,9: NUMBER '1'
528 152,10-152,11: OP '*'
529 152,12-152,13: NUMBER '1'
530 152,14-152,15: OP '%'
531 152,16-152,17: NUMBER '1'
532 152,17-152,18: NEWLINE '\012'
533 153,0-153,1: NL '\012'
534 154,0-154,8: COMMENT '# unary\012'
535 155,0-155,1: NAME 'x'
538 155,5-155,6: NUMBER '1'
540 155,9-155,10: NUMBER '1'
541 155,11-155,12: OP '&'
542 155,13-155,14: NUMBER '1'
543 155,15-155,16: OP '|'
544 155,17-155,18: NUMBER '1'
545 155,19-155,20: OP '&'
546 155,21-155,22: NUMBER '1'
547 155,23-155,24: OP '^'
548 155,25-155,26: OP '-'
549 155,26-155,27: NUMBER '1'
550 155,27-155,28: NEWLINE '\012'
551 156,0-156,1: NAME 'x'
554 156,5-156,6: NUMBER '1'
556 156,7-156,8: NUMBER '1'
558 156,9-156,10: NUMBER '1'
559 156,11-156,12: OP '+'
560 156,13-156,14: NUMBER '1'
561 156,14-156,15: OP '*'
562 156,15-156,16: NUMBER '1'
563 156,17-156,18: OP '-'
564 156,19-156,20: OP '-'
565 156,20-156,21: OP '-'
566 156,21-156,22: OP '-'
567 156,22-156,23: NUMBER '1'
568 156,23-156,24: OP '*'
569 156,24-156,25: NUMBER '1'
570 156,25-156,26: NEWLINE '\012'
571 157,0-157,1: NL '\012'
572 158,0-158,11: COMMENT '# selector\012'
573 159,0-159,6: NAME 'import'
574 159,7-159,10: NAME 'sys'
575 159,10-159,11: OP ','
576 159,12-159,16: NAME 'time'
577 159,16-159,17: NEWLINE '\012'
578 160,0-160,1: NAME 'x'
580 160,4-160,7: NAME 'sys'
582 160,8-160,15: NAME 'modules'
583 160,15-160,16: OP '['
584 160,16-160,22: STRING "'time'"
585 160,22-160,23: OP ']'
586 160,23-160,24: OP '.'
587 160,24-160,28: NAME 'time'
588 160,28-160,29: OP '('
589 160,29-160,30: OP ')'
590 160,30-160,31: NEWLINE '\012'
591 161,0-161,1: NL '\012'
592 162,0-162,0: ENDMARKER ''