2 1,0-1,35: COMMENT "# Tests for the 'tokenize' module.\n"
3 2,0-2,43: COMMENT '# Large bits stolen from test_grammar.py. \n'
5 4,0-4,11: COMMENT '# Comments\n'
8 6,0-6,3: COMMENT "#'\n"
9 7,0-7,3: COMMENT '#"\n'
10 8,0-8,3: COMMENT '#\\\n'
11 9,7-9,9: COMMENT '#\n'
12 10,4-10,10: COMMENT '# abc\n'
13 11,0-12,4: STRING "'''#\n#'''"
14 12,4-12,5: NEWLINE '\n'
19 14,7-14,8: COMMENT '#'
20 14,8-14,9: NEWLINE '\n'
22 16,0-16,25: COMMENT '# Balancing continuation\n'
36 19,7-19,8: NEWLINE '\n'
47 21,4-21,5: NEWLINE '\n'
51 22,5-22,8: STRING "'a'"
53 22,9-22,10: NUMBER '5'
56 23,2-23,5: STRING "'b'"
60 23,8-23,9: NEWLINE '\n'
71 24,16-24,17: NUMBER '5'
84 26,9-26,12: NAME 'len'
93 29,3-29,4: NEWLINE '\n'
95 31,0-31,37: COMMENT '# Backslash means line continuation:\n'
100 33,2-33,3: NUMBER '1'
101 33,3-33,4: NEWLINE '\n'
103 35,0-35,55: COMMENT '# Backslash does not means continuation in comments :\\\n'
106 36,4-36,5: NUMBER '0'
107 36,5-36,6: NEWLINE '\n'
109 38,0-38,20: COMMENT '# Ordinary integers\n'
110 39,0-39,4: NUMBER '0xff'
112 39,8-39,11: NUMBER '255'
113 39,11-39,12: NEWLINE '\n'
114 40,0-40,4: NUMBER '0377'
116 40,8-40,11: NUMBER '255'
117 40,11-40,12: NEWLINE '\n'
118 41,0-41,10: NUMBER '2147483647'
120 41,16-41,28: NUMBER '017777777777'
121 41,28-41,29: NEWLINE '\n'
123 42,1-42,11: NUMBER '2147483647'
125 42,12-42,13: NUMBER '1'
127 42,17-42,29: NUMBER '020000000000'
128 42,29-42,30: NEWLINE '\n'
129 43,0-43,12: NUMBER '037777777777'
132 43,17-43,18: NUMBER '1'
133 43,18-43,19: NEWLINE '\n'
134 44,0-44,10: NUMBER '0xffffffff'
137 44,15-44,16: NUMBER '1'
138 44,16-44,17: NEWLINE '\n'
140 46,0-46,16: COMMENT '# Long integers\n'
143 47,4-47,6: NUMBER '0L'
144 47,6-47,7: NEWLINE '\n'
147 48,4-48,6: NUMBER '0l'
148 48,6-48,7: NEWLINE '\n'
151 49,4-49,23: NUMBER '0xffffffffffffffffL'
152 49,23-49,24: NEWLINE '\n'
155 50,4-50,23: NUMBER '0xffffffffffffffffl'
156 50,23-50,24: NEWLINE '\n'
159 51,4-51,23: NUMBER '077777777777777777L'
160 51,23-51,24: NEWLINE '\n'
163 52,4-52,23: NUMBER '077777777777777777l'
164 52,23-52,24: NEWLINE '\n'
167 53,4-53,35: NUMBER '123456789012345678901234567890L'
168 53,35-53,36: NEWLINE '\n'
171 54,4-54,35: NUMBER '123456789012345678901234567890l'
172 54,35-54,36: NEWLINE '\n'
174 56,0-56,25: COMMENT '# Floating-point numbers\n'
177 57,4-57,8: NUMBER '3.14'
178 57,8-57,9: NEWLINE '\n'
181 58,4-58,8: NUMBER '314.'
182 58,8-58,9: NEWLINE '\n'
185 59,4-59,9: NUMBER '0.314'
186 59,9-59,10: NEWLINE '\n'
187 60,0-60,18: COMMENT '# XXX x = 000.314\n'
190 61,4-61,8: NUMBER '.314'
191 61,8-61,9: NEWLINE '\n'
194 62,4-62,8: NUMBER '3e14'
195 62,8-62,9: NEWLINE '\n'
198 63,4-63,8: NUMBER '3E14'
199 63,8-63,9: NEWLINE '\n'
202 64,4-64,9: NUMBER '3e-14'
203 64,9-64,10: NEWLINE '\n'
206 65,4-65,9: NUMBER '3e+14'
207 65,9-65,10: NEWLINE '\n'
210 66,4-66,9: NUMBER '3.e14'
211 66,9-66,10: NEWLINE '\n'
214 67,4-67,9: NUMBER '.3e14'
215 67,9-67,10: NEWLINE '\n'
218 68,4-68,9: NUMBER '3.1e4'
219 68,9-68,10: NEWLINE '\n'
221 70,0-70,18: COMMENT '# String literals\n'
224 71,4-71,6: STRING "''"
228 71,12-71,14: STRING '""'
230 71,15-71,16: NEWLINE '\n'
233 72,4-72,8: STRING "'\\''"
235 72,10-72,11: NAME 'y'
237 72,14-72,17: STRING '"\'"'
239 72,18-72,19: NEWLINE '\n'
242 73,4-73,7: STRING '\'"\''
246 73,13-73,17: STRING '"\\""'
248 73,18-73,19: NEWLINE '\n'
251 74,4-74,32: STRING '"doesn\'t \\"shrink\\" does it"'
252 74,32-74,33: NEWLINE '\n'
255 75,4-75,31: STRING '\'doesn\\\'t "shrink" does it\''
256 75,31-75,32: NEWLINE '\n'
259 76,4-76,32: STRING '"does \\"shrink\\" doesn\'t it"'
260 76,32-76,33: NEWLINE '\n'
263 77,4-77,31: STRING '\'does "shrink" doesn\\\'t it\''
264 77,31-77,32: NEWLINE '\n'
267 78,4-83,3: STRING '"""\nThe "quick"\nbrown fox\njumps over\nthe \'lazy\' dog.\n"""'
268 83,3-83,4: NEWLINE '\n'
271 84,4-84,63: STRING '\'\\nThe "quick"\\nbrown fox\\njumps over\\nthe \\\'lazy\\\' dog.\\n\''
272 84,63-84,64: NEWLINE '\n'
275 85,4-90,3: STRING '\'\'\'\nThe "quick"\nbrown fox\njumps over\nthe \'lazy\' dog.\n\'\'\''
277 90,4-90,5: NEWLINE '\n'
280 91,4-96,1: STRING '"\\n\\\nThe \\"quick\\"\\n\\\nbrown fox\\n\\\njumps over\\n\\\nthe \'lazy\' dog.\\n\\\n"'
282 96,2-96,3: NEWLINE '\n'
285 97,4-102,1: STRING '\'\\n\\\nThe \\"quick\\"\\n\\\nbrown fox\\n\\\njumps over\\n\\\nthe \\\'lazy\\\' dog.\\n\\\n\''
287 102,2-102,3: NEWLINE '\n'
288 103,0-103,1: NAME 'x'
290 103,4-103,9: STRING "r'\\\\'"
291 103,10-103,11: OP '+'
292 103,12-103,17: STRING "R'\\\\'"
293 103,17-103,18: NEWLINE '\n'
294 104,0-104,1: NAME 'x'
296 104,4-104,9: STRING "r'\\''"
297 104,10-104,11: OP '+'
298 104,12-104,14: STRING "''"
299 104,14-104,15: NEWLINE '\n'
300 105,0-105,1: NAME 'y'
302 105,4-107,6: STRING "r'''\nfoo bar \\\\\nbaz'''"
304 107,9-108,6: STRING "R'''\nfoo'''"
305 108,6-108,7: NEWLINE '\n'
306 109,0-109,1: NAME 'y'
308 109,4-111,3: STRING 'r"""foo\nbar \\\\ baz\n"""'
310 111,6-112,3: STRING "R'''spam\n'''"
311 112,3-112,4: NEWLINE '\n'
312 113,0-113,1: NAME 'x'
314 113,4-113,10: STRING "u'abc'"
315 113,11-113,12: OP '+'
316 113,13-113,19: STRING "U'ABC'"
317 113,19-113,20: NEWLINE '\n'
318 114,0-114,1: NAME 'y'
320 114,4-114,10: STRING 'u"abc"'
321 114,11-114,12: OP '+'
322 114,13-114,19: STRING 'U"ABC"'
323 114,19-114,20: NEWLINE '\n'
324 115,0-115,1: NAME 'x'
326 115,4-115,11: STRING "ur'abc'"
327 115,12-115,13: OP '+'
328 115,14-115,21: STRING "Ur'ABC'"
329 115,22-115,23: OP '+'
330 115,24-115,31: STRING "uR'ABC'"
331 115,32-115,33: OP '+'
332 115,34-115,41: STRING "UR'ABC'"
333 115,41-115,42: NEWLINE '\n'
334 116,0-116,1: NAME 'y'
336 116,4-116,11: STRING 'ur"abc"'
337 116,12-116,13: OP '+'
338 116,14-116,21: STRING 'Ur"ABC"'
339 116,22-116,23: OP '+'
340 116,24-116,31: STRING 'uR"ABC"'
341 116,32-116,33: OP '+'
342 116,34-116,41: STRING 'UR"ABC"'
343 116,41-116,42: NEWLINE '\n'
344 117,0-117,1: NAME 'x'
346 117,4-117,10: STRING "ur'\\\\'"
347 117,11-117,12: OP '+'
348 117,13-117,19: STRING "UR'\\\\'"
349 117,19-117,20: NEWLINE '\n'
350 118,0-118,1: NAME 'x'
352 118,4-118,10: STRING "ur'\\''"
353 118,11-118,12: OP '+'
354 118,13-118,15: STRING "''"
355 118,15-118,16: NEWLINE '\n'
356 119,0-119,1: NAME 'y'
358 119,4-121,6: STRING "ur'''\nfoo bar \\\\\nbaz'''"
360 121,9-122,6: STRING "UR'''\nfoo'''"
361 122,6-122,7: NEWLINE '\n'
362 123,0-123,1: NAME 'y'
364 123,4-125,3: STRING 'Ur"""foo\nbar \\\\ baz\n"""'
366 125,6-126,3: STRING "uR'''spam\n'''"
367 126,3-126,4: NEWLINE '\n'
369 128,0-128,14: COMMENT '# Indentation\n'
370 129,0-129,2: NAME 'if'
371 129,3-129,4: NUMBER '1'
373 129,5-129,6: NEWLINE '\n'
374 130,0-130,4: INDENT ' '
375 130,4-130,5: NAME 'x'
377 130,8-130,9: NUMBER '2'
378 130,9-130,10: NEWLINE '\n'
379 131,0-131,0: DEDENT ''
380 131,0-131,2: NAME 'if'
381 131,3-131,4: NUMBER '1'
383 131,5-131,6: NEWLINE '\n'
384 132,0-132,8: INDENT ' '
385 132,8-132,9: NAME 'x'
386 132,10-132,11: OP '='
387 132,12-132,13: NUMBER '2'
388 132,13-132,14: NEWLINE '\n'
389 133,0-133,0: DEDENT ''
390 133,0-133,2: NAME 'if'
391 133,3-133,4: NUMBER '1'
393 133,5-133,6: NEWLINE '\n'
394 134,0-134,4: INDENT ' '
395 134,4-134,9: NAME 'while'
396 134,10-134,11: NUMBER '0'
397 134,11-134,12: OP ':'
398 134,12-134,13: NEWLINE '\n'
399 135,0-135,5: INDENT ' '
400 135,5-135,7: NAME 'if'
401 135,8-135,9: NUMBER '0'
403 135,10-135,11: NEWLINE '\n'
404 136,0-136,11: INDENT ' '
405 136,11-136,12: NAME 'x'
406 136,13-136,14: OP '='
407 136,15-136,16: NUMBER '2'
408 136,16-136,17: NEWLINE '\n'
409 137,5-137,5: DEDENT ''
410 137,5-137,6: NAME 'x'
412 137,9-137,10: NUMBER '2'
413 137,10-137,11: NEWLINE '\n'
414 138,0-138,0: DEDENT ''
415 138,0-138,0: DEDENT ''
416 138,0-138,2: NAME 'if'
417 138,3-138,4: NUMBER '0'
419 138,5-138,6: NEWLINE '\n'
420 139,0-139,2: INDENT ' '
421 139,2-139,4: NAME 'if'
422 139,5-139,6: NUMBER '2'
424 139,7-139,8: NEWLINE '\n'
425 140,0-140,3: INDENT ' '
426 140,3-140,8: NAME 'while'
427 140,9-140,10: NUMBER '0'
428 140,10-140,11: OP ':'
429 140,11-140,12: NEWLINE '\n'
430 141,0-141,8: INDENT ' '
431 141,8-141,10: NAME 'if'
432 141,11-141,12: NUMBER '1'
433 141,12-141,13: OP ':'
434 141,13-141,14: NEWLINE '\n'
435 142,0-142,10: INDENT ' '
436 142,10-142,11: NAME 'x'
437 142,12-142,13: OP '='
438 142,14-142,15: NUMBER '2'
439 142,15-142,16: NEWLINE '\n'
441 144,0-144,12: COMMENT '# Operators\n'
443 146,0-146,0: DEDENT ''
444 146,0-146,0: DEDENT ''
445 146,0-146,0: DEDENT ''
446 146,0-146,0: DEDENT ''
447 146,0-146,3: NAME 'def'
448 146,4-146,7: NAME 'd22'
450 146,8-146,9: NAME 'a'
452 146,11-146,12: NAME 'b'
453 146,12-146,13: OP ','
454 146,14-146,15: NAME 'c'
455 146,15-146,16: OP '='
456 146,16-146,17: NUMBER '1'
457 146,17-146,18: OP ','
458 146,19-146,20: NAME 'd'
459 146,20-146,21: OP '='
460 146,21-146,22: NUMBER '2'
461 146,22-146,23: OP ')'
462 146,23-146,24: OP ':'
463 146,25-146,29: NAME 'pass'
464 146,29-146,30: NEWLINE '\n'
465 147,0-147,3: NAME 'def'
466 147,4-147,8: NAME 'd01v'
468 147,9-147,10: NAME 'a'
469 147,10-147,11: OP '='
470 147,11-147,12: NUMBER '1'
471 147,12-147,13: OP ','
472 147,14-147,15: OP '*'
473 147,15-147,20: NAME 'restt'
474 147,20-147,21: OP ','
475 147,22-147,24: OP '**'
476 147,24-147,29: NAME 'restd'
477 147,29-147,30: OP ')'
478 147,30-147,31: OP ':'
479 147,32-147,36: NAME 'pass'
480 147,36-147,37: NEWLINE '\n'
483 149,1-149,2: NAME 'x'
485 149,4-149,5: NAME 'y'
488 149,10-149,11: OP '('
489 149,11-149,12: OP '{'
490 149,12-149,15: STRING "'a'"
491 149,15-149,16: OP ':'
492 149,16-149,17: NUMBER '1'
493 149,17-149,18: OP '}'
494 149,18-149,19: OP ','
495 149,20-149,21: OP '{'
496 149,21-149,24: STRING "'b'"
497 149,24-149,25: OP ':'
498 149,25-149,26: NUMBER '2'
499 149,26-149,27: OP '}'
500 149,27-149,28: OP ')'
501 149,28-149,29: NEWLINE '\n'
503 151,0-151,13: COMMENT '# comparison\n'
504 152,0-152,2: NAME 'if'
505 152,3-152,4: NUMBER '1'
507 152,7-152,8: NUMBER '1'
509 152,11-152,12: NUMBER '1'
510 152,13-152,15: OP '=='
511 152,16-152,17: NUMBER '1'
512 152,18-152,20: OP '>='
513 152,21-152,22: NUMBER '1'
514 152,23-152,25: OP '<='
515 152,26-152,27: NUMBER '1'
516 152,28-152,30: OP '<>'
517 152,31-152,32: NUMBER '1'
518 152,33-152,35: OP '!='
519 152,36-152,37: NUMBER '1'
520 152,38-152,40: NAME 'in'
521 152,41-152,42: NUMBER '1'
522 152,43-152,46: NAME 'not'
523 152,47-152,49: NAME 'in'
524 152,50-152,51: NUMBER '1'
525 152,52-152,54: NAME 'is'
526 152,55-152,56: NUMBER '1'
527 152,57-152,59: NAME 'is'
528 152,60-152,63: NAME 'not'
529 152,64-152,65: NUMBER '1'
530 152,65-152,66: OP ':'
531 152,67-152,71: NAME 'pass'
532 152,71-152,72: NEWLINE '\n'
534 154,0-154,9: COMMENT '# binary\n'
535 155,0-155,1: NAME 'x'
537 155,4-155,5: NUMBER '1'
539 155,8-155,9: NUMBER '1'
540 155,9-155,10: NEWLINE '\n'
541 156,0-156,1: NAME 'x'
543 156,4-156,5: NUMBER '1'
545 156,8-156,9: NUMBER '1'
546 156,9-156,10: NEWLINE '\n'
547 157,0-157,1: NAME 'x'
549 157,4-157,5: NUMBER '1'
551 157,8-157,9: NUMBER '1'
552 157,9-157,10: NEWLINE '\n'
554 159,0-159,8: COMMENT '# shift\n'
555 160,0-160,1: NAME 'x'
557 160,4-160,5: NUMBER '1'
559 160,9-160,10: NUMBER '1'
560 160,11-160,13: OP '>>'
561 160,14-160,15: NUMBER '1'
562 160,15-160,16: NEWLINE '\n'
564 162,0-162,11: COMMENT '# additive\n'
565 163,0-163,1: NAME 'x'
567 163,4-163,5: NUMBER '1'
569 163,8-163,9: NUMBER '1'
570 163,10-163,11: OP '+'
571 163,12-163,13: NUMBER '1'
572 163,14-163,15: OP '-'
573 163,16-163,17: NUMBER '1'
574 163,18-163,19: OP '+'
575 163,20-163,21: NUMBER '1'
576 163,21-163,22: NEWLINE '\n'
578 165,0-165,17: COMMENT '# multiplicative\n'
579 166,0-166,1: NAME 'x'
581 166,4-166,5: NUMBER '1'
583 166,8-166,9: NUMBER '1'
584 166,10-166,11: OP '*'
585 166,12-166,13: NUMBER '1'
586 166,14-166,15: OP '%'
587 166,16-166,17: NUMBER '1'
588 166,17-166,18: NEWLINE '\n'
590 168,0-168,8: COMMENT '# unary\n'
591 169,0-169,1: NAME 'x'
594 169,5-169,6: NUMBER '1'
596 169,9-169,10: NUMBER '1'
597 169,11-169,12: OP '&'
598 169,13-169,14: NUMBER '1'
599 169,15-169,16: OP '|'
600 169,17-169,18: NUMBER '1'
601 169,19-169,20: OP '&'
602 169,21-169,22: NUMBER '1'
603 169,23-169,24: OP '^'
604 169,25-169,26: OP '-'
605 169,26-169,27: NUMBER '1'
606 169,27-169,28: NEWLINE '\n'
607 170,0-170,1: NAME 'x'
610 170,5-170,6: NUMBER '1'
612 170,7-170,8: NUMBER '1'
614 170,9-170,10: NUMBER '1'
615 170,11-170,12: OP '+'
616 170,13-170,14: NUMBER '1'
617 170,14-170,15: OP '*'
618 170,15-170,16: NUMBER '1'
619 170,17-170,18: OP '-'
620 170,19-170,20: OP '-'
621 170,20-170,21: OP '-'
622 170,21-170,22: OP '-'
623 170,22-170,23: NUMBER '1'
624 170,23-170,24: OP '*'
625 170,24-170,25: NUMBER '1'
626 170,25-170,26: NEWLINE '\n'
628 172,0-172,11: COMMENT '# selector\n'
629 173,0-173,6: NAME 'import'
630 173,7-173,10: NAME 'sys'
631 173,10-173,11: OP ','
632 173,12-173,16: NAME 'time'
633 173,16-173,17: NEWLINE '\n'
634 174,0-174,1: NAME 'x'
636 174,4-174,7: NAME 'sys'
638 174,8-174,15: NAME 'modules'
639 174,15-174,16: OP '['
640 174,16-174,22: STRING "'time'"
641 174,22-174,23: OP ']'
642 174,23-174,24: OP '.'
643 174,24-174,28: NAME 'time'
644 174,28-174,29: OP '('
645 174,29-174,30: OP ')'
646 174,30-174,31: NEWLINE '\n'
648 176,0-176,0: ENDMARKER ''