third_party/jinja2/parser.py

   1 # -*- coding: utf-8 -*-
   2 """
   3     jinja2.parser
   4     ~~~~~~~~~~~~~
   5
   6     Implements the template parser.
   7
   8     :copyright: (c) 2010 by the Jinja Team.
   9     :license: BSD, see LICENSE for more details.
  10 """
  11 from jinja2 import nodes
  12 from jinja2.exceptions import TemplateSyntaxError, TemplateAssertionError
  13 from jinja2.lexer import describe_token, describe_token_expr
  14 from jinja2._compat import next, imap
  15
  16
  17 #: statements that callinto
  18 _statement_keywords = frozenset(['for', 'if', 'block', 'extends', 'print',
  19                                  'macro', 'include', 'from', 'import',
  20                                  'set'])
  21 _compare_operators = frozenset(['eq', 'ne', 'lt', 'lteq', 'gt', 'gteq'])
  22
  23
  24 class Parser(object):
  25     """This is the central parsing class Jinja2 uses.  It's passed to
  26     extensions and can be used to parse expressions or statements.
  27     """
  28
  29     def __init__(self, environment, source, name=None, filename=None,
  30                  state=None):
  31         self.environment = environment
  32         self.stream = environment._tokenize(source, name, filename, state)
  33         self.name = name
  34         self.filename = filename
  35         self.closed = False
  36         self.extensions = {}
  37         for extension in environment.iter_extensions():
  38             for tag in extension.tags:
  39                 self.extensions[tag] = extension.parse
  40         self._last_identifier = 0
  41         self._tag_stack = []
  42         self._end_token_stack = []
  43
  44     def fail(self, msg, lineno=None, exc=TemplateSyntaxError):
  45         """Convenience method that raises `exc` with the message, passed
  46         line number or last line number as well as the current name and
  47         filename.
  48         """
  49         if lineno is None:
  50             lineno = self.stream.current.lineno
  51         raise exc(msg, lineno, self.name, self.filename)
  52
  53     def _fail_ut_eof(self, name, end_token_stack, lineno):
  54         expected = []
  55         for exprs in end_token_stack:
  56             expected.extend(imap(describe_token_expr, exprs))
  57         if end_token_stack:
  58             currently_looking = ' or '.join(
  59                 "'%s'" % describe_token_expr(expr)
  60                 for expr in end_token_stack[-1])
  61         else:
  62             currently_looking = None
  63
  64         if name is None:
  65             message = ['Unexpected end of template.']
  66         else:
  67             message = ['Encountered unknown tag \'%s\'.' % name]
  68
  69         if currently_looking:
  70             if name is not None and name in expected:
  71                 message.append('You probably made a nesting mistake. Jinja '
  72                                'is expecting this tag, but currently looking '
  73                                'for %s.' % currently_looking)
  74             else:
  75                 message.append('Jinja was looking for the following tags: '
  76                                '%s.' % currently_looking)
  77
  78         if self._tag_stack:
  79             message.append('The innermost block that needs to be '
  80                            'closed is \'%s\'.' % self._tag_stack[-1])
  81
  82         self.fail(' '.join(message), lineno)
  83
  84     def fail_unknown_tag(self, name, lineno=None):
  85         """Called if the parser encounters an unknown tag.  Tries to fail
  86         with a human readable error message that could help to identify
  87         the problem.
  88         """
  89         return self._fail_ut_eof(name, self._end_token_stack, lineno)
  90
  91     def fail_eof(self, end_tokens=None, lineno=None):
  92         """Like fail_unknown_tag but for end of template situations."""
  93         stack = list(self._end_token_stack)
  94         if end_tokens is not None:
  95             stack.append(end_tokens)
  96         return self._fail_ut_eof(None, stack, lineno)
  97
  98     def is_tuple_end(self, extra_end_rules=None):
  99         """Are we at the end of a tuple?"""
 100         if self.stream.current.type in ('variable_end', 'block_end', 'rparen'):
 101             return True
 102         elif extra_end_rules is not None:
 103             return self.stream.current.test_any(extra_end_rules)
 104         return False
 105
 106     def free_identifier(self, lineno=None):
 107         """Return a new free identifier as :class:`~jinja2.nodes.InternalName`."""
 108         self._last_identifier += 1
 109         rv = object.__new__(nodes.InternalName)
 110         nodes.Node.__init__(rv, 'fi%d' % self._last_identifier, lineno=lineno)
 111         return rv
 112
 113     def parse_statement(self):
 114         """Parse a single statement."""
 115         token = self.stream.current
 116         if token.type != 'name':
 117             self.fail('tag name expected', token.lineno)
 118         self._tag_stack.append(token.value)
 119         pop_tag = True
 120         try:
 121             if token.value in _statement_keywords:
 122                 return getattr(self, 'parse_' + self.stream.current.value)()
 123             if token.value == 'call':
 124                 return self.parse_call_block()
 125             if token.value == 'filter':
 126                 return self.parse_filter_block()
 127             ext = self.extensions.get(token.value)
 128             if ext is not None:
 129                 return ext(self)
 130
 131             # did not work out, remove the token we pushed by accident
 132             # from the stack so that the unknown tag fail function can
 133             # produce a proper error message.
 134             self._tag_stack.pop()
 135             pop_tag = False
 136             self.fail_unknown_tag(token.value, token.lineno)
 137         finally:
 138             if pop_tag:
 139                 self._tag_stack.pop()
 140
 141     def parse_statements(self, end_tokens, drop_needle=False):
 142         """Parse multiple statements into a list until one of the end tokens
 143         is reached.  This is used to parse the body of statements as it also
 144         parses template data if appropriate.  The parser checks first if the
 145         current token is a colon and skips it if there is one.  Then it checks
 146         for the block end and parses until if one of the `end_tokens` is
 147         reached.  Per default the active token in the stream at the end of
 148         the call is the matched end token.  If this is not wanted `drop_needle`
 149         can be set to `True` and the end token is removed.
 150         """
 151         # the first token may be a colon for python compatibility
 152         self.stream.skip_if('colon')
 153
 154         # in the future it would be possible to add whole code sections
 155         # by adding some sort of end of statement token and parsing those here.
 156         self.stream.expect('block_end')
 157         result = self.subparse(end_tokens)
 158
 159         # we reached the end of the template too early, the subparser
 160         # does not check for this, so we do that now
 161         if self.stream.current.type == 'eof':
 162             self.fail_eof(end_tokens)
 163
 164         if drop_needle:
 165             next(self.stream)
 166         return result
 167
 168     def parse_set(self):
 169         """Parse an assign statement."""
 170         lineno = next(self.stream).lineno
 171         target = self.parse_assign_target()
 172         self.stream.expect('assign')
 173         expr = self.parse_tuple()
 174         return nodes.Assign(target, expr, lineno=lineno)
 175
 176     def parse_for(self):
 177         """Parse a for loop."""
 178         lineno = self.stream.expect('name:for').lineno
 179         target = self.parse_assign_target(extra_end_rules=('name:in',))
 180         self.stream.expect('name:in')
 181         iter = self.parse_tuple(with_condexpr=False,
 182                                 extra_end_rules=('name:recursive',))
 183         test = None
 184         if self.stream.skip_if('name:if'):
 185             test = self.parse_expression()
 186         recursive = self.stream.skip_if('name:recursive')
 187         body = self.parse_statements(('name:endfor', 'name:else'))
 188         if next(self.stream).value == 'endfor':
 189             else_ = []
 190         else:
 191             else_ = self.parse_statements(('name:endfor',), drop_needle=True)
 192         return nodes.For(target, iter, body, else_, test,
 193                          recursive, lineno=lineno)
 194
 195     def parse_if(self):
 196         """Parse an if construct."""
 197         node = result = nodes.If(lineno=self.stream.expect('name:if').lineno)
 198         while 1:
 199             node.test = self.parse_tuple(with_condexpr=False)
 200             node.body = self.parse_statements(('name:elif', 'name:else',
 201                                                'name:endif'))
 202             token = next(self.stream)
 203             if token.test('name:elif'):
 204                 new_node = nodes.If(lineno=self.stream.current.lineno)
 205                 node.else_ = [new_node]
 206                 node = new_node
 207                 continue
 208             elif token.test('name:else'):
 209                 node.else_ = self.parse_statements(('name:endif',),
 210                                                    drop_needle=True)
 211             else:
 212                 node.else_ = []
 213             break
 214         return result
 215
 216     def parse_block(self):
 217         node = nodes.Block(lineno=next(self.stream).lineno)
 218         node.name = self.stream.expect('name').value
 219         node.scoped = self.stream.skip_if('name:scoped')
 220
 221         # common problem people encounter when switching from django
 222         # to jinja.  we do not support hyphens in block names, so let's
 223         # raise a nicer error message in that case.
 224         if self.stream.current.type == 'sub':
 225             self.fail('Block names in Jinja have to be valid Python '
 226                       'identifiers and may not contain hyphens, use an '
 227                       'underscore instead.')
 228
 229         node.body = self.parse_statements(('name:endblock',), drop_needle=True)
 230         self.stream.skip_if('name:' + node.name)
 231         return node
 232
 233     def parse_extends(self):
 234         node = nodes.Extends(lineno=next(self.stream).lineno)
 235         node.template = self.parse_expression()
 236         return node
 237
 238     def parse_import_context(self, node, default):
 239         if self.stream.current.test_any('name:with', 'name:without') and \
 240            self.stream.look().test('name:context'):
 241             node.with_context = next(self.stream).value == 'with'
 242             self.stream.skip()
 243         else:
 244             node.with_context = default
 245         return node
 246
 247     def parse_include(self):
 248         node = nodes.Include(lineno=next(self.stream).lineno)
 249         node.template = self.parse_expression()
 250         if self.stream.current.test('name:ignore') and \
 251            self.stream.look().test('name:missing'):
 252             node.ignore_missing = True
 253             self.stream.skip(2)
 254         else:
 255             node.ignore_missing = False
 256         return self.parse_import_context(node, True)
 257
 258     def parse_import(self):
 259         node = nodes.Import(lineno=next(self.stream).lineno)
 260         node.template = self.parse_expression()
 261         self.stream.expect('name:as')
 262         node.target = self.parse_assign_target(name_only=True).name
 263         return self.parse_import_context(node, False)
 264
 265     def parse_from(self):
 266         node = nodes.FromImport(lineno=next(self.stream).lineno)
 267         node.template = self.parse_expression()
 268         self.stream.expect('name:import')
 269         node.names = []
 270
 271         def parse_context():
 272             if self.stream.current.value in ('with', 'without') and \
 273                self.stream.look().test('name:context'):
 274                 node.with_context = next(self.stream).value == 'with'
 275                 self.stream.skip()
 276                 return True
 277             return False
 278
 279         while 1:
 280             if node.names:
 281                 self.stream.expect('comma')
 282             if self.stream.current.type == 'name':
 283                 if parse_context():
 284                     break
 285                 target = self.parse_assign_target(name_only=True)
 286                 if target.name.startswith('_'):
 287                     self.fail('names starting with an underline can not '
 288                               'be imported', target.lineno,
 289                               exc=TemplateAssertionError)
 290                 if self.stream.skip_if('name:as'):
 291                     alias = self.parse_assign_target(name_only=True)
 292                     node.names.append((target.name, alias.name))
 293                 else:
 294                     node.names.append(target.name)
 295                 if parse_context() or self.stream.current.type != 'comma':
 296                     break
 297             else:
 298                 break
 299         if not hasattr(node, 'with_context'):
 300             node.with_context = False
 301             self.stream.skip_if('comma')
 302         return node
 303
 304     def parse_signature(self, node):
 305         node.args = args = []
 306         node.defaults = defaults = []
 307         self.stream.expect('lparen')
 308         while self.stream.current.type != 'rparen':
 309             if args:
 310                 self.stream.expect('comma')
 311             arg = self.parse_assign_target(name_only=True)
 312             arg.set_ctx('param')
 313             if self.stream.skip_if('assign'):
 314                 defaults.append(self.parse_expression())
 315             args.append(arg)
 316         self.stream.expect('rparen')
 317
 318     def parse_call_block(self):
 319         node = nodes.CallBlock(lineno=next(self.stream).lineno)
 320         if self.stream.current.type == 'lparen':
 321             self.parse_signature(node)
 322         else:
 323             node.args = []
 324             node.defaults = []
 325
 326         node.call = self.parse_expression()
 327         if not isinstance(node.call, nodes.Call):
 328             self.fail('expected call', node.lineno)
 329         node.body = self.parse_statements(('name:endcall',), drop_needle=True)
 330         return node
 331
 332     def parse_filter_block(self):
 333         node = nodes.FilterBlock(lineno=next(self.stream).lineno)
 334         node.filter = self.parse_filter(None, start_inline=True)
 335         node.body = self.parse_statements(('name:endfilter',),
 336                                           drop_needle=True)
 337         return node
 338
 339     def parse_macro(self):
 340         node = nodes.Macro(lineno=next(self.stream).lineno)
 341         node.name = self.parse_assign_target(name_only=True).name
 342         self.parse_signature(node)
 343         node.body = self.parse_statements(('name:endmacro',),
 344                                           drop_needle=True)
 345         return node
 346
 347     def parse_print(self):
 348         node = nodes.Output(lineno=next(self.stream).lineno)
 349         node.nodes = []
 350         while self.stream.current.type != 'block_end':
 351             if node.nodes:
 352                 self.stream.expect('comma')
 353             node.nodes.append(self.parse_expression())
 354         return node
 355
 356     def parse_assign_target(self, with_tuple=True, name_only=False,
 357                             extra_end_rules=None):
 358         """Parse an assignment target.  As Jinja2 allows assignments to
 359         tuples, this function can parse all allowed assignment targets.  Per
 360         default assignments to tuples are parsed, that can be disable however
 361         by setting `with_tuple` to `False`.  If only assignments to names are
 362         wanted `name_only` can be set to `True`.  The `extra_end_rules`
 363         parameter is forwarded to the tuple parsing function.
 364         """
 365         if name_only:
 366             token = self.stream.expect('name')
 367             target = nodes.Name(token.value, 'store', lineno=token.lineno)
 368         else:
 369             if with_tuple:
 370                 target = self.parse_tuple(simplified=True,
 371                                           extra_end_rules=extra_end_rules)
 372             else:
 373                 target = self.parse_primary()
 374             target.set_ctx('store')
 375         if not target.can_assign():
 376             self.fail('can\'t assign to %r' % target.__class__.
 377                       __name__.lower(), target.lineno)
 378         return target
 379
 380     def parse_expression(self, with_condexpr=True):
 381         """Parse an expression.  Per default all expressions are parsed, if
 382         the optional `with_condexpr` parameter is set to `False` conditional
 383         expressions are not parsed.
 384         """
 385         if with_condexpr:
 386             return self.parse_condexpr()
 387         return self.parse_or()
 388
 389     def parse_condexpr(self):
 390         lineno = self.stream.current.lineno
 391         expr1 = self.parse_or()
 392         while self.stream.skip_if('name:if'):
 393             expr2 = self.parse_or()
 394             if self.stream.skip_if('name:else'):
 395                 expr3 = self.parse_condexpr()
 396             else:
 397                 expr3 = None
 398             expr1 = nodes.CondExpr(expr2, expr1, expr3, lineno=lineno)
 399             lineno = self.stream.current.lineno
 400         return expr1
 401
 402     def parse_or(self):
 403         lineno = self.stream.current.lineno
 404         left = self.parse_and()
 405         while self.stream.skip_if('name:or'):
 406             right = self.parse_and()
 407             left = nodes.Or(left, right, lineno=lineno)
 408             lineno = self.stream.current.lineno
 409         return left
 410
 411     def parse_and(self):
 412         lineno = self.stream.current.lineno
 413         left = self.parse_not()
 414         while self.stream.skip_if('name:and'):
 415             right = self.parse_not()
 416             left = nodes.And(left, right, lineno=lineno)
 417             lineno = self.stream.current.lineno
 418         return left
 419
 420     def parse_not(self):
 421         if self.stream.current.test('name:not'):
 422             lineno = next(self.stream).lineno
 423             return nodes.Not(self.parse_not(), lineno=lineno)
 424         return self.parse_compare()
 425
 426     def parse_compare(self):
 427         lineno = self.stream.current.lineno
 428         expr = self.parse_add()
 429         ops = []
 430         while 1:
 431             token_type = self.stream.current.type
 432             if token_type in _compare_operators:
 433                 next(self.stream)
 434                 ops.append(nodes.Operand(token_type, self.parse_add()))
 435             elif self.stream.skip_if('name:in'):
 436                 ops.append(nodes.Operand('in', self.parse_add()))
 437             elif self.stream.current.test('name:not') and \
 438                  self.stream.look().test('name:in'):
 439                 self.stream.skip(2)
 440                 ops.append(nodes.Operand('notin', self.parse_add()))
 441             else:
 442                 break
 443             lineno = self.stream.current.lineno
 444         if not ops:
 445             return expr
 446         return nodes.Compare(expr, ops, lineno=lineno)
 447
 448     def parse_add(self):
 449         lineno = self.stream.current.lineno
 450         left = self.parse_sub()
 451         while self.stream.current.type == 'add':
 452             next(self.stream)
 453             right = self.parse_sub()
 454             left = nodes.Add(left, right, lineno=lineno)
 455             lineno = self.stream.current.lineno
 456         return left
 457
 458     def parse_sub(self):
 459         lineno = self.stream.current.lineno
 460         left = self.parse_concat()
 461         while self.stream.current.type == 'sub':
 462             next(self.stream)
 463             right = self.parse_concat()
 464             left = nodes.Sub(left, right, lineno=lineno)
 465             lineno = self.stream.current.lineno
 466         return left
 467
 468     def parse_concat(self):
 469         lineno = self.stream.current.lineno
 470         args = [self.parse_mul()]
 471         while self.stream.current.type == 'tilde':
 472             next(self.stream)
 473             args.append(self.parse_mul())
 474         if len(args) == 1:
 475             return args[0]
 476         return nodes.Concat(args, lineno=lineno)
 477
 478     def parse_mul(self):
 479         lineno = self.stream.current.lineno
 480         left = self.parse_div()
 481         while self.stream.current.type == 'mul':
 482             next(self.stream)
 483             right = self.parse_div()
 484             left = nodes.Mul(left, right, lineno=lineno)
 485             lineno = self.stream.current.lineno
 486         return left
 487
 488     def parse_div(self):
 489         lineno = self.stream.current.lineno
 490         left = self.parse_floordiv()
 491         while self.stream.current.type == 'div':
 492             next(self.stream)
 493             right = self.parse_floordiv()
 494             left = nodes.Div(left, right, lineno=lineno)
 495             lineno = self.stream.current.lineno
 496         return left
 497
 498     def parse_floordiv(self):
 499         lineno = self.stream.current.lineno
 500         left = self.parse_mod()
 501         while self.stream.current.type == 'floordiv':
 502             next(self.stream)
 503             right = self.parse_mod()
 504             left = nodes.FloorDiv(left, right, lineno=lineno)
 505             lineno = self.stream.current.lineno
 506         return left
 507
 508     def parse_mod(self):
 509         lineno = self.stream.current.lineno
 510         left = self.parse_pow()
 511         while self.stream.current.type == 'mod':
 512             next(self.stream)
 513             right = self.parse_pow()
 514             left = nodes.Mod(left, right, lineno=lineno)
 515             lineno = self.stream.current.lineno
 516         return left
 517
 518     def parse_pow(self):
 519         lineno = self.stream.current.lineno
 520         left = self.parse_unary()
 521         while self.stream.current.type == 'pow':
 522             next(self.stream)
 523             right = self.parse_unary()
 524             left = nodes.Pow(left, right, lineno=lineno)
 525             lineno = self.stream.current.lineno
 526         return left
 527
 528     def parse_unary(self, with_filter=True):
 529         token_type = self.stream.current.type
 530         lineno = self.stream.current.lineno
 531         if token_type == 'sub':
 532             next(self.stream)
 533             node = nodes.Neg(self.parse_unary(False), lineno=lineno)
 534         elif token_type == 'add':
 535             next(self.stream)
 536             node = nodes.Pos(self.parse_unary(False), lineno=lineno)
 537         else:
 538             node = self.parse_primary()
 539         node = self.parse_postfix(node)
 540         if with_filter:
 541             node = self.parse_filter_expr(node)
 542         return node
 543
 544     def parse_primary(self):
 545         token = self.stream.current
 546         if token.type == 'name':
 547             if token.value in ('true', 'false', 'True', 'False'):
 548                 node = nodes.Const(token.value in ('true', 'True'),
 549                                    lineno=token.lineno)
 550             elif token.value in ('none', 'None'):
 551                 node = nodes.Const(None, lineno=token.lineno)
 552             else:
 553                 node = nodes.Name(token.value, 'load', lineno=token.lineno)
 554             next(self.stream)
 555         elif token.type == 'string':
 556             next(self.stream)
 557             buf = [token.value]
 558             lineno = token.lineno
 559             while self.stream.current.type == 'string':
 560                 buf.append(self.stream.current.value)
 561                 next(self.stream)
 562             node = nodes.Const(''.join(buf), lineno=lineno)
 563         elif token.type in ('integer', 'float'):
 564             next(self.stream)
 565             node = nodes.Const(token.value, lineno=token.lineno)
 566         elif token.type == 'lparen':
 567             next(self.stream)
 568             node = self.parse_tuple(explicit_parentheses=True)
 569             self.stream.expect('rparen')
 570         elif token.type == 'lbracket':
 571             node = self.parse_list()
 572         elif token.type == 'lbrace':
 573             node = self.parse_dict()
 574         else:
 575             self.fail("unexpected '%s'" % describe_token(token), token.lineno)
 576         return node
 577
 578     def parse_tuple(self, simplified=False, with_condexpr=True,
 579                     extra_end_rules=None, explicit_parentheses=False):
 580         """Works like `parse_expression` but if multiple expressions are
 581         delimited by a comma a :class:`~jinja2.nodes.Tuple` node is created.
 582         This method could also return a regular expression instead of a tuple
 583         if no commas where found.
 584
 585         The default parsing mode is a full tuple.  If `simplified` is `True`
 586         only names and literals are parsed.  The `no_condexpr` parameter is
 587         forwarded to :meth:`parse_expression`.
 588
 589         Because tuples do not require delimiters and may end in a bogus comma
 590         an extra hint is needed that marks the end of a tuple.  For example
 591         for loops support tuples between `for` and `in`.  In that case the
 592         `extra_end_rules` is set to ``['name:in']``.
 593
 594         `explicit_parentheses` is true if the parsing was triggered by an
 595         expression in parentheses.  This is used to figure out if an empty
 596         tuple is a valid expression or not.
 597         """
 598         lineno = self.stream.current.lineno
 599         if simplified:
 600             parse = self.parse_primary
 601         elif with_condexpr:
 602             parse = self.parse_expression
 603         else:
 604             parse = lambda: self.parse_expression(with_condexpr=False)
 605         args = []
 606         is_tuple = False
 607         while 1:
 608             if args:
 609                 self.stream.expect('comma')
 610             if self.is_tuple_end(extra_end_rules):
 611                 break
 612             args.append(parse())
 613             if self.stream.current.type == 'comma':
 614                 is_tuple = True
 615             else:
 616                 break
 617             lineno = self.stream.current.lineno
 618
 619         if not is_tuple:
 620             if args:
 621                 return args[0]
 622
 623             # if we don't have explicit parentheses, an empty tuple is
 624             # not a valid expression.  This would mean nothing (literally
 625             # nothing) in the spot of an expression would be an empty
 626             # tuple.
 627             if not explicit_parentheses:
 628                 self.fail('Expected an expression, got \'%s\'' %
 629                           describe_token(self.stream.current))
 630
 631         return nodes.Tuple(args, 'load', lineno=lineno)
 632
 633     def parse_list(self):
 634         token = self.stream.expect('lbracket')
 635         items = []
 636         while self.stream.current.type != 'rbracket':
 637             if items:
 638                 self.stream.expect('comma')
 639             if self.stream.current.type == 'rbracket':
 640                 break
 641             items.append(self.parse_expression())
 642         self.stream.expect('rbracket')
 643         return nodes.List(items, lineno=token.lineno)
 644
 645     def parse_dict(self):
 646         token = self.stream.expect('lbrace')
 647         items = []
 648         while self.stream.current.type != 'rbrace':
 649             if items:
 650                 self.stream.expect('comma')
 651             if self.stream.current.type == 'rbrace':
 652                 break
 653             key = self.parse_expression()
 654             self.stream.expect('colon')
 655             value = self.parse_expression()
 656             items.append(nodes.Pair(key, value, lineno=key.lineno))
 657         self.stream.expect('rbrace')
 658         return nodes.Dict(items, lineno=token.lineno)
 659
 660     def parse_postfix(self, node):
 661         while 1:
 662             token_type = self.stream.current.type
 663             if token_type == 'dot' or token_type == 'lbracket':
 664                 node = self.parse_subscript(node)
 665             # calls are valid both after postfix expressions (getattr
 666             # and getitem) as well as filters and tests
 667             elif token_type == 'lparen':
 668                 node = self.parse_call(node)
 669             else:
 670                 break
 671         return node
 672
 673     def parse_filter_expr(self, node):
 674         while 1:
 675             token_type = self.stream.current.type
 676             if token_type == 'pipe':
 677                 node = self.parse_filter(node)
 678             elif token_type == 'name' and self.stream.current.value == 'is':
 679                 node = self.parse_test(node)
 680             # calls are valid both after postfix expressions (getattr
 681             # and getitem) as well as filters and tests
 682             elif token_type == 'lparen':
 683                 node = self.parse_call(node)
 684             else:
 685                 break
 686         return node
 687
 688     def parse_subscript(self, node):
 689         token = next(self.stream)
 690         if token.type == 'dot':
 691             attr_token = self.stream.current
 692             next(self.stream)
 693             if attr_token.type == 'name':
 694                 return nodes.Getattr(node, attr_token.value, 'load',
 695                                      lineno=token.lineno)
 696             elif attr_token.type != 'integer':
 697                 self.fail('expected name or number', attr_token.lineno)
 698             arg = nodes.Const(attr_token.value, lineno=attr_token.lineno)
 699             return nodes.Getitem(node, arg, 'load', lineno=token.lineno)
 700         if token.type == 'lbracket':
 701             args = []
 702             while self.stream.current.type != 'rbracket':
 703                 if args:
 704                     self.stream.expect('comma')
 705                 args.append(self.parse_subscribed())
 706             self.stream.expect('rbracket')
 707             if len(args) == 1:
 708                 arg = args[0]
 709             else:
 710                 arg = nodes.Tuple(args, 'load', lineno=token.lineno)
 711             return nodes.Getitem(node, arg, 'load', lineno=token.lineno)
 712         self.fail('expected subscript expression', self.lineno)
 713
 714     def parse_subscribed(self):
 715         lineno = self.stream.current.lineno
 716
 717         if self.stream.current.type == 'colon':
 718             next(self.stream)
 719             args = [None]
 720         else:
 721             node = self.parse_expression()
 722             if self.stream.current.type != 'colon':
 723                 return node
 724             next(self.stream)
 725             args = [node]
 726
 727         if self.stream.current.type == 'colon':
 728             args.append(None)
 729         elif self.stream.current.type not in ('rbracket', 'comma'):
 730             args.append(self.parse_expression())
 731         else:
 732             args.append(None)
 733
 734         if self.stream.current.type == 'colon':
 735             next(self.stream)
 736             if self.stream.current.type not in ('rbracket', 'comma'):
 737                 args.append(self.parse_expression())
 738             else:
 739                 args.append(None)
 740         else:
 741             args.append(None)
 742
 743         return nodes.Slice(lineno=lineno, *args)
 744
 745     def parse_call(self, node):
 746         token = self.stream.expect('lparen')
 747         args = []
 748         kwargs = []
 749         dyn_args = dyn_kwargs = None
 750         require_comma = False
 751
 752         def ensure(expr):
 753             if not expr:
 754                 self.fail('invalid syntax for function call expression',
 755                           token.lineno)
 756
 757         while self.stream.current.type != 'rparen':
 758             if require_comma:
 759                 self.stream.expect('comma')
 760                 # support for trailing comma
 761                 if self.stream.current.type == 'rparen':
 762                     break
 763             if self.stream.current.type == 'mul':
 764                 ensure(dyn_args is None and dyn_kwargs is None)
 765                 next(self.stream)
 766                 dyn_args = self.parse_expression()
 767             elif self.stream.current.type == 'pow':
 768                 ensure(dyn_kwargs is None)
 769                 next(self.stream)
 770                 dyn_kwargs = self.parse_expression()
 771             else:
 772                 ensure(dyn_args is None and dyn_kwargs is None)
 773                 if self.stream.current.type == 'name' and \
 774                     self.stream.look().type == 'assign':
 775                     key = self.stream.current.value
 776                     self.stream.skip(2)
 777                     value = self.parse_expression()
 778                     kwargs.append(nodes.Keyword(key, value,
 779                                                 lineno=value.lineno))
 780                 else:
 781                     ensure(not kwargs)
 782                     args.append(self.parse_expression())
 783
 784             require_comma = True
 785         self.stream.expect('rparen')
 786
 787         if node is None:
 788             return args, kwargs, dyn_args, dyn_kwargs
 789         return nodes.Call(node, args, kwargs, dyn_args, dyn_kwargs,
 790                           lineno=token.lineno)
 791
 792     def parse_filter(self, node, start_inline=False):
 793         while self.stream.current.type == 'pipe' or start_inline:
 794             if not start_inline:
 795                 next(self.stream)
 796             token = self.stream.expect('name')
 797             name = token.value
 798             while self.stream.current.type == 'dot':
 799                 next(self.stream)
 800                 name += '.' + self.stream.expect('name').value
 801             if self.stream.current.type == 'lparen':
 802                 args, kwargs, dyn_args, dyn_kwargs = self.parse_call(None)
 803             else:
 804                 args = []
 805                 kwargs = []
 806                 dyn_args = dyn_kwargs = None
 807             node = nodes.Filter(node, name, args, kwargs, dyn_args,
 808                                 dyn_kwargs, lineno=token.lineno)
 809             start_inline = False
 810         return node
 811
 812     def parse_test(self, node):
 813         token = next(self.stream)
 814         if self.stream.current.test('name:not'):
 815             next(self.stream)
 816             negated = True
 817         else:
 818             negated = False
 819         name = self.stream.expect('name').value
 820         while self.stream.current.type == 'dot':
 821             next(self.stream)
 822             name += '.' + self.stream.expect('name').value
 823         dyn_args = dyn_kwargs = None
 824         kwargs = []
 825         if self.stream.current.type == 'lparen':
 826             args, kwargs, dyn_args, dyn_kwargs = self.parse_call(None)
 827         elif self.stream.current.type in ('name', 'string', 'integer',
 828                                           'float', 'lparen', 'lbracket',
 829                                           'lbrace') and not \
 830              self.stream.current.test_any('name:else', 'name:or',
 831                                           'name:and'):
 832             if self.stream.current.test('name:is'):
 833                 self.fail('You cannot chain multiple tests with is')
 834             args = [self.parse_expression()]
 835         else:
 836             args = []
 837         node = nodes.Test(node, name, args, kwargs, dyn_args,
 838                           dyn_kwargs, lineno=token.lineno)
 839         if negated:
 840             node = nodes.Not(node, lineno=token.lineno)
 841         return node
 842
 843     def subparse(self, end_tokens=None):
 844         body = []
 845         data_buffer = []
 846         add_data = data_buffer.append
 847
 848         if end_tokens is not None:
 849             self._end_token_stack.append(end_tokens)
 850
 851         def flush_data():
 852             if data_buffer:
 853                 lineno = data_buffer[0].lineno
 854                 body.append(nodes.Output(data_buffer[:], lineno=lineno))
 855                 del data_buffer[:]
 856
 857         try:
 858             while self.stream:
 859                 token = self.stream.current
 860                 if token.type == 'data':
 861                     if token.value:
 862                         add_data(nodes.TemplateData(token.value,
 863                                                     lineno=token.lineno))
 864                     next(self.stream)
 865                 elif token.type == 'variable_begin':
 866                     next(self.stream)
 867                     add_data(self.parse_tuple(with_condexpr=True))
 868                     self.stream.expect('variable_end')
 869                 elif token.type == 'block_begin':
 870                     flush_data()
 871                     next(self.stream)
 872                     if end_tokens is not None and \
 873                        self.stream.current.test_any(*end_tokens):
 874                         return body
 875                     rv = self.parse_statement()
 876                     if isinstance(rv, list):
 877                         body.extend(rv)
 878                     else:
 879                         body.append(rv)
 880                     self.stream.expect('block_end')
 881                 else:
 882                     raise AssertionError('internal parsing error')
 883
 884             flush_data()
 885         finally:
 886             if end_tokens is not None:
 887                 self._end_token_stack.pop()
 888
 889         return body
 890
 891     def parse(self):
 892         """Parse the whole template into a `Template` node."""
 893         result = nodes.Template(self.subparse(), lineno=1)
 894         result.set_environment(self.environment)
 895         return result