mojo/public/third_party/jinja2/ext.py

   1 # -*- coding: utf-8 -*-
   2 """
   3     jinja2.ext
   4     ~~~~~~~~~~
   5
   6     Jinja extensions allow to add custom tags similar to the way django custom
   7     tags work.  By default two example extensions exist: an i18n and a cache
   8     extension.
   9
  10     :copyright: (c) 2010 by the Jinja Team.
  11     :license: BSD.
  12 """
  13 from jinja2 import nodes
  14 from jinja2.defaults import BLOCK_START_STRING, \
  15      BLOCK_END_STRING, VARIABLE_START_STRING, VARIABLE_END_STRING, \
  16      COMMENT_START_STRING, COMMENT_END_STRING, LINE_STATEMENT_PREFIX, \
  17      LINE_COMMENT_PREFIX, TRIM_BLOCKS, NEWLINE_SEQUENCE, \
  18      KEEP_TRAILING_NEWLINE, LSTRIP_BLOCKS
  19 from jinja2.environment import Environment
  20 from jinja2.runtime import concat
  21 from jinja2.exceptions import TemplateAssertionError, TemplateSyntaxError
  22 from jinja2.utils import contextfunction, import_string, Markup
  23 from jinja2._compat import next, with_metaclass, string_types, iteritems
  24
  25
  26 # the only real useful gettext functions for a Jinja template.  Note
  27 # that ugettext must be assigned to gettext as Jinja doesn't support
  28 # non unicode strings.
  29 GETTEXT_FUNCTIONS = ('_', 'gettext', 'ngettext')
  30
  31
  32 class ExtensionRegistry(type):
  33     """Gives the extension an unique identifier."""
  34
  35     def __new__(cls, name, bases, d):
  36         rv = type.__new__(cls, name, bases, d)
  37         rv.identifier = rv.__module__ + '.' + rv.__name__
  38         return rv
  39
  40
  41 class Extension(with_metaclass(ExtensionRegistry, object)):
  42     """Extensions can be used to add extra functionality to the Jinja template
  43     system at the parser level.  Custom extensions are bound to an environment
  44     but may not store environment specific data on `self`.  The reason for
  45     this is that an extension can be bound to another environment (for
  46     overlays) by creating a copy and reassigning the `environment` attribute.
  47
  48     As extensions are created by the environment they cannot accept any
  49     arguments for configuration.  One may want to work around that by using
  50     a factory function, but that is not possible as extensions are identified
  51     by their import name.  The correct way to configure the extension is
  52     storing the configuration values on the environment.  Because this way the
  53     environment ends up acting as central configuration storage the
  54     attributes may clash which is why extensions have to ensure that the names
  55     they choose for configuration are not too generic.  ``prefix`` for example
  56     is a terrible name, ``fragment_cache_prefix`` on the other hand is a good
  57     name as includes the name of the extension (fragment cache).
  58     """
  59
  60     #: if this extension parses this is the list of tags it's listening to.
  61     tags = set()
  62
  63     #: the priority of that extension.  This is especially useful for
  64     #: extensions that preprocess values.  A lower value means higher
  65     #: priority.
  66     #:
  67     #: .. versionadded:: 2.4
  68     priority = 100
  69
  70     def __init__(self, environment):
  71         self.environment = environment
  72
  73     def bind(self, environment):
  74         """Create a copy of this extension bound to another environment."""
  75         rv = object.__new__(self.__class__)
  76         rv.__dict__.update(self.__dict__)
  77         rv.environment = environment
  78         return rv
  79
  80     def preprocess(self, source, name, filename=None):
  81         """This method is called before the actual lexing and can be used to
  82         preprocess the source.  The `filename` is optional.  The return value
  83         must be the preprocessed source.
  84         """
  85         return source
  86
  87     def filter_stream(self, stream):
  88         """It's passed a :class:`~jinja2.lexer.TokenStream` that can be used
  89         to filter tokens returned.  This method has to return an iterable of
  90         :class:`~jinja2.lexer.Token`\s, but it doesn't have to return a
  91         :class:`~jinja2.lexer.TokenStream`.
  92
  93         In the `ext` folder of the Jinja2 source distribution there is a file
  94         called `inlinegettext.py` which implements a filter that utilizes this
  95         method.
  96         """
  97         return stream
  98
  99     def parse(self, parser):
 100         """If any of the :attr:`tags` matched this method is called with the
 101         parser as first argument.  The token the parser stream is pointing at
 102         is the name token that matched.  This method has to return one or a
 103         list of multiple nodes.
 104         """
 105         raise NotImplementedError()
 106
 107     def attr(self, name, lineno=None):
 108         """Return an attribute node for the current extension.  This is useful
 109         to pass constants on extensions to generated template code.
 110
 111         ::
 112
 113             self.attr('_my_attribute', lineno=lineno)
 114         """
 115         return nodes.ExtensionAttribute(self.identifier, name, lineno=lineno)
 116
 117     def call_method(self, name, args=None, kwargs=None, dyn_args=None,
 118                     dyn_kwargs=None, lineno=None):
 119         """Call a method of the extension.  This is a shortcut for
 120         :meth:`attr` + :class:`jinja2.nodes.Call`.
 121         """
 122         if args is None:
 123             args = []
 124         if kwargs is None:
 125             kwargs = []
 126         return nodes.Call(self.attr(name, lineno=lineno), args, kwargs,
 127                           dyn_args, dyn_kwargs, lineno=lineno)
 128
 129
 130 @contextfunction
 131 def _gettext_alias(__context, *args, **kwargs):
 132     return __context.call(__context.resolve('gettext'), *args, **kwargs)
 133
 134
 135 def _make_new_gettext(func):
 136     @contextfunction
 137     def gettext(__context, __string, **variables):
 138         rv = __context.call(func, __string)
 139         if __context.eval_ctx.autoescape:
 140             rv = Markup(rv)
 141         return rv % variables
 142     return gettext
 143
 144
 145 def _make_new_ngettext(func):
 146     @contextfunction
 147     def ngettext(__context, __singular, __plural, __num, **variables):
 148         variables.setdefault('num', __num)
 149         rv = __context.call(func, __singular, __plural, __num)
 150         if __context.eval_ctx.autoescape:
 151             rv = Markup(rv)
 152         return rv % variables
 153     return ngettext
 154
 155
 156 class InternationalizationExtension(Extension):
 157     """This extension adds gettext support to Jinja2."""
 158     tags = set(['trans'])
 159
 160     # TODO: the i18n extension is currently reevaluating values in a few
 161     # situations.  Take this example:
 162     #   {% trans count=something() %}{{ count }} foo{% pluralize
 163     #     %}{{ count }} fooss{% endtrans %}
 164     # something is called twice here.  One time for the gettext value and
 165     # the other time for the n-parameter of the ngettext function.
 166
 167     def __init__(self, environment):
 168         Extension.__init__(self, environment)
 169         environment.globals['_'] = _gettext_alias
 170         environment.extend(
 171             install_gettext_translations=self._install,
 172             install_null_translations=self._install_null,
 173             install_gettext_callables=self._install_callables,
 174             uninstall_gettext_translations=self._uninstall,
 175             extract_translations=self._extract,
 176             newstyle_gettext=False
 177         )
 178
 179     def _install(self, translations, newstyle=None):
 180         gettext = getattr(translations, 'ugettext', None)
 181         if gettext is None:
 182             gettext = translations.gettext
 183         ngettext = getattr(translations, 'ungettext', None)
 184         if ngettext is None:
 185             ngettext = translations.ngettext
 186         self._install_callables(gettext, ngettext, newstyle)
 187
 188     def _install_null(self, newstyle=None):
 189         self._install_callables(
 190             lambda x: x,
 191             lambda s, p, n: (n != 1 and (p,) or (s,))[0],
 192             newstyle
 193         )
 194
 195     def _install_callables(self, gettext, ngettext, newstyle=None):
 196         if newstyle is not None:
 197             self.environment.newstyle_gettext = newstyle
 198         if self.environment.newstyle_gettext:
 199             gettext = _make_new_gettext(gettext)
 200             ngettext = _make_new_ngettext(ngettext)
 201         self.environment.globals.update(
 202             gettext=gettext,
 203             ngettext=ngettext
 204         )
 205
 206     def _uninstall(self, translations):
 207         for key in 'gettext', 'ngettext':
 208             self.environment.globals.pop(key, None)
 209
 210     def _extract(self, source, gettext_functions=GETTEXT_FUNCTIONS):
 211         if isinstance(source, string_types):
 212             source = self.environment.parse(source)
 213         return extract_from_ast(source, gettext_functions)
 214
 215     def parse(self, parser):
 216         """Parse a translatable tag."""
 217         lineno = next(parser.stream).lineno
 218         num_called_num = False
 219
 220         # find all the variables referenced.  Additionally a variable can be
 221         # defined in the body of the trans block too, but this is checked at
 222         # a later state.
 223         plural_expr = None
 224         plural_expr_assignment = None
 225         variables = {}
 226         while parser.stream.current.type != 'block_end':
 227             if variables:
 228                 parser.stream.expect('comma')
 229
 230             # skip colon for python compatibility
 231             if parser.stream.skip_if('colon'):
 232                 break
 233
 234             name = parser.stream.expect('name')
 235             if name.value in variables:
 236                 parser.fail('translatable variable %r defined twice.' %
 237                             name.value, name.lineno,
 238                             exc=TemplateAssertionError)
 239
 240             # expressions
 241             if parser.stream.current.type == 'assign':
 242                 next(parser.stream)
 243                 variables[name.value] = var = parser.parse_expression()
 244             else:
 245                 variables[name.value] = var = nodes.Name(name.value, 'load')
 246
 247             if plural_expr is None:
 248                 if isinstance(var, nodes.Call):
 249                     plural_expr = nodes.Name('_trans', 'load')
 250                     variables[name.value] = plural_expr
 251                     plural_expr_assignment = nodes.Assign(
 252                         nodes.Name('_trans', 'store'), var)
 253                 else:
 254                     plural_expr = var
 255                 num_called_num = name.value == 'num'
 256
 257         parser.stream.expect('block_end')
 258
 259         plural = plural_names = None
 260         have_plural = False
 261         referenced = set()
 262
 263         # now parse until endtrans or pluralize
 264         singular_names, singular = self._parse_block(parser, True)
 265         if singular_names:
 266             referenced.update(singular_names)
 267             if plural_expr is None:
 268                 plural_expr = nodes.Name(singular_names[0], 'load')
 269                 num_called_num = singular_names[0] == 'num'
 270
 271         # if we have a pluralize block, we parse that too
 272         if parser.stream.current.test('name:pluralize'):
 273             have_plural = True
 274             next(parser.stream)
 275             if parser.stream.current.type != 'block_end':
 276                 name = parser.stream.expect('name')
 277                 if name.value not in variables:
 278                     parser.fail('unknown variable %r for pluralization' %
 279                                 name.value, name.lineno,
 280                                 exc=TemplateAssertionError)
 281                 plural_expr = variables[name.value]
 282                 num_called_num = name.value == 'num'
 283             parser.stream.expect('block_end')
 284             plural_names, plural = self._parse_block(parser, False)
 285             next(parser.stream)
 286             referenced.update(plural_names)
 287         else:
 288             next(parser.stream)
 289
 290         # register free names as simple name expressions
 291         for var in referenced:
 292             if var not in variables:
 293                 variables[var] = nodes.Name(var, 'load')
 294
 295         if not have_plural:
 296             plural_expr = None
 297         elif plural_expr is None:
 298             parser.fail('pluralize without variables', lineno)
 299
 300         node = self._make_node(singular, plural, variables, plural_expr,
 301                                bool(referenced),
 302                                num_called_num and have_plural)
 303         node.set_lineno(lineno)
 304         if plural_expr_assignment is not None:
 305             return [plural_expr_assignment, node]
 306         else:
 307             return node
 308
 309     def _parse_block(self, parser, allow_pluralize):
 310         """Parse until the next block tag with a given name."""
 311         referenced = []
 312         buf = []
 313         while 1:
 314             if parser.stream.current.type == 'data':
 315                 buf.append(parser.stream.current.value.replace('%', '%%'))
 316                 next(parser.stream)
 317             elif parser.stream.current.type == 'variable_begin':
 318                 next(parser.stream)
 319                 name = parser.stream.expect('name').value
 320                 referenced.append(name)
 321                 buf.append('%%(%s)s' % name)
 322                 parser.stream.expect('variable_end')
 323             elif parser.stream.current.type == 'block_begin':
 324                 next(parser.stream)
 325                 if parser.stream.current.test('name:endtrans'):
 326                     break
 327                 elif parser.stream.current.test('name:pluralize'):
 328                     if allow_pluralize:
 329                         break
 330                     parser.fail('a translatable section can have only one '
 331                                 'pluralize section')
 332                 parser.fail('control structures in translatable sections are '
 333                             'not allowed')
 334             elif parser.stream.eos:
 335                 parser.fail('unclosed translation block')
 336             else:
 337                 assert False, 'internal parser error'
 338
 339         return referenced, concat(buf)
 340
 341     def _make_node(self, singular, plural, variables, plural_expr,
 342                    vars_referenced, num_called_num):
 343         """Generates a useful node from the data provided."""
 344         # no variables referenced?  no need to escape for old style
 345         # gettext invocations only if there are vars.
 346         if not vars_referenced and not self.environment.newstyle_gettext:
 347             singular = singular.replace('%%', '%')
 348             if plural:
 349                 plural = plural.replace('%%', '%')
 350
 351         # singular only:
 352         if plural_expr is None:
 353             gettext = nodes.Name('gettext', 'load')
 354             node = nodes.Call(gettext, [nodes.Const(singular)],
 355                               [], None, None)
 356
 357         # singular and plural
 358         else:
 359             ngettext = nodes.Name('ngettext', 'load')
 360             node = nodes.Call(ngettext, [
 361                 nodes.Const(singular),
 362                 nodes.Const(plural),
 363                 plural_expr
 364             ], [], None, None)
 365
 366         # in case newstyle gettext is used, the method is powerful
 367         # enough to handle the variable expansion and autoescape
 368         # handling itself
 369         if self.environment.newstyle_gettext:
 370             for key, value in iteritems(variables):
 371                 # the function adds that later anyways in case num was
 372                 # called num, so just skip it.
 373                 if num_called_num and key == 'num':
 374                     continue
 375                 node.kwargs.append(nodes.Keyword(key, value))
 376
 377         # otherwise do that here
 378         else:
 379             # mark the return value as safe if we are in an
 380             # environment with autoescaping turned on
 381             node = nodes.MarkSafeIfAutoescape(node)
 382             if variables:
 383                 node = nodes.Mod(node, nodes.Dict([
 384                     nodes.Pair(nodes.Const(key), value)
 385                     for key, value in variables.items()
 386                 ]))
 387         return nodes.Output([node])
 388
 389
 390 class ExprStmtExtension(Extension):
 391     """Adds a `do` tag to Jinja2 that works like the print statement just
 392     that it doesn't print the return value.
 393     """
 394     tags = set(['do'])
 395
 396     def parse(self, parser):
 397         node = nodes.ExprStmt(lineno=next(parser.stream).lineno)
 398         node.node = parser.parse_tuple()
 399         return node
 400
 401
 402 class LoopControlExtension(Extension):
 403     """Adds break and continue to the template engine."""
 404     tags = set(['break', 'continue'])
 405
 406     def parse(self, parser):
 407         token = next(parser.stream)
 408         if token.value == 'break':
 409             return nodes.Break(lineno=token.lineno)
 410         return nodes.Continue(lineno=token.lineno)
 411
 412
 413 class WithExtension(Extension):
 414     """Adds support for a django-like with block."""
 415     tags = set(['with'])
 416
 417     def parse(self, parser):
 418         node = nodes.Scope(lineno=next(parser.stream).lineno)
 419         assignments = []
 420         while parser.stream.current.type != 'block_end':
 421             lineno = parser.stream.current.lineno
 422             if assignments:
 423                 parser.stream.expect('comma')
 424             target = parser.parse_assign_target()
 425             parser.stream.expect('assign')
 426             expr = parser.parse_expression()
 427             assignments.append(nodes.Assign(target, expr, lineno=lineno))
 428         node.body = assignments + \
 429             list(parser.parse_statements(('name:endwith',),
 430                                          drop_needle=True))
 431         return node
 432
 433
 434 class AutoEscapeExtension(Extension):
 435     """Changes auto escape rules for a scope."""
 436     tags = set(['autoescape'])
 437
 438     def parse(self, parser):
 439         node = nodes.ScopedEvalContextModifier(lineno=next(parser.stream).lineno)
 440         node.options = [
 441             nodes.Keyword('autoescape', parser.parse_expression())
 442         ]
 443         node.body = parser.parse_statements(('name:endautoescape',),
 444                                             drop_needle=True)
 445         return nodes.Scope([node])
 446
 447
 448 def extract_from_ast(node, gettext_functions=GETTEXT_FUNCTIONS,
 449                      babel_style=True):
 450     """Extract localizable strings from the given template node.  Per
 451     default this function returns matches in babel style that means non string
 452     parameters as well as keyword arguments are returned as `None`.  This
 453     allows Babel to figure out what you really meant if you are using
 454     gettext functions that allow keyword arguments for placeholder expansion.
 455     If you don't want that behavior set the `babel_style` parameter to `False`
 456     which causes only strings to be returned and parameters are always stored
 457     in tuples.  As a consequence invalid gettext calls (calls without a single
 458     string parameter or string parameters after non-string parameters) are
 459     skipped.
 460
 461     This example explains the behavior:
 462
 463     >>> from jinja2 import Environment
 464     >>> env = Environment()
 465     >>> node = env.parse('{{ (_("foo"), _(), ngettext("foo", "bar", 42)) }}')
 466     >>> list(extract_from_ast(node))
 467     [(1, '_', 'foo'), (1, '_', ()), (1, 'ngettext', ('foo', 'bar', None))]
 468     >>> list(extract_from_ast(node, babel_style=False))
 469     [(1, '_', ('foo',)), (1, 'ngettext', ('foo', 'bar'))]
 470
 471     For every string found this function yields a ``(lineno, function,
 472     message)`` tuple, where:
 473
 474     * ``lineno`` is the number of the line on which the string was found,
 475     * ``function`` is the name of the ``gettext`` function used (if the
 476       string was extracted from embedded Python code), and
 477     *  ``message`` is the string itself (a ``unicode`` object, or a tuple
 478        of ``unicode`` objects for functions with multiple string arguments).
 479
 480     This extraction function operates on the AST and is because of that unable
 481     to extract any comments.  For comment support you have to use the babel
 482     extraction interface or extract comments yourself.
 483     """
 484     for node in node.find_all(nodes.Call):
 485         if not isinstance(node.node, nodes.Name) or \
 486            node.node.name not in gettext_functions:
 487             continue
 488
 489         strings = []
 490         for arg in node.args:
 491             if isinstance(arg, nodes.Const) and \
 492                isinstance(arg.value, string_types):
 493                 strings.append(arg.value)
 494             else:
 495                 strings.append(None)
 496
 497         for arg in node.kwargs:
 498             strings.append(None)
 499         if node.dyn_args is not None:
 500             strings.append(None)
 501         if node.dyn_kwargs is not None:
 502             strings.append(None)
 503
 504         if not babel_style:
 505             strings = tuple(x for x in strings if x is not None)
 506             if not strings:
 507                 continue
 508         else:
 509             if len(strings) == 1:
 510                 strings = strings[0]
 511             else:
 512                 strings = tuple(strings)
 513         yield node.lineno, node.node.name, strings
 514
 515
 516 class _CommentFinder(object):
 517     """Helper class to find comments in a token stream.  Can only
 518     find comments for gettext calls forwards.  Once the comment
 519     from line 4 is found, a comment for line 1 will not return a
 520     usable value.
 521     """
 522
 523     def __init__(self, tokens, comment_tags):
 524         self.tokens = tokens
 525         self.comment_tags = comment_tags
 526         self.offset = 0
 527         self.last_lineno = 0
 528
 529     def find_backwards(self, offset):
 530         try:
 531             for _, token_type, token_value in \
 532                     reversed(self.tokens[self.offset:offset]):
 533                 if token_type in ('comment', 'linecomment'):
 534                     try:
 535                         prefix, comment = token_value.split(None, 1)
 536                     except ValueError:
 537                         continue
 538                     if prefix in self.comment_tags:
 539                         return [comment.rstrip()]
 540             return []
 541         finally:
 542             self.offset = offset
 543
 544     def find_comments(self, lineno):
 545         if not self.comment_tags or self.last_lineno > lineno:
 546             return []
 547         for idx, (token_lineno, _, _) in enumerate(self.tokens[self.offset:]):
 548             if token_lineno > lineno:
 549                 return self.find_backwards(self.offset + idx)
 550         return self.find_backwards(len(self.tokens))
 551
 552
 553 def babel_extract(fileobj, keywords, comment_tags, options):
 554     """Babel extraction method for Jinja templates.
 555
 556     .. versionchanged:: 2.3
 557        Basic support for translation comments was added.  If `comment_tags`
 558        is now set to a list of keywords for extraction, the extractor will
 559        try to find the best preceeding comment that begins with one of the
 560        keywords.  For best results, make sure to not have more than one
 561        gettext call in one line of code and the matching comment in the
 562        same line or the line before.
 563
 564     .. versionchanged:: 2.5.1
 565        The `newstyle_gettext` flag can be set to `True` to enable newstyle
 566        gettext calls.
 567
 568     .. versionchanged:: 2.7
 569        A `silent` option can now be provided.  If set to `False` template
 570        syntax errors are propagated instead of being ignored.
 571
 572     :param fileobj: the file-like object the messages should be extracted from
 573     :param keywords: a list of keywords (i.e. function names) that should be
 574                      recognized as translation functions
 575     :param comment_tags: a list of translator tags to search for and include
 576                          in the results.
 577     :param options: a dictionary of additional options (optional)
 578     :return: an iterator over ``(lineno, funcname, message, comments)`` tuples.
 579              (comments will be empty currently)
 580     """
 581     extensions = set()
 582     for extension in options.get('extensions', '').split(','):
 583         extension = extension.strip()
 584         if not extension:
 585             continue
 586         extensions.add(import_string(extension))
 587     if InternationalizationExtension not in extensions:
 588         extensions.add(InternationalizationExtension)
 589
 590     def getbool(options, key, default=False):
 591         return options.get(key, str(default)).lower() in \
 592             ('1', 'on', 'yes', 'true')
 593
 594     silent = getbool(options, 'silent', True)
 595     environment = Environment(
 596         options.get('block_start_string', BLOCK_START_STRING),
 597         options.get('block_end_string', BLOCK_END_STRING),
 598         options.get('variable_start_string', VARIABLE_START_STRING),
 599         options.get('variable_end_string', VARIABLE_END_STRING),
 600         options.get('comment_start_string', COMMENT_START_STRING),
 601         options.get('comment_end_string', COMMENT_END_STRING),
 602         options.get('line_statement_prefix') or LINE_STATEMENT_PREFIX,
 603         options.get('line_comment_prefix') or LINE_COMMENT_PREFIX,
 604         getbool(options, 'trim_blocks', TRIM_BLOCKS),
 605         getbool(options, 'lstrip_blocks', LSTRIP_BLOCKS),
 606         NEWLINE_SEQUENCE,
 607         getbool(options, 'keep_trailing_newline', KEEP_TRAILING_NEWLINE),
 608         frozenset(extensions),
 609         cache_size=0,
 610         auto_reload=False
 611     )
 612
 613     if getbool(options, 'newstyle_gettext'):
 614         environment.newstyle_gettext = True
 615
 616     source = fileobj.read().decode(options.get('encoding', 'utf-8'))
 617     try:
 618         node = environment.parse(source)
 619         tokens = list(environment.lex(environment.preprocess(source)))
 620     except TemplateSyntaxError as e:
 621         if not silent:
 622             raise
 623         # skip templates with syntax errors
 624         return
 625
 626     finder = _CommentFinder(tokens, comment_tags)
 627     for lineno, func, message in extract_from_ast(node, keywords):
 628         yield lineno, func, message, finder.find_comments(lineno)
 629
 630
 631 #: nicer import names
 632 i18n = InternationalizationExtension
 633 do = ExprStmtExtension
 634 loopcontrols = LoopControlExtension
 635 with_ = WithExtension
 636 autoescape = AutoEscapeExtension