third_party/cython/src/Cython/Tempita/_tempita.py

   1 """
   2 A small templating language
   3
   4 This implements a small templating language.  This language implements
   5 if/elif/else, for/continue/break, expressions, and blocks of Python
   6 code.  The syntax is::
   7
   8   {{any expression (function calls etc)}}
   9   {{any expression | filter}}
  10   {{for x in y}}...{{endfor}}
  11   {{if x}}x{{elif y}}y{{else}}z{{endif}}
  12   {{py:x=1}}
  13   {{py:
  14   def foo(bar):
  15       return 'baz'
  16   }}
  17   {{default var = default_value}}
  18   {{# comment}}
  19
  20 You use this with the ``Template`` class or the ``sub`` shortcut.
  21 The ``Template`` class takes the template string and the name of
  22 the template (for errors) and a default namespace.  Then (like
  23 ``string.Template``) you can call the ``tmpl.substitute(**kw)``
  24 method to make a substitution (or ``tmpl.substitute(a_dict)``).
  25
  26 ``sub(content, **kw)`` substitutes the template immediately.  You
  27 can use ``__name='tmpl.html'`` to set the name of the template.
  28
  29 If there are syntax errors ``TemplateError`` will be raised.
  30 """
  31
  32 import re
  33 import sys
  34 import cgi
  35 try:
  36     from urllib import quote as url_quote
  37 except ImportError:  # Py3
  38     from urllib.parse import quote as url_quote
  39 import os
  40 import tokenize
  41 try:
  42     from io import StringIO
  43 except ImportError:
  44     from cStringIO import StringIO
  45 from Cython.Tempita._looper import looper
  46 from Cython.Tempita.compat3 import bytes, basestring_, next, is_unicode, coerce_text
  47
  48 __all__ = ['TemplateError', 'Template', 'sub', 'HTMLTemplate',
  49            'sub_html', 'html', 'bunch']
  50
  51 in_re = re.compile(r'\s+in\s+')
  52 var_re = re.compile(r'^[a-z_][a-z0-9_]*$', re.I)
  53
  54
  55 class TemplateError(Exception):
  56     """Exception raised while parsing a template
  57     """
  58
  59     def __init__(self, message, position, name=None):
  60         Exception.__init__(self, message)
  61         self.position = position
  62         self.name = name
  63
  64     def __str__(self):
  65         msg = ' '.join(self.args)
  66         if self.position:
  67             msg = '%s at line %s column %s' % (
  68                 msg, self.position[0], self.position[1])
  69         if self.name:
  70             msg += ' in %s' % self.name
  71         return msg
  72
  73
  74 class _TemplateContinue(Exception):
  75     pass
  76
  77
  78 class _TemplateBreak(Exception):
  79     pass
  80
  81
  82 def get_file_template(name, from_template):
  83     path = os.path.join(os.path.dirname(from_template.name), name)
  84     return from_template.__class__.from_filename(
  85         path, namespace=from_template.namespace,
  86         get_template=from_template.get_template)
  87
  88
  89 class Template(object):
  90
  91     default_namespace = {
  92         'start_braces': '{{',
  93         'end_braces': '}}',
  94         'looper': looper,
  95         }
  96
  97     default_encoding = 'utf8'
  98     default_inherit = None
  99
 100     def __init__(self, content, name=None, namespace=None, stacklevel=None,
 101                  get_template=None, default_inherit=None, line_offset=0,
 102                  delimeters=None):
 103         self.content = content
 104
 105         # set delimeters
 106         if delimeters is None:
 107             delimeters = (self.default_namespace['start_braces'],
 108                           self.default_namespace['end_braces'])
 109         else:
 110             #assert len(delimeters) == 2 and all([isinstance(delimeter, basestring)
 111             #                                     for delimeter in delimeters])
 112             self.default_namespace = self.__class__.default_namespace.copy()
 113             self.default_namespace['start_braces'] = delimeters[0]
 114             self.default_namespace['end_braces'] = delimeters[1]
 115         self.delimeters = delimeters
 116
 117         self._unicode = is_unicode(content)
 118         if name is None and stacklevel is not None:
 119             try:
 120                 caller = sys._getframe(stacklevel)
 121             except ValueError:
 122                 pass
 123             else:
 124                 globals = caller.f_globals
 125                 lineno = caller.f_lineno
 126                 if '__file__' in globals:
 127                     name = globals['__file__']
 128                     if name.endswith('.pyc') or name.endswith('.pyo'):
 129                         name = name[:-1]
 130                 elif '__name__' in globals:
 131                     name = globals['__name__']
 132                 else:
 133                     name = '<string>'
 134                 if lineno:
 135                     name += ':%s' % lineno
 136         self.name = name
 137         self._parsed = parse(content, name=name, line_offset=line_offset, delimeters=self.delimeters)
 138         if namespace is None:
 139             namespace = {}
 140         self.namespace = namespace
 141         self.get_template = get_template
 142         if default_inherit is not None:
 143             self.default_inherit = default_inherit
 144
 145     def from_filename(cls, filename, namespace=None, encoding=None,
 146                       default_inherit=None, get_template=get_file_template):
 147         f = open(filename, 'rb')
 148         c = f.read()
 149         f.close()
 150         if encoding:
 151             c = c.decode(encoding)
 152         return cls(content=c, name=filename, namespace=namespace,
 153                    default_inherit=default_inherit, get_template=get_template)
 154
 155     from_filename = classmethod(from_filename)
 156
 157     def __repr__(self):
 158         return '<%s %s name=%r>' % (
 159             self.__class__.__name__,
 160             hex(id(self))[2:], self.name)
 161
 162     def substitute(self, *args, **kw):
 163         if args:
 164             if kw:
 165                 raise TypeError(
 166                     "You can only give positional *or* keyword arguments")
 167             if len(args) > 1:
 168                 raise TypeError(
 169                     "You can only give one positional argument")
 170             if not hasattr(args[0], 'items'):
 171                 raise TypeError(
 172                     "If you pass in a single argument, you must pass in a dictionary-like object (with a .items() method); you gave %r"
 173                     % (args[0],))
 174             kw = args[0]
 175         ns = kw
 176         ns['__template_name__'] = self.name
 177         if self.namespace:
 178             ns.update(self.namespace)
 179         result, defs, inherit = self._interpret(ns)
 180         if not inherit:
 181             inherit = self.default_inherit
 182         if inherit:
 183             result = self._interpret_inherit(result, defs, inherit, ns)
 184         return result
 185
 186     def _interpret(self, ns):
 187         __traceback_hide__ = True
 188         parts = []
 189         defs = {}
 190         self._interpret_codes(self._parsed, ns, out=parts, defs=defs)
 191         if '__inherit__' in defs:
 192             inherit = defs.pop('__inherit__')
 193         else:
 194             inherit = None
 195         return ''.join(parts), defs, inherit
 196
 197     def _interpret_inherit(self, body, defs, inherit_template, ns):
 198         __traceback_hide__ = True
 199         if not self.get_template:
 200             raise TemplateError(
 201                 'You cannot use inheritance without passing in get_template',
 202                 position=None, name=self.name)
 203         templ = self.get_template(inherit_template, self)
 204         self_ = TemplateObject(self.name)
 205         for name, value in defs.iteritems():
 206             setattr(self_, name, value)
 207         self_.body = body
 208         ns = ns.copy()
 209         ns['self'] = self_
 210         return templ.substitute(ns)
 211
 212     def _interpret_codes(self, codes, ns, out, defs):
 213         __traceback_hide__ = True
 214         for item in codes:
 215             if isinstance(item, basestring_):
 216                 out.append(item)
 217             else:
 218                 self._interpret_code(item, ns, out, defs)
 219
 220     def _interpret_code(self, code, ns, out, defs):
 221         __traceback_hide__ = True
 222         name, pos = code[0], code[1]
 223         if name == 'py':
 224             self._exec(code[2], ns, pos)
 225         elif name == 'continue':
 226             raise _TemplateContinue()
 227         elif name == 'break':
 228             raise _TemplateBreak()
 229         elif name == 'for':
 230             vars, expr, content = code[2], code[3], code[4]
 231             expr = self._eval(expr, ns, pos)
 232             self._interpret_for(vars, expr, content, ns, out, defs)
 233         elif name == 'cond':
 234             parts = code[2:]
 235             self._interpret_if(parts, ns, out, defs)
 236         elif name == 'expr':
 237             parts = code[2].split('|')
 238             base = self._eval(parts[0], ns, pos)
 239             for part in parts[1:]:
 240                 func = self._eval(part, ns, pos)
 241                 base = func(base)
 242             out.append(self._repr(base, pos))
 243         elif name == 'default':
 244             var, expr = code[2], code[3]
 245             if var not in ns:
 246                 result = self._eval(expr, ns, pos)
 247                 ns[var] = result
 248         elif name == 'inherit':
 249             expr = code[2]
 250             value = self._eval(expr, ns, pos)
 251             defs['__inherit__'] = value
 252         elif name == 'def':
 253             name = code[2]
 254             signature = code[3]
 255             parts = code[4]
 256             ns[name] = defs[name] = TemplateDef(self, name, signature, body=parts, ns=ns,
 257                                                 pos=pos)
 258         elif name == 'comment':
 259             return
 260         else:
 261             assert 0, "Unknown code: %r" % name
 262
 263     def _interpret_for(self, vars, expr, content, ns, out, defs):
 264         __traceback_hide__ = True
 265         for item in expr:
 266             if len(vars) == 1:
 267                 ns[vars[0]] = item
 268             else:
 269                 if len(vars) != len(item):
 270                     raise ValueError(
 271                         'Need %i items to unpack (got %i items)'
 272                         % (len(vars), len(item)))
 273                 for name, value in zip(vars, item):
 274                     ns[name] = value
 275             try:
 276                 self._interpret_codes(content, ns, out, defs)
 277             except _TemplateContinue:
 278                 continue
 279             except _TemplateBreak:
 280                 break
 281
 282     def _interpret_if(self, parts, ns, out, defs):
 283         __traceback_hide__ = True
 284         # @@: if/else/else gets through
 285         for part in parts:
 286             assert not isinstance(part, basestring_)
 287             name, pos = part[0], part[1]
 288             if name == 'else':
 289                 result = True
 290             else:
 291                 result = self._eval(part[2], ns, pos)
 292             if result:
 293                 self._interpret_codes(part[3], ns, out, defs)
 294                 break
 295
 296     def _eval(self, code, ns, pos):
 297         __traceback_hide__ = True
 298         try:
 299             try:
 300                 value = eval(code, self.default_namespace, ns)
 301             except SyntaxError, e:
 302                 raise SyntaxError(
 303                     'invalid syntax in expression: %s' % code)
 304             return value
 305         except:
 306             exc_info = sys.exc_info()
 307             e = exc_info[1]
 308             if getattr(e, 'args', None):
 309                 arg0 = e.args[0]
 310             else:
 311                 arg0 = coerce_text(e)
 312             e.args = (self._add_line_info(arg0, pos),)
 313             raise exc_info[0], e, exc_info[2]
 314
 315     def _exec(self, code, ns, pos):
 316         __traceback_hide__ = True
 317         try:
 318             exec code in self.default_namespace, ns
 319         except:
 320             exc_info = sys.exc_info()
 321             e = exc_info[1]
 322             if e.args:
 323                 e.args = (self._add_line_info(e.args[0], pos),)
 324             else:
 325                 e.args = (self._add_line_info(None, pos),)
 326             raise exc_info[0], e, exc_info[2]
 327
 328     def _repr(self, value, pos):
 329         __traceback_hide__ = True
 330         try:
 331             if value is None:
 332                 return ''
 333             if self._unicode:
 334                 try:
 335                     value = unicode(value)
 336                 except UnicodeDecodeError:
 337                     value = bytes(value)
 338             else:
 339                 if not isinstance(value, basestring_):
 340                     value = coerce_text(value)
 341                 if (is_unicode(value)
 342                     and self.default_encoding):
 343                     value = value.encode(self.default_encoding)
 344         except:
 345             exc_info = sys.exc_info()
 346             e = exc_info[1]
 347             e.args = (self._add_line_info(e.args[0], pos),)
 348             raise exc_info[0], e, exc_info[2]
 349         else:
 350             if self._unicode and isinstance(value, bytes):
 351                 if not self.default_encoding:
 352                     raise UnicodeDecodeError(
 353                         'Cannot decode bytes value %r into unicode '
 354                         '(no default_encoding provided)' % value)
 355                 try:
 356                     value = value.decode(self.default_encoding)
 357                 except UnicodeDecodeError, e:
 358                     raise UnicodeDecodeError(
 359                         e.encoding,
 360                         e.object,
 361                         e.start,
 362                         e.end,
 363                         e.reason + ' in string %r' % value)
 364             elif not self._unicode and is_unicode(value):
 365                 if not self.default_encoding:
 366                     raise UnicodeEncodeError(
 367                         'Cannot encode unicode value %r into bytes '
 368                         '(no default_encoding provided)' % value)
 369                 value = value.encode(self.default_encoding)
 370             return value
 371
 372     def _add_line_info(self, msg, pos):
 373         msg = "%s at line %s column %s" % (
 374             msg, pos[0], pos[1])
 375         if self.name:
 376             msg += " in file %s" % self.name
 377         return msg
 378
 379
 380 def sub(content, delimeters=None, **kw):
 381     name = kw.get('__name')
 382     tmpl = Template(content, name=name, delimeters=delimeters)
 383     return tmpl.substitute(kw)
 384
 385
 386 def paste_script_template_renderer(content, vars, filename=None):
 387     tmpl = Template(content, name=filename)
 388     return tmpl.substitute(vars)
 389
 390
 391 class bunch(dict):
 392
 393     def __init__(self, **kw):
 394         for name, value in kw.iteritems():
 395             setattr(self, name, value)
 396
 397     def __setattr__(self, name, value):
 398         self[name] = value
 399
 400     def __getattr__(self, name):
 401         try:
 402             return self[name]
 403         except KeyError:
 404             raise AttributeError(name)
 405
 406     def __getitem__(self, key):
 407         if 'default' in self:
 408             try:
 409                 return dict.__getitem__(self, key)
 410             except KeyError:
 411                 return dict.__getitem__(self, 'default')
 412         else:
 413             return dict.__getitem__(self, key)
 414
 415     def __repr__(self):
 416         items = [
 417             (k, v) for k, v in self.iteritems()]
 418         items.sort()
 419         return '<%s %s>' % (
 420             self.__class__.__name__,
 421             ' '.join(['%s=%r' % (k, v) for k, v in items]))
 422
 423 ############################################################
 424 ## HTML Templating
 425 ############################################################
 426
 427
 428 class html(object):
 429
 430     def __init__(self, value):
 431         self.value = value
 432
 433     def __str__(self):
 434         return self.value
 435
 436     def __html__(self):
 437         return self.value
 438
 439     def __repr__(self):
 440         return '<%s %r>' % (
 441             self.__class__.__name__, self.value)
 442
 443
 444 def html_quote(value, force=True):
 445     if not force and hasattr(value, '__html__'):
 446         return value.__html__()
 447     if value is None:
 448         return ''
 449     if not isinstance(value, basestring_):
 450         value = coerce_text(value)
 451     if sys.version >= "3" and isinstance(value, bytes):
 452         value = cgi.escape(value.decode('latin1'), 1)
 453         value = value.encode('latin1')
 454     else:
 455         value = cgi.escape(value, 1)
 456     if sys.version < "3":
 457         if is_unicode(value):
 458             value = value.encode('ascii', 'xmlcharrefreplace')
 459     return value
 460
 461
 462 def url(v):
 463     v = coerce_text(v)
 464     if is_unicode(v):
 465         v = v.encode('utf8')
 466     return url_quote(v)
 467
 468
 469 def attr(**kw):
 470     kw = list(kw.iteritems())
 471     kw.sort()
 472     parts = []
 473     for name, value in kw:
 474         if value is None:
 475             continue
 476         if name.endswith('_'):
 477             name = name[:-1]
 478         parts.append('%s="%s"' % (html_quote(name), html_quote(value)))
 479     return html(' '.join(parts))
 480
 481
 482 class HTMLTemplate(Template):
 483
 484     default_namespace = Template.default_namespace.copy()
 485     default_namespace.update(dict(
 486         html=html,
 487         attr=attr,
 488         url=url,
 489         html_quote=html_quote,
 490         ))
 491
 492     def _repr(self, value, pos):
 493         if hasattr(value, '__html__'):
 494             value = value.__html__()
 495             quote = False
 496         else:
 497             quote = True
 498         plain = Template._repr(self, value, pos)
 499         if quote:
 500             return html_quote(plain)
 501         else:
 502             return plain
 503
 504
 505 def sub_html(content, **kw):
 506     name = kw.get('__name')
 507     tmpl = HTMLTemplate(content, name=name)
 508     return tmpl.substitute(kw)
 509
 510
 511 class TemplateDef(object):
 512     def __init__(self, template, func_name, func_signature,
 513                  body, ns, pos, bound_self=None):
 514         self._template = template
 515         self._func_name = func_name
 516         self._func_signature = func_signature
 517         self._body = body
 518         self._ns = ns
 519         self._pos = pos
 520         self._bound_self = bound_self
 521
 522     def __repr__(self):
 523         return '<tempita function %s(%s) at %s:%s>' % (
 524             self._func_name, self._func_signature,
 525             self._template.name, self._pos)
 526
 527     def __str__(self):
 528         return self()
 529
 530     def __call__(self, *args, **kw):
 531         values = self._parse_signature(args, kw)
 532         ns = self._ns.copy()
 533         ns.update(values)
 534         if self._bound_self is not None:
 535             ns['self'] = self._bound_self
 536         out = []
 537         subdefs = {}
 538         self._template._interpret_codes(self._body, ns, out, subdefs)
 539         return ''.join(out)
 540
 541     def __get__(self, obj, type=None):
 542         if obj is None:
 543             return self
 544         return self.__class__(
 545             self._template, self._func_name, self._func_signature,
 546             self._body, self._ns, self._pos, bound_self=obj)
 547
 548     def _parse_signature(self, args, kw):
 549         values = {}
 550         sig_args, var_args, var_kw, defaults = self._func_signature
 551         extra_kw = {}
 552         for name, value in kw.iteritems():
 553             if not var_kw and name not in sig_args:
 554                 raise TypeError(
 555                     'Unexpected argument %s' % name)
 556             if name in sig_args:
 557                 values[sig_args] = value
 558             else:
 559                 extra_kw[name] = value
 560         args = list(args)
 561         sig_args = list(sig_args)
 562         while args:
 563             while sig_args and sig_args[0] in values:
 564                 sig_args.pop(0)
 565             if sig_args:
 566                 name = sig_args.pop(0)
 567                 values[name] = args.pop(0)
 568             elif var_args:
 569                 values[var_args] = tuple(args)
 570                 break
 571             else:
 572                 raise TypeError(
 573                     'Extra position arguments: %s'
 574                     % ', '.join([repr(v) for v in args]))
 575         for name, value_expr in defaults.iteritems():
 576             if name not in values:
 577                 values[name] = self._template._eval(
 578                     value_expr, self._ns, self._pos)
 579         for name in sig_args:
 580             if name not in values:
 581                 raise TypeError(
 582                     'Missing argument: %s' % name)
 583         if var_kw:
 584             values[var_kw] = extra_kw
 585         return values
 586
 587
 588 class TemplateObject(object):
 589
 590     def __init__(self, name):
 591         self.__name = name
 592         self.get = TemplateObjectGetter(self)
 593
 594     def __repr__(self):
 595         return '<%s %s>' % (self.__class__.__name__, self.__name)
 596
 597
 598 class TemplateObjectGetter(object):
 599
 600     def __init__(self, template_obj):
 601         self.__template_obj = template_obj
 602
 603     def __getattr__(self, attr):
 604         return getattr(self.__template_obj, attr, Empty)
 605
 606     def __repr__(self):
 607         return '<%s around %r>' % (self.__class__.__name__, self.__template_obj)
 608
 609
 610 class _Empty(object):
 611     def __call__(self, *args, **kw):
 612         return self
 613
 614     def __str__(self):
 615         return ''
 616
 617     def __repr__(self):
 618         return 'Empty'
 619
 620     def __unicode__(self):
 621         return u''
 622
 623     def __iter__(self):
 624         return iter(())
 625
 626     def __bool__(self):
 627         return False
 628
 629     if sys.version < "3":
 630         __nonzero__ = __bool__
 631
 632 Empty = _Empty()
 633 del _Empty
 634
 635 ############################################################
 636 ## Lexing and Parsing
 637 ############################################################
 638
 639
 640 def lex(s, name=None, trim_whitespace=True, line_offset=0, delimeters=None):
 641     """
 642     Lex a string into chunks:
 643
 644         >>> lex('hey')
 645         ['hey']
 646         >>> lex('hey {{you}}')
 647         ['hey ', ('you', (1, 7))]
 648         >>> lex('hey {{')
 649         Traceback (most recent call last):
 650             ...
 651         TemplateError: No }} to finish last expression at line 1 column 7
 652         >>> lex('hey }}')
 653         Traceback (most recent call last):
 654             ...
 655         TemplateError: }} outside expression at line 1 column 7
 656         >>> lex('hey {{ {{')
 657         Traceback (most recent call last):
 658             ...
 659         TemplateError: {{ inside expression at line 1 column 10
 660
 661     """
 662     if delimeters is None:
 663         delimeters = ( Template.default_namespace['start_braces'],
 664                        Template.default_namespace['end_braces'] )
 665     in_expr = False
 666     chunks = []
 667     last = 0
 668     last_pos = (line_offset + 1, 1)
 669
 670     token_re = re.compile(r'%s|%s' % (re.escape(delimeters[0]),
 671                                       re.escape(delimeters[1])))
 672     for match in token_re.finditer(s):
 673         expr = match.group(0)
 674         pos = find_position(s, match.end(), last, last_pos)
 675         if expr == delimeters[0] and in_expr:
 676             raise TemplateError('%s inside expression' % delimeters[0],
 677                                 position=pos,
 678                                 name=name)
 679         elif expr == delimeters[1] and not in_expr:
 680             raise TemplateError('%s outside expression' % delimeters[1],
 681                                 position=pos,
 682                                 name=name)
 683         if expr == delimeters[0]:
 684             part = s[last:match.start()]
 685             if part:
 686                 chunks.append(part)
 687             in_expr = True
 688         else:
 689             chunks.append((s[last:match.start()], last_pos))
 690             in_expr = False
 691         last = match.end()
 692         last_pos = pos
 693     if in_expr:
 694         raise TemplateError('No %s to finish last expression' % delimeters[1],
 695                             name=name, position=last_pos)
 696     part = s[last:]
 697     if part:
 698         chunks.append(part)
 699     if trim_whitespace:
 700         chunks = trim_lex(chunks)
 701     return chunks
 702
 703 statement_re = re.compile(r'^(?:if |elif |for |def |inherit |default |py:)')
 704 single_statements = ['else', 'endif', 'endfor', 'enddef', 'continue', 'break']
 705 trail_whitespace_re = re.compile(r'\n\r?[\t ]*$')
 706 lead_whitespace_re = re.compile(r'^[\t ]*\n')
 707
 708
 709 def trim_lex(tokens):
 710     r"""
 711     Takes a lexed set of tokens, and removes whitespace when there is
 712     a directive on a line by itself:
 713
 714        >>> tokens = lex('{{if x}}\nx\n{{endif}}\ny', trim_whitespace=False)
 715        >>> tokens
 716        [('if x', (1, 3)), '\nx\n', ('endif', (3, 3)), '\ny']
 717        >>> trim_lex(tokens)
 718        [('if x', (1, 3)), 'x\n', ('endif', (3, 3)), 'y']
 719     """
 720     last_trim = None
 721     for i, current in enumerate(tokens):
 722         if isinstance(current, basestring_):
 723             # we don't trim this
 724             continue
 725         item = current[0]
 726         if not statement_re.search(item) and item not in single_statements:
 727             continue
 728         if not i:
 729             prev = ''
 730         else:
 731             prev = tokens[i - 1]
 732         if i + 1 >= len(tokens):
 733             next_chunk = ''
 734         else:
 735             next_chunk = tokens[i + 1]
 736         if (not isinstance(next_chunk, basestring_)
 737             or not isinstance(prev, basestring_)):
 738             continue
 739         prev_ok = not prev or trail_whitespace_re.search(prev)
 740         if i == 1 and not prev.strip():
 741             prev_ok = True
 742         if last_trim is not None and last_trim + 2 == i and not prev.strip():
 743             prev_ok = 'last'
 744         if (prev_ok
 745             and (not next_chunk or lead_whitespace_re.search(next_chunk)
 746                  or (i == len(tokens) - 2 and not next_chunk.strip()))):
 747             if prev:
 748                 if ((i == 1 and not prev.strip())
 749                     or prev_ok == 'last'):
 750                     tokens[i - 1] = ''
 751                 else:
 752                     m = trail_whitespace_re.search(prev)
 753                     # +1 to leave the leading \n on:
 754                     prev = prev[:m.start() + 1]
 755                     tokens[i - 1] = prev
 756             if next_chunk:
 757                 last_trim = i
 758                 if i == len(tokens) - 2 and not next_chunk.strip():
 759                     tokens[i + 1] = ''
 760                 else:
 761                     m = lead_whitespace_re.search(next_chunk)
 762                     next_chunk = next_chunk[m.end():]
 763                     tokens[i + 1] = next_chunk
 764     return tokens
 765
 766
 767 def find_position(string, index, last_index, last_pos):
 768     """Given a string and index, return (line, column)"""
 769     lines = string.count('\n', last_index, index)
 770     if lines > 0:
 771         column = index - string.rfind('\n', last_index, index)
 772     else:
 773         column = last_pos[1] + (index - last_index)
 774     return (last_pos[0] + lines, column)
 775
 776
 777 def parse(s, name=None, line_offset=0, delimeters=None):
 778     r"""
 779     Parses a string into a kind of AST
 780
 781         >>> parse('{{x}}')
 782         [('expr', (1, 3), 'x')]
 783         >>> parse('foo')
 784         ['foo']
 785         >>> parse('{{if x}}test{{endif}}')
 786         [('cond', (1, 3), ('if', (1, 3), 'x', ['test']))]
 787         >>> parse('series->{{for x in y}}x={{x}}{{endfor}}')
 788         ['series->', ('for', (1, 11), ('x',), 'y', ['x=', ('expr', (1, 27), 'x')])]
 789         >>> parse('{{for x, y in z:}}{{continue}}{{endfor}}')
 790         [('for', (1, 3), ('x', 'y'), 'z', [('continue', (1, 21))])]
 791         >>> parse('{{py:x=1}}')
 792         [('py', (1, 3), 'x=1')]
 793         >>> parse('{{if x}}a{{elif y}}b{{else}}c{{endif}}')
 794         [('cond', (1, 3), ('if', (1, 3), 'x', ['a']), ('elif', (1, 12), 'y', ['b']), ('else', (1, 23), None, ['c']))]
 795
 796     Some exceptions::
 797
 798         >>> parse('{{continue}}')
 799         Traceback (most recent call last):
 800             ...
 801         TemplateError: continue outside of for loop at line 1 column 3
 802         >>> parse('{{if x}}foo')
 803         Traceback (most recent call last):
 804             ...
 805         TemplateError: No {{endif}} at line 1 column 3
 806         >>> parse('{{else}}')
 807         Traceback (most recent call last):
 808             ...
 809         TemplateError: else outside of an if block at line 1 column 3
 810         >>> parse('{{if x}}{{for x in y}}{{endif}}{{endfor}}')
 811         Traceback (most recent call last):
 812             ...
 813         TemplateError: Unexpected endif at line 1 column 25
 814         >>> parse('{{if}}{{endif}}')
 815         Traceback (most recent call last):
 816             ...
 817         TemplateError: if with no expression at line 1 column 3
 818         >>> parse('{{for x y}}{{endfor}}')
 819         Traceback (most recent call last):
 820             ...
 821         TemplateError: Bad for (no "in") in 'x y' at line 1 column 3
 822         >>> parse('{{py:x=1\ny=2}}')
 823         Traceback (most recent call last):
 824             ...
 825         TemplateError: Multi-line py blocks must start with a newline at line 1 column 3
 826     """
 827     if delimeters is None:
 828         delimeters = ( Template.default_namespace['start_braces'],
 829                        Template.default_namespace['end_braces'] )
 830     tokens = lex(s, name=name, line_offset=line_offset, delimeters=delimeters)
 831     result = []
 832     while tokens:
 833         next_chunk, tokens = parse_expr(tokens, name)
 834         result.append(next_chunk)
 835     return result
 836
 837
 838 def parse_expr(tokens, name, context=()):
 839     if isinstance(tokens[0], basestring_):
 840         return tokens[0], tokens[1:]
 841     expr, pos = tokens[0]
 842     expr = expr.strip()
 843     if expr.startswith('py:'):
 844         expr = expr[3:].lstrip(' \t')
 845         if expr.startswith('\n') or expr.startswith('\r'):
 846             expr = expr.lstrip('\r\n')
 847             if '\r' in expr:
 848                 expr = expr.replace('\r\n', '\n')
 849                 expr = expr.replace('\r', '')
 850             expr += '\n'
 851         else:
 852             if '\n' in expr:
 853                 raise TemplateError(
 854                     'Multi-line py blocks must start with a newline',
 855                     position=pos, name=name)
 856         return ('py', pos, expr), tokens[1:]
 857     elif expr in ('continue', 'break'):
 858         if 'for' not in context:
 859             raise TemplateError(
 860                 'continue outside of for loop',
 861                 position=pos, name=name)
 862         return (expr, pos), tokens[1:]
 863     elif expr.startswith('if '):
 864         return parse_cond(tokens, name, context)
 865     elif (expr.startswith('elif ')
 866           or expr == 'else'):
 867         raise TemplateError(
 868             '%s outside of an if block' % expr.split()[0],
 869             position=pos, name=name)
 870     elif expr in ('if', 'elif', 'for'):
 871         raise TemplateError(
 872             '%s with no expression' % expr,
 873             position=pos, name=name)
 874     elif expr in ('endif', 'endfor', 'enddef'):
 875         raise TemplateError(
 876             'Unexpected %s' % expr,
 877             position=pos, name=name)
 878     elif expr.startswith('for '):
 879         return parse_for(tokens, name, context)
 880     elif expr.startswith('default '):
 881         return parse_default(tokens, name, context)
 882     elif expr.startswith('inherit '):
 883         return parse_inherit(tokens, name, context)
 884     elif expr.startswith('def '):
 885         return parse_def(tokens, name, context)
 886     elif expr.startswith('#'):
 887         return ('comment', pos, tokens[0][0]), tokens[1:]
 888     return ('expr', pos, tokens[0][0]), tokens[1:]
 889
 890
 891 def parse_cond(tokens, name, context):
 892     start = tokens[0][1]
 893     pieces = []
 894     context = context + ('if',)
 895     while 1:
 896         if not tokens:
 897             raise TemplateError(
 898                 'Missing {{endif}}',
 899                 position=start, name=name)
 900         if (isinstance(tokens[0], tuple)
 901             and tokens[0][0] == 'endif'):
 902             return ('cond', start) + tuple(pieces), tokens[1:]
 903         next_chunk, tokens = parse_one_cond(tokens, name, context)
 904         pieces.append(next_chunk)
 905
 906
 907 def parse_one_cond(tokens, name, context):
 908     (first, pos), tokens = tokens[0], tokens[1:]
 909     content = []
 910     if first.endswith(':'):
 911         first = first[:-1]
 912     if first.startswith('if '):
 913         part = ('if', pos, first[3:].lstrip(), content)
 914     elif first.startswith('elif '):
 915         part = ('elif', pos, first[5:].lstrip(), content)
 916     elif first == 'else':
 917         part = ('else', pos, None, content)
 918     else:
 919         assert 0, "Unexpected token %r at %s" % (first, pos)
 920     while 1:
 921         if not tokens:
 922             raise TemplateError(
 923                 'No {{endif}}',
 924                 position=pos, name=name)
 925         if (isinstance(tokens[0], tuple)
 926             and (tokens[0][0] == 'endif'
 927                  or tokens[0][0].startswith('elif ')
 928                  or tokens[0][0] == 'else')):
 929             return part, tokens
 930         next_chunk, tokens = parse_expr(tokens, name, context)
 931         content.append(next_chunk)
 932
 933
 934 def parse_for(tokens, name, context):
 935     first, pos = tokens[0]
 936     tokens = tokens[1:]
 937     context = ('for',) + context
 938     content = []
 939     assert first.startswith('for ')
 940     if first.endswith(':'):
 941         first = first[:-1]
 942     first = first[3:].strip()
 943     match = in_re.search(first)
 944     if not match:
 945         raise TemplateError(
 946             'Bad for (no "in") in %r' % first,
 947             position=pos, name=name)
 948     vars = first[:match.start()]
 949     if '(' in vars:
 950         raise TemplateError(
 951             'You cannot have () in the variable section of a for loop (%r)'
 952             % vars, position=pos, name=name)
 953     vars = tuple([
 954         v.strip() for v in first[:match.start()].split(',')
 955         if v.strip()])
 956     expr = first[match.end():]
 957     while 1:
 958         if not tokens:
 959             raise TemplateError(
 960                 'No {{endfor}}',
 961                 position=pos, name=name)
 962         if (isinstance(tokens[0], tuple)
 963             and tokens[0][0] == 'endfor'):
 964             return ('for', pos, vars, expr, content), tokens[1:]
 965         next_chunk, tokens = parse_expr(tokens, name, context)
 966         content.append(next_chunk)
 967
 968
 969 def parse_default(tokens, name, context):
 970     first, pos = tokens[0]
 971     assert first.startswith('default ')
 972     first = first.split(None, 1)[1]
 973     parts = first.split('=', 1)
 974     if len(parts) == 1:
 975         raise TemplateError(
 976             "Expression must be {{default var=value}}; no = found in %r" % first,
 977             position=pos, name=name)
 978     var = parts[0].strip()
 979     if ',' in var:
 980         raise TemplateError(
 981             "{{default x, y = ...}} is not supported",
 982             position=pos, name=name)
 983     if not var_re.search(var):
 984         raise TemplateError(
 985             "Not a valid variable name for {{default}}: %r"
 986             % var, position=pos, name=name)
 987     expr = parts[1].strip()
 988     return ('default', pos, var, expr), tokens[1:]
 989
 990
 991 def parse_inherit(tokens, name, context):
 992     first, pos = tokens[0]
 993     assert first.startswith('inherit ')
 994     expr = first.split(None, 1)[1]
 995     return ('inherit', pos, expr), tokens[1:]
 996
 997
 998 def parse_def(tokens, name, context):
 999     first, start = tokens[0]
1000     tokens = tokens[1:]
1001     assert first.startswith('def ')
1002     first = first.split(None, 1)[1]
1003     if first.endswith(':'):
1004         first = first[:-1]
1005     if '(' not in first:
1006         func_name = first
1007         sig = ((), None, None, {})
1008     elif not first.endswith(')'):
1009         raise TemplateError("Function definition doesn't end with ): %s" % first,
1010                             position=start, name=name)
1011     else:
1012         first = first[:-1]
1013         func_name, sig_text = first.split('(', 1)
1014         sig = parse_signature(sig_text, name, start)
1015     context = context + ('def',)
1016     content = []
1017     while 1:
1018         if not tokens:
1019             raise TemplateError(
1020                 'Missing {{enddef}}',
1021                 position=start, name=name)
1022         if (isinstance(tokens[0], tuple)
1023             and tokens[0][0] == 'enddef'):
1024             return ('def', start, func_name, sig, content), tokens[1:]
1025         next_chunk, tokens = parse_expr(tokens, name, context)
1026         content.append(next_chunk)
1027
1028
1029 def parse_signature(sig_text, name, pos):
1030     tokens = tokenize.generate_tokens(StringIO(sig_text).readline)
1031     sig_args = []
1032     var_arg = None
1033     var_kw = None
1034     defaults = {}
1035
1036     def get_token(pos=False):
1037         try:
1038             tok_type, tok_string, (srow, scol), (erow, ecol), line = next(tokens)
1039         except StopIteration:
1040             return tokenize.ENDMARKER, ''
1041         if pos:
1042             return tok_type, tok_string, (srow, scol), (erow, ecol)
1043         else:
1044             return tok_type, tok_string
1045     while 1:
1046         var_arg_type = None
1047         tok_type, tok_string = get_token()
1048         if tok_type == tokenize.ENDMARKER:
1049             break
1050         if tok_type == tokenize.OP and (tok_string == '*' or tok_string == '**'):
1051             var_arg_type = tok_string
1052             tok_type, tok_string = get_token()
1053         if tok_type != tokenize.NAME:
1054             raise TemplateError('Invalid signature: (%s)' % sig_text,
1055                                 position=pos, name=name)
1056         var_name = tok_string
1057         tok_type, tok_string = get_token()
1058         if tok_type == tokenize.ENDMARKER or (tok_type == tokenize.OP and tok_string == ','):
1059             if var_arg_type == '*':
1060                 var_arg = var_name
1061             elif var_arg_type == '**':
1062                 var_kw = var_name
1063             else:
1064                 sig_args.append(var_name)
1065             if tok_type == tokenize.ENDMARKER:
1066                 break
1067             continue
1068         if var_arg_type is not None:
1069             raise TemplateError('Invalid signature: (%s)' % sig_text,
1070                                 position=pos, name=name)
1071         if tok_type == tokenize.OP and tok_string == '=':
1072             nest_type = None
1073             unnest_type = None
1074             nest_count = 0
1075             start_pos = end_pos = None
1076             parts = []
1077             while 1:
1078                 tok_type, tok_string, s, e = get_token(True)
1079                 if start_pos is None:
1080                     start_pos = s
1081                 end_pos = e
1082                 if tok_type == tokenize.ENDMARKER and nest_count:
1083                     raise TemplateError('Invalid signature: (%s)' % sig_text,
1084                                         position=pos, name=name)
1085                 if (not nest_count and
1086                     (tok_type == tokenize.ENDMARKER or (tok_type == tokenize.OP and tok_string == ','))):
1087                     default_expr = isolate_expression(sig_text, start_pos, end_pos)
1088                     defaults[var_name] = default_expr
1089                     sig_args.append(var_name)
1090                     break
1091                 parts.append((tok_type, tok_string))
1092                 if nest_count and tok_type == tokenize.OP and tok_string == nest_type:
1093                     nest_count += 1
1094                 elif nest_count and tok_type == tokenize.OP and tok_string == unnest_type:
1095                     nest_count -= 1
1096                     if not nest_count:
1097                         nest_type = unnest_type = None
1098                 elif not nest_count and tok_type == tokenize.OP and tok_string in ('(', '[', '{'):
1099                     nest_type = tok_string
1100                     nest_count = 1
1101                     unnest_type = {'(': ')', '[': ']', '{': '}'}[nest_type]
1102     return sig_args, var_arg, var_kw, defaults
1103
1104
1105 def isolate_expression(string, start_pos, end_pos):
1106     srow, scol = start_pos
1107     srow -= 1
1108     erow, ecol = end_pos
1109     erow -= 1
1110     lines = string.splitlines(True)
1111     if srow == erow:
1112         return lines[srow][scol:ecol]
1113     parts = [lines[srow][scol:]]
1114     parts.extend(lines[srow+1:erow])
1115     if erow < len(lines):
1116         # It'll sometimes give (end_row_past_finish, 0)
1117         parts.append(lines[erow][:ecol])
1118     return ''.join(parts)
1119
1120 _fill_command_usage = """\
1121 %prog [OPTIONS] TEMPLATE arg=value
1122
1123 Use py:arg=value to set a Python value; otherwise all values are
1124 strings.
1125 """
1126
1127
1128 def fill_command(args=None):
1129     import sys
1130     import optparse
1131     import pkg_resources
1132     import os
1133     if args is None:
1134         args = sys.argv[1:]
1135     dist = pkg_resources.get_distribution('Paste')
1136     parser = optparse.OptionParser(
1137         version=coerce_text(dist),
1138         usage=_fill_command_usage)
1139     parser.add_option(
1140         '-o', '--output',
1141         dest='output',
1142         metavar="FILENAME",
1143         help="File to write output to (default stdout)")
1144     parser.add_option(
1145         '--html',
1146         dest='use_html',
1147         action='store_true',
1148         help="Use HTML style filling (including automatic HTML quoting)")
1149     parser.add_option(
1150         '--env',
1151         dest='use_env',
1152         action='store_true',
1153         help="Put the environment in as top-level variables")
1154     options, args = parser.parse_args(args)
1155     if len(args) < 1:
1156         print('You must give a template filename')
1157         sys.exit(2)
1158     template_name = args[0]
1159     args = args[1:]
1160     vars = {}
1161     if options.use_env:
1162         vars.update(os.environ)
1163     for value in args:
1164         if '=' not in value:
1165             print('Bad argument: %r' % value)
1166             sys.exit(2)
1167         name, value = value.split('=', 1)
1168         if name.startswith('py:'):
1169             name = name[:3]
1170             value = eval(value)
1171         vars[name] = value
1172     if template_name == '-':
1173         template_content = sys.stdin.read()
1174         template_name = '<stdin>'
1175     else:
1176         f = open(template_name, 'rb')
1177         template_content = f.read()
1178         f.close()
1179     if options.use_html:
1180         TemplateClass = HTMLTemplate
1181     else:
1182         TemplateClass = Template
1183     template = TemplateClass(template_content, name=template_name)
1184     result = template.substitute(vars)
1185     if options.output:
1186         f = open(options.output, 'wb')
1187         f.write(result)
1188         f.close()
1189     else:
1190         sys.stdout.write(result)
1191
1192 if __name__ == '__main__':
1193     fill_command()