third_party/motemplate/motemplate.py

   1 # Copyright 2012 Benjamin Kalman
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #     http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 # TODO: Escaping control characters somehow. e.g. \{{, \{{-.
  16
  17 import json
  18 import re
  19
  20 '''Motemplate templates are data binding templates more-than-loosely inspired by
  21 ctemplate. Use like:
  22
  23   from motemplate import Motemplate
  24
  25   template = Motemplate('hello {{#foo bar/}} world')
  26   input = {
  27     'foo': [
  28       { 'bar': 1 },
  29       { 'bar': 2 },
  30       { 'bar': 3 }
  31     ]
  32   }
  33   print(template.render(input).text)
  34
  35 Motemplate will use get() on contexts to return values, so to create custom
  36 getters (for example, something that populates values lazily from keys), just
  37 provide an object with a get() method.
  38
  39   class CustomContext(object):
  40     def get(self, key):
  41       return 10
  42   print(Motemplate('hello {{world}}').render(CustomContext()).text)
  43
  44 will print 'hello 10'.
  45 '''
  46
  47 class ParseException(Exception):
  48   '''The exception thrown while parsing a template.
  49   '''
  50   def __init__(self, error):
  51     Exception.__init__(self, error)
  52
  53 class RenderResult(object):
  54   '''The result of a render operation.
  55   '''
  56   def __init__(self, text, errors):
  57     self.text = text;
  58     self.errors = errors
  59
  60   def __repr__(self):
  61     return '%s(text=%s, errors=%s)' % (type(self).__name__,
  62                                        self.text,
  63                                        self.errors)
  64
  65   def __str__(self):
  66     return repr(self)
  67
  68 class _StringBuilder(object):
  69   '''Efficiently builds strings.
  70   '''
  71   def __init__(self):
  72     self._buf = []
  73
  74   def __len__(self):
  75     self._Collapse()
  76     return len(self._buf[0])
  77
  78   def Append(self, string):
  79     if not isinstance(string, basestring):
  80       string = str(string)
  81     self._buf.append(string)
  82
  83   def ToString(self):
  84     self._Collapse()
  85     return self._buf[0]
  86
  87   def _Collapse(self):
  88     self._buf = [u''.join(self._buf)]
  89
  90   def __repr__(self):
  91     return self.ToString()
  92
  93   def __str__(self):
  94     return repr(self)
  95
  96 class _Contexts(object):
  97   '''Tracks a stack of context objects, providing efficient key/value retrieval.
  98   '''
  99   class _Node(object):
 100     '''A node within the stack. Wraps a real context and maintains the key/value
 101     pairs seen so far.
 102     '''
 103     def __init__(self, value):
 104       self._value = value
 105       self._value_has_get = hasattr(value, 'get')
 106       self._found = {}
 107
 108     def GetKeys(self):
 109       '''Returns the list of keys that |_value| contains.
 110       '''
 111       return self._found.keys()
 112
 113     def Get(self, key):
 114       '''Returns the value for |key|, or None if not found (including if
 115       |_value| doesn't support key retrieval).
 116       '''
 117       if not self._value_has_get:
 118         return None
 119       value = self._found.get(key)
 120       if value is not None:
 121         return value
 122       value = self._value.get(key)
 123       if value is not None:
 124         self._found[key] = value
 125       return value
 126
 127     def __repr__(self):
 128       return 'Node(value=%s, found=%s)' % (self._value, self._found)
 129
 130     def __str__(self):
 131       return repr(self)
 132
 133   def __init__(self, globals_):
 134     '''Initializes with the initial global contexts, listed in order from most
 135     to least important.
 136     '''
 137     self._nodes = map(_Contexts._Node, globals_)
 138     self._first_local = len(self._nodes)
 139     self._value_info = {}
 140
 141   def CreateFromGlobals(self):
 142     new = _Contexts([])
 143     new._nodes = self._nodes[:self._first_local]
 144     new._first_local = self._first_local
 145     return new
 146
 147   def Push(self, context):
 148     self._nodes.append(_Contexts._Node(context))
 149
 150   def Pop(self):
 151     node = self._nodes.pop()
 152     assert len(self._nodes) >= self._first_local
 153     for found_key in node.GetKeys():
 154       # [0] is the stack of nodes that |found_key| has been found in.
 155       self._value_info[found_key][0].pop()
 156
 157   def FirstLocal(self):
 158     if len(self._nodes) == self._first_local:
 159       return None
 160     return self._nodes[-1]._value
 161
 162   def Resolve(self, path):
 163     # This method is only efficient at finding |key|; if |tail| has a value (and
 164     # |key| evaluates to an indexable value) we'll need to descend into that.
 165     key, tail = path.split('.', 1) if '.' in path else (path, None)
 166     found = self._FindNodeValue(key)
 167     if tail is None:
 168       return found
 169     for part in tail.split('.'):
 170       if not hasattr(found, 'get'):
 171         return None
 172       found = found.get(part)
 173     return found
 174
 175   def Scope(self, context, fn, *args):
 176     self.Push(context)
 177     try:
 178       return fn(*args)
 179     finally:
 180       self.Pop()
 181
 182   def _FindNodeValue(self, key):
 183     # |found_node_list| will be all the nodes that |key| has been found in.
 184     # |checked_node_set| are those that have been checked.
 185     info = self._value_info.get(key)
 186     if info is None:
 187       info = ([], set())
 188       self._value_info[key] = info
 189     found_node_list, checked_node_set = info
 190
 191     # Check all the nodes not yet checked for |key|.
 192     newly_found = []
 193     for node in reversed(self._nodes):
 194       if node in checked_node_set:
 195         break
 196       value = node.Get(key)
 197       if value is not None:
 198         newly_found.append(node)
 199       checked_node_set.add(node)
 200
 201     # The nodes will have been found in reverse stack order. After extending
 202     # the found nodes, the freshest value will be at the tip of the stack.
 203     found_node_list.extend(reversed(newly_found))
 204     if not found_node_list:
 205       return None
 206
 207     return found_node_list[-1]._value.get(key)
 208
 209 class _Stack(object):
 210   class Entry(object):
 211     def __init__(self, name, id_):
 212       self.name = name
 213       self.id_ = id_
 214
 215   def __init__(self, entries=[]):
 216     self.entries = entries
 217
 218   def Descend(self, name, id_):
 219     descended = list(self.entries)
 220     descended.append(_Stack.Entry(name, id_))
 221     return _Stack(entries=descended)
 222
 223 class _InternalContext(object):
 224   def __init__(self):
 225     self._render_state = None
 226
 227   def SetRenderState(self, render_state):
 228     self._render_state = render_state
 229
 230   def get(self, key):
 231     if key == 'errors':
 232       errors = self._render_state._errors
 233       return '\n'.join(errors) if errors else None
 234     return None
 235
 236 class _RenderState(object):
 237   '''The state of a render call.
 238   '''
 239   def __init__(self, name, contexts, _stack=_Stack()):
 240     self.text = _StringBuilder()
 241     self.contexts = contexts
 242     self._name = name
 243     self._errors = []
 244     self._stack = _stack
 245
 246   def AddResolutionError(self, id_, description=None):
 247     message = id_.CreateResolutionErrorMessage(self._name, stack=self._stack)
 248     if description is not None:
 249       message = '%s (%s)' % (message, description)
 250     self._errors.append(message)
 251
 252   def Copy(self):
 253     return _RenderState(
 254         self._name, self.contexts, _stack=self._stack)
 255
 256   def ForkPartial(self, custom_name, id_):
 257     name = custom_name or id_.name
 258     return _RenderState(name,
 259                         self.contexts.CreateFromGlobals(),
 260                         _stack=self._stack.Descend(name, id_))
 261
 262   def Merge(self, render_state, text_transform=None):
 263     self._errors.extend(render_state._errors)
 264     text = render_state.text.ToString()
 265     if text_transform is not None:
 266       text = text_transform(text)
 267     self.text.Append(text)
 268
 269   def GetResult(self):
 270     return RenderResult(self.text.ToString(), self._errors);
 271
 272 class _Identifier(object):
 273   '''An identifier of the form 'foo', 'foo.bar.baz', 'foo-bar.baz', etc.
 274   '''
 275   _VALID_ID_MATCHER = re.compile(r'^[a-zA-Z0-9@_/-]+$')
 276
 277   def __init__(self, name, line, column):
 278     self.name = name
 279     self.line = line
 280     self.column = column
 281     if name == '':
 282       raise ParseException('Empty identifier %s' % self.GetDescription())
 283     for part in name.split('.'):
 284       if not _Identifier._VALID_ID_MATCHER.match(part):
 285         raise ParseException('Invalid identifier %s' % self.GetDescription())
 286
 287   def GetDescription(self):
 288     return '\'%s\' at line %s column %s' % (self.name, self.line, self.column)
 289
 290   def CreateResolutionErrorMessage(self, name, stack=None):
 291     message = _StringBuilder()
 292     message.Append('Failed to resolve %s in %s\n' % (self.GetDescription(),
 293                                                      name))
 294     if stack is not None:
 295       for entry in reversed(stack.entries):
 296         message.Append('  included as %s in %s\n' % (entry.id_.GetDescription(),
 297                                                      entry.name))
 298     return message.ToString().strip()
 299
 300   def __repr__(self):
 301     return self.name
 302
 303   def __str__(self):
 304     return repr(self)
 305
 306 class _Node(object): pass
 307
 308 class _LeafNode(_Node):
 309   def __init__(self, start_line, end_line):
 310     self._start_line = start_line
 311     self._end_line = end_line
 312
 313   def StartsWithNewLine(self):
 314     return False
 315
 316   def TrimStartingNewLine(self):
 317     pass
 318
 319   def TrimEndingSpaces(self):
 320     return 0
 321
 322   def TrimEndingNewLine(self):
 323     pass
 324
 325   def EndsWithEmptyLine(self):
 326     return False
 327
 328   def GetStartLine(self):
 329     return self._start_line
 330
 331   def GetEndLine(self):
 332     return self._end_line
 333
 334   def __str__(self):
 335     return repr(self)
 336
 337 class _DecoratorNode(_Node):
 338   def __init__(self, content):
 339     self._content = content
 340
 341   def StartsWithNewLine(self):
 342     return self._content.StartsWithNewLine()
 343
 344   def TrimStartingNewLine(self):
 345     self._content.TrimStartingNewLine()
 346
 347   def TrimEndingSpaces(self):
 348     return self._content.TrimEndingSpaces()
 349
 350   def TrimEndingNewLine(self):
 351     self._content.TrimEndingNewLine()
 352
 353   def EndsWithEmptyLine(self):
 354     return self._content.EndsWithEmptyLine()
 355
 356   def GetStartLine(self):
 357     return self._content.GetStartLine()
 358
 359   def GetEndLine(self):
 360     return self._content.GetEndLine()
 361
 362   def __repr__(self):
 363     return str(self._content)
 364
 365   def __str__(self):
 366     return repr(self)
 367
 368 class _InlineNode(_DecoratorNode):
 369   def __init__(self, content):
 370     _DecoratorNode.__init__(self, content)
 371
 372   def Render(self, render_state):
 373     content_render_state = render_state.Copy()
 374     self._content.Render(content_render_state)
 375     render_state.Merge(content_render_state,
 376                        text_transform=lambda text: text.replace('\n', ''))
 377
 378 class _IndentedNode(_DecoratorNode):
 379   def __init__(self, content, indentation):
 380     _DecoratorNode.__init__(self, content)
 381     self._indent_str = ' ' * indentation
 382
 383   def Render(self, render_state):
 384     if isinstance(self._content, _CommentNode):
 385       return
 386     def inlinify(text):
 387       if len(text) == 0:  # avoid rendering a blank line
 388         return ''
 389       buf = _StringBuilder()
 390       buf.Append(self._indent_str)
 391       buf.Append(text.replace('\n', '\n%s' % self._indent_str))
 392       if not text.endswith('\n'):  # partials will often already end in a \n
 393         buf.Append('\n')
 394       return buf.ToString()
 395     content_render_state = render_state.Copy()
 396     self._content.Render(content_render_state)
 397     render_state.Merge(content_render_state, text_transform=inlinify)
 398
 399 class _BlockNode(_DecoratorNode):
 400   def __init__(self, content):
 401     _DecoratorNode.__init__(self, content)
 402     content.TrimStartingNewLine()
 403     content.TrimEndingSpaces()
 404
 405   def Render(self, render_state):
 406     self._content.Render(render_state)
 407
 408 class _NodeCollection(_Node):
 409   def __init__(self, nodes):
 410     assert nodes
 411     self._nodes = nodes
 412
 413   def Render(self, render_state):
 414     for node in self._nodes:
 415       node.Render(render_state)
 416
 417   def StartsWithNewLine(self):
 418     return self._nodes[0].StartsWithNewLine()
 419
 420   def TrimStartingNewLine(self):
 421     self._nodes[0].TrimStartingNewLine()
 422
 423   def TrimEndingSpaces(self):
 424     return self._nodes[-1].TrimEndingSpaces()
 425
 426   def TrimEndingNewLine(self):
 427     self._nodes[-1].TrimEndingNewLine()
 428
 429   def EndsWithEmptyLine(self):
 430     return self._nodes[-1].EndsWithEmptyLine()
 431
 432   def GetStartLine(self):
 433     return self._nodes[0].GetStartLine()
 434
 435   def GetEndLine(self):
 436     return self._nodes[-1].GetEndLine()
 437
 438   def __repr__(self):
 439     return ''.join(str(node) for node in self._nodes)
 440
 441 class _StringNode(_Node):
 442   '''Just a string.
 443   '''
 444   def __init__(self, string, start_line, end_line):
 445     self._string = string
 446     self._start_line = start_line
 447     self._end_line = end_line
 448
 449   def Render(self, render_state):
 450     render_state.text.Append(self._string)
 451
 452   def StartsWithNewLine(self):
 453     return self._string.startswith('\n')
 454
 455   def TrimStartingNewLine(self):
 456     if self.StartsWithNewLine():
 457       self._string = self._string[1:]
 458
 459   def TrimEndingSpaces(self):
 460     original_length = len(self._string)
 461     self._string = self._string[:self._LastIndexOfSpaces()]
 462     return original_length - len(self._string)
 463
 464   def TrimEndingNewLine(self):
 465     if self._string.endswith('\n'):
 466       self._string = self._string[:len(self._string) - 1]
 467
 468   def EndsWithEmptyLine(self):
 469     index = self._LastIndexOfSpaces()
 470     return index == 0 or self._string[index - 1] == '\n'
 471
 472   def _LastIndexOfSpaces(self):
 473     index = len(self._string)
 474     while index > 0 and self._string[index - 1] == ' ':
 475       index -= 1
 476     return index
 477
 478   def GetStartLine(self):
 479     return self._start_line
 480
 481   def GetEndLine(self):
 482     return self._end_line
 483
 484   def __repr__(self):
 485     return self._string
 486
 487 class _EscapedVariableNode(_LeafNode):
 488   '''{{foo}}
 489   '''
 490   def __init__(self, id_):
 491     _LeafNode.__init__(self, id_.line, id_.line)
 492     self._id = id_
 493
 494   def Render(self, render_state):
 495     value = render_state.contexts.Resolve(self._id.name)
 496     if value is None:
 497       render_state.AddResolutionError(self._id)
 498       return
 499     string = value if isinstance(value, basestring) else str(value)
 500     render_state.text.Append(string.replace('&', '&amp;')
 501                                    .replace('<', '&lt;')
 502                                    .replace('>', '&gt;'))
 503
 504   def __repr__(self):
 505     return '{{%s}}' % self._id
 506
 507 class _UnescapedVariableNode(_LeafNode):
 508   '''{{{foo}}}
 509   '''
 510   def __init__(self, id_):
 511     _LeafNode.__init__(self, id_.line, id_.line)
 512     self._id = id_
 513
 514   def Render(self, render_state):
 515     value = render_state.contexts.Resolve(self._id.name)
 516     if value is None:
 517       render_state.AddResolutionError(self._id)
 518       return
 519     string = value if isinstance(value, basestring) else str(value)
 520     render_state.text.Append(string)
 521
 522   def __repr__(self):
 523     return '{{{%s}}}' % self._id
 524
 525 class _CommentNode(_LeafNode):
 526   '''{{- This is a comment -}}
 527   An empty placeholder node for correct indented rendering behaviour.
 528   '''
 529   def __init__(self, start_line, end_line):
 530     _LeafNode.__init__(self, start_line, end_line)
 531
 532   def Render(self, render_state):
 533     pass
 534
 535   def __repr__(self):
 536     return '<comment>'
 537
 538 class _SectionNode(_DecoratorNode):
 539   '''{{#var:foo}} ... {{/foo}}
 540   '''
 541   def __init__(self, bind_to, id_, content):
 542     _DecoratorNode.__init__(self, content)
 543     self._bind_to = bind_to
 544     self._id = id_
 545
 546   def Render(self, render_state):
 547     value = render_state.contexts.Resolve(self._id.name)
 548     if isinstance(value, list):
 549       for item in value:
 550         if self._bind_to is not None:
 551           render_state.contexts.Scope({self._bind_to.name: item},
 552                                       self._content.Render, render_state)
 553         else:
 554           self._content.Render(render_state)
 555     elif hasattr(value, 'get'):
 556       if self._bind_to is not None:
 557         render_state.contexts.Scope({self._bind_to.name: value},
 558                                     self._content.Render, render_state)
 559       else:
 560         render_state.contexts.Scope(value, self._content.Render, render_state)
 561     else:
 562       render_state.AddResolutionError(self._id)
 563
 564   def __repr__(self):
 565     return '{{#%s}}%s{{/%s}}' % (
 566         self._id, _DecoratorNode.__repr__(self), self._id)
 567
 568 class _VertedSectionNode(_DecoratorNode):
 569   '''{{?var:foo}} ... {{/foo}}
 570   '''
 571   def __init__(self, bind_to, id_, content):
 572     _DecoratorNode.__init__(self, content)
 573     self._bind_to = bind_to
 574     self._id = id_
 575
 576   def Render(self, render_state):
 577     value = render_state.contexts.Resolve(self._id.name)
 578     if _VertedSectionNode.ShouldRender(value):
 579       if self._bind_to is not None:
 580         render_state.contexts.Scope({self._bind_to.name: value},
 581                                     self._content.Render, render_state)
 582       else:
 583         self._content.Render(render_state)
 584
 585   def __repr__(self):
 586     return '{{?%s}}%s{{/%s}}' % (
 587         self._id, _DecoratorNode.__repr__(self), self._id)
 588
 589   @staticmethod
 590   def ShouldRender(value):
 591     if value is None:
 592       return False
 593     if isinstance(value, bool):
 594       return value
 595     if isinstance(value, list):
 596       return len(value) > 0
 597     return True
 598
 599 class _InvertedSectionNode(_DecoratorNode):
 600   '''{{^foo}} ... {{/foo}}
 601   '''
 602   def __init__(self, bind_to, id_, content):
 603     _DecoratorNode.__init__(self, content)
 604     if bind_to is not None:
 605       raise ParseException('{{^%s:%s}} does not support variable binding'
 606                            % (bind_to, id_))
 607     self._id = id_
 608
 609   def Render(self, render_state):
 610     value = render_state.contexts.Resolve(self._id.name)
 611     if not _VertedSectionNode.ShouldRender(value):
 612       self._content.Render(render_state)
 613
 614   def __repr__(self):
 615     return '{{^%s}}%s{{/%s}}' % (
 616         self._id, _DecoratorNode.__repr__(self), self._id)
 617
 618 class _AssertionNode(_LeafNode):
 619   '''{{!foo Some comment about foo}}
 620   '''
 621   def __init__(self, id_, description):
 622     _LeafNode.__init__(self, id_.line, id_.line)
 623     self._id = id_
 624     self._description = description
 625
 626   def Render(self, render_state):
 627     if render_state.contexts.Resolve(self._id.name) is None:
 628       render_state.AddResolutionError(self._id, description=self._description)
 629
 630   def __repr__(self):
 631     return '{{!%s %s}}' % (self._id, self._description)
 632
 633 class _JsonNode(_LeafNode):
 634   '''{{*foo}}
 635   '''
 636   def __init__(self, id_):
 637     _LeafNode.__init__(self, id_.line, id_.line)
 638     self._id = id_
 639
 640   def Render(self, render_state):
 641     value = render_state.contexts.Resolve(self._id.name)
 642     if value is None:
 643       render_state.AddResolutionError(self._id)
 644       return
 645     render_state.text.Append(json.dumps(value, separators=(',',':')))
 646
 647   def __repr__(self):
 648     return '{{*%s}}' % self._id
 649
 650 class _PartialNodeWithArguments(_DecoratorNode):
 651   def __init__(self, partial, args):
 652     if isinstance(partial, Motemplate):
 653       # Preserve any get() method that the caller has added.
 654       if hasattr(partial, 'get'):
 655         self.get = partial.get
 656       partial = partial._top_node
 657     _DecoratorNode.__init__(self, partial)
 658     self._partial = partial
 659     self._args = args
 660
 661   def Render(self, render_state):
 662     render_state.contexts.Scope(self._args, self._partial.Render, render_state)
 663
 664 class _PartialNodeInContext(_DecoratorNode):
 665   def __init__(self, partial, context):
 666     if isinstance(partial, Motemplate):
 667       # Preserve any get() method that the caller has added.
 668       if hasattr(partial, 'get'):
 669         self.get = partial.get
 670       partial = partial._top_node
 671     _DecoratorNode.__init__(self, partial)
 672     self._partial = partial
 673     self._context = context
 674
 675   def Render(self, render_state):
 676     original_contexts = render_state.contexts
 677     try:
 678       render_state.contexts = self._context
 679       render_state.contexts.Scope(
 680           # The first local context of |original_contexts| will be the
 681           # arguments that were passed to the partial, if any.
 682           original_contexts.FirstLocal() or {},
 683           self._partial.Render, render_state)
 684     finally:
 685       render_state.contexts = original_contexts
 686
 687 class _PartialNode(_LeafNode):
 688   '''{{+var:foo}} ... {{/foo}}
 689   '''
 690   def __init__(self, bind_to, id_, content):
 691     _LeafNode.__init__(self, id_.line, id_.line)
 692     self._bind_to = bind_to
 693     self._id = id_
 694     self._content = content
 695     self._args = None
 696     self._pass_through_id = None
 697
 698   @classmethod
 699   def Inline(cls, id_):
 700     return cls(None, id_, None)
 701
 702   def Render(self, render_state):
 703     value = render_state.contexts.Resolve(self._id.name)
 704     if value is None:
 705       render_state.AddResolutionError(self._id)
 706       return
 707     if not isinstance(value, (Motemplate, _Node)):
 708       render_state.AddResolutionError(self._id, description='not a partial')
 709       return
 710
 711     if isinstance(value, Motemplate):
 712       node, name = value._top_node, value._name
 713     else:
 714       node, name = value, None
 715
 716     partial_render_state = render_state.ForkPartial(name, self._id)
 717
 718     arg_context = {}
 719     if self._pass_through_id is not None:
 720       context = render_state.contexts.Resolve(self._pass_through_id.name)
 721       if context is not None:
 722         arg_context[self._pass_through_id.name] = context
 723     if self._args is not None:
 724       def resolve_args(args):
 725         resolved = {}
 726         for key, value in args.iteritems():
 727           if isinstance(value, dict):
 728             assert len(value.keys()) == 1
 729             id_of_partial, partial_args = value.items()[0]
 730             partial = render_state.contexts.Resolve(id_of_partial.name)
 731             if partial is not None:
 732               resolved[key] = _PartialNodeWithArguments(
 733                   partial, resolve_args(partial_args))
 734           else:
 735             context = render_state.contexts.Resolve(value.name)
 736             if context is not None:
 737               resolved[key] = context
 738         return resolved
 739       arg_context.update(resolve_args(self._args))
 740     if self._bind_to and self._content:
 741       arg_context[self._bind_to.name] = _PartialNodeInContext(
 742           self._content, render_state.contexts)
 743     if arg_context:
 744       partial_render_state.contexts.Push(arg_context)
 745
 746     node.Render(partial_render_state)
 747
 748     render_state.Merge(
 749         partial_render_state,
 750         text_transform=lambda text: text[:-1] if text.endswith('\n') else text)
 751
 752   def SetArguments(self, args):
 753     self._args = args
 754
 755   def PassThroughArgument(self, id_):
 756     self._pass_through_id = id_
 757
 758   def __repr__(self):
 759     return '{{+%s}}' % self._id
 760
 761 _TOKENS = {}
 762
 763 class _Token(object):
 764   '''The tokens that can appear in a template.
 765   '''
 766   class Data(object):
 767     def __init__(self, name, text, clazz):
 768       self.name = name
 769       self.text = text
 770       self.clazz = clazz
 771       _TOKENS[text] = self
 772
 773     def ElseNodeClass(self):
 774       if self.clazz == _VertedSectionNode:
 775         return _InvertedSectionNode
 776       if self.clazz == _InvertedSectionNode:
 777         return _VertedSectionNode
 778       return None
 779
 780     def __repr__(self):
 781       return self.name
 782
 783     def __str__(self):
 784       return repr(self)
 785
 786   OPEN_START_SECTION = Data(
 787       'OPEN_START_SECTION'         , '{{#', _SectionNode)
 788   OPEN_START_VERTED_SECTION = Data(
 789       'OPEN_START_VERTED_SECTION'  , '{{?', _VertedSectionNode)
 790   OPEN_START_INVERTED_SECTION = Data(
 791       'OPEN_START_INVERTED_SECTION', '{{^', _InvertedSectionNode)
 792   OPEN_ASSERTION = Data(
 793       'OPEN_ASSERTION'             , '{{!', _AssertionNode)
 794   OPEN_JSON = Data(
 795       'OPEN_JSON'                  , '{{*', _JsonNode)
 796   OPEN_PARTIAL = Data(
 797       'OPEN_PARTIAL'               , '{{+', _PartialNode)
 798   OPEN_ELSE = Data(
 799       'OPEN_ELSE'                  , '{{:', None)
 800   OPEN_END_SECTION = Data(
 801       'OPEN_END_SECTION'           , '{{/', None)
 802   INLINE_END_SECTION = Data(
 803       'INLINE_END_SECTION'         , '/}}', None)
 804   OPEN_UNESCAPED_VARIABLE = Data(
 805       'OPEN_UNESCAPED_VARIABLE'    , '{{{', _UnescapedVariableNode)
 806   CLOSE_MUSTACHE3 = Data(
 807       'CLOSE_MUSTACHE3'            , '}}}', None)
 808   OPEN_COMMENT = Data(
 809       'OPEN_COMMENT'               , '{{-', _CommentNode)
 810   CLOSE_COMMENT = Data(
 811       'CLOSE_COMMENT'              , '-}}', None)
 812   OPEN_VARIABLE = Data(
 813       'OPEN_VARIABLE'              , '{{' , _EscapedVariableNode)
 814   CLOSE_MUSTACHE = Data(
 815       'CLOSE_MUSTACHE'             , '}}' , None)
 816   CHARACTER = Data(
 817       'CHARACTER'                  , '.'  , None)
 818
 819 class _TokenStream(object):
 820   '''Tokeniser for template parsing.
 821   '''
 822   def __init__(self, string):
 823     self.next_token = None
 824     self.next_line = 1
 825     self.next_column = 0
 826     self._string = string
 827     self._cursor = 0
 828     self.Advance()
 829
 830   def HasNext(self):
 831     return self.next_token is not None
 832
 833   def NextCharacter(self):
 834     if self.next_token is _Token.CHARACTER:
 835       return self._string[self._cursor - 1]
 836     return None
 837
 838   def Advance(self):
 839     if self._cursor > 0 and self._string[self._cursor - 1] == '\n':
 840       self.next_line += 1
 841       self.next_column = 0
 842     elif self.next_token is not None:
 843       self.next_column += len(self.next_token.text)
 844
 845     self.next_token = None
 846
 847     if self._cursor == len(self._string):
 848       return None
 849     assert self._cursor < len(self._string)
 850
 851     if (self._cursor + 1 < len(self._string) and
 852         self._string[self._cursor + 1] in '{}'):
 853       self.next_token = (
 854           _TOKENS.get(self._string[self._cursor:self._cursor+3]) or
 855           _TOKENS.get(self._string[self._cursor:self._cursor+2]))
 856
 857     if self.next_token is None:
 858       self.next_token = _Token.CHARACTER
 859
 860     self._cursor += len(self.next_token.text)
 861     return self
 862
 863   def AdvanceOver(self, token, description=None):
 864     parse_error = None
 865     if not self.next_token:
 866       parse_error = 'Reached EOF but expected %s' % token.name
 867     elif self.next_token is not token:
 868       parse_error = 'Expecting token %s but got %s at line %s' % (
 869                      token.name, self.next_token.name, self.next_line)
 870     if parse_error:
 871       parse_error += ' %s' % description or ''
 872       raise ParseException(parse_error)
 873     return self.Advance()
 874
 875   def AdvanceOverSeparator(self, char, description=None):
 876     self.SkipWhitespace()
 877     next_char = self.NextCharacter()
 878     if next_char != char:
 879       parse_error = 'Expected \'%s\'. got \'%s\'' % (char, next_char)
 880       if description is not None:
 881         parse_error += ' (%s)' % description
 882       raise ParseException(parse_error)
 883     self.AdvanceOver(_Token.CHARACTER)
 884     self.SkipWhitespace()
 885
 886   def AdvanceOverNextString(self, excluded=''):
 887     start = self._cursor - len(self.next_token.text)
 888     while (self.next_token is _Token.CHARACTER and
 889            # Can use -1 here because token length of CHARACTER is 1.
 890            self._string[self._cursor - 1] not in excluded):
 891       self.Advance()
 892     end = self._cursor - (len(self.next_token.text) if self.next_token else 0)
 893     return self._string[start:end]
 894
 895   def AdvanceToNextWhitespace(self):
 896     return self.AdvanceOverNextString(excluded=' \n\r\t')
 897
 898   def SkipWhitespace(self):
 899     while (self.next_token is _Token.CHARACTER and
 900            # Can use -1 here because token length of CHARACTER is 1.
 901            self._string[self._cursor - 1] in ' \n\r\t'):
 902       self.Advance()
 903
 904   def __repr__(self):
 905     return '%s(next_token=%s, remainder=%s)' % (type(self).__name__,
 906                                                 self.next_token,
 907                                                 self._string[self._cursor:])
 908
 909   def __str__(self):
 910     return repr(self)
 911
 912 class Motemplate(object):
 913   '''A motemplate template.
 914   '''
 915   def __init__(self, template, name=None):
 916     self.source = template
 917     self._name = name
 918     tokens = _TokenStream(template)
 919     self._top_node = self._ParseSection(tokens)
 920     if not self._top_node:
 921       raise ParseException('Template is empty')
 922     if tokens.HasNext():
 923       raise ParseException('There are still tokens remaining at %s, '
 924                            'was there an end-section without a start-section?' %
 925                            tokens.next_line)
 926
 927   def _ParseSection(self, tokens):
 928     nodes = []
 929     while tokens.HasNext():
 930       if tokens.next_token in (_Token.OPEN_END_SECTION,
 931                                _Token.OPEN_ELSE):
 932         # Handled after running parseSection within the SECTION cases, so this
 933         # is a terminating condition. If there *is* an orphaned
 934         # OPEN_END_SECTION, it will be caught by noticing that there are
 935         # leftover tokens after termination.
 936         break
 937       elif tokens.next_token in (_Token.CLOSE_MUSTACHE,
 938                                  _Token.CLOSE_MUSTACHE3):
 939         raise ParseException('Orphaned %s at line %s' % (tokens.next_token.name,
 940                                                          tokens.next_line))
 941       nodes += self._ParseNextOpenToken(tokens)
 942
 943     for i, node in enumerate(nodes):
 944       if isinstance(node, _StringNode):
 945         continue
 946
 947       previous_node = nodes[i - 1] if i > 0 else None
 948       next_node = nodes[i + 1] if i < len(nodes) - 1 else None
 949       rendered_node = None
 950
 951       if node.GetStartLine() != node.GetEndLine():
 952         rendered_node = _BlockNode(node)
 953         if previous_node:
 954           previous_node.TrimEndingSpaces()
 955         if next_node:
 956           next_node.TrimStartingNewLine()
 957       elif ((not previous_node or previous_node.EndsWithEmptyLine()) and
 958             (not next_node or next_node.StartsWithNewLine())):
 959         indentation = 0
 960         if previous_node:
 961           indentation = previous_node.TrimEndingSpaces()
 962         if next_node:
 963           next_node.TrimStartingNewLine()
 964         rendered_node = _IndentedNode(node, indentation)
 965       else:
 966         rendered_node = _InlineNode(node)
 967
 968       nodes[i] = rendered_node
 969
 970     if len(nodes) == 0:
 971       return None
 972     if len(nodes) == 1:
 973       return nodes[0]
 974     return _NodeCollection(nodes)
 975
 976   def _ParseNextOpenToken(self, tokens):
 977     next_token = tokens.next_token
 978
 979     if next_token is _Token.CHARACTER:
 980       # Plain strings.
 981       start_line = tokens.next_line
 982       string = tokens.AdvanceOverNextString()
 983       return [_StringNode(string, start_line, tokens.next_line)]
 984     elif next_token in (_Token.OPEN_VARIABLE,
 985                         _Token.OPEN_UNESCAPED_VARIABLE,
 986                         _Token.OPEN_JSON):
 987       # Inline nodes that don't take arguments.
 988       tokens.Advance()
 989       close_token = (_Token.CLOSE_MUSTACHE3
 990                      if next_token is _Token.OPEN_UNESCAPED_VARIABLE else
 991                      _Token.CLOSE_MUSTACHE)
 992       id_ = self._NextIdentifier(tokens)
 993       tokens.AdvanceOver(close_token)
 994       return [next_token.clazz(id_)]
 995     elif next_token is _Token.OPEN_ASSERTION:
 996       # Inline nodes that take arguments.
 997       tokens.Advance()
 998       id_ = self._NextIdentifier(tokens)
 999       node = next_token.clazz(id_, tokens.AdvanceOverNextString())
1000       tokens.AdvanceOver(_Token.CLOSE_MUSTACHE)
1001       return [node]
1002     elif next_token in (_Token.OPEN_PARTIAL,
1003                         _Token.OPEN_START_SECTION,
1004                         _Token.OPEN_START_VERTED_SECTION,
1005                         _Token.OPEN_START_INVERTED_SECTION):
1006       # Block nodes, though they may have inline syntax like {{#foo bar /}}.
1007       tokens.Advance()
1008       bind_to, id_ = None, self._NextIdentifier(tokens)
1009       if tokens.NextCharacter() == ':':
1010         # This section has the format {{#bound:id}} as opposed to just {{id}}.
1011         # That is, |id_| is actually the identifier to bind what the section
1012         # is producing, not the identifier of where to find that content.
1013         tokens.AdvanceOverSeparator(':')
1014         bind_to, id_ = id_, self._NextIdentifier(tokens)
1015       partial_args = None
1016       if next_token is _Token.OPEN_PARTIAL:
1017         partial_args = self._ParsePartialNodeArgs(tokens)
1018         if tokens.next_token is not _Token.CLOSE_MUSTACHE:
1019           # Inline syntax for partial types.
1020           if bind_to is not None:
1021             raise ParseException(
1022                 'Cannot bind %s to a self-closing partial' % bind_to)
1023           tokens.AdvanceOver(_Token.INLINE_END_SECTION)
1024           partial_node = _PartialNode.Inline(id_)
1025           partial_node.SetArguments(partial_args)
1026           return [partial_node]
1027       elif tokens.next_token is not _Token.CLOSE_MUSTACHE:
1028         # Inline syntax for non-partial types. Support select node types:
1029         # variables, partials, JSON.
1030         line, column = tokens.next_line, (tokens.next_column + 1)
1031         name = tokens.AdvanceToNextWhitespace()
1032         clazz = _UnescapedVariableNode
1033         if name.startswith('*'):
1034           clazz = _JsonNode
1035         elif name.startswith('+'):
1036           clazz = _PartialNode.Inline
1037         if clazz is not _UnescapedVariableNode:
1038           name = name[1:]
1039           column += 1
1040         inline_node = clazz(_Identifier(name, line, column))
1041         if isinstance(inline_node, _PartialNode):
1042           inline_node.SetArguments(self._ParsePartialNodeArgs(tokens))
1043           if bind_to is not None:
1044             inline_node.PassThroughArgument(bind_to)
1045         tokens.SkipWhitespace()
1046         tokens.AdvanceOver(_Token.INLINE_END_SECTION)
1047         return [next_token.clazz(bind_to, id_, inline_node)]
1048       # Block syntax.
1049       tokens.AdvanceOver(_Token.CLOSE_MUSTACHE)
1050       section = self._ParseSection(tokens)
1051       else_node_class = next_token.ElseNodeClass()  # may not have one
1052       else_section = None
1053       if (else_node_class is not None and
1054           tokens.next_token is _Token.OPEN_ELSE):
1055         self._OpenElse(tokens, id_)
1056         else_section = self._ParseSection(tokens)
1057       self._CloseSection(tokens, id_)
1058       nodes = []
1059       if section is not None:
1060         node = next_token.clazz(bind_to, id_, section)
1061         if partial_args:
1062           node.SetArguments(partial_args)
1063         nodes.append(node)
1064       if else_section is not None:
1065         nodes.append(else_node_class(bind_to, id_, else_section))
1066       return nodes
1067     elif next_token is _Token.OPEN_COMMENT:
1068       # Comments.
1069       start_line = tokens.next_line
1070       self._AdvanceOverComment(tokens)
1071       return [_CommentNode(start_line, tokens.next_line)]
1072
1073   def _AdvanceOverComment(self, tokens):
1074     tokens.AdvanceOver(_Token.OPEN_COMMENT)
1075     depth = 1
1076     while tokens.HasNext() and depth > 0:
1077       if tokens.next_token is _Token.OPEN_COMMENT:
1078         depth += 1
1079       elif tokens.next_token is _Token.CLOSE_COMMENT:
1080         depth -= 1
1081       tokens.Advance()
1082
1083   def _CloseSection(self, tokens, id_):
1084     tokens.AdvanceOver(_Token.OPEN_END_SECTION,
1085                        description='to match %s' % id_.GetDescription())
1086     next_string = tokens.AdvanceOverNextString()
1087     if next_string != '' and next_string != id_.name:
1088       raise ParseException(
1089           'Start section %s doesn\'t match end %s' % (id_, next_string))
1090     tokens.AdvanceOver(_Token.CLOSE_MUSTACHE)
1091
1092   def _OpenElse(self, tokens, id_):
1093     tokens.AdvanceOver(_Token.OPEN_ELSE)
1094     next_string = tokens.AdvanceOverNextString()
1095     if next_string != '' and next_string != id_.name:
1096       raise ParseException(
1097           'Start section %s doesn\'t match else %s' % (id_, next_string))
1098     tokens.AdvanceOver(_Token.CLOSE_MUSTACHE)
1099
1100   def _ParsePartialNodeArgs(self, tokens):
1101     args = {}
1102     tokens.SkipWhitespace()
1103     while (tokens.next_token is _Token.CHARACTER and
1104            tokens.NextCharacter() != ')'):
1105       key = tokens.AdvanceOverNextString(excluded=':')
1106       tokens.AdvanceOverSeparator(':')
1107       if tokens.NextCharacter() == '(':
1108         tokens.AdvanceOverSeparator('(')
1109         inner_id = self._NextIdentifier(tokens)
1110         inner_args = self._ParsePartialNodeArgs(tokens)
1111         tokens.AdvanceOverSeparator(')')
1112         args[key] = {inner_id: inner_args}
1113       else:
1114         args[key] = self._NextIdentifier(tokens)
1115     return args or None
1116
1117   def _NextIdentifier(self, tokens):
1118     tokens.SkipWhitespace()
1119     column_start = tokens.next_column + 1
1120     id_ = _Identifier(tokens.AdvanceOverNextString(excluded=' \n\r\t:()'),
1121                       tokens.next_line,
1122                       column_start)
1123     tokens.SkipWhitespace()
1124     return id_
1125
1126   def Render(self, *user_contexts):
1127     '''Renders this template given a variable number of contexts to read out
1128     values from (such as those appearing in {{foo}}).
1129     '''
1130     internal_context = _InternalContext()
1131     contexts = list(user_contexts)
1132     contexts.append({
1133       '_': internal_context,
1134       'false': False,
1135       'true': True,
1136     })
1137     render_state = _RenderState(self._name or '<root>', _Contexts(contexts))
1138     internal_context.SetRenderState(render_state)
1139     self._top_node.Render(render_state)
1140     return render_state.GetResult()
1141
1142   def render(self, *contexts):
1143     return self.Render(*contexts)
1144
1145   def __eq__(self, other):
1146     return self.source == other.source and self._name == other._name
1147
1148   def __ne__(self, other):
1149     return not (self == other)
1150
1151   def __repr__(self):
1152     return str('%s(%s)' % (type(self).__name__, self._top_node))
1153
1154   def __str__(self):
1155     return repr(self)