Cheetah/Parser.py

   1 #!/usr/bin/env python
   2 # $Id: Parser.py,v 1.135 2007/11/16 18:26:01 tavis_rudd Exp $
   3 """Parser classes for Cheetah's Compiler
   4
   5 Classes:
   6   ParseError( Exception )
   7   _LowLevelParser( Cheetah.SourceReader.SourceReader ), basically a lexer
   8   _HighLevelParser( _LowLevelParser )
   9   Parser === _HighLevelParser (an alias)
  10
  11 Meta-Data
  12 ================================================================================
  13 Author: Tavis Rudd <tavis@damnsimple.com>
  14 Version: $Revision: 1.135 $
  15 Start Date: 2001/08/01
  16 Last Revision Date: $Date: 2007/11/16 18:26:01 $
  17 """
  18 __author__ = "Tavis Rudd <tavis@damnsimple.com>"
  19 __revision__ = "$Revision: 1.135 $"[11:-2]
  20
  21 import os
  22 import sys
  23 import re
  24 from re import DOTALL, MULTILINE
  25 from types import StringType, ListType, TupleType, ClassType, TypeType
  26 import time
  27 from tokenize import pseudoprog
  28 import inspect
  29 import new
  30 import traceback
  31
  32 from Cheetah.SourceReader import SourceReader
  33 from Cheetah import Filters
  34 from Cheetah import ErrorCatchers
  35 from Cheetah.Unspecified import Unspecified
  36 from Cheetah.Macros.I18n import I18n
  37
  38 # re tools
  39 _regexCache = {}
  40 def cachedRegex(pattern):
  41     if pattern not in _regexCache:
  42         _regexCache[pattern] = re.compile(pattern)
  43     return _regexCache[pattern]
  44
  45 def escapeRegexChars(txt,
  46                      escapeRE=re.compile(r'([\$\^\*\+\.\?\{\}\[\]\(\)\|\\])')):
  47
  48     """Return a txt with all special regular expressions chars escaped."""
  49
  50     return escapeRE.sub(r'\\\1' , txt)
  51
  52 def group(*choices): return '(' + '|'.join(choices) + ')'
  53 def nongroup(*choices): return '(?:' + '|'.join(choices) + ')'
  54 def namedGroup(name, *choices): return '(P:<' + name +'>' + '|'.join(choices) + ')'
  55 def any(*choices): return apply(group, choices) + '*'
  56 def maybe(*choices): return apply(group, choices) + '?'
  57
  58 ##################################################
  59 ## CONSTANTS & GLOBALS ##
  60
  61 NO_CACHE = 0
  62 STATIC_CACHE = 1
  63 REFRESH_CACHE = 2
  64
  65 SET_LOCAL = 0
  66 SET_GLOBAL = 1
  67 SET_MODULE = 2
  68
  69 ##################################################
  70 ## Tokens for the parser ##
  71
  72 #generic
  73 identchars = "abcdefghijklmnopqrstuvwxyz" \
  74             "ABCDEFGHIJKLMNOPQRSTUVWXYZ_"
  75 namechars = identchars + "0123456789"
  76
  77 #operators
  78 powerOp = '**'
  79 unaryArithOps = ('+', '-', '~')
  80 binaryArithOps = ('+', '-', '/', '//','%')
  81 shiftOps = ('>>','<<')
  82 bitwiseOps = ('&','|','^')
  83 assignOp = '='
  84 augAssignOps = ('+=','-=','/=','*=', '**=','^=','%=',
  85           '>>=','<<=','&=','|=', )
  86 assignmentOps = (assignOp,) + augAssignOps
  87
  88 compOps = ('<','>','==','!=','<=','>=', '<>', 'is', 'in',)
  89 booleanOps = ('and','or','not')
  90 operators = (powerOp,) + unaryArithOps + binaryArithOps \
  91             + shiftOps + bitwiseOps + assignmentOps \
  92             + compOps + booleanOps
  93
  94 delimeters = ('(',')','{','}','[',']',
  95               ',','.',':',';','=','`') + augAssignOps
  96
  97
  98 keywords = ('and',       'del',       'for',       'is',        'raise',
  99             'assert',    'elif',      'from',      'lambda',    'return',
 100             'break',     'else',      'global',    'not',       'try',
 101             'class',     'except',    'if',        'or',        'while',
 102             'continue',  'exec',      'import',    'pass',
 103             'def',       'finally',   'in',        'print',
 104             )
 105
 106 single3 = "'''"
 107 double3 = '"""'
 108
 109 tripleQuotedStringStarts =  ("'''", '"""',
 110                              "r'''", 'r"""', "R'''", 'R"""',
 111                              "u'''", 'u"""', "U'''", 'U"""',
 112                              "ur'''", 'ur"""', "Ur'''", 'Ur"""',
 113                              "uR'''", 'uR"""', "UR'''", 'UR"""')
 114
 115 tripleQuotedStringPairs = {"'''": single3, '"""': double3,
 116                            "r'''": single3, 'r"""': double3,
 117                            "u'''": single3, 'u"""': double3,
 118                            "ur'''": single3, 'ur"""': double3,
 119                            "R'''": single3, 'R"""': double3,
 120                            "U'''": single3, 'U"""': double3,
 121                            "uR'''": single3, 'uR"""': double3,
 122                            "Ur'''": single3, 'Ur"""': double3,
 123                            "UR'''": single3, 'UR"""': double3,
 124                            }
 125
 126 closurePairs= {')':'(',']':'[','}':'{'}
 127 closurePairsRev= {'(':')','[':']','{':'}'}
 128
 129 ##################################################
 130 ## Regex chunks for the parser ##
 131
 132 tripleQuotedStringREs = {}
 133 def makeTripleQuoteRe(start, end):
 134     start = escapeRegexChars(start)
 135     end = escapeRegexChars(end)
 136     return re.compile(r'(?:' + start + r').*?' + r'(?:' + end + r')', re.DOTALL)
 137
 138 for start, end in tripleQuotedStringPairs.items():
 139     tripleQuotedStringREs[start] = makeTripleQuoteRe(start, end)
 140
 141 WS = r'[ \f\t]*'
 142 EOL = r'\r\n|\n|\r'
 143 EOLZ = EOL + r'|\Z'
 144 escCharLookBehind = nongroup(r'(?<=\A)',r'(?<!\\)')
 145 nameCharLookAhead = r'(?=[A-Za-z_])'
 146 identRE=re.compile(r'[a-zA-Z_][a-zA-Z_0-9]*')
 147 EOLre=re.compile(r'(?:\r\n|\r|\n)')
 148
 149 specialVarRE=re.compile(r'([a-zA-z_]+)@') # for matching specialVar comments
 150 # e.g. ##author@ Tavis Rudd
 151
 152 unicodeDirectiveRE = re.compile(
 153     r'(?:^|\r\n|\r|\n)\s*#\s{0,5}unicode[:\s]*([-\w.]*)\s*(?:\r\n|\r|\n)', re.MULTILINE)
 154 encodingDirectiveRE = re.compile(
 155     r'(?:^|\r\n|\r|\n)\s*#\s{0,5}encoding[:\s]*([-\w.]*)\s*(?:\r\n|\r|\n)', re.MULTILINE)
 156
 157 escapedNewlineRE = re.compile(r'(?<!\\)\\n')
 158
 159 directiveNamesAndParsers = {
 160     # importing and inheritance
 161     'import':None,
 162     'from':None,
 163     'extends': 'eatExtends',
 164     'implements': 'eatImplements',
 165     'super': 'eatSuper',
 166
 167     # output, filtering, and caching
 168     'slurp': 'eatSlurp',
 169     'raw': 'eatRaw',
 170     'include': 'eatInclude',
 171     'cache': 'eatCache',
 172     'filter': 'eatFilter',
 173     'echo': None,
 174     'silent': None,
 175
 176     'call': 'eatCall',
 177     'arg': 'eatCallArg',
 178
 179     'capture': 'eatCapture',
 180
 181     # declaration, assignment, and deletion
 182     'attr': 'eatAttr',
 183     'def': 'eatDef',
 184     'block': 'eatBlock',
 185     '@': 'eatDecorator',
 186     'defmacro': 'eatDefMacro',
 187
 188     'closure': 'eatClosure',
 189
 190     'set': 'eatSet',
 191     'del': None,
 192
 193     # flow control
 194     'if': 'eatIf',
 195     'while': None,
 196     'for': None,
 197     'else': None,
 198     'elif': None,
 199     'pass': None,
 200     'break': None,
 201     'continue': None,
 202     'stop': None,
 203     'return': None,
 204     'yield': None,
 205
 206     # little wrappers
 207     'repeat': None,
 208     'unless': None,
 209
 210     # error handling
 211     'assert': None,
 212     'raise': None,
 213     'try': None,
 214     'except': None,
 215     'finally': None,
 216     'errorCatcher': 'eatErrorCatcher',
 217
 218     # intructions to the parser and compiler
 219     'breakpoint': 'eatBreakPoint',
 220     'compiler': 'eatCompiler',
 221     'compiler-settings': 'eatCompilerSettings',
 222
 223     # misc
 224     'shBang': 'eatShbang',
 225     'encoding': 'eatEncoding',
 226
 227     'end': 'eatEndDirective',
 228     }
 229
 230 endDirectiveNamesAndHandlers = {
 231     'def': 'handleEndDef',      # has short-form
 232     'block': None,              # has short-form
 233     'closure': None,            # has short-form
 234     'cache': None,              # has short-form
 235     'call': None,               # has short-form
 236     'capture': None,            # has short-form
 237     'filter': None,
 238     'errorCatcher':None,
 239     'while': None,              # has short-form
 240     'for': None,                # has short-form
 241     'if': None,                 # has short-form
 242     'try': None,                # has short-form
 243     'repeat': None,             # has short-form
 244     'unless': None,             # has short-form
 245     }
 246
 247 ##################################################
 248 ## CLASSES ##
 249
 250 # @@TR: SyntaxError doesn't call exception.__str__ for some reason!
 251 #class ParseError(SyntaxError):
 252 class ParseError(ValueError):
 253     def __init__(self, stream, msg='Invalid Syntax', extMsg='', lineno=None, col=None):
 254         self.stream = stream
 255         if stream.pos() >= len(stream):
 256             stream.setPos(len(stream) -1)
 257         self.msg = msg
 258         self.extMsg = extMsg
 259         self.lineno = lineno
 260         self.col = col
 261
 262     def __str__(self):
 263         return self.report()
 264
 265     def report(self):
 266         stream = self.stream
 267         if stream.filename():
 268             f = " in file %s" % stream.filename()
 269         else:
 270             f = ''
 271         report = ''
 272         if self.lineno:
 273             lineno = self.lineno
 274             row, col, line = (lineno, (self.col or 0),
 275                               self.stream.splitlines()[lineno-1])
 276         else:
 277             row, col, line = self.stream.getRowColLine()
 278
 279         ## get the surrounding lines
 280         lines = stream.splitlines()
 281         prevLines = []                  # (rowNum, content)
 282         for i in range(1,4):
 283             if row-1-i <=0:
 284                 break
 285             prevLines.append( (row-i,lines[row-1-i]) )
 286
 287         nextLines = []                  # (rowNum, content)
 288         for i in range(1,4):
 289             if not row-1+i < len(lines):
 290                 break
 291             nextLines.append( (row+i,lines[row-1+i]) )
 292         nextLines.reverse()
 293
 294         ## print the main message
 295         report += "\n\n%s\n" %self.msg
 296         report += "Line %i, column %i%s\n\n" % (row, col, f)
 297         report += 'Line|Cheetah Code\n'
 298         report += '----|-------------------------------------------------------------\n'
 299         while prevLines:
 300             lineInfo = prevLines.pop()
 301             report += "%(row)-4d|%(line)s\n"% {'row':lineInfo[0], 'line':lineInfo[1]}
 302         report += "%(row)-4d|%(line)s\n"% {'row':row, 'line':line}
 303         report += ' '*5 +' '*(col-1) + "^\n"
 304
 305         while nextLines:
 306             lineInfo = nextLines.pop()
 307             report += "%(row)-4d|%(line)s\n"% {'row':lineInfo[0], 'line':lineInfo[1]}
 308         ## add the extra msg
 309         if self.extMsg:
 310             report += self.extMsg + '\n'
 311
 312         return report
 313
 314 class ForbiddenSyntax(ParseError): pass
 315 class ForbiddenExpression(ForbiddenSyntax): pass
 316 class ForbiddenDirective(ForbiddenSyntax): pass
 317
 318 class CheetahVariable:
 319     def __init__(self, nameChunks, useNameMapper=True, cacheToken=None,
 320                  rawSource=None):
 321         self.nameChunks = nameChunks
 322         self.useNameMapper = useNameMapper
 323         self.cacheToken = cacheToken
 324         self.rawSource = rawSource
 325
 326 class Placeholder(CheetahVariable): pass
 327
 328 class ArgList:
 329     """Used by _LowLevelParser.getArgList()"""
 330
 331     def __init__(self):
 332         self.argNames = []
 333         self.defVals = []
 334         self.i = 0
 335
 336     def addArgName(self, name):
 337         self.argNames.append( name )
 338         self.defVals.append( None )
 339
 340     def next(self):
 341         self.i += 1
 342
 343     def addToDefVal(self, token):
 344         i = self.i
 345         if self.defVals[i] == None:
 346             self.defVals[i] = ''
 347         self.defVals[i] += token
 348
 349     def merge(self):
 350         defVals = self.defVals
 351         for i in range(len(defVals)):
 352             if type(defVals[i]) == StringType:
 353                 defVals[i] = defVals[i].strip()
 354
 355         return map(None, [i.strip() for i in self.argNames], defVals)
 356
 357     def __str__(self):
 358         return str(self.merge())
 359
 360 class _LowLevelParser(SourceReader):
 361     """This class implements the methods to match or extract ('get*') the basic
 362     elements of Cheetah's grammar.  It does NOT handle any code generation or
 363     state management.
 364     """
 365
 366     _settingsManager = None
 367
 368     def setSettingsManager(self, settingsManager):
 369         self._settingsManager = settingsManager
 370
 371     def setting(self, key, default=Unspecified):
 372         if default is Unspecified:
 373             return self._settingsManager.setting(key)
 374         else:
 375             return self._settingsManager.setting(key, default=default)
 376
 377     def setSetting(self, key, val):
 378         self._settingsManager.setSetting(key, val)
 379
 380     def settings(self):
 381         return self._settingsManager.settings()
 382
 383     def updateSettings(self, settings):
 384         self._settingsManager.updateSettings(settings)
 385
 386     def _initializeSettings(self):
 387         self._settingsManager._initializeSettings()
 388
 389     def configureParser(self):
 390         """Is called by the Compiler instance after the parser has had a
 391         settingsManager assigned with self.setSettingsManager()
 392         """
 393         self._makeCheetahVarREs()
 394         self._makeCommentREs()
 395         self._makeDirectiveREs()
 396         self._makePspREs()
 397         self._possibleNonStrConstantChars = (
 398             self.setting('commentStartToken')[0] +
 399             self.setting('multiLineCommentStartToken')[0] +
 400             self.setting('cheetahVarStartToken')[0] +
 401             self.setting('directiveStartToken')[0] +
 402             self.setting('PSPStartToken')[0])
 403         self._nonStrConstMatchers = [
 404             self.matchCommentStartToken,
 405             self.matchMultiLineCommentStartToken,
 406             self.matchVariablePlaceholderStart,
 407             self.matchExpressionPlaceholderStart,
 408             self.matchDirective,
 409             self.matchPSPStartToken,
 410             self.matchEOLSlurpToken,
 411             ]
 412
 413     ## regex setup ##
 414
 415     def _makeCheetahVarREs(self):
 416
 417         """Setup the regexs for Cheetah $var parsing."""
 418
 419         num = r'[0-9\.]+'
 420         interval =   (r'(?P<interval>' +
 421                       num + r's|' +
 422                       num + r'm|' +
 423                       num + r'h|' +
 424                       num + r'd|' +
 425                       num + r'w|' +
 426                       num + ')'
 427                       )
 428
 429         cacheToken = (r'(?:' +
 430                       r'(?P<REFRESH_CACHE>\*' + interval + '\*)'+
 431                       '|' +
 432                       r'(?P<STATIC_CACHE>\*)' +
 433                       '|' +
 434                       r'(?P<NO_CACHE>)' +
 435                       ')')
 436         self.cacheTokenRE = cachedRegex(cacheToken)
 437
 438         silentPlaceholderToken = (r'(?:' +
 439                                   r'(?P<SILENT>' +escapeRegexChars('!')+')'+
 440                                   '|' +
 441                                   r'(?P<NOT_SILENT>)' +
 442                                   ')')
 443         self.silentPlaceholderTokenRE = cachedRegex(silentPlaceholderToken)
 444
 445         self.cheetahVarStartRE = cachedRegex(
 446             escCharLookBehind +
 447             r'(?P<startToken>'+escapeRegexChars(self.setting('cheetahVarStartToken'))+')'+
 448             r'(?P<silenceToken>'+silentPlaceholderToken+')'+
 449             r'(?P<cacheToken>'+cacheToken+')'+
 450             r'(?P<enclosure>|(?:(?:\{|\(|\[)[ \t\f]*))' + # allow WS after enclosure
 451             r'(?=[A-Za-z_])')
 452         validCharsLookAhead = r'(?=[A-Za-z_\*!\{\(\[])'
 453         self.cheetahVarStartToken = self.setting('cheetahVarStartToken')
 454         self.cheetahVarStartTokenRE = cachedRegex(
 455             escCharLookBehind +
 456             escapeRegexChars(self.setting('cheetahVarStartToken'))
 457             +validCharsLookAhead
 458             )
 459
 460         self.cheetahVarInExpressionStartTokenRE = cachedRegex(
 461             escapeRegexChars(self.setting('cheetahVarStartToken'))
 462             +r'(?=[A-Za-z_])'
 463             )
 464
 465         self.expressionPlaceholderStartRE = cachedRegex(
 466             escCharLookBehind +
 467             r'(?P<startToken>' + escapeRegexChars(self.setting('cheetahVarStartToken')) + ')' +
 468             r'(?P<cacheToken>' + cacheToken + ')' +
 469             #r'\[[ \t\f]*'
 470             r'(?:\{|\(|\[)[ \t\f]*'
 471             + r'(?=[^\)\}\]])'
 472             )
 473
 474         if self.setting('EOLSlurpToken'):
 475             self.EOLSlurpRE = cachedRegex(
 476                 escapeRegexChars(self.setting('EOLSlurpToken'))
 477                 + r'[ \t\f]*'
 478                 + r'(?:'+EOL+')'
 479                 )
 480         else:
 481             self.EOLSlurpRE = None
 482
 483
 484     def _makeCommentREs(self):
 485         """Construct the regex bits that are used in comment parsing."""
 486         startTokenEsc = escapeRegexChars(self.setting('commentStartToken'))
 487         self.commentStartTokenRE = cachedRegex(escCharLookBehind + startTokenEsc)
 488         del startTokenEsc
 489
 490         startTokenEsc = escapeRegexChars(
 491             self.setting('multiLineCommentStartToken'))
 492         endTokenEsc = escapeRegexChars(
 493             self.setting('multiLineCommentEndToken'))
 494         self.multiLineCommentTokenStartRE = cachedRegex(escCharLookBehind +
 495                                                        startTokenEsc)
 496         self.multiLineCommentEndTokenRE = cachedRegex(escCharLookBehind +
 497                                                      endTokenEsc)
 498
 499     def _makeDirectiveREs(self):
 500         """Construct the regexs that are used in directive parsing."""
 501         startToken = self.setting('directiveStartToken')
 502         endToken = self.setting('directiveEndToken')
 503         startTokenEsc = escapeRegexChars(startToken)
 504         endTokenEsc = escapeRegexChars(endToken)
 505         validSecondCharsLookAhead = r'(?=[A-Za-z_@])'
 506         reParts = [escCharLookBehind, startTokenEsc]
 507         if self.setting('allowWhitespaceAfterDirectiveStartToken'):
 508             reParts.append('[ \t]*')
 509         reParts.append(validSecondCharsLookAhead)
 510         self.directiveStartTokenRE = cachedRegex(''.join(reParts))
 511         self.directiveEndTokenRE = cachedRegex(escCharLookBehind + endTokenEsc)
 512
 513     def _makePspREs(self):
 514         """Setup the regexs for PSP parsing."""
 515         startToken = self.setting('PSPStartToken')
 516         startTokenEsc = escapeRegexChars(startToken)
 517         self.PSPStartTokenRE = cachedRegex(escCharLookBehind + startTokenEsc)
 518         endToken = self.setting('PSPEndToken')
 519         endTokenEsc = escapeRegexChars(endToken)
 520         self.PSPEndTokenRE = cachedRegex(escCharLookBehind + endTokenEsc)
 521
 522
 523     def isLineClearToStartToken(self, pos=None):
 524         return self.isLineClearToPos(pos)
 525
 526     def matchTopLevelToken(self):
 527         """Returns the first match found from the following methods:
 528             self.matchCommentStartToken
 529             self.matchMultiLineCommentStartToken
 530             self.matchVariablePlaceholderStart
 531             self.matchExpressionPlaceholderStart
 532             self.matchDirective
 533             self.matchPSPStartToken
 534             self.matchEOLSlurpToken
 535
 536         Returns None if no match.
 537         """
 538         match = None
 539         if self.peek() in self._possibleNonStrConstantChars:
 540             for matcher in self._nonStrConstMatchers:
 541                 match = matcher()
 542                 if match:
 543                     break
 544         return match
 545
 546     def matchPyToken(self):
 547         match = pseudoprog.match(self.src(), self.pos())
 548
 549         if match and match.group() in tripleQuotedStringStarts:
 550             TQSmatch = tripleQuotedStringREs[match.group()].match(self.src(), self.pos())
 551             if TQSmatch:
 552                 return TQSmatch
 553         return match
 554
 555     def getPyToken(self):
 556         match = self.matchPyToken()
 557         if match is None:
 558             raise ParseError(self)
 559         elif match.group() in tripleQuotedStringStarts:
 560             raise ParseError(self, msg='Malformed triple-quoted string')
 561         return self.readTo(match.end())
 562
 563     def matchEOLSlurpToken(self):
 564         if self.EOLSlurpRE:
 565             return self.EOLSlurpRE.match(self.src(), self.pos())
 566
 567     def getEOLSlurpToken(self):
 568         match = self.matchEOLSlurpToken()
 569         if not match:
 570             raise ParseError(self, msg='Invalid EOL slurp token')
 571         return self.readTo(match.end())
 572
 573     def matchCommentStartToken(self):
 574         return self.commentStartTokenRE.match(self.src(), self.pos())
 575
 576     def getCommentStartToken(self):
 577         match = self.matchCommentStartToken()
 578         if not match:
 579             raise ParseError(self, msg='Invalid single-line comment start token')
 580         return self.readTo(match.end())
 581
 582     def matchMultiLineCommentStartToken(self):
 583         return self.multiLineCommentTokenStartRE.match(self.src(), self.pos())
 584
 585     def getMultiLineCommentStartToken(self):
 586         match = self.matchMultiLineCommentStartToken()
 587         if not match:
 588             raise ParseError(self, msg='Invalid multi-line comment start token')
 589         return self.readTo(match.end())
 590
 591     def matchMultiLineCommentEndToken(self):
 592         return self.multiLineCommentEndTokenRE.match(self.src(), self.pos())
 593
 594     def getMultiLineCommentEndToken(self):
 595         match = self.matchMultiLineCommentEndToken()
 596         if not match:
 597             raise ParseError(self, msg='Invalid multi-line comment end token')
 598         return self.readTo(match.end())
 599
 600     def getDottedName(self):
 601         srcLen = len(self)
 602         nameChunks = []
 603
 604         if not self.peek() in identchars:
 605             raise ParseError(self)
 606
 607         while self.pos() < srcLen:
 608             c = self.peek()
 609             if c in namechars:
 610                 nameChunk = self.getIdentifier()
 611                 nameChunks.append(nameChunk)
 612             elif c == '.':
 613                 if self.pos()+1 <srcLen and self.peek(1) in identchars:
 614                     nameChunks.append(self.getc())
 615                 else:
 616                     break
 617             else:
 618                 break
 619
 620         return ''.join(nameChunks)
 621
 622     def matchIdentifier(self):
 623         return identRE.match(self.src(), self.pos())
 624
 625     def getIdentifier(self):
 626         match = self.matchIdentifier()
 627         if not match:
 628             raise ParseError(self, msg='Invalid identifier')
 629         return self.readTo(match.end())
 630
 631     def matchOperator(self):
 632         match = self.matchPyToken()
 633         if match and match.group() not in operators:
 634             match = None
 635         return match
 636
 637     def getOperator(self):
 638         match = self.matchOperator()
 639         if not match:
 640             raise ParseError(self, msg='Expected operator')
 641         return self.readTo( match.end() )
 642
 643     def matchAssignmentOperator(self):
 644         match = self.matchPyToken()
 645         if match and match.group() not in assignmentOps:
 646             match = None
 647         return match
 648
 649     def getAssignmentOperator(self):
 650         match = self.matchAssignmentOperator()
 651         if not match:
 652             raise ParseError(self, msg='Expected assignment operator')
 653         return self.readTo( match.end() )
 654
 655     def matchDirective(self):
 656         """Returns False or the name of the directive matched.
 657         """
 658         startPos = self.pos()
 659         if not self.matchDirectiveStartToken():
 660             return False
 661         self.getDirectiveStartToken()
 662         directiveName = self.matchDirectiveName()
 663         self.setPos(startPos)
 664         return directiveName
 665
 666     def matchDirectiveName(self, directiveNameChars=identchars+'0123456789-@'):
 667         startPos = self.pos()
 668         possibleMatches = self._directiveNamesAndParsers.keys()
 669         name = ''
 670         match = None
 671
 672         while not self.atEnd():
 673             c = self.getc()
 674             if not c in directiveNameChars:
 675                 break
 676             name += c
 677             if name == '@':
 678                 if not self.atEnd() and self.peek() in identchars:
 679                     match = '@'
 680                 break
 681             possibleMatches = [dn for dn in possibleMatches if dn.startswith(name)]
 682             if not possibleMatches:
 683                 break
 684             elif (name in possibleMatches and (self.atEnd() or self.peek() not in directiveNameChars)):
 685                 match = name
 686                 break
 687
 688         self.setPos(startPos)
 689         return match
 690
 691     def matchDirectiveStartToken(self):
 692         return self.directiveStartTokenRE.match(self.src(), self.pos())
 693
 694     def getDirectiveStartToken(self):
 695         match = self.matchDirectiveStartToken()
 696         if not match:
 697             raise ParseError(self, msg='Invalid directive start token')
 698         return self.readTo(match.end())
 699
 700     def matchDirectiveEndToken(self):
 701         return self.directiveEndTokenRE.match(self.src(), self.pos())
 702
 703     def getDirectiveEndToken(self):
 704         match = self.matchDirectiveEndToken()
 705         if not match:
 706             raise ParseError(self, msg='Invalid directive end token')
 707         return self.readTo(match.end())
 708
 709
 710     def matchColonForSingleLineShortFormDirective(self):
 711         if not self.atEnd() and self.peek()==':':
 712             restOfLine = self[self.pos()+1:self.findEOL()]
 713             restOfLine = restOfLine.strip()
 714             if not restOfLine:
 715                 return False
 716             elif self.commentStartTokenRE.match(restOfLine):
 717                 return False
 718             else: # non-whitespace, non-commment chars found
 719                 return True
 720         return False
 721
 722     def matchPSPStartToken(self):
 723         return self.PSPStartTokenRE.match(self.src(), self.pos())
 724
 725     def matchPSPEndToken(self):
 726         return self.PSPEndTokenRE.match(self.src(), self.pos())
 727
 728     def getPSPStartToken(self):
 729         match = self.matchPSPStartToken()
 730         if not match:
 731             raise ParseError(self, msg='Invalid psp start token')
 732         return self.readTo(match.end())
 733
 734     def getPSPEndToken(self):
 735         match = self.matchPSPEndToken()
 736         if not match:
 737             raise ParseError(self, msg='Invalid psp end token')
 738         return self.readTo(match.end())
 739
 740     def matchCheetahVarStart(self):
 741         """includes the enclosure and cache token"""
 742         return self.cheetahVarStartRE.match(self.src(), self.pos())
 743
 744     def matchCheetahVarStartToken(self):
 745         """includes the enclosure and cache token"""
 746         return self.cheetahVarStartTokenRE.match(self.src(), self.pos())
 747
 748     def matchCheetahVarInExpressionStartToken(self):
 749         """no enclosures or cache tokens allowed"""
 750         return self.cheetahVarInExpressionStartTokenRE.match(self.src(), self.pos())
 751
 752     def matchVariablePlaceholderStart(self):
 753         """includes the enclosure and cache token"""
 754         return self.cheetahVarStartRE.match(self.src(), self.pos())
 755
 756     def matchExpressionPlaceholderStart(self):
 757         """includes the enclosure and cache token"""
 758         return self.expressionPlaceholderStartRE.match(self.src(), self.pos())
 759
 760     def getCheetahVarStartToken(self):
 761         """just the start token, not the enclosure or cache token"""
 762         match = self.matchCheetahVarStartToken()
 763         if not match:
 764             raise ParseError(self, msg='Expected Cheetah $var start token')
 765         return self.readTo( match.end() )
 766
 767
 768     def getCacheToken(self):
 769         try:
 770             token = self.cacheTokenRE.match(self.src(), self.pos())
 771             self.setPos( token.end() )
 772             return token.group()
 773         except:
 774             raise ParseError(self, msg='Expected cache token')
 775
 776     def getSilentPlaceholderToken(self):
 777         try:
 778             token = self.silentPlaceholderTokenRE.match(self.src(), self.pos())
 779             self.setPos( token.end() )
 780             return token.group()
 781         except:
 782             raise ParseError(self, msg='Expected silent placeholder token')
 783
 784
 785
 786     def getTargetVarsList(self):
 787         varnames = []
 788         while not self.atEnd():
 789             if self.peek() in ' \t\f':
 790                 self.getWhiteSpace()
 791             elif self.peek() in '\r\n':
 792                 break
 793             elif self.startswith(','):
 794                 self.advance()
 795             elif self.startswith('in ') or self.startswith('in\t'):
 796                 break
 797             #elif self.matchCheetahVarStart():
 798             elif self.matchCheetahVarInExpressionStartToken():
 799                 self.getCheetahVarStartToken()
 800                 self.getSilentPlaceholderToken()
 801                 self.getCacheToken()
 802                 varnames.append( self.getDottedName() )
 803             elif self.matchIdentifier():
 804                 varnames.append( self.getDottedName() )
 805             else:
 806                 break
 807         return varnames
 808
 809     def getCheetahVar(self, plain=False, skipStartToken=False):
 810         """This is called when parsing inside expressions. Cache tokens are only
 811         valid in placeholders so this method discards any cache tokens found.
 812         """
 813         if not skipStartToken:
 814             self.getCheetahVarStartToken()
 815         self.getSilentPlaceholderToken()
 816         self.getCacheToken()
 817         return self.getCheetahVarBody(plain=plain)
 818
 819     def getCheetahVarBody(self, plain=False):
 820         # @@TR: this should be in the compiler
 821         return self._compiler.genCheetahVar(self.getCheetahVarNameChunks(), plain=plain)
 822
 823     def getCheetahVarNameChunks(self):
 824
 825         """
 826         nameChunks = list of Cheetah $var subcomponents represented as tuples
 827           [ (namemapperPart,autoCall,restOfName),
 828           ]
 829         where:
 830           namemapperPart = the dottedName base
 831           autocall = where NameMapper should use autocalling on namemapperPart
 832           restOfName = any arglist, index, or slice
 833
 834         If restOfName contains a call arglist (e.g. '(1234)') then autocall is
 835         False, otherwise it defaults to True.
 836
 837         EXAMPLE
 838         ------------------------------------------------------------------------
 839
 840         if the raw CheetahVar is
 841           $a.b.c[1].d().x.y.z
 842
 843         nameChunks is the list
 844           [ ('a.b.c',True,'[1]'),
 845             ('d',False,'()'),
 846             ('x.y.z',True,''),
 847           ]
 848
 849         """
 850
 851         chunks = []
 852         while self.pos() < len(self):
 853             rest = ''
 854             autoCall = True
 855             if not self.peek() in identchars + '.':
 856                 break
 857             elif self.peek() == '.':
 858
 859                 if self.pos()+1 < len(self) and self.peek(1) in identchars:
 860                     self.advance()  # discard the period as it isn't needed with NameMapper
 861                 else:
 862                     break
 863
 864             dottedName = self.getDottedName()
 865             if not self.atEnd() and self.peek() in '([':
 866                 if self.peek() == '(':
 867                     rest = self.getCallArgString()
 868                 else:
 869                     rest = self.getExpression(enclosed=True)
 870
 871                 period = max(dottedName.rfind('.'), 0)
 872                 if period:
 873                     chunks.append( (dottedName[:period], autoCall, '') )
 874                     dottedName = dottedName[period+1:]
 875                 if rest and rest[0]=='(':
 876                     autoCall = False
 877             chunks.append( (dottedName, autoCall, rest) )
 878
 879         return chunks
 880
 881
 882     def getCallArgString(self,
 883                          enclosures=[],  # list of tuples (char, pos), where char is ({ or [
 884                          useNameMapper=Unspecified):
 885
 886         """ Get a method/function call argument string.
 887
 888         This method understands *arg, and **kw
 889         """
 890
 891         # @@TR: this settings mangling should be removed
 892         if useNameMapper is not Unspecified:
 893             useNameMapper_orig = self.setting('useNameMapper')
 894             self.setSetting('useNameMapper', useNameMapper)
 895
 896         if enclosures:
 897             pass
 898         else:
 899             if not self.peek() == '(':
 900                 raise ParseError(self, msg="Expected '('")
 901             startPos = self.pos()
 902             self.getc()
 903             enclosures = [('(', startPos),
 904                           ]
 905
 906         argStringBits = ['(']
 907         addBit = argStringBits.append
 908
 909         while 1:
 910             if self.atEnd():
 911                 open = enclosures[-1][0]
 912                 close = closurePairsRev[open]
 913                 self.setPos(enclosures[-1][1])
 914                 raise ParseError(
 915                     self, msg="EOF was reached before a matching '" + close +
 916                     "' was found for the '" + open + "'")
 917
 918             c = self.peek()
 919             if c in ")}]": # get the ending enclosure and break
 920                 if not enclosures:
 921                     raise ParseError(self)
 922                 c = self.getc()
 923                 open = closurePairs[c]
 924                 if enclosures[-1][0] == open:
 925                     enclosures.pop()
 926                     addBit(')')
 927                     break
 928                 else:
 929                     raise ParseError(self)
 930             elif c in " \t\f\r\n":
 931                 addBit(self.getc())
 932             elif self.matchCheetahVarInExpressionStartToken():
 933                 startPos = self.pos()
 934                 codeFor1stToken = self.getCheetahVar()
 935                 WS = self.getWhiteSpace()
 936                 if not self.atEnd() and self.peek() == '=':
 937                     nextToken = self.getPyToken()
 938                     if nextToken == '=':
 939                         endPos = self.pos()
 940                         self.setPos(startPos)
 941                         codeFor1stToken = self.getCheetahVar(plain=True)
 942                         self.setPos(endPos)
 943
 944                     ## finally
 945                     addBit( codeFor1stToken + WS + nextToken )
 946                 else:
 947                     addBit( codeFor1stToken + WS)
 948             elif self.matchCheetahVarStart():
 949                 # it has syntax that is only valid at the top level
 950                 self._raiseErrorAboutInvalidCheetahVarSyntaxInExpr()
 951             else:
 952                 beforeTokenPos = self.pos()
 953                 token = self.getPyToken()
 954                 if token in ('{','(','['):
 955                     self.rev()
 956                     token = self.getExpression(enclosed=True)
 957                 token = self.transformToken(token, beforeTokenPos)
 958                 addBit(token)
 959
 960         if useNameMapper is not Unspecified:
 961             self.setSetting('useNameMapper', useNameMapper_orig) # @@TR: see comment above
 962
 963         return ''.join(argStringBits)
 964
 965     def getDefArgList(self, exitPos=None, useNameMapper=False):
 966
 967         """ Get an argument list. Can be used for method/function definition
 968         argument lists or for #directive argument lists. Returns a list of
 969         tuples in the form (argName, defVal=None) with one tuple for each arg
 970         name.
 971
 972         These defVals are always strings, so (argName, defVal=None) is safe even
 973         with a case like (arg1, arg2=None, arg3=1234*2), which would be returned as
 974         [('arg1', None),
 975          ('arg2', 'None'),
 976          ('arg3', '1234*2'),
 977         ]
 978
 979         This method understands *arg, and **kw
 980
 981         """
 982
 983         if self.peek() == '(':
 984             self.advance()
 985         else:
 986             exitPos = self.findEOL()  # it's a directive so break at the EOL
 987         argList = ArgList()
 988         onDefVal = False
 989
 990         # @@TR: this settings mangling should be removed
 991         useNameMapper_orig = self.setting('useNameMapper')
 992         self.setSetting('useNameMapper', useNameMapper)
 993
 994         while 1:
 995             if self.atEnd():
 996                 raise ParseError(
 997                     self, msg="EOF was reached before a matching ')'"+
 998                     " was found for the '('")
 999
1000             if self.pos() == exitPos:
1001                 break
1002
1003             c = self.peek()
1004             if c == ")" or self.matchDirectiveEndToken():
1005                 break
1006             elif c == ":":
1007                 break
1008             elif c in " \t\f\r\n":
1009                 if onDefVal:
1010                     argList.addToDefVal(c)
1011                 self.advance()
1012             elif c == '=':
1013                 onDefVal = True
1014                 self.advance()
1015             elif c == ",":
1016                 argList.next()
1017                 onDefVal = False
1018                 self.advance()
1019             elif self.startswith(self.cheetahVarStartToken) and not onDefVal:
1020                 self.advance(len(self.cheetahVarStartToken))
1021             elif self.matchIdentifier() and not onDefVal:
1022                 argList.addArgName( self.getIdentifier() )
1023             elif onDefVal:
1024                 if self.matchCheetahVarInExpressionStartToken():
1025                     token = self.getCheetahVar()
1026                 elif self.matchCheetahVarStart():
1027                     # it has syntax that is only valid at the top level
1028                     self._raiseErrorAboutInvalidCheetahVarSyntaxInExpr()
1029                 else:
1030                     beforeTokenPos = self.pos()
1031                     token = self.getPyToken()
1032                     if token in ('{','(','['):
1033                         self.rev()
1034                         token = self.getExpression(enclosed=True)
1035                     token = self.transformToken(token, beforeTokenPos)
1036                 argList.addToDefVal(token)
1037             elif c == '*' and not onDefVal:
1038                 varName = self.getc()
1039                 if self.peek() == '*':
1040                     varName += self.getc()
1041                 if not self.matchIdentifier():
1042                     raise ParseError(self)
1043                 varName += self.getIdentifier()
1044                 argList.addArgName(varName)
1045             else:
1046                 raise ParseError(self)
1047
1048
1049         self.setSetting('useNameMapper', useNameMapper_orig) # @@TR: see comment above
1050         return argList.merge()
1051
1052     def getExpressionParts(self,
1053                            enclosed=False,
1054                            enclosures=None, # list of tuples (char, pos), where char is ({ or [
1055                            pyTokensToBreakAt=None, # only works if not enclosed
1056                            useNameMapper=Unspecified,
1057                            ):
1058
1059         """ Get a Cheetah expression that includes $CheetahVars and break at
1060         directive end tokens, the end of an enclosure, or at a specified
1061         pyToken.
1062         """
1063
1064         if useNameMapper is not Unspecified:
1065             useNameMapper_orig = self.setting('useNameMapper')
1066             self.setSetting('useNameMapper', useNameMapper)
1067
1068         if enclosures is None:
1069             enclosures = []
1070
1071         srcLen = len(self)
1072         exprBits = []
1073         while 1:
1074             if self.atEnd():
1075                 if enclosures:
1076                     open = enclosures[-1][0]
1077                     close = closurePairsRev[open]
1078                     self.setPos(enclosures[-1][1])
1079                     raise ParseError(
1080                         self, msg="EOF was reached before a matching '" + close +
1081                         "' was found for the '" + open + "'")
1082                 else:
1083                     break
1084
1085             c = self.peek()
1086             if c in "{([":
1087                 exprBits.append(c)
1088                 enclosures.append( (c, self.pos()) )
1089                 self.advance()
1090             elif enclosed and not enclosures:
1091                 break
1092             elif c in "])}":
1093                 if not enclosures:
1094                     raise ParseError(self)
1095                 open = closurePairs[c]
1096                 if enclosures[-1][0] == open:
1097                     enclosures.pop()
1098                     exprBits.append(c)
1099                 else:
1100                     open = enclosures[-1][0]
1101                     close = closurePairsRev[open]
1102                     row, col = self.getRowCol()
1103                     self.setPos(enclosures[-1][1])
1104                     raise ParseError(
1105                         self, msg= "A '" + c + "' was found at line " + str(row) +
1106                         ", col " + str(col) +
1107                         " before a matching '" + close +
1108                         "' was found\nfor the '" + open + "'")
1109                 self.advance()
1110
1111             elif c in " \f\t":
1112                 exprBits.append(self.getWhiteSpace())
1113             elif self.matchDirectiveEndToken() and not enclosures:
1114                 break
1115             elif c == "\\" and self.pos()+1 < srcLen:
1116                 eolMatch = EOLre.match(self.src(), self.pos()+1)
1117                 if not eolMatch:
1118                     self.advance()
1119                     raise ParseError(self, msg='Line ending expected')
1120                 self.setPos( eolMatch.end() )
1121             elif c in '\r\n':
1122                 if enclosures:
1123                     self.advance()
1124                 else:
1125                     break
1126             elif self.matchCheetahVarInExpressionStartToken():
1127                 expr = self.getCheetahVar()
1128                 exprBits.append(expr)
1129             elif self.matchCheetahVarStart():
1130                 # it has syntax that is only valid at the top level
1131                 self._raiseErrorAboutInvalidCheetahVarSyntaxInExpr()
1132             else:
1133                 beforeTokenPos = self.pos()
1134                 token = self.getPyToken()
1135                 if (not enclosures
1136                     and pyTokensToBreakAt
1137                     and token in pyTokensToBreakAt):
1138
1139                     self.setPos(beforeTokenPos)
1140                     break
1141
1142                 token = self.transformToken(token, beforeTokenPos)
1143
1144                 exprBits.append(token)
1145                 if identRE.match(token):
1146                     if token == 'for':
1147                         expr = self.getExpression(useNameMapper=False, pyTokensToBreakAt=['in'])
1148                         exprBits.append(expr)
1149                     else:
1150                         exprBits.append(self.getWhiteSpace())
1151                         if not self.atEnd() and self.peek() == '(':
1152                             exprBits.append(self.getCallArgString())
1153         ##
1154         if useNameMapper is not Unspecified:
1155             self.setSetting('useNameMapper', useNameMapper_orig) # @@TR: see comment above
1156         return exprBits
1157
1158     def getExpression(self,
1159                       enclosed=False,
1160                       enclosures=None, # list of tuples (char, pos), where # char is ({ or [
1161                       pyTokensToBreakAt=None,
1162                       useNameMapper=Unspecified,
1163                       ):
1164         """Returns the output of self.getExpressionParts() as a concatenated
1165         string rather than as a list.
1166         """
1167         return ''.join(self.getExpressionParts(
1168             enclosed=enclosed, enclosures=enclosures,
1169             pyTokensToBreakAt=pyTokensToBreakAt,
1170             useNameMapper=useNameMapper))
1171
1172
1173     def transformToken(self, token, beforeTokenPos):
1174         """Takes a token from the expression being parsed and performs and
1175         special transformations required by Cheetah.
1176
1177         At the moment only Cheetah's c'$placeholder strings' are transformed.
1178         """
1179         if token=='c' and not self.atEnd() and self.peek() in '\'"':
1180             nextToken = self.getPyToken()
1181             token = nextToken.upper()
1182             theStr = eval(token)
1183             endPos = self.pos()
1184             if not theStr:
1185                 return
1186
1187             if token.startswith(single3) or token.startswith(double3):
1188                 startPosIdx = 3
1189             else:
1190                 startPosIdx = 1
1191             #print 'CHEETAH STRING', nextToken, theStr, startPosIdx
1192             self.setPos(beforeTokenPos+startPosIdx+1)
1193             outputExprs = []
1194             strConst = ''
1195             while self.pos() < (endPos-startPosIdx):
1196                 if self.matchCheetahVarStart() or self.matchExpressionPlaceholderStart():
1197                     if strConst:
1198                         outputExprs.append(repr(strConst))
1199                         strConst = ''
1200                     placeholderExpr = self.getPlaceholder()
1201                     outputExprs.append('str('+placeholderExpr+')')
1202                 else:
1203                     strConst += self.getc()
1204             self.setPos(endPos)
1205             if strConst:
1206                 outputExprs.append(repr(strConst))
1207             #if not self.atEnd() and self.matches('.join('):
1208             #    print 'DEBUG***'
1209             token = "''.join(["+','.join(outputExprs)+"])"
1210         return token
1211
1212     def _raiseErrorAboutInvalidCheetahVarSyntaxInExpr(self):
1213         match = self.matchCheetahVarStart()
1214         groupdict = match.groupdict()
1215         if groupdict.get('cacheToken'):
1216             raise ParseError(
1217                 self,
1218                 msg='Cache tokens are not valid inside expressions. '
1219                 'Use them in top-level $placeholders only.')
1220         elif groupdict.get('enclosure'):
1221             raise ParseError(
1222                 self,
1223                 msg='Long-form placeholders - ${}, $(), $[], etc. are not valid inside expressions. '
1224                 'Use them in top-level $placeholders only.')
1225         else:
1226             raise ParseError(
1227                 self,
1228                 msg='This form of $placeholder syntax is not valid here.')
1229
1230
1231     def getPlaceholder(self, allowCacheTokens=False, plain=False, returnEverything=False):
1232         # filtered
1233         for callback in self.setting('preparsePlaceholderHooks'):
1234             callback(parser=self)
1235
1236         startPos = self.pos()
1237         lineCol = self.getRowCol(startPos)
1238         startToken = self.getCheetahVarStartToken()
1239         silentPlaceholderToken = self.getSilentPlaceholderToken()
1240         if silentPlaceholderToken:
1241             isSilentPlaceholder = True
1242         else:
1243             isSilentPlaceholder = False
1244
1245
1246         if allowCacheTokens:
1247             cacheToken = self.getCacheToken()
1248             cacheTokenParts = self.cacheTokenRE.match(cacheToken).groupdict()
1249         else:
1250             cacheTokenParts = {}
1251
1252         if self.peek() in '({[':
1253             pos = self.pos()
1254             enclosureOpenChar = self.getc()
1255             enclosures = [ (enclosureOpenChar, pos) ]
1256             self.getWhiteSpace()
1257         else:
1258             enclosures = []
1259
1260         filterArgs = None
1261         if self.matchIdentifier():
1262             nameChunks = self.getCheetahVarNameChunks()
1263             expr = self._compiler.genCheetahVar(nameChunks[:], plain=plain)
1264             restOfExpr = None
1265             if enclosures:
1266                 WS = self.getWhiteSpace()
1267                 expr += WS
1268                 if self.setting('allowPlaceholderFilterArgs') and self.peek()==',':
1269                     filterArgs = self.getCallArgString(enclosures=enclosures)[1:-1]
1270                 else:
1271                     if self.peek()==closurePairsRev[enclosureOpenChar]:
1272                         self.getc()
1273                     else:
1274                         restOfExpr = self.getExpression(enclosed=True, enclosures=enclosures)
1275                         if restOfExpr[-1] == closurePairsRev[enclosureOpenChar]:
1276                             restOfExpr = restOfExpr[:-1]
1277                         expr += restOfExpr
1278             rawPlaceholder = self[startPos: self.pos()]
1279         else:
1280             expr = self.getExpression(enclosed=True, enclosures=enclosures)
1281             if expr[-1] == closurePairsRev[enclosureOpenChar]:
1282                 expr = expr[:-1]
1283             rawPlaceholder=self[startPos: self.pos()]
1284
1285         expr = self._applyExpressionFilters(expr,'placeholder',
1286                                             rawExpr=rawPlaceholder,startPos=startPos)
1287         for callback in self.setting('postparsePlaceholderHooks'):
1288             callback(parser=self)
1289
1290         if returnEverything:
1291             return (expr, rawPlaceholder, lineCol, cacheTokenParts,
1292                     filterArgs, isSilentPlaceholder)
1293         else:
1294             return expr
1295
1296
1297 class _HighLevelParser(_LowLevelParser):
1298     """This class is a StateMachine for parsing Cheetah source and
1299     sending state dependent code generation commands to
1300     Cheetah.Compiler.Compiler.
1301     """
1302     def __init__(self, src, filename=None, breakPoint=None, compiler=None):
1303         _LowLevelParser.__init__(self, src, filename=filename, breakPoint=breakPoint)
1304         self.setSettingsManager(compiler)
1305         self._compiler = compiler
1306         self.setupState()
1307         self.configureParser()
1308
1309     def setupState(self):
1310         self._macros = {}
1311         self._macroDetails = {}
1312         self._openDirectivesStack = []
1313
1314     def cleanup(self):
1315         """Cleanup to remove any possible reference cycles
1316         """
1317         self._macros.clear()
1318         for macroname, macroDetails in self._macroDetails.items():
1319             macroDetails.template.shutdown()
1320             del macroDetails.template
1321         self._macroDetails.clear()
1322
1323     def configureParser(self):
1324         _LowLevelParser.configureParser(self)
1325         self._initDirectives()
1326
1327     def _initDirectives(self):
1328         def normalizeParserVal(val):
1329             if isinstance(val, (str,unicode)):
1330                 handler = getattr(self, val)
1331             elif type(val) in (ClassType, TypeType):
1332                 handler = val(self)
1333             elif callable(val):
1334                 handler = val
1335             elif val is None:
1336                 handler = val
1337             else:
1338                 raise Exception('Invalid parser/handler value %r for %s'%(val, name))
1339             return handler
1340
1341         normalizeHandlerVal = normalizeParserVal
1342
1343         _directiveNamesAndParsers = directiveNamesAndParsers.copy()
1344         customNamesAndParsers = self.setting('directiveNamesAndParsers',{})
1345         _directiveNamesAndParsers.update(customNamesAndParsers)
1346
1347         _endDirectiveNamesAndHandlers = endDirectiveNamesAndHandlers.copy()
1348         customNamesAndHandlers = self.setting('endDirectiveNamesAndHandlers',{})
1349         _endDirectiveNamesAndHandlers.update(customNamesAndHandlers)
1350
1351         self._directiveNamesAndParsers = {}
1352         for name, val in _directiveNamesAndParsers.items():
1353             if val in (False, 0):
1354                 continue
1355             self._directiveNamesAndParsers[name] = normalizeParserVal(val)
1356
1357         self._endDirectiveNamesAndHandlers = {}
1358         for name, val in _endDirectiveNamesAndHandlers.items():
1359             if val in (False, 0):
1360                 continue
1361             self._endDirectiveNamesAndHandlers[name] = normalizeHandlerVal(val)
1362
1363         self._closeableDirectives = ['def','block','closure','defmacro',
1364                                      'call',
1365                                      'capture',
1366                                      'cache',
1367                                      'filter',
1368                                      'if','unless',
1369                                      'for','while','repeat',
1370                                      'try',
1371                                      ]
1372         for directiveName in self.setting('closeableDirectives',[]):
1373             self._closeableDirectives.append(directiveName)
1374
1375
1376
1377         macroDirectives = self.setting('macroDirectives',{})
1378         macroDirectives['i18n'] = I18n
1379
1380
1381         for macroName, callback in macroDirectives.items():
1382             if type(callback) in (ClassType, TypeType):
1383                 callback = callback(parser=self)
1384             assert callback
1385             self._macros[macroName] = callback
1386             self._directiveNamesAndParsers[macroName] = self.eatMacroCall
1387
1388     def _applyExpressionFilters(self, expr, exprType, rawExpr=None, startPos=None):
1389         """Pipes cheetah expressions through a set of optional filter hooks.
1390
1391         The filters are functions which may modify the expressions or raise
1392         a ForbiddenExpression exception if the expression is not allowed.  They
1393         are defined in the compiler setting 'expressionFilterHooks'.
1394
1395         Some intended use cases:
1396
1397          - to implement 'restricted execution' safeguards in cases where you
1398            can't trust the author of the template.
1399
1400          - to enforce style guidelines
1401
1402         filter call signature:  (parser, expr, exprType, rawExpr=None, startPos=None)
1403          - parser is the Cheetah parser
1404          - expr is the expression to filter.  In some cases the parser will have
1405            already modified it from the original source code form.  For example,
1406            placeholders will have been translated into namemapper calls.  If you
1407            need to work with the original source, see rawExpr.
1408          - exprType is the name of the directive, 'psp', or 'placeholder'. All
1409            lowercase.  @@TR: These will eventually be replaced with a set of
1410            constants.
1411          - rawExpr is the original source string that Cheetah parsed.  This
1412            might be None in some cases.
1413          - startPos is the character position in the source string/file
1414            where the parser started parsing the current expression.
1415
1416         @@TR: I realize this use of the term 'expression' is a bit wonky as many
1417          of the 'expressions' are actually statements, but I haven't thought of
1418          a better name yet.  Suggestions?
1419         """
1420         for callback in self.setting('expressionFilterHooks'):
1421             expr = callback(parser=self, expr=expr,  exprType=exprType,
1422                             rawExpr=rawExpr, startPos=startPos)
1423         return expr
1424
1425     def _filterDisabledDirectives(self, directiveName):
1426         directiveName = directiveName.lower()
1427         if (directiveName in self.setting('disabledDirectives')
1428             or (self.setting('enabledDirectives')
1429                 and directiveName not in self.setting('enabledDirectives'))):
1430             for callback in self.setting('disabledDirectiveHooks'):
1431                 callback(parser=self, directiveName=directiveName)
1432             raise ForbiddenDirective(self, msg='This %r directive is disabled'%directiveName)
1433
1434     ## main parse loop
1435
1436     def parse(self, breakPoint=None, assertEmptyStack=True):
1437         if breakPoint:
1438             origBP = self.breakPoint()
1439             self.setBreakPoint(breakPoint)
1440             assertEmptyStack = False
1441
1442         while not self.atEnd():
1443             if self.matchCommentStartToken():
1444                 self.eatComment()
1445             elif self.matchMultiLineCommentStartToken():
1446                 self.eatMultiLineComment()
1447             elif self.matchVariablePlaceholderStart():
1448                 self.eatPlaceholder()
1449             elif self.matchExpressionPlaceholderStart():
1450                 self.eatPlaceholder()
1451             elif self.matchDirective():
1452                 self.eatDirective()
1453             elif self.matchPSPStartToken():
1454                 self.eatPSP()
1455             elif self.matchEOLSlurpToken():
1456                 self.eatEOLSlurpToken()
1457             else:
1458                 self.eatPlainText()
1459         if assertEmptyStack:
1460             self.assertEmptyOpenDirectivesStack()
1461         if breakPoint:
1462             self.setBreakPoint(origBP)
1463
1464     ## non-directive eat methods
1465
1466     def eatPlainText(self):
1467         startPos = self.pos()
1468         match = None
1469         while not self.atEnd():
1470             match = self.matchTopLevelToken()
1471             if match:
1472                 break
1473             else:
1474                 self.advance()
1475         strConst = self.readTo(self.pos(), start=startPos)
1476         self._compiler.addStrConst(strConst)
1477         return match
1478
1479     def eatComment(self):
1480         isLineClearToStartToken = self.isLineClearToStartToken()
1481         if isLineClearToStartToken:
1482             self._compiler.handleWSBeforeDirective()
1483         self.getCommentStartToken()
1484         comm = self.readToEOL(gobble=isLineClearToStartToken)
1485         self._compiler.addComment(comm)
1486
1487     def eatMultiLineComment(self):
1488         isLineClearToStartToken = self.isLineClearToStartToken()
1489         endOfFirstLine = self.findEOL()
1490
1491         self.getMultiLineCommentStartToken()
1492         endPos = startPos = self.pos()
1493         level = 1
1494         while 1:
1495             endPos = self.pos()
1496             if self.atEnd():
1497                 break
1498             if self.matchMultiLineCommentStartToken():
1499                 self.getMultiLineCommentStartToken()
1500                 level += 1
1501             elif self.matchMultiLineCommentEndToken():
1502                 self.getMultiLineCommentEndToken()
1503                 level -= 1
1504             if not level:
1505                 break
1506             self.advance()
1507         comm = self.readTo(endPos, start=startPos)
1508
1509         if not self.atEnd():
1510             self.getMultiLineCommentEndToken()
1511
1512         if (not self.atEnd()) and self.setting('gobbleWhitespaceAroundMultiLineComments'):
1513             restOfLine = self[self.pos():self.findEOL()]
1514             if not restOfLine.strip(): # WS only to EOL
1515                 self.readToEOL(gobble=isLineClearToStartToken)
1516
1517             if isLineClearToStartToken and (self.atEnd() or self.pos() > endOfFirstLine):
1518                 self._compiler.handleWSBeforeDirective()
1519
1520         self._compiler.addComment(comm)
1521
1522     def eatPlaceholder(self):
1523         (expr, rawPlaceholder,
1524          lineCol, cacheTokenParts,
1525          filterArgs, isSilentPlaceholder) = self.getPlaceholder(
1526             allowCacheTokens=True, returnEverything=True)
1527
1528         self._compiler.addPlaceholder(
1529             expr,
1530             filterArgs=filterArgs,
1531             rawPlaceholder=rawPlaceholder,
1532             cacheTokenParts=cacheTokenParts,
1533             lineCol=lineCol,
1534             silentMode=isSilentPlaceholder)
1535         return
1536
1537     def eatPSP(self):
1538         # filtered
1539         self._filterDisabledDirectives(directiveName='psp')
1540         self.getPSPStartToken()
1541         endToken = self.setting('PSPEndToken')
1542         startPos = self.pos()
1543         while not self.atEnd():
1544             if self.peek() == endToken[0]:
1545                 if self.matchPSPEndToken():
1546                     break
1547             self.advance()
1548         pspString = self.readTo(self.pos(), start=startPos).strip()
1549         pspString = self._applyExpressionFilters(pspString, 'psp', startPos=startPos)
1550         self._compiler.addPSP(pspString)
1551         self.getPSPEndToken()
1552
1553     ## generic directive eat methods
1554     _simpleIndentingDirectives = '''
1555     else elif for while repeat unless try except finally'''.split()
1556     _simpleExprDirectives = '''
1557     pass continue stop return yield break
1558     del assert raise
1559     silent echo
1560     import from'''.split()
1561     _directiveHandlerNames = {'import':'addImportStatement',
1562                               'from':'addImportStatement', }
1563     def eatDirective(self):
1564         directiveName = self.matchDirective()
1565         self._filterDisabledDirectives(directiveName)
1566
1567         for callback in self.setting('preparseDirectiveHooks'):
1568             callback(parser=self, directiveName=directiveName)
1569
1570         # subclasses can override the default behaviours here by providing an
1571         # eater method in self._directiveNamesAndParsers[directiveName]
1572         directiveParser = self._directiveNamesAndParsers.get(directiveName)
1573         if directiveParser:
1574             directiveParser()
1575         elif directiveName in self._simpleIndentingDirectives:
1576             handlerName = self._directiveHandlerNames.get(directiveName)
1577             if not handlerName:
1578                 handlerName = 'add'+directiveName.capitalize()
1579             handler = getattr(self._compiler, handlerName)
1580             self.eatSimpleIndentingDirective(directiveName, callback=handler)
1581         elif directiveName in self._simpleExprDirectives:
1582             handlerName = self._directiveHandlerNames.get(directiveName)
1583             if not handlerName:
1584                 handlerName = 'add'+directiveName.capitalize()
1585             handler = getattr(self._compiler, handlerName)
1586             if directiveName in ('silent', 'echo'):
1587                 includeDirectiveNameInExpr = False
1588             else:
1589                 includeDirectiveNameInExpr = True
1590             expr = self.eatSimpleExprDirective(
1591                 directiveName,
1592                 includeDirectiveNameInExpr=includeDirectiveNameInExpr)
1593             handler(expr)
1594         ##
1595         for callback in self.setting('postparseDirectiveHooks'):
1596             callback(parser=self, directiveName=directiveName)
1597
1598     def _eatRestOfDirectiveTag(self, isLineClearToStartToken, endOfFirstLinePos):
1599         foundComment = False
1600         if self.matchCommentStartToken():
1601             pos = self.pos()
1602             self.advance()
1603             if not self.matchDirective():
1604                 self.setPos(pos)
1605                 foundComment = True
1606                 self.eatComment() # this won't gobble the EOL
1607             else:
1608                 self.setPos(pos)
1609
1610         if not foundComment and self.matchDirectiveEndToken():
1611                 self.getDirectiveEndToken()
1612         elif isLineClearToStartToken and (not self.atEnd()) and self.peek() in '\r\n':
1613             # still gobble the EOL if a comment was found.
1614             self.readToEOL(gobble=True)
1615
1616         if isLineClearToStartToken and (self.atEnd() or self.pos() > endOfFirstLinePos):
1617             self._compiler.handleWSBeforeDirective()
1618
1619     def _eatToThisEndDirective(self, directiveName):
1620         finalPos = endRawPos = startPos = self.pos()
1621         directiveChar = self.setting('directiveStartToken')[0]
1622         isLineClearToStartToken = False
1623         while not self.atEnd():
1624             if self.peek() == directiveChar:
1625                 if self.matchDirective() == 'end':
1626                     endRawPos = self.pos()
1627                     self.getDirectiveStartToken()
1628                     self.advance(len('end'))
1629                     self.getWhiteSpace()
1630                     if self.startswith(directiveName):
1631                         if self.isLineClearToStartToken(endRawPos):
1632                             isLineClearToStartToken = True
1633                             endRawPos = self.findBOL(endRawPos)
1634                         self.advance(len(directiveName)) # to end of directiveName
1635                         self.getWhiteSpace()
1636                         finalPos = self.pos()
1637                         break
1638             self.advance()
1639             finalPos = endRawPos = self.pos()
1640
1641         textEaten = self.readTo(endRawPos, start=startPos)
1642         self.setPos(finalPos)
1643
1644         endOfFirstLinePos = self.findEOL()
1645
1646         if self.matchDirectiveEndToken():
1647             self.getDirectiveEndToken()
1648         elif isLineClearToStartToken and (not self.atEnd()) and self.peek() in '\r\n':
1649             self.readToEOL(gobble=True)
1650
1651         if isLineClearToStartToken and self.pos() > endOfFirstLinePos:
1652             self._compiler.handleWSBeforeDirective()
1653         return textEaten
1654
1655
1656     def eatSimpleExprDirective(self, directiveName, includeDirectiveNameInExpr=True):
1657         # filtered
1658         isLineClearToStartToken = self.isLineClearToStartToken()
1659         endOfFirstLine = self.findEOL()
1660         self.getDirectiveStartToken()
1661         if not includeDirectiveNameInExpr:
1662             self.advance(len(directiveName))
1663         startPos = self.pos()
1664         expr = self.getExpression().strip()
1665         directiveName = expr.split()[0]
1666         expr = self._applyExpressionFilters(expr, directiveName, startPos=startPos)
1667         if directiveName in self._closeableDirectives:
1668             self.pushToOpenDirectivesStack(directiveName)
1669         self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLine)
1670         return expr
1671
1672     def eatSimpleIndentingDirective(self, directiveName, callback,
1673                                     includeDirectiveNameInExpr=False):
1674         # filtered
1675         isLineClearToStartToken = self.isLineClearToStartToken()
1676         endOfFirstLinePos = self.findEOL()
1677         lineCol = self.getRowCol()
1678         self.getDirectiveStartToken()
1679         if directiveName not in 'else elif for while try except finally'.split():
1680             self.advance(len(directiveName))
1681         startPos = self.pos()
1682
1683         self.getWhiteSpace()
1684
1685         expr = self.getExpression(pyTokensToBreakAt=[':'])
1686         expr = self._applyExpressionFilters(expr, directiveName, startPos=startPos)
1687         if self.matchColonForSingleLineShortFormDirective():
1688             self.advance() # skip over :
1689             if directiveName in 'else elif except finally'.split():
1690                 callback(expr, dedent=False, lineCol=lineCol)
1691             else:
1692                 callback(expr, lineCol=lineCol)
1693
1694             self.getWhiteSpace(max=1)
1695             self.parse(breakPoint=self.findEOL(gobble=True))
1696             self._compiler.commitStrConst()
1697             self._compiler.dedent()
1698         else:
1699             if self.peek()==':':
1700                 self.advance()
1701             self.getWhiteSpace()
1702             self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
1703             if directiveName in self._closeableDirectives:
1704                 self.pushToOpenDirectivesStack(directiveName)
1705             callback(expr, lineCol=lineCol)
1706
1707     def eatEndDirective(self):
1708         isLineClearToStartToken = self.isLineClearToStartToken()
1709         self.getDirectiveStartToken()
1710         self.advance(3)                 # to end of 'end'
1711         self.getWhiteSpace()
1712         pos = self.pos()
1713         directiveName = False
1714         for key in self._endDirectiveNamesAndHandlers.keys():
1715             if self.find(key, pos) == pos:
1716                 directiveName = key
1717                 break
1718         if not directiveName:
1719             raise ParseError(self, msg='Invalid end directive')
1720
1721         endOfFirstLinePos = self.findEOL()
1722         self.getExpression() # eat in any extra comment-like crap
1723         self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
1724         if directiveName in self._closeableDirectives:
1725             self.popFromOpenDirectivesStack(directiveName)
1726
1727         # subclasses can override the default behaviours here by providing an
1728         # end-directive handler in self._endDirectiveNamesAndHandlers[directiveName]
1729         if self._endDirectiveNamesAndHandlers.get(directiveName):
1730             handler = self._endDirectiveNamesAndHandlers[directiveName]
1731             handler()
1732         elif directiveName in 'block capture cache call filter errorCatcher'.split():
1733             if key == 'block':
1734                 self._compiler.closeBlock()
1735             elif key == 'capture':
1736                 self._compiler.endCaptureRegion()
1737             elif key == 'cache':
1738                 self._compiler.endCacheRegion()
1739             elif key == 'call':
1740                 self._compiler.endCallRegion()
1741             elif key == 'filter':
1742                 self._compiler.closeFilterBlock()
1743             elif key == 'errorCatcher':
1744                 self._compiler.turnErrorCatcherOff()
1745         elif directiveName in 'while for if try repeat unless'.split():
1746             self._compiler.commitStrConst()
1747             self._compiler.dedent()
1748         elif directiveName=='closure':
1749             self._compiler.commitStrConst()
1750             self._compiler.dedent()
1751             # @@TR: temporary hack of useSearchList
1752             self.setSetting('useSearchList', self._useSearchList_orig)
1753
1754     ## specific directive eat methods
1755
1756     def eatBreakPoint(self):
1757         """Tells the parser to stop parsing at this point and completely ignore
1758         everything else.
1759
1760         This is a debugging tool.
1761         """
1762         self.setBreakPoint(self.pos())
1763
1764     def eatShbang(self):
1765         # filtered
1766         self.getDirectiveStartToken()
1767         self.advance(len('shBang'))
1768         self.getWhiteSpace()
1769         startPos = self.pos()
1770         shBang = self.readToEOL()
1771         shBang = self._applyExpressionFilters(shBang, 'shbang', startPos=startPos)
1772         self._compiler.setShBang(shBang.strip())
1773
1774     def eatEncoding(self):
1775         # filtered
1776         self.getDirectiveStartToken()
1777         self.advance(len('encoding'))
1778         self.getWhiteSpace()
1779         startPos = self.pos()
1780         encoding = self.readToEOL()
1781         encoding = self._applyExpressionFilters(encoding, 'encoding', startPos=startPos)
1782         self._compiler.setModuleEncoding(encoding.strip())
1783
1784     def eatCompiler(self):
1785         # filtered
1786         isLineClearToStartToken = self.isLineClearToStartToken()
1787         endOfFirstLine = self.findEOL()
1788         startPos = self.pos()
1789         self.getDirectiveStartToken()
1790         self.advance(len('compiler'))   # to end of 'compiler'
1791         self.getWhiteSpace()
1792
1793         startPos = self.pos()
1794         settingName = self.getIdentifier()
1795
1796         if settingName.lower() == 'reset':
1797             self.getExpression() # gobble whitespace & junk
1798             self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLine)
1799             self._initializeSettings()
1800             self.configureParser()
1801             return
1802
1803         self.getWhiteSpace()
1804         if self.peek() == '=':
1805             self.advance()
1806         else:
1807             raise ParseError(self)
1808         valueExpr = self.getExpression()
1809         endPos = self.pos()
1810
1811         # @@TR: it's unlikely that anyone apply filters would have left this
1812         # directive enabled:
1813         # @@TR: fix up filtering, regardless
1814         self._applyExpressionFilters('%s=%r'%(settingName, valueExpr),
1815                                      'compiler', startPos=startPos)
1816
1817         self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLine)
1818         try:
1819             self._compiler.setCompilerSetting(settingName, valueExpr)
1820         except:
1821             out = sys.stderr
1822             print >> out, 'An error occurred while processing the following #compiler directive.'
1823             print >> out, '-'*80
1824             print >> out, self[startPos:endPos]
1825             print >> out, '-'*80
1826             print >> out, 'Please check the syntax of these settings.'
1827             print >> out, 'A full Python exception traceback follows.'
1828             raise
1829
1830
1831     def eatCompilerSettings(self):
1832         # filtered
1833         isLineClearToStartToken = self.isLineClearToStartToken()
1834         endOfFirstLine = self.findEOL()
1835         self.getDirectiveStartToken()
1836         self.advance(len('compiler-settings'))   # to end of 'settings'
1837
1838         keywords = self.getTargetVarsList()
1839         self.getExpression()            # gobble any garbage
1840
1841         self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLine)
1842
1843         if 'reset' in keywords:
1844             self._compiler._initializeSettings()
1845             self.configureParser()
1846             # @@TR: this implies a single-line #compiler-settings directive, and
1847             # thus we should parse forward for an end directive.
1848             # Subject to change in the future
1849             return
1850         startPos = self.pos()
1851         settingsStr = self._eatToThisEndDirective('compiler-settings')
1852         settingsStr = self._applyExpressionFilters(settingsStr, 'compilerSettings',
1853                                                    startPos=startPos)
1854         try:
1855             self._compiler.setCompilerSettings(keywords=keywords, settingsStr=settingsStr)
1856         except:
1857             out = sys.stderr
1858             print >> out, 'An error occurred while processing the following compiler settings.'
1859             print >> out, '-'*80
1860             print >> out, settingsStr.strip()
1861             print >> out, '-'*80
1862             print >> out, 'Please check the syntax of these settings.'
1863             print >> out, 'A full Python exception traceback follows.'
1864             raise
1865
1866     def eatAttr(self):
1867         # filtered
1868         isLineClearToStartToken = self.isLineClearToStartToken()
1869         endOfFirstLinePos = self.findEOL()
1870         startPos = self.pos()
1871         self.getDirectiveStartToken()
1872         self.advance(len('attr'))
1873         self.getWhiteSpace()
1874         startPos = self.pos()
1875         if self.matchCheetahVarStart():
1876             self.getCheetahVarStartToken()
1877         attribName = self.getIdentifier()
1878         self.getWhiteSpace()
1879         self.getAssignmentOperator()
1880         expr = self.getExpression()
1881         expr = self._applyExpressionFilters(expr, 'attr', startPos=startPos)
1882         self._compiler.addAttribute(attribName, expr)
1883         self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
1884
1885     def eatDecorator(self):
1886         isLineClearToStartToken = self.isLineClearToStartToken()
1887         endOfFirstLinePos = self.findEOL()
1888         startPos = self.pos()
1889         self.getDirectiveStartToken()
1890         #self.advance() # eat @
1891         startPos = self.pos()
1892         decoratorExpr = self.getExpression()
1893         decoratorExpr = self._applyExpressionFilters(decoratorExpr, 'decorator', startPos=startPos)
1894         self._compiler.addDecorator(decoratorExpr)
1895         self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
1896         self.getWhiteSpace()
1897
1898         directiveName = self.matchDirective()
1899         if not directiveName or directiveName not in ('def', 'block', 'closure'):
1900             raise ParseError(self, msg='Expected #def, #block or #closure')
1901         self.eatDirective()
1902
1903     def eatDef(self):
1904         # filtered
1905         self._eatDefOrBlock('def')
1906
1907     def eatBlock(self):
1908         # filtered
1909         startPos = self.pos()
1910         methodName, rawSignature = self._eatDefOrBlock('block')
1911         self._compiler._blockMetaData[methodName] = {
1912             'raw':rawSignature,
1913             'lineCol':self.getRowCol(startPos),
1914             }
1915
1916     def eatClosure(self):
1917         # filtered
1918         self._eatDefOrBlock('closure')
1919
1920     def _eatDefOrBlock(self, directiveName):
1921         # filtered
1922         assert directiveName in ('def','block','closure')
1923         isLineClearToStartToken = self.isLineClearToStartToken()
1924         endOfFirstLinePos = self.findEOL()
1925         startPos = self.pos()
1926         self.getDirectiveStartToken()
1927         self.advance(len(directiveName))
1928         self.getWhiteSpace()
1929         if self.matchCheetahVarStart():
1930             self.getCheetahVarStartToken()
1931         methodName = self.getIdentifier()
1932         self.getWhiteSpace()
1933         if self.peek() == '(':
1934             argsList = self.getDefArgList()
1935             self.advance()              # past the closing ')'
1936             if argsList and argsList[0][0] == 'self':
1937                 del argsList[0]
1938         else:
1939             argsList=[]
1940
1941         def includeBlockMarkers():
1942             if self.setting('includeBlockMarkers'):
1943                 startMarker = self.setting('blockMarkerStart')
1944                 self._compiler.addStrConst(startMarker[0] + methodName + startMarker[1])
1945
1946         # @@TR: fix up filtering
1947         self._applyExpressionFilters(self[startPos:self.pos()], 'def', startPos=startPos)
1948
1949         if self.matchColonForSingleLineShortFormDirective():
1950             isNestedDef = (self.setting('allowNestedDefScopes')
1951                            and [name for name in self._openDirectivesStack if name=='def'])
1952             self.getc()
1953             rawSignature = self[startPos:endOfFirstLinePos]
1954             self._eatSingleLineDef(directiveName=directiveName,
1955                                    methodName=methodName,
1956                                    argsList=argsList,
1957                                    startPos=startPos,
1958                                    endPos=endOfFirstLinePos)
1959             if directiveName == 'def' and not isNestedDef:
1960                 #@@TR: must come before _eatRestOfDirectiveTag ... for some reason
1961                 self._compiler.closeDef()
1962             elif directiveName == 'block':
1963                 includeBlockMarkers()
1964                 self._compiler.closeBlock()
1965             elif directiveName == 'closure' or isNestedDef:
1966                 self._compiler.dedent()
1967
1968             self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
1969         else:
1970             if self.peek()==':':
1971                 self.getc()
1972             self.pushToOpenDirectivesStack(directiveName)
1973             rawSignature = self[startPos:self.pos()]
1974             self._eatMultiLineDef(directiveName=directiveName,
1975                                   methodName=methodName,
1976                                   argsList=argsList,
1977                                   startPos=startPos,
1978                                   isLineClearToStartToken=isLineClearToStartToken)
1979             if directiveName == 'block':
1980                 includeBlockMarkers()
1981
1982         return methodName, rawSignature
1983
1984     def _eatMultiLineDef(self, directiveName, methodName, argsList, startPos,
1985                          isLineClearToStartToken=False):
1986         # filtered in calling method
1987         self.getExpression()            # slurp up any garbage left at the end
1988         signature = self[startPos:self.pos()]
1989         endOfFirstLinePos = self.findEOL()
1990         self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
1991         parserComment = ('## CHEETAH: generated from ' + signature +
1992                          ' at line %s, col %s' % self.getRowCol(startPos)
1993                          + '.')
1994
1995         isNestedDef = (self.setting('allowNestedDefScopes')
1996                        and len([name for name in self._openDirectivesStack if name=='def'])>1)
1997         if directiveName=='block' or (directiveName=='def' and not isNestedDef):
1998             self._compiler.startMethodDef(methodName, argsList, parserComment)
1999         else: #closure
2000             self._useSearchList_orig = self.setting('useSearchList')
2001             self.setSetting('useSearchList', False)
2002             self._compiler.addClosure(methodName, argsList, parserComment)
2003
2004         return methodName
2005
2006     def _eatSingleLineDef(self, directiveName, methodName, argsList, startPos, endPos):
2007         # filtered in calling method
2008         fullSignature = self[startPos:endPos]
2009         parserComment = ('## Generated from ' + fullSignature +
2010                          ' at line %s, col %s' % self.getRowCol(startPos)
2011                          + '.')
2012         isNestedDef = (self.setting('allowNestedDefScopes')
2013                        and [name for name in self._openDirectivesStack if name=='def'])
2014         if directiveName=='block' or (directiveName=='def' and not isNestedDef):
2015             self._compiler.startMethodDef(methodName, argsList, parserComment)
2016         else: #closure
2017             # @@TR: temporary hack of useSearchList
2018             useSearchList_orig = self.setting('useSearchList')
2019             self.setSetting('useSearchList', False)
2020             self._compiler.addClosure(methodName, argsList, parserComment)
2021
2022         self.getWhiteSpace(max=1)
2023         self.parse(breakPoint=endPos)
2024         if directiveName=='closure' or isNestedDef: # @@TR: temporary hack of useSearchList
2025             self.setSetting('useSearchList', useSearchList_orig)
2026
2027     def eatExtends(self):
2028         # filtered
2029         isLineClearToStartToken = self.isLineClearToStartToken()
2030         endOfFirstLine = self.findEOL()
2031         self.getDirectiveStartToken()
2032         self.advance(len('extends'))
2033         self.getWhiteSpace()
2034         startPos = self.pos()
2035         if self.setting('allowExpressionsInExtendsDirective'):
2036             baseName = self.getExpression()
2037         else:
2038             baseName = self.getDottedName()
2039
2040         baseName = self._applyExpressionFilters(baseName, 'extends', startPos=startPos)
2041         self._compiler.setBaseClass(baseName) # in compiler
2042         self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLine)
2043
2044     def eatImplements(self):
2045         # filtered
2046         isLineClearToStartToken = self.isLineClearToStartToken()
2047         endOfFirstLine = self.findEOL()
2048         self.getDirectiveStartToken()
2049         self.advance(len('implements'))
2050         self.getWhiteSpace()
2051         startPos = self.pos()
2052         methodName = self.getIdentifier()
2053         if not self.atEnd() and self.peek() == '(':
2054             argsList = self.getDefArgList()
2055             self.advance()              # past the closing ')'
2056             if argsList and argsList[0][0] == 'self':
2057                 del argsList[0]
2058         else:
2059             argsList=[]
2060
2061         # @@TR: need to split up filtering of the methodname and the args
2062         #methodName = self._applyExpressionFilters(methodName, 'implements', startPos=startPos)
2063         self._applyExpressionFilters(self[startPos:self.pos()], 'implements', startPos=startPos)
2064
2065         self._compiler.setMainMethodName(methodName)
2066         self._compiler.setMainMethodArgs(argsList)
2067
2068         self.getExpression()  # throw away and unwanted crap that got added in
2069         self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLine)
2070
2071     def eatSuper(self):
2072         # filtered
2073         isLineClearToStartToken = self.isLineClearToStartToken()
2074         endOfFirstLine = self.findEOL()
2075         self.getDirectiveStartToken()
2076         self.advance(len('super'))
2077         self.getWhiteSpace()
2078         startPos = self.pos()
2079         if not self.atEnd() and self.peek() == '(':
2080             argsList = self.getDefArgList()
2081             self.advance()              # past the closing ')'
2082             if argsList and argsList[0][0] == 'self':
2083                 del argsList[0]
2084         else:
2085             argsList=[]
2086
2087         self._applyExpressionFilters(self[startPos:self.pos()], 'super', startPos=startPos)
2088
2089         #parserComment = ('## CHEETAH: generated from ' + signature +
2090         #                 ' at line %s, col %s' % self.getRowCol(startPos)
2091         #                 + '.')
2092
2093         self.getExpression()  # throw away and unwanted crap that got added in
2094         self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLine)
2095         self._compiler.addSuper(argsList)
2096
2097     def eatSet(self):
2098         # filtered
2099         isLineClearToStartToken = self.isLineClearToStartToken()
2100         endOfFirstLine = self.findEOL()
2101         self.getDirectiveStartToken()
2102         self.advance(3)
2103         self.getWhiteSpace()
2104         style = SET_LOCAL
2105         if self.startswith('local'):
2106             self.getIdentifier()
2107             self.getWhiteSpace()
2108         elif self.startswith('global'):
2109             self.getIdentifier()
2110             self.getWhiteSpace()
2111             style = SET_GLOBAL
2112         elif self.startswith('module'):
2113             self.getIdentifier()
2114             self.getWhiteSpace()
2115             style = SET_MODULE
2116
2117         startsWithDollar = self.matchCheetahVarStart()
2118         startPos = self.pos()
2119         LVALUE = self.getExpression(pyTokensToBreakAt=assignmentOps, useNameMapper=False).strip()
2120         OP = self.getAssignmentOperator()
2121         RVALUE = self.getExpression()
2122         expr = LVALUE + ' ' + OP + ' ' + RVALUE.strip()
2123
2124         expr = self._applyExpressionFilters(expr, 'set', startPos=startPos)
2125         self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLine)
2126
2127         class Components: pass # used for 'set global'
2128         exprComponents = Components()
2129         exprComponents.LVALUE = LVALUE
2130         exprComponents.OP = OP
2131         exprComponents.RVALUE = RVALUE
2132         self._compiler.addSet(expr, exprComponents, style)
2133
2134     def eatSlurp(self):
2135         if self.isLineClearToStartToken():
2136             self._compiler.handleWSBeforeDirective()
2137         self._compiler.commitStrConst()
2138         self.readToEOL(gobble=True)
2139
2140     def eatEOLSlurpToken(self):
2141         if self.isLineClearToStartToken():
2142             self._compiler.handleWSBeforeDirective()
2143         self._compiler.commitStrConst()
2144         self.readToEOL(gobble=True)
2145
2146     def eatRaw(self):
2147         isLineClearToStartToken = self.isLineClearToStartToken()
2148         endOfFirstLinePos = self.findEOL()
2149         self.getDirectiveStartToken()
2150         self.advance(len('raw'))
2151         self.getWhiteSpace()
2152         if self.matchColonForSingleLineShortFormDirective():
2153             self.advance() # skip over :
2154             self.getWhiteSpace(max=1)
2155             rawBlock = self.readToEOL(gobble=False)
2156         else:
2157             if self.peek()==':':
2158                 self.advance()
2159             self.getWhiteSpace()
2160             self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
2161             rawBlock = self._eatToThisEndDirective('raw')
2162         self._compiler.addRawText(rawBlock)
2163
2164     def eatInclude(self):
2165         # filtered
2166         isLineClearToStartToken = self.isLineClearToStartToken()
2167         endOfFirstLinePos = self.findEOL()
2168         self.getDirectiveStartToken()
2169         self.advance(len('include'))
2170
2171         self.getWhiteSpace()
2172         includeFrom = 'file'
2173         isRaw = False
2174         if self.startswith('raw'):
2175             self.advance(3)
2176             isRaw=True
2177
2178         self.getWhiteSpace()
2179         if self.startswith('source'):
2180             self.advance(len('source'))
2181             includeFrom = 'str'
2182             self.getWhiteSpace()
2183             if not self.peek() == '=':
2184                 raise ParseError(self)
2185             self.advance()
2186         startPos = self.pos()
2187         sourceExpr = self.getExpression()
2188         sourceExpr = self._applyExpressionFilters(sourceExpr, 'include', startPos=startPos)
2189         self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
2190         self._compiler.addInclude(sourceExpr, includeFrom, isRaw)
2191
2192
2193     def eatDefMacro(self):
2194         # @@TR: not filtered yet
2195         isLineClearToStartToken = self.isLineClearToStartToken()
2196         endOfFirstLinePos = self.findEOL()
2197         self.getDirectiveStartToken()
2198         self.advance(len('defmacro'))
2199
2200         self.getWhiteSpace()
2201         if self.matchCheetahVarStart():
2202             self.getCheetahVarStartToken()
2203         macroName = self.getIdentifier()
2204         self.getWhiteSpace()
2205         if self.peek() == '(':
2206             argsList = self.getDefArgList(useNameMapper=False)
2207             self.advance()              # past the closing ')'
2208             if argsList and argsList[0][0] == 'self':
2209                 del argsList[0]
2210         else:
2211             argsList=[]
2212
2213         assert not self._directiveNamesAndParsers.has_key(macroName)
2214         argsList.insert(0, ('src',None))
2215         argsList.append(('parser','None'))
2216         argsList.append(('macros','None'))
2217         argsList.append(('compilerSettings','None'))
2218         argsList.append(('isShortForm','None'))
2219         argsList.append(('EOLCharsInShortForm','None'))
2220         argsList.append(('startPos','None'))
2221         argsList.append(('endPos','None'))
2222
2223         if self.matchColonForSingleLineShortFormDirective():
2224             self.advance() # skip over :
2225             self.getWhiteSpace(max=1)
2226             macroSrc = self.readToEOL(gobble=False)
2227             self.readToEOL(gobble=True)
2228         else:
2229             if self.peek()==':':
2230                 self.advance()
2231             self.getWhiteSpace()
2232             self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
2233             macroSrc = self._eatToThisEndDirective('defmacro')
2234
2235         #print argsList
2236         normalizedMacroSrc = ''.join(
2237             ['%def callMacro('+','.join([defv and '%s=%s'%(n,defv) or n
2238                                          for n,defv in argsList])
2239              +')\n',
2240              macroSrc,
2241              '%end def'])
2242
2243
2244         from Cheetah.Template import Template
2245         templateAPIClass = self.setting('templateAPIClassForDefMacro', default=Template)
2246         compilerSettings = self.setting('compilerSettingsForDefMacro', default={})
2247         searchListForMacros = self.setting('searchListForDefMacro', default=[])
2248         searchListForMacros = list(searchListForMacros) # copy to avoid mutation bugs
2249         searchListForMacros.append({'macros':self._macros,
2250                                     'parser':self,
2251                                     'compilerSettings':self.settings(),
2252                                     })
2253
2254         templateAPIClass._updateSettingsWithPreprocessTokens(
2255             compilerSettings, placeholderToken='@', directiveToken='%')
2256         macroTemplateClass = templateAPIClass.compile(source=normalizedMacroSrc,
2257                                                       compilerSettings=compilerSettings)
2258         #print normalizedMacroSrc
2259         #t = macroTemplateClass()
2260         #print t.callMacro('src')
2261         #print t.generatedClassCode()
2262
2263         class MacroDetails: pass
2264         macroDetails = MacroDetails()
2265         macroDetails.macroSrc = macroSrc
2266         macroDetails.argsList = argsList
2267         macroDetails.template = macroTemplateClass(searchList=searchListForMacros)
2268
2269         self._macroDetails[macroName] = macroDetails
2270         self._macros[macroName] = macroDetails.template.callMacro
2271         self._directiveNamesAndParsers[macroName] = self.eatMacroCall
2272
2273     def eatMacroCall(self):
2274         isLineClearToStartToken = self.isLineClearToStartToken()
2275         endOfFirstLinePos = self.findEOL()
2276         startPos = self.pos()
2277         self.getDirectiveStartToken()
2278         macroName = self.getIdentifier()
2279         macro = self._macros[macroName]
2280         if hasattr(macro, 'parse'):
2281             return macro.parse(parser=self, startPos=startPos)
2282
2283         if hasattr(macro, 'parseArgs'):
2284             args = macro.parseArgs(parser=self, startPos=startPos)
2285         else:
2286             self.getWhiteSpace()
2287             args = self.getExpression(useNameMapper=False,
2288                                       pyTokensToBreakAt=[':']).strip()
2289
2290         if self.matchColonForSingleLineShortFormDirective():
2291             isShortForm = True
2292             self.advance() # skip over :
2293             self.getWhiteSpace(max=1)
2294             srcBlock = self.readToEOL(gobble=False)
2295             EOLCharsInShortForm = self.readToEOL(gobble=True)
2296             #self.readToEOL(gobble=False)
2297         else:
2298             isShortForm = False
2299             if self.peek()==':':
2300                 self.advance()
2301             self.getWhiteSpace()
2302             self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
2303             srcBlock = self._eatToThisEndDirective(macroName)
2304
2305
2306         if hasattr(macro, 'convertArgStrToDict'):
2307             kwArgs = macro.convertArgStrToDict(args, parser=self, startPos=startPos)
2308         else:
2309             def getArgs(*pargs, **kws):
2310                 return pargs, kws
2311             exec 'positionalArgs, kwArgs = getArgs(%(args)s)'%locals()
2312
2313         assert not kwArgs.has_key('src')
2314         kwArgs['src'] = srcBlock
2315
2316         if type(macro)==new.instancemethod:
2317             co = macro.im_func.func_code
2318         elif (hasattr(macro, '__call__')
2319               and hasattr(macro.__call__, 'im_func')):
2320             co = macro.__call__.im_func.func_code
2321         else:
2322             co = macro.func_code
2323         availableKwArgs = inspect.getargs(co)[0]
2324
2325         if 'parser' in availableKwArgs:
2326             kwArgs['parser'] = self
2327         if 'macros' in availableKwArgs:
2328             kwArgs['macros'] = self._macros
2329         if 'compilerSettings' in availableKwArgs:
2330             kwArgs['compilerSettings'] = self.settings()
2331         if 'isShortForm' in availableKwArgs:
2332             kwArgs['isShortForm'] = isShortForm
2333         if isShortForm and 'EOLCharsInShortForm' in availableKwArgs:
2334             kwArgs['EOLCharsInShortForm'] = EOLCharsInShortForm
2335
2336         if 'startPos' in availableKwArgs:
2337             kwArgs['startPos'] = startPos
2338         if 'endPos' in availableKwArgs:
2339             kwArgs['endPos'] = self.pos()
2340
2341         srcFromMacroOutput = macro(**kwArgs)
2342
2343         origParseSrc = self._src
2344         origBreakPoint = self.breakPoint()
2345         origPos = self.pos()
2346         # add a comment to the output about the macro src that is being parsed
2347         # or add a comment prefix to all the comments added by the compiler
2348         self._src = srcFromMacroOutput
2349         self.setPos(0)
2350         self.setBreakPoint(len(srcFromMacroOutput))
2351
2352         self.parse(assertEmptyStack=False)
2353
2354         self._src = origParseSrc
2355         self.setBreakPoint(origBreakPoint)
2356         self.setPos(origPos)
2357
2358
2359         #self._compiler.addRawText('end')
2360
2361     def eatCache(self):
2362         isLineClearToStartToken = self.isLineClearToStartToken()
2363         endOfFirstLinePos = self.findEOL()
2364         lineCol = self.getRowCol()
2365         self.getDirectiveStartToken()
2366         self.advance(len('cache'))
2367
2368         startPos = self.pos()
2369         argList = self.getDefArgList(useNameMapper=True)
2370         argList = self._applyExpressionFilters(argList, 'cache', startPos=startPos)
2371
2372         def startCache():
2373             cacheInfo = self._compiler.genCacheInfoFromArgList(argList)
2374             self._compiler.startCacheRegion(cacheInfo, lineCol)
2375
2376         if self.matchColonForSingleLineShortFormDirective():
2377             self.advance() # skip over :
2378             self.getWhiteSpace(max=1)
2379             startCache()
2380             self.parse(breakPoint=self.findEOL(gobble=True))
2381             self._compiler.endCacheRegion()
2382         else:
2383             if self.peek()==':':
2384                 self.advance()
2385             self.getWhiteSpace()
2386             self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
2387             self.pushToOpenDirectivesStack('cache')
2388             startCache()
2389
2390     def eatCall(self):
2391         # @@TR: need to enable single line version of this
2392         isLineClearToStartToken = self.isLineClearToStartToken()
2393         endOfFirstLinePos = self.findEOL()
2394         lineCol = self.getRowCol()
2395         self.getDirectiveStartToken()
2396         self.advance(len('call'))
2397         startPos = self.pos()
2398
2399         useAutocallingOrig = self.setting('useAutocalling')
2400         self.setSetting('useAutocalling', False)
2401         self.getWhiteSpace()
2402         if self.matchCheetahVarStart():
2403             functionName = self.getCheetahVar()
2404         else:
2405             functionName = self.getCheetahVar(plain=True, skipStartToken=True)
2406         self.setSetting('useAutocalling', useAutocallingOrig)
2407         # @@TR: fix up filtering
2408         self._applyExpressionFilters(self[startPos:self.pos()], 'call', startPos=startPos)
2409
2410         self.getWhiteSpace()
2411         args = self.getExpression(pyTokensToBreakAt=[':']).strip()
2412         if self.matchColonForSingleLineShortFormDirective():
2413             self.advance() # skip over :
2414             self._compiler.startCallRegion(functionName, args, lineCol)
2415             self.getWhiteSpace(max=1)
2416             self.parse(breakPoint=self.findEOL(gobble=False))
2417             self._compiler.endCallRegion()
2418         else:
2419             if self.peek()==':':
2420                 self.advance()
2421             self.getWhiteSpace()
2422             self.pushToOpenDirectivesStack("call")
2423             self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
2424             self._compiler.startCallRegion(functionName, args, lineCol)
2425
2426     def eatCallArg(self):
2427         isLineClearToStartToken = self.isLineClearToStartToken()
2428         endOfFirstLinePos = self.findEOL()
2429         lineCol = self.getRowCol()
2430         self.getDirectiveStartToken()
2431
2432         self.advance(len('arg'))
2433         startPos = self.pos()
2434         self.getWhiteSpace()
2435         argName = self.getIdentifier()
2436         self.getWhiteSpace()
2437         argName = self._applyExpressionFilters(argName, 'arg', startPos=startPos)
2438         self._compiler.setCallArg(argName, lineCol)
2439         if self.peek() == ':':
2440             self.getc()
2441         else:
2442             self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
2443
2444     def eatFilter(self):
2445         isLineClearToStartToken = self.isLineClearToStartToken()
2446         endOfFirstLinePos = self.findEOL()
2447
2448         self.getDirectiveStartToken()
2449         self.advance(len('filter'))
2450         self.getWhiteSpace()
2451         startPos = self.pos()
2452         if self.matchCheetahVarStart():
2453             isKlass = True
2454             theFilter = self.getExpression(pyTokensToBreakAt=[':'])
2455         else:
2456             isKlass = False
2457             theFilter = self.getIdentifier()
2458             self.getWhiteSpace()
2459         theFilter = self._applyExpressionFilters(theFilter, 'filter', startPos=startPos)
2460
2461         if self.matchColonForSingleLineShortFormDirective():
2462             self.advance() # skip over :
2463             self.getWhiteSpace(max=1)
2464             self._compiler.setFilter(theFilter, isKlass)
2465             self.parse(breakPoint=self.findEOL(gobble=False))
2466             self._compiler.closeFilterBlock()
2467         else:
2468             if self.peek()==':':
2469                 self.advance()
2470             self.getWhiteSpace()
2471             self.pushToOpenDirectivesStack("filter")
2472             self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
2473             self._compiler.setFilter(theFilter, isKlass)
2474
2475     def eatErrorCatcher(self):
2476         isLineClearToStartToken = self.isLineClearToStartToken()
2477         endOfFirstLinePos = self.findEOL()
2478         self.getDirectiveStartToken()
2479         self.advance(len('errorCatcher'))
2480         self.getWhiteSpace()
2481         startPos = self.pos()
2482         errorCatcherName = self.getIdentifier()
2483         errorCatcherName = self._applyExpressionFilters(
2484             errorCatcherName, 'errorcatcher', startPos=startPos)
2485         self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
2486         self._compiler.setErrorCatcher(errorCatcherName)
2487
2488     def eatCapture(self):
2489         # @@TR:  this could be refactored to use the code in eatSimpleIndentingDirective
2490         # filtered
2491         isLineClearToStartToken = self.isLineClearToStartToken()
2492         endOfFirstLinePos = self.findEOL()
2493         lineCol = self.getRowCol()
2494
2495         self.getDirectiveStartToken()
2496         self.advance(len('capture'))
2497         startPos = self.pos()
2498         self.getWhiteSpace()
2499
2500         expr = self.getExpression(pyTokensToBreakAt=[':'])
2501         expr = self._applyExpressionFilters(expr, 'capture', startPos=startPos)
2502         if self.matchColonForSingleLineShortFormDirective():
2503             self.advance() # skip over :
2504             self._compiler.startCaptureRegion(assignTo=expr, lineCol=lineCol)
2505             self.getWhiteSpace(max=1)
2506             self.parse(breakPoint=self.findEOL(gobble=False))
2507             self._compiler.endCaptureRegion()
2508         else:
2509             if self.peek()==':':
2510                 self.advance()
2511             self.getWhiteSpace()
2512             self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
2513             self.pushToOpenDirectivesStack("capture")
2514             self._compiler.startCaptureRegion(assignTo=expr, lineCol=lineCol)
2515
2516
2517     def eatIf(self):
2518         # filtered
2519         isLineClearToStartToken = self.isLineClearToStartToken()
2520         endOfFirstLine = self.findEOL()
2521         lineCol = self.getRowCol()
2522         self.getDirectiveStartToken()
2523         startPos = self.pos()
2524
2525         expressionParts = self.getExpressionParts(pyTokensToBreakAt=[':'])
2526         expr = ''.join(expressionParts).strip()
2527         expr = self._applyExpressionFilters(expr, 'if', startPos=startPos)
2528
2529         isTernaryExpr = ('then' in expressionParts and 'else' in expressionParts)
2530         if isTernaryExpr:
2531             conditionExpr = []
2532             trueExpr = []
2533             falseExpr = []
2534             currentExpr = conditionExpr
2535             for part in expressionParts:
2536                 if part.strip()=='then':
2537                     currentExpr = trueExpr
2538                 elif part.strip()=='else':
2539                     currentExpr = falseExpr
2540                 else:
2541                     currentExpr.append(part)
2542
2543             conditionExpr = ''.join(conditionExpr)
2544             trueExpr = ''.join(trueExpr)
2545             falseExpr = ''.join(falseExpr)
2546             self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLine)
2547             self._compiler.addTernaryExpr(conditionExpr, trueExpr, falseExpr, lineCol=lineCol)
2548         elif self.matchColonForSingleLineShortFormDirective():
2549             self.advance() # skip over :
2550             self._compiler.addIf(expr, lineCol=lineCol)
2551             self.getWhiteSpace(max=1)
2552             self.parse(breakPoint=self.findEOL(gobble=True))
2553             self._compiler.commitStrConst()
2554             self._compiler.dedent()
2555         else:
2556             if self.peek()==':':
2557                 self.advance()
2558             self.getWhiteSpace()
2559             self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLine)
2560             self.pushToOpenDirectivesStack('if')
2561             self._compiler.addIf(expr, lineCol=lineCol)
2562
2563     ## end directive handlers
2564     def handleEndDef(self):
2565         isNestedDef = (self.setting('allowNestedDefScopes')
2566                        and [name for name in self._openDirectivesStack if name=='def'])
2567         if not isNestedDef:
2568             self._compiler.closeDef()
2569         else:
2570             # @@TR: temporary hack of useSearchList
2571             self.setSetting('useSearchList', self._useSearchList_orig)
2572             self._compiler.commitStrConst()
2573             self._compiler.dedent()
2574     ###
2575
2576     def pushToOpenDirectivesStack(self, directiveName):
2577         assert directiveName in self._closeableDirectives
2578         self._openDirectivesStack.append(directiveName)
2579
2580     def popFromOpenDirectivesStack(self, directiveName):
2581         if not self._openDirectivesStack:
2582             raise ParseError(self, msg="#end found, but nothing to end")
2583
2584         if self._openDirectivesStack[-1] == directiveName:
2585             del self._openDirectivesStack[-1]
2586         else:
2587             raise ParseError(self, msg="#end %s found, expected #end %s" %(
2588                 directiveName, self._openDirectivesStack[-1]))
2589
2590     def assertEmptyOpenDirectivesStack(self):
2591         if self._openDirectivesStack:
2592             errorMsg = (
2593                 "Some #directives are missing their corresponding #end ___ tag: %s" %(
2594                 ', '.join(self._openDirectivesStack)))
2595             raise ParseError(self, msg=errorMsg)
2596
2597 ##################################################
2598 ## Make an alias to export
2599 Parser = _HighLevelParser