Cheetah/Parser.py

   1 #!/usr/bin/env python
   2 # $Id: Parser.py,v 1.130 2006/06/21 23:49:14 tavis_rudd Exp $
   3 """Parser classes for Cheetah's Compiler
   4
   5 Classes:
   6   ParseError( Exception )
   7   _LowLevelParser( Cheetah.SourceReader.SourceReader ), basically a lexer
   8   _HighLevelParser( _LowLevelParser )
   9   Parser === _HighLevelParser (an alias)
  10
  11 Meta-Data
  12 ================================================================================
  13 Author: Tavis Rudd <tavis@damnsimple.com>
  14 Version: $Revision: 1.130 $
  15 Start Date: 2001/08/01
  16 Last Revision Date: $Date: 2006/06/21 23:49:14 $
  17 """
  18 __author__ = "Tavis Rudd <tavis@damnsimple.com>"
  19 __revision__ = "$Revision: 1.130 $"[11:-2]
  20
  21 import os
  22 import sys
  23 import re
  24 from re import DOTALL, MULTILINE
  25 from types import StringType, ListType, TupleType, ClassType, TypeType
  26 import time
  27 from tokenize import pseudoprog
  28 import inspect
  29 import new
  30 import traceback
  31
  32 from Cheetah.SourceReader import SourceReader
  33 from Cheetah import Filters
  34 from Cheetah import ErrorCatchers
  35 from Cheetah.Unspecified import Unspecified
  36
  37 # re tools
  38 def escapeRegexChars(txt,
  39                      escapeRE=re.compile(r'([\$\^\*\+\.\?\{\}\[\]\(\)\|\\])')):
  40
  41     """Return a txt with all special regular expressions chars escaped."""
  42
  43     return escapeRE.sub(r'\\\1' , txt)
  44
  45 def group(*choices): return '(' + '|'.join(choices) + ')'
  46 def nongroup(*choices): return '(?:' + '|'.join(choices) + ')'
  47 def namedGroup(name, *choices): return '(P:<' + name +'>' + '|'.join(choices) + ')'
  48 def any(*choices): return apply(group, choices) + '*'
  49 def maybe(*choices): return apply(group, choices) + '?'
  50
  51 ##################################################
  52 ## CONSTANTS & GLOBALS ##
  53
  54 NO_CACHE = 0
  55 STATIC_CACHE = 1
  56 REFRESH_CACHE = 2
  57
  58 SET_LOCAL = 0
  59 SET_GLOBAL = 1
  60 SET_MODULE = 2
  61
  62 ##################################################
  63 ## Tokens for the parser ##
  64
  65 #generic
  66 identchars = "abcdefghijklmnopqrstuvwxyz" \
  67             "ABCDEFGHIJKLMNOPQRSTUVWXYZ_"
  68 namechars = identchars + "0123456789"
  69
  70 #operators
  71 powerOp = '**'
  72 unaryArithOps = ('+', '-', '~')
  73 binaryArithOps = ('+', '-', '/', '//','%')
  74 shiftOps = ('>>','<<')
  75 bitwiseOps = ('&','|','^')
  76 assignOp = '='
  77 augAssignOps = ('+=','-=','/=','*=', '**=','^=','%=',
  78           '>>=','<<=','&=','|=', )
  79 assignmentOps = (assignOp,) + augAssignOps
  80
  81 compOps = ('<','>','==','!=','<=','>=', '<>', 'is', 'in',)
  82 booleanOps = ('and','or','not')
  83 operators = (powerOp,) + unaryArithOps + binaryArithOps \
  84             + shiftOps + bitwiseOps + assignmentOps \
  85             + compOps + booleanOps
  86
  87 delimeters = ('(',')','{','}','[',']',
  88               ',','.',':',';','=','`') + augAssignOps
  89
  90
  91 keywords = ('and',       'del',       'for',       'is',        'raise',
  92             'assert',    'elif',      'from',      'lambda',    'return',
  93             'break',     'else',      'global',    'not',       'try',
  94             'class',     'except',    'if',        'or',        'while',
  95             'continue',  'exec',      'import',    'pass',
  96             'def',       'finally',   'in',        'print',
  97             )
  98
  99 single3 = "'''"
 100 double3 = '"""'
 101
 102 tripleQuotedStringStarts =  ("'''", '"""',
 103                              "r'''", 'r"""', "R'''", 'R"""',
 104                              "u'''", 'u"""', "U'''", 'U"""',
 105                              "ur'''", 'ur"""', "Ur'''", 'Ur"""',
 106                              "uR'''", 'uR"""', "UR'''", 'UR"""')
 107
 108 tripleQuotedStringPairs = {"'''": single3, '"""': double3,
 109                            "r'''": single3, 'r"""': double3,
 110                            "u'''": single3, 'u"""': double3,
 111                            "ur'''": single3, 'ur"""': double3,
 112                            "R'''": single3, 'R"""': double3,
 113                            "U'''": single3, 'U"""': double3,
 114                            "uR'''": single3, 'uR"""': double3,
 115                            "Ur'''": single3, 'Ur"""': double3,
 116                            "UR'''": single3, 'UR"""': double3,
 117                            }
 118
 119 closurePairs= {')':'(',']':'[','}':'{'}
 120 closurePairsRev= {'(':')','[':']','{':'}'}
 121
 122 ##################################################
 123 ## Regex chunks for the parser ##
 124
 125 tripleQuotedStringREs = {}
 126 def makeTripleQuoteRe(start, end):
 127     start = escapeRegexChars(start)
 128     end = escapeRegexChars(end)
 129     return re.compile(r'(?:' + start + r').*?' + r'(?:' + end + r')', re.DOTALL)
 130
 131 for start, end in tripleQuotedStringPairs.items():
 132     tripleQuotedStringREs[start] = makeTripleQuoteRe(start, end)
 133
 134 WS = r'[ \f\t]*'
 135 EOL = r'\r\n|\n|\r'
 136 EOLZ = EOL + r'|\Z'
 137 escCharLookBehind = nongroup(r'(?<=\A)',r'(?<!\\)')
 138 nameCharLookAhead = r'(?=[A-Za-z_])'
 139 identRE=re.compile(r'[a-zA-Z_][a-zA-Z_0-9]*')
 140 EOLre=re.compile(r'(?:\r\n|\r|\n)')
 141
 142 specialVarRE=re.compile(r'([a-zA-z_]+)@') # for matching specialVar comments
 143 # e.g. ##author@ Tavis Rudd
 144
 145 directiveNamesAndParsers = {
 146     # importing and inheritance
 147     'import':None,
 148     'from':None,
 149     'extends': 'eatExtends',
 150     'implements': 'eatImplements',
 151
 152     # output, filtering, and caching
 153     'slurp': 'eatSlurp',
 154     'raw': 'eatRaw',
 155     'include': 'eatInclude',
 156     'cache': 'eatCache',
 157     'filter': 'eatFilter',
 158     'echo': None,
 159     'silent': None,
 160
 161     'call': 'eatCall',
 162     'arg': 'eatCallArg',
 163
 164     'capture': 'eatCapture',
 165
 166     # declaration, assignment, and deletion
 167     'attr': 'eatAttr',
 168     'def': 'eatDef',
 169     'block': 'eatBlock',
 170     '@': 'eatDecorator',
 171     'defmacro': 'eatDefMacro',
 172
 173     'closure': 'eatClosure',
 174
 175     'set': 'eatSet',
 176     'del': None,
 177
 178     # flow control
 179     'if': 'eatIf',
 180     'while': None,
 181     'for': None,
 182     'else': None,
 183     'elif': None,
 184     'pass': None,
 185     'break': None,
 186     'continue': None,
 187     'stop': None,
 188     'return': None,
 189     'yield': None,
 190
 191     # little wrappers
 192     'repeat': None,
 193     'unless': None,
 194
 195     # error handling
 196     'assert': None,
 197     'raise': None,
 198     'try': None,
 199     'except': None,
 200     'finally': None,
 201     'errorCatcher': 'eatErrorCatcher',
 202
 203     # intructions to the parser and compiler
 204     'breakpoint': 'eatBreakPoint',
 205     'compiler': 'eatCompiler',
 206     'compiler-settings': 'eatCompilerSettings',
 207
 208     # misc
 209     'shBang': 'eatShbang',
 210     'encoding': 'eatEncoding',
 211
 212     'end': 'eatEndDirective',
 213     }
 214
 215 endDirectiveNamesAndHandlers = {
 216     'def': 'handleEndDef',      # has short-form
 217     'block': None,              # has short-form
 218     'closure': None,            # has short-form
 219     'cache': None,              # has short-form
 220     'call': None,               # has short-form
 221     'capture': None,            # has short-form
 222     'filter': None,
 223     'errorCatcher':None,
 224     'while': None,              # has short-form
 225     'for': None,                # has short-form
 226     'if': None,                 # has short-form
 227     'try': None,                # has short-form
 228     'repeat': None,             # has short-form
 229     'unless': None,             # has short-form
 230     }
 231
 232 ##################################################
 233 ## CLASSES ##
 234
 235 # @@TR: SyntaxError doesn't call exception.__str__ for some reason!
 236 #class ParseError(SyntaxError):
 237 class ParseError(ValueError):
 238     def __init__(self, stream, msg='Invalid Syntax', extMsg='', lineno=None, col=None):
 239         self.stream = stream
 240         if stream.pos() >= len(stream):
 241             stream.setPos(len(stream) -1)
 242         self.msg = msg
 243         self.extMsg = extMsg
 244         self.lineno = lineno
 245         self.col = col
 246
 247     def __str__(self):
 248         return self.report()
 249
 250     def report(self):
 251         stream = self.stream
 252         if stream.filename():
 253             f = " in file %s" % stream.filename()
 254         else:
 255             f = ''
 256         report = ''
 257         if self.lineno:
 258             lineno = self.lineno
 259             row, col, line = (lineno, (self.col or 0),
 260                               self.stream.splitlines()[lineno-1])
 261         else:
 262             row, col, line = self.stream.getRowColLine()
 263
 264         ## get the surrounding lines
 265         lines = stream.splitlines()
 266         prevLines = []                  # (rowNum, content)
 267         for i in range(1,4):
 268             if row-1-i <=0:
 269                 break
 270             prevLines.append( (row-i,lines[row-1-i]) )
 271
 272         nextLines = []                  # (rowNum, content)
 273         for i in range(1,4):
 274             if not row-1+i < len(lines):
 275                 break
 276             nextLines.append( (row+i,lines[row-1+i]) )
 277         nextLines.reverse()
 278
 279         ## print the main message
 280         report += "\n\n%s\n" %self.msg
 281         report += "Line %i, column %i%s\n\n" % (row, col, f)
 282         report += 'Line|Cheetah Code\n'
 283         report += '----|-------------------------------------------------------------\n'
 284         while prevLines:
 285             lineInfo = prevLines.pop()
 286             report += "%(row)-4d|%(line)s\n"% {'row':lineInfo[0], 'line':lineInfo[1]}
 287         report += "%(row)-4d|%(line)s\n"% {'row':row, 'line':line}
 288         report += ' '*5 +' '*(col-1) + "^\n"
 289
 290         while nextLines:
 291             lineInfo = nextLines.pop()
 292             report += "%(row)-4d|%(line)s\n"% {'row':lineInfo[0], 'line':lineInfo[1]}
 293         ## add the extra msg
 294         if self.extMsg:
 295             report += self.extMsg + '\n'
 296
 297         return report
 298
 299 class ForbiddenSyntax(ParseError): pass
 300 class ForbiddenExpression(ForbiddenSyntax): pass
 301 class ForbiddenDirective(ForbiddenSyntax): pass
 302
 303 class CheetahVariable:
 304     def __init__(self, nameChunks, useNameMapper=True, cacheToken=None,
 305                  rawSource=None):
 306         self.nameChunks = nameChunks
 307         self.useNameMapper = useNameMapper
 308         self.cacheToken = cacheToken
 309         self.rawSource = rawSource
 310
 311 class Placeholder(CheetahVariable): pass
 312
 313 class ArgList:
 314     """Used by _LowLevelParser.getArgList()"""
 315
 316     def __init__(self):
 317         self.argNames = []
 318         self.defVals = []
 319         self.i = 0
 320
 321     def addArgName(self, name):
 322         self.argNames.append( name )
 323         self.defVals.append( None )
 324
 325     def next(self):
 326         self.i += 1
 327
 328     def addToDefVal(self, token):
 329         i = self.i
 330         if self.defVals[i] == None:
 331             self.defVals[i] = ''
 332         self.defVals[i] += token
 333
 334     def merge(self):
 335         defVals = self.defVals
 336         for i in range(len(defVals)):
 337             if type(defVals[i]) == StringType:
 338                 defVals[i] = defVals[i].strip()
 339
 340         return map(None, [i.strip() for i in self.argNames], defVals)
 341
 342     def __str__(self):
 343         return str(self.merge())
 344
 345 class _LowLevelParser(SourceReader):
 346     """This class implements the methods to match or extract ('get*') the basic
 347     elements of Cheetah's grammar.  It does NOT handle any code generation or
 348     state management.
 349     """
 350
 351     _settingsManager = None
 352
 353     def setSettingsManager(self, settingsManager):
 354         self._settingsManager = settingsManager
 355
 356     def setting(self, key, default=Unspecified):
 357         if default is Unspecified:
 358             return self._settingsManager.setting(key)
 359         else:
 360             return self._settingsManager.setting(key, default=default)
 361
 362     def setSetting(self, key, val):
 363         self._settingsManager.setSetting(key, val)
 364
 365     def settings(self):
 366         return self._settingsManager.settings()
 367
 368     def updateSettings(self, settings):
 369         self._settingsManager.updateSettings(settings)
 370
 371     def _initializeSettings(self):
 372         self._settingsManager._initializeSettings()
 373
 374     def configureParser(self):
 375         """Is called by the Compiler instance after the parser has had a
 376         settingsManager assigned with self.setSettingsManager()
 377         """
 378         self._makeCheetahVarREs()
 379         self._makeCommentREs()
 380         self._makeDirectiveREs()
 381         self._makePspREs()
 382         self._possibleNonStrConstantChars = (
 383             self.setting('commentStartToken')[0] +
 384             self.setting('multiLineCommentStartToken')[0] +
 385             self.setting('cheetahVarStartToken')[0] +
 386             self.setting('directiveStartToken')[0] +
 387             self.setting('PSPStartToken')[0])
 388         self._nonStrConstMatchers = [
 389             self.matchCommentStartToken,
 390             self.matchMultiLineCommentStartToken,
 391             self.matchVariablePlaceholderStart,
 392             self.matchExpressionPlaceholderStart,
 393             self.matchDirective,
 394             self.matchPSPStartToken,
 395             self.matchEOLSlurpToken,
 396             ]
 397
 398     ## regex setup ##
 399
 400     def _makeCheetahVarREs(self):
 401
 402         """Setup the regexs for Cheetah $var parsing."""
 403
 404         num = r'[0-9\.]+'
 405         interval =   (r'(?P<interval>' +
 406                       num + r's|' +
 407                       num + r'm|' +
 408                       num + r'h|' +
 409                       num + r'd|' +
 410                       num + r'w|' +
 411                       num + ')'
 412                       )
 413
 414         cacheToken = (r'(?:' +
 415                       r'(?P<REFRESH_CACHE>\*' + interval + '\*)'+
 416                       '|' +
 417                       r'(?P<STATIC_CACHE>\*)' +
 418                       '|' +
 419                       r'(?P<NO_CACHE>)' +
 420                       ')')
 421         self.cacheTokenRE = re.compile(cacheToken)
 422
 423         silentPlaceholderToken = (r'(?:' +
 424                                   r'(?P<SILENT>' +escapeRegexChars('!')+')'+
 425                                   '|' +
 426                                   r'(?P<NOT_SILENT>)' +
 427                                   ')')
 428         self.silentPlaceholderTokenRE = re.compile(silentPlaceholderToken)
 429
 430         self.cheetahVarStartRE = re.compile(
 431             escCharLookBehind +
 432             r'(?P<startToken>'+escapeRegexChars(self.setting('cheetahVarStartToken'))+')'+
 433             r'(?P<silenceToken>'+silentPlaceholderToken+')'+
 434             r'(?P<cacheToken>'+cacheToken+')'+
 435             r'(?P<enclosure>|(?:(?:\{|\(|\[)[ \t\f]*))' + # allow WS after enclosure
 436             r'(?=[A-Za-z_])')
 437         validCharsLookAhead = r'(?=[A-Za-z_\*!\{\(\[])'
 438         self.cheetahVarStartToken = self.setting('cheetahVarStartToken')
 439         self.cheetahVarStartTokenRE = re.compile(
 440             escCharLookBehind +
 441             escapeRegexChars(self.setting('cheetahVarStartToken'))
 442             +validCharsLookAhead
 443             )
 444
 445         self.cheetahVarInExpressionStartTokenRE = re.compile(
 446             escapeRegexChars(self.setting('cheetahVarStartToken'))
 447             +r'(?=[A-Za-z_])'
 448             )
 449
 450         self.expressionPlaceholderStartRE = re.compile(
 451             escCharLookBehind +
 452             r'(?P<startToken>' + escapeRegexChars(self.setting('cheetahVarStartToken')) + ')' +
 453             r'(?P<cacheToken>' + cacheToken + ')' +
 454             #r'\[[ \t\f]*'
 455             r'(?:\{|\(|\[)[ \t\f]*'
 456             + r'(?=[^\)\}\]])'
 457             )
 458
 459         if self.setting('EOLSlurpToken'):
 460             self.EOLSlurpRE = re.compile(
 461                 escapeRegexChars(self.setting('EOLSlurpToken'))
 462                 + r'[ \t\f]*'
 463                 + r'(?:'+EOL+')'
 464                 )
 465         else:
 466             self.EOLSlurpRE = None
 467
 468
 469     def _makeCommentREs(self):
 470         """Construct the regex bits that are used in comment parsing."""
 471         startTokenEsc = escapeRegexChars(self.setting('commentStartToken'))
 472         self.commentStartTokenRE = re.compile(escCharLookBehind + startTokenEsc)
 473         del startTokenEsc
 474
 475         startTokenEsc = escapeRegexChars(
 476             self.setting('multiLineCommentStartToken'))
 477         endTokenEsc = escapeRegexChars(
 478             self.setting('multiLineCommentEndToken'))
 479         self.multiLineCommentTokenStartRE = re.compile(escCharLookBehind +
 480                                                        startTokenEsc)
 481         self.multiLineCommentEndTokenRE = re.compile(escCharLookBehind +
 482                                                      endTokenEsc)
 483
 484     def _makeDirectiveREs(self):
 485         """Construct the regexs that are used in directive parsing."""
 486         startToken = self.setting('directiveStartToken')
 487         endToken = self.setting('directiveEndToken')
 488         startTokenEsc = escapeRegexChars(startToken)
 489         endTokenEsc = escapeRegexChars(endToken)
 490         validSecondCharsLookAhead = r'(?=[A-Za-z_@])'
 491         reParts = [escCharLookBehind, startTokenEsc]
 492         if self.setting('allowWhitespaceAfterDirectiveStartToken'):
 493             reParts.append('[ \t]*')
 494         reParts.append(validSecondCharsLookAhead)
 495         self.directiveStartTokenRE = re.compile(''.join(reParts))
 496         self.directiveEndTokenRE = re.compile(escCharLookBehind + endTokenEsc)
 497
 498     def _makePspREs(self):
 499         """Setup the regexs for PSP parsing."""
 500         startToken = self.setting('PSPStartToken')
 501         startTokenEsc = escapeRegexChars(startToken)
 502         self.PSPStartTokenRE = re.compile(escCharLookBehind + startTokenEsc)
 503         endToken = self.setting('PSPEndToken')
 504         endTokenEsc = escapeRegexChars(endToken)
 505         self.PSPEndTokenRE = re.compile(escCharLookBehind + endTokenEsc)
 506
 507
 508     def isLineClearToStartToken(self, pos=None):
 509         return self.isLineClearToPos(pos)
 510
 511     def matchTopLevelToken(self):
 512         """Returns the first match found from the following methods:
 513             self.matchCommentStartToken
 514             self.matchMultiLineCommentStartToken
 515             self.matchVariablePlaceholderStart
 516             self.matchExpressionPlaceholderStart
 517             self.matchDirective
 518             self.matchPSPStartToken
 519             self.matchEOLSlurpToken
 520
 521         Returns None if no match.
 522         """
 523         match = None
 524         if self.peek() in self._possibleNonStrConstantChars:
 525             for matcher in self._nonStrConstMatchers:
 526                 match = matcher()
 527                 if match:
 528                     break
 529         return match
 530
 531     def matchPyToken(self):
 532         match = pseudoprog.match(self.src(), self.pos())
 533
 534         if match and match.group() in tripleQuotedStringStarts:
 535             TQSmatch = tripleQuotedStringREs[match.group()].match(self.src(), self.pos())
 536             if TQSmatch:
 537                 return TQSmatch
 538         return match
 539
 540     def getPyToken(self):
 541         match = self.matchPyToken()
 542         if match is None:
 543             raise ParseError(self)
 544         elif match.group() in tripleQuotedStringStarts:
 545             raise ParseError(self, msg='Malformed triple-quoted string')
 546         return self.readTo(match.end())
 547
 548     def matchEOLSlurpToken(self):
 549         if self.EOLSlurpRE:
 550             return self.EOLSlurpRE.match(self.src(), self.pos())
 551
 552     def getEOLSlurpToken(self):
 553         match = self.matchEOLSlurpToken()
 554         if not match:
 555             raise ParseError(self, msg='Invalid EOL slurp token')
 556         return self.readTo(match.end())
 557
 558     def matchCommentStartToken(self):
 559         return self.commentStartTokenRE.match(self.src(), self.pos())
 560
 561     def getCommentStartToken(self):
 562         match = self.matchCommentStartToken()
 563         if not match:
 564             raise ParseError(self, msg='Invalid single-line comment start token')
 565         return self.readTo(match.end())
 566
 567     def matchMultiLineCommentStartToken(self):
 568         return self.multiLineCommentTokenStartRE.match(self.src(), self.pos())
 569
 570     def getMultiLineCommentStartToken(self):
 571         match = self.matchMultiLineCommentStartToken()
 572         if not match:
 573             raise ParseError(self, msg='Invalid multi-line comment start token')
 574         return self.readTo(match.end())
 575
 576     def matchMultiLineCommentEndToken(self):
 577         return self.multiLineCommentEndTokenRE.match(self.src(), self.pos())
 578
 579     def getMultiLineCommentEndToken(self):
 580         match = self.matchMultiLineCommentEndToken()
 581         if not match:
 582             raise ParseError(self, msg='Invalid multi-line comment end token')
 583         return self.readTo(match.end())
 584
 585     def getDottedName(self):
 586         srcLen = len(self)
 587         nameChunks = []
 588
 589         if not self.peek() in identchars:
 590             raise ParseError(self)
 591
 592         while self.pos() < srcLen:
 593             c = self.peek()
 594             if c in namechars:
 595                 nameChunk = self.getIdentifier()
 596                 nameChunks.append(nameChunk)
 597             elif c == '.':
 598                 if self.pos()+1 <srcLen and self.peek(1) in identchars:
 599                     nameChunks.append(self.getc())
 600                 else:
 601                     break
 602             else:
 603                 break
 604
 605         return ''.join(nameChunks)
 606
 607     def matchIdentifier(self):
 608         return identRE.match(self.src(), self.pos())
 609
 610     def getIdentifier(self):
 611         match = self.matchIdentifier()
 612         if not match:
 613             raise ParseError(self, msg='Invalid identifier')
 614         return self.readTo(match.end())
 615
 616     def matchOperator(self):
 617         match = self.matchPyToken()
 618         if match and match.group() not in operators:
 619             match = None
 620         return match
 621
 622     def getOperator(self):
 623         match = self.matchOperator()
 624         if not match:
 625             raise ParseError(self, msg='Expected operator')
 626         return self.readTo( match.end() )
 627
 628     def matchAssignmentOperator(self):
 629         match = self.matchPyToken()
 630         if match and match.group() not in assignmentOps:
 631             match = None
 632         return match
 633
 634     def getAssignmentOperator(self):
 635         match = self.matchAssignmentOperator()
 636         if not match:
 637             raise ParseError(self, msg='Expected assignment operator')
 638         return self.readTo( match.end() )
 639
 640     def matchDirective(self):
 641         """Returns False or the name of the directive matched.
 642         """
 643         startPos = self.pos()
 644         if not self.matchDirectiveStartToken():
 645             return False
 646         self.getDirectiveStartToken()
 647         directiveName = self.matchDirectiveName()
 648         self.setPos(startPos)
 649         return directiveName
 650
 651     def matchDirectiveName(self, directiveNameChars=identchars+'0123456789-@'):
 652         startPos = self.pos()
 653         directives = self._directiveNamesAndParsers.keys()
 654         possibleMatches = []
 655         name = ''
 656         while not self.atEnd():
 657             c = self.getc()
 658             if not c in directiveNameChars:
 659                 break
 660             name += c
 661             if name in directives:
 662                 possibleMatches.append(name)
 663
 664         possibleMatches.sort()
 665         possibleMatches.reverse() # longest match first
 666
 667         directiveName = False
 668         if possibleMatches:
 669             directiveName = possibleMatches[0]
 670
 671         self.setPos(startPos)
 672         return directiveName
 673
 674     def matchDirectiveStartToken(self):
 675         return self.directiveStartTokenRE.match(self.src(), self.pos())
 676
 677     def getDirectiveStartToken(self):
 678         match = self.matchDirectiveStartToken()
 679         if not match:
 680             raise ParseError(self, msg='Invalid directive start token')
 681         return self.readTo(match.end())
 682
 683     def matchDirectiveEndToken(self):
 684         return self.directiveEndTokenRE.match(self.src(), self.pos())
 685
 686     def getDirectiveEndToken(self):
 687         match = self.matchDirectiveEndToken()
 688         if not match:
 689             raise ParseError(self, msg='Invalid directive end token')
 690         return self.readTo(match.end())
 691
 692
 693     def matchColonForSingleLineShortFormDirective(self):
 694         if not self.atEnd() and self.peek()==':':
 695             restOfLine = self[self.pos()+1:self.findEOL()]
 696             restOfLine = restOfLine.strip()
 697             if not restOfLine:
 698                 return False
 699             elif self.commentStartTokenRE.match(restOfLine):
 700                 return False
 701             else: # non-whitespace, non-commment chars found
 702                 return True
 703         return False
 704
 705     def matchPSPStartToken(self):
 706         return self.PSPStartTokenRE.match(self.src(), self.pos())
 707
 708     def matchPSPEndToken(self):
 709         return self.PSPEndTokenRE.match(self.src(), self.pos())
 710
 711     def getPSPStartToken(self):
 712         match = self.matchPSPStartToken()
 713         if not match:
 714             raise ParseError(self, msg='Invalid psp start token')
 715         return self.readTo(match.end())
 716
 717     def getPSPEndToken(self):
 718         match = self.matchPSPEndToken()
 719         if not match:
 720             raise ParseError(self, msg='Invalid psp end token')
 721         return self.readTo(match.end())
 722
 723     def matchCheetahVarStart(self):
 724         """includes the enclosure and cache token"""
 725         return self.cheetahVarStartRE.match(self.src(), self.pos())
 726
 727     def matchCheetahVarStartToken(self):
 728         """includes the enclosure and cache token"""
 729         return self.cheetahVarStartTokenRE.match(self.src(), self.pos())
 730
 731     def matchCheetahVarInExpressionStartToken(self):
 732         """no enclosures or cache tokens allowed"""
 733         return self.cheetahVarInExpressionStartTokenRE.match(self.src(), self.pos())
 734
 735     def matchVariablePlaceholderStart(self):
 736         """includes the enclosure and cache token"""
 737         return self.cheetahVarStartRE.match(self.src(), self.pos())
 738
 739     def matchExpressionPlaceholderStart(self):
 740         """includes the enclosure and cache token"""
 741         return self.expressionPlaceholderStartRE.match(self.src(), self.pos())
 742
 743     def getCheetahVarStartToken(self):
 744         """just the start token, not the enclosure or cache token"""
 745         match = self.matchCheetahVarStartToken()
 746         if not match:
 747             raise ParseError(self, msg='Expected Cheetah $var start token')
 748         return self.readTo( match.end() )
 749
 750
 751     def getCacheToken(self):
 752         try:
 753             token = self.cacheTokenRE.match(self.src(), self.pos())
 754             self.setPos( token.end() )
 755             return token.group()
 756         except:
 757             raise ParseError(self, msg='Expected cache token')
 758
 759     def getSilentPlaceholderToken(self):
 760         try:
 761             token = self.silentPlaceholderTokenRE.match(self.src(), self.pos())
 762             self.setPos( token.end() )
 763             return token.group()
 764         except:
 765             raise ParseError(self, msg='Expected silent placeholder token')
 766
 767
 768
 769     def getTargetVarsList(self):
 770         varnames = []
 771         while not self.atEnd():
 772             if self.peek() in ' \t\f':
 773                 self.getWhiteSpace()
 774             elif self.peek() in '\r\n':
 775                 break
 776             elif self.startswith(','):
 777                 self.advance()
 778             elif self.startswith('in ') or self.startswith('in\t'):
 779                 break
 780             #elif self.matchCheetahVarStart():
 781             elif self.matchCheetahVarInExpressionStartToken():
 782                 self.getCheetahVarStartToken()
 783                 self.getSilentPlaceholderToken()
 784                 self.getCacheToken()
 785                 varnames.append( self.getDottedName() )
 786             elif self.matchIdentifier():
 787                 varnames.append( self.getDottedName() )
 788             else:
 789                 break
 790         return varnames
 791
 792     def getCheetahVar(self, plain=False, skipStartToken=False):
 793         """This is called when parsing inside expressions. Cache tokens are only
 794         valid in placeholders so this method discards any cache tokens found.
 795         """
 796         if not skipStartToken:
 797             self.getCheetahVarStartToken()
 798         self.getSilentPlaceholderToken()
 799         self.getCacheToken()
 800         return self.getCheetahVarBody(plain=plain)
 801
 802     def getCheetahVarBody(self, plain=False):
 803         # @@TR: this should be in the compiler
 804         return self._compiler.genCheetahVar(self.getCheetahVarNameChunks(), plain=plain)
 805
 806     def getCheetahVarNameChunks(self):
 807
 808         """
 809         nameChunks = list of Cheetah $var subcomponents represented as tuples
 810           [ (namemapperPart,autoCall,restOfName),
 811           ]
 812         where:
 813           namemapperPart = the dottedName base
 814           autocall = where NameMapper should use autocalling on namemapperPart
 815           restOfName = any arglist, index, or slice
 816
 817         If restOfName contains a call arglist (e.g. '(1234)') then autocall is
 818         False, otherwise it defaults to True.
 819
 820         EXAMPLE
 821         ------------------------------------------------------------------------
 822
 823         if the raw CheetahVar is
 824           $a.b.c[1].d().x.y.z
 825
 826         nameChunks is the list
 827           [ ('a.b.c',True,'[1]'),
 828             ('d',False,'()'),
 829             ('x.y.z',True,''),
 830           ]
 831
 832         """
 833
 834         chunks = []
 835         while self.pos() < len(self):
 836             rest = ''
 837             autoCall = True
 838             if not self.peek() in identchars + '.':
 839                 break
 840             elif self.peek() == '.':
 841
 842                 if self.pos()+1 < len(self) and self.peek(1) in identchars:
 843                     self.advance()  # discard the period as it isn't needed with NameMapper
 844                 else:
 845                     break
 846
 847             dottedName = self.getDottedName()
 848             if not self.atEnd() and self.peek() in '([':
 849                 if self.peek() == '(':
 850                     rest = self.getCallArgString()
 851                 else:
 852                     rest = self.getExpression(enclosed=True)
 853
 854                 period = max(dottedName.rfind('.'), 0)
 855                 if period:
 856                     chunks.append( (dottedName[:period], autoCall, '') )
 857                     dottedName = dottedName[period+1:]
 858                 if rest and rest[0]=='(':
 859                     autoCall = False
 860             chunks.append( (dottedName, autoCall, rest) )
 861
 862         return chunks
 863
 864
 865     def getCallArgString(self,
 866                          enclosures=[],  # list of tuples (char, pos), where char is ({ or [
 867                          useNameMapper=Unspecified):
 868
 869         """ Get a method/function call argument string.
 870
 871         This method understands *arg, and **kw
 872         """
 873
 874         # @@TR: this settings mangling should be removed
 875         if useNameMapper is not Unspecified:
 876             useNameMapper_orig = self.setting('useNameMapper')
 877             self.setSetting('useNameMapper', useNameMapper)
 878
 879         if enclosures:
 880             pass
 881         else:
 882             if not self.peek() == '(':
 883                 raise ParseError(self, msg="Expected '('")
 884             startPos = self.pos()
 885             self.getc()
 886             enclosures = [('(', startPos),
 887                           ]
 888
 889         argStringBits = ['(']
 890         addBit = argStringBits.append
 891
 892         while 1:
 893             if self.atEnd():
 894                 open = enclosures[-1][0]
 895                 close = closurePairsRev[open]
 896                 self.setPos(enclosures[-1][1])
 897                 raise ParseError(
 898                     self, msg="EOF was reached before a matching '" + close +
 899                     "' was found for the '" + open + "'")
 900
 901             c = self.peek()
 902             if c in ")}]": # get the ending enclosure and break
 903                 if not enclosures:
 904                     raise ParseError(self)
 905                 c = self.getc()
 906                 open = closurePairs[c]
 907                 if enclosures[-1][0] == open:
 908                     enclosures.pop()
 909                     addBit(')')
 910                     break
 911                 else:
 912                     raise ParseError(self)
 913             elif c in " \t\f\r\n":
 914                 addBit(self.getc())
 915             elif self.matchCheetahVarInExpressionStartToken():
 916                 startPos = self.pos()
 917                 codeFor1stToken = self.getCheetahVar()
 918                 WS = self.getWhiteSpace()
 919                 if not self.atEnd() and self.peek() == '=':
 920                     nextToken = self.getPyToken()
 921                     if nextToken == '=':
 922                         endPos = self.pos()
 923                         self.setPos(startPos)
 924                         codeFor1stToken = self.getCheetahVar(plain=True)
 925                         self.setPos(endPos)
 926
 927                     ## finally
 928                     addBit( codeFor1stToken + WS + nextToken )
 929                 else:
 930                     addBit( codeFor1stToken + WS)
 931             elif self.matchCheetahVarStart():
 932                 # it has syntax that is only valid at the top level
 933                 self._raiseErrorAboutInvalidCheetahVarSyntaxInExpr()
 934             else:
 935                 beforeTokenPos = self.pos()
 936                 token = self.getPyToken()
 937                 if token in ('{','(','['):
 938                     self.rev()
 939                     token = self.getExpression(enclosed=True)
 940                 token = self.transformToken(token, beforeTokenPos)
 941                 addBit(token)
 942
 943         if useNameMapper is not Unspecified:
 944             self.setSetting('useNameMapper', useNameMapper_orig) # @@TR: see comment above
 945
 946         return ''.join(argStringBits)
 947
 948     def getDefArgList(self, exitPos=None, useNameMapper=False):
 949
 950         """ Get an argument list. Can be used for method/function definition
 951         argument lists or for #directive argument lists. Returns a list of
 952         tuples in the form (argName, defVal=None) with one tuple for each arg
 953         name.
 954
 955         These defVals are always strings, so (argName, defVal=None) is safe even
 956         with a case like (arg1, arg2=None, arg3=1234*2), which would be returned as
 957         [('arg1', None),
 958          ('arg2', 'None'),
 959          ('arg3', '1234*2'),
 960         ]
 961
 962         This method understands *arg, and **kw
 963
 964         """
 965
 966         if self.peek() == '(':
 967             self.advance()
 968         else:
 969             exitPos = self.findEOL()  # it's a directive so break at the EOL
 970         argList = ArgList()
 971         onDefVal = False
 972
 973         # @@TR: this settings mangling should be removed
 974         useNameMapper_orig = self.setting('useNameMapper')
 975         self.setSetting('useNameMapper', useNameMapper)
 976
 977         while 1:
 978             if self.atEnd():
 979                 self.setPos(enclosures[-1][1])
 980                 raise ParseError(
 981                     self, msg="EOF was reached before a matching ')'"+
 982                     " was found for the '('")
 983
 984             if self.pos() == exitPos:
 985                 break
 986
 987             c = self.peek()
 988             if c == ")" or self.matchDirectiveEndToken():
 989                 break
 990             elif c == ":":
 991                 break
 992             elif c in " \t\f\r\n":
 993                 if onDefVal:
 994                     argList.addToDefVal(c)
 995                 self.advance()
 996             elif c == '=':
 997                 onDefVal = True
 998                 self.advance()
 999             elif c == ",":
1000                 argList.next()
1001                 onDefVal = False
1002                 self.advance()
1003             elif self.startswith(self.cheetahVarStartToken) and not onDefVal:
1004                 self.advance(len(self.cheetahVarStartToken))
1005             elif self.matchIdentifier() and not onDefVal:
1006                 argList.addArgName( self.getIdentifier() )
1007             elif onDefVal:
1008                 if self.matchCheetahVarInExpressionStartToken():
1009                     token = self.getCheetahVar()
1010                 elif self.matchCheetahVarStart():
1011                     # it has syntax that is only valid at the top level
1012                     self._raiseErrorAboutInvalidCheetahVarSyntaxInExpr()
1013                 else:
1014                     beforeTokenPos = self.pos()
1015                     token = self.getPyToken()
1016                     if token in ('{','(','['):
1017                         self.rev()
1018                         token = self.getExpression(enclosed=True)
1019                     token = self.transformToken(token, beforeTokenPos)
1020                 argList.addToDefVal(token)
1021             elif c == '*' and not onDefVal:
1022                 varName = self.getc()
1023                 if self.peek() == '*':
1024                     varName += self.getc()
1025                 if not self.matchIdentifier():
1026                     raise ParseError(self)
1027                 varName += self.getIdentifier()
1028                 argList.addArgName(varName)
1029             else:
1030                 raise ParseError(self)
1031
1032
1033         self.setSetting('useNameMapper', useNameMapper_orig) # @@TR: see comment above
1034         return argList.merge()
1035
1036     def getExpressionParts(self,
1037                            enclosed=False,
1038                            enclosures=None, # list of tuples (char, pos), where char is ({ or [
1039                            pyTokensToBreakAt=None, # only works if not enclosed
1040                            useNameMapper=Unspecified,
1041                            ):
1042
1043         """ Get a Cheetah expression that includes $CheetahVars and break at
1044         directive end tokens, the end of an enclosure, or at a specified
1045         pyToken.
1046         """
1047
1048         if useNameMapper is not Unspecified:
1049             useNameMapper_orig = self.setting('useNameMapper')
1050             self.setSetting('useNameMapper', useNameMapper)
1051
1052         if enclosures is None:
1053             enclosures = []
1054
1055         srcLen = len(self)
1056         exprBits = []
1057         while 1:
1058             if self.atEnd():
1059                 if enclosures:
1060                     open = enclosures[-1][0]
1061                     close = closurePairsRev[open]
1062                     self.setPos(enclosures[-1][1])
1063                     raise ParseError(
1064                         self, msg="EOF was reached before a matching '" + close +
1065                         "' was found for the '" + open + "'")
1066                 else:
1067                     break
1068
1069             c = self.peek()
1070             if c in "{([":
1071                 exprBits.append(c)
1072                 enclosures.append( (c, self.pos()) )
1073                 self.advance()
1074             elif enclosed and not enclosures:
1075                 break
1076             elif c in "])}":
1077                 if not enclosures:
1078                     raise ParseError(self)
1079                 open = closurePairs[c]
1080                 if enclosures[-1][0] == open:
1081                     enclosures.pop()
1082                     exprBits.append(c)
1083                 else:
1084                     open = enclosures[-1][0]
1085                     close = closurePairsRev[open]
1086                     row, col = self.getRowCol()
1087                     self.setPos(enclosures[-1][1])
1088                     raise ParseError(
1089                         self, msg= "A '" + c + "' was found at line " + str(row) +
1090                         ", col " + str(col) +
1091                         " before a matching '" + close +
1092                         "' was found\nfor the '" + open + "'")
1093                 self.advance()
1094
1095             elif c in " \f\t":
1096                 exprBits.append(self.getWhiteSpace())
1097             elif self.matchDirectiveEndToken() and not enclosures:
1098                 break
1099             elif c == "\\" and self.pos()+1 < srcLen:
1100                 eolMatch = EOLre.match(self.src(), self.pos()+1)
1101                 if not eolMatch:
1102                     self.advance()
1103                     raise ParseError(self, msg='Line ending expected')
1104                 self.setPos( eolMatch.end() )
1105             elif c in '\r\n':
1106                 if enclosures:
1107                     self.advance()
1108                 else:
1109                     break
1110             elif self.matchCheetahVarInExpressionStartToken():
1111                 expr = self.getCheetahVar()
1112                 exprBits.append(expr)
1113             elif self.matchCheetahVarStart():
1114                 # it has syntax that is only valid at the top level
1115                 self._raiseErrorAboutInvalidCheetahVarSyntaxInExpr()
1116             else:
1117                 beforeTokenPos = self.pos()
1118                 token = self.getPyToken()
1119                 if (not enclosures
1120                     and pyTokensToBreakAt
1121                     and token in pyTokensToBreakAt):
1122
1123                     self.setPos(beforeTokenPos)
1124                     break
1125
1126                 token = self.transformToken(token, beforeTokenPos)
1127
1128                 exprBits.append(token)
1129                 if identRE.match(token):
1130                     if token == 'for':
1131                         expr = self.getExpression(useNameMapper=False, pyTokensToBreakAt=['in'])
1132                         exprBits.append(expr)
1133                     else:
1134                         exprBits.append(self.getWhiteSpace())
1135                         if not self.atEnd() and self.peek() == '(':
1136                             exprBits.append(self.getCallArgString())
1137         ##
1138         if useNameMapper is not Unspecified:
1139             self.setSetting('useNameMapper', useNameMapper_orig) # @@TR: see comment above
1140         return exprBits
1141
1142     def getExpression(self,
1143                       enclosed=False,
1144                       enclosures=None, # list of tuples (char, pos), where # char is ({ or [
1145                       pyTokensToBreakAt=None,
1146                       useNameMapper=Unspecified,
1147                       ):
1148         """Returns the output of self.getExpressionParts() as a concatenated
1149         string rather than as a list.
1150         """
1151         return ''.join(self.getExpressionParts(
1152             enclosed=enclosed, enclosures=enclosures,
1153             pyTokensToBreakAt=pyTokensToBreakAt,
1154             useNameMapper=useNameMapper))
1155
1156
1157     def transformToken(self, token, beforeTokenPos):
1158         """Takes a token from the expression being parsed and performs and
1159         special transformations required by Cheetah.
1160
1161         At the moment only Cheetah's c'$placeholder strings' are transformed.
1162         """
1163         if token=='c' and not self.atEnd() and self.peek() in '\'"':
1164             nextToken = self.getPyToken()
1165             token = nextToken.upper()
1166             theStr = eval(token)
1167             endPos = self.pos()
1168             if not theStr:
1169                 return
1170
1171             if token.startswith(single3) or token.startswith(double3):
1172                 startPosIdx = 3
1173             else:
1174                 startPosIdx = 1
1175             #print 'CHEETAH STRING', nextToken, theStr, startPosIdx
1176             self.setPos(beforeTokenPos+startPosIdx+1)
1177             outputExprs = []
1178             strConst = ''
1179             while self.pos() < (endPos-startPosIdx):
1180                 if self.matchCheetahVarStart() or self.matchExpressionPlaceholderStart():
1181                     if strConst:
1182                         outputExprs.append(repr(strConst))
1183                         strConst = ''
1184                     placeholderExpr = self.getPlaceholder()
1185                     outputExprs.append('str('+placeholderExpr+')')
1186                 else:
1187                     strConst += self.getc()
1188             self.setPos(endPos)
1189             if strConst:
1190                 outputExprs.append(repr(strConst))
1191             #if not self.atEnd() and self.matches('.join('):
1192             #    print 'DEBUG***'
1193             token = "''.join(["+','.join(outputExprs)+"])"
1194         return token
1195
1196     def _raiseErrorAboutInvalidCheetahVarSyntaxInExpr(self):
1197         match = self.matchCheetahVarStart()
1198         groupdict = match.groupdict()
1199         if groupdict.get('cacheToken'):
1200             raise ParseError(
1201                 self,
1202                 msg='Cache tokens are not valid inside expressions. '
1203                 'Use them in top-level $placeholders only.')
1204         elif groupdict.get('enclosure'):
1205             raise ParseError(
1206                 self,
1207                 msg='Long-form placeholders - ${}, $(), $[], etc. are not valid inside expressions. '
1208                 'Use them in top-level $placeholders only.')
1209         else:
1210             raise ParseError(
1211                 self,
1212                 msg='This form of $placeholder syntax is not valid here.')
1213
1214
1215     def getPlaceholder(self, allowCacheTokens=False, plain=False, returnEverything=False):
1216         # filtered
1217         for callback in self.setting('preparsePlaceholderHooks'):
1218             callback(parser=self)
1219
1220         startPos = self.pos()
1221         lineCol = self.getRowCol(startPos)
1222         startToken = self.getCheetahVarStartToken()
1223         silentPlaceholderToken = self.getSilentPlaceholderToken()
1224         if silentPlaceholderToken:
1225             isSilentPlaceholder = True
1226         else:
1227             isSilentPlaceholder = False
1228
1229
1230         if allowCacheTokens:
1231             cacheToken = self.getCacheToken()
1232             cacheTokenParts = self.cacheTokenRE.match(cacheToken).groupdict()
1233         else:
1234             cacheTokenParts = {}
1235
1236         if self.peek() in '({[':
1237             pos = self.pos()
1238             enclosureOpenChar = self.getc()
1239             enclosures = [ (enclosureOpenChar, pos) ]
1240             self.getWhiteSpace()
1241         else:
1242             enclosures = []
1243
1244         filterArgs = None
1245         if self.matchIdentifier():
1246             nameChunks = self.getCheetahVarNameChunks()
1247             expr = self._compiler.genCheetahVar(nameChunks[:], plain=plain)
1248             restOfExpr = None
1249             if enclosures:
1250                 WS = self.getWhiteSpace()
1251                 expr += WS
1252                 if self.setting('allowPlaceholderFilterArgs') and self.peek()==',':
1253                     filterArgs = self.getCallArgString(enclosures=enclosures)[1:-1]
1254                 else:
1255                     if self.peek()==closurePairsRev[enclosureOpenChar]:
1256                         self.getc()
1257                     else:
1258                         restOfExpr = self.getExpression(enclosed=True, enclosures=enclosures)
1259                         if restOfExpr[-1] == closurePairsRev[enclosureOpenChar]:
1260                             restOfExpr = restOfExpr[:-1]
1261                         expr += restOfExpr
1262             rawPlaceholder = self[startPos: self.pos()]
1263         else:
1264             expr = self.getExpression(enclosed=True, enclosures=enclosures)
1265             if expr[-1] == closurePairsRev[enclosureOpenChar]:
1266                 expr = expr[:-1]
1267             rawPlaceholder=self[startPos: self.pos()]
1268
1269         expr = self._applyExpressionFilters(expr,'placeholder',
1270                                             rawExpr=rawPlaceholder,startPos=startPos)
1271         for callback in self.setting('postparsePlaceholderHooks'):
1272             callback(parser=self)
1273
1274         if returnEverything:
1275             return (expr, rawPlaceholder, lineCol, cacheTokenParts,
1276                     filterArgs, isSilentPlaceholder)
1277         else:
1278             return expr
1279
1280
1281 class _HighLevelParser(_LowLevelParser):
1282     """This class is a StateMachine for parsing Cheetah source and
1283     sending state dependent code generation commands to
1284     Cheetah.Compiler.Compiler.
1285     """
1286     def __init__(self, src, filename=None, breakPoint=None, compiler=None):
1287         _LowLevelParser.__init__(self, src, filename=filename, breakPoint=breakPoint)
1288         self.setSettingsManager(compiler)
1289         self._compiler = compiler
1290         self.setupState()
1291         self.configureParser()
1292
1293     def setupState(self):
1294         self._macros = {}
1295         self._macroDetails = {}
1296         self._openDirectivesStack = []
1297
1298     def cleanup(self):
1299         """Cleanup to remove any possible reference cycles
1300         """
1301         self._macros.clear()
1302         for macroname, macroDetails in self._macroDetails.items():
1303             macroDetails.template.shutdown()
1304             del macroDetails.template
1305         self._macroDetails.clear()
1306
1307     def configureParser(self):
1308         _LowLevelParser.configureParser(self)
1309         self._initDirectives()
1310
1311     def _initDirectives(self):
1312         def normalizeParserVal(val):
1313             if isinstance(val, (str,unicode)):
1314                 handler = getattr(self, val)
1315             elif type(val) in (ClassType, TypeType):
1316                 handler = val(self)
1317             elif callable(val):
1318                 handler = val
1319             elif val is None:
1320                 handler = val
1321             else:
1322                 raise Exception('Invalid parser/handler value %r for %s'%(val, name))
1323             return handler
1324
1325         normalizeHandlerVal = normalizeParserVal
1326
1327         _directiveNamesAndParsers = directiveNamesAndParsers.copy()
1328         customNamesAndParsers = self.setting('directiveNamesAndParsers',{})
1329         _directiveNamesAndParsers.update(customNamesAndParsers)
1330
1331         _endDirectiveNamesAndHandlers = endDirectiveNamesAndHandlers.copy()
1332         customNamesAndHandlers = self.setting('endDirectiveNamesAndHandlers',{})
1333         _endDirectiveNamesAndHandlers.update(customNamesAndHandlers)
1334
1335         self._directiveNamesAndParsers = {}
1336         for name, val in _directiveNamesAndParsers.items():
1337             if val in (False, 0):
1338                 continue
1339             self._directiveNamesAndParsers[name] = normalizeParserVal(val)
1340
1341         self._endDirectiveNamesAndHandlers = {}
1342         for name, val in _endDirectiveNamesAndHandlers.items():
1343             if val in (False, 0):
1344                 continue
1345             self._endDirectiveNamesAndHandlers[name] = normalizeHandlerVal(val)
1346
1347         self._closeableDirectives = ['def','block','closure','defmacro',
1348                                      'call',
1349                                      'capture',
1350                                      'cache',
1351                                      'filter',
1352                                      'if','unless',
1353                                      'for','while','repeat',
1354                                      'try',
1355                                      ]
1356         for directiveName in self.setting('closeableDirectives',[]):
1357             self._closeableDirectives.append(directiveName)
1358
1359
1360
1361         macroDirectives = self.setting('macroDirectives',{})
1362         from Cheetah.Macros.I18n import I18n
1363         macroDirectives['i18n'] = I18n
1364
1365
1366         for macroName, callback in macroDirectives.items():
1367             if type(callback) in (ClassType, TypeType):
1368                 callback = callback(parser=self)
1369             assert callback
1370             self._macros[macroName] = callback
1371             self._directiveNamesAndParsers[macroName] = self.eatMacroCall
1372
1373     def _applyExpressionFilters(self, expr, exprType, rawExpr=None, startPos=None):
1374         """Pipes cheetah expressions through a set of optional filter hooks.
1375
1376         The filters are functions which may modify the expressions or raise
1377         a ForbiddenExpression exception if the expression is not allowed.  They
1378         are defined in the compiler setting 'expressionFilterHooks'.
1379
1380         Some intended use cases:
1381
1382          - to implement 'restricted execution' safeguards in cases where you
1383            can't trust the author of the template.
1384
1385          - to enforce style guidelines
1386
1387         filter call signature:  (parser, expr, exprType, rawExpr=None, startPos=None)
1388          - parser is the Cheetah parser
1389          - expr is the expression to filter.  In some cases the parser will have
1390            already modified it from the original source code form.  For example,
1391            placeholders will have been translated into namemapper calls.  If you
1392            need to work with the original source, see rawExpr.
1393          - exprType is the name of the directive, 'psp', or 'placeholder'. All
1394            lowercase.  @@TR: These will eventually be replaced with a set of
1395            constants.
1396          - rawExpr is the original source string that Cheetah parsed.  This
1397            might be None in some cases.
1398          - startPos is the character position in the source string/file
1399            where the parser started parsing the current expression.
1400
1401         @@TR: I realize this use of the term 'expression' is a bit wonky as many
1402          of the 'expressions' are actually statements, but I haven't thought of
1403          a better name yet.  Suggestions?
1404         """
1405         for callback in self.setting('expressionFilterHooks'):
1406             expr = callback(parser=self, expr=expr,  exprType=exprType,
1407                             rawExpr=rawExpr, startPos=startPos)
1408         return expr
1409
1410     def _filterDisabledDirectives(self, directiveName):
1411         directiveName = directiveName.lower()
1412         if (directiveName in self.setting('disabledDirectives')
1413             or (self.setting('enabledDirectives')
1414                 and directiveName not in self.setting('enabledDirectives'))):
1415             for callback in self.setting('disabledDirectiveHooks'):
1416                 callback(parser=self, directiveName=directiveName)
1417             raise ForbiddenDirective(self, msg='This %r directive is disabled'%directiveName)
1418
1419     ## main parse loop
1420
1421     def parse(self, breakPoint=None, assertEmptyStack=True):
1422         if breakPoint:
1423             origBP = self.breakPoint()
1424             self.setBreakPoint(breakPoint)
1425             assertEmptyStack = False
1426
1427         while not self.atEnd():
1428             if self.matchCommentStartToken():
1429                 self.eatComment()
1430             elif self.matchMultiLineCommentStartToken():
1431                 self.eatMultiLineComment()
1432             elif self.matchVariablePlaceholderStart():
1433                 self.eatPlaceholder()
1434             elif self.matchExpressionPlaceholderStart():
1435                 self.eatPlaceholder()
1436             elif self.matchDirective():
1437                 self.eatDirective()
1438             elif self.matchPSPStartToken():
1439                 self.eatPSP()
1440             elif self.matchEOLSlurpToken():
1441                 self.eatEOLSlurpToken()
1442             else:
1443                 self.eatPlainText()
1444         if assertEmptyStack:
1445             self.assertEmptyOpenDirectivesStack()
1446         if breakPoint:
1447             self.setBreakPoint(origBP)
1448
1449     ## non-directive eat methods
1450
1451     def eatPlainText(self):
1452         startPos = self.pos()
1453         match = None
1454         while not self.atEnd():
1455             match = self.matchTopLevelToken()
1456             if match:
1457                 break
1458             else:
1459                 self.advance()
1460         strConst = self.readTo(self.pos(), start=startPos)
1461         self._compiler.addStrConst(strConst)
1462         return match
1463
1464     def eatComment(self):
1465         isLineClearToStartToken = self.isLineClearToStartToken()
1466         if isLineClearToStartToken:
1467             self._compiler.handleWSBeforeDirective()
1468         self.getCommentStartToken()
1469         comm = self.readToEOL(gobble=isLineClearToStartToken)
1470         self._compiler.addComment(comm)
1471
1472     def eatMultiLineComment(self):
1473         isLineClearToStartToken = self.isLineClearToStartToken()
1474         endOfFirstLine = self.findEOL()
1475
1476         self.getMultiLineCommentStartToken()
1477         endPos = startPos = self.pos()
1478         level = 1
1479         while 1:
1480             endPos = self.pos()
1481             if self.atEnd():
1482                 break
1483             if self.matchMultiLineCommentStartToken():
1484                 self.getMultiLineCommentStartToken()
1485                 level += 1
1486             elif self.matchMultiLineCommentEndToken():
1487                 self.getMultiLineCommentEndToken()
1488                 level -= 1
1489             if not level:
1490                 break
1491             self.advance()
1492         comm = self.readTo(endPos, start=startPos)
1493
1494         if not self.atEnd():
1495             self.getMultiLineCommentEndToken()
1496
1497         if (not self.atEnd()) and self.setting('gobbleWhitespaceAroundMultiLineComments'):
1498             restOfLine = self[self.pos():self.findEOL()]
1499             if not restOfLine.strip(): # WS only to EOL
1500                 self.readToEOL(gobble=isLineClearToStartToken)
1501
1502             if isLineClearToStartToken and (self.atEnd() or self.pos() > endOfFirstLine):
1503                 self._compiler.handleWSBeforeDirective()
1504
1505         self._compiler.addComment(comm)
1506
1507     def eatPlaceholder(self):
1508         (expr, rawPlaceholder,
1509          lineCol, cacheTokenParts,
1510          filterArgs, isSilentPlaceholder) = self.getPlaceholder(
1511             allowCacheTokens=True, returnEverything=True)
1512
1513         self._compiler.addPlaceholder(
1514             expr,
1515             filterArgs=filterArgs,
1516             rawPlaceholder=rawPlaceholder,
1517             cacheTokenParts=cacheTokenParts,
1518             lineCol=lineCol,
1519             silentMode=isSilentPlaceholder)
1520         return
1521
1522     def eatPSP(self):
1523         # filtered
1524         self._filterDisabledDirectives(directiveName='psp')
1525         self.getPSPStartToken()
1526         endToken = self.setting('PSPEndToken')
1527         startPos = self.pos()
1528         while not self.atEnd():
1529             if self.peek() == endToken[0]:
1530                 if self.matchPSPEndToken():
1531                     break
1532             self.advance()
1533         pspString = self.readTo(self.pos(), start=startPos).strip()
1534         pspString = self._applyExpressionFilters(pspString, 'psp', startPos=startPos)
1535         self._compiler.addPSP(pspString)
1536         self.getPSPEndToken()
1537
1538     ## generic directive eat methods
1539     _simpleIndentingDirectives = '''
1540     else elif for while repeat unless try except finally'''.split()
1541     _simpleExprDirectives = '''
1542     pass continue stop return yield break
1543     del assert raise
1544     silent echo
1545     import from'''.split()
1546     _directiveHandlerNames = {'import':'addImportStatement',
1547                               'from':'addImportStatement', }
1548     def eatDirective(self):
1549         directiveName = self.matchDirective()
1550         self._filterDisabledDirectives(directiveName)
1551
1552         for callback in self.setting('preparseDirectiveHooks'):
1553             callback(parser=self, directiveName=directiveName)
1554
1555         # subclasses can override the default behaviours here by providing an
1556         # eater method in self._directiveNamesAndParsers[directiveName]
1557         directiveParser = self._directiveNamesAndParsers.get(directiveName)
1558         if directiveParser:
1559             directiveParser()
1560         elif directiveName in self._simpleIndentingDirectives:
1561             handlerName = self._directiveHandlerNames.get(directiveName)
1562             if not handlerName:
1563                 handlerName = 'add'+directiveName.capitalize()
1564             handler = getattr(self._compiler, handlerName)
1565             self.eatSimpleIndentingDirective(directiveName, callback=handler)
1566         elif directiveName in self._simpleExprDirectives:
1567             handlerName = self._directiveHandlerNames.get(directiveName)
1568             if not handlerName:
1569                 handlerName = 'add'+directiveName.capitalize()
1570             handler = getattr(self._compiler, handlerName)
1571             if directiveName in ('silent', 'echo'):
1572                 includeDirectiveNameInExpr = False
1573             else:
1574                 includeDirectiveNameInExpr = True
1575             expr = self.eatSimpleExprDirective(
1576                 directiveName,
1577                 includeDirectiveNameInExpr=includeDirectiveNameInExpr)
1578             handler(expr)
1579         ##
1580         for callback in self.setting('postparseDirectiveHooks'):
1581             callback(parser=self, directiveName=directiveName)
1582
1583     def _eatRestOfDirectiveTag(self, isLineClearToStartToken, endOfFirstLinePos):
1584         foundComment = False
1585         if self.matchCommentStartToken():
1586             pos = self.pos()
1587             self.advance()
1588             if not self.matchDirective():
1589                 self.setPos(pos)
1590                 foundComment = True
1591                 self.eatComment() # this won't gobble the EOL
1592             else:
1593                 self.setPos(pos)
1594
1595         if not foundComment and self.matchDirectiveEndToken():
1596                 self.getDirectiveEndToken()
1597         elif isLineClearToStartToken and (not self.atEnd()) and self.peek() in '\r\n':
1598             # still gobble the EOL if a comment was found.
1599             self.readToEOL(gobble=True)
1600
1601         if isLineClearToStartToken and (self.atEnd() or self.pos() > endOfFirstLinePos):
1602             self._compiler.handleWSBeforeDirective()
1603
1604     def _eatToThisEndDirective(self, directiveName):
1605         finalPos = endRawPos = startPos = self.pos()
1606         directiveChar = self.setting('directiveStartToken')[0]
1607         isLineClearToStartToken = False
1608         while not self.atEnd():
1609             if self.peek() == directiveChar:
1610                 if self.matchDirective() == 'end':
1611                     endRawPos = self.pos()
1612                     self.getDirectiveStartToken()
1613                     self.advance(len('end'))
1614                     self.getWhiteSpace()
1615                     if self.startswith(directiveName):
1616                         if self.isLineClearToStartToken(endRawPos):
1617                             isLineClearToStartToken = True
1618                             endRawPos = self.findBOL(endRawPos)
1619                         self.advance(len(directiveName)) # to end of directiveName
1620                         self.getWhiteSpace()
1621                         finalPos = self.pos()
1622                         break
1623             self.advance()
1624             finalPos = endRawPos = self.pos()
1625
1626         textEaten = self.readTo(endRawPos, start=startPos)
1627         self.setPos(finalPos)
1628
1629         endOfFirstLinePos = self.findEOL()
1630
1631         if self.matchDirectiveEndToken():
1632             self.getDirectiveEndToken()
1633         elif isLineClearToStartToken and (not self.atEnd()) and self.peek() in '\r\n':
1634             self.readToEOL(gobble=True)
1635
1636         if isLineClearToStartToken and self.pos() > endOfFirstLinePos:
1637             self._compiler.handleWSBeforeDirective()
1638         return textEaten
1639
1640
1641     def eatSimpleExprDirective(self, directiveName, includeDirectiveNameInExpr=True):
1642         # filtered
1643         isLineClearToStartToken = self.isLineClearToStartToken()
1644         endOfFirstLine = self.findEOL()
1645         self.getDirectiveStartToken()
1646         if not includeDirectiveNameInExpr:
1647             self.advance(len(directiveName))
1648         startPos = self.pos()
1649         expr = self.getExpression().strip()
1650         directiveName = expr.split()[0]
1651         expr = self._applyExpressionFilters(expr, directiveName, startPos=startPos)
1652         if directiveName in self._closeableDirectives:
1653             self.pushToOpenDirectivesStack(directiveName)
1654         self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLine)
1655         return expr
1656
1657     def eatSimpleIndentingDirective(self, directiveName, callback,
1658                                     includeDirectiveNameInExpr=False):
1659         # filtered
1660         isLineClearToStartToken = self.isLineClearToStartToken()
1661         endOfFirstLinePos = self.findEOL()
1662         lineCol = self.getRowCol()
1663         self.getDirectiveStartToken()
1664         if directiveName not in 'else elif for while try except finally'.split():
1665             self.advance(len(directiveName))
1666         startPos = self.pos()
1667
1668         self.getWhiteSpace()
1669
1670         expr = self.getExpression(pyTokensToBreakAt=[':'])
1671         expr = self._applyExpressionFilters(expr, directiveName, startPos=startPos)
1672         if self.matchColonForSingleLineShortFormDirective():
1673             self.advance() # skip over :
1674             if directiveName in 'else elif except finally'.split():
1675                 callback(expr, dedent=False, lineCol=lineCol)
1676             else:
1677                 callback(expr, lineCol=lineCol)
1678
1679             self.getWhiteSpace(max=1)
1680             self.parse(breakPoint=self.findEOL(gobble=True))
1681             self._compiler.commitStrConst()
1682             self._compiler.dedent()
1683         else:
1684             if self.peek()==':':
1685                 self.advance()
1686             self.getWhiteSpace()
1687             self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
1688             if directiveName in self._closeableDirectives:
1689                 self.pushToOpenDirectivesStack(directiveName)
1690             callback(expr, lineCol=lineCol)
1691
1692     def eatEndDirective(self):
1693         isLineClearToStartToken = self.isLineClearToStartToken()
1694         self.getDirectiveStartToken()
1695         self.advance(3)                 # to end of 'end'
1696         self.getWhiteSpace()
1697         pos = self.pos()
1698         directiveName = False
1699         for key in self._endDirectiveNamesAndHandlers.keys():
1700             if self.find(key, pos) == pos:
1701                 directiveName = key
1702                 break
1703         if not directiveName:
1704             raise ParseError(self, msg='Invalid end directive')
1705
1706         endOfFirstLinePos = self.findEOL()
1707         self.getExpression() # eat in any extra comment-like crap
1708         self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
1709         if directiveName in self._closeableDirectives:
1710             self.popFromOpenDirectivesStack(directiveName)
1711
1712         # subclasses can override the default behaviours here by providing an
1713         # end-directive handler in self._endDirectiveNamesAndHandlers[directiveName]
1714         if self._endDirectiveNamesAndHandlers.get(directiveName):
1715             handler = self._endDirectiveNamesAndHandlers[directiveName]
1716             handler()
1717         elif directiveName in 'block capture cache call filter errorCatcher'.split():
1718             if key == 'block':
1719                 self._compiler.closeBlock()
1720             elif key == 'capture':
1721                 self._compiler.endCaptureRegion()
1722             elif key == 'cache':
1723                 self._compiler.endCacheRegion()
1724             elif key == 'call':
1725                 self._compiler.endCallRegion()
1726             elif key == 'filter':
1727                 self._compiler.closeFilterBlock()
1728             elif key == 'errorCatcher':
1729                 self._compiler.turnErrorCatcherOff()
1730         elif directiveName in 'while for if try repeat unless'.split():
1731             self._compiler.commitStrConst()
1732             self._compiler.dedent()
1733         elif directiveName=='closure':
1734             self._compiler.commitStrConst()
1735             self._compiler.dedent()
1736             # @@TR: temporary hack of useSearchList
1737             self.setSetting('useSearchList', self._useSearchList_orig)
1738
1739     ## specific directive eat methods
1740
1741     def eatBreakPoint(self):
1742         """Tells the parser to stop parsing at this point and completely ignore
1743         everything else.
1744
1745         This is a debugging tool.
1746         """
1747         self.setBreakPoint(self.pos())
1748
1749     def eatShbang(self):
1750         # filtered
1751         self.getDirectiveStartToken()
1752         self.advance(len('shBang'))
1753         self.getWhiteSpace()
1754         startPos = self.pos()
1755         shBang = self.readToEOL()
1756         shBang = self._applyExpressionFilters(shBang, 'shbang', startPos=startPos)
1757         self._compiler.setShBang(shBang.strip())
1758
1759     def eatEncoding(self):
1760         # filtered
1761         self.getDirectiveStartToken()
1762         self.advance(len('encoding'))
1763         self.getWhiteSpace()
1764         startPos = self.pos()
1765         encoding = self.readToEOL()
1766         encoding = self._applyExpressionFilters(encoding, 'encoding', startPos=startPos)
1767         self._compiler.setModuleEncoding(encoding.strip())
1768
1769     def eatCompiler(self):
1770         # filtered
1771         isLineClearToStartToken = self.isLineClearToStartToken()
1772         endOfFirstLine = self.findEOL()
1773         startPos = self.pos()
1774         self.getDirectiveStartToken()
1775         self.advance(len('compiler'))   # to end of 'compiler'
1776         self.getWhiteSpace()
1777
1778         startPos = self.pos()
1779         settingName = self.getIdentifier()
1780
1781         if settingName.lower() == 'reset':
1782             self.getExpression() # gobble whitespace & junk
1783             self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLine)
1784             self._initializeSettings()
1785             self.configureParser()
1786             return
1787
1788         self.getWhiteSpace()
1789         if self.peek() == '=':
1790             self.advance()
1791         else:
1792             raise ParserError(self)
1793         valueExpr = self.getExpression()
1794         endPos = self.pos()
1795
1796         # @@TR: it's unlikely that anyone apply filters would have left this
1797         # directive enabled:
1798         # @@TR: fix up filtering, regardless
1799         self._applyExpressionFilters('%s=%r'%(settingName, valueExpr),
1800                                      'compiler', startPos=startPos)
1801
1802         self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLine)
1803         try:
1804             self._compiler.setCompilerSetting(settingName, valueExpr)
1805         except:
1806             out = sys.stderr
1807             print >> out, 'An error occurred while processing the following #compiler directive.'
1808             print >> out, '-'*80
1809             print >> out, self[startPos:endPos]
1810             print >> out, '-'*80
1811             print >> out, 'Please check the syntax of these settings.'
1812             print >> out, 'A full Python exception traceback follows.'
1813             raise
1814
1815
1816     def eatCompilerSettings(self):
1817         # filtered
1818         isLineClearToStartToken = self.isLineClearToStartToken()
1819         endOfFirstLine = self.findEOL()
1820         self.getDirectiveStartToken()
1821         self.advance(len('compiler-settings'))   # to end of 'settings'
1822
1823         keywords = self.getTargetVarsList()
1824         self.getExpression()            # gobble any garbage
1825
1826         self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLine)
1827
1828         if 'reset' in keywords:
1829             self._compiler._initializeSettings()
1830             self.configureParser()
1831             # @@TR: this implies a single-line #compiler-settings directive, and
1832             # thus we should parse forward for an end directive.
1833             # Subject to change in the future
1834             return
1835         startPos = self.pos()
1836         settingsStr = self._eatToThisEndDirective('compiler-settings')
1837         settingsStr = self._applyExpressionFilters(settingsStr, 'compilerSettings',
1838                                                    startPos=startPos)
1839         try:
1840             self._compiler.setCompilerSettings(keywords=keywords, settingsStr=settingsStr)
1841         except:
1842             out = sys.stderr
1843             print >> out, 'An error occurred while processing the following compiler settings.'
1844             print >> out, '-'*80
1845             print >> out, settingsStr.strip()
1846             print >> out, '-'*80
1847             print >> out, 'Please check the syntax of these settings.'
1848             print >> out, 'A full Python exception traceback follows.'
1849             raise
1850
1851     def eatAttr(self):
1852         # filtered
1853         isLineClearToStartToken = self.isLineClearToStartToken()
1854         endOfFirstLinePos = self.findEOL()
1855         startPos = self.pos()
1856         self.getDirectiveStartToken()
1857         self.advance(len('attr'))
1858         self.getWhiteSpace()
1859         startPos = self.pos()
1860         if self.matchCheetahVarStart():
1861             self.getCheetahVarStartToken()
1862         attribName = self.getIdentifier()
1863         self.getWhiteSpace()
1864         self.getAssignmentOperator()
1865         expr = self.getExpression()
1866         expr = self._applyExpressionFilters(expr, 'attr', startPos=startPos)
1867         self._compiler.addAttribute(attribName, expr)
1868         self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
1869
1870     def eatDecorator(self):
1871         isLineClearToStartToken = self.isLineClearToStartToken()
1872         endOfFirstLinePos = self.findEOL()
1873         startPos = self.pos()
1874         self.getDirectiveStartToken()
1875         #self.advance() # eat @
1876         startPos = self.pos()
1877         decoratorExpr = self.getExpression()
1878         decoratorExpr = self._applyExpressionFilters(decoratorExpr, 'decorator', startPos=startPos)
1879         self._compiler.addDecorator(decoratorExpr)
1880         self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
1881         self.getWhiteSpace()
1882
1883         directiveName = self.matchDirective()
1884         if not directiveName or directiveName not in ('def', 'block', 'closure'):
1885             raise ParseError(self, msg='Expected #def, #block or #closure')
1886         self.eatDirective()
1887
1888     def eatDef(self):
1889         # filtered
1890         self._eatDefOrBlock('def')
1891
1892     def eatBlock(self):
1893         # filtered
1894         startPos = self.pos()
1895         methodName, rawSignature = self._eatDefOrBlock('block')
1896         self._compiler._blockMetaData[methodName] = {
1897             'raw':rawSignature,
1898             'lineCol':self.getRowCol(startPos),
1899             }
1900
1901     def eatClosure(self):
1902         # filtered
1903         self._eatDefOrBlock('closure')
1904
1905     def _eatDefOrBlock(self, directiveName):
1906         # filtered
1907         assert directiveName in ('def','block','closure')
1908         isLineClearToStartToken = self.isLineClearToStartToken()
1909         endOfFirstLinePos = self.findEOL()
1910         startPos = self.pos()
1911         self.getDirectiveStartToken()
1912         self.advance(len(directiveName))
1913         self.getWhiteSpace()
1914         if self.matchCheetahVarStart():
1915             self.getCheetahVarStartToken()
1916         methodName = self.getIdentifier()
1917         self.getWhiteSpace()
1918         if self.peek() == '(':
1919             argsList = self.getDefArgList()
1920             self.advance()              # past the closing ')'
1921             if argsList and argsList[0][0] == 'self':
1922                 del argsList[0]
1923         else:
1924             argsList=[]
1925
1926         def includeBlockMarkers():
1927             if self.setting('includeBlockMarkers'):
1928                 startMarker = self.setting('blockMarkerStart')
1929                 self._compiler.addStrConst(startMarker[0] + methodName + startMarker[1])
1930
1931         # @@TR: fix up filtering
1932         self._applyExpressionFilters(self[startPos:self.pos()], 'def', startPos=startPos)
1933
1934         if self.matchColonForSingleLineShortFormDirective():
1935             isNestedDef = (self.setting('allowNestedDefScopes')
1936                            and [name for name in self._openDirectivesStack if name=='def'])
1937             self.getc()
1938             rawSignature = self[startPos:endOfFirstLinePos]
1939             self._eatSingleLineDef(directiveName=directiveName,
1940                                    methodName=methodName,
1941                                    argsList=argsList,
1942                                    startPos=startPos,
1943                                    endPos=endOfFirstLinePos)
1944             if directiveName == 'def' and not isNestedDef:
1945                 #@@TR: must come before _eatRestOfDirectiveTag ... for some reason
1946                 self._compiler.closeDef()
1947             elif directiveName == 'block':
1948                 includeBlockMarkers()
1949                 self._compiler.closeBlock()
1950             elif directiveName == 'closure' or isNestedDef:
1951                 self._compiler.dedent()
1952
1953             self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
1954         else:
1955             if self.peek()==':':
1956                 self.getc()
1957             self.pushToOpenDirectivesStack(directiveName)
1958             rawSignature = self[startPos:self.pos()]
1959             self._eatMultiLineDef(directiveName=directiveName,
1960                                   methodName=methodName,
1961                                   argsList=argsList,
1962                                   startPos=startPos,
1963                                   isLineClearToStartToken=isLineClearToStartToken)
1964             if directiveName == 'block':
1965                 includeBlockMarkers()
1966
1967         return methodName, rawSignature
1968
1969     def _eatMultiLineDef(self, directiveName, methodName, argsList, startPos,
1970                          isLineClearToStartToken=False):
1971         # filtered in calling method
1972         self.getExpression()            # slurp up any garbage left at the end
1973         signature = self[startPos:self.pos()]
1974         endOfFirstLinePos = self.findEOL()
1975         self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
1976         parserComment = ('## CHEETAH: generated from ' + signature +
1977                          ' at line %s, col %s' % self.getRowCol(startPos)
1978                          + '.')
1979
1980         isNestedDef = (self.setting('allowNestedDefScopes')
1981                        and len([name for name in self._openDirectivesStack if name=='def'])>1)
1982         if directiveName=='block' or (directiveName=='def' and not isNestedDef):
1983             self._compiler.startMethodDef(methodName, argsList, parserComment)
1984         else: #closure
1985             self._useSearchList_orig = self.setting('useSearchList')
1986             self.setSetting('useSearchList', False)
1987             self._compiler.addClosure(methodName, argsList, parserComment)
1988
1989         return methodName
1990
1991     def _eatSingleLineDef(self, directiveName, methodName, argsList, startPos, endPos):
1992         # filtered in calling method
1993         fullSignature = self[startPos:endPos]
1994         parserComment = ('## Generated from ' + fullSignature +
1995                          ' at line %s, col %s' % self.getRowCol(startPos)
1996                          + '.')
1997         isNestedDef = (self.setting('allowNestedDefScopes')
1998                        and [name for name in self._openDirectivesStack if name=='def'])
1999         if directiveName=='block' or (directiveName=='def' and not isNestedDef):
2000             self._compiler.startMethodDef(methodName, argsList, parserComment)
2001         else: #closure
2002             # @@TR: temporary hack of useSearchList
2003             useSearchList_orig = self.setting('useSearchList')
2004             self.setSetting('useSearchList', False)
2005             self._compiler.addClosure(methodName, argsList, parserComment)
2006
2007         self.getWhiteSpace(max=1)
2008         self.parse(breakPoint=endPos)
2009         if directiveName=='closure' or isNestedDef: # @@TR: temporary hack of useSearchList
2010             self.setSetting('useSearchList', useSearchList_orig)
2011
2012     def eatExtends(self):
2013         # filtered
2014         isLineClearToStartToken = self.isLineClearToStartToken()
2015         endOfFirstLine = self.findEOL()
2016         self.getDirectiveStartToken()
2017         self.advance(len('extends'))
2018         self.getWhiteSpace()
2019         startPos = self.pos()
2020         if self.setting('allowExpressionsInExtendsDirective'):
2021             baseName = self.getExpression()
2022         else:
2023             baseName = self.getDottedName()
2024
2025         baseName = self._applyExpressionFilters(baseName, 'extends', startPos=startPos)
2026         self._compiler.setBaseClass(baseName) # in compiler
2027         self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLine)
2028
2029     def eatImplements(self):
2030         # filtered
2031         isLineClearToStartToken = self.isLineClearToStartToken()
2032         endOfFirstLine = self.findEOL()
2033         self.getDirectiveStartToken()
2034         self.advance(len('implements'))
2035         self.getWhiteSpace()
2036         startPos = self.pos()
2037         methodName = self.getIdentifier()
2038         if not self.atEnd() and self.peek() == '(':
2039             argsList = self.getDefArgList()
2040             self.advance()              # past the closing ')'
2041             if argsList and argsList[0][0] == 'self':
2042                 del argsList[0]
2043         else:
2044             argsList=[]
2045
2046         # @@TR: need to split up filtering of the methodname and the args
2047         #methodName = self._applyExpressionFilters(methodName, 'implements', startPos=startPos)
2048         self._applyExpressionFilters(self[startPos:self.pos()], 'implements', startPos=startPos)
2049
2050         self._compiler.setMainMethodName(methodName)
2051         self._compiler.setMainMethodArgs(argsList)
2052
2053         self.getExpression()  # throw away and unwanted crap that got added in
2054         self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLine)
2055
2056     def eatSet(self):
2057         # filtered
2058         isLineClearToStartToken = self.isLineClearToStartToken()
2059         endOfFirstLine = self.findEOL()
2060         self.getDirectiveStartToken()
2061         self.advance(3)
2062         self.getWhiteSpace()
2063         style = SET_LOCAL
2064         if self.startswith('local'):
2065             self.getIdentifier()
2066             self.getWhiteSpace()
2067         elif self.startswith('global'):
2068             self.getIdentifier()
2069             self.getWhiteSpace()
2070             style = SET_GLOBAL
2071         elif self.startswith('module'):
2072             self.getIdentifier()
2073             self.getWhiteSpace()
2074             style = SET_MODULE
2075
2076         startsWithDollar = self.matchCheetahVarStart()
2077         startPos = self.pos()
2078         LVALUE = self.getExpression(pyTokensToBreakAt=assignmentOps, useNameMapper=False).strip()
2079         OP = self.getAssignmentOperator()
2080         RVALUE = self.getExpression()
2081         expr = LVALUE + ' ' + OP + ' ' + RVALUE.strip()
2082
2083         expr = self._applyExpressionFilters(expr, 'set', startPos=startPos)
2084         self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLine)
2085
2086         class Components: pass # used for 'set global'
2087         exprComponents = Components()
2088         exprComponents.LVALUE = LVALUE
2089         exprComponents.OP = OP
2090         exprComponents.RVALUE = RVALUE
2091         self._compiler.addSet(expr, exprComponents, style)
2092
2093     def eatSlurp(self):
2094         if self.isLineClearToStartToken():
2095             self._compiler.handleWSBeforeDirective()
2096         self._compiler.commitStrConst()
2097         self.readToEOL(gobble=True)
2098
2099     def eatEOLSlurpToken(self):
2100         if self.isLineClearToStartToken():
2101             self._compiler.handleWSBeforeDirective()
2102         self._compiler.commitStrConst()
2103         self.readToEOL(gobble=True)
2104
2105     def eatRaw(self):
2106         isLineClearToStartToken = self.isLineClearToStartToken()
2107         endOfFirstLinePos = self.findEOL()
2108         self.getDirectiveStartToken()
2109         self.advance(len('raw'))
2110         self.getWhiteSpace()
2111         if self.matchColonForSingleLineShortFormDirective():
2112             self.advance() # skip over :
2113             self.getWhiteSpace(max=1)
2114             rawBlock = self.readToEOL(gobble=False)
2115         else:
2116             if self.peek()==':':
2117                 self.advance()
2118             self.getWhiteSpace()
2119             self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
2120             rawBlock = self._eatToThisEndDirective('raw')
2121         self._compiler.addRawText(rawBlock)
2122
2123     def eatInclude(self):
2124         # filtered
2125         isLineClearToStartToken = self.isLineClearToStartToken()
2126         endOfFirstLinePos = self.findEOL()
2127         self.getDirectiveStartToken()
2128         self.advance(len('include'))
2129
2130         self.getWhiteSpace()
2131         includeFrom = 'file'
2132         isRaw = False
2133         if self.startswith('raw'):
2134             self.advance(3)
2135             isRaw=True
2136
2137         self.getWhiteSpace()
2138         if self.startswith('source'):
2139             self.advance(len('source'))
2140             includeFrom = 'str'
2141             self.getWhiteSpace()
2142             if not self.peek() == '=':
2143                 raise ParseError(self)
2144             self.advance()
2145         startPos = self.pos()
2146         sourceExpr = self.getExpression()
2147         sourceExpr = self._applyExpressionFilters(sourceExpr, 'include', startPos=startPos)
2148         self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
2149         self._compiler.addInclude(sourceExpr, includeFrom, isRaw)
2150
2151
2152     def eatDefMacro(self):
2153         # @@TR: not filtered yet
2154         isLineClearToStartToken = self.isLineClearToStartToken()
2155         endOfFirstLinePos = self.findEOL()
2156         self.getDirectiveStartToken()
2157         self.advance(len('defmacro'))
2158
2159         self.getWhiteSpace()
2160         if self.matchCheetahVarStart():
2161             self.getCheetahVarStartToken()
2162         macroName = self.getIdentifier()
2163         self.getWhiteSpace()
2164         if self.peek() == '(':
2165             argsList = self.getDefArgList(useNameMapper=False)
2166             self.advance()              # past the closing ')'
2167             if argsList and argsList[0][0] == 'self':
2168                 del argsList[0]
2169         else:
2170             argsList=[]
2171
2172         assert not self._directiveNamesAndParsers.has_key(macroName)
2173         argsList.insert(0, ('src',None))
2174         argsList.append(('parser','None'))
2175         argsList.append(('macros','None'))
2176         argsList.append(('compilerSettings','None'))
2177         argsList.append(('isShortForm','None'))
2178         argsList.append(('EOLCharsInShortForm','None'))
2179         argsList.append(('startPos','None'))
2180         argsList.append(('endPos','None'))
2181
2182         if self.matchColonForSingleLineShortFormDirective():
2183             self.advance() # skip over :
2184             self.getWhiteSpace(max=1)
2185             macroSrc = self.readToEOL(gobble=False)
2186             self.readToEOL(gobble=True)
2187         else:
2188             if self.peek()==':':
2189                 self.advance()
2190             self.getWhiteSpace()
2191             self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
2192             macroSrc = self._eatToThisEndDirective('defmacro')
2193
2194         #print argsList
2195         normalizedMacroSrc = ''.join(
2196             ['%def callMacro('+','.join([defv and '%s=%s'%(n,defv) or n
2197                                          for n,defv in argsList])
2198              +')\n',
2199              macroSrc,
2200              '%end def'])
2201
2202
2203         from Cheetah.Template import Template
2204         templateAPIClass = self.setting('templateAPIClassForDefMacro', default=Template)
2205         compilerSettings = self.setting('compilerSettingsForDefMacro', default={})
2206         searchListForMacros = self.setting('searchListForDefMacro', default=[])
2207         searchListForMacros = list(searchListForMacros) # copy to avoid mutation bugs
2208         searchListForMacros.append({'macros':self._macros,
2209                                     'parser':self,
2210                                     'compilerSettings':self.settings(),
2211                                     })
2212
2213         templateAPIClass._updateSettingsWithPreprocessTokens(
2214             compilerSettings, placeholderToken='@', directiveToken='%')
2215         macroTemplateClass = templateAPIClass.compile(source=normalizedMacroSrc,
2216                                                       compilerSettings=compilerSettings)
2217         #print normalizedMacroSrc
2218         #t = macroTemplateClass()
2219         #print t.callMacro('src')
2220         #print t.generatedClassCode()
2221
2222         class MacroDetails: pass
2223         macroDetails = MacroDetails()
2224         macroDetails.macroSrc = macroSrc
2225         macroDetails.argsList = argsList
2226         macroDetails.template = macroTemplateClass(searchList=searchListForMacros)
2227
2228         self._macroDetails[macroName] = macroDetails
2229         self._macros[macroName] = macroDetails.template.callMacro
2230         self._directiveNamesAndParsers[macroName] = self.eatMacroCall
2231
2232     def eatMacroCall(self):
2233         isLineClearToStartToken = self.isLineClearToStartToken()
2234         endOfFirstLinePos = self.findEOL()
2235         startPos = self.pos()
2236         self.getDirectiveStartToken()
2237         macroName = self.getIdentifier()
2238         macro = self._macros[macroName]
2239         if hasattr(macro, 'parse'):
2240             return macro.parse(parser=self, startPos=startPos)
2241
2242         if hasattr(macro, 'parseArgs'):
2243             args = macro.parseArgs(parser=self, startPos=startPos)
2244         else:
2245             self.getWhiteSpace()
2246             args = self.getExpression(useNameMapper=False,
2247                                       pyTokensToBreakAt=[':']).strip()
2248
2249         if self.matchColonForSingleLineShortFormDirective():
2250             isShortForm = True
2251             self.advance() # skip over :
2252             self.getWhiteSpace(max=1)
2253             srcBlock = self.readToEOL(gobble=False)
2254             EOLCharsInShortForm = self.readToEOL(gobble=True)
2255             #self.readToEOL(gobble=False)
2256         else:
2257             isShortForm = False
2258             if self.peek()==':':
2259                 self.advance()
2260             self.getWhiteSpace()
2261             self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
2262             srcBlock = self._eatToThisEndDirective(macroName)
2263
2264
2265         if hasattr(macro, 'convertArgStrToDict'):
2266             kwArgs = macro.convertArgStrToDict(args, parser=self, startPos=startPos)
2267         else:
2268             def getArgs(*pargs, **kws):
2269                 return pargs, kws
2270             exec 'positionalArgs, kwArgs = getArgs(%(args)s)'%locals()
2271
2272         assert not kwArgs.has_key('src')
2273         kwArgs['src'] = srcBlock
2274
2275         if type(macro)==new.instancemethod:
2276             co = macro.im_func.func_code
2277         elif (hasattr(macro, '__call__')
2278               and hasattr(macro.__call__, 'im_func')):
2279             co = macro.__call__.im_func.func_code
2280         else:
2281             co = macro.func_code
2282         availableKwArgs = inspect.getargs(co)[0]
2283
2284         if 'parser' in availableKwArgs:
2285             kwArgs['parser'] = self
2286         if 'macros' in availableKwArgs:
2287             kwArgs['macros'] = self._macros
2288         if 'compilerSettings' in availableKwArgs:
2289             kwArgs['compilerSettings'] = self.settings()
2290         if 'isShortForm' in availableKwArgs:
2291             kwArgs['isShortForm'] = isShortForm
2292         if isShortForm and 'EOLCharsInShortForm' in availableKwArgs:
2293             kwArgs['EOLCharsInShortForm'] = EOLCharsInShortForm
2294
2295         if 'startPos' in availableKwArgs:
2296             kwArgs['startPos'] = startPos
2297         if 'endPos' in availableKwArgs:
2298             kwArgs['endPos'] = self.pos()
2299
2300         srcFromMacroOutput = macro(**kwArgs)
2301
2302         origParseSrc = self._src
2303         origBreakPoint = self.breakPoint()
2304         origPos = self.pos()
2305         # add a comment to the output about the macro src that is being parsed
2306         # or add a comment prefix to all the comments added by the compiler
2307         self._src = srcFromMacroOutput
2308         self.setPos(0)
2309         self.setBreakPoint(len(srcFromMacroOutput))
2310
2311         self.parse(assertEmptyStack=False)
2312
2313         self._src = origParseSrc
2314         self.setBreakPoint(origBreakPoint)
2315         self.setPos(origPos)
2316
2317
2318         #self._compiler.addRawText('end')
2319
2320     def eatCache(self):
2321         isLineClearToStartToken = self.isLineClearToStartToken()
2322         endOfFirstLinePos = self.findEOL()
2323         lineCol = self.getRowCol()
2324         self.getDirectiveStartToken()
2325         self.advance(len('cache'))
2326
2327         startPos = self.pos()
2328         argList = self.getDefArgList(useNameMapper=True)
2329         argList = self._applyExpressionFilters(argList, 'cache', startPos=startPos)
2330
2331         def startCache():
2332             cacheInfo = self._compiler.genCacheInfoFromArgList(argList)
2333             self._compiler.startCacheRegion(cacheInfo, lineCol)
2334
2335         if self.matchColonForSingleLineShortFormDirective():
2336             self.advance() # skip over :
2337             self.getWhiteSpace(max=1)
2338             startCache()
2339             self.parse(breakPoint=self.findEOL(gobble=True))
2340             self._compiler.endCacheRegion()
2341         else:
2342             if self.peek()==':':
2343                 self.advance()
2344             self.getWhiteSpace()
2345             self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
2346             self.pushToOpenDirectivesStack('cache')
2347             startCache()
2348
2349     def eatCall(self):
2350         # @@TR: need to enable single line version of this
2351         isLineClearToStartToken = self.isLineClearToStartToken()
2352         endOfFirstLinePos = self.findEOL()
2353         lineCol = self.getRowCol()
2354         self.getDirectiveStartToken()
2355         self.advance(len('call'))
2356         startPos = self.pos()
2357
2358         useAutocallingOrig = self.setting('useAutocalling')
2359         self.setSetting('useAutocalling', False)
2360         self.getWhiteSpace()
2361         if self.matchCheetahVarStart():
2362             functionName = self.getCheetahVar()
2363         else:
2364             functionName = self.getCheetahVar(plain=True, skipStartToken=True)
2365         self.setSetting('useAutocalling', useAutocallingOrig)
2366         # @@TR: fix up filtering
2367         self._applyExpressionFilters(self[startPos:self.pos()], 'call', startPos=startPos)
2368
2369         self.getWhiteSpace()
2370         args = self.getExpression(pyTokensToBreakAt=[':']).strip()
2371         if self.matchColonForSingleLineShortFormDirective():
2372             self.advance() # skip over :
2373             self._compiler.startCallRegion(functionName, args, lineCol)
2374             self.getWhiteSpace(max=1)
2375             self.parse(breakPoint=self.findEOL(gobble=False))
2376             self._compiler.endCallRegion()
2377         else:
2378             if self.peek()==':':
2379                 self.advance()
2380             self.getWhiteSpace()
2381             self.pushToOpenDirectivesStack("call")
2382             self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
2383             self._compiler.startCallRegion(functionName, args, lineCol)
2384
2385     def eatCallArg(self):
2386         isLineClearToStartToken = self.isLineClearToStartToken()
2387         endOfFirstLinePos = self.findEOL()
2388         lineCol = self.getRowCol()
2389         self.getDirectiveStartToken()
2390
2391         self.advance(len('arg'))
2392         startPos = self.pos()
2393         self.getWhiteSpace()
2394         argName = self.getIdentifier()
2395         self.getWhiteSpace()
2396         argName = self._applyExpressionFilters(argName, 'arg', startPos=startPos)
2397         self._compiler.setCallArg(argName, lineCol)
2398         if self.peek() == ':':
2399             self.getc()
2400         else:
2401             self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
2402
2403     def eatFilter(self):
2404         isLineClearToStartToken = self.isLineClearToStartToken()
2405         endOfFirstLinePos = self.findEOL()
2406
2407         self.getDirectiveStartToken()
2408         self.advance(len('filter'))
2409         self.getWhiteSpace()
2410         startPos = self.pos()
2411         if self.matchCheetahVarStart():
2412             isKlass = True
2413             theFilter = self.getExpression(pyTokensToBreakAt=[':'])
2414         else:
2415             isKlass = False
2416             theFilter = self.getIdentifier()
2417             self.getWhiteSpace()
2418         theFilter = self._applyExpressionFilters(theFilter, 'filter', startPos=startPos)
2419
2420         if self.matchColonForSingleLineShortFormDirective():
2421             self.advance() # skip over :
2422             self.getWhiteSpace(max=1)
2423             self._compiler.setFilter(theFilter, isKlass)
2424             self.parse(breakPoint=self.findEOL(gobble=False))
2425             self._compiler.closeFilterBlock()
2426         else:
2427             if self.peek()==':':
2428                 self.advance()
2429             self.getWhiteSpace()
2430             self.pushToOpenDirectivesStack("filter")
2431             self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
2432             self._compiler.setFilter(theFilter, isKlass)
2433
2434     def eatErrorCatcher(self):
2435         isLineClearToStartToken = self.isLineClearToStartToken()
2436         endOfFirstLinePos = self.findEOL()
2437         self.getDirectiveStartToken()
2438         self.advance(len('errorCatcher'))
2439         self.getWhiteSpace()
2440         startPos = self.pos()
2441         errorCatcherName = self.getIdentifier()
2442         errorCatcherName = self._applyExpressionFilters(
2443             errorCatcherName, 'errorcatcher', startPos=startPos)
2444         self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
2445         self._compiler.setErrorCatcher(errorCatcherName)
2446
2447     def eatCapture(self):
2448         # @@TR:  this could be refactored to use the code in eatSimpleIndentingDirective
2449         # filtered
2450         isLineClearToStartToken = self.isLineClearToStartToken()
2451         endOfFirstLinePos = self.findEOL()
2452         lineCol = self.getRowCol()
2453
2454         self.getDirectiveStartToken()
2455         self.advance(len('capture'))
2456         startPos = self.pos()
2457         self.getWhiteSpace()
2458
2459         expr = self.getExpression(pyTokensToBreakAt=[':'])
2460         expr = self._applyExpressionFilters(expr, 'capture', startPos=startPos)
2461         if self.matchColonForSingleLineShortFormDirective():
2462             self.advance() # skip over :
2463             self._compiler.startCaptureRegion(assignTo=expr, lineCol=lineCol)
2464             self.getWhiteSpace(max=1)
2465             self.parse(breakPoint=self.findEOL(gobble=False))
2466             self._compiler.endCaptureRegion()
2467         else:
2468             if self.peek()==':':
2469                 self.advance()
2470             self.getWhiteSpace()
2471             self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
2472             self.pushToOpenDirectivesStack("capture")
2473             self._compiler.startCaptureRegion(assignTo=expr, lineCol=lineCol)
2474
2475
2476     def eatIf(self):
2477         # filtered
2478         isLineClearToStartToken = self.isLineClearToStartToken()
2479         endOfFirstLine = self.findEOL()
2480         lineCol = self.getRowCol()
2481         self.getDirectiveStartToken()
2482         startPos = self.pos()
2483
2484         expressionParts = self.getExpressionParts(pyTokensToBreakAt=[':'])
2485         expr = ''.join(expressionParts).strip()
2486         expr = self._applyExpressionFilters(expr, 'if', startPos=startPos)
2487
2488         isTernaryExpr = ('then' in expressionParts and 'else' in expressionParts)
2489         if isTernaryExpr:
2490             conditionExpr = []
2491             trueExpr = []
2492             falseExpr = []
2493             currentExpr = conditionExpr
2494             for part in expressionParts:
2495                 if part.strip()=='then':
2496                     currentExpr = trueExpr
2497                 elif part.strip()=='else':
2498                     currentExpr = falseExpr
2499                 else:
2500                     currentExpr.append(part)
2501
2502             conditionExpr = ''.join(conditionExpr)
2503             trueExpr = ''.join(trueExpr)
2504             falseExpr = ''.join(falseExpr)
2505             self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLine)
2506             self._compiler.addTernaryExpr(conditionExpr, trueExpr, falseExpr, lineCol=lineCol)
2507         elif self.matchColonForSingleLineShortFormDirective():
2508             self.advance() # skip over :
2509             self._compiler.addIf(expr, lineCol=lineCol)
2510             self.getWhiteSpace(max=1)
2511             self.parse(breakPoint=self.findEOL(gobble=True))
2512             self._compiler.commitStrConst()
2513             self._compiler.dedent()
2514         else:
2515             if self.peek()==':':
2516                 self.advance()
2517             self.getWhiteSpace()
2518             self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLine)
2519             self.pushToOpenDirectivesStack('if')
2520             self._compiler.addIf(expr, lineCol=lineCol)
2521
2522     ## end directive handlers
2523     def handleEndDef(self):
2524         isNestedDef = (self.setting('allowNestedDefScopes')
2525                        and [name for name in self._openDirectivesStack if name=='def'])
2526         if not isNestedDef:
2527             self._compiler.closeDef()
2528         else:
2529             # @@TR: temporary hack of useSearchList
2530             self.setSetting('useSearchList', self._useSearchList_orig)
2531             self._compiler.commitStrConst()
2532             self._compiler.dedent()
2533     ###
2534
2535     def pushToOpenDirectivesStack(self, directiveName):
2536         assert directiveName in self._closeableDirectives
2537         self._openDirectivesStack.append(directiveName)
2538
2539     def popFromOpenDirectivesStack(self, directiveName):
2540         if not self._openDirectivesStack:
2541             raise ParseError(self, msg="#end found, but nothing to end")
2542
2543         if self._openDirectivesStack[-1] == directiveName:
2544             del self._openDirectivesStack[-1]
2545         else:
2546             raise ParseError(self, msg="#end %s found, expected #end %s" %(
2547                 directiveName, self._openDirectivesStack[-1]))
2548
2549     def assertEmptyOpenDirectivesStack(self):
2550         if self._openDirectivesStack:
2551             errorMsg = (
2552                 "Some #directives are missing their corresponding #end ___ tag: %s" %(
2553                 ', '.join(self._openDirectivesStack)))
2554             raise ParseError(self, msg=errorMsg)
2555
2556 ##################################################
2557 ## Make an alias to export
2558 Parser = _HighLevelParser