applied swfrender --help patch from Sergey Vinokurov
[swftools.git] / rendertest / athana.py
blob5edd53297023544ff42822fb2f0c0148c9f419eb
1 #!/usr/bin/python
2 """
3 Athana - standalone web server including the TAL template language
5 Copyright (C) 2007 Matthias Kramm <kramm@in.tum.de>
7 This program is free software: you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation, either version 3 of the License, or
10 (at your option) any later version.
12 This program is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with this program. If not, see <http://www.gnu.org/licenses/>.
19 """
21 #===============================================================
23 # Athana
25 # A standalone webserver based on Medusa and the Zope TAL Parser
27 # This file is distributed under the GPL, see file COPYING for details.
29 #===============================================================
30 """
31 Parse HTML and compile to TALInterpreter intermediate code.
32 """
34 RCS_ID = '$Id: athana.py,v 1.15 2007/11/23 10:13:32 kramm Exp $'
36 import sys
38 from HTMLParser import HTMLParser, HTMLParseError
40 BOOLEAN_HTML_ATTRS = [
41 "compact", "nowrap", "ismap", "declare", "noshade", "checked",
42 "disabled", "readonly", "multiple", "selected", "noresize",
43 "defer"
46 EMPTY_HTML_TAGS = [
47 "base", "meta", "link", "hr", "br", "param", "img", "area",
48 "input", "col", "basefont", "isindex", "frame",
51 PARA_LEVEL_HTML_TAGS = [
52 "h1", "h2", "h3", "h4", "h5", "h6", "p",
55 BLOCK_CLOSING_TAG_MAP = {
56 "tr": ("tr", "td", "th"),
57 "td": ("td", "th"),
58 "th": ("td", "th"),
59 "li": ("li",),
60 "dd": ("dd", "dt"),
61 "dt": ("dd", "dt"),
64 BLOCK_LEVEL_HTML_TAGS = [
65 "blockquote", "table", "tr", "th", "td", "thead", "tfoot", "tbody",
66 "noframe", "ul", "ol", "li", "dl", "dt", "dd", "div",
69 TIGHTEN_IMPLICIT_CLOSE_TAGS = (PARA_LEVEL_HTML_TAGS
70 + BLOCK_CLOSING_TAG_MAP.keys())
73 class NestingError(HTMLParseError):
74 """Exception raised when elements aren't properly nested."""
76 def __init__(self, tagstack, endtag, position=(None, None)):
77 self.endtag = endtag
78 if tagstack:
79 if len(tagstack) == 1:
80 msg = ('Open tag <%s> does not match close tag </%s>'
81 % (tagstack[0], endtag))
82 else:
83 msg = ('Open tags <%s> do not match close tag </%s>'
84 % ('>, <'.join(tagstack), endtag))
85 else:
86 msg = 'No tags are open to match </%s>' % endtag
87 HTMLParseError.__init__(self, msg, position)
89 class EmptyTagError(NestingError):
90 """Exception raised when empty elements have an end tag."""
92 def __init__(self, tag, position=(None, None)):
93 self.tag = tag
94 msg = 'Close tag </%s> should be removed' % tag
95 HTMLParseError.__init__(self, msg, position)
97 class OpenTagError(NestingError):
98 """Exception raised when a tag is not allowed in another tag."""
100 def __init__(self, tagstack, tag, position=(None, None)):
101 self.tag = tag
102 msg = 'Tag <%s> is not allowed in <%s>' % (tag, tagstack[-1])
103 HTMLParseError.__init__(self, msg, position)
105 class HTMLTALParser(HTMLParser):
108 def __init__(self, gen=None):
109 HTMLParser.__init__(self)
110 if gen is None:
111 gen = TALGenerator(xml=0)
112 self.gen = gen
113 self.tagstack = []
114 self.nsstack = []
115 self.nsdict = {'tal': ZOPE_TAL_NS,
116 'metal': ZOPE_METAL_NS,
117 'i18n': ZOPE_I18N_NS,
120 def parseFile(self, file):
121 f = open(file)
122 data = f.read()
123 f.close()
124 try:
125 self.parseString(data)
126 except TALError, e:
127 e.setFile(file)
128 raise
130 def parseString(self, data):
131 self.feed(data)
132 self.close()
133 while self.tagstack:
134 self.implied_endtag(self.tagstack[-1], 2)
135 assert self.nsstack == [], self.nsstack
137 def getCode(self):
138 return self.gen.getCode()
140 def getWarnings(self):
141 return ()
144 def handle_starttag(self, tag, attrs):
145 self.close_para_tags(tag)
146 self.scan_xmlns(attrs)
147 tag, attrlist, taldict, metaldict, i18ndict \
148 = self.process_ns(tag, attrs)
149 if tag in EMPTY_HTML_TAGS and taldict.get("content"):
150 raise TALError(
151 "empty HTML tags cannot use tal:content: %s" % `tag`,
152 self.getpos())
153 self.tagstack.append(tag)
154 self.gen.emitStartElement(tag, attrlist, taldict, metaldict, i18ndict,
155 self.getpos())
156 if tag in EMPTY_HTML_TAGS:
157 self.implied_endtag(tag, -1)
159 def handle_startendtag(self, tag, attrs):
160 self.close_para_tags(tag)
161 self.scan_xmlns(attrs)
162 tag, attrlist, taldict, metaldict, i18ndict \
163 = self.process_ns(tag, attrs)
164 if taldict.get("content"):
165 if tag in EMPTY_HTML_TAGS:
166 raise TALError(
167 "empty HTML tags cannot use tal:content: %s" % `tag`,
168 self.getpos())
169 self.gen.emitStartElement(tag, attrlist, taldict, metaldict,
170 i18ndict, self.getpos())
171 self.gen.emitEndElement(tag, implied=-1)
172 else:
173 self.gen.emitStartElement(tag, attrlist, taldict, metaldict,
174 i18ndict, self.getpos(), isend=1)
175 self.pop_xmlns()
177 def handle_endtag(self, tag):
178 if tag in EMPTY_HTML_TAGS:
179 raise EmptyTagError(tag, self.getpos())
180 self.close_enclosed_tags(tag)
181 self.gen.emitEndElement(tag)
182 self.pop_xmlns()
183 self.tagstack.pop()
185 def close_para_tags(self, tag):
186 if tag in EMPTY_HTML_TAGS:
187 return
188 close_to = -1
189 if BLOCK_CLOSING_TAG_MAP.has_key(tag):
190 blocks_to_close = BLOCK_CLOSING_TAG_MAP[tag]
191 for i in range(len(self.tagstack)):
192 t = self.tagstack[i]
193 if t in blocks_to_close:
194 if close_to == -1:
195 close_to = i
196 elif t in BLOCK_LEVEL_HTML_TAGS:
197 close_to = -1
198 elif tag in PARA_LEVEL_HTML_TAGS + BLOCK_LEVEL_HTML_TAGS:
199 i = len(self.tagstack) - 1
200 while i >= 0:
201 closetag = self.tagstack[i]
202 if closetag in BLOCK_LEVEL_HTML_TAGS:
203 break
204 if closetag in PARA_LEVEL_HTML_TAGS:
205 if closetag != "p":
206 raise OpenTagError(self.tagstack, tag, self.getpos())
207 close_to = i
208 i = i - 1
209 if close_to >= 0:
210 while len(self.tagstack) > close_to:
211 self.implied_endtag(self.tagstack[-1], 1)
213 def close_enclosed_tags(self, tag):
214 if tag not in self.tagstack:
215 raise NestingError(self.tagstack, tag, self.getpos())
216 while tag != self.tagstack[-1]:
217 self.implied_endtag(self.tagstack[-1], 1)
218 assert self.tagstack[-1] == tag
220 def implied_endtag(self, tag, implied):
221 assert tag == self.tagstack[-1]
222 assert implied in (-1, 1, 2)
223 isend = (implied < 0)
224 if tag in TIGHTEN_IMPLICIT_CLOSE_TAGS:
225 white = self.gen.unEmitWhitespace()
226 else:
227 white = None
228 self.gen.emitEndElement(tag, isend=isend, implied=implied)
229 if white:
230 self.gen.emitRawText(white)
231 self.tagstack.pop()
232 self.pop_xmlns()
234 def handle_charref(self, name):
235 self.gen.emitRawText("&#%s;" % name)
237 def handle_entityref(self, name):
238 self.gen.emitRawText("&%s;" % name)
240 def handle_data(self, data):
241 self.gen.emitRawText(data)
243 def handle_comment(self, data):
244 self.gen.emitRawText("<!--%s-->" % data)
246 def handle_decl(self, data):
247 self.gen.emitRawText("<!%s>" % data)
249 def handle_pi(self, data):
250 self.gen.emitRawText("<?%s>" % data)
253 def scan_xmlns(self, attrs):
254 nsnew = {}
255 for key, value in attrs:
256 if key.startswith("xmlns:"):
257 nsnew[key[6:]] = value
258 if nsnew:
259 self.nsstack.append(self.nsdict)
260 self.nsdict = self.nsdict.copy()
261 self.nsdict.update(nsnew)
262 else:
263 self.nsstack.append(self.nsdict)
265 def pop_xmlns(self):
266 self.nsdict = self.nsstack.pop()
268 def fixname(self, name):
269 if ':' in name:
270 prefix, suffix = name.split(':', 1)
271 if prefix == 'xmlns':
272 nsuri = self.nsdict.get(suffix)
273 if nsuri in (ZOPE_TAL_NS, ZOPE_METAL_NS, ZOPE_I18N_NS):
274 return name, name, prefix
275 else:
276 nsuri = self.nsdict.get(prefix)
277 if nsuri == ZOPE_TAL_NS:
278 return name, suffix, 'tal'
279 elif nsuri == ZOPE_METAL_NS:
280 return name, suffix, 'metal'
281 elif nsuri == ZOPE_I18N_NS:
282 return name, suffix, 'i18n'
283 return name, name, 0
285 def process_ns(self, name, attrs):
286 attrlist = []
287 taldict = {}
288 metaldict = {}
289 i18ndict = {}
290 name, namebase, namens = self.fixname(name)
291 for item in attrs:
292 key, value = item
293 key, keybase, keyns = self.fixname(key)
294 ns = keyns or namens # default to tag namespace
295 if ns and ns != 'unknown':
296 item = (key, value, ns)
297 if ns == 'tal':
298 if taldict.has_key(keybase):
299 raise TALError("duplicate TAL attribute " +
300 `keybase`, self.getpos())
301 taldict[keybase] = value
302 elif ns == 'metal':
303 if metaldict.has_key(keybase):
304 raise METALError("duplicate METAL attribute " +
305 `keybase`, self.getpos())
306 metaldict[keybase] = value
307 elif ns == 'i18n':
308 if i18ndict.has_key(keybase):
309 raise I18NError("duplicate i18n attribute " +
310 `keybase`, self.getpos())
311 i18ndict[keybase] = value
312 attrlist.append(item)
313 if namens in ('metal', 'tal'):
314 taldict['tal tag'] = namens
315 return name, attrlist, taldict, metaldict, i18ndict
317 Generic expat-based XML parser base class.
321 class XMLParser:
323 ordered_attributes = 0
325 handler_names = [
326 "StartElementHandler",
327 "EndElementHandler",
328 "ProcessingInstructionHandler",
329 "CharacterDataHandler",
330 "UnparsedEntityDeclHandler",
331 "NotationDeclHandler",
332 "StartNamespaceDeclHandler",
333 "EndNamespaceDeclHandler",
334 "CommentHandler",
335 "StartCdataSectionHandler",
336 "EndCdataSectionHandler",
337 "DefaultHandler",
338 "DefaultHandlerExpand",
339 "NotStandaloneHandler",
340 "ExternalEntityRefHandler",
341 "XmlDeclHandler",
342 "StartDoctypeDeclHandler",
343 "EndDoctypeDeclHandler",
344 "ElementDeclHandler",
345 "AttlistDeclHandler"
348 def __init__(self, encoding=None):
349 self.parser = p = self.createParser()
350 if self.ordered_attributes:
351 try:
352 self.parser.ordered_attributes = self.ordered_attributes
353 except AttributeError:
354 print "Can't set ordered_attributes"
355 self.ordered_attributes = 0
356 for name in self.handler_names:
357 method = getattr(self, name, None)
358 if method is not None:
359 try:
360 setattr(p, name, method)
361 except AttributeError:
362 print "Can't set expat handler %s" % name
364 def createParser(self, encoding=None):
365 global XMLParseError
366 try:
367 from Products.ParsedXML.Expat import pyexpat
368 XMLParseError = pyexpat.ExpatError
369 return pyexpat.ParserCreate(encoding, ' ')
370 except ImportError:
371 from xml.parsers import expat
372 XMLParseError = expat.ExpatError
373 return expat.ParserCreate(encoding, ' ')
375 def parseFile(self, filename):
376 f = open(filename)
377 self.parseStream(f)
378 #self.parseStream(open(filename))
380 def parseString(self, s):
381 self.parser.Parse(s, 1)
383 def parseURL(self, url):
384 import urllib
385 self.parseStream(urllib.urlopen(url))
387 def parseStream(self, stream):
388 self.parser.ParseFile(stream)
390 def parseFragment(self, s, end=0):
391 self.parser.Parse(s, end)
392 """Interface that a TALES engine provides to the METAL/TAL implementation."""
394 try:
395 from Interface import Interface
396 from Interface.Attribute import Attribute
397 except:
398 class Interface: pass
399 def Attribute(*args): pass
402 class ITALESCompiler(Interface):
403 """Compile-time interface provided by a TALES implementation.
405 The TAL compiler needs an instance of this interface to support
406 compilation of TALES expressions embedded in documents containing
407 TAL and METAL constructs.
410 def getCompilerError():
411 """Return the exception class raised for compilation errors.
414 def compile(expression):
415 """Return a compiled form of 'expression' for later evaluation.
417 'expression' is the source text of the expression.
419 The return value may be passed to the various evaluate*()
420 methods of the ITALESEngine interface. No compatibility is
421 required for the values of the compiled expression between
422 different ITALESEngine implementations.
426 class ITALESEngine(Interface):
427 """Render-time interface provided by a TALES implementation.
429 The TAL interpreter uses this interface to TALES to support
430 evaluation of the compiled expressions returned by
431 ITALESCompiler.compile().
434 def getCompiler():
435 """Return an object that supports ITALESCompiler."""
437 def getDefault():
438 """Return the value of the 'default' TALES expression.
440 Checking a value for a match with 'default' should be done
441 using the 'is' operator in Python.
444 def setPosition((lineno, offset)):
445 """Inform the engine of the current position in the source file.
447 This is used to allow the evaluation engine to report
448 execution errors so that site developers can more easily
449 locate the offending expression.
452 def setSourceFile(filename):
453 """Inform the engine of the name of the current source file.
455 This is used to allow the evaluation engine to report
456 execution errors so that site developers can more easily
457 locate the offending expression.
460 def beginScope():
461 """Push a new scope onto the stack of open scopes.
464 def endScope():
465 """Pop one scope from the stack of open scopes.
468 def evaluate(compiled_expression):
469 """Evaluate an arbitrary expression.
471 No constraints are imposed on the return value.
474 def evaluateBoolean(compiled_expression):
475 """Evaluate an expression that must return a Boolean value.
478 def evaluateMacro(compiled_expression):
479 """Evaluate an expression that must return a macro program.
482 def evaluateStructure(compiled_expression):
483 """Evaluate an expression that must return a structured
484 document fragment.
486 The result of evaluating 'compiled_expression' must be a
487 string containing a parsable HTML or XML fragment. Any TAL
488 markup cnotained in the result string will be interpreted.
491 def evaluateText(compiled_expression):
492 """Evaluate an expression that must return text.
494 The returned text should be suitable for direct inclusion in
495 the output: any HTML or XML escaping or quoting is the
496 responsibility of the expression itself.
499 def evaluateValue(compiled_expression):
500 """Evaluate an arbitrary expression.
502 No constraints are imposed on the return value.
505 def createErrorInfo(exception, (lineno, offset)):
506 """Returns an ITALESErrorInfo object.
508 The returned object is used to provide information about the
509 error condition for the on-error handler.
512 def setGlobal(name, value):
513 """Set a global variable.
515 The variable will be named 'name' and have the value 'value'.
518 def setLocal(name, value):
519 """Set a local variable in the current scope.
521 The variable will be named 'name' and have the value 'value'.
524 def setRepeat(name, compiled_expression):
528 def translate(domain, msgid, mapping, default=None):
530 See ITranslationService.translate()
534 class ITALESErrorInfo(Interface):
536 type = Attribute("type",
537 "The exception class.")
539 value = Attribute("value",
540 "The exception instance.")
542 lineno = Attribute("lineno",
543 "The line number the error occurred on in the source.")
545 offset = Attribute("offset",
546 "The character offset at which the error occurred.")
548 Common definitions used by TAL and METAL compilation an transformation.
551 from types import ListType, TupleType
554 TAL_VERSION = "1.5"
556 XML_NS = "http://www.w3.org/XML/1998/namespace" # URI for XML namespace
557 XMLNS_NS = "http://www.w3.org/2000/xmlns/" # URI for XML NS declarations
559 ZOPE_TAL_NS = "http://xml.zope.org/namespaces/tal"
560 ZOPE_METAL_NS = "http://xml.zope.org/namespaces/metal"
561 ZOPE_I18N_NS = "http://xml.zope.org/namespaces/i18n"
563 NAME_RE = "[a-zA-Z_][-a-zA-Z0-9_]*"
565 KNOWN_METAL_ATTRIBUTES = [
566 "define-macro",
567 "use-macro",
568 "define-slot",
569 "fill-slot",
570 "slot",
573 KNOWN_TAL_ATTRIBUTES = [
574 "define",
575 "condition",
576 "content",
577 "replace",
578 "repeat",
579 "attributes",
580 "on-error",
581 "omit-tag",
582 "tal tag",
585 KNOWN_I18N_ATTRIBUTES = [
586 "translate",
587 "domain",
588 "target",
589 "source",
590 "attributes",
591 "data",
592 "name",
595 class TALError(Exception):
597 def __init__(self, msg, position=(None, None)):
598 assert msg != ""
599 self.msg = msg
600 self.lineno = position[0]
601 self.offset = position[1]
602 self.filename = None
604 def setFile(self, filename):
605 self.filename = filename
607 def __str__(self):
608 result = self.msg
609 if self.lineno is not None:
610 result = result + ", at line %d" % self.lineno
611 if self.offset is not None:
612 result = result + ", column %d" % (self.offset + 1)
613 if self.filename is not None:
614 result = result + ', in file %s' % self.filename
615 return result
617 class METALError(TALError):
618 pass
620 class TALESError(TALError):
621 pass
623 class I18NError(TALError):
624 pass
627 class ErrorInfo:
629 __implements__ = ITALESErrorInfo
631 def __init__(self, err, position=(None, None)):
632 if isinstance(err, Exception):
633 self.type = err.__class__
634 self.value = err
635 else:
636 self.type = err
637 self.value = None
638 self.lineno = position[0]
639 self.offset = position[1]
643 import re
644 _attr_re = re.compile(r"\s*([^\s]+)\s+([^\s].*)\Z", re.S)
645 _subst_re = re.compile(r"\s*(?:(text|raw|structure)\s+)?(.*)\Z", re.S)
646 del re
648 def parseAttributeReplacements(arg, xml):
649 dict = {}
650 for part in splitParts(arg):
651 m = _attr_re.match(part)
652 if not m:
653 raise TALError("Bad syntax in attributes: " + `part`)
654 name, expr = m.group(1, 2)
655 if not xml:
656 name = name.lower()
657 if dict.has_key(name):
658 raise TALError("Duplicate attribute name in attributes: " + `part`)
659 dict[name] = expr
660 return dict
662 def parseSubstitution(arg, position=(None, None)):
663 m = _subst_re.match(arg)
664 if not m:
665 raise TALError("Bad syntax in substitution text: " + `arg`, position)
666 key, expr = m.group(1, 2)
667 if not key:
668 key = "text"
669 return key, expr
671 def splitParts(arg):
672 arg = arg.replace(";;", "\0")
673 parts = arg.split(';')
674 parts = [p.replace("\0", ";") for p in parts]
675 if len(parts) > 1 and not parts[-1].strip():
676 del parts[-1] # It ended in a semicolon
677 return parts
679 def isCurrentVersion(program):
680 version = getProgramVersion(program)
681 return version == TAL_VERSION
683 def getProgramMode(program):
684 version = getProgramVersion(program)
685 if (version == TAL_VERSION and isinstance(program[1], TupleType) and
686 len(program[1]) == 2):
687 opcode, mode = program[1]
688 if opcode == "mode":
689 return mode
690 return None
692 def getProgramVersion(program):
693 if (len(program) >= 2 and
694 isinstance(program[0], TupleType) and len(program[0]) == 2):
695 opcode, version = program[0]
696 if opcode == "version":
697 return version
698 return None
700 import re
701 _ent1_re = re.compile('&(?![A-Z#])', re.I)
702 _entch_re = re.compile('&([A-Z][A-Z0-9]*)(?![A-Z0-9;])', re.I)
703 _entn1_re = re.compile('&#(?![0-9X])', re.I)
704 _entnx_re = re.compile('&(#X[A-F0-9]*)(?![A-F0-9;])', re.I)
705 _entnd_re = re.compile('&(#[0-9][0-9]*)(?![0-9;])')
706 del re
708 def attrEscape(s):
709 """Replace special characters '&<>' by character entities,
710 except when '&' already begins a syntactically valid entity."""
711 s = _ent1_re.sub('&amp;', s)
712 s = _entch_re.sub(r'&amp;\1', s)
713 s = _entn1_re.sub('&amp;#', s)
714 s = _entnx_re.sub(r'&amp;\1', s)
715 s = _entnd_re.sub(r'&amp;\1', s)
716 s = s.replace('<', '&lt;')
717 s = s.replace('>', '&gt;')
718 s = s.replace('"', '&quot;')
719 return s
721 Code generator for TALInterpreter intermediate code.
724 import re
725 import cgi
729 I18N_REPLACE = 1
730 I18N_CONTENT = 2
731 I18N_EXPRESSION = 3
733 _name_rx = re.compile(NAME_RE)
736 class TALGenerator:
738 inMacroUse = 0
739 inMacroDef = 0
740 source_file = None
742 def __init__(self, expressionCompiler=None, xml=1, source_file=None):
743 if not expressionCompiler:
744 expressionCompiler = AthanaTALEngine()
745 self.expressionCompiler = expressionCompiler
746 self.CompilerError = expressionCompiler.getCompilerError()
747 self.program = []
748 self.stack = []
749 self.todoStack = []
750 self.macros = {}
751 self.slots = {}
752 self.slotStack = []
753 self.xml = xml
754 self.emit("version", TAL_VERSION)
755 self.emit("mode", xml and "xml" or "html")
756 if source_file is not None:
757 self.source_file = source_file
758 self.emit("setSourceFile", source_file)
759 self.i18nContext = TranslationContext()
760 self.i18nLevel = 0
762 def getCode(self):
763 assert not self.stack
764 assert not self.todoStack
765 return self.optimize(self.program), self.macros
767 def optimize(self, program):
768 output = []
769 collect = []
770 cursor = 0
771 if self.xml:
772 endsep = "/>"
773 else:
774 endsep = " />"
775 for cursor in xrange(len(program)+1):
776 try:
777 item = program[cursor]
778 except IndexError:
779 item = (None, None)
780 opcode = item[0]
781 if opcode == "rawtext":
782 collect.append(item[1])
783 continue
784 if opcode == "endTag":
785 collect.append("</%s>" % item[1])
786 continue
787 if opcode == "startTag":
788 if self.optimizeStartTag(collect, item[1], item[2], ">"):
789 continue
790 if opcode == "startEndTag":
791 if self.optimizeStartTag(collect, item[1], item[2], endsep):
792 continue
793 if opcode in ("beginScope", "endScope"):
794 output.append(self.optimizeArgsList(item))
795 continue
796 if opcode == 'noop':
797 opcode = None
798 pass
799 text = "".join(collect)
800 if text:
801 i = text.rfind("\n")
802 if i >= 0:
803 i = len(text) - (i + 1)
804 output.append(("rawtextColumn", (text, i)))
805 else:
806 output.append(("rawtextOffset", (text, len(text))))
807 if opcode != None:
808 output.append(self.optimizeArgsList(item))
809 collect = []
810 return self.optimizeCommonTriple(output)
812 def optimizeArgsList(self, item):
813 if len(item) == 2:
814 return item
815 else:
816 return item[0], tuple(item[1:])
818 def optimizeStartTag(self, collect, name, attrlist, end):
819 if not attrlist:
820 collect.append("<%s%s" % (name, end))
821 return 1
822 opt = 1
823 new = ["<" + name]
824 for i in range(len(attrlist)):
825 item = attrlist[i]
826 if len(item) > 2:
827 opt = 0
828 name, value, action = item[:3]
829 attrlist[i] = (name, value, action) + item[3:]
830 else:
831 if item[1] is None:
832 s = item[0]
833 else:
834 s = '%s="%s"' % (item[0], attrEscape(item[1]))
835 attrlist[i] = item[0], s
836 new.append(" " + s)
837 if opt:
838 new.append(end)
839 collect.extend(new)
840 return opt
842 def optimizeCommonTriple(self, program):
843 if len(program) < 3:
844 return program
845 output = program[:2]
846 prev2, prev1 = output
847 for item in program[2:]:
848 if ( item[0] == "beginScope"
849 and prev1[0] == "setPosition"
850 and prev2[0] == "rawtextColumn"):
851 position = output.pop()[1]
852 text, column = output.pop()[1]
853 prev1 = None, None
854 closeprev = 0
855 if output and output[-1][0] == "endScope":
856 closeprev = 1
857 output.pop()
858 item = ("rawtextBeginScope",
859 (text, column, position, closeprev, item[1]))
860 output.append(item)
861 prev2 = prev1
862 prev1 = item
863 return output
865 def todoPush(self, todo):
866 self.todoStack.append(todo)
868 def todoPop(self):
869 return self.todoStack.pop()
871 def compileExpression(self, expr):
872 try:
873 return self.expressionCompiler.compile(expr)
874 except self.CompilerError, err:
875 raise TALError('%s in expression %s' % (err.args[0], `expr`),
876 self.position)
878 def pushProgram(self):
879 self.stack.append(self.program)
880 self.program = []
882 def popProgram(self):
883 program = self.program
884 self.program = self.stack.pop()
885 return self.optimize(program)
887 def pushSlots(self):
888 self.slotStack.append(self.slots)
889 self.slots = {}
891 def popSlots(self):
892 slots = self.slots
893 self.slots = self.slotStack.pop()
894 return slots
896 def emit(self, *instruction):
897 self.program.append(instruction)
899 def emitStartTag(self, name, attrlist, isend=0):
900 if isend:
901 opcode = "startEndTag"
902 else:
903 opcode = "startTag"
904 self.emit(opcode, name, attrlist)
906 def emitEndTag(self, name):
907 if self.xml and self.program and self.program[-1][0] == "startTag":
908 self.program[-1] = ("startEndTag",) + self.program[-1][1:]
909 else:
910 self.emit("endTag", name)
912 def emitOptTag(self, name, optTag, isend):
913 program = self.popProgram() #block
914 start = self.popProgram() #start tag
915 if (isend or not program) and self.xml:
916 start[-1] = ("startEndTag",) + start[-1][1:]
917 isend = 1
918 cexpr = optTag[0]
919 if cexpr:
920 cexpr = self.compileExpression(optTag[0])
921 self.emit("optTag", name, cexpr, optTag[1], isend, start, program)
923 def emitRawText(self, text):
924 self.emit("rawtext", text)
926 def emitText(self, text):
927 self.emitRawText(cgi.escape(text))
929 def emitDefines(self, defines):
930 for part in splitParts(defines):
931 m = re.match(
932 r"(?s)\s*(?:(global|local)\s+)?(%s)\s+(.*)\Z" % NAME_RE, part)
933 if not m:
934 raise TALError("invalid define syntax: " + `part`,
935 self.position)
936 scope, name, expr = m.group(1, 2, 3)
937 scope = scope or "local"
938 cexpr = self.compileExpression(expr)
939 if scope == "local":
940 self.emit("setLocal", name, cexpr)
941 else:
942 self.emit("setGlobal", name, cexpr)
944 def emitOnError(self, name, onError, TALtag, isend):
945 block = self.popProgram()
946 key, expr = parseSubstitution(onError)
947 cexpr = self.compileExpression(expr)
948 if key == "text":
949 self.emit("insertText", cexpr, [])
950 elif key == "raw":
951 self.emit("insertRaw", cexpr, [])
952 else:
953 assert key == "structure"
954 self.emit("insertStructure", cexpr, {}, [])
955 if TALtag:
956 self.emitOptTag(name, (None, 1), isend)
957 else:
958 self.emitEndTag(name)
959 handler = self.popProgram()
960 self.emit("onError", block, handler)
962 def emitCondition(self, expr):
963 cexpr = self.compileExpression(expr)
964 program = self.popProgram()
965 self.emit("condition", cexpr, program)
967 def emitRepeat(self, arg):
970 m = re.match("(?s)\s*(%s)\s+(.*)\Z" % NAME_RE, arg)
971 if not m:
972 raise TALError("invalid repeat syntax: " + `arg`,
973 self.position)
974 name, expr = m.group(1, 2)
975 cexpr = self.compileExpression(expr)
976 program = self.popProgram()
977 self.emit("loop", name, cexpr, program)
980 def emitSubstitution(self, arg, attrDict={}):
981 key, expr = parseSubstitution(arg)
982 cexpr = self.compileExpression(expr)
983 program = self.popProgram()
984 if key == "text":
985 self.emit("insertText", cexpr, program)
986 elif key == "raw":
987 self.emit("insertRaw", cexpr, program)
988 else:
989 assert key == "structure"
990 self.emit("insertStructure", cexpr, attrDict, program)
992 def emitI18nVariable(self, stuff):
993 varname, action, expression = stuff
994 m = _name_rx.match(varname)
995 if m is None or m.group() != varname:
996 raise TALError("illegal i18n:name: %r" % varname, self.position)
997 key = cexpr = None
998 program = self.popProgram()
999 if action == I18N_REPLACE:
1000 program = program[1:-1]
1001 elif action == I18N_CONTENT:
1002 pass
1003 else:
1004 assert action == I18N_EXPRESSION
1005 key, expr = parseSubstitution(expression)
1006 cexpr = self.compileExpression(expr)
1007 self.emit('i18nVariable',
1008 varname, program, cexpr, int(key == "structure"))
1010 def emitTranslation(self, msgid, i18ndata):
1011 program = self.popProgram()
1012 if i18ndata is None:
1013 self.emit('insertTranslation', msgid, program)
1014 else:
1015 key, expr = parseSubstitution(i18ndata)
1016 cexpr = self.compileExpression(expr)
1017 assert key == 'text'
1018 self.emit('insertTranslation', msgid, program, cexpr)
1020 def emitDefineMacro(self, macroName):
1021 program = self.popProgram()
1022 macroName = macroName.strip()
1023 if self.macros.has_key(macroName):
1024 raise METALError("duplicate macro definition: %s" % `macroName`,
1025 self.position)
1026 if not re.match('%s$' % NAME_RE, macroName):
1027 raise METALError("invalid macro name: %s" % `macroName`,
1028 self.position)
1029 self.macros[macroName] = program
1030 self.inMacroDef = self.inMacroDef - 1
1031 self.emit("defineMacro", macroName, program)
1033 def emitUseMacro(self, expr):
1034 cexpr = self.compileExpression(expr)
1035 program = self.popProgram()
1036 self.inMacroUse = 0
1037 self.emit("useMacro", expr, cexpr, self.popSlots(), program)
1039 def emitDefineSlot(self, slotName):
1040 program = self.popProgram()
1041 slotName = slotName.strip()
1042 if not re.match('%s$' % NAME_RE, slotName):
1043 raise METALError("invalid slot name: %s" % `slotName`,
1044 self.position)
1045 self.emit("defineSlot", slotName, program)
1047 def emitFillSlot(self, slotName):
1048 program = self.popProgram()
1049 slotName = slotName.strip()
1050 if self.slots.has_key(slotName):
1051 raise METALError("duplicate fill-slot name: %s" % `slotName`,
1052 self.position)
1053 if not re.match('%s$' % NAME_RE, slotName):
1054 raise METALError("invalid slot name: %s" % `slotName`,
1055 self.position)
1056 self.slots[slotName] = program
1057 self.inMacroUse = 1
1058 self.emit("fillSlot", slotName, program)
1060 def unEmitWhitespace(self):
1061 collect = []
1062 i = len(self.program) - 1
1063 while i >= 0:
1064 item = self.program[i]
1065 if item[0] != "rawtext":
1066 break
1067 text = item[1]
1068 if not re.match(r"\A\s*\Z", text):
1069 break
1070 collect.append(text)
1071 i = i-1
1072 del self.program[i+1:]
1073 if i >= 0 and self.program[i][0] == "rawtext":
1074 text = self.program[i][1]
1075 m = re.search(r"\s+\Z", text)
1076 if m:
1077 self.program[i] = ("rawtext", text[:m.start()])
1078 collect.append(m.group())
1079 collect.reverse()
1080 return "".join(collect)
1082 def unEmitNewlineWhitespace(self):
1083 collect = []
1084 i = len(self.program)
1085 while i > 0:
1086 i = i-1
1087 item = self.program[i]
1088 if item[0] != "rawtext":
1089 break
1090 text = item[1]
1091 if re.match(r"\A[ \t]*\Z", text):
1092 collect.append(text)
1093 continue
1094 m = re.match(r"(?s)^(.*)(\n[ \t]*)\Z", text)
1095 if not m:
1096 break
1097 text, rest = m.group(1, 2)
1098 collect.reverse()
1099 rest = rest + "".join(collect)
1100 del self.program[i:]
1101 if text:
1102 self.emit("rawtext", text)
1103 return rest
1104 return None
1106 def replaceAttrs(self, attrlist, repldict):
1107 if not repldict:
1108 return attrlist
1109 newlist = []
1110 for item in attrlist:
1111 key = item[0]
1112 if repldict.has_key(key):
1113 expr, xlat, msgid = repldict[key]
1114 item = item[:2] + ("replace", expr, xlat, msgid)
1115 del repldict[key]
1116 newlist.append(item)
1117 for key, (expr, xlat, msgid) in repldict.items():
1118 newlist.append((key, None, "insert", expr, xlat, msgid))
1119 return newlist
1121 def emitStartElement(self, name, attrlist, taldict, metaldict, i18ndict,
1122 position=(None, None), isend=0):
1123 if not taldict and not metaldict and not i18ndict:
1124 self.emitStartTag(name, attrlist, isend)
1125 self.todoPush({})
1126 if isend:
1127 self.emitEndElement(name, isend)
1128 return
1130 self.position = position
1131 for key, value in taldict.items():
1132 if key not in KNOWN_TAL_ATTRIBUTES:
1133 raise TALError("bad TAL attribute: " + `key`, position)
1134 if not (value or key == 'omit-tag'):
1135 raise TALError("missing value for TAL attribute: " +
1136 `key`, position)
1137 for key, value in metaldict.items():
1138 if key not in KNOWN_METAL_ATTRIBUTES:
1139 raise METALError("bad METAL attribute: " + `key`,
1140 position)
1141 if not value:
1142 raise TALError("missing value for METAL attribute: " +
1143 `key`, position)
1144 for key, value in i18ndict.items():
1145 if key not in KNOWN_I18N_ATTRIBUTES:
1146 raise I18NError("bad i18n attribute: " + `key`, position)
1147 if not value and key in ("attributes", "data", "id"):
1148 raise I18NError("missing value for i18n attribute: " +
1149 `key`, position)
1150 todo = {}
1151 defineMacro = metaldict.get("define-macro")
1152 useMacro = metaldict.get("use-macro")
1153 defineSlot = metaldict.get("define-slot")
1154 fillSlot = metaldict.get("fill-slot")
1155 define = taldict.get("define")
1156 condition = taldict.get("condition")
1157 repeat = taldict.get("repeat")
1158 content = taldict.get("content")
1159 replace = taldict.get("replace")
1160 attrsubst = taldict.get("attributes")
1161 onError = taldict.get("on-error")
1162 omitTag = taldict.get("omit-tag")
1163 TALtag = taldict.get("tal tag")
1164 i18nattrs = i18ndict.get("attributes")
1165 msgid = i18ndict.get("translate")
1166 varname = i18ndict.get('name')
1167 i18ndata = i18ndict.get('data')
1169 if varname and not self.i18nLevel:
1170 raise I18NError(
1171 "i18n:name can only occur inside a translation unit",
1172 position)
1174 if i18ndata and not msgid:
1175 raise I18NError("i18n:data must be accompanied by i18n:translate",
1176 position)
1178 if len(metaldict) > 1 and (defineMacro or useMacro):
1179 raise METALError("define-macro and use-macro cannot be used "
1180 "together or with define-slot or fill-slot",
1181 position)
1182 if replace:
1183 if content:
1184 raise TALError(
1185 "tal:content and tal:replace are mutually exclusive",
1186 position)
1187 if msgid is not None:
1188 raise I18NError(
1189 "i18n:translate and tal:replace are mutually exclusive",
1190 position)
1192 repeatWhitespace = None
1193 if repeat:
1194 repeatWhitespace = self.unEmitNewlineWhitespace()
1195 if position != (None, None):
1196 self.emit("setPosition", position)
1197 if self.inMacroUse:
1198 if fillSlot:
1199 self.pushProgram()
1200 if self.source_file is not None:
1201 self.emit("setSourceFile", self.source_file)
1202 todo["fillSlot"] = fillSlot
1203 self.inMacroUse = 0
1204 else:
1205 if fillSlot:
1206 raise METALError("fill-slot must be within a use-macro",
1207 position)
1208 if not self.inMacroUse:
1209 if defineMacro:
1210 self.pushProgram()
1211 self.emit("version", TAL_VERSION)
1212 self.emit("mode", self.xml and "xml" or "html")
1213 if self.source_file is not None:
1214 self.emit("setSourceFile", self.source_file)
1215 todo["defineMacro"] = defineMacro
1216 self.inMacroDef = self.inMacroDef + 1
1217 if useMacro:
1218 self.pushSlots()
1219 self.pushProgram()
1220 todo["useMacro"] = useMacro
1221 self.inMacroUse = 1
1222 if defineSlot:
1223 if not self.inMacroDef:
1224 raise METALError(
1225 "define-slot must be within a define-macro",
1226 position)
1227 self.pushProgram()
1228 todo["defineSlot"] = defineSlot
1230 if defineSlot or i18ndict:
1232 domain = i18ndict.get("domain") or self.i18nContext.domain
1233 source = i18ndict.get("source") or self.i18nContext.source
1234 target = i18ndict.get("target") or self.i18nContext.target
1235 if ( domain != DEFAULT_DOMAIN
1236 or source is not None
1237 or target is not None):
1238 self.i18nContext = TranslationContext(self.i18nContext,
1239 domain=domain,
1240 source=source,
1241 target=target)
1242 self.emit("beginI18nContext",
1243 {"domain": domain, "source": source,
1244 "target": target})
1245 todo["i18ncontext"] = 1
1246 if taldict or i18ndict:
1247 dict = {}
1248 for item in attrlist:
1249 key, value = item[:2]
1250 dict[key] = value
1251 self.emit("beginScope", dict)
1252 todo["scope"] = 1
1253 if onError:
1254 self.pushProgram() # handler
1255 if TALtag:
1256 self.pushProgram() # start
1257 self.emitStartTag(name, list(attrlist)) # Must copy attrlist!
1258 if TALtag:
1259 self.pushProgram() # start
1260 self.pushProgram() # block
1261 todo["onError"] = onError
1262 if define:
1263 self.emitDefines(define)
1264 todo["define"] = define
1265 if condition:
1266 self.pushProgram()
1267 todo["condition"] = condition
1268 if repeat:
1269 todo["repeat"] = repeat
1270 self.pushProgram()
1271 if repeatWhitespace:
1272 self.emitText(repeatWhitespace)
1273 if content:
1274 if varname:
1275 todo['i18nvar'] = (varname, I18N_CONTENT, None)
1276 todo["content"] = content
1277 self.pushProgram()
1278 else:
1279 todo["content"] = content
1280 elif replace:
1281 if varname:
1282 todo['i18nvar'] = (varname, I18N_EXPRESSION, replace)
1283 else:
1284 todo["replace"] = replace
1285 self.pushProgram()
1286 elif varname:
1287 todo['i18nvar'] = (varname, I18N_REPLACE, None)
1288 self.pushProgram()
1289 if msgid is not None:
1290 self.i18nLevel += 1
1291 todo['msgid'] = msgid
1292 if i18ndata:
1293 todo['i18ndata'] = i18ndata
1294 optTag = omitTag is not None or TALtag
1295 if optTag:
1296 todo["optional tag"] = omitTag, TALtag
1297 self.pushProgram()
1298 if attrsubst or i18nattrs:
1299 if attrsubst:
1300 repldict = parseAttributeReplacements(attrsubst,
1301 self.xml)
1302 else:
1303 repldict = {}
1304 if i18nattrs:
1305 i18nattrs = _parseI18nAttributes(i18nattrs, attrlist, repldict,
1306 self.position, self.xml,
1307 self.source_file)
1308 else:
1309 i18nattrs = {}
1310 for key, value in repldict.items():
1311 if i18nattrs.get(key, None):
1312 raise I18NError(
1313 ("attribute [%s] cannot both be part of tal:attributes" +
1314 " and have a msgid in i18n:attributes") % key,
1315 position)
1316 ce = self.compileExpression(value)
1317 repldict[key] = ce, key in i18nattrs, i18nattrs.get(key)
1318 for key in i18nattrs:
1319 if not repldict.has_key(key):
1320 repldict[key] = None, 1, i18nattrs.get(key)
1321 else:
1322 repldict = {}
1323 if replace:
1324 todo["repldict"] = repldict
1325 repldict = {}
1326 self.emitStartTag(name, self.replaceAttrs(attrlist, repldict), isend)
1327 if optTag:
1328 self.pushProgram()
1329 if content and not varname:
1330 self.pushProgram()
1331 if msgid is not None:
1332 self.pushProgram()
1333 if content and varname:
1334 self.pushProgram()
1335 if todo and position != (None, None):
1336 todo["position"] = position
1337 self.todoPush(todo)
1338 if isend:
1339 self.emitEndElement(name, isend)
1341 def emitEndElement(self, name, isend=0, implied=0):
1342 todo = self.todoPop()
1343 if not todo:
1344 if not isend:
1345 self.emitEndTag(name)
1346 return
1348 self.position = position = todo.get("position", (None, None))
1349 defineMacro = todo.get("defineMacro")
1350 useMacro = todo.get("useMacro")
1351 defineSlot = todo.get("defineSlot")
1352 fillSlot = todo.get("fillSlot")
1353 repeat = todo.get("repeat")
1354 content = todo.get("content")
1355 replace = todo.get("replace")
1356 condition = todo.get("condition")
1357 onError = todo.get("onError")
1358 repldict = todo.get("repldict", {})
1359 scope = todo.get("scope")
1360 optTag = todo.get("optional tag")
1361 msgid = todo.get('msgid')
1362 i18ncontext = todo.get("i18ncontext")
1363 varname = todo.get('i18nvar')
1364 i18ndata = todo.get('i18ndata')
1366 if implied > 0:
1367 if defineMacro or useMacro or defineSlot or fillSlot:
1368 exc = METALError
1369 what = "METAL"
1370 else:
1371 exc = TALError
1372 what = "TAL"
1373 raise exc("%s attributes on <%s> require explicit </%s>" %
1374 (what, name, name), position)
1376 if content:
1377 self.emitSubstitution(content, {})
1378 if msgid is not None:
1379 if (not varname) or (
1380 varname and (varname[1] == I18N_CONTENT)):
1381 self.emitTranslation(msgid, i18ndata)
1382 self.i18nLevel -= 1
1383 if optTag:
1384 self.emitOptTag(name, optTag, isend)
1385 elif not isend:
1386 if varname:
1387 self.emit('noop')
1388 self.emitEndTag(name)
1389 if replace:
1390 self.emitSubstitution(replace, repldict)
1391 elif varname:
1392 assert (varname[1]
1393 in [I18N_REPLACE, I18N_CONTENT, I18N_EXPRESSION])
1394 self.emitI18nVariable(varname)
1395 if msgid is not None:
1396 if varname and (varname[1] <> I18N_CONTENT):
1397 self.emitTranslation(msgid, i18ndata)
1398 if repeat:
1399 self.emitRepeat(repeat)
1400 if condition:
1401 self.emitCondition(condition)
1402 if onError:
1403 self.emitOnError(name, onError, optTag and optTag[1], isend)
1404 if scope:
1405 self.emit("endScope")
1406 if i18ncontext:
1407 self.emit("endI18nContext")
1408 assert self.i18nContext.parent is not None
1409 self.i18nContext = self.i18nContext.parent
1410 if defineSlot:
1411 self.emitDefineSlot(defineSlot)
1412 if fillSlot:
1413 self.emitFillSlot(fillSlot)
1414 if useMacro:
1415 self.emitUseMacro(useMacro)
1416 if defineMacro:
1417 self.emitDefineMacro(defineMacro)
1419 def _parseI18nAttributes(i18nattrs, attrlist, repldict, position,
1420 xml, source_file):
1422 def addAttribute(dic, attr, msgid, position, xml):
1423 if not xml:
1424 attr = attr.lower()
1425 if attr in dic:
1426 raise TALError(
1427 "attribute may only be specified once in i18n:attributes: "
1428 + attr,
1429 position)
1430 dic[attr] = msgid
1432 d = {}
1433 if ';' in i18nattrs:
1434 i18nattrlist = i18nattrs.split(';')
1435 i18nattrlist = [attr.strip().split()
1436 for attr in i18nattrlist if attr.strip()]
1437 for parts in i18nattrlist:
1438 if len(parts) > 2:
1439 raise TALError("illegal i18n:attributes specification: %r"
1440 % parts, position)
1441 if len(parts) == 2:
1442 attr, msgid = parts
1443 else:
1444 attr = parts[0]
1445 msgid = None
1446 addAttribute(d, attr, msgid, position, xml)
1447 else:
1448 i18nattrlist = i18nattrs.split()
1449 if len(i18nattrlist) == 1:
1450 addAttribute(d, i18nattrlist[0], None, position, xml)
1451 elif len(i18nattrlist) == 2:
1452 staticattrs = [attr[0] for attr in attrlist if len(attr) == 2]
1453 if (not i18nattrlist[1] in staticattrs) and (
1454 not i18nattrlist[1] in repldict):
1455 attr, msgid = i18nattrlist
1456 addAttribute(d, attr, msgid, position, xml)
1457 else:
1458 import warnings
1459 warnings.warn(I18N_ATTRIBUTES_WARNING
1460 % (source_file, str(position), i18nattrs)
1461 , DeprecationWarning)
1462 msgid = None
1463 for attr in i18nattrlist:
1464 addAttribute(d, attr, msgid, position, xml)
1465 else:
1466 import warnings
1467 warnings.warn(I18N_ATTRIBUTES_WARNING
1468 % (source_file, str(position), i18nattrs)
1469 , DeprecationWarning)
1470 msgid = None
1471 for attr in i18nattrlist:
1472 addAttribute(d, attr, msgid, position, xml)
1473 return d
1475 I18N_ATTRIBUTES_WARNING = (
1476 'Space separated attributes in i18n:attributes'
1477 ' are deprecated (i18n:attributes="value title"). Please use'
1478 ' semicolon to separate attributes'
1479 ' (i18n:attributes="value; title").'
1480 '\nFile %s at row, column %s\nAttributes %s')
1482 """Interpreter for a pre-compiled TAL program.
1485 import cgi
1486 import sys
1487 import getopt
1488 import re
1489 from cgi import escape
1491 from StringIO import StringIO
1495 class ConflictError:
1496 pass
1498 class MessageID:
1499 pass
1503 BOOLEAN_HTML_ATTRS = [
1504 "compact", "nowrap", "ismap", "declare", "noshade", "checked",
1505 "disabled", "readonly", "multiple", "selected", "noresize",
1506 "defer"
1509 def _init():
1510 d = {}
1511 for s in BOOLEAN_HTML_ATTRS:
1512 d[s] = 1
1513 return d
1515 BOOLEAN_HTML_ATTRS = _init()
1517 _nulljoin = ''.join
1518 _spacejoin = ' '.join
1520 def normalize(text):
1521 return _spacejoin(text.split())
1524 NAME_RE = r"[a-zA-Z][a-zA-Z0-9_]*"
1525 _interp_regex = re.compile(r'(?<!\$)(\$(?:%(n)s|{%(n)s}))' %({'n': NAME_RE}))
1526 _get_var_regex = re.compile(r'%(n)s' %({'n': NAME_RE}))
1528 def interpolate(text, mapping):
1529 """Interpolate ${keyword} substitutions.
1531 This is called when no translation is provided by the translation
1532 service.
1534 if not mapping:
1535 return text
1536 to_replace = _interp_regex.findall(text)
1537 for string in to_replace:
1538 var = _get_var_regex.findall(string)[0]
1539 if mapping.has_key(var):
1540 subst = ustr(mapping[var])
1541 try:
1542 text = text.replace(string, subst)
1543 except UnicodeError:
1544 subst = `subst`[1:-1]
1545 text = text.replace(string, subst)
1546 return text
1549 class AltTALGenerator(TALGenerator):
1551 def __init__(self, repldict, expressionCompiler=None, xml=0):
1552 self.repldict = repldict
1553 self.enabled = 1
1554 TALGenerator.__init__(self, expressionCompiler, xml)
1556 def enable(self, enabled):
1557 self.enabled = enabled
1559 def emit(self, *args):
1560 if self.enabled:
1561 TALGenerator.emit(self, *args)
1563 def emitStartElement(self, name, attrlist, taldict, metaldict, i18ndict,
1564 position=(None, None), isend=0):
1565 metaldict = {}
1566 taldict = {}
1567 i18ndict = {}
1568 if self.enabled and self.repldict:
1569 taldict["attributes"] = "x x"
1570 TALGenerator.emitStartElement(self, name, attrlist,
1571 taldict, metaldict, i18ndict,
1572 position, isend)
1574 def replaceAttrs(self, attrlist, repldict):
1575 if self.enabled and self.repldict:
1576 repldict = self.repldict
1577 self.repldict = None
1578 return TALGenerator.replaceAttrs(self, attrlist, repldict)
1581 class TALInterpreter:
1583 def __init__(self, program, macros, engine, stream=None,
1584 debug=0, wrap=60, metal=1, tal=1, showtal=-1,
1585 strictinsert=1, stackLimit=100, i18nInterpolate=1):
1586 self.program = program
1587 self.macros = macros
1588 self.engine = engine # Execution engine (aka context)
1589 self.Default = engine.getDefault()
1590 self.stream = stream or sys.stdout
1591 self._stream_write = self.stream.write
1592 self.debug = debug
1593 self.wrap = wrap
1594 self.metal = metal
1595 self.tal = tal
1596 if tal:
1597 self.dispatch = self.bytecode_handlers_tal
1598 else:
1599 self.dispatch = self.bytecode_handlers
1600 assert showtal in (-1, 0, 1)
1601 if showtal == -1:
1602 showtal = (not tal)
1603 self.showtal = showtal
1604 self.strictinsert = strictinsert
1605 self.stackLimit = stackLimit
1606 self.html = 0
1607 self.endsep = "/>"
1608 self.endlen = len(self.endsep)
1609 self.macroStack = []
1610 self.position = None, None # (lineno, offset)
1611 self.col = 0
1612 self.level = 0
1613 self.scopeLevel = 0
1614 self.sourceFile = None
1615 self.i18nStack = []
1616 self.i18nInterpolate = i18nInterpolate
1617 self.i18nContext = TranslationContext()
1619 def StringIO(self):
1620 return FasterStringIO()
1622 def saveState(self):
1623 return (self.position, self.col, self.stream,
1624 self.scopeLevel, self.level, self.i18nContext)
1626 def restoreState(self, state):
1627 (self.position, self.col, self.stream,
1628 scopeLevel, level, i18n) = state
1629 self._stream_write = self.stream.write
1630 assert self.level == level
1631 while self.scopeLevel > scopeLevel:
1632 self.engine.endScope()
1633 self.scopeLevel = self.scopeLevel - 1
1634 self.engine.setPosition(self.position)
1635 self.i18nContext = i18n
1637 def restoreOutputState(self, state):
1638 (dummy, self.col, self.stream,
1639 scopeLevel, level, i18n) = state
1640 self._stream_write = self.stream.write
1641 assert self.level == level
1642 assert self.scopeLevel == scopeLevel
1644 def pushMacro(self, macroName, slots, entering=1):
1645 if len(self.macroStack) >= self.stackLimit:
1646 raise METALError("macro nesting limit (%d) exceeded "
1647 "by %s" % (self.stackLimit, `macroName`))
1648 self.macroStack.append([macroName, slots, entering, self.i18nContext])
1650 def popMacro(self):
1651 return self.macroStack.pop()
1653 def __call__(self):
1654 assert self.level == 0
1655 assert self.scopeLevel == 0
1656 assert self.i18nContext.parent is None
1657 self.interpret(self.program)
1658 assert self.level == 0
1659 assert self.scopeLevel == 0
1660 assert self.i18nContext.parent is None
1661 if self.col > 0:
1662 self._stream_write("\n")
1663 self.col = 0
1665 def interpretWithStream(self, program, stream):
1666 oldstream = self.stream
1667 self.stream = stream
1668 self._stream_write = stream.write
1669 try:
1670 self.interpret(program)
1671 finally:
1672 self.stream = oldstream
1673 self._stream_write = oldstream.write
1675 def stream_write(self, s,
1676 len=len):
1677 self._stream_write(s)
1678 i = s.rfind('\n')
1679 if i < 0:
1680 self.col = self.col + len(s)
1681 else:
1682 self.col = len(s) - (i + 1)
1684 bytecode_handlers = {}
1686 def interpret(self, program):
1687 oldlevel = self.level
1688 self.level = oldlevel + 1
1689 handlers = self.dispatch
1690 try:
1691 if self.debug:
1692 for (opcode, args) in program:
1693 s = "%sdo_%s(%s)\n" % (" "*self.level, opcode,
1694 repr(args))
1695 if len(s) > 80:
1696 s = s[:76] + "...\n"
1697 sys.stderr.write(s)
1698 handlers[opcode](self, args)
1699 else:
1700 for (opcode, args) in program:
1701 handlers[opcode](self, args)
1702 finally:
1703 self.level = oldlevel
1705 def do_version(self, version):
1706 assert version == TAL_VERSION
1707 bytecode_handlers["version"] = do_version
1709 def do_mode(self, mode):
1710 assert mode in ("html", "xml")
1711 self.html = (mode == "html")
1712 if self.html:
1713 self.endsep = " />"
1714 else:
1715 self.endsep = "/>"
1716 self.endlen = len(self.endsep)
1717 bytecode_handlers["mode"] = do_mode
1719 def do_setSourceFile(self, source_file):
1720 self.sourceFile = source_file
1721 self.engine.setSourceFile(source_file)
1722 bytecode_handlers["setSourceFile"] = do_setSourceFile
1724 def do_setPosition(self, position):
1725 self.position = position
1726 self.engine.setPosition(position)
1727 bytecode_handlers["setPosition"] = do_setPosition
1729 def do_startEndTag(self, stuff):
1730 self.do_startTag(stuff, self.endsep, self.endlen)
1731 bytecode_handlers["startEndTag"] = do_startEndTag
1733 def do_startTag(self, (name, attrList),
1734 end=">", endlen=1, _len=len):
1735 self._currentTag = name
1736 L = ["<", name]
1737 append = L.append
1738 col = self.col + _len(name) + 1
1739 wrap = self.wrap
1740 align = col + 1
1741 if align >= wrap/2:
1742 align = 4 # Avoid a narrow column far to the right
1743 attrAction = self.dispatch["<attrAction>"]
1744 try:
1745 for item in attrList:
1746 if _len(item) == 2:
1747 name, s = item
1748 else:
1749 if item[2] in ('metal', 'tal', 'xmlns', 'i18n'):
1750 if not self.showtal:
1751 continue
1752 ok, name, s = self.attrAction(item)
1753 else:
1754 ok, name, s = attrAction(self, item)
1755 if not ok:
1756 continue
1757 slen = _len(s)
1758 if (wrap and
1759 col >= align and
1760 col + 1 + slen > wrap):
1761 append("\n")
1762 append(" "*align)
1763 col = align + slen
1764 else:
1765 append(" ")
1766 col = col + 1 + slen
1767 append(s)
1768 append(end)
1769 col = col + endlen
1770 finally:
1771 self._stream_write(_nulljoin(L))
1772 self.col = col
1773 bytecode_handlers["startTag"] = do_startTag
1775 def attrAction(self, item):
1776 name, value, action = item[:3]
1777 if action == 'insert':
1778 return 0, name, value
1779 macs = self.macroStack
1780 if action == 'metal' and self.metal and macs:
1781 if len(macs) > 1 or not macs[-1][2]:
1782 return 0, name, value
1783 macs[-1][2] = 0
1784 i = name.rfind(":") + 1
1785 prefix, suffix = name[:i], name[i:]
1786 if suffix == "define-macro":
1787 name = prefix + "use-macro"
1788 value = macs[-1][0] # Macro name
1789 elif suffix == "define-slot":
1790 name = prefix + "fill-slot"
1791 elif suffix == "fill-slot":
1792 pass
1793 else:
1794 return 0, name, value
1796 if value is None:
1797 value = name
1798 else:
1799 value = '%s="%s"' % (name, attrEscape(value))
1800 return 1, name, value
1802 def attrAction_tal(self, item):
1803 name, value, action = item[:3]
1804 ok = 1
1805 expr, xlat, msgid = item[3:]
1806 if self.html and name.lower() in BOOLEAN_HTML_ATTRS:
1807 evalue = self.engine.evaluateBoolean(item[3])
1808 if evalue is self.Default:
1809 if action == 'insert': # Cancelled insert
1810 ok = 0
1811 elif evalue:
1812 value = None
1813 else:
1814 ok = 0
1815 elif expr is not None:
1816 evalue = self.engine.evaluateText(item[3])
1817 if evalue is self.Default:
1818 if action == 'insert': # Cancelled insert
1819 ok = 0
1820 else:
1821 if evalue is None:
1822 ok = 0
1823 value = evalue
1824 else:
1825 evalue = None
1827 if ok:
1828 if xlat:
1829 translated = self.translate(msgid or value, value, {})
1830 if translated is not None:
1831 value = translated
1832 if value is None:
1833 value = name
1834 elif evalue is self.Default:
1835 value = attrEscape(value)
1836 else:
1837 value = escape(value, quote=1)
1838 value = '%s="%s"' % (name, value)
1839 return ok, name, value
1840 bytecode_handlers["<attrAction>"] = attrAction
1842 def no_tag(self, start, program):
1843 state = self.saveState()
1844 self.stream = stream = self.StringIO()
1845 self._stream_write = stream.write
1846 self.interpret(start)
1847 self.restoreOutputState(state)
1848 self.interpret(program)
1850 def do_optTag(self, (name, cexpr, tag_ns, isend, start, program),
1851 omit=0):
1852 if tag_ns and not self.showtal:
1853 return self.no_tag(start, program)
1855 self.interpret(start)
1856 if not isend:
1857 self.interpret(program)
1858 s = '</%s>' % name
1859 self._stream_write(s)
1860 self.col = self.col + len(s)
1862 def do_optTag_tal(self, stuff):
1863 cexpr = stuff[1]
1864 if cexpr is not None and (cexpr == '' or
1865 self.engine.evaluateBoolean(cexpr)):
1866 self.no_tag(stuff[-2], stuff[-1])
1867 else:
1868 self.do_optTag(stuff)
1869 bytecode_handlers["optTag"] = do_optTag
1871 def do_rawtextBeginScope(self, (s, col, position, closeprev, dict)):
1872 self._stream_write(s)
1873 self.col = col
1874 self.position = position
1875 self.engine.setPosition(position)
1876 if closeprev:
1877 engine = self.engine
1878 engine.endScope()
1879 engine.beginScope()
1880 else:
1881 self.engine.beginScope()
1882 self.scopeLevel = self.scopeLevel + 1
1884 def do_rawtextBeginScope_tal(self, (s, col, position, closeprev, dict)):
1885 self._stream_write(s)
1886 self.col = col
1887 engine = self.engine
1888 self.position = position
1889 engine.setPosition(position)
1890 if closeprev:
1891 engine.endScope()
1892 engine.beginScope()
1893 else:
1894 engine.beginScope()
1895 self.scopeLevel = self.scopeLevel + 1
1896 engine.setLocal("attrs", dict)
1897 bytecode_handlers["rawtextBeginScope"] = do_rawtextBeginScope
1899 def do_beginScope(self, dict):
1900 self.engine.beginScope()
1901 self.scopeLevel = self.scopeLevel + 1
1903 def do_beginScope_tal(self, dict):
1904 engine = self.engine
1905 engine.beginScope()
1906 engine.setLocal("attrs", dict)
1907 self.scopeLevel = self.scopeLevel + 1
1908 bytecode_handlers["beginScope"] = do_beginScope
1910 def do_endScope(self, notused=None):
1911 self.engine.endScope()
1912 self.scopeLevel = self.scopeLevel - 1
1913 bytecode_handlers["endScope"] = do_endScope
1915 def do_setLocal(self, notused):
1916 pass
1918 def do_setLocal_tal(self, (name, expr)):
1919 self.engine.setLocal(name, self.engine.evaluateValue(expr))
1920 bytecode_handlers["setLocal"] = do_setLocal
1922 def do_setGlobal_tal(self, (name, expr)):
1923 self.engine.setGlobal(name, self.engine.evaluateValue(expr))
1924 bytecode_handlers["setGlobal"] = do_setLocal
1926 def do_beginI18nContext(self, settings):
1927 get = settings.get
1928 self.i18nContext = TranslationContext(self.i18nContext,
1929 domain=get("domain"),
1930 source=get("source"),
1931 target=get("target"))
1932 bytecode_handlers["beginI18nContext"] = do_beginI18nContext
1934 def do_endI18nContext(self, notused=None):
1935 self.i18nContext = self.i18nContext.parent
1936 assert self.i18nContext is not None
1937 bytecode_handlers["endI18nContext"] = do_endI18nContext
1939 def do_insertText(self, stuff):
1940 self.interpret(stuff[1])
1942 def do_insertText_tal(self, stuff):
1943 text = self.engine.evaluateText(stuff[0])
1944 if text is None:
1945 return
1946 if text is self.Default:
1947 self.interpret(stuff[1])
1948 return
1949 if isinstance(text, MessageID):
1950 text = self.engine.translate(text.domain, text, text.mapping)
1951 s = escape(text)
1952 self._stream_write(s)
1953 i = s.rfind('\n')
1954 if i < 0:
1955 self.col = self.col + len(s)
1956 else:
1957 self.col = len(s) - (i + 1)
1958 bytecode_handlers["insertText"] = do_insertText
1960 def do_insertRawText_tal(self, stuff):
1961 text = self.engine.evaluateText(stuff[0])
1962 if text is None:
1963 return
1964 if text is self.Default:
1965 self.interpret(stuff[1])
1966 return
1967 if isinstance(text, MessageID):
1968 text = self.engine.translate(text.domain, text, text.mapping)
1969 s = text
1970 self._stream_write(s)
1971 i = s.rfind('\n')
1972 if i < 0:
1973 self.col = self.col + len(s)
1974 else:
1975 self.col = len(s) - (i + 1)
1977 def do_i18nVariable(self, stuff):
1978 varname, program, expression, structure = stuff
1979 if expression is None:
1980 state = self.saveState()
1981 try:
1982 tmpstream = self.StringIO()
1983 self.interpretWithStream(program, tmpstream)
1984 if self.html and self._currentTag == "pre":
1985 value = tmpstream.getvalue()
1986 else:
1987 value = normalize(tmpstream.getvalue())
1988 finally:
1989 self.restoreState(state)
1990 else:
1991 if structure:
1992 value = self.engine.evaluateStructure(expression)
1993 else:
1994 value = self.engine.evaluate(expression)
1996 if isinstance(value, MessageID):
1997 value = self.engine.translate(value.domain, value,
1998 value.mapping)
2000 if not structure:
2001 value = cgi.escape(ustr(value))
2003 i18ndict, srepr = self.i18nStack[-1]
2004 i18ndict[varname] = value
2005 placeholder = '${%s}' % varname
2006 srepr.append(placeholder)
2007 self._stream_write(placeholder)
2008 bytecode_handlers['i18nVariable'] = do_i18nVariable
2010 def do_insertTranslation(self, stuff):
2011 i18ndict = {}
2012 srepr = []
2013 obj = None
2014 self.i18nStack.append((i18ndict, srepr))
2015 msgid = stuff[0]
2016 currentTag = self._currentTag
2017 tmpstream = self.StringIO()
2018 self.interpretWithStream(stuff[1], tmpstream)
2019 default = tmpstream.getvalue()
2020 if not msgid:
2021 if self.html and currentTag == "pre":
2022 msgid = default
2023 else:
2024 msgid = normalize(default)
2025 self.i18nStack.pop()
2026 if len(stuff) > 2:
2027 obj = self.engine.evaluate(stuff[2])
2028 xlated_msgid = self.translate(msgid, default, i18ndict, obj)
2029 assert xlated_msgid is not None
2030 self._stream_write(xlated_msgid)
2031 bytecode_handlers['insertTranslation'] = do_insertTranslation
2033 def do_insertStructure(self, stuff):
2034 self.interpret(stuff[2])
2036 def do_insertStructure_tal(self, (expr, repldict, block)):
2037 structure = self.engine.evaluateStructure(expr)
2038 if structure is None:
2039 return
2040 if structure is self.Default:
2041 self.interpret(block)
2042 return
2043 text = ustr(structure)
2044 if not (repldict or self.strictinsert):
2045 self.stream_write(text)
2046 return
2047 if self.html:
2048 self.insertHTMLStructure(text, repldict)
2049 else:
2050 self.insertXMLStructure(text, repldict)
2051 bytecode_handlers["insertStructure"] = do_insertStructure
2053 def insertHTMLStructure(self, text, repldict):
2054 gen = AltTALGenerator(repldict, self.engine.getCompiler(), 0)
2055 p = HTMLTALParser(gen) # Raises an exception if text is invalid
2056 p.parseString(text)
2057 program, macros = p.getCode()
2058 self.interpret(program)
2060 def insertXMLStructure(self, text, repldict):
2061 gen = AltTALGenerator(repldict, self.engine.getCompiler(), 0)
2062 p = TALParser(gen)
2063 gen.enable(0)
2064 p.parseFragment('<!DOCTYPE foo PUBLIC "foo" "bar"><foo>')
2065 gen.enable(1)
2066 p.parseFragment(text) # Raises an exception if text is invalid
2067 gen.enable(0)
2068 p.parseFragment('</foo>', 1)
2069 program, macros = gen.getCode()
2070 self.interpret(program)
2072 def do_loop(self, (name, expr, block)):
2073 self.interpret(block)
2075 def do_loop_tal(self, (name, expr, block)):
2076 iterator = self.engine.setRepeat(name, expr)
2077 while iterator.next():
2078 self.interpret(block)
2079 bytecode_handlers["loop"] = do_loop
2081 def translate(self, msgid, default, i18ndict, obj=None):
2082 if obj:
2083 i18ndict.update(obj)
2084 if not self.i18nInterpolate:
2085 return msgid
2086 return self.engine.translate(self.i18nContext.domain,
2087 msgid, i18ndict, default=default)
2089 def do_rawtextColumn(self, (s, col)):
2090 self._stream_write(s)
2091 self.col = col
2092 bytecode_handlers["rawtextColumn"] = do_rawtextColumn
2094 def do_rawtextOffset(self, (s, offset)):
2095 self._stream_write(s)
2096 self.col = self.col + offset
2097 bytecode_handlers["rawtextOffset"] = do_rawtextOffset
2099 def do_condition(self, (condition, block)):
2100 if not self.tal or self.engine.evaluateBoolean(condition):
2101 self.interpret(block)
2102 bytecode_handlers["condition"] = do_condition
2104 def do_defineMacro(self, (macroName, macro)):
2105 macs = self.macroStack
2106 if len(macs) == 1:
2107 entering = macs[-1][2]
2108 if not entering:
2109 macs.append(None)
2110 self.interpret(macro)
2111 assert macs[-1] is None
2112 macs.pop()
2113 return
2114 self.interpret(macro)
2115 bytecode_handlers["defineMacro"] = do_defineMacro
2117 def do_useMacro(self, (macroName, macroExpr, compiledSlots, block)):
2118 if not self.metal:
2119 self.interpret(block)
2120 return
2121 macro = self.engine.evaluateMacro(macroExpr)
2122 if macro is self.Default:
2123 macro = block
2124 else:
2125 if not isCurrentVersion(macro):
2126 raise METALError("macro %s has incompatible version %s" %
2127 (`macroName`, `getProgramVersion(macro)`),
2128 self.position)
2129 mode = getProgramMode(macro)
2130 #if mode != (self.html and "html" or "xml"):
2131 # raise METALError("macro %s has incompatible mode %s" %
2132 # (`macroName`, `mode`), self.position)
2134 self.pushMacro(macroName, compiledSlots)
2135 prev_source = self.sourceFile
2136 self.interpret(macro)
2137 if self.sourceFile != prev_source:
2138 self.engine.setSourceFile(prev_source)
2139 self.sourceFile = prev_source
2140 self.popMacro()
2141 bytecode_handlers["useMacro"] = do_useMacro
2143 def do_fillSlot(self, (slotName, block)):
2144 self.interpret(block)
2145 bytecode_handlers["fillSlot"] = do_fillSlot
2147 def do_defineSlot(self, (slotName, block)):
2148 if not self.metal:
2149 self.interpret(block)
2150 return
2151 macs = self.macroStack
2152 if macs and macs[-1] is not None:
2153 macroName, slots = self.popMacro()[:2]
2154 slot = slots.get(slotName)
2155 if slot is not None:
2156 prev_source = self.sourceFile
2157 self.interpret(slot)
2158 if self.sourceFile != prev_source:
2159 self.engine.setSourceFile(prev_source)
2160 self.sourceFile = prev_source
2161 self.pushMacro(macroName, slots, entering=0)
2162 return
2163 self.pushMacro(macroName, slots)
2164 self.interpret(block)
2165 bytecode_handlers["defineSlot"] = do_defineSlot
2167 def do_onError(self, (block, handler)):
2168 self.interpret(block)
2170 def do_onError_tal(self, (block, handler)):
2171 state = self.saveState()
2172 self.stream = stream = self.StringIO()
2173 self._stream_write = stream.write
2174 try:
2175 self.interpret(block)
2176 except ConflictError:
2177 raise
2178 except:
2179 exc = sys.exc_info()[1]
2180 self.restoreState(state)
2181 engine = self.engine
2182 engine.beginScope()
2183 error = engine.createErrorInfo(exc, self.position)
2184 engine.setLocal('error', error)
2185 try:
2186 self.interpret(handler)
2187 finally:
2188 engine.endScope()
2189 else:
2190 self.restoreOutputState(state)
2191 self.stream_write(stream.getvalue())
2192 bytecode_handlers["onError"] = do_onError
2194 bytecode_handlers_tal = bytecode_handlers.copy()
2195 bytecode_handlers_tal["rawtextBeginScope"] = do_rawtextBeginScope_tal
2196 bytecode_handlers_tal["beginScope"] = do_beginScope_tal
2197 bytecode_handlers_tal["setLocal"] = do_setLocal_tal
2198 bytecode_handlers_tal["setGlobal"] = do_setGlobal_tal
2199 bytecode_handlers_tal["insertStructure"] = do_insertStructure_tal
2200 bytecode_handlers_tal["insertText"] = do_insertText_tal
2201 bytecode_handlers_tal["insertRaw"] = do_insertRawText_tal
2202 bytecode_handlers_tal["loop"] = do_loop_tal
2203 bytecode_handlers_tal["onError"] = do_onError_tal
2204 bytecode_handlers_tal["<attrAction>"] = attrAction_tal
2205 bytecode_handlers_tal["optTag"] = do_optTag_tal
2208 class FasterStringIO(StringIO):
2209 """Append-only version of StringIO.
2211 This let's us have a much faster write() method.
2213 def close(self):
2214 if not self.closed:
2215 self.write = _write_ValueError
2216 StringIO.close(self)
2218 def seek(self, pos, mode=0):
2219 raise RuntimeError("FasterStringIO.seek() not allowed")
2221 def write(self, s):
2222 self.buflist.append(s)
2223 self.len = self.pos = self.pos + len(s)
2226 def _write_ValueError(s):
2227 raise ValueError, "I/O operation on closed file"
2229 Parse XML and compile to TALInterpreter intermediate code.
2233 class TALParser(XMLParser):
2235 ordered_attributes = 1
2237 def __init__(self, gen=None): # Override
2238 XMLParser.__init__(self)
2239 if gen is None:
2240 gen = TALGenerator()
2241 self.gen = gen
2242 self.nsStack = []
2243 self.nsDict = {XML_NS: 'xml'}
2244 self.nsNew = []
2246 def getCode(self):
2247 return self.gen.getCode()
2249 def getWarnings(self):
2250 return ()
2252 def StartNamespaceDeclHandler(self, prefix, uri):
2253 self.nsStack.append(self.nsDict.copy())
2254 self.nsDict[uri] = prefix
2255 self.nsNew.append((prefix, uri))
2257 def EndNamespaceDeclHandler(self, prefix):
2258 self.nsDict = self.nsStack.pop()
2260 def StartElementHandler(self, name, attrs):
2261 if self.ordered_attributes:
2262 attrlist = []
2263 for i in range(0, len(attrs), 2):
2264 key = attrs[i]
2265 value = attrs[i+1]
2266 attrlist.append((key, value))
2267 else:
2268 attrlist = attrs.items()
2269 attrlist.sort() # For definiteness
2270 name, attrlist, taldict, metaldict, i18ndict \
2271 = self.process_ns(name, attrlist)
2272 attrlist = self.xmlnsattrs() + attrlist
2273 self.gen.emitStartElement(name, attrlist, taldict, metaldict, i18ndict)
2275 def process_ns(self, name, attrlist):
2276 taldict = {}
2277 metaldict = {}
2278 i18ndict = {}
2279 fixedattrlist = []
2280 name, namebase, namens = self.fixname(name)
2281 for key, value in attrlist:
2282 key, keybase, keyns = self.fixname(key)
2283 ns = keyns or namens # default to tag namespace
2284 item = key, value
2285 if ns == 'metal':
2286 metaldict[keybase] = value
2287 item = item + ("metal",)
2288 elif ns == 'tal':
2289 taldict[keybase] = value
2290 item = item + ("tal",)
2291 elif ns == 'i18n':
2292 i18ndict[keybase] = value
2293 item = item + ('i18n',)
2294 fixedattrlist.append(item)
2295 if namens in ('metal', 'tal', 'i18n'):
2296 taldict['tal tag'] = namens
2297 return name, fixedattrlist, taldict, metaldict, i18ndict
2299 def xmlnsattrs(self):
2300 newlist = []
2301 for prefix, uri in self.nsNew:
2302 if prefix:
2303 key = "xmlns:" + prefix
2304 else:
2305 key = "xmlns"
2306 if uri in (ZOPE_METAL_NS, ZOPE_TAL_NS, ZOPE_I18N_NS):
2307 item = (key, uri, "xmlns")
2308 else:
2309 item = (key, uri)
2310 newlist.append(item)
2311 self.nsNew = []
2312 return newlist
2314 def fixname(self, name):
2315 if ' ' in name:
2316 uri, name = name.split(' ')
2317 prefix = self.nsDict[uri]
2318 prefixed = name
2319 if prefix:
2320 prefixed = "%s:%s" % (prefix, name)
2321 ns = 'x'
2322 if uri == ZOPE_TAL_NS:
2323 ns = 'tal'
2324 elif uri == ZOPE_METAL_NS:
2325 ns = 'metal'
2326 elif uri == ZOPE_I18N_NS:
2327 ns = 'i18n'
2328 return (prefixed, name, ns)
2329 return (name, name, None)
2331 def EndElementHandler(self, name):
2332 name = self.fixname(name)[0]
2333 self.gen.emitEndElement(name)
2335 def DefaultHandler(self, text):
2336 self.gen.emitRawText(text)
2338 """Translation context object for the TALInterpreter's I18N support.
2340 The translation context provides a container for the information
2341 needed to perform translation of a marked string from a page template.
2345 DEFAULT_DOMAIN = "default"
2347 class TranslationContext:
2348 """Information about the I18N settings of a TAL processor."""
2350 def __init__(self, parent=None, domain=None, target=None, source=None):
2351 if parent:
2352 if not domain:
2353 domain = parent.domain
2354 if not target:
2355 target = parent.target
2356 if not source:
2357 source = parent.source
2358 elif domain is None:
2359 domain = DEFAULT_DOMAIN
2361 self.parent = parent
2362 self.domain = domain
2363 self.target = target
2364 self.source = source
2366 Dummy TALES engine so that I can test out the TAL implementation.
2369 import re
2370 import sys
2371 import stat
2372 import os
2373 import traceback
2375 class _Default:
2376 pass
2377 Default = _Default()
2379 name_match = re.compile(r"(?s)(%s):(.*)\Z" % NAME_RE).match
2381 class CompilerError(Exception):
2382 pass
2384 class AthanaTALEngine:
2386 position = None
2387 source_file = None
2389 __implements__ = ITALESCompiler, ITALESEngine
2391 def __init__(self, macros=None, context=None, webcontext=None, language=None, request=None):
2392 if macros is None:
2393 macros = {}
2394 self.macros = macros
2395 dict = {'nothing': None, 'default': Default}
2396 if context is not None:
2397 dict.update(context)
2399 self.locals = self.globals = dict
2400 self.stack = [dict]
2401 self.webcontext = webcontext
2402 self.language = language
2403 self.request = request
2405 def compilefile(self, file, mode=None):
2406 assert mode in ("html", "xml", None)
2407 #file = join_paths(GLOBAL_ROOT_DIR,join_paths(self.webcontext.root, file))
2408 if mode is None:
2409 ext = os.path.splitext(file)[1]
2410 if ext.lower() in (".html", ".htm"):
2411 mode = "html"
2412 else:
2413 mode = "xml"
2414 if mode == "html":
2415 p = HTMLTALParser(TALGenerator(self))
2416 else:
2417 p = TALParser(TALGenerator(self))
2418 p.parseFile(file)
2419 return p.getCode()
2421 def getCompilerError(self):
2422 return CompilerError
2424 def getCompiler(self):
2425 return self
2427 def setSourceFile(self, source_file):
2428 self.source_file = source_file
2430 def setPosition(self, position):
2431 self.position = position
2433 def compile(self, expr):
2434 return "$%s$" % expr
2436 def uncompile(self, expression):
2437 assert (expression.startswith("$") and expression.endswith("$"),
2438 expression)
2439 return expression[1:-1]
2441 def beginScope(self):
2442 self.stack.append(self.locals)
2444 def endScope(self):
2445 assert len(self.stack) > 1, "more endScope() than beginScope() calls"
2446 self.locals = self.stack.pop()
2448 def setLocal(self, name, value):
2449 if self.locals is self.stack[-1]:
2450 self.locals = self.locals.copy()
2451 self.locals[name] = value
2453 def setGlobal(self, name, value):
2454 self.globals[name] = value
2456 def evaluate(self, expression):
2457 assert (expression.startswith("$") and expression.endswith("$"),
2458 expression)
2459 expression = expression[1:-1]
2460 m = name_match(expression)
2461 if m:
2462 type, expr = m.group(1, 2)
2463 else:
2464 type = "path"
2465 expr = expression
2466 if type in ("string", "str"):
2467 return expr
2468 if type in ("path", "var", "global", "local"):
2469 return self.evaluatePathOrVar(expr)
2470 if type == "not":
2471 return not self.evaluate(expr)
2472 if type == "exists":
2473 return self.locals.has_key(expr) or self.globals.has_key(expr)
2474 if type == "python":
2475 try:
2476 return eval(expr, self.globals, self.locals)
2477 except:
2478 print "Error in python expression"
2479 print sys.exc_info()[0], sys.exc_info()[1]
2480 traceback.print_tb(sys.exc_info()[2])
2481 raise TALESError("evaluation error in %s" % `expr`)
2483 if type == "position":
2484 if self.position:
2485 lineno, offset = self.position
2486 else:
2487 lineno, offset = None, None
2488 return '%s (%s,%s)' % (self.source_file, lineno, offset)
2489 raise TALESError("unrecognized expression: " + `expression`)
2491 def evaluatePathOrVar(self, expr):
2492 expr = expr.strip()
2493 _expr=expr
2494 _f=None
2495 if expr.rfind("/")>0:
2496 pos=expr.rfind("/")
2497 _expr = expr[0:pos]
2498 _f = expr[pos+1:]
2499 if self.locals.has_key(_expr):
2500 if _f:
2501 return getattr(self.locals[_expr],_f)
2502 else:
2503 return self.locals[_expr]
2504 elif self.globals.has_key(_expr):
2505 if _f:
2506 return getattr(self.globals[_expr], _f)
2507 else:
2508 return self.globals[_expr]
2509 else:
2510 raise TALESError("unknown variable: %s" % `_expr`)
2512 def evaluateValue(self, expr):
2513 return self.evaluate(expr)
2515 def evaluateBoolean(self, expr):
2516 return self.evaluate(expr)
2518 def evaluateText(self, expr):
2519 text = self.evaluate(expr)
2520 if text is not None and text is not Default:
2521 text = ustr(text)
2522 return text
2524 def evaluateStructure(self, expr):
2525 return self.evaluate(expr)
2527 def evaluateSequence(self, expr):
2528 return self.evaluate(expr)
2530 def evaluateMacro(self, macroName):
2531 assert (macroName.startswith("$") and macroName.endswith("$"),
2532 macroName)
2533 macroName = macroName[1:-1]
2534 file, localName = self.findMacroFile(macroName)
2535 if not file:
2536 macro = self.macros[localName]
2537 else:
2538 program, macros = self.compilefile(file)
2539 macro = macros.get(localName)
2540 if not macro:
2541 raise TALESError("macro %s not found in file %s" %
2542 (localName, file))
2543 return macro
2545 def findMacroDocument(self, macroName):
2546 file, localName = self.findMacroFile(macroName)
2547 if not file:
2548 return file, localName
2549 doc = parsefile(file)
2550 return doc, localName
2552 def findMacroFile(self, macroName):
2553 if not macroName:
2554 raise TALESError("empty macro name")
2555 i = macroName.rfind('/')
2556 if i < 0:
2557 print "NO Macro"
2558 return None, macroName
2559 else:
2560 fileName = getMacroFile(macroName[:i])
2561 localName = macroName[i+1:]
2562 return fileName, localName
2564 def setRepeat(self, name, expr):
2565 seq = self.evaluateSequence(expr)
2566 self.locals[name] = Iterator(name, seq, self)
2567 return self.locals[name]
2569 def createErrorInfo(self, err, position):
2570 return ErrorInfo(err, position)
2572 def getDefault(self):
2573 return Default
2575 def translate(self, domain, msgid, mapping, default=None):
2576 global translators
2577 text = default or msgid
2578 for f in translators:
2579 text = f(msgid, language=self.language, request=self.request)
2580 try:
2581 text = f(msgid, language=self.language, request=self.request)
2582 if text and text!=msgid:
2583 break
2584 except:
2585 pass
2586 def repl(m, mapping=mapping):
2587 return ustr(mapping[m.group(m.lastindex).lower()])
2588 return VARIABLE.sub(repl, text)
2591 class Iterator:
2593 def __init__(self, name, seq, engine):
2594 self.name = name
2595 self.seq = seq
2596 self.engine = engine
2597 self.nextIndex = 0
2599 def next(self):
2600 self.index = i = self.nextIndex
2601 try:
2602 item = self.seq[i]
2603 except IndexError:
2604 return 0
2605 self.nextIndex = i+1
2606 self.engine.setLocal(self.name, item)
2607 return 1
2609 def even(self):
2610 print "-even-"
2611 return not self.index % 2
2613 def odd(self):
2614 print "-odd-"
2615 return self.index % 2
2617 def number(self):
2618 return self.nextIndex
2620 def parity(self):
2621 if self.index % 2:
2622 return 'odd'
2623 return 'even'
2625 def first(self, name=None):
2626 if self.start: return 1
2627 return not self.same_part(name, self._last, self.item)
2629 def last(self, name=None):
2630 if self.end: return 1
2631 return not self.same_part(name, self.item, self._next)
2633 def length(self):
2634 return len(self.seq)
2637 VARIABLE = re.compile(r'\$(?:(%s)|\{(%s)\})' % (NAME_RE, NAME_RE))
2639 parsed_files = {}
2640 parsed_strings = {}
2642 def runTAL(writer, context=None, string=None, file=None, macro=None, language=None, request=None):
2644 if file:
2645 file = getMacroFile(file)
2647 if context is None:
2648 context = {}
2650 if string and not file:
2651 if string in parsed_strings:
2652 program,macros = parsed_strings[string]
2653 else:
2654 program,macros = None,None
2655 elif file and not string:
2656 if file in parsed_files:
2657 (program,macros,mtime) = parsed_files[file]
2658 mtime_file = os.stat(file)[stat.ST_MTIME]
2659 if mtime != mtime_file:
2660 program,macros = None,None
2661 mtime = mtime_file
2662 else:
2663 program,macros,mtime = None,None,None
2665 if not (program and macros):
2666 if file and file.endswith("xml"):
2667 talparser = TALParser(TALGenerator(AthanaTALEngine()))
2668 else:
2669 talparser = HTMLTALParser(TALGenerator(AthanaTALEngine()))
2670 if string:
2671 talparser.parseString(string)
2672 (program, macros) = talparser.getCode()
2673 parsed_strings[string] = (program,macros)
2674 else:
2675 talparser.parseFile(file)
2676 (program, macros) = talparser.getCode()
2677 parsed_files[file] = (program,macros,mtime)
2679 if macro and macro in macros:
2680 program = macros[macro]
2681 engine = AthanaTALEngine(macros, context, language=language, request=request)
2682 TALInterpreter(program, macros, engine, writer, wrap=0)()
2684 def processTAL(context=None, string=None, file=None, macro=None, language=None, request=None):
2685 class STRWriter:
2686 def __init__(self):
2687 self.string = ""
2688 def write(self,text):
2689 if type(text) == type(u''):
2690 self.string += text.encode("utf-8")
2691 else:
2692 self.string += text
2693 def getvalue(self):
2694 return self.string
2695 wr = STRWriter()
2696 runTAL(wr, context, string=string, file=file, macro=macro, language=language, request=request)
2697 return wr.getvalue()
2700 class MyWriter:
2701 def write(self,s):
2702 sys.stdout.write(s)
2704 def test():
2705 p = TALParser(TALGenerator(AthanaTALEngine()))
2706 file = "test.xml"
2707 if sys.argv[1:]:
2708 file = sys.argv[1]
2709 p.parseFile(file)
2710 program, macros = p.getCode()
2712 class Node:
2713 def getText(self):
2714 return "TEST"
2716 engine = AthanaTALEngine(macros, {'node': Node()})
2717 TALInterpreter(program, macros, engine, MyWriter(), wrap=0)()
2720 def ustr(v):
2721 """Convert any object to a plain string or unicode string,
2722 minimising the chance of raising a UnicodeError. This
2723 even works with uncooperative objects like Exceptions
2725 if type(v) == type(""): #isinstance(v, basestring):
2726 return v
2727 else:
2728 fn = getattr(v,'__str__',None)
2729 if fn is not None:
2730 v = fn()
2731 if isinstance(v, basestring):
2732 return v
2733 else:
2734 raise ValueError('__str__ returned wrong type')
2735 return str(v)
2738 # ================ MEDUSA ===============
2740 # python modules
2741 import os
2742 import re
2743 import select
2744 import socket
2745 import string
2746 import sys
2747 import time
2748 import stat
2749 import string
2750 import mimetypes
2751 import glob
2752 from cgi import escape
2753 from urllib import unquote, splitquery
2755 # async modules
2756 import asyncore
2757 import socket
2759 class async_chat (asyncore.dispatcher):
2760 """This is an abstract class. You must derive from this class, and add
2761 the two methods collect_incoming_data() and found_terminator()"""
2763 # these are overridable defaults
2765 ac_in_buffer_size = 4096
2766 ac_out_buffer_size = 4096
2768 def __init__ (self, conn=None):
2769 self.ac_in_buffer = ''
2770 self.ac_out_buffer = ''
2771 self.producer_fifo = fifo()
2772 asyncore.dispatcher.__init__ (self, conn)
2774 def collect_incoming_data(self, data):
2775 raise NotImplementedError, "must be implemented in subclass"
2777 def found_terminator(self):
2778 raise NotImplementedError, "must be implemented in subclass"
2780 def set_terminator (self, term):
2781 "Set the input delimiter. Can be a fixed string of any length, an integer, or None"
2782 self.terminator = term
2784 def get_terminator (self):
2785 return self.terminator
2787 # grab some more data from the socket,
2788 # throw it to the collector method,
2789 # check for the terminator,
2790 # if found, transition to the next state.
2792 def handle_read (self):
2794 try:
2795 data = self.recv (self.ac_in_buffer_size)
2796 except socket.error, why:
2797 self.handle_error()
2798 return
2800 self.ac_in_buffer = self.ac_in_buffer + data
2802 # Continue to search for self.terminator in self.ac_in_buffer,
2803 # while calling self.collect_incoming_data. The while loop
2804 # is necessary because we might read several data+terminator
2805 # combos with a single recv(1024).
2807 while self.ac_in_buffer:
2808 lb = len(self.ac_in_buffer)
2809 terminator = self.get_terminator()
2810 if terminator is None or terminator == '':
2811 # no terminator, collect it all
2812 self.collect_incoming_data (self.ac_in_buffer)
2813 self.ac_in_buffer = ''
2814 elif isinstance(terminator, int):
2815 # numeric terminator
2816 n = terminator
2817 if lb < n:
2818 self.collect_incoming_data (self.ac_in_buffer)
2819 self.ac_in_buffer = ''
2820 self.terminator = self.terminator - lb
2821 else:
2822 self.collect_incoming_data (self.ac_in_buffer[:n])
2823 self.ac_in_buffer = self.ac_in_buffer[n:]
2824 self.terminator = 0
2825 self.found_terminator()
2826 else:
2827 # 3 cases:
2828 # 1) end of buffer matches terminator exactly:
2829 # collect data, transition
2830 # 2) end of buffer matches some prefix:
2831 # collect data to the prefix
2832 # 3) end of buffer does not match any prefix:
2833 # collect data
2834 terminator_len = len(terminator)
2835 index = self.ac_in_buffer.find(terminator)
2836 if index != -1:
2837 # we found the terminator
2838 if index > 0:
2839 # don't bother reporting the empty string (source of subtle bugs)
2840 self.collect_incoming_data (self.ac_in_buffer[:index])
2841 self.ac_in_buffer = self.ac_in_buffer[index+terminator_len:]
2842 # This does the Right Thing if the terminator is changed here.
2843 self.found_terminator()
2844 else:
2845 # check for a prefix of the terminator
2846 index = find_prefix_at_end (self.ac_in_buffer, terminator)
2847 if index:
2848 if index != lb:
2849 # we found a prefix, collect up to the prefix
2850 self.collect_incoming_data (self.ac_in_buffer[:-index])
2851 self.ac_in_buffer = self.ac_in_buffer[-index:]
2852 break
2853 else:
2854 # no prefix, collect it all
2855 self.collect_incoming_data (self.ac_in_buffer)
2856 self.ac_in_buffer = ''
2858 def handle_write (self):
2859 self.initiate_send ()
2861 def handle_close (self):
2862 self.close()
2864 def push (self, data):
2865 self.producer_fifo.push (simple_producer (data))
2866 self.initiate_send()
2868 def push_with_producer (self, producer):
2869 self.producer_fifo.push (producer)
2870 self.initiate_send()
2872 def readable (self):
2873 "predicate for inclusion in the readable for select()"
2874 return (len(self.ac_in_buffer) <= self.ac_in_buffer_size)
2876 def writable (self):
2877 "predicate for inclusion in the writable for select()"
2878 # return len(self.ac_out_buffer) or len(self.producer_fifo) or (not self.connected)
2879 # this is about twice as fast, though not as clear.
2880 return not (
2881 (self.ac_out_buffer == '') and
2882 self.producer_fifo.is_empty() and
2883 self.connected
2886 def close_when_done (self):
2887 "automatically close this channel once the outgoing queue is empty"
2888 self.producer_fifo.push (None)
2890 # refill the outgoing buffer by calling the more() method
2891 # of the first producer in the queue
2892 def refill_buffer (self):
2893 while 1:
2894 if len(self.producer_fifo):
2895 p = self.producer_fifo.first()
2896 # a 'None' in the producer fifo is a sentinel,
2897 # telling us to close the channel.
2898 if p is None:
2899 if not self.ac_out_buffer:
2900 self.producer_fifo.pop()
2901 self.close()
2902 return
2903 elif isinstance(p, str):
2904 self.producer_fifo.pop()
2905 self.ac_out_buffer = self.ac_out_buffer + p
2906 return
2907 data = p.more()
2908 if data:
2909 self.ac_out_buffer = self.ac_out_buffer + data
2910 return
2911 else:
2912 self.producer_fifo.pop()
2913 else:
2914 return
2916 def initiate_send (self):
2917 obs = self.ac_out_buffer_size
2918 # try to refill the buffer
2919 if (len (self.ac_out_buffer) < obs):
2920 self.refill_buffer()
2922 if self.ac_out_buffer and self.connected:
2923 # try to send the buffer
2924 try:
2925 num_sent = self.send (self.ac_out_buffer[:obs])
2926 if num_sent:
2927 self.ac_out_buffer = self.ac_out_buffer[num_sent:]
2929 except socket.error, why:
2930 self.handle_error()
2931 return
2933 def discard_buffers (self):
2934 # Emergencies only!
2935 self.ac_in_buffer = ''
2936 self.ac_out_buffer = ''
2937 while self.producer_fifo:
2938 self.producer_fifo.pop()
2941 class simple_producer:
2943 def __init__ (self, data, buffer_size=512):
2944 self.data = data
2945 self.buffer_size = buffer_size
2947 def more (self):
2948 if len (self.data) > self.buffer_size:
2949 result = self.data[:self.buffer_size]
2950 self.data = self.data[self.buffer_size:]
2951 return result
2952 else:
2953 result = self.data
2954 self.data = ''
2955 return result
2957 class fifo:
2958 def __init__ (self, list=None):
2959 if not list:
2960 self.list = []
2961 else:
2962 self.list = list
2964 def __len__ (self):
2965 return len(self.list)
2967 def is_empty (self):
2968 return self.list == []
2970 def first (self):
2971 return self.list[0]
2973 def push (self, data):
2974 self.list.append (data)
2976 def pop (self):
2977 if self.list:
2978 return (1, self.list.pop(0))
2979 else:
2980 return (0, None)
2982 # Given 'haystack', see if any prefix of 'needle' is at its end. This
2983 # assumes an exact match has already been checked. Return the number of
2984 # characters matched.
2985 # for example:
2986 # f_p_a_e ("qwerty\r", "\r\n") => 1
2987 # f_p_a_e ("qwertydkjf", "\r\n") => 0
2988 # f_p_a_e ("qwerty\r\n", "\r\n") => <undefined>
2990 # this could maybe be made faster with a computed regex?
2991 # [answer: no; circa Python-2.0, Jan 2001]
2992 # new python: 28961/s
2993 # old python: 18307/s
2994 # re: 12820/s
2995 # regex: 14035/s
2997 def find_prefix_at_end (haystack, needle):
2998 l = len(needle) - 1
2999 while l and not haystack.endswith(needle[:l]):
3000 l -= 1
3001 return l
3003 class counter:
3004 "general-purpose counter"
3006 def __init__ (self, initial_value=0):
3007 self.value = initial_value
3009 def increment (self, delta=1):
3010 result = self.value
3011 try:
3012 self.value = self.value + delta
3013 except OverflowError:
3014 self.value = long(self.value) + delta
3015 return result
3017 def decrement (self, delta=1):
3018 result = self.value
3019 try:
3020 self.value = self.value - delta
3021 except OverflowError:
3022 self.value = long(self.value) - delta
3023 return result
3025 def as_long (self):
3026 return long(self.value)
3028 def __nonzero__ (self):
3029 return self.value != 0
3031 def __repr__ (self):
3032 return '<counter value=%s at %x>' % (self.value, id(self))
3034 def __str__ (self):
3035 s = str(long(self.value))
3036 if s[-1:] == 'L':
3037 s = s[:-1]
3038 return s
3041 # http_date
3042 def concat (*args):
3043 return ''.join (args)
3045 def join (seq, field=' '):
3046 return field.join (seq)
3048 def group (s):
3049 return '(' + s + ')'
3051 short_days = ['sun','mon','tue','wed','thu','fri','sat']
3052 long_days = ['sunday','monday','tuesday','wednesday','thursday','friday','saturday']
3054 short_day_reg = group (join (short_days, '|'))
3055 long_day_reg = group (join (long_days, '|'))
3057 daymap = {}
3058 for i in range(7):
3059 daymap[short_days[i]] = i
3060 daymap[long_days[i]] = i
3062 hms_reg = join (3 * [group('[0-9][0-9]')], ':')
3064 months = ['jan','feb','mar','apr','may','jun','jul','aug','sep','oct','nov','dec']
3066 monmap = {}
3067 for i in range(12):
3068 monmap[months[i]] = i+1
3070 months_reg = group (join (months, '|'))
3072 # From draft-ietf-http-v11-spec-07.txt/3.3.1
3073 # Sun, 06 Nov 1994 08:49:37 GMT ; RFC 822, updated by RFC 1123
3074 # Sunday, 06-Nov-94 08:49:37 GMT ; RFC 850, obsoleted by RFC 1036
3075 # Sun Nov 6 08:49:37 1994 ; ANSI C's asctime() format
3077 # rfc822 format
3078 rfc822_date = join (
3079 [concat (short_day_reg,','), # day
3080 group('[0-9][0-9]?'), # date
3081 months_reg, # month
3082 group('[0-9]+'), # year
3083 hms_reg, # hour minute second
3084 'gmt'
3089 rfc822_reg = re.compile (rfc822_date)
3091 def unpack_rfc822 (m):
3092 g = m.group
3093 a = string.atoi
3094 return (
3095 a(g(4)), # year
3096 monmap[g(3)], # month
3097 a(g(2)), # day
3098 a(g(5)), # hour
3099 a(g(6)), # minute
3100 a(g(7)), # second
3106 # rfc850 format
3107 rfc850_date = join (
3108 [concat (long_day_reg,','),
3109 join (
3110 [group ('[0-9][0-9]?'),
3111 months_reg,
3112 group ('[0-9]+')
3116 hms_reg,
3117 'gmt'
3122 rfc850_reg = re.compile (rfc850_date)
3123 # they actually unpack the same way
3124 def unpack_rfc850 (m):
3125 g = m.group
3126 a = string.atoi
3127 return (
3128 a(g(4)), # year
3129 monmap[g(3)], # month
3130 a(g(2)), # day
3131 a(g(5)), # hour
3132 a(g(6)), # minute
3133 a(g(7)), # second
3139 # parsdate.parsedate - ~700/sec.
3140 # parse_http_date - ~1333/sec.
3142 def build_http_date (when):
3143 return time.strftime ('%a, %d %b %Y %H:%M:%S GMT', time.gmtime(when))
3145 time_offset = 0
3147 def parse_http_date (d):
3148 global time_offset
3149 d = string.lower (d)
3150 tz = time.timezone
3151 m = rfc850_reg.match (d)
3152 if m and m.end() == len(d):
3153 retval = int (time.mktime (unpack_rfc850(m)) - tz)
3154 else:
3155 m = rfc822_reg.match (d)
3156 if m and m.end() == len(d):
3157 try:
3158 retval = int (time.mktime (unpack_rfc822(m)) - tz)
3159 except OverflowError:
3160 return 0
3161 else:
3162 return 0
3163 # Thanks to Craig Silverstein <csilvers@google.com> for pointing
3164 # out the DST discrepancy
3165 if time.daylight and time.localtime(retval)[-1] == 1: # DST correction
3166 retval = retval + (tz - time.altzone)
3167 return retval - time_offset
3169 def check_date():
3170 global time_offset
3171 tmpfile = join_paths(GLOBAL_TEMP_DIR, "datetest"+str(random.random())+".tmp")
3172 open(tmpfile,"wb").close()
3173 time1 = os.stat(tmpfile)[stat.ST_MTIME]
3174 os.unlink(tmpfile)
3175 time2 = parse_http_date(build_http_date(time.time()))
3176 time_offset = time2-time1
3177 print time_offset
3179 # producers
3181 class simple_producer:
3182 "producer for a string"
3183 def __init__ (self, data, buffer_size=1024):
3184 self.data = data
3185 self.buffer_size = buffer_size
3187 def more (self):
3188 if len (self.data) > self.buffer_size:
3189 result = self.data[:self.buffer_size]
3190 self.data = self.data[self.buffer_size:]
3191 return result
3192 else:
3193 result = self.data
3194 self.data = ''
3195 return result
3197 class file_producer:
3198 "producer wrapper for file[-like] objects"
3200 # match http_channel's outgoing buffer size
3201 out_buffer_size = 1<<16
3203 def __init__ (self, file):
3204 self.done = 0
3205 self.file = file
3207 def more (self):
3208 if self.done:
3209 return ''
3210 else:
3211 data = self.file.read (self.out_buffer_size)
3212 if not data:
3213 self.file.close()
3214 del self.file
3215 self.done = 1
3216 return ''
3217 else:
3218 return data
3220 # A simple output producer. This one does not [yet] have
3221 # the safety feature builtin to the monitor channel: runaway
3222 # output will not be caught.
3224 # don't try to print from within any of the methods
3225 # of this object.
3227 class output_producer:
3228 "Acts like an output file; suitable for capturing sys.stdout"
3229 def __init__ (self):
3230 self.data = ''
3232 def write (self, data):
3233 lines = string.splitfields (data, '\n')
3234 data = string.join (lines, '\r\n')
3235 self.data = self.data + data
3237 def writeline (self, line):
3238 self.data = self.data + line + '\r\n'
3240 def writelines (self, lines):
3241 self.data = self.data + string.joinfields (
3242 lines,
3243 '\r\n'
3244 ) + '\r\n'
3246 def flush (self):
3247 pass
3249 def softspace (self, *args):
3250 pass
3252 def more (self):
3253 if self.data:
3254 result = self.data[:512]
3255 self.data = self.data[512:]
3256 return result
3257 else:
3258 return ''
3260 class composite_producer:
3261 "combine a fifo of producers into one"
3262 def __init__ (self, producers):
3263 self.producers = producers
3265 def more (self):
3266 while len(self.producers):
3267 p = self.producers[0]
3268 d = p.more()
3269 if d:
3270 return d
3271 else:
3272 self.producers.pop(0)
3273 else:
3274 return ''
3277 class globbing_producer:
3279 'glob' the output from a producer into a particular buffer size.
3280 helps reduce the number of calls to send(). [this appears to
3281 gain about 30% performance on requests to a single channel]
3284 def __init__ (self, producer, buffer_size=1<<16):
3285 self.producer = producer
3286 self.buffer = ''
3287 self.buffer_size = buffer_size
3289 def more (self):
3290 while len(self.buffer) < self.buffer_size:
3291 data = self.producer.more()
3292 if data:
3293 self.buffer = self.buffer + data
3294 else:
3295 break
3296 r = self.buffer
3297 self.buffer = ''
3298 return r
3301 class hooked_producer:
3303 A producer that will call <function> when it empties,.
3304 with an argument of the number of bytes produced. Useful
3305 for logging/instrumentation purposes.
3308 def __init__ (self, producer, function):
3309 self.producer = producer
3310 self.function = function
3311 self.bytes = 0
3313 def more (self):
3314 if self.producer:
3315 result = self.producer.more()
3316 if not result:
3317 self.producer = None
3318 self.function (self.bytes)
3319 else:
3320 self.bytes = self.bytes + len(result)
3321 return result
3322 else:
3323 return ''
3325 # HTTP 1.1 emphasizes that an advertised Content-Length header MUST be
3326 # correct. In the face of Strange Files, it is conceivable that
3327 # reading a 'file' may produce an amount of data not matching that
3328 # reported by os.stat() [text/binary mode issues, perhaps the file is
3329 # being appended to, etc..] This makes the chunked encoding a True
3330 # Blessing, and it really ought to be used even with normal files.
3331 # How beautifully it blends with the concept of the producer.
3333 class chunked_producer:
3334 """A producer that implements the 'chunked' transfer coding for HTTP/1.1.
3335 Here is a sample usage:
3336 request['Transfer-Encoding'] = 'chunked'
3337 request.push (
3338 producers.chunked_producer (your_producer)
3340 request.done()
3343 def __init__ (self, producer, footers=None):
3344 self.producer = producer
3345 self.footers = footers
3347 def more (self):
3348 if self.producer:
3349 data = self.producer.more()
3350 if data:
3351 return '%x\r\n%s\r\n' % (len(data), data)
3352 else:
3353 self.producer = None
3354 if self.footers:
3355 return string.join (
3356 ['0'] + self.footers,
3357 '\r\n'
3358 ) + '\r\n\r\n'
3359 else:
3360 return '0\r\n\r\n'
3361 else:
3362 return ''
3364 class escaping_producer:
3366 "A producer that escapes a sequence of characters"
3367 " Common usage: escaping the CRLF.CRLF sequence in SMTP, NNTP, etc..."
3369 def __init__ (self, producer, esc_from='\r\n.', esc_to='\r\n..'):
3370 self.producer = producer
3371 self.esc_from = esc_from
3372 self.esc_to = esc_to
3373 self.buffer = ''
3374 self.find_prefix_at_end = find_prefix_at_end
3376 def more (self):
3377 esc_from = self.esc_from
3378 esc_to = self.esc_to
3380 buffer = self.buffer + self.producer.more()
3382 if buffer:
3383 buffer = string.replace (buffer, esc_from, esc_to)
3384 i = self.find_prefix_at_end (buffer, esc_from)
3385 if i:
3386 # we found a prefix
3387 self.buffer = buffer[-i:]
3388 return buffer[:-i]
3389 else:
3390 # no prefix, return it all
3391 self.buffer = ''
3392 return buffer
3393 else:
3394 return buffer
3396 class tail_logger:
3397 "Keep track of the last <size> log messages"
3398 def __init__ (self, logger, size=500):
3399 self.size = size
3400 self.logger = logger
3401 self.messages = []
3403 def log (self, message):
3404 self.messages.append (strip_eol (message))
3405 if len (self.messages) > self.size:
3406 del self.messages[0]
3407 self.logger.log (message)
3410 def html_repr (object):
3411 so = escape (repr (object))
3412 if hasattr (object, 'hyper_respond'):
3413 return '<a href="/status/object/%d/">%s</a>' % (id (object), so)
3414 else:
3415 return so
3417 def html_reprs (list, front='', back=''):
3418 reprs = map (
3419 lambda x,f=front,b=back: '%s%s%s' % (f,x,b),
3420 map (lambda x: escape (html_repr(x)), list)
3422 reprs.sort()
3423 return reprs
3425 # for example, tera, giga, mega, kilo
3426 # p_d (n, (1024, 1024, 1024, 1024))
3427 # smallest divider goes first - for example
3428 # minutes, hours, days
3429 # p_d (n, (60, 60, 24))
3431 def progressive_divide (n, parts):
3432 result = []
3433 for part in parts:
3434 n, rem = divmod (n, part)
3435 result.append (rem)
3436 result.append (n)
3437 return result
3439 # b,k,m,g,t
3440 def split_by_units (n, units, dividers, format_string):
3441 divs = progressive_divide (n, dividers)
3442 result = []
3443 for i in range(len(units)):
3444 if divs[i]:
3445 result.append (format_string % (divs[i], units[i]))
3446 result.reverse()
3447 if not result:
3448 return [format_string % (0, units[0])]
3449 else:
3450 return result
3452 def english_bytes (n):
3453 return split_by_units (
3455 ('','K','M','G','T'),
3456 (1024, 1024, 1024, 1024, 1024),
3457 '%d %sB'
3460 def english_time (n):
3461 return split_by_units (
3463 ('secs', 'mins', 'hours', 'days', 'weeks', 'years'),
3464 ( 60, 60, 24, 7, 52),
3465 '%d %s'
3468 class file_logger:
3470 # pass this either a path or a file object.
3471 def __init__ (self, file, flush=1, mode='a'):
3472 if type(file) == type(''):
3473 if (file == '-'):
3474 self.file = sys.stdout
3475 else:
3476 self.file = open (file, mode)
3477 else:
3478 self.file = file
3479 self.do_flush = flush
3481 def __repr__ (self):
3482 return '<file logger: %s>' % self.file
3484 def write (self, data):
3485 self.file.write (data)
3486 self.maybe_flush()
3488 def writeline (self, line):
3489 self.file.writeline (line)
3490 self.maybe_flush()
3492 def writelines (self, lines):
3493 self.file.writelines (lines)
3494 self.maybe_flush()
3496 def maybe_flush (self):
3497 if self.do_flush:
3498 self.file.flush()
3500 def flush (self):
3501 self.file.flush()
3503 def softspace (self, *args):
3504 pass
3506 def log (self, message):
3507 if message[-1] not in ('\r', '\n'):
3508 self.write (message + '\n')
3509 else:
3510 self.write (message)
3512 def debug(self, message):
3513 self.log(message)
3515 class unresolving_logger:
3516 "Just in case you don't want to resolve"
3517 def __init__ (self, logger):
3518 self.logger = logger
3520 def log (self, ip, message):
3521 self.logger.log ('%s:%s' % (ip, message))
3524 def strip_eol (line):
3525 while line and line[-1] in '\r\n':
3526 line = line[:-1]
3527 return line
3529 VERSION_STRING = string.split(RCS_ID)[2]
3530 ATHANA_VERSION = "0.2.1"
3532 # ===========================================================================
3533 # Request Object
3534 # ===========================================================================
3536 class http_request:
3538 # default reply code
3539 reply_code = 200
3541 request_counter = counter()
3543 # Whether to automatically use chunked encoding when
3545 # HTTP version is 1.1
3546 # Content-Length is not set
3547 # Chunked encoding is not already in effect
3549 # If your clients are having trouble, you might want to disable this.
3550 use_chunked = 1
3552 # by default, this request object ignores user data.
3553 collector = None
3555 def __init__ (self, *args):
3556 # unpack information about the request
3557 (self.channel, self.request,
3558 self.command, self.uri, self.version,
3559 self.header) = args
3561 self.outgoing = []
3562 self.reply_headers = {
3563 'Server' : 'Athana/%s' % ATHANA_VERSION,
3564 'Date' : build_http_date (time.time()),
3565 'Expires' : build_http_date (time.time())
3567 self.request_number = http_request.request_counter.increment()
3568 self._split_uri = None
3569 self._header_cache = {}
3571 # --------------------------------------------------
3572 # reply header management
3573 # --------------------------------------------------
3574 def __setitem__ (self, key, value):
3575 try:
3576 if key=='Set-Cookie':
3577 self.reply_headers[key] += [value]
3578 else:
3579 self.reply_headers[key] = [value]
3580 except:
3581 self.reply_headers[key] = [value]
3583 def __getitem__ (self, key):
3584 return self.reply_headers[key][0]
3586 def has_key (self, key):
3587 return self.reply_headers.has_key(key)
3589 def build_reply_header (self):
3590 h = []
3591 for k,vv in self.reply_headers.items():
3592 if type(vv) != type([]):
3593 h += ["%s: %s" % (k,vv)]
3594 else:
3595 for v in vv:
3596 h += ["%s: %s" % (k,v)]
3597 return string.join([self.response(self.reply_code)] + h, '\r\n') + '\r\n\r\n'
3599 # --------------------------------------------------
3600 # split a uri
3601 # --------------------------------------------------
3603 # <path>;<params>?<query>#<fragment>
3604 path_regex = re.compile (
3605 # path params query fragment
3606 r'([^;?#]*)(;[^?#]*)?(\?[^#]*)?(#.*)?'
3609 def split_uri (self):
3610 if self._split_uri is None:
3611 m = self.path_regex.match (self.uri)
3612 if m.end() != len(self.uri):
3613 raise ValueError, "Broken URI"
3614 else:
3615 self._split_uri = m.groups()
3616 return self._split_uri
3618 def get_header_with_regex (self, head_reg, group):
3619 for line in self.header:
3620 m = head_reg.match (line)
3621 if m.end() == len(line):
3622 return m.group (group)
3623 return ''
3625 def get_header (self, header):
3626 header = string.lower (header)
3627 hc = self._header_cache
3628 if not hc.has_key (header):
3629 h = header + ': '
3630 hl = len(h)
3631 for line in self.header:
3632 if string.lower (line[:hl]) == h:
3633 r = line[hl:]
3634 hc[header] = r
3635 return r
3636 hc[header] = None
3637 return None
3638 else:
3639 return hc[header]
3641 # --------------------------------------------------
3642 # user data
3643 # --------------------------------------------------
3645 def collect_incoming_data (self, data):
3646 if self.collector:
3647 self.collector.collect_incoming_data (data)
3648 else:
3649 self.log_info(
3650 'Dropping %d bytes of incoming request data' % len(data),
3651 'warning'
3654 def found_terminator (self):
3655 if self.collector:
3656 self.collector.found_terminator()
3657 else:
3658 self.log_info (
3659 'Unexpected end-of-record for incoming request',
3660 'warning'
3663 def push (self, thing):
3664 if type(thing) == type(''):
3665 self.outgoing.append(simple_producer (thing))
3666 else:
3667 thing.more
3668 self.outgoing.append(thing)
3670 def response (self, code=200):
3671 message = self.responses[code]
3672 self.reply_code = code
3673 return 'HTTP/%s %d %s' % (self.version, code, message)
3675 def error (self, code, s=None):
3676 self.reply_code = code
3677 self.outgoing = []
3678 message = self.responses[code]
3679 if s is None:
3680 s = self.DEFAULT_ERROR_MESSAGE % {
3681 'code': code,
3682 'message': message,
3684 self['Content-Length'] = len(s)
3685 self['Content-Type'] = 'text/html'
3686 # make an error reply
3687 self.push (s)
3688 self.done()
3690 # can also be used for empty replies
3691 reply_now = error
3693 def done (self):
3694 "finalize this transaction - send output to the http channel"
3696 if hasattr(self,"tempfiles"):
3697 for f in self.tempfiles:
3698 os.unlink(f)
3700 # ----------------------------------------
3701 # persistent connection management
3702 # ----------------------------------------
3704 # --- BUCKLE UP! ----
3706 connection = string.lower (get_header (CONNECTION, self.header))
3708 close_it = 0
3709 wrap_in_chunking = 0
3711 if self.version == '1.0':
3712 if connection == 'keep-alive':
3713 if not self.has_key ('Content-Length'):
3714 close_it = 1
3715 else:
3716 self['Connection'] = 'Keep-Alive'
3717 else:
3718 close_it = 1
3719 elif self.version == '1.1':
3720 if connection == 'close':
3721 close_it = 1
3722 elif not self.has_key ('Content-Length'):
3723 if self.has_key ('Transfer-Encoding'):
3724 if not self['Transfer-Encoding'] == 'chunked':
3725 close_it = 1
3726 elif self.use_chunked:
3727 self['Transfer-Encoding'] = 'chunked'
3728 wrap_in_chunking = 1
3729 else:
3730 close_it = 1
3731 elif self.version is None:
3732 # Although we don't *really* support http/0.9 (because we'd have to
3733 # use \r\n as a terminator, and it would just yuck up a lot of stuff)
3734 # it's very common for developers to not want to type a version number
3735 # when using telnet to debug a server.
3736 close_it = 1
3738 outgoing_header = simple_producer (self.build_reply_header())
3740 if close_it:
3741 self['Connection'] = 'close'
3743 if wrap_in_chunking:
3744 outgoing_producer = chunked_producer (
3745 composite_producer (list(self.outgoing))
3747 # prepend the header
3748 outgoing_producer = composite_producer(
3749 [outgoing_header, outgoing_producer]
3751 else:
3752 # prepend the header
3753 self.outgoing.insert(0, outgoing_header)
3754 outgoing_producer = composite_producer (list(self.outgoing))
3756 # actually, this is already set to None by the handler:
3757 self.channel.current_request = None
3759 # apply a few final transformations to the output
3760 self.channel.push_with_producer (
3761 # globbing gives us large packets
3762 globbing_producer (
3763 outgoing_producer
3767 if close_it:
3768 self.channel.close_when_done()
3770 def log_date_string (self, when):
3771 t = time.localtime(when)
3772 return time.strftime ( '%d/%b/%Y:%H:%M:%S ', t)
3774 def log (self):
3775 self.channel.server.logger.log (
3776 self.channel.addr[0],
3777 '%d - - [%s] "%s"\n' % (
3778 self.channel.addr[1],
3779 self.log_date_string (time.time()),
3780 self.request,
3784 def write(self,text):
3785 if type(text) == type(''):
3786 self.push(text)
3787 elif type(text) == type(u''):
3788 self.push(text.encode("utf-8"))
3789 else:
3790 text.more
3791 self.push(text)
3793 def setStatus(self,status):
3794 self.reply_code = status
3796 def makeLink(self,page,params=None):
3797 query = ""
3798 if params is not None:
3799 first = 1
3800 for k,v in params.items():
3801 if first:
3802 query += "?"
3803 else:
3804 query += "&"
3805 query += urllib.quote(k)+"="+urllib.quote(v)
3806 first = 0
3807 return page+";"+self.sessionid+query
3809 def sendFile(self,path,content_type,force=0):
3811 try:
3812 file_length = os.stat(path)[stat.ST_SIZE]
3813 except OSError:
3814 self.error (404)
3815 return
3817 ims = get_header_match (IF_MODIFIED_SINCE, self.header)
3818 length_match = 1
3819 if ims:
3820 length = ims.group (4)
3821 if length:
3822 try:
3823 length = string.atoi (length)
3824 if length != file_length:
3825 length_match = 0
3826 except:
3827 pass
3828 ims_date = 0
3829 if ims:
3830 ims_date = parse_http_date (ims.group (1))
3832 try:
3833 mtime = os.stat (path)[stat.ST_MTIME]
3834 except:
3835 self.error (404)
3836 return
3837 if length_match and ims_date:
3838 if mtime <= ims_date and not force:
3839 print "File "+path+" was not modified since "+str(ims_date)+" (current filedate is "+str(mtime)+")-> 304"
3840 self.reply_code = 304
3841 return
3842 try:
3843 file = open (path, 'rb')
3844 except IOError:
3845 self.error (404)
3846 print "404"
3847 return
3849 self.reply_headers['Last-Modified'] = build_http_date (mtime)
3850 self.reply_headers['Content-Length'] = file_length
3851 self.reply_headers['Content-Type'] = content_type
3852 self.reply_headers['Connection'] = 'close';
3853 if self.command == 'GET':
3854 self.push(file_producer(file))
3855 return
3857 def setCookie(self, name, value, expire=None):
3858 if expire is None:
3859 s = name+'='+value;
3860 else:
3861 datestr = time.strftime("%a, %d-%b-%Y %H:%M:%S GMT", time.gmtime(expire))
3862 s = name+'='+value+'; expires='+datestr; #+'; path=PATH; domain=DOMAIN_NAME; secure';
3864 if 'Set-Cookie' not in self.reply_headers:
3865 self.reply_headers['Set-Cookie'] = [s]
3866 else:
3867 self.reply_headers['Set-Cookie'] += [s]
3869 def makeSelfLink(self,params):
3870 params2 = self.params.copy()
3871 for k,v in params.items():
3872 if v is not None:
3873 params2[k] = v
3874 else:
3875 try: del params2[k]
3876 except: pass
3877 ret = self.makeLink(self.fullpath, params2)
3878 return ret
3880 def writeTAL(self,page,context,macro=None):
3881 runTAL(self, context, file=page, macro=macro, request=self)
3883 def writeTALstr(self,string,context,macro=None):
3884 runTAL(self, context, string=string, macro=macro, request=self)
3886 def getTAL(self,page,context,macro=None):
3887 return processTAL(context,file=page, macro=macro, request=self)
3889 def getTALstr(self,string,context,macro=None):
3890 return processTAL(context,string=string, macro=macro, request=self)
3893 responses = {
3894 100: "Continue",
3895 101: "Switching Protocols",
3896 200: "OK",
3897 201: "Created",
3898 202: "Accepted",
3899 203: "Non-Authoritative Information",
3900 204: "No Content",
3901 205: "Reset Content",
3902 206: "Partial Content",
3903 300: "Multiple Choices",
3904 301: "Moved Permanently",
3905 302: "Moved Temporarily",
3906 303: "See Other",
3907 304: "Not Modified",
3908 305: "Use Proxy",
3909 400: "Bad Request",
3910 401: "Unauthorized",
3911 402: "Payment Required",
3912 403: "Forbidden",
3913 404: "Not Found",
3914 405: "Method Not Allowed",
3915 406: "Not Acceptable",
3916 407: "Proxy Authentication Required",
3917 408: "Request Time-out",
3918 409: "Conflict",
3919 410: "Gone",
3920 411: "Length Required",
3921 412: "Precondition Failed",
3922 413: "Request Entity Too Large",
3923 414: "Request-URI Too Large",
3924 415: "Unsupported Media Type",
3925 500: "Internal Server Error",
3926 501: "Not Implemented",
3927 502: "Bad Gateway",
3928 503: "Service Unavailable",
3929 504: "Gateway Time-out",
3930 505: "HTTP Version not supported"
3933 # Default error message
3934 DEFAULT_ERROR_MESSAGE = string.join (
3935 ['<html><head>',
3936 '<title>Error response</title>',
3937 '</head>',
3938 '<body>',
3939 '<h1>Error response</h1>',
3940 '<p>Error code %(code)d.</p>',
3941 '<p>Message: %(message)s.</p>',
3942 '</body></html>',
3945 '\r\n'
3948 def getTAL(page,context,macro=None,language=None):
3949 return processTAL(context,file=page, macro=macro, language=language)
3951 def getTALstr(string,context,macro=None,language=None):
3952 return processTAL(context,string=string, macro=macro, language=language)
3954 # ===========================================================================
3955 # HTTP Channel Object
3956 # ===========================================================================
3958 class http_channel (async_chat):
3960 # use a larger default output buffer
3961 ac_out_buffer_size = 1<<16
3963 current_request = None
3964 channel_counter = counter()
3966 def __init__ (self, server, conn, addr):
3967 self.channel_number = http_channel.channel_counter.increment()
3968 self.request_counter = counter()
3969 async_chat.__init__ (self, conn)
3970 self.server = server
3971 self.addr = addr
3972 self.set_terminator ('\r\n\r\n')
3973 self.in_buffer = ''
3974 self.creation_time = int (time.time())
3975 self.check_maintenance()
3976 self.producer_lock = thread.allocate_lock()
3978 def initiate_send (self):
3979 self.producer_lock.acquire()
3980 try:
3981 async_chat.initiate_send(self)
3982 finally:
3983 self.producer_lock.release()
3985 def push (self, data):
3986 data.more
3987 self.producer_lock.acquire()
3988 try:
3989 self.producer_fifo.push (simple_producer (data))
3990 finally:
3991 self.producer_lock.release()
3992 self.initiate_send()
3994 def push_with_producer (self, producer):
3995 self.producer_lock.acquire()
3996 try:
3997 self.producer_fifo.push (producer)
3998 finally:
3999 self.producer_lock.release()
4000 self.initiate_send()
4002 def close_when_done (self):
4003 self.producer_lock.acquire()
4004 try:
4005 self.producer_fifo.push (None)
4006 finally:
4007 self.producer_lock.release()
4009 #results in select.error: (9, 'Bad file descriptor') if the socket map is poll'ed
4010 #while this socket is being closed
4011 #we do it anyway, and catch the select.error in the main loop
4013 #XXX on Ubuntu's 2.6.10-5-386, the socket won't be closed until the select finishes (or
4014 #times out). We probably need to send a SIGINT signal or something. For now, we just
4015 #set a very small timeout (0.01) in the main loop, so that select() will be called often
4016 #enough.
4018 #it also results in a "NoneType has no attribute more" error if refill_buffer tries
4019 #to run data = p.more() on the None terminator (which we catch)
4020 try:
4021 self.initiate_send()
4022 except AttributeError:
4023 pass
4025 def __repr__ (self):
4026 ar = async_chat.__repr__(self)[1:-1]
4027 return '<%s channel#: %s requests:%s>' % (
4029 self.channel_number,
4030 self.request_counter
4033 # Channel Counter, Maintenance Interval...
4034 maintenance_interval = 500
4036 def check_maintenance (self):
4037 if not self.channel_number % self.maintenance_interval:
4038 self.maintenance()
4040 def maintenance (self):
4041 self.kill_zombies()
4043 # 30-minute zombie timeout. status_handler also knows how to kill zombies.
4044 zombie_timeout = 30 * 60
4046 def kill_zombies (self):
4047 now = int (time.time())
4048 for channel in asyncore.socket_map.values():
4049 if channel.__class__ == self.__class__:
4050 if (now - channel.creation_time) > channel.zombie_timeout:
4051 channel.close()
4053 # --------------------------------------------------
4054 # send/recv overrides, good place for instrumentation.
4055 # --------------------------------------------------
4057 # this information needs to get into the request object,
4058 # so that it may log correctly.
4059 def send (self, data):
4060 result = async_chat.send (self, data)
4061 self.server.bytes_out.increment (len(data))
4062 return result
4064 def recv (self, buffer_size):
4065 try:
4066 result = async_chat.recv (self, buffer_size)
4067 self.server.bytes_in.increment (len(result))
4068 return result
4069 except MemoryError:
4070 # --- Save a Trip to Your Service Provider ---
4071 # It's possible for a process to eat up all the memory of
4072 # the machine, and put it in an extremely wedged state,
4073 # where medusa keeps running and can't be shut down. This
4074 # is where MemoryError tends to get thrown, though of
4075 # course it could get thrown elsewhere.
4076 sys.exit ("Out of Memory!")
4078 def handle_error (self):
4079 t, v = sys.exc_info()[:2]
4080 if t is SystemExit:
4081 raise t, v
4082 else:
4083 async_chat.handle_error (self)
4085 def log (self, *args):
4086 pass
4088 # --------------------------------------------------
4089 # async_chat methods
4090 # --------------------------------------------------
4092 def collect_incoming_data (self, data):
4093 if self.current_request:
4094 # we are receiving data (probably POST data) for a request
4095 self.current_request.collect_incoming_data (data)
4096 else:
4097 # we are receiving header (request) data
4098 self.in_buffer = self.in_buffer + data
4100 def found_terminator (self):
4101 if self.current_request:
4102 self.current_request.found_terminator()
4103 else:
4104 header = self.in_buffer
4105 self.in_buffer = ''
4106 lines = string.split (header, '\r\n')
4108 # --------------------------------------------------
4109 # crack the request header
4110 # --------------------------------------------------
4112 while lines and not lines[0]:
4113 # as per the suggestion of http-1.1 section 4.1, (and
4114 # Eric Parker <eparker@zyvex.com>), ignore a leading
4115 # blank lines (buggy browsers tack it onto the end of
4116 # POST requests)
4117 lines = lines[1:]
4119 if not lines:
4120 self.close_when_done()
4121 return
4123 request = lines[0]
4125 command, uri, version = crack_request (request)
4126 header = join_headers (lines[1:])
4128 # unquote path if necessary (thanks to Skip Montanaro for pointing
4129 # out that we must unquote in piecemeal fashion).
4130 rpath, rquery = splitquery(uri)
4131 if '%' in rpath:
4132 if rquery:
4133 uri = unquote (rpath) + '?' + rquery
4134 else:
4135 uri = unquote (rpath)
4137 r = http_request (self, request, command, uri, version, header)
4138 self.request_counter.increment()
4139 self.server.total_requests.increment()
4141 if command is None:
4142 self.log_info ('Bad HTTP request: %s' % repr(request), 'error')
4143 r.error (400)
4144 return
4146 # --------------------------------------------------
4147 # handler selection and dispatch
4148 # --------------------------------------------------
4149 for h in self.server.handlers:
4150 if h.match (r):
4151 try:
4152 self.current_request = r
4153 # This isn't used anywhere.
4154 # r.handler = h # CYCLE
4155 h.handle_request (r)
4156 except:
4157 self.server.exceptions.increment()
4158 (file, fun, line), t, v, tbinfo = asyncore.compact_traceback()
4159 self.log_info(
4160 'Server Error: %s, %s: file: %s line: %s' % (t,v,file,line),
4161 'error')
4162 try:
4163 r.error (500)
4164 except:
4165 pass
4166 return
4168 # no handlers, so complain
4169 r.error (404)
4171 # ===========================================================================
4172 # HTTP Server Object
4173 # ===========================================================================
4175 class http_server (asyncore.dispatcher):
4177 SERVER_IDENT = 'HTTP Server (V%s)' % VERSION_STRING
4179 channel_class = http_channel
4181 def __init__ (self, ip, port, resolver=None, logger_object=None):
4182 self.ip = ip
4183 self.port = port
4184 asyncore.dispatcher.__init__ (self)
4185 self.create_socket (socket.AF_INET, socket.SOCK_STREAM)
4187 self.handlers = []
4189 if not logger_object:
4190 logger_object = file_logger (sys.stdout)
4192 self.set_reuse_addr()
4193 self.bind ((ip, port))
4195 # lower this to 5 if your OS complains
4196 self.listen (1024)
4198 host, port = self.socket.getsockname()
4199 if not ip:
4200 self.log_info('Computing default hostname', 'warning')
4201 ip = socket.gethostbyname (socket.gethostname())
4202 try:
4203 self.server_name = socket.gethostbyaddr (ip)[0]
4204 except socket.error:
4205 self.log_info('Cannot do reverse lookup', 'warning')
4206 self.server_name = ip # use the IP address as the "hostname"
4208 self.server_port = port
4209 self.total_clients = counter()
4210 self.total_requests = counter()
4211 self.exceptions = counter()
4212 self.bytes_out = counter()
4213 self.bytes_in = counter()
4215 if not logger_object:
4216 logger_object = file_logger (sys.stdout)
4218 self.logger = unresolving_logger (logger_object)
4220 self.log_info (
4221 'Athana (%s) started at %s'
4222 '\n\n'
4223 'The server is running! You can now direct your browser to:\n'
4224 '\thttp://%s:%d/'
4225 '\n' % (
4226 ATHANA_VERSION,
4227 time.ctime(time.time()),
4228 self.server_name,
4229 port,
4233 def writable (self):
4234 return 0
4236 def handle_read (self):
4237 pass
4239 def readable (self):
4240 return self.accepting
4242 def handle_connect (self):
4243 pass
4245 def handle_accept (self):
4246 self.total_clients.increment()
4247 try:
4248 conn, addr = self.accept()
4249 except socket.error:
4250 # linux: on rare occasions we get a bogus socket back from
4251 # accept. socketmodule.c:makesockaddr complains that the
4252 # address family is unknown. We don't want the whole server
4253 # to shut down because of this.
4254 self.log_info ('warning: server accept() threw an exception', 'warning')
4255 return
4256 except TypeError:
4257 # unpack non-sequence. this can happen when a read event
4258 # fires on a listening socket, but when we call accept()
4259 # we get EWOULDBLOCK, so dispatcher.accept() returns None.
4260 # Seen on FreeBSD3.
4261 self.log_info ('warning: server accept() threw EWOULDBLOCK', 'warning')
4262 return
4264 self.channel_class (self, conn, addr)
4266 def install_handler (self, handler, back=0):
4267 if back:
4268 self.handlers.append (handler)
4269 else:
4270 self.handlers.insert (0, handler)
4272 def remove_handler (self, handler):
4273 self.handlers.remove (handler)
4276 CONNECTION = re.compile ('Connection: (.*)', re.IGNORECASE)
4278 # merge multi-line headers
4279 # [486dx2: ~500/sec]
4280 def join_headers (headers):
4281 r = []
4282 for i in range(len(headers)):
4283 if headers[i][0] in ' \t':
4284 r[-1] = r[-1] + headers[i][1:]
4285 else:
4286 r.append (headers[i])
4287 return r
4289 def get_header (head_reg, lines, group=1):
4290 for line in lines:
4291 m = head_reg.match (line)
4292 if m and m.end() == len(line):
4293 return m.group (group)
4294 return ''
4296 def get_header_match (head_reg, lines):
4297 for line in lines:
4298 m = head_reg.match (line)
4299 if m and m.end() == len(line):
4300 return m
4301 return ''
4303 REQUEST = re.compile ('([^ ]+) ([^ ]+)(( HTTP/([0-9.]+))$|$)')
4305 def crack_request (r):
4306 m = REQUEST.match (r)
4307 if m and m.end() == len(r):
4308 if m.group(3):
4309 version = m.group(5)
4310 else:
4311 version = None
4312 return m.group(1), m.group(2), version
4313 else:
4314 return None, None, None
4317 # This is the 'default' handler. it implements the base set of
4318 # features expected of a simple file-delivering HTTP server. file
4319 # services are provided through a 'filesystem' object, the very same
4320 # one used by the FTP server.
4322 # You can replace or modify this handler if you want a non-standard
4323 # HTTP server. You can also derive your own handler classes from
4324 # it.
4326 # support for handling POST requests is available in the derived
4327 # class <default_with_post_handler>, defined below.
4330 class default_handler:
4332 valid_commands = ['GET', 'HEAD']
4334 IDENT = 'Default HTTP Request Handler'
4336 # Pathnames that are tried when a URI resolves to a directory name
4337 directory_defaults = [
4338 'index.html',
4339 'default.html'
4342 default_file_producer = file_producer
4344 def __init__ (self, filesystem):
4345 self.filesystem = filesystem
4346 # count total hits
4347 self.hit_counter = counter()
4348 # count file deliveries
4349 self.file_counter = counter()
4350 # count cache hits
4351 self.cache_counter = counter()
4353 hit_counter = 0
4355 def __repr__ (self):
4356 return '<%s (%s hits) at %x>' % (
4357 self.IDENT,
4358 self.hit_counter,
4359 id (self)
4362 # always match, since this is a default
4363 def match (self, request):
4364 return 1
4366 def can_handle(self, request):
4367 path, params, query, fragment = request.split_uri()
4368 if '%' in path:
4369 path = unquote (path)
4370 while path and path[0] == '/':
4371 path = path[1:]
4372 if self.filesystem.isdir (path):
4373 if path and path[-1] != '/':
4374 return 0
4375 found = 0
4376 if path and path[-1] != '/':
4377 path = path + '/'
4378 for default in self.directory_defaults:
4379 p = path + default
4380 if self.filesystem.isfile (p):
4381 path = p
4382 found = 1
4383 break
4384 if not found:
4385 return 0
4386 elif not self.filesystem.isfile (path):
4387 return 0
4388 return 1
4390 # handle a file request, with caching.
4392 def handle_request (self, request):
4394 if request.command not in self.valid_commands:
4395 request.error (400) # bad request
4396 return
4398 self.hit_counter.increment()
4400 path, params, query, fragment = request.split_uri()
4402 if '%' in path:
4403 path = unquote (path)
4405 # strip off all leading slashes
4406 while path and path[0] == '/':
4407 path = path[1:]
4409 if self.filesystem.isdir (path):
4410 if path and path[-1] != '/':
4411 request['Location'] = 'http://%s/%s/' % (
4412 request.channel.server.server_name,
4413 path
4415 request.error (301)
4416 return
4418 # we could also generate a directory listing here,
4419 # may want to move this into another method for that
4420 # purpose
4421 found = 0
4422 if path and path[-1] != '/':
4423 path = path + '/'
4424 for default in self.directory_defaults:
4425 p = path + default
4426 if self.filesystem.isfile (p):
4427 path = p
4428 found = 1
4429 break
4430 if not found:
4431 request.error (404) # Not Found
4432 return
4434 elif not self.filesystem.isfile (path):
4435 request.error (404) # Not Found
4436 return
4438 file_length = self.filesystem.stat (path)[stat.ST_SIZE]
4440 ims = get_header_match (IF_MODIFIED_SINCE, request.header)
4442 length_match = 1
4443 if ims:
4444 length = ims.group (4)
4445 if length:
4446 try:
4447 length = string.atoi (length)
4448 if length != file_length:
4449 length_match = 0
4450 except:
4451 pass
4453 ims_date = 0
4455 if ims:
4456 ims_date = parse_http_date (ims.group (1))
4458 try:
4459 mtime = self.filesystem.stat (path)[stat.ST_MTIME]
4460 except:
4461 request.error (404)
4462 return
4464 if length_match and ims_date:
4465 if mtime <= ims_date:
4466 request.reply_code = 304
4467 request.done()
4468 self.cache_counter.increment()
4469 print "File "+path+" was not modified since "+str(ims_date)+" (current filedate is "+str(mtime)+")"
4470 return
4471 try:
4472 file = self.filesystem.open (path, 'rb')
4473 except IOError:
4474 request.error (404)
4475 return
4477 request['Last-Modified'] = build_http_date (mtime)
4478 request['Content-Length'] = file_length
4479 self.set_content_type (path, request)
4481 if request.command == 'GET':
4482 request.push (self.default_file_producer (file))
4484 self.file_counter.increment()
4485 request.done()
4487 def set_content_type (self, path, request):
4488 ext = string.lower (get_extension (path))
4489 typ, encoding = mimetypes.guess_type(path)
4490 if typ is not None:
4491 request['Content-Type'] = typ
4492 else:
4493 # TODO: test a chunk off the front of the file for 8-bit
4494 # characters, and use application/octet-stream instead.
4495 request['Content-Type'] = 'text/plain'
4497 def status (self):
4498 return simple_producer (
4499 '<li>%s' % html_repr (self)
4500 + '<ul>'
4501 + ' <li><b>Total Hits:</b> %s' % self.hit_counter
4502 + ' <li><b>Files Delivered:</b> %s' % self.file_counter
4503 + ' <li><b>Cache Hits:</b> %s' % self.cache_counter
4504 + '</ul>'
4507 # HTTP/1.0 doesn't say anything about the "; length=nnnn" addition
4508 # to this header. I suppose its purpose is to avoid the overhead
4509 # of parsing dates...
4510 IF_MODIFIED_SINCE = re.compile (
4511 'If-Modified-Since: ([^;]+)((; length=([0-9]+)$)|$)',
4512 re.IGNORECASE
4515 USER_AGENT = re.compile ('User-Agent: (.*)', re.IGNORECASE)
4517 CONTENT_TYPE = re.compile (
4518 r'Content-Type: ([^;]+)((; boundary=([A-Za-z0-9\'\(\)+_,./:=?-]+)$)|$)',
4519 re.IGNORECASE
4522 get_header = get_header
4523 get_header_match = get_header_match
4525 def get_extension (path):
4526 dirsep = string.rfind (path, '/')
4527 dotsep = string.rfind (path, '.')
4528 if dotsep > dirsep:
4529 return path[dotsep+1:]
4530 else:
4531 return ''
4533 class abstract_filesystem:
4534 def __init__ (self):
4535 pass
4537 def current_directory (self):
4538 "Return a string representing the current directory."
4539 pass
4541 def listdir (self, path, long=0):
4542 """Return a listing of the directory at 'path' The empty string
4543 indicates the current directory. If 'long' is set, instead
4544 return a list of (name, stat_info) tuples
4546 pass
4548 def open (self, path, mode):
4549 "Return an open file object"
4550 pass
4552 def stat (self, path):
4553 "Return the equivalent of os.stat() on the given path."
4554 pass
4556 def isdir (self, path):
4557 "Does the path represent a directory?"
4558 pass
4560 def isfile (self, path):
4561 "Does the path represent a plain file?"
4562 pass
4564 def cwd (self, path):
4565 "Change the working directory."
4566 pass
4568 def cdup (self):
4569 "Change to the parent of the current directory."
4570 pass
4573 def longify (self, path):
4574 """Return a 'long' representation of the filename
4575 [for the output of the LIST command]"""
4576 pass
4578 # standard wrapper around a unix-like filesystem, with a 'false root'
4579 # capability.
4581 # security considerations: can symbolic links be used to 'escape' the
4582 # root? should we allow it? if not, then we could scan the
4583 # filesystem on startup, but that would not help if they were added
4584 # later. We will probably need to check for symlinks in the cwd method.
4586 # what to do if wd is an invalid directory?
4588 def safe_stat (path):
4589 try:
4590 return (path, os.stat (path))
4591 except:
4592 return None
4594 class os_filesystem:
4595 path_module = os.path
4597 # set this to zero if you want to disable pathname globbing.
4598 # [we currently don't glob, anyway]
4599 do_globbing = 1
4601 def __init__ (self, root, wd='/'):
4602 self.root = root
4603 self.wd = wd
4605 def current_directory (self):
4606 return self.wd
4608 def isfile (self, path):
4609 p = self.normalize (self.path_module.join (self.wd, path))
4610 return self.path_module.isfile (self.translate(p))
4612 def isdir (self, path):
4613 p = self.normalize (self.path_module.join (self.wd, path))
4614 return self.path_module.isdir (self.translate(p))
4616 def cwd (self, path):
4617 p = self.normalize (self.path_module.join (self.wd, path))
4618 translated_path = self.translate(p)
4619 if not self.path_module.isdir (translated_path):
4620 return 0
4621 else:
4622 old_dir = os.getcwd()
4623 # temporarily change to that directory, in order
4624 # to see if we have permission to do so.
4625 try:
4626 can = 0
4627 try:
4628 os.chdir (translated_path)
4629 can = 1
4630 self.wd = p
4631 except:
4632 pass
4633 finally:
4634 if can:
4635 os.chdir (old_dir)
4636 return can
4638 def cdup (self):
4639 return self.cwd ('..')
4641 def listdir (self, path, long=0):
4642 p = self.translate (path)
4643 # I think we should glob, but limit it to the current
4644 # directory only.
4645 ld = os.listdir (p)
4646 if not long:
4647 return list_producer (ld, None)
4648 else:
4649 old_dir = os.getcwd()
4650 try:
4651 os.chdir (p)
4652 # if os.stat fails we ignore that file.
4653 result = filter (None, map (safe_stat, ld))
4654 finally:
4655 os.chdir (old_dir)
4656 return list_producer (result, self.longify)
4658 # TODO: implement a cache w/timeout for stat()
4659 def stat (self, path):
4660 p = self.translate (path)
4661 return os.stat (p)
4663 def open (self, path, mode):
4664 p = self.translate (path)
4665 return open (p, mode)
4667 def unlink (self, path):
4668 p = self.translate (path)
4669 return os.unlink (p)
4671 def mkdir (self, path):
4672 p = self.translate (path)
4673 return os.mkdir (p)
4675 def rmdir (self, path):
4676 p = self.translate (path)
4677 return os.rmdir (p)
4679 # utility methods
4680 def normalize (self, path):
4681 # watch for the ever-sneaky '/+' path element
4682 path = re.sub('/+', '/', path)
4683 p = self.path_module.normpath (path)
4684 # remove 'dangling' cdup's.
4685 if len(p) > 2 and p[:3] == '/..':
4686 p = '/'
4687 return p
4689 def translate (self, path):
4690 # we need to join together three separate
4691 # path components, and do it safely.
4692 # <real_root>/<current_directory>/<path>
4693 # use the operating system's path separator.
4694 path = string.join (string.split (path, '/'), os.sep)
4695 p = self.normalize (self.path_module.join (self.wd, path))
4696 p = self.normalize (self.path_module.join (self.root, p[1:]))
4697 return p
4699 def longify (self, (path, stat_info)):
4700 return unix_longify (path, stat_info)
4702 def __repr__ (self):
4703 return '<unix-style fs root:%s wd:%s>' % (
4704 self.root,
4705 self.wd
4708 # this matches the output of NT's ftp server (when in
4709 # MSDOS mode) exactly.
4711 def msdos_longify (file, stat_info):
4712 if stat.S_ISDIR (stat_info[stat.ST_MODE]):
4713 dir = '<DIR>'
4714 else:
4715 dir = ' '
4716 date = msdos_date (stat_info[stat.ST_MTIME])
4717 return '%s %s %8d %s' % (
4718 date,
4719 dir,
4720 stat_info[stat.ST_SIZE],
4721 file
4724 def msdos_date (t):
4725 try:
4726 info = time.gmtime (t)
4727 except:
4728 info = time.gmtime (0)
4729 # year, month, day, hour, minute, second, ...
4730 if info[3] > 11:
4731 merid = 'PM'
4732 info[3] = info[3] - 12
4733 else:
4734 merid = 'AM'
4735 return '%02d-%02d-%02d %02d:%02d%s' % (
4736 info[1],
4737 info[2],
4738 info[0]%100,
4739 info[3],
4740 info[4],
4741 merid
4744 months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun',
4745 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']
4747 mode_table = {
4748 '0':'---',
4749 '1':'--x',
4750 '2':'-w-',
4751 '3':'-wx',
4752 '4':'r--',
4753 '5':'r-x',
4754 '6':'rw-',
4755 '7':'rwx'
4758 def unix_longify (file, stat_info):
4759 # for now, only pay attention to the lower bits
4760 mode = ('%o' % stat_info[stat.ST_MODE])[-3:]
4761 mode = string.join (map (lambda x: mode_table[x], mode), '')
4762 if stat.S_ISDIR (stat_info[stat.ST_MODE]):
4763 dirchar = 'd'
4764 else:
4765 dirchar = '-'
4766 date = ls_date (long(time.time()), stat_info[stat.ST_MTIME])
4767 return '%s%s %3d %-8d %-8d %8d %s %s' % (
4768 dirchar,
4769 mode,
4770 stat_info[stat.ST_NLINK],
4771 stat_info[stat.ST_UID],
4772 stat_info[stat.ST_GID],
4773 stat_info[stat.ST_SIZE],
4774 date,
4775 file
4778 # Emulate the unix 'ls' command's date field.
4779 # it has two formats - if the date is more than 180
4780 # days in the past, then it's like this:
4781 # Oct 19 1995
4782 # otherwise, it looks like this:
4783 # Oct 19 17:33
4785 def ls_date (now, t):
4786 try:
4787 info = time.gmtime (t)
4788 except:
4789 info = time.gmtime (0)
4790 # 15,600,000 == 86,400 * 180
4791 if (now - t) > 15600000:
4792 return '%s %2d %d' % (
4793 months[info[1]-1],
4794 info[2],
4795 info[0]
4797 else:
4798 return '%s %2d %02d:%02d' % (
4799 months[info[1]-1],
4800 info[2],
4801 info[3],
4802 info[4]
4805 class list_producer:
4806 def __init__ (self, list, func=None):
4807 self.list = list
4808 self.func = func
4810 # this should do a pushd/popd
4811 def more (self):
4812 if not self.list:
4813 return ''
4814 else:
4815 # do a few at a time
4816 bunch = self.list[:50]
4817 if self.func is not None:
4818 bunch = map (self.func, bunch)
4819 self.list = self.list[50:]
4820 return string.joinfields (bunch, '\r\n') + '\r\n'
4822 class hooked_callback:
4823 def __init__ (self, hook, callback):
4824 self.hook, self.callback = hook, callback
4826 def __call__ (self, *args):
4827 apply (self.hook, args)
4828 apply (self.callback, args)
4830 # An extensible, configurable, asynchronous FTP server.
4832 # All socket I/O is non-blocking, however file I/O is currently
4833 # blocking. Eventually file I/O may be made non-blocking, too, if it
4834 # seems necessary. Currently the only CPU-intensive operation is
4835 # getting and formatting a directory listing. [this could be moved
4836 # into another process/directory server, or another thread?]
4838 # Only a subset of RFC 959 is implemented, but much of that RFC is
4839 # vestigial anyway. I've attempted to include the most commonly-used
4840 # commands, using the feature set of wu-ftpd as a guide.
4843 # TODO: implement a directory listing cache. On very-high-load
4844 # servers this could save a lot of disk abuse, and possibly the
4845 # work of computing emulated unix ls output.
4847 # Potential security problem with the FTP protocol? I don't think
4848 # there's any verification of the origin of a data connection. Not
4849 # really a problem for the server (since it doesn't send the port
4850 # command, except when in PASV mode) But I think a data connection
4851 # could be spoofed by a program with access to a sniffer - it could
4852 # watch for a PORT command to go over a command channel, and then
4853 # connect to that port before the server does.
4855 # Unix user id's:
4856 # In order to support assuming the id of a particular user,
4857 # it seems there are two options:
4858 # 1) fork, and seteuid in the child
4859 # 2) carefully control the effective uid around filesystem accessing
4860 # methods, using try/finally. [this seems to work]
4862 VERSION = string.split(RCS_ID)[2]
4864 class ftp_channel (async_chat):
4866 # defaults for a reliable __repr__
4867 addr = ('unknown','0')
4869 # unset this in a derived class in order
4870 # to enable the commands in 'self.write_commands'
4871 read_only = 1
4872 write_commands = ['appe','dele','mkd','rmd','rnfr','rnto','stor','stou']
4874 restart_position = 0
4876 # comply with (possibly troublesome) RFC959 requirements
4877 # This is necessary to correctly run an active data connection
4878 # through a firewall that triggers on the source port (expected
4879 # to be 'L-1', or 20 in the normal case).
4880 bind_local_minus_one = 0
4882 def __init__ (self, server, conn, addr):
4883 self.server = server
4884 self.current_mode = 'a'
4885 self.addr = addr
4886 async_chat.__init__ (self, conn)
4887 self.set_terminator ('\r\n')
4889 # client data port. Defaults to 'the same as the control connection'.
4890 self.client_addr = (addr[0], 21)
4892 self.client_dc = None
4893 self.in_buffer = ''
4894 self.closing = 0
4895 self.passive_acceptor = None
4896 self.passive_connection = None
4897 self.filesystem = None
4898 self.authorized = 0
4899 # send the greeting
4900 self.respond (
4901 '220 %s FTP server (Medusa Async V%s [experimental]) ready.' % (
4902 self.server.hostname,
4903 VERSION
4907 # def __del__ (self):
4908 # print 'ftp_channel.__del__()'
4910 # --------------------------------------------------
4911 # async-library methods
4912 # --------------------------------------------------
4914 def handle_expt (self):
4915 # this is handled below. not sure what I could
4916 # do here to make that code less kludgish.
4917 pass
4919 def collect_incoming_data (self, data):
4920 self.in_buffer = self.in_buffer + data
4921 if len(self.in_buffer) > 4096:
4922 # silently truncate really long lines
4923 # (possible denial-of-service attack)
4924 self.in_buffer = ''
4926 def found_terminator (self):
4928 line = self.in_buffer
4930 if not len(line):
4931 return
4933 sp = string.find (line, ' ')
4934 if sp != -1:
4935 line = [line[:sp], line[sp+1:]]
4936 else:
4937 line = [line]
4939 command = string.lower (line[0])
4940 # watch especially for 'urgent' abort commands.
4941 if string.find (command, 'abor') != -1:
4942 # strip off telnet sync chars and the like...
4943 while command and command[0] not in string.letters:
4944 command = command[1:]
4945 fun_name = 'cmd_%s' % command
4946 if command != 'pass':
4947 self.log ('<== %s' % repr(self.in_buffer)[1:-1])
4948 else:
4949 self.log ('<== %s' % line[0]+' <password>')
4950 self.in_buffer = ''
4951 if not hasattr (self, fun_name):
4952 self.command_not_understood (line[0])
4953 return
4954 fun = getattr (self, fun_name)
4955 if (not self.authorized) and (command not in ('user', 'pass', 'help', 'quit')):
4956 self.respond ('530 Please log in with USER and PASS')
4957 elif (not self.check_command_authorization (command)):
4958 self.command_not_authorized (command)
4959 else:
4960 try:
4961 result = apply (fun, (line,))
4962 except:
4963 self.server.total_exceptions.increment()
4964 (file, fun, line), t,v, tbinfo = asyncore.compact_traceback()
4965 if self.client_dc:
4966 try:
4967 self.client_dc.close()
4968 except:
4969 pass
4970 self.respond (
4971 '451 Server Error: %s, %s: file: %s line: %s' % (
4972 t,v,file,line,
4976 closed = 0
4977 def close (self):
4978 if not self.closed:
4979 self.closed = 1
4980 if self.passive_acceptor:
4981 self.passive_acceptor.close()
4982 if self.client_dc:
4983 self.client_dc.close()
4984 self.server.closed_sessions.increment()
4985 async_chat.close (self)
4987 # --------------------------------------------------
4988 # filesystem interface functions.
4989 # override these to provide access control or perform
4990 # other functions.
4991 # --------------------------------------------------
4993 def cwd (self, line):
4994 return self.filesystem.cwd (line[1])
4996 def cdup (self, line):
4997 return self.filesystem.cdup()
4999 def open (self, path, mode):
5000 return self.filesystem.open (path, mode)
5002 # returns a producer
5003 def listdir (self, path, long=0):
5004 return self.filesystem.listdir (path, long)
5006 def get_dir_list (self, line, long=0):
5007 # we need to scan the command line for arguments to '/bin/ls'...
5008 args = line[1:]
5009 path_args = []
5010 for arg in args:
5011 if arg[0] != '-':
5012 path_args.append (arg)
5013 else:
5014 # ignore arguments
5015 pass
5016 if len(path_args) < 1:
5017 dir = '.'
5018 else:
5019 dir = path_args[0]
5020 return self.listdir (dir, long)
5022 # --------------------------------------------------
5023 # authorization methods
5024 # --------------------------------------------------
5026 def check_command_authorization (self, command):
5027 if command in self.write_commands and self.read_only:
5028 return 0
5029 else:
5030 return 1
5032 # --------------------------------------------------
5033 # utility methods
5034 # --------------------------------------------------
5036 def log (self, message):
5037 self.server.logger.log (
5038 self.addr[0],
5039 '%d %s' % (
5040 self.addr[1], message
5044 def respond (self, resp):
5045 self.log ('==> %s' % resp)
5046 self.push (resp + '\r\n')
5048 def command_not_understood (self, command):
5049 self.respond ("500 '%s': command not understood." % command)
5051 def command_not_authorized (self, command):
5052 self.respond (
5053 "530 You are not authorized to perform the '%s' command" % (
5054 command
5058 def make_xmit_channel (self):
5059 # In PASV mode, the connection may or may _not_ have been made
5060 # yet. [although in most cases it is... FTP Explorer being
5061 # the only exception I've yet seen]. This gets somewhat confusing
5062 # because things may happen in any order...
5063 pa = self.passive_acceptor
5064 if pa:
5065 if pa.ready:
5066 # a connection has already been made.
5067 conn, addr = self.passive_acceptor.ready
5068 cdc = xmit_channel (self, addr)
5069 cdc.set_socket (conn)
5070 cdc.connected = 1
5071 self.passive_acceptor.close()
5072 self.passive_acceptor = None
5073 else:
5074 # we're still waiting for a connect to the PASV port.
5075 cdc = xmit_channel (self)
5076 else:
5077 # not in PASV mode.
5078 ip, port = self.client_addr
5079 cdc = xmit_channel (self, self.client_addr)
5080 cdc.create_socket (socket.AF_INET, socket.SOCK_STREAM)
5081 if self.bind_local_minus_one:
5082 cdc.bind (('', self.server.port - 1))
5083 try:
5084 cdc.connect ((ip, port))
5085 except socket.error, why:
5086 self.respond ("425 Can't build data connection")
5087 self.client_dc = cdc
5089 # pretty much the same as xmit, but only right on the verge of
5090 # being worth a merge.
5091 def make_recv_channel (self, fd):
5092 pa = self.passive_acceptor
5093 if pa:
5094 if pa.ready:
5095 # a connection has already been made.
5096 conn, addr = pa.ready
5097 cdc = recv_channel (self, addr, fd)
5098 cdc.set_socket (conn)
5099 cdc.connected = 1
5100 self.passive_acceptor.close()
5101 self.passive_acceptor = None
5102 else:
5103 # we're still waiting for a connect to the PASV port.
5104 cdc = recv_channel (self, None, fd)
5105 else:
5106 # not in PASV mode.
5107 ip, port = self.client_addr
5108 cdc = recv_channel (self, self.client_addr, fd)
5109 cdc.create_socket (socket.AF_INET, socket.SOCK_STREAM)
5110 try:
5111 cdc.connect ((ip, port))
5112 except socket.error, why:
5113 self.respond ("425 Can't build data connection")
5114 self.client_dc = cdc
5116 type_map = {
5117 'a':'ASCII',
5118 'i':'Binary',
5119 'e':'EBCDIC',
5120 'l':'Binary'
5123 type_mode_map = {
5124 'a':'t',
5125 'i':'b',
5126 'e':'b',
5127 'l':'b'
5130 # --------------------------------------------------
5131 # command methods
5132 # --------------------------------------------------
5134 def cmd_type (self, line):
5135 'specify data transfer type'
5136 # ascii, ebcdic, image, local <byte size>
5137 t = string.lower (line[1])
5138 # no support for EBCDIC
5139 # if t not in ['a','e','i','l']:
5140 if t not in ['a','i','l']:
5141 self.command_not_understood (string.join (line))
5142 elif t == 'l' and (len(line) > 2 and line[2] != '8'):
5143 self.respond ('504 Byte size must be 8')
5144 else:
5145 self.current_mode = t
5146 self.respond ('200 Type set to %s.' % self.type_map[t])
5149 def cmd_quit (self, line):
5150 'terminate session'
5151 self.respond ('221 Goodbye.')
5152 self.close_when_done()
5154 def cmd_port (self, line):
5155 'specify data connection port'
5156 info = string.split (line[1], ',')
5157 ip = string.join (info[:4], '.')
5158 port = string.atoi(info[4])*256 + string.atoi(info[5])
5159 # how many data connections at a time?
5160 # I'm assuming one for now...
5161 # TODO: we should (optionally) verify that the
5162 # ip number belongs to the client. [wu-ftpd does this?]
5163 self.client_addr = (ip, port)
5164 self.respond ('200 PORT command successful.')
5166 def new_passive_acceptor (self):
5167 # ensure that only one of these exists at a time.
5168 if self.passive_acceptor is not None:
5169 self.passive_acceptor.close()
5170 self.passive_acceptor = None
5171 self.passive_acceptor = passive_acceptor (self)
5172 return self.passive_acceptor
5174 def cmd_pasv (self, line):
5175 'prepare for server-to-server transfer'
5176 pc = self.new_passive_acceptor()
5177 port = pc.addr[1]
5178 ip_addr = pc.control_channel.getsockname()[0]
5179 self.respond (
5180 '227 Entering Passive Mode (%s,%d,%d)' % (
5181 string.replace(ip_addr, '.', ','),
5182 port/256,
5183 port%256
5186 self.client_dc = None
5188 def cmd_nlst (self, line):
5189 'give name list of files in directory'
5190 # ncftp adds the -FC argument for the user-visible 'nlist'
5191 # command. We could try to emulate ls flags, but not just yet.
5192 if '-FC' in line:
5193 line.remove ('-FC')
5194 try:
5195 dir_list_producer = self.get_dir_list (line, 0)
5196 except os.error, why:
5197 self.respond ('550 Could not list directory: %s' % why)
5198 return
5199 self.respond (
5200 '150 Opening %s mode data connection for file list' % (
5201 self.type_map[self.current_mode]
5204 self.make_xmit_channel()
5205 self.client_dc.push_with_producer (dir_list_producer)
5206 self.client_dc.close_when_done()
5208 def cmd_list (self, line):
5209 'give a list of files in a directory'
5210 try:
5211 dir_list_producer = self.get_dir_list (line, 1)
5212 except os.error, why:
5213 self.respond ('550 Could not list directory: %s' % why)
5214 return
5215 self.respond (
5216 '150 Opening %s mode data connection for file list' % (
5217 self.type_map[self.current_mode]
5220 self.make_xmit_channel()
5221 self.client_dc.push_with_producer (dir_list_producer)
5222 self.client_dc.close_when_done()
5224 def cmd_cwd (self, line):
5225 'change working directory'
5226 if self.cwd (line):
5227 self.respond ('250 CWD command successful.')
5228 else:
5229 self.respond ('550 No such directory.')
5231 def cmd_cdup (self, line):
5232 'change to parent of current working directory'
5233 if self.cdup(line):
5234 self.respond ('250 CDUP command successful.')
5235 else:
5236 self.respond ('550 No such directory.')
5238 def cmd_pwd (self, line):
5239 'print the current working directory'
5240 self.respond (
5241 '257 "%s" is the current directory.' % (
5242 self.filesystem.current_directory()
5246 # modification time
5247 # example output:
5248 # 213 19960301204320
5249 def cmd_mdtm (self, line):
5250 'show last modification time of file'
5251 filename = line[1]
5252 if not self.filesystem.isfile (filename):
5253 self.respond ('550 "%s" is not a file' % filename)
5254 else:
5255 mtime = time.gmtime(self.filesystem.stat(filename)[stat.ST_MTIME])
5256 self.respond (
5257 '213 %4d%02d%02d%02d%02d%02d' % (
5258 mtime[0],
5259 mtime[1],
5260 mtime[2],
5261 mtime[3],
5262 mtime[4],
5263 mtime[5]
5267 def cmd_noop (self, line):
5268 'do nothing'
5269 self.respond ('200 NOOP command successful.')
5271 def cmd_size (self, line):
5272 'return size of file'
5273 filename = line[1]
5274 if not self.filesystem.isfile (filename):
5275 self.respond ('550 "%s" is not a file' % filename)
5276 else:
5277 self.respond (
5278 '213 %d' % (self.filesystem.stat(filename)[stat.ST_SIZE])
5281 def cmd_retr (self, line):
5282 'retrieve a file'
5283 if len(line) < 2:
5284 self.command_not_understood (string.join (line))
5285 else:
5286 file = line[1]
5287 if not self.filesystem.isfile (file):
5288 self.log_info ('checking %s' % file)
5289 self.respond ('550 No such file')
5290 else:
5291 try:
5292 # FIXME: for some reason, 'rt' isn't working on win95
5293 mode = 'r'+self.type_mode_map[self.current_mode]
5294 fd = self.open (file, mode)
5295 except IOError, why:
5296 self.respond ('553 could not open file for reading: %s' % (repr(why)))
5297 return
5298 self.respond (
5299 "150 Opening %s mode data connection for file '%s'" % (
5300 self.type_map[self.current_mode],
5301 file
5304 self.make_xmit_channel()
5306 if self.restart_position:
5307 # try to position the file as requested, but
5308 # give up silently on failure (the 'file object'
5309 # may not support seek())
5310 try:
5311 fd.seek (self.restart_position)
5312 except:
5313 pass
5314 self.restart_position = 0
5316 self.client_dc.push_with_producer (
5317 file_producer (fd)
5319 self.client_dc.close_when_done()
5321 def cmd_stor (self, line, mode='wb'):
5322 'store a file'
5323 if len (line) < 2:
5324 self.command_not_understood (string.join (line))
5325 else:
5326 if self.restart_position:
5327 restart_position = 0
5328 self.respond ('553 restart on STOR not yet supported')
5329 return
5330 file = line[1]
5331 # todo: handle that type flag
5332 try:
5333 fd = self.open (file, mode)
5334 except IOError, why:
5335 self.respond ('553 could not open file for writing: %s' % (repr(why)))
5336 return
5337 self.respond (
5338 '150 Opening %s connection for %s' % (
5339 self.type_map[self.current_mode],
5340 file
5343 self.make_recv_channel (fd)
5345 def cmd_abor (self, line):
5346 'abort operation'
5347 if self.client_dc:
5348 self.client_dc.close()
5349 self.respond ('226 ABOR command successful.')
5351 def cmd_appe (self, line):
5352 'append to a file'
5353 return self.cmd_stor (line, 'ab')
5355 def cmd_dele (self, line):
5356 if len (line) != 2:
5357 self.command_not_understood (string.join (line))
5358 else:
5359 file = line[1]
5360 if self.filesystem.isfile (file):
5361 try:
5362 self.filesystem.unlink (file)
5363 self.respond ('250 DELE command successful.')
5364 except:
5365 self.respond ('550 error deleting file.')
5366 else:
5367 self.respond ('550 %s: No such file.' % file)
5369 def cmd_mkd (self, line):
5370 if len (line) != 2:
5371 self.command_not_understood (string.join (line))
5372 else:
5373 path = line[1]
5374 try:
5375 self.filesystem.mkdir (path)
5376 self.respond ('257 MKD command successful.')
5377 except:
5378 self.respond ('550 error creating directory.')
5380 def cmd_rmd (self, line):
5381 if len (line) != 2:
5382 self.command_not_understood (string.join (line))
5383 else:
5384 path = line[1]
5385 try:
5386 self.filesystem.rmdir (path)
5387 self.respond ('250 RMD command successful.')
5388 except:
5389 self.respond ('550 error removing directory.')
5391 def cmd_user (self, line):
5392 'specify user name'
5393 if len(line) > 1:
5394 self.user = line[1]
5395 self.respond ('331 Password required.')
5396 else:
5397 self.command_not_understood (string.join (line))
5399 def cmd_pass (self, line):
5400 'specify password'
5401 if len(line) < 2:
5402 pw = ''
5403 else:
5404 pw = line[1]
5405 result, message, fs = self.server.authorizer.authorize (self, self.user, pw)
5406 if result:
5407 self.respond ('230 %s' % message)
5408 self.filesystem = fs
5409 self.authorized = 1
5410 self.log_info('Successful login: Filesystem=%s' % repr(fs))
5411 else:
5412 self.respond ('530 %s' % message)
5414 def cmd_rest (self, line):
5415 'restart incomplete transfer'
5416 try:
5417 pos = string.atoi (line[1])
5418 except ValueError:
5419 self.command_not_understood (string.join (line))
5420 self.restart_position = pos
5421 self.respond (
5422 '350 Restarting at %d. Send STORE or RETRIEVE to initiate transfer.' % pos
5425 def cmd_stru (self, line):
5426 'obsolete - set file transfer structure'
5427 if line[1] in 'fF':
5428 # f == 'file'
5429 self.respond ('200 STRU F Ok')
5430 else:
5431 self.respond ('504 Unimplemented STRU type')
5433 def cmd_mode (self, line):
5434 'obsolete - set file transfer mode'
5435 if line[1] in 'sS':
5436 # f == 'file'
5437 self.respond ('200 MODE S Ok')
5438 else:
5439 self.respond ('502 Unimplemented MODE type')
5441 # The stat command has two personalities. Normally it returns status
5442 # information about the current connection. But if given an argument,
5443 # it is equivalent to the LIST command, with the data sent over the
5444 # control connection. Strange. But wuftpd, ftpd, and nt's ftp server
5445 # all support it.
5447 ## def cmd_stat (self, line):
5448 ## 'return status of server'
5449 ## pass
5451 def cmd_syst (self, line):
5452 'show operating system type of server system'
5453 # Replying to this command is of questionable utility, because
5454 # this server does not behave in a predictable way w.r.t. the
5455 # output of the LIST command. We emulate Unix ls output, but
5456 # on win32 the pathname can contain drive information at the front
5457 # Currently, the combination of ensuring that os.sep == '/'
5458 # and removing the leading slash when necessary seems to work.
5459 # [cd'ing to another drive also works]
5461 # This is how wuftpd responds, and is probably
5462 # the most expected. The main purpose of this reply is so that
5463 # the client knows to expect Unix ls-style LIST output.
5464 self.respond ('215 UNIX Type: L8')
5465 # one disadvantage to this is that some client programs
5466 # assume they can pass args to /bin/ls.
5467 # a few typical responses:
5468 # 215 UNIX Type: L8 (wuftpd)
5469 # 215 Windows_NT version 3.51
5470 # 215 VMS MultiNet V3.3
5471 # 500 'SYST': command not understood. (SVR4)
5473 def cmd_help (self, line):
5474 'give help information'
5475 # find all the methods that match 'cmd_xxxx',
5476 # use their docstrings for the help response.
5477 attrs = dir(self.__class__)
5478 help_lines = []
5479 for attr in attrs:
5480 if attr[:4] == 'cmd_':
5481 x = getattr (self, attr)
5482 if type(x) == type(self.cmd_help):
5483 if x.__doc__:
5484 help_lines.append ('\t%s\t%s' % (attr[4:], x.__doc__))
5485 if help_lines:
5486 self.push ('214-The following commands are recognized\r\n')
5487 self.push_with_producer (lines_producer (help_lines))
5488 self.push ('214\r\n')
5489 else:
5490 self.push ('214-\r\n\tHelp Unavailable\r\n214\r\n')
5492 class ftp_server (asyncore.dispatcher):
5493 # override this to spawn a different FTP channel class.
5494 ftp_channel_class = ftp_channel
5496 SERVER_IDENT = 'FTP Server (V%s)' % VERSION
5498 def __init__ (
5499 self,
5500 authorizer,
5501 hostname =None,
5502 ip ='',
5503 port =21,
5504 logger_object=file_logger (sys.stdout)
5506 self.ip = ip
5507 self.port = port
5508 self.authorizer = authorizer
5510 if hostname is None:
5511 self.hostname = socket.gethostname()
5512 else:
5513 self.hostname = hostname
5515 # statistics
5516 self.total_sessions = counter()
5517 self.closed_sessions = counter()
5518 self.total_files_out = counter()
5519 self.total_files_in = counter()
5520 self.total_bytes_out = counter()
5521 self.total_bytes_in = counter()
5522 self.total_exceptions = counter()
5524 asyncore.dispatcher.__init__ (self)
5525 self.create_socket (socket.AF_INET, socket.SOCK_STREAM)
5527 self.set_reuse_addr()
5528 self.bind ((self.ip, self.port))
5529 self.listen (5)
5531 if not logger_object:
5532 logger_object = sys.stdout
5534 self.logger = unresolving_logger (logger_object)
5536 self.log_info('FTP server started at %s\n\tAuthorizer:%s\n\tHostname: %s\n\tPort: %d' % (
5537 time.ctime(time.time()),
5538 repr (self.authorizer),
5539 self.hostname,
5540 self.port)
5543 def writable (self):
5544 return 0
5546 def handle_read (self):
5547 pass
5549 def handle_connect (self):
5550 pass
5552 def handle_accept (self):
5553 conn, addr = self.accept()
5554 self.total_sessions.increment()
5555 self.log_info('Incoming connection from %s:%d' % (addr[0], addr[1]))
5556 self.ftp_channel_class (self, conn, addr)
5558 # return a producer describing the state of the server
5559 def status (self):
5561 def nice_bytes (n):
5562 return string.join (english_bytes (n))
5564 return lines_producer (
5565 ['<h2>%s</h2>' % self.SERVER_IDENT,
5566 '<br>Listening on <b>Host:</b> %s' % self.hostname,
5567 '<b>Port:</b> %d' % self.port,
5568 '<br>Sessions',
5569 '<b>Total:</b> %s' % self.total_sessions,
5570 '<b>Current:</b> %d' % (self.total_sessions.as_long() - self.closed_sessions.as_long()),
5571 '<br>Files',
5572 '<b>Sent:</b> %s' % self.total_files_out,
5573 '<b>Received:</b> %s' % self.total_files_in,
5574 '<br>Bytes',
5575 '<b>Sent:</b> %s' % nice_bytes (self.total_bytes_out.as_long()),
5576 '<b>Received:</b> %s' % nice_bytes (self.total_bytes_in.as_long()),
5577 '<br>Exceptions: %s' % self.total_exceptions,
5581 # ======================================================================
5582 # Data Channel Classes
5583 # ======================================================================
5585 # This socket accepts a data connection, used when the server has been
5586 # placed in passive mode. Although the RFC implies that we ought to
5587 # be able to use the same acceptor over and over again, this presents
5588 # a problem: how do we shut it off, so that we are accepting
5589 # connections only when we expect them? [we can't]
5591 # wuftpd, and probably all the other servers, solve this by allowing
5592 # only one connection to hit this acceptor. They then close it. Any
5593 # subsequent data-connection command will then try for the default
5594 # port on the client side [which is of course never there]. So the
5595 # 'always-send-PORT/PASV' behavior seems required.
5597 # Another note: wuftpd will also be listening on the channel as soon
5598 # as the PASV command is sent. It does not wait for a data command
5599 # first.
5601 # --- we need to queue up a particular behavior:
5602 # 1) xmit : queue up producer[s]
5603 # 2) recv : the file object
5605 # It would be nice if we could make both channels the same. Hmmm..
5608 class passive_acceptor (asyncore.dispatcher):
5609 ready = None
5611 def __init__ (self, control_channel):
5612 # connect_fun (conn, addr)
5613 asyncore.dispatcher.__init__ (self)
5614 self.control_channel = control_channel
5615 self.create_socket (socket.AF_INET, socket.SOCK_STREAM)
5616 # bind to an address on the interface that the
5617 # control connection is coming from.
5618 self.bind ((
5619 self.control_channel.getsockname()[0],
5622 self.addr = self.getsockname()
5623 self.listen (1)
5625 # def __del__ (self):
5626 # print 'passive_acceptor.__del__()'
5628 def log (self, *ignore):
5629 pass
5631 def handle_accept (self):
5632 conn, addr = self.accept()
5633 dc = self.control_channel.client_dc
5634 if dc is not None:
5635 dc.set_socket (conn)
5636 dc.addr = addr
5637 dc.connected = 1
5638 self.control_channel.passive_acceptor = None
5639 else:
5640 self.ready = conn, addr
5641 self.close()
5644 class xmit_channel (async_chat):
5646 # for an ethernet, you want this to be fairly large, in fact, it
5647 # _must_ be large for performance comparable to an ftpd. [64k] we
5648 # ought to investigate automatically-sized buffers...
5650 ac_out_buffer_size = 16384
5651 bytes_out = 0
5653 def __init__ (self, channel, client_addr=None):
5654 self.channel = channel
5655 self.client_addr = client_addr
5656 async_chat.__init__ (self)
5658 # def __del__ (self):
5659 # print 'xmit_channel.__del__()'
5661 def log (self, *args):
5662 pass
5664 def readable (self):
5665 return not self.connected
5667 def writable (self):
5668 return 1
5670 def send (self, data):
5671 result = async_chat.send (self, data)
5672 self.bytes_out = self.bytes_out + result
5673 return result
5675 def handle_error (self):
5676 # usually this is to catch an unexpected disconnect.
5677 self.log_info ('unexpected disconnect on data xmit channel', 'error')
5678 try:
5679 self.close()
5680 except:
5681 pass
5683 # TODO: there's a better way to do this. we need to be able to
5684 # put 'events' in the producer fifo. to do this cleanly we need
5685 # to reposition the 'producer' fifo as an 'event' fifo.
5687 def close (self):
5688 c = self.channel
5689 s = c.server
5690 c.client_dc = None
5691 s.total_files_out.increment()
5692 s.total_bytes_out.increment (self.bytes_out)
5693 if not len(self.producer_fifo):
5694 c.respond ('226 Transfer complete')
5695 elif not c.closed:
5696 c.respond ('426 Connection closed; transfer aborted')
5697 del c
5698 del s
5699 del self.channel
5700 async_chat.close (self)
5702 class recv_channel (asyncore.dispatcher):
5703 def __init__ (self, channel, client_addr, fd):
5704 self.channel = channel
5705 self.client_addr = client_addr
5706 self.fd = fd
5707 asyncore.dispatcher.__init__ (self)
5708 self.bytes_in = counter()
5710 def log (self, *ignore):
5711 pass
5713 def handle_connect (self):
5714 pass
5716 def writable (self):
5717 return 0
5719 def recv (*args):
5720 result = apply (asyncore.dispatcher.recv, args)
5721 self = args[0]
5722 self.bytes_in.increment(len(result))
5723 return result
5725 buffer_size = 8192
5727 def handle_read (self):
5728 block = self.recv (self.buffer_size)
5729 if block:
5730 try:
5731 self.fd.write (block)
5732 except IOError:
5733 self.log_info ('got exception writing block...', 'error')
5735 def handle_close (self):
5736 s = self.channel.server
5737 s.total_files_in.increment()
5738 s.total_bytes_in.increment(self.bytes_in.as_long())
5739 self.fd.close()
5740 self.channel.respond ('226 Transfer complete.')
5741 self.close()
5744 import getopt
5745 import re, sys
5746 import asyncore
5747 import os
5748 import random
5749 import imp
5750 import time
5751 import thread
5752 import stat
5753 import urllib
5754 import traceback
5755 import logging
5756 import zipfile
5758 HTTP_CONTINUE = 100
5759 HTTP_SWITCHING_PROTOCOLS = 101
5760 HTTP_PROCESSING = 102
5761 HTTP_OK = 200
5762 HTTP_CREATED = 201
5763 HTTP_ACCEPTED = 202
5764 HTTP_NON_AUTHORITATIVE = 203
5765 HTTP_NO_CONTENT = 204
5766 HTTP_RESET_CONTENT = 205
5767 HTTP_PARTIAL_CONTENT = 206
5768 HTTP_MULTI_STATUS = 207
5769 HTTP_MULTIPLE_CHOICES = 300
5770 HTTP_MOVED_PERMANENTLY = 301
5771 HTTP_MOVED_TEMPORARILY = 302
5772 HTTP_SEE_OTHER = 303
5773 HTTP_NOT_MODIFIED = 304
5774 HTTP_USE_PROXY = 305
5775 HTTP_TEMPORARY_REDIRECT = 307
5776 HTTP_BAD_REQUEST = 400
5777 HTTP_UNAUTHORIZED = 401
5778 HTTP_PAYMENT_REQUIRED = 402
5779 HTTP_FORBIDDEN = 403
5780 HTTP_NOT_FOUND = 404
5781 HTTP_METHOD_NOT_ALLOWED = 405
5782 HTTP_NOT_ACCEPTABLE = 406
5783 HTTP_PROXY_AUTHENTICATION_REQUIRED= 407
5784 HTTP_REQUEST_TIME_OUT = 408
5785 HTTP_CONFLICT = 409
5786 HTTP_GONE = 410
5787 HTTP_LENGTH_REQUIRED = 411
5788 HTTP_PRECONDITION_FAILED = 412
5789 HTTP_REQUEST_ENTITY_TOO_LARGE = 413
5790 HTTP_REQUEST_URI_TOO_LARGE = 414
5791 HTTP_UNSUPPORTED_MEDIA_TYPE = 415
5792 HTTP_RANGE_NOT_SATISFIABLE = 416
5793 HTTP_EXPECTATION_FAILED = 417
5794 HTTP_UNPROCESSABLE_ENTITY = 422
5795 HTTP_LOCKED = 423
5796 HTTP_FAILED_DEPENDENCY = 424
5797 HTTP_INTERNAL_SERVER_ERROR = 500
5798 HTTP_NOT_IMPLEMENTED = 501
5799 HTTP_BAD_GATEWAY = 502
5800 HTTP_SERVICE_UNAVAILABLE = 503
5801 HTTP_GATEWAY_TIME_OUT = 504
5802 HTTP_VERSION_NOT_SUPPORTED = 505
5803 HTTP_VARIANT_ALSO_VARIES = 506
5804 HTTP_INSUFFICIENT_STORAGE = 507
5805 HTTP_NOT_EXTENDED = 510
5807 GLOBAL_TEMP_DIR="/tmp/"
5808 GLOBAL_ROOT_DIR="no-root-dir-set"
5809 verbose = 0
5810 multithreading_enabled = 0
5811 number_of_threads = 32
5813 def qualify_path(p):
5814 if p[-1] != '/':
5815 return p + "/"
5816 return p
5818 def join_paths(p1,p2):
5819 if p1.endswith("/"):
5820 if p2.startswith("/"):
5821 return p1[:-1] + p2
5822 else:
5823 return p1 + p2
5824 else:
5825 if p2.startswith("/"):
5826 return p1 + p2
5827 else:
5828 return p1 + "/" + p2
5831 translators = []
5832 macroresolvers = []
5833 ftphandlers = []
5834 contexts = []
5836 def getMacroFile(filename):
5837 global macrofile_callback
5838 for r in macroresolvers:
5839 try:
5840 f = r(filename)
5841 if f is not None and os.path.isfile(f):
5842 return f
5843 except:
5844 pass
5845 if os.path.isfile(filename):
5846 return filename
5847 filename2 = join_paths(GLOBAL_ROOT_DIR,filename)
5848 if os.path.isfile(filename2):
5849 return filename2
5850 raise IOError("No such file: "+filename2)
5853 global_modules={}
5855 def _make_inifiles(root, path):
5856 dirs = path.split("/")
5857 path = root
5858 for dir in dirs:
5859 path = join_paths(path, dir)
5860 inifile = join_paths(path, "__init__.py")
5861 # create missing __init__.py
5862 if not os.path.isfile(inifile):
5863 if lg:
5864 lg.log("creating file "+inifile)
5865 open(inifile, "wb").close()
5867 def _load_module(filename):
5868 global global_modules
5869 b = BASENAME.match(filename)
5870 # filename e.g. /my/modules/test.py
5871 # b.group(1) = /my/modules/
5872 # b.group(2) = test.py
5873 if b is None:
5874 raise "Internal error with filename "+filename
5875 module = b.group(2)
5876 if module is None:
5877 raise "Internal error with filename "+filename
5879 while filename.startswith("./"):
5880 filename = filename[2:]
5882 if filename in global_modules:
5883 return global_modules[filename]
5885 dir = os.path.dirname(filename)
5886 path = dir.replace("/",".")
5888 _make_inifiles(GLOBAL_ROOT_DIR, dir)
5890 # strip tailing/leading dots
5891 while len(path) and path[0] == '.':
5892 path = path[1:]
5893 while len(path) and path[-1] != '.':
5894 path = path + "."
5896 module2 = (path + module)
5897 if lg:
5898 lg.log("Loading module "+module2)
5900 m = __import__(module2)
5901 try:
5902 i = module2.index(".")
5903 m = eval("m."+module2[i+1:])
5904 global_modules[filename] = m
5905 except:
5906 pass
5907 return m
5909 system_modules = sys.modules.copy()
5910 stdlib, x = os.path.split(os.__file__)
5911 def _purge_all_modules():
5912 for m,mod in sys.modules.items():
5913 if m not in system_modules:
5914 if hasattr(mod, "__file__"):
5915 f = mod.__file__
5916 path, x = os.path.split(f)
5917 if not path.startswith(stdlib):
5918 del sys.modules[m]
5920 class WebContext:
5921 def __init__(self, name, root=None):
5922 self.name = name
5923 self.files = []
5924 self.startupfile = None
5925 if root:
5926 self.root = qualify_path(root)
5927 self.pattern_to_function = {}
5928 self.id_to_function = {}
5930 def addFile(self, filename):
5931 file = WebFile(self, filename)
5932 self.files += [file]
5933 return file
5935 def setRoot(self, root):
5936 self.root = qualify_path(root)
5937 while self.root.startswith("./"):
5938 self.root = self.root[2:]
5940 def setStartupFile(self, startupfile):
5941 self.startupfile = startupfile
5942 lg.log(" executing startupfile")
5943 self._load_module(self.startupfile)
5945 def getStartupFile(self):
5946 return self.startupfile
5948 def match(self, path):
5949 function = None
5950 for pattern,call in self.pattern_to_function.items():
5951 if pattern.match(path):
5952 function,desc = call
5953 if verbose:
5954 lg.log("Request %s matches (%s)" % (req.path, desc))
5955 if function is None:
5956 for id,call in self.id_to_function.items():
5957 if path == id:
5958 function,desc = call
5959 if verbose:
5960 lg.log("Request %s matches handler (%s)" % (req.path, desc))
5961 if not function:
5962 return None
5963 def call_and_close(f,req):
5964 status = f(req)
5965 if status is not None and type(1)==type(status) and status>10:
5966 req.reply_code = status
5967 if status == 404:
5968 return req.error(status, "not found")
5969 elif(status >= 400 and status <= 500):
5970 return req.error(status)
5971 return req.done()
5972 return lambda req: call_and_close(function,req)
5974 class FileStore:
5975 def __init__(self, name, root=None):
5976 self.name = name
5977 self.handlers = []
5978 if type(root) == type(""):
5979 self.addRoot(root)
5980 elif type(root) == type([]):
5981 for dir in root:
5982 self.addRoot(dir)
5984 def match(self, path):
5985 return lambda req: self.findfile(req)
5987 def findfile(self, request):
5988 for handler in self.handlers:
5989 if handler.can_handle(request):
5990 return handler.handle_request(request)
5991 return request.error(404, "File "+request.path+" not found")
5993 def addRoot(self, dir):
5994 dir = qualify_path(dir)
5995 while dir.startswith("./"):
5996 dir = dir[2:]
5997 if zipfile.is_zipfile(GLOBAL_ROOT_DIR + dir[:-1]) and dir.lower().endswith("zip/"):
5998 self.handlers += [default_handler (zip_filesystem (GLOBAL_ROOT_DIR + dir[:-1]))]
5999 else:
6000 self.handlers += [default_handler (os_filesystem (GLOBAL_ROOT_DIR + dir))]
6002 class WebFile:
6003 def __init__(self, context, filename):
6004 self.context = context
6005 if filename[0] == '/':
6006 filename = filename[1:]
6007 self.filename = filename
6008 self.m = _load_module(filename)
6009 self.handlers = []
6011 def addHandler(self, function):
6012 handler = WebHandler(self, function)
6013 self.handlers += [handler]
6014 return handler
6016 def addFTPHandler(self, ftpclass):
6017 global ftphandlers
6018 m = self.m
6019 try:
6020 c = eval("m."+ftpclass)
6021 if c is None:
6022 raise
6023 ftphandlers += [c]
6024 except:
6025 lgerr.log("Error in FTP Handler:" + str(sys.exc_info()[0]) + " " + str(sys.exc_info()[1]))
6026 traceback.print_tb(sys.exc_info()[2],None,lgerr)
6027 raise "No such function "+ftpclass+" in file "+self.filename
6029 def addMacroResolver(self, macroresolver):
6030 global macroresolvers
6031 m = self.m
6032 try:
6033 f = eval("m."+macroresolver)
6034 if f is None:
6035 raise
6036 macroresolvers += [f]
6037 except:
6038 lgerr.log("Error in Macro Resolver:" + str(sys.exc_info()[0]) + " " + str(sys.exc_info()[1]))
6039 traceback.print_tb(sys.exc_info()[2],None,lgerr)
6040 raise "No such function "+macroresolver+" in file "+self.filename
6042 def addTranslator(self, handler):
6043 global translators
6044 m = self.m
6045 try:
6046 f = eval("m."+translator)
6047 if f is None:
6048 raise
6049 translators += [f]
6050 except:
6051 lgerr.log("Error in Macro Resolver:" + str(sys.exc_info()[0]) + " " + str(sys.exc_info()[1]))
6052 traceback.print_tb(sys.exc_info()[2],None,lgerr)
6053 raise "No such function "+translator+" in file "+self.filename
6055 def getFileName(self):
6056 return self.context.root + self.filename
6058 class WebHandler:
6059 def __init__(self, file, function):
6060 self.file = file
6061 self.function = function
6062 m = file.m
6063 try:
6064 self.f = eval("m."+function)
6065 if self.f is None:
6066 raise
6067 except:
6068 lgerr.log("Error in Handler:" + str(sys.exc_info()[0]) + " " + str(sys.exc_info()[1]))
6069 traceback.print_tb(sys.exc_info()[2],None,lgerr)
6070 raise "No such function "+function+" in file "+self.file.filename
6072 def addPattern(self, pattern):
6073 p = WebPattern(self,pattern)
6074 desc = "pattern %s, file %s, function %s" % (pattern,self.file.filename,self.function)
6075 desc2 = "file %s, function %s" % (self.file.filename,self.function)
6076 self.file.context.pattern_to_function[p.getPattern()] = (self.f,desc)
6077 self.file.context.id_to_function["/"+self.function] = (self.f,desc2)
6078 return p
6080 class WebPattern:
6081 def __init__(self, handler, pattern):
6082 self.handler = handler
6083 self.pattern = pattern
6084 if not pattern.endswith('$'):
6085 pattern = pattern + "$"
6086 self.compiled = re.compile(pattern)
6087 def getPattern(self):
6088 return self.compiled
6089 def getPatternString(self):
6090 return self.pattern
6092 def read_ini_file(filename):
6093 global GLOBAL_TEMP_DIR,GLOBAL_ROOT_DIR,number_of_threads,multithreading_enabled,contexts
6094 lineno = 0
6095 fi = open(filename, "rb")
6096 file = None
6097 function = None
6098 context = None
6099 GLOBAL_ROOT_DIR = '/'
6100 for line in fi.readlines():
6101 lineno=lineno+1
6102 hashpos = line.find("#")
6103 if hashpos>=0:
6104 line = line[0:hashpos]
6105 line = line.strip()
6107 if line == "":
6108 continue #skip empty line
6110 equals = line.find(":")
6111 if equals<0:
6112 continue
6113 key = line[0:equals].strip()
6114 value = line[equals+1:].strip()
6115 if key == "tempdir":
6116 GLOBAL_TEMP_DIR = qualify_path(value)
6117 elif key == "threads":
6118 number_of_threads = int(value)
6119 multithreading_enabled = 1
6120 elif key == "base":
6121 GLOBAL_ROOT_DIR = qualify_path(value)
6122 sys.path += [GLOBAL_ROOT_DIR]
6123 elif key == "filestore":
6124 if len(value) and value[0] != '/':
6125 value = "/" + value
6126 filestore = FileStore(value)
6127 contexts += [filestore]
6128 context = None
6129 elif key == "context":
6130 if len(value) and value[0] != '/':
6131 value = "/" + value
6132 contextname = value
6133 context = WebContext(contextname)
6134 contexts += [context]
6135 filestore = None
6136 elif key == "startupfile":
6137 if context is not None:
6138 context.setStartupFile(value)
6139 else:
6140 raise "Error: startupfile must be below a context"
6141 elif key == "root":
6142 if value.startswith('/'):
6143 value = value[1:]
6144 if context:
6145 context.setRoot(value)
6146 if filestore:
6147 filestore.addRoot(value)
6148 elif key == "file":
6149 filename = value
6150 context.addFile(filename)
6151 elif key == "ftphandler":
6152 file.addFTPHandler(value)
6153 elif key == "handler":
6154 function = value
6155 file.addHandler(function)
6156 elif key == "macroresolver":
6157 file.addMacroResolver(value)
6158 elif key == "translator":
6159 file.addTranslator(value)
6160 elif key == "pattern":
6161 handler.addPattern(value)
6162 else:
6163 raise "Syntax error in line "+str(lineno)+" of file "+filename+":\n"+line
6164 fi.close()
6166 def headers_to_map(mylist):
6167 headers={}
6168 for h in mylist:
6169 try:
6170 i = h.index(':')
6171 except:
6172 i = -1
6173 if i >= 0:
6174 key = h[0:i].lower()
6175 value = h[i+1:]
6176 if len(value)>0 and value[0] == ' ':
6177 value = value[1:]
6178 headers[key] = value
6179 else:
6180 if len(h.strip())>0:
6181 lg.log("invalid header: "+str(h))
6182 return headers
6184 class AthanaFile:
6185 def __init__(self,fieldname, parammap,filename,content_type):
6186 self.fieldname = fieldname
6187 self.parammap = parammap
6188 self.filename = filename
6189 self.content_type = content_type
6190 self.tempname = GLOBAL_TEMP_DIR+str(int(random.random()*999999))+os.path.splitext(filename)[1]
6191 self.filesize = 0
6192 self.fi = open(self.tempname, "wb")
6193 def adddata(self,data):
6194 self.filesize += len(data)
6195 self.fi.write(data)
6196 def close(self):
6197 self.fi.close()
6198 # only append file to parameters if it contains some data
6199 if self.filename or self.filesize:
6200 self.parammap[self.fieldname] = self
6201 del self.fieldname
6202 del self.parammap
6203 del self.fi
6204 def __str__(self):
6205 return "file %s (%s), %d bytes, content-type: %s" % (self.filename, self.tempname, self.filesize, self.content_type)
6207 class AthanaField:
6208 def __init__(self,fieldname,parammap):
6209 self.fieldname = fieldname
6210 self.data = ""
6211 self.parammap = parammap
6212 def adddata(self,data):
6213 self.data += data
6214 def close(self):
6215 try:
6216 oldvalue = self.parammap[self.fieldname] + ";"
6217 except KeyError:
6218 oldvalue = ""
6219 self.parammap[self.fieldname] = oldvalue + self.data
6220 del self.data
6221 del self.parammap
6223 class simple_input_collector:
6224 def __init__ (self, handler, request, length):
6225 self.request = request
6226 self.length = length
6227 self.handler = handler
6228 request.channel.set_terminator(length)
6229 self.data = ""
6231 def collect_incoming_data (self, data):
6232 self.data += data
6234 def found_terminator(self):
6235 self.request.channel.set_terminator('\r\n\r\n')
6236 self.request.collector = None
6237 d=self.data;del self.data
6238 r=self.request;del self.request
6239 parameters={}
6240 data = d.split('&')
6241 for e in data:
6242 if '=' in e:
6243 key,value = e.split('=')
6244 key = urllib.unquote_plus(key)
6245 try:
6246 oldvalue = parameters[key]+";"
6247 except KeyError:
6248 oldvalue = ""
6249 parameters[key] = oldvalue + urllib.unquote_plus(value)
6250 else:
6251 if len(e.strip())>0:
6252 lg.log("Unknown parameter: "+e)
6253 self.handler.continue_request(r,parameters)
6255 class upload_input_collector:
6256 def __init__ (self, handler, request, length, boundary):
6257 self.request = request
6258 self.length = length
6259 self.handler = handler
6260 self.boundary = boundary
6261 request.channel.set_terminator(length)
6262 self.data = ""
6263 self.pos = 0
6264 self.start_marker = "--"+boundary+"\r\n"
6265 self.end_marker = "--"+boundary+"--\r\n"
6266 self.prefix = "--"+boundary
6267 self.marker = "\r\n--"+boundary
6268 self.header_end_marker = "\r\n\r\n"
6269 self.current_file = None
6270 self.boundary = boundary
6271 self.file = None
6272 self.parameters = {}
6273 self.files = []
6275 def parse_semicolon_parameters(self,params):
6276 params = params.split("; ")
6277 parmap = {}
6278 for a in params:
6279 if '=' in a:
6280 key,value = a.split('=')
6281 if value.startswith('"') and value.endswith('"'):
6282 value = value[1:-1]
6283 parmap[key] = value
6284 return parmap
6286 def startFile(self,headers):
6287 fieldname = None
6288 filename = None
6289 if self.file is not None:
6290 raise "Illegal state"
6291 if "content-disposition" in headers:
6292 cd = headers["content-disposition"]
6293 l = self.parse_semicolon_parameters(cd)
6294 if "name" in l:
6295 fieldname = l["name"]
6296 if "filename" in l:
6297 filename = l["filename"]
6298 if "content-type" in headers:
6299 content_type = headers["content-type"]
6300 self.file = AthanaFile(fieldname,self.parameters,filename,content_type)
6301 self.files += [self.file]
6302 else:
6303 self.file = AthanaField(fieldname,self.parameters)
6305 def split_headers(self,string):
6306 return string.split("\r\n")
6308 def collect_incoming_data (self, newdata):
6309 self.pos += len(newdata)
6310 self.data += newdata
6312 while len(self.data)>0:
6313 if self.data.startswith(self.end_marker):
6314 self.data = self.data[len(self.end_marker):]
6315 if self.file is not None:
6316 self.file.close()
6317 self.file = None
6318 return
6319 elif self.data.startswith(self.start_marker):
6320 try:
6321 i = self.data.index(self.header_end_marker, len(self.start_marker))
6322 except:
6323 i = -1
6324 if i>=0:
6325 headerstr = self.data[len(self.start_marker):i+2]
6326 headers = headers_to_map(self.split_headers(headerstr))
6327 self.startFile(headers)
6328 self.data = self.data[i+len(self.header_end_marker):]
6329 else:
6330 return # wait for more data (inside headers)
6331 elif self.data.startswith(self.prefix):
6332 return
6333 else:
6334 try:
6335 bindex = self.data.index(self.marker)
6336 self.file.adddata(self.data[0:bindex])
6337 self.file.close()
6338 self.file = None
6339 self.data = self.data[bindex+2:] # cut to position after \r\n
6340 except ValueError: #not found
6341 if(len(self.data) <= len(self.marker)):
6342 return #wait for more data before we make a decision or pass through data
6343 else:
6344 self.file.adddata(self.data[0:-len(self.marker)])
6345 self.data = self.data[-len(self.marker):]
6347 def found_terminator(self):
6348 if len(self.data)>0:# and self.file is not None:
6349 if self.file is not None:
6350 self.file.close()
6351 self.file = None
6352 raise "Unfinished/malformed multipart request"
6353 if self.file is not None:
6354 self.file.close()
6355 self.file = None
6357 self.request.collector = None
6358 self.request.channel.set_terminator('\r\n\r\n')
6359 d=self.data;del self.data
6360 r=self.request;del self.request
6361 r.tempfiles = [f.tempname for f in self.files]
6362 self.handler.continue_request(r,self.parameters)
6364 class Session(dict):
6365 def __init__(self, id):
6366 self.id = id
6367 def use(self):
6368 self.lastuse = time.time()
6370 def exception_string():
6371 s = "Exception "+str(sys.exc_info()[0])
6372 info = sys.exc_info()[1]
6373 if info:
6374 s += " "+str(info)
6375 s += "\n"
6376 for l in traceback.extract_tb(sys.exc_info()[2]):
6377 s += " File \"%s\", line %d, in %s\n" % (l[0],l[1],l[2])
6378 s += " %s\n" % l[3]
6379 return s
6381 BASENAME = re.compile("([^/]*/)*([^/.]*)(.py)?")
6382 MULTIPART = re.compile ('multipart/form-data.*boundary=([^ ]*)', re.IGNORECASE)
6383 SESSION_PATTERN = re.compile("^;[a-z0-9]{6}-[a-z0-9]{6}-[a-z0-9]{6}$")
6385 use_cookies = 1
6387 class AthanaHandler:
6388 def __init__(self):
6389 self.sessions = {}
6390 self.queue = []
6391 self.queuelock = thread.allocate_lock()
6393 def match(self, request):
6394 path, params, query, fragment = request.split_uri()
6395 #lg.log("===== request:"+path+"=====")
6396 return 1
6398 def handle_request (self, request):
6399 headers = headers_to_map(request.header)
6400 request.request_headers = headers
6402 size=headers.get("content-length",None)
6404 if size and size != '0':
6405 size=int(size)
6406 ctype=headers.get("content-type",None)
6407 b = MULTIPART.match(ctype)
6408 if b is not None:
6409 request.type = "MULTIPART"
6410 boundary = b.group(1)
6411 request.collector = upload_input_collector(self,request,size,boundary)
6412 else:
6413 request.type = "POST"
6414 request.collector = simple_input_collector(self,request,size)
6415 else:
6416 request.type = "GET"
6417 self.continue_request(request, {})
6419 def create_session_id(self):
6420 pid = abs((str(random.random())).__hash__())
6421 now = abs((str(time.time())).__hash__())
6422 rand = abs((str(random.random())).__hash__())
6423 x = "abcdefghijklmnopqrstuvwxyz0123456789"
6424 result = ""
6425 for a in range(0,6):
6426 result += x[pid%36]
6427 pid = pid / 36
6428 result += "-"
6429 for a in range(0,6):
6430 result += x[now%36]
6431 now = now / 36
6432 result += "-"
6433 for a in range(0,6):
6434 result += x[rand%36]
6435 rand = rand / 36
6436 return result
6438 def continue_request(self, request, parameters):
6440 path, params, query, fragment = request.split_uri()
6442 ip = request.request_headers.get("x-forwarded-for",None)
6443 if ip is None:
6444 try: ip = request.channel.addr[0]
6445 except: pass
6446 if ip:
6447 request.channel.addr = (ip,request.channel.addr[1])
6449 request.log()
6451 if query is not None:
6452 if query[0] == '?':
6453 query=query[1:]
6454 query = query.split('&')
6455 for e in query:
6456 key,value = e.split('=')
6457 key = urllib.unquote_plus(key)
6458 try:
6459 oldvalue = parameters[key]+";"
6460 except KeyError:
6461 oldvalue = ""
6462 parameters[key] = oldvalue + urllib.unquote_plus(value) #_plus?
6464 cookies = {}
6465 if "cookie" in request.request_headers:
6466 cookiestr = request.request_headers["cookie"]
6467 if cookiestr.rfind(";") == len(cookiestr)-1:
6468 cookiestr = cookiestr[:-1]
6469 items = cookiestr.split(';')
6470 for a in items:
6471 key,value = a.strip().split('=')
6472 cookies[key] = value
6474 request.Cookies = cookies
6476 sessionid = None
6477 if params is not None and SESSION_PATTERN.match(params):
6478 sessionid = params
6479 if sessionid[0] == ';':
6480 sessionid = sessionid[1:]
6481 elif use_cookies and "PSESSION" in cookies:
6482 sessionid = cookies["PSESSION"]
6484 if sessionid is not None:
6485 if sessionid in self.sessions:
6486 session = self.sessions[sessionid]
6487 session.use()
6488 else:
6489 session = Session(sessionid)
6490 self.sessions[sessionid] = session
6491 else:
6492 sessionid = self.create_session_id()
6493 session = Session(sessionid)
6494 self.sessions[sessionid] = session
6497 request['Connection'] = 'close';
6498 request['Content-Type'] = 'text/html; encoding=utf-8; charset=utf-8';
6500 maxlen = -1
6501 context = None
6502 global contexts
6503 for c in contexts:
6504 #lg.debug("Compare context "+c.name+" with request "+path)
6505 if path.startswith(c.name) and len(c.name)>maxlen:
6506 context = c
6507 maxlen = len(context.name)
6508 if context is None:
6509 request.error (404)
6510 return
6512 #print "Request ",'"'+path+'"',"maps to context",context.name
6513 fullpath = path
6514 path = path[len(context.name):]
6515 if len(path)==0 or path[0] != '/':
6516 path = "/" + path
6518 request.session = session
6519 request.sessionid = sessionid
6520 request.context = context
6521 request.path = path
6522 request.fullpath = fullpath
6523 request.paramstring = params
6524 request.query = query
6525 request.fragment = fragment
6526 request.params = parameters
6527 request.request = request
6528 request.ip = ip
6529 request.uri = request.uri.replace(context.name, "/")
6530 request._split_uri = None
6532 if use_cookies:
6533 request.setCookie('PSESSION', sessionid, time.time()+3600*2)
6535 request.channel.current_request = None
6537 function = context.match(path)
6539 if function is not None:
6540 if not multithreading_enabled:
6541 self.callhandler(function, request)
6542 else:
6543 self.queuelock.acquire()
6544 self.queue += [(function,request)]
6545 self.queuelock.release()
6546 return
6547 else:
6548 lg.log("Request %s matches no pattern (context: %s)" % (request.path,context.name))
6549 return request.error(404, "File %s not found" % request.path)
6551 def callhandler(self, function, req):
6552 request = req.request
6553 s = None
6554 try:
6555 status = function(req)
6556 except:
6557 lgerr.log("Error in page :" + str(sys.exc_info()[0]) + " " + str(sys.exc_info()[1]))
6558 traceback.print_tb(sys.exc_info()[2],None,lgerr)
6559 s = "<pre>"+exception_string()+"</pre>"
6560 return request.error(500,s)
6562 def worker_thread(server):
6563 while 1:
6564 server.queuelock.acquire()
6565 if len(server.queue) == 0:
6566 server.queuelock.release()
6567 time.sleep(0.01)
6568 else:
6569 function,req = server.queue.pop()
6570 server.queuelock.release()
6571 try:
6572 server.callhandler(function,req)
6573 except:
6574 lgerr.log("Error while processing request:" + str(sys.exc_info()[0]) + " " + str(sys.exc_info()[1]))
6575 traceback.print_tb(sys.exc_info()[2],None,lgerr)
6577 class fs:
6578 pass
6580 class virtual_authorizer:
6581 def __init__ (self):
6582 pass
6583 def authorize (self, channel, username, password):
6584 channel.persona = -1, -1
6585 channel.read_only = 1
6586 #return 1, 'Ok.', fs()
6587 return 1, 'Ok.', os_filesystem("/home/kramm")
6589 def __repr__(self):
6590 return 'virtual'
6592 class logging_logger:
6593 def __init__(self,name="athana"):
6594 self.logger = logging.getLogger(name)
6595 def log (self, message):
6596 self.logger.info(message.rstrip())
6597 def debug (self, message):
6598 self.logger.debug(message.rstrip())
6599 def write (self, message):
6600 self.logger.info(message.rstrip())
6601 def error (self, message):
6602 self.logger.error(message.rstrip())
6604 lg = logging_logger()
6605 lgerr = logging_logger("errors")
6607 class zip_filesystem:
6608 def __init__(self, filename):
6609 self.filename = filename
6610 self.wd = '/'
6611 self.m = {}
6612 self.z = zipfile.ZipFile(filename)
6613 self.lock = thread.allocate_lock()
6614 for f in self.z.filelist:
6615 self.m['/' + f.filename] = f
6617 def current_directory(self):
6618 return self.wd
6620 def isfile(self, path):
6621 if len(path) and path[-1]=='/':
6622 return 0
6623 return (self.wd + path) in self.m
6625 def isdir (self, path):
6626 if not (len(path) and path[-1]=='/'):
6627 path += '/'
6628 return path in self.m
6630 def cwd (self, path):
6631 path = join_paths(self.wd, path)
6632 if not self.isdir (path):
6633 return 0
6634 else:
6635 self.wd = path
6636 return 1
6638 def cdup (self):
6639 try:
6640 i = self.wd[:-1].rindex('/')
6641 self.wd = self.wd[0:i+1]
6642 except ValueError:
6643 self.wd = '/'
6644 return 1
6646 def listdir (self, path, long=0):
6647 raise "Not implemented"
6649 # TODO: implement a cache w/timeout for stat()
6650 def stat (self, path):
6651 fullpath = join_paths(self.wd, path)
6652 if self.isfile(path):
6653 size = self.m[fullpath].file_size
6654 return (33188, 77396L, 10L, 1, 1000, 1000, size, 0,0,0)
6655 elif self.isdir(path):
6656 return (16895, 117481L, 10L, 20, 1000, 1000, 4096L, 0,0,0)
6657 else:
6658 raise "No such file or directory "+path
6660 def open (self, path, mode):
6661 class zFile:
6662 def __init__(self, content):
6663 self.content = content
6664 self.pos = 0
6665 self.len = len(content)
6666 def read(self,l=None):
6667 if l is None:
6668 l = self.len - self.pos
6669 if self.len < self.pos + l:
6670 l = self.len - self.pos
6671 s = self.content[self.pos : self.pos + l]
6672 self.pos += l
6673 return s
6674 def close(self):
6675 del self.content
6676 del self.len
6677 del self.pos
6678 self.lock.acquire()
6679 try:
6680 data = self.z.read(path)
6681 finally:
6682 self.lock.release()
6683 return zFile(data)
6685 def unlink (self, path):
6686 raise "Not implemented"
6687 def mkdir (self, path):
6688 raise "Not implemented"
6689 def rmdir (self, path):
6690 raise "Not implemented"
6692 def longify (self, (path, stat_info)):
6693 return unix_longify (path, stat_info)
6695 def __repr__ (self):
6696 return '<zipfile fs root:%s wd:%s>' % (self.filename, self.wd)
6699 def setBase(base):
6700 global GLOBAL_ROOT_DIR
6701 GLOBAL_ROOT_DIR = qualify_path(base)
6703 def setTempDir(tempdir):
6704 global GLOBAL_TEMP_DIR
6705 GLOBAL_TEMP_DIR = qualify_path(tempdir)
6707 def addMacroResolver(m):
6708 global macroresolvers
6709 macroresolvers += [m]
6711 def addTranslator(m):
6712 global translators
6713 translators += [m]
6715 def addFTPHandler(m):
6716 global ftphandlers
6717 ftphandlers += [m]
6719 def addContext(webpath, localpath):
6720 global contexts
6721 c = WebContext(webpath, localpath)
6722 contexts += [c]
6723 return c
6725 def flush():
6726 global contexts,translators,ftphandlers,macroresolvers,global_modules
6727 contexts[:] = []
6728 translators[:] = []
6729 ftphandlers[:] = []
6730 macroresolvers[:] = []
6731 global_modules.clear()
6732 _purge_all_modules()
6734 def addFileStore(webpath, localpaths):
6735 global contexts
6736 if len(webpath) and webpath[0] != '/':
6737 webpath = "/" + webpath
6738 c = FileStore(webpath, localpaths)
6739 contexts += [c]
6740 return c
6742 def setThreads(number):
6743 global number_of_threads
6744 global multithreading_enabled
6745 if number>1:
6746 multithreading_enabled=1
6747 number_of_threads=number
6748 else:
6749 multithreading_enabled=0
6750 number_of_threads=1
6752 def run(port=8081):
6753 check_date()
6754 ph = AthanaHandler()
6755 hs = http_server ('', port, logger_object = lg)
6756 hs.install_handler (ph)
6758 if len(ftphandlers) > 0:
6759 ftp = ftp_server (virtual_authorizer(), port=8021, logger_object=lg)
6761 if multithreading_enabled:
6762 threadlist = []
6763 for i in range(number_of_threads):
6764 threadlist += [thread.start_new_thread(worker_thread, (ph,))]
6766 while 1:
6767 try:
6768 asyncore.loop(timeout=0.01)
6769 except select.error:
6770 continue
6773 TODO:
6774 * session clearup
6775 * temp directory in .cfg file
6778 def setTempDir(path):
6779 global GLOBAL_TEMP_DIR
6780 GLOBAL_TEMP_DIR = path
6782 def mainfunction():
6783 global verbose,port,init_file,log_file,temp_path,multithreading_enabled,number_of_threads,GLOBAL_TEMP_DIR,contexts,lg,lgerr
6784 os.putenv('ATHANA_VERSION',ATHANA_VERSION)
6786 from optparse import OptionParser
6788 parser = OptionParser()
6790 parser.add_option("-v", "--verbose", dest="verbose", help="Be more verbose", action="store_true")
6791 parser.add_option("-q", "--quiet", dest="quiet", help="Be quiet", action="store_true")
6792 parser.add_option("-d", "--debug", dest="debug", help="Turn on debugging", action="store_true")
6793 parser.add_option("-p", "--port", dest="port", help="Set the port number", action="store",type="string")
6794 parser.add_option("-i", "--init-file", dest="init", help="Set the init file to use",action="store",type="string")
6795 parser.add_option("-l", "--log-file", dest="log", help="Set the logging file to use",action="store",type="string")
6796 parser.add_option("-t", "--temp-path", dest="temp", help="Set the temporary directory (default: /tmp/)",action="store",type="string")
6797 parser.add_option("-m", "--multithread", dest="multithreading_enabled", help="Enable multithreading",action="store_true")
6798 parser.add_option("-n", "--number-of-threads", dest="threads", help="Number of threads",action="store",type="int")
6799 parser.add_option("-T", "--talfile", dest="talfile", help="execute TAL File",action="store",type="string")
6801 (options, args) = parser.parse_args()
6803 verbose = 0
6804 init_file="web.cfg"
6805 log_file=None
6806 temp_path="/tmp/"
6807 port=8081
6809 if options.verbose != None : verbose = 2
6810 if options.quiet != None : verbose = 0
6811 if options.debug != None : verbose = 3
6812 if options.port != None : port = int(options.port)
6813 if options.init != None : init_file = options.init
6814 if options.log != None : log_file = options.log
6815 if options.temp != None : GLOBAL_TEMP_DIR = options.temp
6816 if options.multithreading_enabled : multithreading_enabled = 1
6817 if options.threads != None : number_of_threads = options.threads
6819 if options.talfile:
6820 print getTAL(options.talfile, {"mynone":None})
6821 sys.exit(0)
6823 if inifile:
6824 contexts += read_ini_file(inifile)
6826 if logfile is not None:
6827 fi = open(logfile, "wb")
6828 lg = file_logger (fi)
6829 lgerr = lg
6831 print "-"*72
6832 if multithreading_enabled:
6833 print "Starting Athana (%d threads)..." % number_of_threads
6834 else:
6835 print "Starting Athana..."
6836 print "Init-File:",init_file
6837 print "Log-File:",log_file
6838 print "Temp-Path:",GLOBAL_TEMP_DIR
6839 print "-"*72
6841 run(port)
6843 if __name__ == '__main__':
6844 import athana
6845 athana.mainfunction()