libs/elementtree/HTMLTreeBuilder.py

   1 #
   2 # ElementTree
   3 # $Id: HTMLTreeBuilder.py 2325 2005-03-16 15:50:43Z fredrik $
   4 #
   5 # a simple tree builder, for HTML input
   6 #
   7 # history:
   8 # 2002-04-06 fl   created
   9 # 2002-04-07 fl   ignore IMG and HR end tags
  10 # 2002-04-07 fl   added support for 1.5.2 and later
  11 # 2003-04-13 fl   added HTMLTreeBuilder alias
  12 # 2004-12-02 fl   don't feed non-ASCII charrefs/entities as 8-bit strings
  13 # 2004-12-05 fl   don't feed non-ASCII CDATA as 8-bit strings
  14 #
  15 # Copyright (c) 1999-2004 by Fredrik Lundh.  All rights reserved.
  16 #
  17 # fredrik@pythonware.com
  18 # http://www.pythonware.com
  19 #
  20 # --------------------------------------------------------------------
  21 # The ElementTree toolkit is
  22 #
  23 # Copyright (c) 1999-2004 by Fredrik Lundh
  24 #
  25 # By obtaining, using, and/or copying this software and/or its
  26 # associated documentation, you agree that you have read, understood,
  27 # and will comply with the following terms and conditions:
  28 #
  29 # Permission to use, copy, modify, and distribute this software and
  30 # its associated documentation for any purpose and without fee is
  31 # hereby granted, provided that the above copyright notice appears in
  32 # all copies, and that both that copyright notice and this permission
  33 # notice appear in supporting documentation, and that the name of
  34 # Secret Labs AB or the author not be used in advertising or publicity
  35 # pertaining to distribution of the software without specific, written
  36 # prior permission.
  37 #
  38 # SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD
  39 # TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANT-
  40 # ABILITY AND FITNESS.  IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR
  41 # BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY
  42 # DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
  43 # WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
  44 # ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
  45 # OF THIS SOFTWARE.
  46 # --------------------------------------------------------------------
  47
  48 ##
  49 # Tools to build element trees from HTML files.
  50 ##
  51
  52 import htmlentitydefs
  53 import re, string, sys
  54 import mimetools, StringIO
  55
  56 import ElementTree
  57
  58 AUTOCLOSE = "p", "li", "tr", "th", "td", "head", "body"
  59 IGNOREEND = "img", "hr", "meta", "link", "br"
  60
  61 if sys.version[:3] == "1.5":
  62     is_not_ascii = re.compile(r"[\x80-\xff]").search # 1.5.2
  63 else:
  64     is_not_ascii = re.compile(eval(r'u"[\u0080-\uffff]"')).search
  65
  66 try:
  67     from HTMLParser import HTMLParser
  68 except ImportError:
  69     from sgmllib import SGMLParser
  70     # hack to use sgmllib's SGMLParser to emulate 2.2's HTMLParser
  71     class HTMLParser(SGMLParser):
  72         # the following only works as long as this class doesn't
  73         # provide any do, start, or end handlers
  74         def unknown_starttag(self, tag, attrs):
  75             self.handle_starttag(tag, attrs)
  76         def unknown_endtag(self, tag):
  77             self.handle_endtag(tag)
  78
  79 ##
  80 # ElementTree builder for HTML source code.  This builder converts an
  81 # HTML document or fragment to an ElementTree.
  82 # <p>
  83 # The parser is relatively picky, and requires balanced tags for most
  84 # elements.  However, elements belonging to the following group are
  85 # automatically closed: P, LI, TR, TH, and TD.  In addition, the
  86 # parser automatically inserts end tags immediately after the start
  87 # tag, and ignores any end tags for the following group: IMG, HR,
  88 # META, and LINK.
  89 #
  90 # @keyparam builder Optional builder object.  If omitted, the parser
  91 #     uses the standard <b>elementtree</b> builder.
  92 # @keyparam encoding Optional character encoding, if known.  If omitted,
  93 #     the parser looks for META tags inside the document.  If no tags
  94 #     are found, the parser defaults to ISO-8859-1.  Note that if your
  95 #     document uses a non-ASCII compatible encoding, you must decode
  96 #     the document before parsing.
  97 #
  98 # @see elementtree.ElementTree
  99
 100 class HTMLTreeBuilder(HTMLParser):
 101
 102     # FIXME: shouldn't this class be named Parser, not Builder?
 103
 104     def __init__(self, builder=None, encoding=None):
 105         self.__stack = []
 106         if builder is None:
 107             builder = ElementTree.TreeBuilder()
 108         self.__builder = builder
 109         self.encoding = encoding or "iso-8859-1"
 110         HTMLParser.__init__(self)
 111
 112     ##
 113     # Flushes parser buffers, and return the root element.
 114     #
 115     # @return An Element instance.
 116
 117     def close(self):
 118         HTMLParser.close(self)
 119         return self.__builder.close()
 120
 121     ##
 122     # (Internal) Handles start tags.
 123
 124     def handle_starttag(self, tag, attrs):
 125         if tag == "meta":
 126             # look for encoding directives
 127             http_equiv = content = None
 128             for k, v in attrs:
 129                 if k == "http-equiv":
 130                     http_equiv = string.lower(v)
 131                 elif k == "content":
 132                     content = v
 133             if http_equiv == "content-type" and content:
 134                 # use mimetools to parse the http header
 135                 header = mimetools.Message(
 136                     StringIO.StringIO("%s: %s\n\n" % (http_equiv, content))
 137                     )
 138                 encoding = header.getparam("charset")
 139                 if encoding:
 140                     self.encoding = encoding
 141         if tag in AUTOCLOSE:
 142             if self.__stack and self.__stack[-1] == tag:
 143                 self.handle_endtag(tag)
 144         self.__stack.append(tag)
 145         attrib = {}
 146         if attrs:
 147             for k, v in attrs:
 148                 attrib[string.lower(k)] = v
 149         self.__builder.start(tag, attrib)
 150         if tag in IGNOREEND:
 151             self.__stack.pop()
 152             self.__builder.end(tag)
 153
 154     ##
 155     # (Internal) Handles end tags.
 156
 157     def handle_endtag(self, tag):
 158         if tag in IGNOREEND:
 159             return
 160         lasttag = self.__stack.pop()
 161         if tag != lasttag and lasttag in AUTOCLOSE:
 162             self.handle_endtag(lasttag)
 163         self.__builder.end(tag)
 164
 165     ##
 166     # (Internal) Handles character references.
 167
 168     def handle_charref(self, char):
 169         if char[:1] == "x":
 170             char = int(char[1:], 16)
 171         else:
 172             char = int(char)
 173         if 0 <= char < 128:
 174             self.__builder.data(chr(char))
 175         else:
 176             self.__builder.data(unichr(char))
 177
 178     ##
 179     # (Internal) Handles entity references.
 180
 181     def handle_entityref(self, name):
 182         entity = htmlentitydefs.entitydefs.get(name)
 183         if entity:
 184             if len(entity) == 1:
 185                 entity = ord(entity)
 186             else:
 187                 entity = int(entity[2:-1])
 188             if 0 <= entity < 128:
 189                 self.__builder.data(chr(entity))
 190             else:
 191                 self.__builder.data(unichr(entity))
 192         else:
 193             self.unknown_entityref(name)
 194
 195     ##
 196     # (Internal) Handles character data.
 197
 198     def handle_data(self, data):
 199         if isinstance(data, type('')) and is_not_ascii(data):
 200             # convert to unicode, but only if necessary
 201             data = unicode(data, self.encoding, "ignore")
 202         self.__builder.data(data)
 203
 204     ##
 205     # (Hook) Handles unknown entity references.  The default action
 206     # is to ignore unknown entities.
 207
 208     def unknown_entityref(self, name):
 209         pass # ignore by default; override if necessary
 210
 211 ##
 212 # An alias for the <b>HTMLTreeBuilder</b> class.
 213
 214 TreeBuilder = HTMLTreeBuilder
 215
 216 ##
 217 # Parse an HTML document or document fragment.
 218 #
 219 # @param source A filename or file object containing HTML data.
 220 # @param encoding Optional character encoding, if known.  If omitted,
 221 #     the parser looks for META tags inside the document.  If no tags
 222 #     are found, the parser defaults to ISO-8859-1.
 223 # @return An ElementTree instance
 224
 225 def parse(source, encoding=None):
 226     return ElementTree.parse(source, HTMLTreeBuilder(encoding=encoding))
 227
 228 if __name__ == "__main__":
 229     import sys
 230     ElementTree.dump(parse(open(sys.argv[1])))