Lib/test/test_htmlparser.py

   1 """Tests for HTMLParser.py."""
   2
   3 import HTMLParser
   4 import pprint
   5 import sys
   6 import test_support
   7 import unittest
   8
   9
  10 class EventCollector(HTMLParser.HTMLParser):
  11
  12     def __init__(self):
  13         self.events = []
  14         self.append = self.events.append
  15         HTMLParser.HTMLParser.__init__(self)
  16
  17     def get_events(self):
  18         # Normalize the list of events so that buffer artefacts don't
  19         # separate runs of contiguous characters.
  20         L = []
  21         prevtype = None
  22         for event in self.events:
  23             type = event[0]
  24             if type == prevtype == "data":
  25                 L[-1] = ("data", L[-1][1] + event[1])
  26             else:
  27                 L.append(event)
  28             prevtype = type
  29         self.events = L
  30         return L
  31
  32     # structure markup
  33
  34     def handle_starttag(self, tag, attrs):
  35         self.append(("starttag", tag, attrs))
  36
  37     def handle_startendtag(self, tag, attrs):
  38         self.append(("startendtag", tag, attrs))
  39
  40     def handle_endtag(self, tag):
  41         self.append(("endtag", tag))
  42
  43     # all other markup
  44
  45     def handle_comment(self, data):
  46         self.append(("comment", data))
  47
  48     def handle_charref(self, data):
  49         self.append(("charref", data))
  50
  51     def handle_data(self, data):
  52         self.append(("data", data))
  53
  54     def handle_decl(self, data):
  55         self.append(("decl", data))
  56
  57     def handle_entityref(self, data):
  58         self.append(("entityref", data))
  59
  60     def handle_pi(self, data):
  61         self.append(("pi", data))
  62
  63     def unknown_decl(self, decl):
  64         self.append(("unknown decl", decl))
  65
  66
  67 class EventCollectorExtra(EventCollector):
  68
  69     def handle_starttag(self, tag, attrs):
  70         EventCollector.handle_starttag(self, tag, attrs)
  71         self.append(("starttag_text", self.get_starttag_text()))
  72
  73
  74 class TestCaseBase(unittest.TestCase):
  75
  76     def _run_check(self, source, expected_events, collector=EventCollector):
  77         parser = collector()
  78         for s in source:
  79             parser.feed(s)
  80         parser.close()
  81         events = parser.get_events()
  82         if events != expected_events:
  83             self.fail("received events did not match expected events\n"
  84                       "Expected:\n" + pprint.pformat(expected_events) +
  85                       "\nReceived:\n" + pprint.pformat(events))
  86
  87     def _run_check_extra(self, source, events):
  88         self._run_check(source, events, EventCollectorExtra)
  89
  90     def _parse_error(self, source):
  91         def parse(source=source):
  92             parser = HTMLParser.HTMLParser()
  93             parser.feed(source)
  94             parser.close()
  95         self.assertRaises(HTMLParser.HTMLParseError, parse)
  96
  97
  98 class HTMLParserTestCase(TestCaseBase):
  99
 100     def test_processing_instruction_only(self):
 101         self._run_check("<?processing instruction>", [
 102             ("pi", "processing instruction"),
 103             ])
 104
 105     def test_simple_html(self):
 106         self._run_check("""
 107 <!DOCTYPE html PUBLIC 'foo'>
 108 <HTML>&entity;&#32;
 109 <!--comment1a
 110 -></foo><bar>&lt;<?pi?></foo<bar
 111 comment1b-->
 112 <Img sRc='Bar' isMAP>sample
 113 text
 114 &#x201C;
 115 <!--comment2a-- --comment2b-->
 116 </Html>
 117 """, [
 118     ("data", "\n"),
 119     ("decl", "DOCTYPE html PUBLIC 'foo'"),
 120     ("data", "\n"),
 121     ("starttag", "html", []),
 122     ("entityref", "entity"),
 123     ("charref", "32"),
 124     ("data", "\n"),
 125     ("comment", "comment1a\n-></foo><bar>&lt;<?pi?></foo<bar\ncomment1b"),
 126     ("data", "\n"),
 127     ("starttag", "img", [("src", "Bar"), ("ismap", None)]),
 128     ("data", "sample\ntext\n"),
 129     ("charref", "x201C"),
 130     ("data", "\n"),
 131     ("comment", "comment2a-- --comment2b"),
 132     ("data", "\n"),
 133     ("endtag", "html"),
 134     ("data", "\n"),
 135     ])
 136
 137     def test_unclosed_entityref(self):
 138         self._run_check("&entityref foo", [
 139             ("entityref", "entityref"),
 140             ("data", " foo"),
 141             ])
 142
 143     def test_doctype_decl(self):
 144         inside = """\
 145 DOCTYPE html [
 146   <!ELEMENT html - O EMPTY>
 147   <!ATTLIST html
 148       version CDATA #IMPLIED
 149       profile CDATA 'DublinCore'>
 150   <!NOTATION datatype SYSTEM 'http://xml.python.org/notations/python-module'>
 151   <!ENTITY myEntity 'internal parsed entity'>
 152   <!ENTITY anEntity SYSTEM 'http://xml.python.org/entities/something.xml'>
 153   <!ENTITY % paramEntity 'name|name|name'>
 154   %paramEntity;
 155   <!-- comment -->
 156 ]"""
 157         self._run_check("<!%s>" % inside, [
 158             ("decl", inside),
 159             ])
 160
 161     def test_bad_nesting(self):
 162         # Strangely, this *is* supposed to test that overlapping
 163         # elements are allowed.  HTMLParser is more geared toward
 164         # lexing the input that parsing the structure.
 165         self._run_check("<a><b></a></b>", [
 166             ("starttag", "a", []),
 167             ("starttag", "b", []),
 168             ("endtag", "a"),
 169             ("endtag", "b"),
 170             ])
 171
 172     def test_bare_ampersands(self):
 173         self._run_check("this text & contains & ampersands &", [
 174             ("data", "this text & contains & ampersands &"),
 175             ])
 176
 177     def test_bare_pointy_brackets(self):
 178         self._run_check("this < text > contains < bare>pointy< brackets", [
 179             ("data", "this < text > contains < bare>pointy< brackets"),
 180             ])
 181
 182     def test_attr_syntax(self):
 183         output = [
 184           ("starttag", "a", [("b", "v"), ("c", "v"), ("d", "v"), ("e", None)])
 185           ]
 186         self._run_check("""<a b='v' c="v" d=v e>""", output)
 187         self._run_check("""<a  b = 'v' c = "v" d = v e>""", output)
 188         self._run_check("""<a\nb\n=\n'v'\nc\n=\n"v"\nd\n=\nv\ne>""", output)
 189         self._run_check("""<a\tb\t=\t'v'\tc\t=\t"v"\td\t=\tv\te>""", output)
 190
 191     def test_attr_values(self):
 192         self._run_check("""<a b='xxx\n\txxx' c="yyy\t\nyyy" d='\txyz\n'>""",
 193                         [("starttag", "a", [("b", "xxx\n\txxx"),
 194                                             ("c", "yyy\t\nyyy"),
 195                                             ("d", "\txyz\n")])
 196                          ])
 197         self._run_check("""<a b='' c="">""", [
 198             ("starttag", "a", [("b", ""), ("c", "")]),
 199             ])
 200
 201     def test_attr_entity_replacement(self):
 202         self._run_check("""<a b='&amp;&gt;&lt;&quot;&apos;'>""", [
 203             ("starttag", "a", [("b", "&><\"'")]),
 204             ])
 205
 206     def test_attr_funky_names(self):
 207         self._run_check("""<a a.b='v' c:d=v e-f=v>""", [
 208             ("starttag", "a", [("a.b", "v"), ("c:d", "v"), ("e-f", "v")]),
 209             ])
 210
 211     def test_illegal_declarations(self):
 212         self._parse_error('<!spacer type="block" height="25">')
 213
 214     def test_starttag_end_boundary(self):
 215         self._run_check("""<a b='<'>""", [("starttag", "a", [("b", "<")])])
 216         self._run_check("""<a b='>'>""", [("starttag", "a", [("b", ">")])])
 217
 218     def test_buffer_artefacts(self):
 219         output = [("starttag", "a", [("b", "<")])]
 220         self._run_check(["<a b='<'>"], output)
 221         self._run_check(["<a ", "b='<'>"], output)
 222         self._run_check(["<a b", "='<'>"], output)
 223         self._run_check(["<a b=", "'<'>"], output)
 224         self._run_check(["<a b='<", "'>"], output)
 225         self._run_check(["<a b='<'", ">"], output)
 226
 227         output = [("starttag", "a", [("b", ">")])]
 228         self._run_check(["<a b='>'>"], output)
 229         self._run_check(["<a ", "b='>'>"], output)
 230         self._run_check(["<a b", "='>'>"], output)
 231         self._run_check(["<a b=", "'>'>"], output)
 232         self._run_check(["<a b='>", "'>"], output)
 233         self._run_check(["<a b='>'", ">"], output)
 234
 235     def test_starttag_junk_chars(self):
 236         self._parse_error("</>")
 237         self._parse_error("</$>")
 238         self._parse_error("</")
 239         self._parse_error("</a")
 240         self._parse_error("<a<a>")
 241         self._parse_error("</a<a>")
 242         self._parse_error("<!")
 243         self._parse_error("<a $>")
 244         self._parse_error("<a")
 245         self._parse_error("<a foo='bar'")
 246         self._parse_error("<a foo='bar")
 247         self._parse_error("<a foo='>'")
 248         self._parse_error("<a foo='>")
 249         self._parse_error("<a foo=>")
 250
 251     def test_declaration_junk_chars(self):
 252         self._parse_error("<!DOCTYPE foo $ >")
 253
 254     def test_startendtag(self):
 255         self._run_check("<p/>", [
 256             ("startendtag", "p", []),
 257             ])
 258         self._run_check("<p></p>", [
 259             ("starttag", "p", []),
 260             ("endtag", "p"),
 261             ])
 262         self._run_check("<p><img src='foo' /></p>", [
 263             ("starttag", "p", []),
 264             ("startendtag", "img", [("src", "foo")]),
 265             ("endtag", "p"),
 266             ])
 267
 268     def test_get_starttag_text(self):
 269         s = """<foo:bar   \n   one="1"\ttwo=2   >"""
 270         self._run_check_extra(s, [
 271             ("starttag", "foo:bar", [("one", "1"), ("two", "2")]),
 272             ("starttag_text", s)])
 273
 274     def test_cdata_content(self):
 275         s = """<script> <!-- not a comment --> &not-an-entity-ref; </script>"""
 276         self._run_check(s, [
 277             ("starttag", "script", []),
 278             ("data", " <!-- not a comment --> &not-an-entity-ref; "),
 279             ("endtag", "script"),
 280             ])
 281         s = """<script> <not a='start tag'> </script>"""
 282         self._run_check(s, [
 283             ("starttag", "script", []),
 284             ("data", " <not a='start tag'> "),
 285             ("endtag", "script"),
 286             ])
 287
 288
 289 def test_main():
 290     test_support.run_unittest(HTMLParserTestCase)
 291
 292
 293 if __name__ == "__main__":
 294     test_main()