Lib/test/test_sgmllib.py

   1 import pprint
   2 import sgmllib
   3 import unittest
   4 from test import test_support
   5
   6
   7 class EventCollector(sgmllib.SGMLParser):
   8
   9     def __init__(self):
  10         self.events = []
  11         self.append = self.events.append
  12         sgmllib.SGMLParser.__init__(self)
  13
  14     def get_events(self):
  15         # Normalize the list of events so that buffer artefacts don't
  16         # separate runs of contiguous characters.
  17         L = []
  18         prevtype = None
  19         for event in self.events:
  20             type = event[0]
  21             if type == prevtype == "data":
  22                 L[-1] = ("data", L[-1][1] + event[1])
  23             else:
  24                 L.append(event)
  25             prevtype = type
  26         self.events = L
  27         return L
  28
  29     # structure markup
  30
  31     def unknown_starttag(self, tag, attrs):
  32         self.append(("starttag", tag, attrs))
  33
  34     def unknown_endtag(self, tag):
  35         self.append(("endtag", tag))
  36
  37     # all other markup
  38
  39     def handle_comment(self, data):
  40         self.append(("comment", data))
  41
  42     def handle_charref(self, data):
  43         self.append(("charref", data))
  44
  45     def handle_data(self, data):
  46         self.append(("data", data))
  47
  48     def handle_decl(self, decl):
  49         self.append(("decl", decl))
  50
  51     def handle_entityref(self, data):
  52         self.append(("entityref", data))
  53
  54     def handle_pi(self, data):
  55         self.append(("pi", data))
  56
  57     def unknown_decl(self, decl):
  58         self.append(("unknown decl", decl))
  59
  60
  61 class CDATAEventCollector(EventCollector):
  62     def start_cdata(self, attrs):
  63         self.append(("starttag", "cdata", attrs))
  64         self.setliteral()
  65
  66
  67 class SGMLParserTestCase(unittest.TestCase):
  68
  69     collector = EventCollector
  70
  71     def get_events(self, source):
  72         parser = self.collector()
  73         try:
  74             for s in source:
  75                 parser.feed(s)
  76             parser.close()
  77         except:
  78             #self.events = parser.events
  79             raise
  80         return parser.get_events()
  81
  82     def check_events(self, source, expected_events):
  83         try:
  84             events = self.get_events(source)
  85         except:
  86             import sys
  87             #print >>sys.stderr, pprint.pformat(self.events)
  88             raise
  89         if events != expected_events:
  90             self.fail("received events did not match expected events\n"
  91                       "Expected:\n" + pprint.pformat(expected_events) +
  92                       "\nReceived:\n" + pprint.pformat(events))
  93
  94     def check_parse_error(self, source):
  95         parser = EventCollector()
  96         try:
  97             parser.feed(source)
  98             parser.close()
  99         except sgmllib.SGMLParseError:
 100             pass
 101         else:
 102             self.fail("expected SGMLParseError for %r\nReceived:\n%s"
 103                       % (source, pprint.pformat(parser.get_events())))
 104
 105     def test_doctype_decl_internal(self):
 106         inside = """\
 107 DOCTYPE html PUBLIC '-//W3C//DTD HTML 4.01//EN'
 108              SYSTEM 'http://www.w3.org/TR/html401/strict.dtd' [
 109   <!ELEMENT html - O EMPTY>
 110   <!ATTLIST html
 111       version CDATA #IMPLIED
 112       profile CDATA 'DublinCore'>
 113   <!NOTATION datatype SYSTEM 'http://xml.python.org/notations/python-module'>
 114   <!ENTITY myEntity 'internal parsed entity'>
 115   <!ENTITY anEntity SYSTEM 'http://xml.python.org/entities/something.xml'>
 116   <!ENTITY % paramEntity 'name|name|name'>
 117   %paramEntity;
 118   <!-- comment -->
 119 ]"""
 120         self.check_events(["<!%s>" % inside], [
 121             ("decl", inside),
 122             ])
 123
 124     def test_doctype_decl_external(self):
 125         inside = "DOCTYPE html PUBLIC '-//W3C//DTD HTML 4.01//EN'"
 126         self.check_events("<!%s>" % inside, [
 127             ("decl", inside),
 128             ])
 129
 130     def test_underscore_in_attrname(self):
 131         # SF bug #436621
 132         """Make sure attribute names with underscores are accepted"""
 133         self.check_events("<a has_under _under>", [
 134             ("starttag", "a", [("has_under", "has_under"),
 135                                ("_under", "_under")]),
 136             ])
 137
 138     def test_underscore_in_tagname(self):
 139         # SF bug #436621
 140         """Make sure tag names with underscores are accepted"""
 141         self.check_events("<has_under></has_under>", [
 142             ("starttag", "has_under", []),
 143             ("endtag", "has_under"),
 144             ])
 145
 146     def test_quotes_in_unquoted_attrs(self):
 147         # SF bug #436621
 148         """Be sure quotes in unquoted attributes are made part of the value"""
 149         self.check_events("<a href=foo'bar\"baz>", [
 150             ("starttag", "a", [("href", "foo'bar\"baz")]),
 151             ])
 152
 153     def test_xhtml_empty_tag(self):
 154         """Handling of XHTML-style empty start tags"""
 155         self.check_events("<br />text<i></i>", [
 156             ("starttag", "br", []),
 157             ("data", "text"),
 158             ("starttag", "i", []),
 159             ("endtag", "i"),
 160             ])
 161
 162     def test_processing_instruction_only(self):
 163         self.check_events("<?processing instruction>", [
 164             ("pi", "processing instruction"),
 165             ])
 166
 167     def test_bad_nesting(self):
 168         self.check_events("<a><b></a></b>", [
 169             ("starttag", "a", []),
 170             ("starttag", "b", []),
 171             ("endtag", "a"),
 172             ("endtag", "b"),
 173             ])
 174
 175     def test_bare_ampersands(self):
 176         self.check_events("this text & contains & ampersands &", [
 177             ("data", "this text & contains & ampersands &"),
 178             ])
 179
 180     def test_bare_pointy_brackets(self):
 181         self.check_events("this < text > contains < bare>pointy< brackets", [
 182             ("data", "this < text > contains < bare>pointy< brackets"),
 183             ])
 184
 185     def test_attr_syntax(self):
 186         output = [
 187           ("starttag", "a", [("b", "v"), ("c", "v"), ("d", "v"), ("e", "e")])
 188           ]
 189         self.check_events("""<a b='v' c="v" d=v e>""", output)
 190         self.check_events("""<a  b = 'v' c = "v" d = v e>""", output)
 191         self.check_events("""<a\nb\n=\n'v'\nc\n=\n"v"\nd\n=\nv\ne>""", output)
 192         self.check_events("""<a\tb\t=\t'v'\tc\t=\t"v"\td\t=\tv\te>""", output)
 193
 194     def test_attr_values(self):
 195         self.check_events("""<a b='xxx\n\txxx' c="yyy\t\nyyy" d='\txyz\n'>""",
 196                         [("starttag", "a", [("b", "xxx\n\txxx"),
 197                                             ("c", "yyy\t\nyyy"),
 198                                             ("d", "\txyz\n")])
 199                          ])
 200         self.check_events("""<a b='' c="">""", [
 201             ("starttag", "a", [("b", ""), ("c", "")]),
 202             ])
 203         # Regression test for SF patch #669683.
 204         self.check_events("<e a=rgb(1,2,3)>", [
 205             ("starttag", "e", [("a", "rgb(1,2,3)")]),
 206             ])
 207
 208     def test_attr_funky_names(self):
 209         self.check_events("""<a a.b='v' c:d=v e-f=v>""", [
 210             ("starttag", "a", [("a.b", "v"), ("c:d", "v"), ("e-f", "v")]),
 211             ])
 212
 213     def test_illegal_declarations(self):
 214         s = 'abc<!spacer type="block" height="25">def'
 215         self.check_events(s, [
 216             ("data", "abc"),
 217             ("unknown decl", 'spacer type="block" height="25"'),
 218             ("data", "def"),
 219             ])
 220
 221     def test_weird_starttags(self):
 222         self.check_events("<a<a>", [
 223             ("starttag", "a", []),
 224             ("starttag", "a", []),
 225             ])
 226         self.check_events("</a<a>", [
 227             ("endtag", "a"),
 228             ("starttag", "a", []),
 229             ])
 230
 231     def test_declaration_junk_chars(self):
 232         self.check_parse_error("<!DOCTYPE foo $ >")
 233
 234     def test_get_starttag_text(self):
 235         s = """<foobar   \n   one="1"\ttwo=2   >"""
 236         self.check_events(s, [
 237             ("starttag", "foobar", [("one", "1"), ("two", "2")]),
 238             ])
 239
 240     def test_cdata_content(self):
 241         s = ("<cdata> <!-- not a comment --> &not-an-entity-ref; </cdata>"
 242              "<notcdata> <!-- comment --> </notcdata>")
 243         self.collector = CDATAEventCollector
 244         self.check_events(s, [
 245             ("starttag", "cdata", []),
 246             ("data", " <!-- not a comment --> &not-an-entity-ref; "),
 247             ("endtag", "cdata"),
 248             ("starttag", "notcdata", []),
 249             ("data", " "),
 250             ("comment", " comment "),
 251             ("data", " "),
 252             ("endtag", "notcdata"),
 253             ])
 254         s = """<cdata> <not a='start tag'> </cdata>"""
 255         self.check_events(s, [
 256             ("starttag", "cdata", []),
 257             ("data", " <not a='start tag'> "),
 258             ("endtag", "cdata"),
 259             ])
 260
 261     def test_illegal_declarations(self):
 262         s = 'abc<!spacer type="block" height="25">def'
 263         self.check_events(s, [
 264             ("data", "abc"),
 265             ("unknown decl", 'spacer type="block" height="25"'),
 266             ("data", "def"),
 267             ])
 268
 269     def test_enumerated_attr_type(self):
 270         s = "<!DOCTYPE doc [<!ATTLIST doc attr (a | b) >]>"
 271         self.check_events(s, [
 272             ('decl', 'DOCTYPE doc [<!ATTLIST doc attr (a | b) >]'),
 273             ])
 274
 275     # XXX These tests have been disabled by prefixing their names with
 276     # an underscore.  The first two exercise outstanding bugs in the
 277     # sgmllib module, and the third exhibits questionable behavior
 278     # that needs to be carefully considered before changing it.
 279
 280     def _test_starttag_end_boundary(self):
 281         self.check_events("""<a b='<'>""", [("starttag", "a", [("b", "<")])])
 282         self.check_events("""<a b='>'>""", [("starttag", "a", [("b", ">")])])
 283
 284     def _test_buffer_artefacts(self):
 285         output = [("starttag", "a", [("b", "<")])]
 286         self.check_events(["<a b='<'>"], output)
 287         self.check_events(["<a ", "b='<'>"], output)
 288         self.check_events(["<a b", "='<'>"], output)
 289         self.check_events(["<a b=", "'<'>"], output)
 290         self.check_events(["<a b='<", "'>"], output)
 291         self.check_events(["<a b='<'", ">"], output)
 292
 293         output = [("starttag", "a", [("b", ">")])]
 294         self.check_events(["<a b='>'>"], output)
 295         self.check_events(["<a ", "b='>'>"], output)
 296         self.check_events(["<a b", "='>'>"], output)
 297         self.check_events(["<a b=", "'>'>"], output)
 298         self.check_events(["<a b='>", "'>"], output)
 299         self.check_events(["<a b='>'", ">"], output)
 300
 301     def _test_starttag_junk_chars(self):
 302         self.check_parse_error("<")
 303         self.check_parse_error("<>")
 304         self.check_parse_error("</$>")
 305         self.check_parse_error("</")
 306         self.check_parse_error("</a")
 307         self.check_parse_error("<$")
 308         self.check_parse_error("<$>")
 309         self.check_parse_error("<!")
 310         self.check_parse_error("<a $>")
 311         self.check_parse_error("<a")
 312         self.check_parse_error("<a foo='bar'")
 313         self.check_parse_error("<a foo='bar")
 314         self.check_parse_error("<a foo='>'")
 315         self.check_parse_error("<a foo='>")
 316         self.check_parse_error("<a foo=>")
 317
 318
 319 def test_main():
 320     test_support.run_unittest(SGMLParserTestCase)
 321
 322
 323 if __name__ == "__main__":
 324     test_main()