Lib/sgmllib.py

   1 """A parser for SGML, using the derived class as a static DTD."""
   2
   3 # XXX This only supports those SGML features used by HTML.
   4
   5 # XXX There should be a way to distinguish between PCDATA (parsed
   6 # character data -- the normal case), RCDATA (replaceable character
   7 # data -- only char and entity references and end tags are special)
   8 # and CDATA (character data -- only end tags are special).
   9
  10
  11 import re
  12 import string
  13
  14 __all__ = ["SGMLParser"]
  15
  16 # Regular expressions used for parsing
  17
  18 interesting = re.compile('[&<]')
  19 incomplete = re.compile('&([a-zA-Z][a-zA-Z0-9]*|#[0-9]*)?|'
  20                            '<([a-zA-Z][^<>]*|'
  21                               '/([a-zA-Z][^<>]*)?|'
  22                               '![^<>]*)?')
  23
  24 entityref = re.compile('&([a-zA-Z][-.a-zA-Z0-9]*)[^a-zA-Z0-9]')
  25 charref = re.compile('&#([0-9]+)[^0-9]')
  26
  27 starttagopen = re.compile('<[>a-zA-Z]')
  28 shorttagopen = re.compile('<[a-zA-Z][-.a-zA-Z0-9]*/')
  29 shorttag = re.compile('<([a-zA-Z][-.a-zA-Z0-9]*)/([^/]*)/')
  30 piopen = re.compile('<\?')
  31 piclose = re.compile('>')
  32 endtagopen = re.compile('</[<>a-zA-Z]')
  33 endbracket = re.compile('[<>]')
  34 special = re.compile('<![^<>]*>')
  35 commentopen = re.compile('<!--')
  36 commentclose = re.compile(r'--\s*>')
  37 tagfind = re.compile('[a-zA-Z][-.a-zA-Z0-9]*')
  38 attrfind = re.compile(
  39     r'\s*([a-zA-Z_][-.a-zA-Z_0-9]*)(\s*=\s*'
  40     r'(\'[^\']*\'|"[^"]*"|[-a-zA-Z0-9./:;+*%?!&$\(\)_#=~]*))?')
  41
  42 declname = re.compile(r'[a-zA-Z][-_.a-zA-Z0-9]*\s*')
  43 declstringlit = re.compile(r'(\'[^\']*\'|"[^"]*")\s*')
  44
  45
  46 class SGMLParseError(RuntimeError):
  47     """Exception raised for all parse errors."""
  48     pass
  49
  50
  51 # SGML parser base class -- find tags and call handler functions.
  52 # Usage: p = SGMLParser(); p.feed(data); ...; p.close().
  53 # The dtd is defined by deriving a class which defines methods
  54 # with special names to handle tags: start_foo and end_foo to handle
  55 # <foo> and </foo>, respectively, or do_foo to handle <foo> by itself.
  56 # (Tags are converted to lower case for this purpose.)  The data
  57 # between tags is passed to the parser by calling self.handle_data()
  58 # with some data as argument (the data may be split up in arbitrary
  59 # chunks).  Entity references are passed by calling
  60 # self.handle_entityref() with the entity reference as argument.
  61
  62 class SGMLParser:
  63
  64     # Interface -- initialize and reset this instance
  65     def __init__(self, verbose=0):
  66         self.verbose = verbose
  67         self.reset()
  68
  69     # Interface -- reset this instance.  Loses all unprocessed data
  70     def reset(self):
  71         self.rawdata = ''
  72         self.stack = []
  73         self.lasttag = '???'
  74         self.nomoretags = 0
  75         self.literal = 0
  76
  77     # For derived classes only -- enter literal mode (CDATA) till EOF
  78     def setnomoretags(self):
  79         self.nomoretags = self.literal = 1
  80
  81     # For derived classes only -- enter literal mode (CDATA)
  82     def setliteral(self, *args):
  83         self.literal = 1
  84
  85     # Interface -- feed some data to the parser.  Call this as
  86     # often as you want, with as little or as much text as you
  87     # want (may include '\n').  (This just saves the text, all the
  88     # processing is done by goahead().)
  89     def feed(self, data):
  90         self.rawdata = self.rawdata + data
  91         self.goahead(0)
  92
  93     # Interface -- handle the remaining data
  94     def close(self):
  95         self.goahead(1)
  96
  97     # Internal -- handle data as far as reasonable.  May leave state
  98     # and data to be processed by a subsequent call.  If 'end' is
  99     # true, force handling all data as if followed by EOF marker.
 100     def goahead(self, end):
 101         rawdata = self.rawdata
 102         i = 0
 103         n = len(rawdata)
 104         while i < n:
 105             if self.nomoretags:
 106                 self.handle_data(rawdata[i:n])
 107                 i = n
 108                 break
 109             match = interesting.search(rawdata, i)
 110             if match: j = match.start(0)
 111             else: j = n
 112             if i < j: self.handle_data(rawdata[i:j])
 113             i = j
 114             if i == n: break
 115             if rawdata[i] == '<':
 116                 if starttagopen.match(rawdata, i):
 117                     if self.literal:
 118                         self.handle_data(rawdata[i])
 119                         i = i+1
 120                         continue
 121                     k = self.parse_starttag(i)
 122                     if k < 0: break
 123                     i = k
 124                     continue
 125                 if endtagopen.match(rawdata, i):
 126                     k = self.parse_endtag(i)
 127                     if k < 0: break
 128                     i =  k
 129                     self.literal = 0
 130                     continue
 131                 if commentopen.match(rawdata, i):
 132                     if self.literal:
 133                         self.handle_data(rawdata[i])
 134                         i = i+1
 135                         continue
 136                     k = self.parse_comment(i)
 137                     if k < 0: break
 138                     i = i+k
 139                     continue
 140                 if piopen.match(rawdata, i):
 141                     if self.literal:
 142                         self.handle_data(rawdata[i])
 143                         i = i+1
 144                         continue
 145                     k = self.parse_pi(i)
 146                     if k < 0: break
 147                     i = i+k
 148                     continue
 149                 match = special.match(rawdata, i)
 150                 if match:
 151                     if self.literal:
 152                         self.handle_data(rawdata[i])
 153                         i = i+1
 154                         continue
 155                     # This is some sort of declaration; in "HTML as
 156                     # deployed," this should only be the document type
 157                     # declaration ("<!DOCTYPE html...>").
 158                     k = self.parse_declaration(i)
 159                     if k < 0: break
 160                     i = k
 161                     continue
 162             elif rawdata[i] == '&':
 163                 match = charref.match(rawdata, i)
 164                 if match:
 165                     name = match.group(1)
 166                     self.handle_charref(name)
 167                     i = match.end(0)
 168                     if rawdata[i-1] != ';': i = i-1
 169                     continue
 170                 match = entityref.match(rawdata, i)
 171                 if match:
 172                     name = match.group(1)
 173                     self.handle_entityref(name)
 174                     i = match.end(0)
 175                     if rawdata[i-1] != ';': i = i-1
 176                     continue
 177             else:
 178                 raise SGMLParseError('neither < nor & ??')
 179             # We get here only if incomplete matches but
 180             # nothing else
 181             match = incomplete.match(rawdata, i)
 182             if not match:
 183                 self.handle_data(rawdata[i])
 184                 i = i+1
 185                 continue
 186             j = match.end(0)
 187             if j == n:
 188                 break # Really incomplete
 189             self.handle_data(rawdata[i:j])
 190             i = j
 191         # end while
 192         if end and i < n:
 193             self.handle_data(rawdata[i:n])
 194             i = n
 195         self.rawdata = rawdata[i:]
 196         # XXX if end: check for empty stack
 197
 198     # Internal -- parse comment, return length or -1 if not terminated
 199     def parse_comment(self, i):
 200         rawdata = self.rawdata
 201         if rawdata[i:i+4] != '<!--':
 202             raise SGMLParseError('unexpected call to parse_comment()')
 203         match = commentclose.search(rawdata, i+4)
 204         if not match:
 205             return -1
 206         j = match.start(0)
 207         self.handle_comment(rawdata[i+4: j])
 208         j = match.end(0)
 209         return j-i
 210
 211     # Internal -- parse declaration.
 212     def parse_declaration(self, i):
 213         rawdata = self.rawdata
 214         j = i + 2
 215         # in practice, this should look like: ((name|stringlit) S*)+ '>'
 216         while 1:
 217             c = rawdata[j:j+1]
 218             if c == ">":
 219                 # end of declaration syntax
 220                 self.handle_decl(rawdata[i+2:j])
 221                 return j + 1
 222             if c in "\"'":
 223                 m = declstringlit.match(rawdata, j)
 224                 if not m:
 225                     # incomplete or an error?
 226                     return -1
 227                 j = m.end()
 228             elif c in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ":
 229                 m = declname.match(rawdata, j)
 230                 if not m:
 231                     # incomplete or an error?
 232                     return -1
 233                 j = m.end()
 234             elif i == len(rawdata):
 235                 # end of buffer between tokens
 236                 return -1
 237             else:
 238                 raise SGMLParseError(
 239                     "unexpected char in declaration: %s" % `rawdata[i]`)
 240         assert 0, "can't get here!"
 241
 242     # Internal -- parse processing instr, return length or -1 if not terminated
 243     def parse_pi(self, i):
 244         rawdata = self.rawdata
 245         if rawdata[i:i+2] != '<?':
 246             raise SGMLParseError('unexpected call to parse_pi()')
 247         match = piclose.search(rawdata, i+2)
 248         if not match:
 249             return -1
 250         j = match.start(0)
 251         self.handle_pi(rawdata[i+2: j])
 252         j = match.end(0)
 253         return j-i
 254
 255     __starttag_text = None
 256     def get_starttag_text(self):
 257         return self.__starttag_text
 258
 259     # Internal -- handle starttag, return length or -1 if not terminated
 260     def parse_starttag(self, i):
 261         self.__starttag_text = None
 262         start_pos = i
 263         rawdata = self.rawdata
 264         if shorttagopen.match(rawdata, i):
 265             # SGML shorthand: <tag/data/ == <tag>data</tag>
 266             # XXX Can data contain &... (entity or char refs)?
 267             # XXX Can data contain < or > (tag characters)?
 268             # XXX Can there be whitespace before the first /?
 269             match = shorttag.match(rawdata, i)
 270             if not match:
 271                 return -1
 272             tag, data = match.group(1, 2)
 273             self.__starttag_text = '<%s/' % tag
 274             tag = tag.lower()
 275             k = match.end(0)
 276             self.finish_shorttag(tag, data)
 277             self.__starttag_text = rawdata[start_pos:match.end(1) + 1]
 278             return k
 279         # XXX The following should skip matching quotes (' or ")
 280         match = endbracket.search(rawdata, i+1)
 281         if not match:
 282             return -1
 283         j = match.start(0)
 284         # Now parse the data between i+1 and j into a tag and attrs
 285         attrs = []
 286         if rawdata[i:i+2] == '<>':
 287             # SGML shorthand: <> == <last open tag seen>
 288             k = j
 289             tag = self.lasttag
 290         else:
 291             match = tagfind.match(rawdata, i+1)
 292             if not match:
 293                 raise SGMLParseError('unexpected call to parse_starttag')
 294             k = match.end(0)
 295             tag = rawdata[i+1:k].lower()
 296             self.lasttag = tag
 297         while k < j:
 298             match = attrfind.match(rawdata, k)
 299             if not match: break
 300             attrname, rest, attrvalue = match.group(1, 2, 3)
 301             if not rest:
 302                 attrvalue = attrname
 303             elif attrvalue[:1] == '\'' == attrvalue[-1:] or \
 304                  attrvalue[:1] == '"' == attrvalue[-1:]:
 305                 attrvalue = attrvalue[1:-1]
 306             attrs.append((attrname.lower(), attrvalue))
 307             k = match.end(0)
 308         if rawdata[j] == '>':
 309             j = j+1
 310         self.__starttag_text = rawdata[start_pos:j]
 311         self.finish_starttag(tag, attrs)
 312         return j
 313
 314     # Internal -- parse endtag
 315     def parse_endtag(self, i):
 316         rawdata = self.rawdata
 317         match = endbracket.search(rawdata, i+1)
 318         if not match:
 319             return -1
 320         j = match.start(0)
 321         tag = rawdata[i+2:j].strip().lower()
 322         if rawdata[j] == '>':
 323             j = j+1
 324         self.finish_endtag(tag)
 325         return j
 326
 327     # Internal -- finish parsing of <tag/data/ (same as <tag>data</tag>)
 328     def finish_shorttag(self, tag, data):
 329         self.finish_starttag(tag, [])
 330         self.handle_data(data)
 331         self.finish_endtag(tag)
 332
 333     # Internal -- finish processing of start tag
 334     # Return -1 for unknown tag, 0 for open-only tag, 1 for balanced tag
 335     def finish_starttag(self, tag, attrs):
 336         try:
 337             method = getattr(self, 'start_' + tag)
 338         except AttributeError:
 339             try:
 340                 method = getattr(self, 'do_' + tag)
 341             except AttributeError:
 342                 self.unknown_starttag(tag, attrs)
 343                 return -1
 344             else:
 345                 self.handle_starttag(tag, method, attrs)
 346                 return 0
 347         else:
 348             self.stack.append(tag)
 349             self.handle_starttag(tag, method, attrs)
 350             return 1
 351
 352     # Internal -- finish processing of end tag
 353     def finish_endtag(self, tag):
 354         if not tag:
 355             found = len(self.stack) - 1
 356             if found < 0:
 357                 self.unknown_endtag(tag)
 358                 return
 359         else:
 360             if tag not in self.stack:
 361                 try:
 362                     method = getattr(self, 'end_' + tag)
 363                 except AttributeError:
 364                     self.unknown_endtag(tag)
 365                 else:
 366                     self.report_unbalanced(tag)
 367                 return
 368             found = len(self.stack)
 369             for i in range(found):
 370                 if self.stack[i] == tag: found = i
 371         while len(self.stack) > found:
 372             tag = self.stack[-1]
 373             try:
 374                 method = getattr(self, 'end_' + tag)
 375             except AttributeError:
 376                 method = None
 377             if method:
 378                 self.handle_endtag(tag, method)
 379             else:
 380                 self.unknown_endtag(tag)
 381             del self.stack[-1]
 382
 383     # Overridable -- handle start tag
 384     def handle_starttag(self, tag, method, attrs):
 385         method(attrs)
 386
 387     # Overridable -- handle end tag
 388     def handle_endtag(self, tag, method):
 389         method()
 390
 391     # Example -- report an unbalanced </...> tag.
 392     def report_unbalanced(self, tag):
 393         if self.verbose:
 394             print '*** Unbalanced </' + tag + '>'
 395             print '*** Stack:', self.stack
 396
 397     # Example -- handle character reference, no need to override
 398     def handle_charref(self, name):
 399         try:
 400             n = int(name)
 401         except ValueError:
 402             self.unknown_charref(name)
 403             return
 404         if not 0 <= n <= 255:
 405             self.unknown_charref(name)
 406             return
 407         self.handle_data(chr(n))
 408
 409     # Definition of entities -- derived classes may override
 410     entitydefs = \
 411             {'lt': '<', 'gt': '>', 'amp': '&', 'quot': '"', 'apos': '\''}
 412
 413     # Example -- handle entity reference, no need to override
 414     def handle_entityref(self, name):
 415         table = self.entitydefs
 416         if table.has_key(name):
 417             self.handle_data(table[name])
 418         else:
 419             self.unknown_entityref(name)
 420             return
 421
 422     # Example -- handle data, should be overridden
 423     def handle_data(self, data):
 424         pass
 425
 426     # Example -- handle comment, could be overridden
 427     def handle_comment(self, data):
 428         pass
 429
 430     # Example -- handle declaration, could be overridden
 431     def handle_decl(self, decl):
 432         pass
 433
 434     # Example -- handle processing instruction, could be overridden
 435     def handle_pi(self, data):
 436         pass
 437
 438     # To be overridden -- handlers for unknown objects
 439     def unknown_starttag(self, tag, attrs): pass
 440     def unknown_endtag(self, tag): pass
 441     def unknown_charref(self, ref): pass
 442     def unknown_entityref(self, ref): pass
 443
 444
 445 class TestSGMLParser(SGMLParser):
 446
 447     def __init__(self, verbose=0):
 448         self.testdata = ""
 449         SGMLParser.__init__(self, verbose)
 450
 451     def handle_data(self, data):
 452         self.testdata = self.testdata + data
 453         if len(`self.testdata`) >= 70:
 454             self.flush()
 455
 456     def flush(self):
 457         data = self.testdata
 458         if data:
 459             self.testdata = ""
 460             print 'data:', `data`
 461
 462     def handle_comment(self, data):
 463         self.flush()
 464         r = `data`
 465         if len(r) > 68:
 466             r = r[:32] + '...' + r[-32:]
 467         print 'comment:', r
 468
 469     def unknown_starttag(self, tag, attrs):
 470         self.flush()
 471         if not attrs:
 472             print 'start tag: <' + tag + '>'
 473         else:
 474             print 'start tag: <' + tag,
 475             for name, value in attrs:
 476                 print name + '=' + '"' + value + '"',
 477             print '>'
 478
 479     def unknown_endtag(self, tag):
 480         self.flush()
 481         print 'end tag: </' + tag + '>'
 482
 483     def unknown_entityref(self, ref):
 484         self.flush()
 485         print '*** unknown entity ref: &' + ref + ';'
 486
 487     def unknown_charref(self, ref):
 488         self.flush()
 489         print '*** unknown char ref: &#' + ref + ';'
 490
 491     def close(self):
 492         SGMLParser.close(self)
 493         self.flush()
 494
 495
 496 def test(args = None):
 497     import sys
 498
 499     if not args:
 500         args = sys.argv[1:]
 501
 502     if args and args[0] == '-s':
 503         args = args[1:]
 504         klass = SGMLParser
 505     else:
 506         klass = TestSGMLParser
 507
 508     if args:
 509         file = args[0]
 510     else:
 511         file = 'test.html'
 512
 513     if file == '-':
 514         f = sys.stdin
 515     else:
 516         try:
 517             f = open(file, 'r')
 518         except IOError, msg:
 519             print file, ":", msg
 520             sys.exit(1)
 521
 522     data = f.read()
 523     if f is not sys.stdin:
 524         f.close()
 525
 526     x = klass()
 527     for c in data:
 528         x.feed(c)
 529     x.close()
 530
 531
 532 if __name__ == '__main__':
 533     test()