Lib/sgmllib.py

   1 """A parser for SGML, using the derived class as a static DTD."""
   2
   3 # XXX This only supports those SGML features used by HTML.
   4
   5 # XXX There should be a way to distinguish between PCDATA (parsed
   6 # character data -- the normal case), RCDATA (replaceable character
   7 # data -- only char and entity references and end tags are special)
   8 # and CDATA (character data -- only end tags are special).
   9
  10
  11 import re
  12 import string
  13
  14
  15 # Regular expressions used for parsing
  16
  17 interesting = re.compile('[&<]')
  18 incomplete = re.compile('&([a-zA-Z][a-zA-Z0-9]*|#[0-9]*)?|'
  19                            '<([a-zA-Z][^<>]*|'
  20                               '/([a-zA-Z][^<>]*)?|'
  21                               '![^<>]*)?')
  22
  23 entityref = re.compile('&([a-zA-Z][-.a-zA-Z0-9]*)[^a-zA-Z0-9]')
  24 charref = re.compile('&#([0-9]+)[^0-9]')
  25
  26 starttagopen = re.compile('<[>a-zA-Z]')
  27 shorttagopen = re.compile('<[a-zA-Z][-.a-zA-Z0-9]*/')
  28 shorttag = re.compile('<([a-zA-Z][-.a-zA-Z0-9]*)/([^/]*)/')
  29 piopen = re.compile('<\?')
  30 piclose = re.compile('>')
  31 endtagopen = re.compile('</[<>a-zA-Z]')
  32 endbracket = re.compile('[<>]')
  33 special = re.compile('<![^<>]*>')
  34 commentopen = re.compile('<!--')
  35 commentclose = re.compile('--[%s]*>' % string.whitespace)
  36 tagfind = re.compile('[a-zA-Z][-.a-zA-Z0-9]*')
  37 attrfind = re.compile(
  38     '[%s]*([a-zA-Z_][-.a-zA-Z_0-9]*)' % string.whitespace
  39     + ('([%s]*=[%s]*' % (string.whitespace, string.whitespace))
  40     + r'(\'[^\']*\'|"[^"]*"|[-a-zA-Z0-9./:+*%?!&$\(\)_#=~]*))?')
  41
  42
  43 # SGML parser base class -- find tags and call handler functions.
  44 # Usage: p = SGMLParser(); p.feed(data); ...; p.close().
  45 # The dtd is defined by deriving a class which defines methods
  46 # with special names to handle tags: start_foo and end_foo to handle
  47 # <foo> and </foo>, respectively, or do_foo to handle <foo> by itself.
  48 # (Tags are converted to lower case for this purpose.)  The data
  49 # between tags is passed to the parser by calling self.handle_data()
  50 # with some data as argument (the data may be split up in arbitrary
  51 # chunks).  Entity references are passed by calling
  52 # self.handle_entityref() with the entity reference as argument.
  53
  54 class SGMLParser:
  55
  56     # Interface -- initialize and reset this instance
  57     def __init__(self, verbose=0):
  58         self.verbose = verbose
  59         self.reset()
  60
  61     # Interface -- reset this instance.  Loses all unprocessed data
  62     def reset(self):
  63         self.rawdata = ''
  64         self.stack = []
  65         self.lasttag = '???'
  66         self.nomoretags = 0
  67         self.literal = 0
  68
  69     # For derived classes only -- enter literal mode (CDATA) till EOF
  70     def setnomoretags(self):
  71         self.nomoretags = self.literal = 1
  72
  73     # For derived classes only -- enter literal mode (CDATA)
  74     def setliteral(self, *args):
  75         self.literal = 1
  76
  77     # Interface -- feed some data to the parser.  Call this as
  78     # often as you want, with as little or as much text as you
  79     # want (may include '\n').  (This just saves the text, all the
  80     # processing is done by goahead().)
  81     def feed(self, data):
  82         self.rawdata = self.rawdata + data
  83         self.goahead(0)
  84
  85     # Interface -- handle the remaining data
  86     def close(self):
  87         self.goahead(1)
  88
  89     # Internal -- handle data as far as reasonable.  May leave state
  90     # and data to be processed by a subsequent call.  If 'end' is
  91     # true, force handling all data as if followed by EOF marker.
  92     def goahead(self, end):
  93         rawdata = self.rawdata
  94         i = 0
  95         n = len(rawdata)
  96         while i < n:
  97             if self.nomoretags:
  98                 self.handle_data(rawdata[i:n])
  99                 i = n
 100                 break
 101             match = interesting.search(rawdata, i)
 102             if match: j = match.start(0)
 103             else: j = n
 104             if i < j: self.handle_data(rawdata[i:j])
 105             i = j
 106             if i == n: break
 107             if rawdata[i] == '<':
 108                 if starttagopen.match(rawdata, i):
 109                     if self.literal:
 110                         self.handle_data(rawdata[i])
 111                         i = i+1
 112                         continue
 113                     k = self.parse_starttag(i)
 114                     if k < 0: break
 115                     i = k
 116                     continue
 117                 if endtagopen.match(rawdata, i):
 118                     k = self.parse_endtag(i)
 119                     if k < 0: break
 120                     i =  k
 121                     self.literal = 0
 122                     continue
 123                 if commentopen.match(rawdata, i):
 124                     if self.literal:
 125                         self.handle_data(rawdata[i])
 126                         i = i+1
 127                         continue
 128                     k = self.parse_comment(i)
 129                     if k < 0: break
 130                     i = i+k
 131                     continue
 132                 if piopen.match(rawdata, i):
 133                     if self.literal:
 134                         self.handle_data(rawdata[i])
 135                         i = i+1
 136                         continue
 137                     k = self.parse_pi(i)
 138                     if k < 0: break
 139                     i = i+k
 140                     continue
 141                 match = special.match(rawdata, i)
 142                 if match:
 143                     if self.literal:
 144                         self.handle_data(rawdata[i])
 145                         i = i+1
 146                         continue
 147                     i = match.end(0)
 148                     continue
 149             elif rawdata[i] == '&':
 150                 match = charref.match(rawdata, i)
 151                 if match:
 152                     name = match.group(1)
 153                     self.handle_charref(name)
 154                     i = match.end(0)
 155                     if rawdata[i-1] != ';': i = i-1
 156                     continue
 157                 match = entityref.match(rawdata, i)
 158                 if match:
 159                     name = match.group(1)
 160                     self.handle_entityref(name)
 161                     i = match.end(0)
 162                     if rawdata[i-1] != ';': i = i-1
 163                     continue
 164             else:
 165                 raise RuntimeError, 'neither < nor & ??'
 166             # We get here only if incomplete matches but
 167             # nothing else
 168             match = incomplete.match(rawdata, i)
 169             if not match:
 170                 self.handle_data(rawdata[i])
 171                 i = i+1
 172                 continue
 173             j = match.end(0)
 174             if j == n:
 175                 break # Really incomplete
 176             self.handle_data(rawdata[i:j])
 177             i = j
 178         # end while
 179         if end and i < n:
 180             self.handle_data(rawdata[i:n])
 181             i = n
 182         self.rawdata = rawdata[i:]
 183         # XXX if end: check for empty stack
 184
 185     # Internal -- parse comment, return length or -1 if not terminated
 186     def parse_comment(self, i):
 187         rawdata = self.rawdata
 188         if rawdata[i:i+4] <> '<!--':
 189             raise RuntimeError, 'unexpected call to handle_comment'
 190         match = commentclose.search(rawdata, i+4)
 191         if not match:
 192             return -1
 193         j = match.start(0)
 194         self.handle_comment(rawdata[i+4: j])
 195         j = match.end(0)
 196         return j-i
 197
 198     # Internal -- parse processing instr, return length or -1 if not terminated
 199     def parse_pi(self, i):
 200         rawdata = self.rawdata
 201         if rawdata[i:i+2] <> '<?':
 202             raise RuntimeError, 'unexpected call to handle_pi'
 203         match = piclose.search(rawdata, i+2)
 204         if not match:
 205             return -1
 206         j = match.start(0)
 207         self.handle_pi(rawdata[i+2: j])
 208         j = match.end(0)
 209         return j-i
 210
 211     __starttag_text = None
 212     def get_starttag_text(self):
 213         return self.__starttag_text
 214
 215     # Internal -- handle starttag, return length or -1 if not terminated
 216     def parse_starttag(self, i):
 217         self.__starttag_text = None
 218         start_pos = i
 219         rawdata = self.rawdata
 220         if shorttagopen.match(rawdata, i):
 221             # SGML shorthand: <tag/data/ == <tag>data</tag>
 222             # XXX Can data contain &... (entity or char refs)?
 223             # XXX Can data contain < or > (tag characters)?
 224             # XXX Can there be whitespace before the first /?
 225             match = shorttag.match(rawdata, i)
 226             if not match:
 227                 return -1
 228             tag, data = match.group(1, 2)
 229             self.__starttag_text = '<%s/' % tag
 230             tag = string.lower(tag)
 231             k = match.end(0)
 232             self.finish_shorttag(tag, data)
 233             self.__starttag_text = rawdata[start_pos:match.end(1) + 1]
 234             return k
 235         # XXX The following should skip matching quotes (' or ")
 236         match = endbracket.search(rawdata, i+1)
 237         if not match:
 238             return -1
 239         j = match.start(0)
 240         # Now parse the data between i+1 and j into a tag and attrs
 241         attrs = []
 242         if rawdata[i:i+2] == '<>':
 243             # SGML shorthand: <> == <last open tag seen>
 244             k = j
 245             tag = self.lasttag
 246         else:
 247             match = tagfind.match(rawdata, i+1)
 248             if not match:
 249                 raise RuntimeError, 'unexpected call to parse_starttag'
 250             k = match.end(0)
 251             tag = string.lower(rawdata[i+1:k])
 252             self.lasttag = tag
 253         while k < j:
 254             match = attrfind.match(rawdata, k)
 255             if not match: break
 256             attrname, rest, attrvalue = match.group(1, 2, 3)
 257             if not rest:
 258                 attrvalue = attrname
 259             elif attrvalue[:1] == '\'' == attrvalue[-1:] or \
 260                  attrvalue[:1] == '"' == attrvalue[-1:]:
 261                 attrvalue = attrvalue[1:-1]
 262             attrs.append((string.lower(attrname), attrvalue))
 263             k = match.end(0)
 264         if rawdata[j] == '>':
 265             j = j+1
 266         self.__starttag_text = rawdata[start_pos:j]
 267         self.finish_starttag(tag, attrs)
 268         return j
 269
 270     # Internal -- parse endtag
 271     def parse_endtag(self, i):
 272         rawdata = self.rawdata
 273         match = endbracket.search(rawdata, i+1)
 274         if not match:
 275             return -1
 276         j = match.start(0)
 277         tag = string.lower(string.strip(rawdata[i+2:j]))
 278         if rawdata[j] == '>':
 279             j = j+1
 280         self.finish_endtag(tag)
 281         return j
 282
 283     # Internal -- finish parsing of <tag/data/ (same as <tag>data</tag>)
 284     def finish_shorttag(self, tag, data):
 285         self.finish_starttag(tag, [])
 286         self.handle_data(data)
 287         self.finish_endtag(tag)
 288
 289     # Internal -- finish processing of start tag
 290     # Return -1 for unknown tag, 0 for open-only tag, 1 for balanced tag
 291     def finish_starttag(self, tag, attrs):
 292         try:
 293             method = getattr(self, 'start_' + tag)
 294         except AttributeError:
 295             try:
 296                 method = getattr(self, 'do_' + tag)
 297             except AttributeError:
 298                 self.unknown_starttag(tag, attrs)
 299                 return -1
 300             else:
 301                 self.handle_starttag(tag, method, attrs)
 302                 return 0
 303         else:
 304             self.stack.append(tag)
 305             self.handle_starttag(tag, method, attrs)
 306             return 1
 307
 308     # Internal -- finish processing of end tag
 309     def finish_endtag(self, tag):
 310         if not tag:
 311             found = len(self.stack) - 1
 312             if found < 0:
 313                 self.unknown_endtag(tag)
 314                 return
 315         else:
 316             if tag not in self.stack:
 317                 try:
 318                     method = getattr(self, 'end_' + tag)
 319                 except AttributeError:
 320                     self.unknown_endtag(tag)
 321                 else:
 322                     self.report_unbalanced(tag)
 323                 return
 324             found = len(self.stack)
 325             for i in range(found):
 326                 if self.stack[i] == tag: found = i
 327         while len(self.stack) > found:
 328             tag = self.stack[-1]
 329             try:
 330                 method = getattr(self, 'end_' + tag)
 331             except AttributeError:
 332                 method = None
 333             if method:
 334                 self.handle_endtag(tag, method)
 335             else:
 336                 self.unknown_endtag(tag)
 337             del self.stack[-1]
 338
 339     # Overridable -- handle start tag
 340     def handle_starttag(self, tag, method, attrs):
 341         method(attrs)
 342
 343     # Overridable -- handle end tag
 344     def handle_endtag(self, tag, method):
 345         method()
 346
 347     # Example -- report an unbalanced </...> tag.
 348     def report_unbalanced(self, tag):
 349         if self.verbose:
 350             print '*** Unbalanced </' + tag + '>'
 351             print '*** Stack:', self.stack
 352
 353     # Example -- handle character reference, no need to override
 354     def handle_charref(self, name):
 355         try:
 356             n = string.atoi(name)
 357         except string.atoi_error:
 358             self.unknown_charref(name)
 359             return
 360         if not 0 <= n <= 255:
 361             self.unknown_charref(name)
 362             return
 363         self.handle_data(chr(n))
 364
 365     # Definition of entities -- derived classes may override
 366     entitydefs = \
 367             {'lt': '<', 'gt': '>', 'amp': '&', 'quot': '"', 'apos': '\''}
 368
 369     # Example -- handle entity reference, no need to override
 370     def handle_entityref(self, name):
 371         table = self.entitydefs
 372         if table.has_key(name):
 373             self.handle_data(table[name])
 374         else:
 375             self.unknown_entityref(name)
 376             return
 377
 378     # Example -- handle data, should be overridden
 379     def handle_data(self, data):
 380         pass
 381
 382     # Example -- handle comment, could be overridden
 383     def handle_comment(self, data):
 384         pass
 385
 386     # Example -- handle processing instruction, could be overridden
 387     def handle_pi(self, data):
 388         pass
 389
 390     # To be overridden -- handlers for unknown objects
 391     def unknown_starttag(self, tag, attrs): pass
 392     def unknown_endtag(self, tag): pass
 393     def unknown_charref(self, ref): pass
 394     def unknown_entityref(self, ref): pass
 395
 396
 397 class TestSGMLParser(SGMLParser):
 398
 399     def __init__(self, verbose=0):
 400         self.testdata = ""
 401         SGMLParser.__init__(self, verbose)
 402
 403     def handle_data(self, data):
 404         self.testdata = self.testdata + data
 405         if len(`self.testdata`) >= 70:
 406             self.flush()
 407
 408     def flush(self):
 409         data = self.testdata
 410         if data:
 411             self.testdata = ""
 412             print 'data:', `data`
 413
 414     def handle_comment(self, data):
 415         self.flush()
 416         r = `data`
 417         if len(r) > 68:
 418             r = r[:32] + '...' + r[-32:]
 419         print 'comment:', r
 420
 421     def unknown_starttag(self, tag, attrs):
 422         self.flush()
 423         if not attrs:
 424             print 'start tag: <' + tag + '>'
 425         else:
 426             print 'start tag: <' + tag,
 427             for name, value in attrs:
 428                 print name + '=' + '"' + value + '"',
 429             print '>'
 430
 431     def unknown_endtag(self, tag):
 432         self.flush()
 433         print 'end tag: </' + tag + '>'
 434
 435     def unknown_entityref(self, ref):
 436         self.flush()
 437         print '*** unknown entity ref: &' + ref + ';'
 438
 439     def unknown_charref(self, ref):
 440         self.flush()
 441         print '*** unknown char ref: &#' + ref + ';'
 442
 443     def close(self):
 444         SGMLParser.close(self)
 445         self.flush()
 446
 447
 448 def test(args = None):
 449     import sys
 450
 451     if not args:
 452         args = sys.argv[1:]
 453
 454     if args and args[0] == '-s':
 455         args = args[1:]
 456         klass = SGMLParser
 457     else:
 458         klass = TestSGMLParser
 459
 460     if args:
 461         file = args[0]
 462     else:
 463         file = 'test.html'
 464
 465     if file == '-':
 466         f = sys.stdin
 467     else:
 468         try:
 469             f = open(file, 'r')
 470         except IOError, msg:
 471             print file, ":", msg
 472             sys.exit(1)
 473
 474     data = f.read()
 475     if f is not sys.stdin:
 476         f.close()
 477
 478     x = klass()
 479     for c in data:
 480         x.feed(c)
 481     x.close()
 482
 483
 484 if __name__ == '__main__':
 485     test()