Demo/tkinter/www/sgmllib.py

   1 # A parser for SGML, using the derived class as static DTD.
   2
   3 # XXX This only supports those SGML features used by HTML.
   4
   5 # XXX There should be a way to distinguish between PCDATA (parsed
   6 # character data -- the normal case), RCDATA (replaceable character
   7 # data -- only char and entity references and end tags are special)
   8 # and CDATA (character data -- only end tags are special).
   9
  10
  11 import regex
  12 import string
  13
  14
  15 # Regular expressions used for parsing
  16
  17 incomplete = regex.compile( \
  18           '<!-?\|</[a-zA-Z][a-zA-Z0-9]*[ \t\n]*\|</?\|' + \
  19           '&#[a-zA-Z0-9]*\|&[a-zA-Z][a-zA-Z0-9]*\|&')
  20 entityref = regex.compile('&[a-zA-Z][a-zA-Z0-9]*[;.]')
  21 charref = regex.compile('&#[a-zA-Z0-9]+;')
  22 starttagopen = regex.compile('<[a-zA-Z]')
  23 endtag = regex.compile('</[a-zA-Z][a-zA-Z0-9]*[ \t\n]*>')
  24 commentopen = regex.compile('<!--')
  25
  26
  27 # SGML parser base class -- find tags and call handler functions.
  28 # Usage: p = SGMLParser(); p.feed(data); ...; p.close().
  29 # The dtd is defined by deriving a class which defines methods
  30 # with special names to handle tags: start_foo and end_foo to handle
  31 # <foo> and </foo>, respectively, or do_foo to handle <foo> by itself.
  32 # (Tags are converted to lower case for this purpose.)  The data
  33 # between tags is passed to the parser by calling self.handle_data()
  34 # with some data as argument (the data may be split up in arbutrary
  35 # chunks).  Entity references are passed by calling
  36 # self.handle_entityref() with the entity reference as argument.
  37
  38 class SGMLParser:
  39
  40         # Interface -- initialize and reset this instance
  41         def __init__(self):
  42                 self.reset()
  43
  44         # Interface -- reset this instance.  Loses all unprocessed data
  45         def reset(self):
  46                 self.rawdata = ''
  47                 self.stack = []
  48                 self.nomoretags = 0
  49                 self.literal = 0
  50
  51         # For derived classes only -- enter literal mode (CDATA) till EOF
  52         def setnomoretags(self):
  53                 self.nomoretags = self.literal = 1
  54
  55         # For derived classes only -- enter literal mode (CDATA)
  56         def setliteral(self, *args):
  57                 self.literal = 1
  58
  59         # Interface -- feed some data to the parser.  Call this as
  60         # often as you want, with as little or as much text as you
  61         # want (may include '\n').  (This just saves the text, all the
  62         # processing is done by process() or close().)
  63         def feed(self, data):
  64                 self.rawdata = self.rawdata + data
  65                 self.goahead(0)
  66
  67         # Interface -- handle the remaining data
  68         def close(self):
  69                 self.goahead(1)
  70
  71         # Internal -- handle data as far as reasonable.  May leave state
  72         # and data to be processed by a subsequent call.  If 'end' is
  73         # true, force handling all data as if followed by EOF marker.
  74         def goahead(self, end):
  75                 rawdata = self.rawdata
  76                 i = 0
  77                 n = len(rawdata)
  78                 while i < n:
  79                         if self.nomoretags:
  80                                 self.handle_data(rawdata[i:n])
  81                                 i = n
  82                                 break
  83                         j = incomplete.search(rawdata, i)
  84                         if j < 0: j = n
  85                         if i < j: self.handle_data(rawdata[i:j])
  86                         i = j
  87                         if i == n: break
  88                         if rawdata[i] == '<':
  89                                 if starttagopen.match(rawdata, i) >= 0:
  90                                         if self.literal:
  91                                                 self.handle_data(rawdata[i])
  92                                                 i = i+1
  93                                                 continue
  94                                         k = self.parse_starttag(i)
  95                                         if k < 0: break
  96                                         i = i + k
  97                                         continue
  98                                 k = endtag.match(rawdata, i)
  99                                 if k >= 0:
 100                                         j = i+k
 101                                         self.parse_endtag(rawdata[i:j])
 102                                         i = j
 103                                         self.literal = 0
 104                                         continue
 105                                 if commentopen.match(rawdata, i) >= 0:
 106                                         if self.literal:
 107                                                 self.handle_data(rawdata[i])
 108                                                 i = i+1
 109                                                 continue
 110                                         k = self.parse_comment(i)
 111                                         if k < 0: break
 112                                         i = i+k
 113                                         continue
 114                         elif rawdata[i] == '&':
 115                                 k = charref.match(rawdata, i)
 116                                 if k >= 0:
 117                                         j = i+k
 118                                         self.handle_charref(rawdata[i+2:j-1])
 119                                         i = j
 120                                         continue
 121                                 k = entityref.match(rawdata, i)
 122                                 if k >= 0:
 123                                         j = i+k
 124                                         self.handle_entityref(rawdata[i+1:j-1])
 125                                         i = j
 126                                         continue
 127                         else:
 128                                 raise RuntimeError, 'neither < nor & ??'
 129                         # We get here only if incomplete matches but
 130                         # nothing else
 131                         k = incomplete.match(rawdata, i)
 132                         if k < 0: raise RuntimeError, 'no incomplete match ??'
 133                         j = i+k
 134                         if j == n: break # Really incomplete
 135                         self.handle_data(rawdata[i:j])
 136                         i = j
 137                 # end while
 138                 if end and i < n:
 139                         self.handle_data(rawdata[i:n])
 140                         i = n
 141                 self.rawdata = rawdata[i:]
 142                 # XXX if end: check for empty stack
 143
 144         # Internal -- parse comment, return length or -1 if not ternimated
 145         def parse_comment(self, i):
 146                 rawdata = self.rawdata
 147                 if rawdata[i:i+4] <> '<!--':
 148                         raise RuntimeError, 'unexpected call to handle_comment'
 149                 try:
 150                         j = string.index(rawdata, '--', i+4)
 151                 except string.index_error:
 152                         return -1
 153                 self.handle_comment(rawdata[i+4: j])
 154                 j = j+2
 155                 n = len(rawdata)
 156                 while j < n and rawdata[j] in ' \t\n': j = j+1
 157                 if j == n: return -1 # Wait for final '>'
 158                 if rawdata[j] == '>':
 159                         j = j+1
 160                 else:
 161                         print '*** comment not terminated with >'
 162                         print repr(rawdata[j-5:j]), '*!*', repr(rawdata[j:j+5])
 163                 return j-i
 164
 165         # Internal -- handle starttag, return length or -1 if not terminated
 166         def parse_starttag(self, i):
 167                 rawdata = self.rawdata
 168                 try:
 169                         j = string.index(rawdata, '>', i)
 170                 except string.index_error:
 171                         return -1
 172                 # Now parse the data between i+1 and j into a tag and attrs
 173                 attrs = []
 174                 tagfind = regex.compile('[a-zA-Z][a-zA-Z0-9]*')
 175                 attrfind = regex.compile( \
 176                   '[ \t\n]+\([a-zA-Z][a-zA-Z0-9]*\)' + \
 177                   '\([ \t\n]*=[ \t\n]*' + \
 178                      '\(\'[^\']*\';\|"[^"]*"\|[-a-zA-Z0-9./:+*%?!()_#]+\)\)?')
 179                 k = tagfind.match(rawdata, i+1)
 180                 if k < 0:
 181                         raise RuntimeError, 'unexpected call to parse_starttag'
 182                 k = i+1+k
 183                 tag = string.lower(rawdata[i+1:k])
 184                 while k < j:
 185                         l = attrfind.match(rawdata, k)
 186                         if l < 0: break
 187                         regs = attrfind.regs
 188                         a1, b1 = regs[1]
 189                         a2, b2 = regs[2]
 190                         a3, b3 = regs[3]
 191                         attrname = rawdata[a1:b1]
 192                         if '=' in rawdata[k:k+l]:
 193                                 attrvalue = rawdata[a3:b3]
 194                                 if attrvalue[:1] == '\'' == attrvalue[-1:] or \
 195                                    attrvalue[:1] == '"' == attrvalue[-1:]:
 196                                         attrvalue = attrvalue[1:-1]
 197                         else:
 198                                 attrvalue = ''
 199                         attrs.append(string.lower(attrname), attrvalue)
 200                         k = k + l
 201                 j = j+1
 202                 try:
 203                         method = getattr(self, 'start_' + tag)
 204                 except AttributeError:
 205                         try:
 206                                 method = getattr(self, 'do_' + tag)
 207                         except AttributeError:
 208                                 self.unknown_starttag(tag, attrs)
 209                                 return j-i
 210                         method(attrs)
 211                         return j-i
 212                 self.stack.append(tag)
 213                 method(attrs)
 214                 return j-i
 215
 216         # Internal -- parse endtag
 217         def parse_endtag(self, data):
 218                 if data[:2] <> '</' or data[-1:] <> '>':
 219                         raise RuntimeError, 'unexpected call to parse_endtag'
 220                 tag = string.lower(string.strip(data[2:-1]))
 221                 try:
 222                         method = getattr(self, 'end_' + tag)
 223                 except AttributeError:
 224                         self.unknown_endtag(tag)
 225                         return
 226                 if self.stack and self.stack[-1] == tag:
 227                         del self.stack[-1]
 228                 else:
 229                         print '*** Unbalanced </' + tag + '>'
 230                         print '*** Stack:', self.stack
 231                         found = None
 232                         for i in range(len(self.stack)):
 233                                 if self.stack[i] == tag: found = i
 234                         if found <> None:
 235                                 del self.stack[found:]
 236                 method()
 237
 238         # Example -- handle character reference, no need to override
 239         def handle_charref(self, name):
 240                 try:
 241                         n = string.atoi(name)
 242                 except string.atoi_error:
 243                         self.unknown_charref(name)
 244                         return
 245                 if not 0 <= n <= 255:
 246                         self.unknown_charref(name)
 247                         return
 248                 self.handle_data(chr(n))
 249
 250         # Definition of entities -- derived classes may override
 251         entitydefs = \
 252                 {'lt': '<', 'gt': '>', 'amp': '&', 'quot': '"', 'apos': '\''}
 253
 254         # Example -- handle entity reference, no need to override
 255         def handle_entityref(self, name):
 256                 table = self.__class__.entitydefs
 257                 name = string.lower(name)
 258                 if table.has_key(name):
 259                         self.handle_data(table[name])
 260                 else:
 261                         self.unknown_entityref(name)
 262                         return
 263
 264         # Example -- handle data, should be overridden
 265         def handle_data(self, data):
 266                 pass
 267
 268         # Example -- handle comment, could be overridden
 269         def handle_comment(self, data):
 270                 pass
 271
 272         # To be overridden -- handlers for unknown objects
 273         def unknown_starttag(self, tag, attrs): pass
 274         def unknown_endtag(self, tag): pass
 275         def unknown_charref(self, ref): pass
 276         def unknown_entityref(self, ref): pass
 277
 278
 279 class TestSGML(SGMLParser):
 280
 281         def handle_data(self, data):
 282                 r = repr(data)
 283                 if len(r) > 72:
 284                         r = r[:35] + '...' + r[-35:]
 285                 print 'data:', r
 286
 287         def handle_comment(self, data):
 288                 r = repr(data)
 289                 if len(r) > 68:
 290                         r = r[:32] + '...' + r[-32:]
 291                 print 'comment:', r
 292
 293         def unknown_starttag(self, tag, attrs):
 294                 print 'start tag: <' + tag,
 295                 for name, value in attrs:
 296                         print name + '=' + '"' + value + '"',
 297                 print '>'
 298
 299         def unknown_endtag(self, tag):
 300                 print 'end tag: </' + tag + '>'
 301
 302         def unknown_entityref(self, ref):
 303                 print '*** unknown entity ref: &' + ref + ';'
 304
 305         def unknown_charref(self, ref):
 306                 print '*** unknown char ref: &#' + ref + ';'
 307
 308
 309 def test():
 310         file = 'test.html'
 311         f = open(file, 'r')
 312         x = TestSGML()
 313         while 1:
 314                 line = f.readline()
 315                 if not line:
 316                         x.close()
 317                         break
 318                 x.feed(line)
 319
 320
 321 #test()