Doc/tools/buildindex.py

   1 #! /usr/bin/env python
   2
   3 __version__ = '$Revision$'
   4
   5 import os.path
   6 import re
   7 import string
   8 import sys
   9
  10
  11 bang_join = "!".join
  12 null_join = "".join
  13
  14
  15 class Node:
  16     __rmjunk = re.compile("<#\d+#>")
  17
  18     continuation = 0
  19
  20     def __init__(self, link, str, seqno):
  21         self.links = [link]
  22         self.seqno = seqno
  23         # remove <#\d+#> left in by moving the data out of LaTeX2HTML
  24         str = self.__rmjunk.sub('', str)
  25         # build up the text
  26         self.text = split_entry_text(str)
  27         self.key = split_entry_key(str)
  28
  29     def __cmp__(self, other):
  30         """Comparison operator includes sequence number, for use with
  31         list.sort()."""
  32         return self.cmp_entry(other) or cmp(self.seqno, other.seqno)
  33
  34     def cmp_entry(self, other):
  35         """Comparison 'operator' that ignores sequence number."""
  36         c = 0
  37         for i in range(min(len(self.key), len(other.key))):
  38             c = (cmp_part(self.key[i], other.key[i])
  39                  or cmp_part(self.text[i], other.text[i]))
  40             if c:
  41                 break
  42         return c or cmp(self.key, other.key) or cmp(self.text, other.text)
  43
  44     def __repr__(self):
  45         return "<Node for %s (%s)>" % (bang_join(self.text), self.seqno)
  46
  47     def __str__(self):
  48         return bang_join(self.key)
  49
  50     def dump(self):
  51         return "%s\1%s###%s\n" \
  52                % ("\1".join(self.links),
  53                   bang_join(self.text),
  54                   self.seqno)
  55
  56
  57 def cmp_part(s1, s2):
  58     result = cmp(s1, s2)
  59     if result == 0:
  60         return 0
  61     l1 = s1.lower()
  62     l2 = s2.lower()
  63     minlen = min(len(s1), len(s2))
  64     if len(s1) < len(s2) and l1 == l2[:len(s1)]:
  65         result = -1
  66     elif len(s2) < len(s1) and l2 == l1[:len(s2)]:
  67         result = 1
  68     else:
  69         result = cmp(l1, l2) or cmp(s1, s2)
  70     return result
  71
  72
  73 def split_entry(str, which):
  74     stuff = []
  75     parts = str.split('!')
  76     parts = [part.split('@') for part in parts]
  77     for entry in parts:
  78         if len(entry) != 1:
  79             key = entry[which]
  80         else:
  81             key = entry[0]
  82         stuff.append(key)
  83     return stuff
  84
  85
  86 _rmtt = re.compile(r"""(.*)<tt(?: class=['"][a-z0-9]+["'])?>(.*)</tt>(.*)$""",
  87                    re.IGNORECASE)
  88 _rmparens = re.compile(r"\(\)")
  89
  90 def split_entry_key(str):
  91     parts = split_entry(str, 1)
  92     for i in range(len(parts)):
  93         m = _rmtt.match(parts[i])
  94         if m:
  95             parts[i] = null_join(m.group(1, 2, 3))
  96         else:
  97             parts[i] = parts[i].lower()
  98         # remove '()' from the key:
  99         parts[i] = _rmparens.sub('', parts[i])
 100     return map(trim_ignored_letters, parts)
 101
 102
 103 def split_entry_text(str):
 104     if '<' in str:
 105         m = _rmtt.match(str)
 106         if m:
 107             str = null_join(m.group(1, 2, 3))
 108     return split_entry(str, 1)
 109
 110
 111 def load(fp):
 112     nodes = []
 113     rx = re.compile("(.*)\1(.*)###(.*)$")
 114     while 1:
 115         line = fp.readline()
 116         if not line:
 117             break
 118         m = rx.match(line)
 119         if m:
 120             link, str, seqno = m.group(1, 2, 3)
 121             nodes.append(Node(link, str, seqno))
 122     return nodes
 123
 124
 125 def trim_ignored_letters(s):
 126     # ignore $ to keep environment variables with the
 127     # leading letter from the name
 128     if s.startswith("$"):
 129         return s[1:].lower()
 130     else:
 131         return s.lower()
 132
 133 def get_first_letter(s):
 134     if s.startswith("<tex2html_percent_mark>"):
 135         return "%"
 136     else:
 137         return trim_ignored_letters(s)[0]
 138
 139
 140 def split_letters(nodes):
 141     letter_groups = []
 142     if nodes:
 143         group = []
 144         append = group.append
 145         letter = get_first_letter(nodes[0].text[0])
 146         letter_groups.append((letter, group))
 147         for node in nodes:
 148             nletter = get_first_letter(node.text[0])
 149             if letter != nletter:
 150                 letter = nletter
 151                 group = []
 152                 letter_groups.append((letter, group))
 153                 append = group.append
 154             append(node)
 155     return letter_groups
 156
 157
 158 def group_symbols(groups):
 159     entries = []
 160     ident_letters = string.ascii_letters + "_"
 161     while groups[0][0] not in ident_letters:
 162         entries += groups[0][1]
 163         del groups[0]
 164     if entries:
 165         groups.insert(0, ("Symbols", entries))
 166
 167
 168 # need a function to separate the nodes into columns...
 169 def split_columns(nodes, columns=1):
 170     if columns <= 1:
 171         return [nodes]
 172     # This is a rough height; we may have to increase to avoid breaks before
 173     # a subitem.
 174     colheight = int(len(nodes) / columns)
 175     numlong = int(len(nodes) % columns)
 176     if numlong:
 177         colheight = colheight + 1
 178     else:
 179         numlong = columns
 180     cols = []
 181     for i in range(numlong):
 182         start = i * colheight
 183         end = start + colheight
 184         cols.append(nodes[start:end])
 185     del nodes[:end]
 186     colheight = colheight - 1
 187     try:
 188         numshort = int(len(nodes) / colheight)
 189     except ZeroDivisionError:
 190         cols = cols + (columns - len(cols)) * [[]]
 191     else:
 192         for i in range(numshort):
 193             start = i * colheight
 194             end = start + colheight
 195             cols.append(nodes[start:end])
 196     #
 197     # If items continue across columns, make sure they are marked
 198     # as continuations so the user knows to look at the previous column.
 199     #
 200     for i in range(len(cols) - 1):
 201         try:
 202             prev = cols[i][-1]
 203             next = cols[i + 1][0]
 204         except IndexError:
 205             return cols
 206         else:
 207             n = min(len(prev.key), len(next.key))
 208             for j in range(n):
 209                 if prev.key[j] != next.key[j]:
 210                     break
 211                 next.continuation = j + 1
 212     return cols
 213
 214
 215 DL_LEVEL_INDENT = "  "
 216
 217 def format_column(nodes):
 218     strings = ["<dl compact>"]
 219     append = strings.append
 220     level = 0
 221     previous = []
 222     for node in nodes:
 223         current = node.text
 224         count = 0
 225         for i in range(min(len(current), len(previous))):
 226             if previous[i] != current[i]:
 227                 break
 228             count = i + 1
 229         if count > level:
 230             append("<dl compact>" * (count - level) + "\n")
 231             level = count
 232         elif level > count:
 233             append("\n")
 234             append(level * DL_LEVEL_INDENT)
 235             append("</dl>" * (level - count))
 236             level = count
 237         # else: level == count
 238         for i in range(count, len(current) - 1):
 239             term = node.text[i]
 240             level = level + 1
 241             if node.continuation > i:
 242                 extra = " (continued)"
 243             else:
 244                 extra = ""
 245             append("\n<dt>%s%s\n<dd>\n%s<dl compact>"
 246                    % (term, extra, level * DL_LEVEL_INDENT))
 247         append("\n%s<dt>%s%s</a>"
 248                % (level * DL_LEVEL_INDENT, node.links[0], node.text[-1]))
 249         for link in node.links[1:]:
 250             append(",\n%s    %s[Link]</a>" % (level * DL_LEVEL_INDENT, link))
 251         previous = current
 252     append("\n")
 253     append("</dl>" * (level + 1))
 254     return null_join(strings)
 255
 256
 257 def format_nodes(nodes, columns=1):
 258     strings = []
 259     append = strings.append
 260     if columns > 1:
 261         colnos = range(columns)
 262         colheight = int(len(nodes) / columns)
 263         if len(nodes) % columns:
 264             colheight = colheight + 1
 265         colwidth = int(100 / columns)
 266         append('<table width="100%"><tr valign="top">')
 267         for col in split_columns(nodes, columns):
 268             append('<td width="%d%%">\n' % colwidth)
 269             append(format_column(col))
 270             append("\n</td>")
 271         append("\n</tr></table>")
 272     else:
 273         append(format_column(nodes))
 274     append("\n<p>\n")
 275     return null_join(strings)
 276
 277
 278 def format_letter(letter):
 279     if letter == '.':
 280         lettername = ". (dot)"
 281     elif letter == '_':
 282         lettername = "_ (underscore)"
 283     else:
 284         lettername = letter.capitalize()
 285     return "\n<hr>\n<h2><a name=\"letter-%s\">%s</a></h2>\n\n" \
 286            % (letter, lettername)
 287
 288
 289 def format_html_letters(nodes, columns, group_symbol_nodes):
 290     letter_groups = split_letters(nodes)
 291     if group_symbol_nodes:
 292         group_symbols(letter_groups)
 293     items = []
 294     for letter, nodes in letter_groups:
 295         s = "<b><a href=\"#letter-%s\">%s</a></b>" % (letter, letter)
 296         items.append(s)
 297     s = ["<hr><center>\n%s</center>\n" % " |\n".join(items)]
 298     for letter, nodes in letter_groups:
 299         s.append(format_letter(letter))
 300         s.append(format_nodes(nodes, columns))
 301     return null_join(s)
 302
 303 def format_html(nodes, columns):
 304     return format_nodes(nodes, columns)
 305
 306
 307 def collapse(nodes):
 308     """Collapse sequences of nodes with matching keys into a single node.
 309     Destructive."""
 310     if len(nodes) < 2:
 311         return
 312     prev = nodes[0]
 313     i = 1
 314     while i < len(nodes):
 315         node = nodes[i]
 316         if not node.cmp_entry(prev):
 317             prev.links.append(node.links[0])
 318             del nodes[i]
 319         else:
 320             i = i + 1
 321             prev = node
 322
 323
 324 def dump(nodes, fp):
 325     for node in nodes:
 326         fp.write(node.dump())
 327
 328
 329 def process_nodes(nodes, columns, letters=0, group_symbol_nodes=0):
 330     nodes.sort()
 331     collapse(nodes)
 332     if letters:
 333         return format_html_letters(nodes, columns, group_symbol_nodes)
 334     else:
 335         return format_html(nodes, columns)
 336
 337
 338 def main():
 339     import getopt
 340     ifn = "-"
 341     ofn = "-"
 342     columns = 1
 343     letters = 0
 344     group_symbol_nodes = 1
 345     opts, args = getopt.getopt(sys.argv[1:], "c:lo:",
 346                                ["columns=", "dont-group-symbols",
 347                                 "group-symbols", "letters", "output="])
 348     for opt, val in opts:
 349         if opt in ("-o", "--output"):
 350             ofn = val
 351         elif opt in ("-c", "--columns"):
 352             columns = int(val, 10)
 353         elif opt in ("-l", "--letters"):
 354             letters = 1
 355         elif opt == "--group-symbols":
 356             group_symbol_nodes = 1
 357         elif opt == "--dont-group-symbols":
 358             group_symbol_nodes = 0
 359     if not args:
 360         args = [ifn]
 361     nodes = []
 362     for fn in args:
 363         nodes = nodes + load(open(fn))
 364     num_nodes = len(nodes)
 365     html = process_nodes(nodes, columns, letters, group_symbol_nodes)
 366     program = os.path.basename(sys.argv[0])
 367     if ofn == "-":
 368         sys.stdout.write(html)
 369         sys.stderr.write("\n%s: %d index nodes" % (program, num_nodes))
 370     else:
 371         open(ofn, "w").write(html)
 372         print
 373         print "%s: %d index nodes" % (program, num_nodes)
 374
 375
 376 if __name__ == "__main__":
 377     main()