gtkdoc/fixxref.py

   1 # -*- python -*-
   2 #
   3 # gtk-doc - GTK DocBook documentation generator.
   4 # Copyright (C) 1998  Damon Chaplin
   5 #               2007-2016  Stefan Sauer
   6 #
   7 # This program is free software; you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation; either version 2 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program; if not, write to the Free Software
  19 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  20 #
  21
  22 ''"Fix cross-references in the HTML documentation.''"
  23
  24 # Support both Python 2 and 3
  25 from __future__ import print_function
  26
  27 import logging
  28 import os
  29 import re
  30 import shlex
  31 import subprocess
  32 import sys
  33 import tempfile
  34
  35 from . import common, config
  36
  37 # This contains all the entities and their relative URLs.
  38 Links = {}
  39
  40 # failing link targets we don't warn about even once
  41 NoLinks = {
  42     'char',
  43     'double',
  44     'float',
  45     'int',
  46     'long',
  47     'main',
  48     'signed',
  49     'unsigned',
  50     'va-list',
  51     'void',
  52 }
  53
  54
  55 def Run(options):
  56     logging.info('options: %s', str(options.__dict__))
  57
  58     LoadIndicies(options.module_dir, options.html_dir, options.extra_dir)
  59     ReadSections(options.module)
  60     FixCrossReferences(options.module_dir, options.module, options.src_lang)
  61
  62
  63 # TODO(ensonic): try to refactor so that we get a list of path's and then just
  64 # loop over them.
  65 # - module_dir is by default 'html'
  66 # - html_dir can be set by configure, defaults to $(docdir)
  67 def LoadIndicies(module_dir, html_dir, extra_dirs):
  68     # Cache of dirs we already scanned for index files
  69     dir_cache = {}
  70
  71     path_prefix = ''
  72     m = re.search(r'(.*?)/share/gtk-doc/html', html_dir)
  73     if m:
  74         path_prefix = m.group(1)
  75         logging.info('Path prefix: %s', path_prefix)
  76     prefix_match = r'^' + re.escape(path_prefix) + r'/'
  77
  78     # We scan the directory containing GLib and any directories in GNOME2_PATH
  79     # first, but these will be overriden by any later scans.
  80     dir = common.GetModuleDocDir('glib-2.0')
  81     if dir and os.path.exists(dir):
  82         # Some predefined link targets to get links into type hierarchies as these
  83         # have no targets. These are always absolute for now.
  84         Links['GBoxed'] = dir + '/gobject/gobject-Boxed-Types.html'
  85         Links['GEnum'] = dir + '/gobject/gobject-Enumeration-and-Flag-Types.html'
  86         Links['GFlags'] = dir + '/gobject/gobject-Enumeration-and-Flag-Types.html'
  87         Links['GInterface'] = dir + '/gobject/GTypeModule.html'
  88
  89         if dir != html_dir:
  90             logging.info('Scanning GLib directory: %s', dir)
  91             ScanIndices(dir, (re.search(prefix_match, dir) is None), dir_cache)
  92     else:
  93         NoLinks.add('GBoxed')
  94         NoLinks.add('GEnum')
  95         NoLinks.add('GFlags')
  96         NoLinks.add('GInterface')
  97
  98     path = os.environ.get('GNOME2_PATH')
  99     if path:
 100         for dir in path.split(':'):
 101             dir += 'share/gtk-doc/html'
 102             if os.path.exists(dir) and dir != html_dir:
 103                 logging.info('Scanning GNOME2_PATH directory: %s', dir)
 104                 ScanIndices(dir, (re.search(prefix_match, dir) is None), dir_cache)
 105
 106     logging.info('Scanning HTML_DIR directory: %s', html_dir)
 107     ScanIndices(html_dir, False, dir_cache)
 108     logging.info('Scanning MODULE_DIR directory: %s', module_dir)
 109     ScanIndices(module_dir, False, dir_cache)
 110
 111     # check all extra dirs, but skip already scanned dirs or subdirs of those
 112     for dir in extra_dirs:
 113         dir = dir.rstrip('/')
 114         logging.info('Scanning EXTRA_DIR directory: %s', dir)
 115
 116         # If the --extra-dir option is not relative and is not sharing the same
 117         # prefix as the target directory of the docs, we need to use absolute
 118         # directories for the links
 119         if not dir.startswith('..') and re.search(prefix_match, dir) is None:
 120             ScanIndices(dir, True, dir_cache)
 121         else:
 122             ScanIndices(dir, False, dir_cache)
 123
 124
 125 def ScanIndices(scan_dir, use_absolute_links, dir_cache):
 126     if not scan_dir or scan_dir in dir_cache:
 127         return
 128     dir_cache[scan_dir] = 1
 129
 130     logging.info('Scanning index directory: %s, absolute: %d', scan_dir, use_absolute_links)
 131
 132     # TODO(ensonic): this code is the same as in rebase.py
 133     if not os.path.isdir(scan_dir):
 134         logging.info('Cannot open dir "%s"', scan_dir)
 135         return
 136
 137     subdirs = []
 138     for entry in sorted(os.listdir(scan_dir)):
 139         full_entry = os.path.join(scan_dir, entry)
 140         if os.path.isdir(full_entry):
 141             subdirs.append(full_entry)
 142             continue
 143
 144         if entry.endswith('.devhelp2'):
 145             # if devhelp-file is good don't read index.sgml
 146             ReadDevhelp(full_entry, use_absolute_links)
 147         elif entry == "index.sgml.gz" and not os.path.exists(os.path.join(scan_dir, 'index.sgml')):
 148             # debian/ubuntu started to compress this as index.sgml.gz :/
 149             print(''' Please fix https://bugs.launchpad.net/ubuntu/+source/gtk-doc/+bug/77138 . For now run:
 150 gunzip %s
 151 ''' % full_entry)
 152         elif entry.endswith('.devhelp2.gz') and not os.path.exists(full_entry[:-3]):
 153             # debian/ubuntu started to compress this as *devhelp2.gz :/
 154             print('''Please fix https://bugs.launchpad.net/ubuntu/+source/gtk-doc/+bug/1466210 . For now run:
 155 gunzip %s
 156 ''' % full_entry)
 157         # we could consider supporting: gzip module
 158
 159     # Now recursively scan the subdirectories.
 160     for subdir in subdirs:
 161         ScanIndices(subdir, use_absolute_links, dir_cache)
 162
 163
 164 def ReadDevhelp(file, use_absolute_links):
 165     # Determine the absolute directory, to be added to links in $file
 166     # if we need to use an absolute link.
 167     # $file will be something like /prefix/gnome/share/gtk-doc/html/gtk/$file
 168     # We want the part up to 'html/.*' since the links in $file include
 169     # the rest.
 170     dir = "../"
 171     if use_absolute_links:
 172         # For uninstalled index files we'd need to map the path to where it
 173         # will be installed to
 174         if not file.startswith('./'):
 175             m = re.search(r'(.*\/)(.*?)\/.*?\.devhelp2', file)
 176             dir = m.group(1) + m.group(2) + '/'
 177     else:
 178         m = re.search(r'(.*\/)(.*?)\/.*?\.devhelp2', file)
 179         if m:
 180             dir += m.group(2) + '/'
 181         else:
 182             dir = ''
 183
 184     logging.info('Scanning index file=%s, absolute=%d, dir=%s', file, use_absolute_links, dir)
 185
 186     for line in common.open_text(file):
 187         m = re.search(r' link="([^#]*)#([^"]*)"', line)
 188         if m:
 189             link = m.group(1) + '#' + m.group(2)
 190             logging.debug('Found id: %s href: %s', m.group(2), link)
 191             Links[m.group(2)] = dir + link
 192
 193
 194 def ReadSections(module):
 195     """We don't warn on missing links to non-public sysmbols."""
 196     for line in common.open_text(module + '-sections.txt'):
 197         m1 = re.search(r'^<SUBSECTION\s*(.*)>', line)
 198         if line.startswith('#') or line.strip() == '':
 199             continue
 200         elif line.startswith('<SECTION>'):
 201             subsection = ''
 202         elif m1:
 203             subsection = m1.group(1)
 204         elif line.startswith('<SUBSECTION>') or line.startswith('</SECTION>'):
 205             continue
 206         elif re.search(r'^<TITLE>(.*)<\/TITLE>', line):
 207             continue
 208         elif re.search(r'^<FILE>(.*)<\/FILE>', line):
 209             continue
 210         elif re.search(r'^<INCLUDE>(.*)<\/INCLUDE>', line):
 211             continue
 212         else:
 213             symbol = line.strip()
 214             if subsection == "Standard" or subsection == "Private":
 215                 NoLinks.add(common.CreateValidSGMLID(symbol))
 216
 217
 218 def FixCrossReferences(module_dir, module, src_lang):
 219     # TODO(ensonic): use glob.glob()?
 220     for entry in sorted(os.listdir(module_dir)):
 221         full_entry = os.path.join(module_dir, entry)
 222         if os.path.isdir(full_entry):
 223             continue
 224         elif entry.endswith('.html') or entry.endswith('.htm'):
 225             FixHTMLFile(src_lang, module, full_entry)
 226
 227
 228 def FixHTMLFile(src_lang, module, file):
 229     logging.info('Fixing file: %s', file)
 230
 231     content = common.open_text(file).read()
 232
 233     if config.highlight:
 234         # FIXME: ideally we'd pass a clue about the example language to the highligher
 235         # unfortunately the "language" attribute is not appearing in the html output
 236         # we could patch the customization to have <code class="xxx"> inside of <pre>
 237         if config.highlight.endswith('vim'):
 238             def repl_func(m):
 239                 return HighlightSourceVim(src_lang, m.group(1), m.group(2))
 240             content = re.sub(
 241                 r'<div class=\"(example-contents|informalexample)\"><pre class=\"programlisting\">(.*?)</pre></div>',
 242                 repl_func, content, flags=re.DOTALL)
 243         else:
 244             def repl_func(m):
 245                 return HighlightSource(src_lang, m.group(1), m.group(2))
 246             content = re.sub(
 247                 r'<div class=\"(example-contents|informalexample)\"><pre class=\"programlisting\">(.*?)</pre></div>',
 248                 repl_func, content, flags=re.DOTALL)
 249
 250         content = re.sub(r'\&lt;GTKDOCLINK\s+HREF=\&quot;(.*?)\&quot;\&gt;(.*?)\&lt;/GTKDOCLINK\&gt;',
 251                          r'\<GTKDOCLINK\ HREF=\"\1\"\>\2\</GTKDOCLINK\>', content, flags=re.DOTALL)
 252
 253         # From the highlighter we get all the functions marked up. Now we can turn them into GTKDOCLINK items
 254         def repl_func(m):
 255             return MakeGtkDocLink(m.group(1), m.group(2), m.group(3))
 256         content = re.sub(r'(<span class=\"function\">)(.*?)(</span>)', repl_func, content, flags=re.DOTALL)
 257         # We can also try the first item in stuff marked up as 'normal'
 258         content = re.sub(
 259             r'(<span class=\"normal\">\s*)(.+?)((\s+.+?)?\s*</span>)', repl_func, content, flags=re.DOTALL)
 260
 261     lines = content.rstrip().split('\n')
 262
 263     def repl_func_with_ix(i):
 264         def repl_func(m):
 265             return MakeXRef(module, file, i + 1, m.group(1), m.group(2))
 266         return repl_func
 267
 268     for i in range(len(lines)):
 269         lines[i] = re.sub(r'<GTKDOCLINK\s+HREF="([^"]*)"\s*>(.*?)</GTKDOCLINK\s*>', repl_func_with_ix(i), lines[i])
 270         if 'GTKDOCLINK' in lines[i]:
 271             logging.info('make xref failed for line %d: "%s"', i, lines[i])
 272
 273     new_file = file + '.new'
 274     content = '\n'.join(lines)
 275     with common.open_text(new_file, 'w') as h:
 276         h.write(content)
 277
 278     os.unlink(file)
 279     os.rename(new_file, file)
 280
 281
 282 def GetXRef(id):
 283     href = Links.get(id)
 284     if href:
 285         return (id, href)
 286
 287     # This is a workaround for some inconsistency we have with CreateValidSGMLID
 288     if ':' in id:
 289         tid = id.replace(':', '--')
 290         href = Links.get(tid)
 291         if href:
 292             return (tid, href)
 293
 294     # poor mans plural support
 295     if id.endswith('s'):
 296         tid = id[:-1]
 297         href = Links.get(tid)
 298         if href:
 299             return (tid, href)
 300         tid += '-struct'
 301         href = Links.get(tid)
 302         if href:
 303             return (tid, href)
 304
 305     tid = id + '-struct'
 306     href = Links.get(tid)
 307     if href:
 308         return (tid, href)
 309
 310     return (id, None)
 311
 312
 313 def ReportBadXRef(file, line, id, text):
 314     logging.info('no link for: id=%s, linktext=%s', id, text)
 315
 316     # don't warn multiple times and also skip blacklisted (ctypes)
 317     if id in NoLinks:
 318         return
 319     # if it's a function, don't warn if it does not contain a "_"
 320     # (transformed to "-")
 321     # - gnome coding style would use '_'
 322     # - will avoid wrong warnings for ansi c functions
 323     if re.search(r' class=\"function\"', text) and '-' not in id:
 324         return
 325     # if it's a 'return value', don't warn (implicitly created link)
 326     if re.search(r' class=\"returnvalue\"', text):
 327         return
 328     # if it's a 'type', don't warn if it starts with lowercase
 329     # - gnome coding style would use CamelCase
 330     if re.search(r' class=\"type\"', text) and id[0].islower():
 331         return
 332     # don't warn for self links
 333     if text == id:
 334         return
 335
 336     common.LogWarning(file, line, 'no link for: "%s" -> (%s).' % (id, text))
 337     NoLinks.add(id)
 338
 339
 340 def MakeRelativeXRef(module, href):
 341     # if it is a link to same module, remove path to make it work uninstalled
 342     m = re.search(r'^\.\./' + module + '/(.*)$', href)
 343     if m:
 344         href = m.group(1)
 345     return href
 346
 347
 348 def MakeXRef(module, file, line, id, text):
 349     href = GetXRef(id)[1]
 350
 351     if href:
 352         href = MakeRelativeXRef(module, href)
 353         logging.info('Fixing link: %s, %s, %s', id, href, text)
 354         return "<a href=\"%s\">%s</a>" % (href, text)
 355     else:
 356         ReportBadXRef(file, line, id, text)
 357         return text
 358
 359
 360 def MakeGtkDocLink(pre, symbol, post):
 361     id = common.CreateValidSGMLID(symbol)
 362
 363     # these are implicitely created links in highlighed sources
 364     # we don't want warnings for those if the links cannot be resolved.
 365     NoLinks.add(id)
 366
 367     return pre + '<GTKDOCLINK HREF="' + id + '">' + symbol + '</GTKDOCLINK>' + post
 368
 369
 370 def HighlightSource(src_lang, type, source):
 371     # write source to a temp file
 372     # FIXME: use .c for now to hint the language to the highlighter
 373     with tempfile.NamedTemporaryFile(mode='w+', suffix='.c') as f:
 374         temp_source_file = HighlightSourcePreProcess(f, source)
 375         highlight_options = config.highlight_options.replace('$SRC_LANG', src_lang)
 376
 377         logging.info('running %s %s %s', config.highlight, highlight_options, temp_source_file)
 378
 379         # format source
 380         highlighted_source = subprocess.check_output(
 381             [config.highlight] + shlex.split(highlight_options) + [temp_source_file]).decode('utf-8')
 382         logging.debug('result: [%s]', highlighted_source)
 383         if config.highlight.endswith('/source-highlight'):
 384             highlighted_source = re.sub(r'^<\!-- .*? -->', '', highlighted_source, flags=re.MULTILINE | re.DOTALL)
 385             highlighted_source = re.sub(
 386                 r'<pre><tt>(.*?)</tt></pre>', r'\1', highlighted_source, flags=re.MULTILINE | re.DOTALL)
 387         elif config.highlight.endswith('/highlight'):
 388             # need to rewrite the stylesheet classes
 389             highlighted_source = highlighted_source.replace('<span class="gtkdoc com">', '<span class="comment">')
 390             highlighted_source = highlighted_source.replace('<span class="gtkdoc dir">', '<span class="preproc">')
 391             highlighted_source = highlighted_source.replace('<span class="gtkdoc kwd">', '<span class="function">')
 392             highlighted_source = highlighted_source.replace('<span class="gtkdoc kwa">', '<span class="keyword">')
 393             highlighted_source = highlighted_source.replace('<span class="gtkdoc line">', '<span class="linenum">')
 394             highlighted_source = highlighted_source.replace('<span class="gtkdoc num">', '<span class="number">')
 395             highlighted_source = highlighted_source.replace('<span class="gtkdoc str">', '<span class="string">')
 396             highlighted_source = highlighted_source.replace('<span class="gtkdoc sym">', '<span class="symbol">')
 397             # maybe also do
 398             # highlighted_source = re.sub(r'</span>(.+)<span', '</span><span class="normal">\1</span><span')
 399
 400     return HighlightSourcePostprocess(type, highlighted_source)
 401
 402
 403 def HighlightSourceVim(src_lang, type, source):
 404     # write source to a temp file
 405     with tempfile.NamedTemporaryFile(mode='w+', suffix='.h') as f:
 406         temp_source_file = HighlightSourcePreProcess(f, source)
 407
 408         # format source
 409         # TODO(ensonic): use p.communicate()
 410         script = "echo 'let html_number_lines=0|let html_use_css=1|let html_use_xhtml=1|e %s|syn on|set syntax=%s|run! plugin/tohtml.vim|run! syntax/2html.vim|w! %s.html|qa!' | " % (
 411             temp_source_file, src_lang, temp_source_file)
 412         script += "%s -n -e -u NONE -T xterm >/dev/null" % config.highlight
 413         subprocess.check_call([script], shell=True)
 414
 415         highlighted_source = common.open_text(temp_source_file + ".html").read()
 416         highlighted_source = re.sub(r'.*<pre\b[^>]*>\n', '', highlighted_source, flags=re.DOTALL)
 417         highlighted_source = re.sub(r'</pre>.*', '', highlighted_source, flags=re.DOTALL)
 418
 419         # need to rewrite the stylesheet classes
 420         highlighted_source = highlighted_source.replace('<span class="Comment">', '<span class="comment">')
 421         highlighted_source = highlighted_source.replace('<span class="PreProc">', '<span class="preproc">')
 422         highlighted_source = highlighted_source.replace('<span class="Statement">', '<span class="keyword">')
 423         highlighted_source = highlighted_source.replace('<span class="Identifier">', '<span class="function">')
 424         highlighted_source = highlighted_source.replace('<span class="Constant">', '<span class="number">')
 425         highlighted_source = highlighted_source.replace('<span class="Special">', '<span class="symbol">')
 426         highlighted_source = highlighted_source.replace('<span class="Type">', '<span class="type">')
 427
 428         # remove temp files
 429         os.unlink(temp_source_file + '.html')
 430
 431     return HighlightSourcePostprocess(type, highlighted_source)
 432
 433
 434 def HighlightSourcePreProcess(f, source):
 435     # chop of leading and trailing empty lines, leave leading space in first real line
 436     source = source.strip(' ')
 437     source = source.strip('\n')
 438     source = source.rstrip()
 439
 440     # cut common indent
 441     m = re.search(r'^(\s+)', source)
 442     if m:
 443         source = re.sub(r'^' + m.group(1), '', source, flags=re.MULTILINE)
 444     # avoid double entity replacement
 445     source = source.replace('&lt;', '<')
 446     source = source.replace('&gt;', '>')
 447     source = source.replace('&amp;', '&')
 448     if sys.version_info < (3,):
 449         source = source.encode('utf-8')
 450     f.write(source)
 451     f.flush()
 452     return f.name
 453
 454
 455 def HighlightSourcePostprocess(type, highlighted_source):
 456     # chop of leading and trailing empty lines
 457     highlighted_source = highlighted_source.strip()
 458
 459     # turn common urls in comments into links
 460     highlighted_source = re.sub(r'<span class="url">(.*?)</span>',
 461                                 r'<span class="url"><a href="\1">\1</a></span>',
 462                                 highlighted_source, flags=re.DOTALL)
 463
 464     # we do own line-numbering
 465     line_count = highlighted_source.count('\n')
 466     source_lines = '\n'.join([str(i) for i in range(1, line_count + 2)])
 467
 468     return """<div class="%s">
 469   <table class="listing_frame" border="0" cellpadding="0" cellspacing="0">
 470     <tbody>
 471       <tr>
 472         <td class="listing_lines" align="right"><pre>%s</pre></td>
 473         <td class="listing_code"><pre class="programlisting">%s</pre></td>
 474       </tr>
 475     </tbody>
 476   </table>
 477 </div>
 478 """ % (type, source_lines, highlighted_source)