3 # gtk-doc - GTK DocBook documentation generator.
4 # Copyright (C) 1998 Damon Chaplin
5 # 2007-2016 Stefan Sauer
7 # This program is free software; you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License as published by
9 # the Free Software Foundation; either version 2 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU General Public License for more details.
17 # You should have received a copy of the GNU General Public License
18 # along with this program; if not, write to the Free Software
19 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
22 ''"Fix cross-references in the HTML documentation.''"
32 from . import common
, config
34 # This contains all the entities and their relative URLs.
37 # failing link targets we don't warn about even once
53 logging
.info('options: %s', str(options
.__dict
__))
55 LoadIndicies(options
.module_dir
, options
.html_dir
, options
.extra_dir
)
56 ReadSections(options
.module
)
57 FixCrossReferences(options
.module_dir
, options
.module
, options
.src_lang
)
60 # TODO(ensonic): try to refactor so that we get a list of path's and then just
62 # - module_dir is by default 'html'
63 # - html_dir can be set by configure, defaults to $(docdir)
64 def LoadIndicies(module_dir
, html_dir
, extra_dirs
):
65 # Cache of dirs we already scanned for index files
69 m
= re
.search(r
'(.*?)/share/gtk-doc/html', html_dir
)
71 path_prefix
= m
.group(1)
72 logging
.info('Path prefix: %s', path_prefix
)
73 prefix_match
= r
'^' + re
.escape(path_prefix
) + r
'/'
75 # We scan the directory containing GLib and any directories in GNOME2_PATH
76 # first, but these will be overriden by any later scans.
77 dir = common
.GetModuleDocDir('glib-2.0')
78 if dir and os
.path
.exists(dir):
79 # Some predefined link targets to get links into type hierarchies as these
80 # have no targets. These are always absolute for now.
81 Links
['GBoxed'] = dir + '/gobject/gobject-Boxed-Types.html'
82 Links
['GEnum'] = dir + '/gobject/gobject-Enumeration-and-Flag-Types.html'
83 Links
['GFlags'] = dir + '/gobject/gobject-Enumeration-and-Flag-Types.html'
84 Links
['GInterface'] = dir + '/gobject/GTypeModule.html'
87 logging
.info('Scanning GLib directory: %s', dir)
88 ScanIndices(dir, (re
.search(prefix_match
, dir) is None), dir_cache
)
93 NoLinks
.add('GInterface')
95 path
= os
.environ
.get('GNOME2_PATH')
97 for dir in path
.split(':'):
98 dir += 'share/gtk-doc/html'
99 if os
.path
.exists(dir) and dir != html_dir
:
100 logging
.info('Scanning GNOME2_PATH directory: %s', dir)
101 ScanIndices(dir, (re
.search(prefix_match
, dir) is None), dir_cache
)
103 logging
.info('Scanning HTML_DIR directory: %s', html_dir
)
104 ScanIndices(html_dir
, False, dir_cache
)
105 logging
.info('Scanning MODULE_DIR directory: %s', module_dir
)
106 ScanIndices(module_dir
, False, dir_cache
)
108 # check all extra dirs, but skip already scanned dirs or subdirs of those
109 for dir in extra_dirs
:
110 dir = dir.rstrip('/')
111 logging
.info('Scanning EXTRA_DIR directory: %s', dir)
113 # If the --extra-dir option is not relative and is not sharing the same
114 # prefix as the target directory of the docs, we need to use absolute
115 # directories for the links
116 if not dir.startswith('..') and re
.search(prefix_match
, dir) is None:
117 ScanIndices(dir, True, dir_cache
)
119 ScanIndices(dir, False, dir_cache
)
122 def ScanIndices(scan_dir
, use_absolute_links
, dir_cache
):
123 if not scan_dir
or scan_dir
in dir_cache
:
125 dir_cache
[scan_dir
] = 1
127 logging
.info('Scanning index directory: %s, absolute: %d', scan_dir
, use_absolute_links
)
129 # TODO(ensonic): this code is the same as in rebase.py
130 if not os
.path
.isdir(scan_dir
):
131 logging
.info('Cannot open dir "%s"', scan_dir
)
135 for entry
in sorted(os
.listdir(scan_dir
)):
136 full_entry
= os
.path
.join(scan_dir
, entry
)
137 if os
.path
.isdir(full_entry
):
138 subdirs
.append(full_entry
)
141 if entry
.endswith('.devhelp2'):
142 # if devhelp-file is good don't read index.sgml
143 ReadDevhelp(full_entry
, use_absolute_links
)
144 elif entry
== "index.sgml.gz" and not os
.path
.exists(os
.path
.join(scan_dir
, 'index.sgml')):
145 # debian/ubuntu started to compress this as index.sgml.gz :/
146 print(''' Please fix https://bugs.launchpad.net/ubuntu/+source/gtk-doc/+bug/77138 . For now run:
149 elif entry
.endswith('.devhelp2.gz') and not os
.path
.exists(full_entry
[:-3]):
150 # debian/ubuntu started to compress this as *devhelp2.gz :/
151 print('''Please fix https://bugs.launchpad.net/ubuntu/+source/gtk-doc/+bug/1466210 . For now run:
154 # we could consider supporting: gzip module
156 # Now recursively scan the subdirectories.
157 for subdir
in subdirs
:
158 ScanIndices(subdir
, use_absolute_links
, dir_cache
)
161 def ReadDevhelp(file, use_absolute_links
):
162 # Determine the absolute directory, to be added to links in $file
163 # if we need to use an absolute link.
164 # $file will be something like /prefix/gnome/share/gtk-doc/html/gtk/$file
165 # We want the part up to 'html/.*' since the links in $file include
168 if use_absolute_links
:
169 # For uninstalled index files we'd need to map the path to where it
170 # will be installed to
171 if not file.startswith('./'):
172 m
= re
.search(r
'(.*\/)(.*?)\/.*?\.devhelp2', file)
173 dir = m
.group(1) + m
.group(2) + '/'
175 m
= re
.search(r
'(.*\/)(.*?)\/.*?\.devhelp2', file)
177 dir += m
.group(2) + '/'
181 logging
.info('Scanning index file=%s, absolute=%d, dir=%s', file, use_absolute_links
, dir)
183 for line
in open(file, 'r', encoding
='utf-8'):
184 m
= re
.search(r
' link="([^#]*)#([^"]*)"', line
)
186 link
= m
.group(1) + '#' + m
.group(2)
187 logging
.debug('Found id: %s href: %s', m
.group(2), link
)
188 Links
[m
.group(2)] = dir + link
191 def ReadSections(module
):
192 """We don't warn on missing links to non-public sysmbols."""
193 for line
in open(module
+ '-sections.txt', 'r', encoding
='utf-8'):
194 m1
= re
.search(r
'^<SUBSECTION\s*(.*)>', line
)
195 if line
.startswith('#') or line
.strip() == '':
197 elif line
.startswith('<SECTION>'):
200 subsection
= m1
.group(1)
201 elif line
.startswith('<SUBSECTION>') or line
.startswith('</SECTION>'):
203 elif re
.search(r
'^<TITLE>(.*)<\/TITLE>', line
):
205 elif re
.search(r
'^<FILE>(.*)<\/FILE>', line
):
207 elif re
.search(r
'^<INCLUDE>(.*)<\/INCLUDE>', line
):
210 symbol
= line
.strip()
211 if subsection
== "Standard" or subsection
== "Private":
212 NoLinks
.add(common
.CreateValidSGMLID(symbol
))
215 def FixCrossReferences(module_dir
, module
, src_lang
):
216 # TODO(ensonic): use glob.glob()?
217 for entry
in sorted(os
.listdir(module_dir
)):
218 full_entry
= os
.path
.join(module_dir
, entry
)
219 if os
.path
.isdir(full_entry
):
221 elif entry
.endswith('.html') or entry
.endswith('.htm'):
222 FixHTMLFile(src_lang
, module
, full_entry
)
225 def FixHTMLFile(src_lang
, module
, file):
226 logging
.info('Fixing file: %s', file)
228 content
= open(file, 'r', encoding
='utf-8').read()
231 # FIXME: ideally we'd pass a clue about the example language to the highligher
232 # unfortunately the "language" attribute is not appearing in the html output
233 # we could patch the customization to have <code class="xxx"> inside of <pre>
234 if config
.highlight
.endswith('vim'):
236 return HighlightSourceVim(src_lang
, m
.group(1), m
.group(2))
238 r
'<div class=\"(example-contents|informalexample)\"><pre class=\"programlisting\">(.*?)</pre></div>',
239 repl_func
, content
, flags
=re
.DOTALL
)
242 return HighlightSource(src_lang
, m
.group(1), m
.group(2))
244 r
'<div class=\"(example-contents|informalexample)\"><pre class=\"programlisting\">(.*?)</pre></div>',
245 repl_func
, content
, flags
=re
.DOTALL
)
247 content
= re
.sub(r
'\<GTKDOCLINK\s+HREF=\"(.*?)\"\>(.*?)\</GTKDOCLINK\>',
248 r
'\<GTKDOCLINK\ HREF=\"\1\"\>\2\</GTKDOCLINK\>', content
, flags
=re
.DOTALL
)
250 # From the highlighter we get all the functions marked up. Now we can turn them into GTKDOCLINK items
252 return MakeGtkDocLink(m
.group(1), m
.group(2), m
.group(3))
253 content
= re
.sub(r
'(<span class=\"function\">)(.*?)(</span>)', repl_func
, content
, flags
=re
.DOTALL
)
254 # We can also try the first item in stuff marked up as 'normal'
256 r
'(<span class=\"normal\">\s*)(.+?)((\s+.+?)?\s*</span>)', repl_func
, content
, flags
=re
.DOTALL
)
258 lines
= content
.rstrip().split('\n')
260 def repl_func_with_ix(i
):
262 return MakeXRef(module
, file, i
+ 1, m
.group(1), m
.group(2))
265 for i
in range(len(lines
)):
266 lines
[i
] = re
.sub(r
'<GTKDOCLINK\s+HREF="([^"]*)"\s*>(.*?)</GTKDOCLINK\s*>', repl_func_with_ix(i
), lines
[i
])
267 if 'GTKDOCLINK' in lines
[i
]:
268 logging
.info('make xref failed for line %d: "%s"', i
, lines
[i
])
270 new_file
= file + '.new'
271 content
= '\n'.join(lines
)
272 with
open(new_file
, 'w', encoding
='utf-8') as h
:
276 os
.rename(new_file
, file)
284 # This is a workaround for some inconsistency we have with CreateValidSGMLID
286 tid
= id.replace(':', '--')
287 href
= Links
.get(tid
)
291 # poor mans plural support
294 href
= Links
.get(tid
)
298 href
= Links
.get(tid
)
303 href
= Links
.get(tid
)
310 def ReportBadXRef(file, line
, id, text
):
311 logging
.info('no link for: id=%s, linktext=%s', id, text
)
313 # don't warn multiple times and also skip blacklisted (ctypes)
316 # if it's a function, don't warn if it does not contain a "_"
317 # (transformed to "-")
318 # - gnome coding style would use '_'
319 # - will avoid wrong warnings for ansi c functions
320 if re
.search(r
' class=\"function\"', text
) and '-' not in id:
322 # if it's a 'return value', don't warn (implicitly created link)
323 if re
.search(r
' class=\"returnvalue\"', text
):
325 # if it's a 'type', don't warn if it starts with lowercase
326 # - gnome coding style would use CamelCase
327 if re
.search(r
' class=\"type\"', text
) and id[0].islower():
329 # don't warn for self links
333 common
.LogWarning(file, line
, 'no link for: "%s" -> (%s).' % (id, text
))
337 def MakeRelativeXRef(module
, href
):
338 # if it is a link to same module, remove path to make it work uninstalled
339 m
= re
.search(r
'^\.\./' + module
+ '/(.*)$', href
)
345 def MakeXRef(module
, file, line
, id, text
):
346 href
= GetXRef(id)[1]
349 href
= MakeRelativeXRef(module
, href
)
350 logging
.info('Fixing link: %s, %s, %s', id, href
, text
)
351 return "<a href=\"%s\">%s</a>" % (href
, text
)
353 ReportBadXRef(file, line
, id, text
)
357 def MakeGtkDocLink(pre
, symbol
, post
):
358 id = common
.CreateValidSGMLID(symbol
)
360 # these are implicitely created links in highlighed sources
361 # we don't want warnings for those if the links cannot be resolved.
364 return pre
+ '<GTKDOCLINK HREF="' + id + '">' + symbol
+ '</GTKDOCLINK>' + post
367 def HighlightSource(src_lang
, type, source
):
368 # write source to a temp file
369 # FIXME: use .c for now to hint the language to the highlighter
370 with tempfile
.NamedTemporaryFile(mode
='w+', suffix
='.c') as f
:
371 temp_source_file
= HighlightSourcePreProcess(f
, source
)
372 highlight_options
= config
.highlight_options
.replace('$SRC_LANG', src_lang
)
374 logging
.info('running %s %s %s', config
.highlight
, highlight_options
, temp_source_file
)
377 highlighted_source
= subprocess
.check_output(
378 [config
.highlight
] + shlex
.split(highlight_options
) + [temp_source_file
]).decode('utf-8')
379 logging
.debug('result: [%s]', highlighted_source
)
380 if config
.highlight
.endswith('/source-highlight'):
381 highlighted_source
= re
.sub(r
'^<\!-- .*? -->', '', highlighted_source
, flags
=re
.MULTILINE | re
.DOTALL
)
382 highlighted_source
= re
.sub(
383 r
'<pre><tt>(.*?)</tt></pre>', r
'\1', highlighted_source
, flags
=re
.MULTILINE | re
.DOTALL
)
384 elif config
.highlight
.endswith('/highlight'):
385 # need to rewrite the stylesheet classes
386 highlighted_source
= highlighted_source
.replace('<span class="gtkdoc com">', '<span class="comment">')
387 highlighted_source
= highlighted_source
.replace('<span class="gtkdoc dir">', '<span class="preproc">')
388 highlighted_source
= highlighted_source
.replace('<span class="gtkdoc kwd">', '<span class="function">')
389 highlighted_source
= highlighted_source
.replace('<span class="gtkdoc kwa">', '<span class="keyword">')
390 highlighted_source
= highlighted_source
.replace('<span class="gtkdoc line">', '<span class="linenum">')
391 highlighted_source
= highlighted_source
.replace('<span class="gtkdoc num">', '<span class="number">')
392 highlighted_source
= highlighted_source
.replace('<span class="gtkdoc str">', '<span class="string">')
393 highlighted_source
= highlighted_source
.replace('<span class="gtkdoc sym">', '<span class="symbol">')
395 # highlighted_source = re.sub(r'</span>(.+)<span', '</span><span class="normal">\1</span><span')
397 return HighlightSourcePostprocess(type, highlighted_source
)
400 def HighlightSourceVim(src_lang
, type, source
):
401 # write source to a temp file
402 with tempfile
.NamedTemporaryFile(mode
='w+', suffix
='.h') as f
:
403 temp_source_file
= HighlightSourcePreProcess(f
, source
)
406 # TODO(ensonic): use p.communicate()
407 script
= "echo 'let html_number_lines=0|let html_use_css=1|let html_use_xhtml=1|e %s|syn on|set syntax=%s|run! plugin/tohtml.vim|run! syntax/2html.vim|w! %s.html|qa!' | " % (
408 temp_source_file
, src_lang
, temp_source_file
)
409 script
+= "%s -n -e -u NONE -T xterm >/dev/null" % config
.highlight
410 subprocess
.check_call([script
], shell
=True)
412 highlighted_source
= open(temp_source_file
+ ".html", 'r', encoding
='utf-8').read()
413 highlighted_source
= re
.sub(r
'.*<pre\b[^>]*>\n', '', highlighted_source
, flags
=re
.DOTALL
)
414 highlighted_source
= re
.sub(r
'</pre>.*', '', highlighted_source
, flags
=re
.DOTALL
)
416 # need to rewrite the stylesheet classes
417 highlighted_source
= highlighted_source
.replace('<span class="Comment">', '<span class="comment">')
418 highlighted_source
= highlighted_source
.replace('<span class="PreProc">', '<span class="preproc">')
419 highlighted_source
= highlighted_source
.replace('<span class="Statement">', '<span class="keyword">')
420 highlighted_source
= highlighted_source
.replace('<span class="Identifier">', '<span class="function">')
421 highlighted_source
= highlighted_source
.replace('<span class="Constant">', '<span class="number">')
422 highlighted_source
= highlighted_source
.replace('<span class="Special">', '<span class="symbol">')
423 highlighted_source
= highlighted_source
.replace('<span class="Type">', '<span class="type">')
426 os
.unlink(temp_source_file
+ '.html')
428 return HighlightSourcePostprocess(type, highlighted_source
)
431 def HighlightSourcePreProcess(f
, source
):
432 # chop of leading and trailing empty lines, leave leading space in first real line
433 source
= source
.strip(' ')
434 source
= source
.strip('\n')
435 source
= source
.rstrip()
438 m
= re
.search(r
'^(\s+)', source
)
440 source
= re
.sub(r
'^' + m
.group(1), '', source
, flags
=re
.MULTILINE
)
441 # avoid double entity replacement
442 source
= source
.replace('<', '<')
443 source
= source
.replace('>', '>')
444 source
= source
.replace('&', '&')
445 if sys
.version_info
< (3,):
446 source
= source
.encode('utf-8')
452 def HighlightSourcePostprocess(type, highlighted_source
):
453 # chop of leading and trailing empty lines
454 highlighted_source
= highlighted_source
.strip()
456 # turn common urls in comments into links
457 highlighted_source
= re
.sub(r
'<span class="url">(.*?)</span>',
458 r
'<span class="url"><a href="\1">\1</a></span>',
459 highlighted_source
, flags
=re
.DOTALL
)
461 # we do own line-numbering
462 line_count
= highlighted_source
.count('\n')
463 source_lines
= '\n'.join([str(i
) for i
in range(1, line_count
+ 2)])
465 return """<div class="%s">
466 <table class="listing_frame" border="0" cellpadding="0" cellspacing="0">
469 <td class="listing_lines" align="right"><pre>%s</pre></td>
470 <td class="listing_code"><pre class="programlisting">%s</pre></td>
475 """ % (type, source_lines
, highlighted_source
)