gtkdoc/mkdb.py

   1 # -*- python; coding: utf-8 -*-
   2 #
   3 # gtk-doc - GTK DocBook documentation generator.
   4 # Copyright (C) 1998  Damon Chaplin
   5 #               2007-2016  Stefan Sauer
   6 #
   7 # This program is free software; you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation; either version 2 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program; if not, write to the Free Software
  19 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  20 #
  21
  22 """
  23 Creates the DocBook files from the source comments.
  24 """
  25
  26 from __future__ import print_function
  27 from six import iteritems, iterkeys
  28
  29 from collections import OrderedDict
  30 import logging
  31 import os
  32 import re
  33 import string
  34
  35 from . import common, md_to_db
  36
  37 # Options
  38 MODULE = None
  39 DB_OUTPUT_DIR = None
  40 INLINE_MARKUP_MODE = None
  41 DEFAULT_STABILITY = None
  42 NAME_SPACE = ''
  43 ROOT_DIR = '.'
  44
  45 # These global arrays store information on signals. Each signal has an entry
  46 # in each of these arrays at the same index, like a multi-dimensional array.
  47 SignalObjects = []        # The GtkObject which emits the signal.
  48 SignalNames = []        # The signal name.
  49 SignalReturns = []        # The return type.
  50 SignalFlags = []        # Flags for the signal
  51 SignalPrototypes = []        # The rest of the prototype of the signal handler.
  52
  53 # These global arrays store information on Args. Each Arg has an entry
  54 # in each of these arrays at the same index, like a multi-dimensional array.
  55 ArgObjects = []                # The GtkObject which has the Arg.
  56 ArgNames = []                # The Arg name.
  57 ArgTypes = []                # The Arg type - gint, GtkArrowType etc.
  58 ArgFlags = []                # How the Arg can be used - readable/writable etc.
  59 ArgNicks = []                # The nickname of the Arg.
  60 ArgBlurbs = []          # Docstring of the Arg.
  61 ArgDefaults = []        # Default value of the Arg.
  62 ArgRanges = []                # The range of the Arg type
  63
  64 # These global hashes store declaration info keyed on a symbol name.
  65 Declarations = {}
  66 DeclarationTypes = {}
  67 DeclarationConditional = {}
  68 DeclarationOutput = {}
  69 Deprecated = {}
  70 Since = {}
  71 StabilityLevel = {}
  72 StructHasTypedef = {}
  73
  74 # These global hashes store the existing documentation.
  75 SymbolDocs = {}
  76 SymbolParams = {}
  77 SymbolAnnotations = {}
  78
  79 # These global hashes store documentation scanned from the source files.
  80 SourceSymbolDocs = {}
  81 SourceSymbolParams = {}
  82 SourceSymbolSourceFile = {}
  83 SourceSymbolSourceLine = {}
  84
  85 # all documentation goes in here, so we can do coverage analysis
  86 AllSymbols = {}
  87 AllIncompleteSymbols = {}
  88 AllUnusedSymbols = {}
  89 AllDocumentedSymbols = {}
  90
  91 # Undeclared yet documented symbols
  92 UndeclaredSymbols = {}
  93
  94 # These global arrays store GObject, subclasses and the hierarchy (also of
  95 # non-object derived types).
  96 Objects = []
  97 ObjectLevels = []
  98 ObjectRoots = {}
  99
 100 Interfaces = {}
 101 Prerequisites = {}
 102
 103 # holds the symbols which are mentioned in <MODULE>-sections.txt and in which
 104 # section they are defined
 105 KnownSymbols = {}
 106 SymbolSection = {}
 107 SymbolSectionId = {}
 108
 109 # collects index entries
 110 IndexEntriesFull = {}
 111 IndexEntriesSince = {}
 112 IndexEntriesDeprecated = {}
 113
 114 # Standard C preprocessor directives, which we ignore for '#' abbreviations.
 115 PreProcessorDirectives = {
 116     'assert', 'define', 'elif', 'else', 'endif', 'error', 'if', 'ifdef', 'ifndef',
 117     'include', 'line', 'pragma', 'unassert', 'undef', 'warning'
 118 }
 119
 120 # remember used annotation (to write minimal glossary)
 121 AnnotationsUsed = {}
 122
 123 # the regexp that parses the annotation is in ScanSourceFile()
 124 AnnotationDefinition = {
 125     # the GObjectIntrospection annotations are defined at:
 126     # https://live.gnome.org/GObjectIntrospection/Annotations
 127     'allow-none': "NULL is OK, both for passing and for returning.",
 128     'nullable': "NULL may be passed as the value in, out, in-out; or as a return value.",
 129     'not nullable': "NULL must not be passed as the value in, out, in-out; or as a return value.",
 130     'optional': "NULL may be passed instead of a pointer to a location.",
 131     'array': "Parameter points to an array of items.",
 132     'attribute': "Deprecated free-form custom annotation, replaced by (attributes) annotation.",
 133     'attributes': "Free-form key-value pairs.",
 134     'closure': "This parameter is a 'user_data', for callbacks; many bindings can pass NULL here.",
 135     'constructor': "This symbol is a constructor, not a static method.",
 136     'destroy': "This parameter is a 'destroy_data', for callbacks.",
 137     'default': "Default parameter value (for in case the <acronym>shadows</acronym>-to function has less parameters).",
 138     'element-type': "Generics and defining elements of containers and arrays.",
 139     'error-domains': "Typed errors. Similar to throws in Java.",
 140     'foreign': "This is a foreign struct.",
 141     'get-value-func': "The specified function is used to convert a struct from a GValue, must be a GTypeInstance.",
 142     'in': "Parameter for input. Default is <acronym>transfer none</acronym>.",
 143     'inout': "Parameter for input and for returning results. Default is <acronym>transfer full</acronym>.",
 144     'in-out': "Parameter for input and for returning results. Default is <acronym>transfer full</acronym>.",
 145     'method': "This is a method",
 146     'not-error': "A GError parameter is not to be handled like a normal GError.",
 147     'out': "Parameter for returning results. Default is <acronym>transfer full</acronym>.",
 148     'out caller-allocates': "Out parameter, where caller must allocate storage.",
 149     'out callee-allocates': "Out parameter, where caller must allocate storage.",
 150     'ref-func': "The specified function is used to ref a struct, must be a GTypeInstance.",
 151     'rename-to': "Rename the original symbol's name to SYMBOL.",
 152     'scope call': "The callback is valid only during the call to the method.",
 153     'scope async': "The callback is valid until first called.",
 154     'scope notified': "The callback is valid until the GDestroyNotify argument is called.",
 155     'set-value-func': "The specified function is used to convert from a struct to a GValue, must be a GTypeInstance.",
 156     'skip': "Exposed in C code, not necessarily available in other languages.",
 157     'transfer container': "Free data container after the code is done.",
 158     'transfer floating': "Alias for <acronym>transfer none</acronym>, used for objects with floating refs.",
 159     'transfer full': "Free data after the code is done.",
 160     'transfer none': "Don't free data after the code is done.",
 161     'type': "Override the parsed C type with given type.",
 162     'unref-func': "The specified function is used to unref a struct, must be a GTypeInstance.",
 163     'virtual': "This is the invoker for a virtual method.",
 164     'value': "The specified value overrides the evaluated value of the constant.",
 165     # Stability Level definition
 166     # https://bugzilla.gnome.org/show_bug.cgi?id=170860
 167     'Stable': '''The intention of a Stable interface is to enable arbitrary third parties to
 168 develop applications to these interfaces, release them, and have confidence that
 169 they will run on all minor releases of the product (after the one in which the
 170 interface was introduced, and within the same major release). Even at a major
 171 release, incompatible changes are expected to be rare, and to have strong
 172 justifications.
 173 ''',
 174     'Unstable': '''Unstable interfaces are experimental or transitional. They are typically used to
 175 give outside developers early access to new or rapidly changing technology, or
 176 to provide an interim solution to a problem where a more general solution is
 177 anticipated. No claims are made about either source or binary compatibility from
 178 one minor release to the next.
 179
 180 The Unstable interface level is a warning that these interfaces are  subject to
 181 change without warning and should not be used in unbundled products.
 182
 183 Given such caveats, customer impact need not be a factor when considering
 184 incompatible changes to an Unstable interface in a major or minor release.
 185 Nonetheless, when such changes are introduced, the changes should still be
 186 mentioned in the release notes for the affected release.
 187 ''',
 188     'Private': '''An interface that can be used within the GNOME stack itself, but that is not
 189 documented for end-users.  Such functions should only be used in specified and
 190 documented ways.
 191 ''',
 192 }
 193
 194 # Function and other declaration output settings.
 195 RETURN_TYPE_FIELD_WIDTH = 20
 196 MAX_SYMBOL_FIELD_WIDTH = 40
 197
 198 # XML header
 199 doctype_header = None
 200
 201 # refentry template
 202 REFENTRY = string.Template('''${header}
 203 <refentry id="${section_id}">
 204 <refmeta>
 205 <refentrytitle role="top_of_page" id="${section_id}.top_of_page">${title}</refentrytitle>
 206 <manvolnum>3</manvolnum>
 207 <refmiscinfo>${MODULE} Library${image}</refmiscinfo>
 208 </refmeta>
 209 <refnamediv>
 210 <refname>${title}</refname>
 211 <refpurpose>${short_desc}</refpurpose>
 212 </refnamediv>
 213 ${stability}
 214 ${functions_synop}${args_synop}${signals_synop}${object_anchors}${other_synop}${hierarchy}${prerequisites}${derived}${interfaces}${implementations}
 215 ${include_output}
 216 <refsect1 id="${section_id}.description" role="desc">
 217 <title role="desc.title">Description</title>
 218 ${extralinks}${long_desc}
 219 </refsect1>
 220 <refsect1 id="${section_id}.functions_details" role="details">
 221 <title role="details.title">Functions</title>
 222 ${functions_details}
 223 </refsect1>
 224 <refsect1 id="${section_id}.other_details" role="details">
 225 <title role="details.title">Types and Values</title>
 226 ${other_details}
 227 </refsect1>
 228 ${args_desc}${signals_desc}${see_also}
 229 </refentry>
 230 ''')
 231
 232
 233 def Run(options):
 234     global MODULE, INLINE_MARKUP_MODE, DEFAULT_STABILITY, NAME_SPACE, \
 235         DB_OUTPUT_DIR, doctype_header
 236
 237     # We should pass the options variable around instead of this global variable horror
 238     # but too much of the code expects these to be around. Fix this once the transition is done.
 239     MODULE = options.module
 240     INLINE_MARKUP_MODE = options.xml_mode or options.sgml_mode
 241     DEFAULT_STABILITY = options.default_stability
 242     NAME_SPACE = options.name_space
 243
 244     main_sgml_file = options.main_sgml_file
 245     if not main_sgml_file:
 246         # backwards compatibility
 247         if os.path.exists(MODULE + "-docs.sgml"):
 248             main_sgml_file = MODULE + "-docs.sgml"
 249         else:
 250             main_sgml_file = MODULE + "-docs.xml"
 251
 252     # extract docbook header or define default
 253     doctype_header = GetDocbookHeader(main_sgml_file)
 254
 255     # This is where we put all the DocBook output.
 256     DB_OUTPUT_DIR = DB_OUTPUT_DIR if DB_OUTPUT_DIR else os.path.join(ROOT_DIR, "xml")
 257     if not os.path.isdir(DB_OUTPUT_DIR):
 258         os.mkdir(DB_OUTPUT_DIR)
 259
 260     ReadKnownSymbols(os.path.join(ROOT_DIR, MODULE + "-sections.txt"))
 261     ReadSignalsFile(os.path.join(ROOT_DIR, MODULE + ".signals"))
 262     ReadArgsFile(os.path.join(ROOT_DIR, MODULE + ".args"))
 263     ReadObjectHierarchy(os.path.join(ROOT_DIR, MODULE + ".hierarchy"))
 264     ReadInterfaces(os.path.join(ROOT_DIR, MODULE + ".interfaces"))
 265     ReadPrerequisites(os.path.join(ROOT_DIR, MODULE + ".prerequisites"))
 266
 267     ReadDeclarationsFile(os.path.join(ROOT_DIR, MODULE + "-decl.txt"), 0)
 268     if os.path.isfile(os.path.join(ROOT_DIR, MODULE + "-overrides.txt")):
 269         ReadDeclarationsFile(os.path.join(ROOT_DIR, MODULE + "-overrides.txt"), 1)
 270
 271     # Scan sources
 272     if options.source_suffixes:
 273         suffix_list = ['.' + ext for ext in options.source_suffixes.split(',')]
 274     else:
 275         suffix_list = ['.c', '.h']
 276
 277     source_dirs = options.source_dir
 278     ignore_files = options.ignore_files
 279     logging.info(" ignore files: " + ignore_files)
 280     for sdir in source_dirs:
 281         ReadSourceDocumentation(sdir, suffix_list, source_dirs, ignore_files)
 282
 283     changed, book_top, book_bottom = OutputDB(os.path.join(ROOT_DIR, MODULE + "-sections.txt"), options)
 284     OutputBook(main_sgml_file, book_top, book_bottom)
 285
 286     logging.info("All files created: %d", changed)
 287
 288     # If any of the DocBook files have changed, update the timestamp file (so
 289     # it can be used for Makefile dependencies).
 290     if changed or not os.path.exists(os.path.join(ROOT_DIR, "sgml.stamp")):
 291
 292         # try to detect the common prefix
 293         # GtkWidget, GTK_WIDGET, gtk_widget -> gtk
 294         if NAME_SPACE == '':
 295             NAME_SPACE = DetermineNamespace()
 296
 297         logging.info('namespace prefix ="%s"', NAME_SPACE)
 298
 299         OutputIndex("api-index-full", IndexEntriesFull)
 300         OutputIndex("api-index-deprecated", IndexEntriesDeprecated)
 301         OutputSinceIndexes()
 302         OutputAnnotationGlossary()
 303
 304         with open(os.path.join(ROOT_DIR, 'sgml.stamp'), 'w') as h:
 305             h.write('timestamp')
 306
 307
 308 def OutputObjectList():
 309     """This outputs the alphabetical list of objects, in a columned table."""
 310     # FIXME: Currently this also outputs ancestor objects which may not actually
 311     # be in this module.
 312     cols = 3
 313
 314     # FIXME: use .xml
 315     old_object_index = os.path.join(DB_OUTPUT_DIR, "object_index.sgml")
 316     new_object_index = os.path.join(DB_OUTPUT_DIR, "object_index.new")
 317
 318     OUTPUT = common.open_text(new_object_index, 'w')
 319
 320     OUTPUT.write('''%s
 321 <informaltable pgwide="1" frame="none">
 322 <tgroup cols="%s">
 323 <colspec colwidth="1*"/>
 324 <colspec colwidth="1*"/>
 325 <colspec colwidth="1*"/>
 326 <tbody>
 327 ''' % (MakeDocHeader("informaltable"), cols))
 328
 329     count = 0
 330     object = None
 331     for object in sorted(Objects):
 332         xref = MakeXRef(object)
 333         if count % cols == 0:
 334             OUTPUT.write("<row>\n")
 335         OUTPUT.write("<entry>%s</entry>\n" % xref)
 336         if count % cols == cols - 1:
 337             OUTPUT.write("</row>\n")
 338         count += 1
 339
 340     if count == 0:
 341         # emit an empty row, since empty tables are invalid
 342         OUTPUT.write("<row><entry> </entry></row>\n")
 343
 344     else:
 345         if count % cols > 0:
 346             OUTPUT.write("</row>\n")
 347
 348     OUTPUT.write('''</tbody></tgroup></informaltable>\n''')
 349     OUTPUT.close()
 350
 351     common.UpdateFileIfChanged(old_object_index, new_object_index, 0)
 352
 353
 354 def TrimTextBlock(desc):
 355     """Trims extra whitespace.
 356
 357     Empty lines inside a block are preserved.
 358     Args:
 359         desc (str): the text block to trim. May contain newlines.
 360     """
 361
 362     # strip trailing spaces on every line
 363     return re.sub(r'\s+$', '\n', desc.lstrip(), flags=re.MULTILINE)
 364
 365
 366 def OutputDB(file, options):
 367     """Generate docbook files.
 368
 369     This collects the output for each section of the docs, and outputs each file
 370     when the end of the section is found.
 371
 372     Args:
 373         file (str): the $MODULE-sections.txt file which contains all of the
 374                     functions/macros/structs etc. being documented, organised
 375                     into sections and subsections.
 376         options:    commandline options
 377     """
 378
 379     logging.info("Reading: %s", file)
 380     INPUT = common.open_text(file)
 381     filename = ''
 382     book_top = ''
 383     book_bottom = ''
 384     includes = options.default_includes or ''
 385     section_includes = ''
 386     in_section = 0
 387     title = ''
 388     section_id = ''
 389     subsection = ''
 390     num_symbols = 0
 391     changed = 0
 392     functions_synop = ''
 393     other_synop = ''
 394     functions_details = ''
 395     other_details = ''
 396     signals_synop = ''
 397     signals_desc = ''
 398     args_synop = ''
 399     child_args_synop = ''
 400     style_args_synop = ''
 401     args_desc = ''
 402     child_args_desc = ''
 403     style_args_desc = ''
 404     hierarchy_str = ''
 405     hierarchy = []
 406     interfaces = ''
 407     implementations = ''
 408     prerequisites = ''
 409     derived = ''
 410     file_objects = []
 411     file_def_line = {}
 412     symbol_def_line = {}
 413
 414     MergeSourceDocumentation()
 415
 416     line_number = 0
 417     for line in INPUT:
 418         line_number += 1
 419
 420         if line.startswith('#'):
 421             continue
 422
 423         logging.info("section file data: %d: %s", line_number, line)
 424
 425         m1 = re.search(r'^<SUBSECTION\s*(.*)>', line, re.I)
 426         m2 = re.search(r'^<TITLE>(.*)<\/TITLE', line)
 427         m3 = re.search(r'^<FILE>(.*)<\/FILE>', line)
 428         m4 = re.search(r'^<INCLUDE>(.*)<\/INCLUDE>', line)
 429         m5 = re.search(r'^(\S+)', line)
 430
 431         if line.startswith('<SECTION>'):
 432             num_symbols = 0
 433             in_section = False
 434             file_objects = []
 435             symbol_def_line = {}
 436
 437         elif m1:
 438             other_synop += "\n"
 439             functions_synop += "\n"
 440             subsection = m1.group(1)
 441
 442         elif line.startswith('<SUBSECTION>'):
 443             continue
 444         elif m2:
 445             title = m2.group(1)
 446             logging.info("Section: %s", title)
 447
 448             # We don't want warnings if object & class structs aren't used.
 449             DeclarationOutput[title] = 1
 450             DeclarationOutput["%sClass" % title] = 1
 451             DeclarationOutput["%sIface" % title] = 1
 452             DeclarationOutput["%sInterface" % title] = 1
 453
 454         elif m3:
 455             filename = m3.group(1)
 456             if not filename in file_def_line:
 457                 file_def_line[filename] = line_number
 458             else:
 459                 common.LogWarning(file, line_number, "Double <FILE>%s</FILE> entry. Previous occurrence on line %s." %
 460                                   (filename, file_def_line[filename]))
 461             if title == '':
 462                 key = filename + ":Title"
 463                 if key in SourceSymbolDocs:
 464                     title = SourceSymbolDocs[key].rstrip()
 465
 466         elif m4:
 467             if in_section:
 468                 section_includes = m4.group(1)
 469             else:
 470                 if options.default_includes:
 471                     common.LogWarning(file, line_number, "Default <INCLUDE> being overridden by command line option.")
 472                 else:
 473                     includes = m4.group(1)
 474
 475         elif re.search(r'^<\/SECTION>', line):
 476             logging.info("End of section: %s", title)
 477             # TODO: also output if we have sections docs?
 478             # long_desc = SymbolDocs.get(filename + ":Long_Description")
 479             if num_symbols > 0:
 480                 # collect documents
 481                 book_bottom += "    <xi:include href=\"xml/%s.xml\"/>\n" % filename
 482
 483                 key = filename + ":Include"
 484                 if key in SourceSymbolDocs:
 485                     if section_includes:
 486                         common.LogWarning(file, line_number, "Section <INCLUDE> being overridden by inline comments.")
 487                     section_includes = SourceSymbolDocs[key]
 488
 489                 if section_includes == '':
 490                     section_includes = includes
 491
 492                 signals_synop = re.sub(r'^\n*', '', signals_synop)
 493                 signals_synop = re.sub(r'\n+$', '\n', signals_synop)
 494
 495                 if signals_synop != '':
 496                     signals_synop = '''<refsect1 id="%s.signals" role="signal_proto">
 497 <title role="signal_proto.title">Signals</title>
 498 <informaltable frame="none">
 499 <tgroup cols="3">
 500 <colspec colname="signals_return" colwidth="150px"/>
 501 <colspec colname="signals_name" colwidth="300px"/>
 502 <colspec colname="signals_flags" colwidth="200px"/>
 503 <tbody>
 504 %s
 505 </tbody>
 506 </tgroup>
 507 </informaltable>
 508 </refsect1>
 509 ''' % (section_id, signals_synop)
 510                     signals_desc = TrimTextBlock(signals_desc)
 511                     signals_desc = '''<refsect1 id="%s.signal-details" role="signals">
 512 <title role="signals.title">Signal Details</title>
 513 %s
 514 </refsect1>
 515 ''' % (section_id, signals_desc)
 516
 517                 args_synop = re.sub(r'^\n*', '', args_synop)
 518                 args_synop = re.sub(r'\n+$', '\n', args_synop)
 519                 if args_synop != '':
 520                     args_synop = '''<refsect1 id="%s.properties" role="properties">
 521 <title role="properties.title">Properties</title>
 522 <informaltable frame="none">
 523 <tgroup cols="3">
 524 <colspec colname="properties_type" colwidth="150px"/>
 525 <colspec colname="properties_name" colwidth="300px"/>
 526 <colspec colname="properties_flags" colwidth="200px"/>
 527 <tbody>
 528 %s
 529 </tbody>
 530 </tgroup>
 531 </informaltable>
 532 </refsect1>
 533 ''' % (section_id, args_synop)
 534                     args_desc = TrimTextBlock(args_desc)
 535                     args_desc = '''<refsect1 id="%s.property-details" role="property_details">
 536 <title role="property_details.title">Property Details</title>
 537 %s
 538 </refsect1>
 539 ''' % (section_id, args_desc)
 540
 541                 child_args_synop = re.sub(r'^\n*', '', child_args_synop)
 542                 child_args_synop = re.sub(r'\n+$', '\n', child_args_synop)
 543                 if child_args_synop != '':
 544                     args_synop += '''<refsect1 id="%s.child-properties" role="child_properties">
 545 <title role="child_properties.title">Child Properties</title>
 546 <informaltable frame="none">
 547 <tgroup cols="3">
 548 <colspec colname="child_properties_type" colwidth="150px"/>
 549 <colspec colname="child_properties_name" colwidth="300px"/>
 550 <colspec colname="child_properties_flags" colwidth="200px"/>
 551 <tbody>
 552 %s
 553 </tbody>
 554 </tgroup>
 555 </informaltable>
 556 </refsect1>
 557 ''' % (section_id, child_args_synop)
 558                     child_args_desc = TrimTextBlock(child_args_desc)
 559                     args_desc += '''<refsect1 id="%s.child-property-details" role="child_property_details">
 560 <title role="child_property_details.title">Child Property Details</title>
 561 %s
 562 </refsect1>
 563 ''' % (section_id, child_args_desc)
 564
 565                 style_args_synop = re.sub(r'^\n*', '', style_args_synop)
 566                 style_args_synop = re.sub(r'\n+$', '\n', style_args_synop)
 567                 if style_args_synop != '':
 568                     args_synop += '''<refsect1 id="%s.style-properties" role="style_properties">
 569 <title role="style_properties.title">Style Properties</title>
 570 <informaltable frame="none">
 571 <tgroup cols="3">
 572 <colspec colname="style_properties_type" colwidth="150px"/>
 573 <colspec colname="style_properties_name" colwidth="300px"/>
 574 <colspec colname="style_properties_flags" colwidth="200px"/>
 575 <tbody>
 576 %s
 577 </tbody>
 578 </tgroup>
 579 </informaltable>
 580 </refsect1>
 581 ''' % (section_id, style_args_synop)
 582                     style_args_desc = TrimTextBlock(style_args_desc)
 583                     args_desc += '''<refsect1 id="%s.style-property-details" role="style_properties_details">
 584 <title role="style_properties_details.title">Style Property Details</title>
 585 %s
 586 </refsect1>
 587 ''' % (section_id, style_args_desc)
 588
 589                 hierarchy_str = AddTreeLineArt(hierarchy)
 590                 if hierarchy_str != '':
 591                     hierarchy_str = '''<refsect1 id="%s.object-hierarchy" role="object_hierarchy">
 592 <title role="object_hierarchy.title">Object Hierarchy</title>
 593 <screen>%s
 594 </screen>
 595 </refsect1>
 596 ''' % (section_id, hierarchy_str)
 597
 598                 interfaces = TrimTextBlock(interfaces)
 599                 if interfaces != '':
 600                     interfaces = '''<refsect1 id="%s.implemented-interfaces" role="impl_interfaces">
 601 <title role="impl_interfaces.title">Implemented Interfaces</title>
 602 %s
 603 </refsect1>
 604 ''' % (section_id, interfaces)
 605
 606                 implementations = TrimTextBlock(implementations)
 607                 if implementations != '':
 608                     implementations = '''<refsect1 id="%s.implementations" role="implementations">
 609 <title role="implementations.title">Known Implementations</title>
 610 %s
 611 </refsect1>
 612 ''' % (section_id, implementations)
 613
 614                 prerequisites = TrimTextBlock(prerequisites)
 615                 if prerequisites != '':
 616                     prerequisites = '''<refsect1 id="%s.prerequisites" role="prerequisites">
 617 <title role="prerequisites.title">Prerequisites</title>
 618 %s
 619 </refsect1>
 620 ''' % (section_id, prerequisites)
 621
 622                 derived = TrimTextBlock(derived)
 623                 if derived != '':
 624                     derived = '''<refsect1 id="%s.derived-interfaces" role="derived_interfaces">
 625 <title role="derived_interfaces.title">Known Derived Interfaces</title>
 626 %s
 627 </refsect1>
 628 ''' % (section_id, derived)
 629
 630                 functions_synop = re.sub(r'^\n*', '', functions_synop)
 631                 functions_synop = re.sub(r'\n+$', '\n', functions_synop)
 632                 if functions_synop != '':
 633                     functions_synop = '''<refsect1 id="%s.functions" role="functions_proto">
 634 <title role="functions_proto.title">Functions</title>
 635 <informaltable pgwide="1" frame="none">
 636 <tgroup cols="2">
 637 <colspec colname="functions_return" colwidth="150px"/>
 638 <colspec colname="functions_name"/>
 639 <tbody>
 640 %s
 641 </tbody>
 642 </tgroup>
 643 </informaltable>
 644 </refsect1>
 645 ''' % (section_id, functions_synop)
 646
 647                 other_synop = re.sub(r'^\n*', '', other_synop)
 648                 other_synop = re.sub(r'\n+$', '\n', other_synop)
 649                 if other_synop != '':
 650                     other_synop = '''<refsect1 id="%s.other" role="other_proto">
 651 <title role="other_proto.title">Types and Values</title>
 652 <informaltable role="enum_members_table" pgwide="1" frame="none">
 653 <tgroup cols="2">
 654 <colspec colname="name" colwidth="150px"/>
 655 <colspec colname="description"/>
 656 <tbody>
 657 %s
 658 </tbody>
 659 </tgroup>
 660 </informaltable>
 661 </refsect1>
 662 ''' % (section_id, other_synop)
 663
 664                 file_changed = OutputDBFile(filename, title, section_id,
 665                                             section_includes,
 666                                             functions_synop, other_synop,
 667                                             functions_details, other_details,
 668                                             signals_synop, signals_desc,
 669                                             args_synop, args_desc,
 670                                             hierarchy_str, interfaces,
 671                                             implementations,
 672                                             prerequisites, derived,
 673                                             file_objects)
 674                 if file_changed:
 675                     changed = True
 676
 677             title = ''
 678             section_id = ''
 679             subsection = ''
 680             in_section = 0
 681             section_includes = ''
 682             functions_synop = ''
 683             other_synop = ''
 684             functions_details = ''
 685             other_details = ''
 686             signals_synop = ''
 687             signals_desc = ''
 688             args_synop = ''
 689             child_args_synop = ''
 690             style_args_synop = ''
 691             args_desc = ''
 692             child_args_desc = ''
 693             style_args_desc = ''
 694             hierarchy_str = ''
 695             hierarchy = []
 696             interfaces = ''
 697             implementations = ''
 698             prerequisites = ''
 699             derived = ''
 700
 701         elif m5:
 702             symbol = m5.group(1)
 703             logging.info('  Symbol: "%s" in subsection: "%s"', symbol, subsection)
 704
 705             # check for duplicate entries
 706             if symbol not in symbol_def_line:
 707                 declaration = Declarations.get(symbol)
 708                 # FIXME: with this we'll output empty declaration
 709                 if declaration is not None:
 710                     if CheckIsObject(symbol):
 711                         file_objects.append(symbol)
 712
 713                     # We don't want standard macros/functions of GObjects,
 714                     # or private declarations.
 715                     if subsection != "Standard" and subsection != "Private":
 716                         synop, desc = OutputDeclaration(symbol, declaration)
 717                         type = DeclarationTypes[symbol]
 718
 719                         if type == 'FUNCTION' or type == 'USER_FUNCTION':
 720                             functions_synop += synop
 721                             functions_details += desc
 722                         elif type == 'MACRO' and re.search(symbol + r'\(', declaration):
 723                             functions_synop += synop
 724                             functions_details += desc
 725                         else:
 726                             other_synop += synop
 727                             other_details += desc
 728
 729                     sig_synop, sig_desc = GetSignals(symbol)
 730                     arg_synop, child_arg_synop, style_arg_synop, arg_desc, child_arg_desc, style_arg_desc = GetArgs(
 731                         symbol)
 732                     ifaces = GetInterfaces(symbol)
 733                     impls = GetImplementations(symbol)
 734                     prereqs = GetPrerequisites(symbol)
 735                     der = GetDerived(symbol)
 736                     hierarchy = GetHierarchy(symbol, hierarchy)
 737
 738                     signals_synop += sig_synop
 739                     signals_desc += sig_desc
 740                     args_synop += arg_synop
 741                     child_args_synop += child_arg_synop
 742                     style_args_synop += style_arg_synop
 743                     args_desc += arg_desc
 744                     child_args_desc += child_arg_desc
 745                     style_args_desc += style_arg_desc
 746                     interfaces += ifaces
 747                     implementations += impls
 748                     prerequisites += prereqs
 749                     derived += der
 750
 751                     # Note that the declaration has been output.
 752                     DeclarationOutput[symbol] = True
 753                 elif subsection != "Standard" and subsection != "Private":
 754                     UndeclaredSymbols[symbol] = True
 755                     common.LogWarning(file, line_number, "No declaration found for %s." % symbol)
 756
 757                 num_symbols += 1
 758                 symbol_def_line[symbol] = line_number
 759
 760                 if section_id == '':
 761                     if title == '' and filename == '':
 762                         common.LogWarning(file, line_number, "Section has no title and no file.")
 763
 764                     # FIXME: one of those would be enough
 765                     # filename should be an internal detail for gtk-doc
 766                     if title == '':
 767                         title = filename
 768                     elif filename == '':
 769                         filename = title
 770
 771                     filename = filename.replace(' ', '_')
 772
 773                     section_id = SourceSymbolDocs.get(filename + ":Section_Id")
 774                     if section_id and section_id.strip() != '':
 775                         # Remove trailing blanks and use as is
 776                         section_id = section_id.rstrip()
 777                     elif CheckIsObject(title):
 778                         # GObjects use their class name as the ID.
 779                         section_id = common.CreateValidSGMLID(title)
 780                     else:
 781                         section_id = common.CreateValidSGMLID(MODULE + '-' + title)
 782
 783                 SymbolSection[symbol] = title
 784                 SymbolSectionId[symbol] = section_id
 785
 786             else:
 787                 common.LogWarning(file, line_number, "Double symbol entry for %s. "
 788                                   "Previous occurrence on line %d." % (symbol, symbol_def_line[symbol]))
 789     INPUT.close()
 790
 791     OutputMissingDocumentation()
 792     OutputUndeclaredSymbols()
 793     OutputUnusedSymbols()
 794
 795     if options.outputallsymbols:
 796         OutputAllSymbols()
 797
 798     if options.outputsymbolswithoutsince:
 799         OutputSymbolsWithoutSince()
 800
 801     for filename in options.expand_content_files.split():
 802         file_changed = OutputExtraFile(filename)
 803         if file_changed:
 804             changed = True
 805
 806     return (changed, book_top, book_bottom)
 807
 808
 809 def DetermineNamespace():
 810     """Find common set of characters."""
 811     name_space = ''
 812     pos = 0
 813     ratio = 0.0
 814     while True:
 815         prefix = {}
 816         letter = ''
 817         for symbol in iterkeys(IndexEntriesFull):
 818             if name_space == '' or name_space.lower() in symbol.lower():
 819                 if len(symbol) > pos:
 820                     letter = symbol[pos:pos + 1]
 821                     # stop prefix scanning
 822                     if letter == "_":
 823                         # stop on "_"
 824                         break
 825                     # Should we also stop on a uppercase char, if last was lowercase
 826                     #   GtkWidget, if we have the 'W' and had the 't' before
 827                     # or should we count upper and lowercase, and stop one 2nd uppercase, if we already had a lowercase
 828                     #   GtkWidget, the 'W' would be the 2nd uppercase and with 't','k' we had lowercase chars before
 829                     # need to recound each time as this is per symbol
 830                     ul = letter.upper()
 831                     if ul in prefix:
 832                         prefix[ul] += 1
 833                     else:
 834                         prefix[ul] = 1
 835
 836         if letter != '' and letter != "_":
 837             maxletter = ''
 838             maxsymbols = 0
 839             for letter in iterkeys(prefix):
 840                 logging.debug("ns prefix: %s: %s", letter, prefix[letter])
 841                 if prefix[letter] > maxsymbols:
 842                     maxletter = letter
 843                     maxsymbols = prefix[letter]
 844
 845             ratio = float(len(IndexEntriesFull)) / prefix[maxletter]
 846             logging.debug('most symbols start with %s, that is %f', maxletter, (100 * ratio))
 847             if ratio > 0.9:
 848                 # do another round
 849                 name_space += maxletter
 850
 851             pos += 1
 852
 853         else:
 854             ratio = 0.0
 855
 856         if ratio < 0.9:
 857             break
 858     return name_space
 859
 860
 861 def OutputIndex(basename, apiindex):
 862     """Writes an index that can be included into the main-document into an <index> tag.
 863
 864     Args:
 865         basename (str): name of the index file without extension
 866         apiindex (dict): the index data
 867     """
 868     old_index = os.path.join(DB_OUTPUT_DIR, basename + '.xml')
 869     new_index = os.path.join(DB_OUTPUT_DIR, basename + '.new')
 870     lastletter = " "
 871     divopen = 0
 872     symbol = None
 873     short_symbol = None
 874
 875     OUTPUT = open(new_index, 'w')
 876
 877     OUTPUT.write(MakeDocHeader("indexdiv") + "\n<indexdiv id=\"%s\">\n" % basename)
 878
 879     logging.info("generate %s index (%d entries) with namespace %s", basename, len(apiindex), NAME_SPACE)
 880
 881     # do a case insensitive sort while chopping off the prefix
 882     mapped_keys = [
 883         {
 884             'original': x,
 885             'short': re.sub(r'^' + NAME_SPACE + r'\_?(.*)', r'\1', x.upper(), flags=re.I),
 886         } for x in iterkeys(apiindex)]
 887     sorted_keys = sorted(mapped_keys, key=lambda d: (d['short'], d['original']))
 888
 889     for key in sorted_keys:
 890         symbol = key['original']
 891         short = key['short']
 892         if short != '':
 893             short_symbol = short
 894         else:
 895             short_symbol = symbol
 896
 897         # generate a short symbol description
 898         symbol_desc = ''
 899         symbol_section = ''
 900         symbol_section_id = ''
 901         symbol_type = ''
 902         if symbol in DeclarationTypes:
 903             symbol_type = DeclarationTypes[symbol].lower()
 904
 905         if symbol_type == '':
 906             logging.info("trying symbol %s", symbol)
 907             m1 = re.search(r'(.*)::(.*)', symbol)
 908             m2 = re.search(r'(.*):(.*)', symbol)
 909             if m1:
 910                 oname = m1.group(1)
 911                 osym = m1.group(2)
 912                 logging.info("  trying object signal %s:%s in %d signals", oname, osym, len(SignalNames))
 913                 for name in SignalNames:
 914                     logging.info("    " + name)
 915                     if name == osym:
 916                         symbol_type = "object signal"
 917                         if oname in SymbolSection:
 918                             symbol_section = SymbolSection[oname]
 919                             symbol_section_id = SymbolSectionId[oname]
 920                         break
 921             elif m2:
 922                 oname = m2.group(1)
 923                 osym = m2.group(2)
 924                 logging.info("  trying object property %s::%s in %d properties", oname, osym, len(ArgNames))
 925                 for name in ArgNames:
 926                     logging.info("    " + name)
 927                     if name == osym:
 928                         symbol_type = "object property"
 929                         if oname in SymbolSection:
 930                             symbol_section = SymbolSection[oname]
 931                             symbol_section_id = SymbolSectionId[oname]
 932                         break
 933         else:
 934             if symbol in SymbolSection:
 935                 symbol_section = SymbolSection[symbol]
 936                 symbol_section_id = SymbolSectionId[symbol]
 937
 938         if symbol_type != '':
 939             symbol_desc = ", " + symbol_type
 940             if symbol_section != '':
 941                 symbol_desc += " in <link linkend=\"%s\">%s</link>" % (symbol_section_id, symbol_section)
 942                 # symbol_desc +=" in " + ExpandAbbreviations(symbol, "#symbol_section")
 943
 944         curletter = short_symbol[0].upper()
 945         ixid = apiindex[symbol]
 946
 947         logging.info("  add symbol %s with %s to index in section '%s' (derived from %s)",
 948                      symbol, ixid, curletter, short_symbol)
 949
 950         if curletter != lastletter:
 951             lastletter = curletter
 952
 953             if divopen:
 954                 OUTPUT.write("</indexdiv>\n")
 955
 956             OUTPUT.write("<indexdiv><title>%s</title>\n" % curletter)
 957             divopen = True
 958
 959         OUTPUT.write('<indexentry><primaryie linkends="%s"><link linkend="%s">%s</link>%s</primaryie></indexentry>\n' %
 960                      (ixid, ixid, symbol, symbol_desc))
 961
 962     if divopen:
 963         OUTPUT.write("</indexdiv>\n")
 964
 965     OUTPUT.write("</indexdiv>\n")
 966     OUTPUT.close()
 967
 968     common.UpdateFileIfChanged(old_index, new_index, 0)
 969
 970
 971 def OutputSinceIndexes():
 972     """Generate the 'since' api index files."""
 973     for version in set(Since.values()):
 974         logging.info("Since : [%s]", version)
 975         index = {x: IndexEntriesSince[x] for x in iterkeys(IndexEntriesSince) if Since[x] == version}
 976         OutputIndex("api-index-" + version, index)
 977
 978
 979 def OutputAnnotationGlossary():
 980     """Writes a glossary of the used annotation terms.
 981
 982     The glossary file can be included into the main document.
 983     """
 984     # if there are no annotations used return
 985     if not AnnotationsUsed:
 986         return
 987
 988     old_glossary = os.path.join(DB_OUTPUT_DIR, "annotation-glossary.xml")
 989     new_glossary = os.path.join(DB_OUTPUT_DIR, "annotation-glossary.new")
 990     lastletter = " "
 991     divopen = False
 992
 993     # add acronyms that are referenced from acronym text
 994     rerun = True
 995     while rerun:
 996         rerun = False
 997         for annotation in AnnotationsUsed:
 998             if annotation not in AnnotationDefinition:
 999                 continue
1000             m = re.search(r'<acronym>([\w ]+)<\/acronym>', AnnotationDefinition[annotation])
1001             if m and m.group(1) not in AnnotationsUsed:
1002                 AnnotationsUsed[m.group(1)] = 1
1003                 rerun = True
1004                 break
1005
1006     OUTPUT = common.open_text(new_glossary, 'w')
1007
1008     OUTPUT.write('''%s
1009 <glossary id="annotation-glossary">
1010   <title>Annotation Glossary</title>
1011 ''' % MakeDocHeader("glossary"))
1012
1013     for annotation in sorted(iterkeys(AnnotationsUsed), key=str.lower):
1014         if annotation in AnnotationDefinition:
1015             definition = AnnotationDefinition[annotation]
1016             curletter = annotation[0].upper()
1017
1018             if curletter != lastletter:
1019                 lastletter = curletter
1020
1021                 if divopen:
1022                     OUTPUT.write("</glossdiv>\n")
1023
1024                 OUTPUT.write("<glossdiv><title>%s</title>\n" % curletter)
1025                 divopen = True
1026
1027             OUTPUT.write('''    <glossentry>
1028       <glossterm><anchor id="annotation-glossterm-%s"/>%s</glossterm>
1029       <glossdef>
1030         <para>%s</para>
1031       </glossdef>
1032     </glossentry>
1033 ''' % (annotation, annotation, definition))
1034
1035     if divopen:
1036         OUTPUT.write("</glossdiv>\n")
1037
1038     OUTPUT.write("</glossary>\n")
1039     OUTPUT.close()
1040
1041     common.UpdateFileIfChanged(old_glossary, new_glossary, 0)
1042
1043
1044 def ReadKnownSymbols(file):
1045     """Collect the names of non-private symbols from the $MODULE-sections.txt file.
1046
1047     Args:
1048         file: the $MODULE-sections.txt file
1049     """
1050
1051     subsection = ''
1052
1053     logging.info("Reading: %s", file)
1054     INPUT = common.open_text(file)
1055     for line in INPUT:
1056         if line.startswith('#'):
1057             continue
1058
1059         if line.startswith('<SECTION>'):
1060             subsection = ''
1061             continue
1062
1063         m = re.search(r'^<SUBSECTION\s*(.*)>', line, flags=re.I)
1064         if m:
1065             subsection = m.group(1)
1066             continue
1067
1068         if line.startswith('<SUBSECTION>'):
1069             continue
1070
1071         if re.search(r'^<TITLE>(.*)<\/TITLE>', line):
1072             continue
1073
1074         m = re.search(r'^<FILE>(.*)<\/FILE>', line)
1075         if m:
1076             KnownSymbols[m.group(1) + ":Long_Description"] = 1
1077             KnownSymbols[m.group(1) + ":Short_Description"] = 1
1078             continue
1079
1080         m = re.search(r'^<INCLUDE>(.*)<\/INCLUDE>', line)
1081         if m:
1082             continue
1083
1084         m = re.search(r'^<\/SECTION>', line)
1085         if m:
1086             continue
1087
1088         m = re.search(r'^(\S+)', line)
1089         if m:
1090             symbol = m.group(1)
1091             if subsection != "Standard" and subsection != "Private":
1092                 KnownSymbols[symbol] = 1
1093             else:
1094                 KnownSymbols[symbol] = 0
1095     INPUT.close()
1096
1097
1098 def OutputDeclaration(symbol, declaration):
1099     """Returns the formatted documentation block for a symbol.
1100
1101     Args:
1102         symbol (str): the name of the function/macro/...
1103         declaration (str): the declaration of the function/macro.
1104
1105     Returns:
1106         str: the formatted documentation
1107     """
1108
1109     dtype = DeclarationTypes[symbol]
1110     if dtype == 'MACRO':
1111         return OutputMacro(symbol, declaration)
1112     elif dtype == 'TYPEDEF':
1113         return OutputTypedef(symbol, declaration)
1114     elif dtype == 'STRUCT':
1115         return OutputStruct(symbol, declaration)
1116     elif dtype == 'ENUM':
1117         return OutputEnum(symbol, declaration)
1118     elif dtype == 'UNION':
1119         return OutputUnion(symbol, declaration)
1120     elif dtype == 'VARIABLE':
1121         return OutputVariable(symbol, declaration)
1122     elif dtype == 'FUNCTION':
1123         return OutputFunction(symbol, declaration, dtype)
1124     elif dtype == 'USER_FUNCTION':
1125         return OutputFunction(symbol, declaration, dtype)
1126     else:
1127         logging.warning("Unknown symbol type %s for symbol %s", dtype, symbol)
1128         return ('', '')
1129
1130
1131 def OutputSymbolTraits(symbol):
1132     """Returns the Since and StabilityLevel paragraphs for a symbol.
1133
1134     Args:
1135         symbol (str): the name to describe
1136
1137     Returns:
1138         str: paragraph or empty string
1139     """
1140
1141     desc = ''
1142
1143     if symbol in Since:
1144         link_id = "api-index-" + Since[symbol]
1145         desc += "<para role=\"since\">Since: <link linkend=\"%s\">%s</link></para>" % (link_id, Since[symbol])
1146
1147     if symbol in StabilityLevel:
1148         stability = StabilityLevel[symbol]
1149         AnnotationsUsed[stability] = True
1150         desc += "<para role=\"stability\">Stability Level: <acronym>%s</acronym></para>" % stability
1151     return desc
1152
1153
1154 def uri_escape(text):
1155     if text is None:
1156         return None
1157
1158     # Build a char to hex map
1159     escapes = {chr(i): ("%%%02X" % i) for i in range(256)}
1160
1161     # Default unsafe characters.  RFC 2732 ^(uric - reserved)
1162     def do_escape(char):
1163         return escapes[char]
1164     return re.sub(r"([^A-Za-z0-9\-_.!~*'()]", do_escape, text)
1165
1166
1167 def OutputSymbolExtraLinks(symbol):
1168     """Returns extralinks for the symbol (if enabled).
1169
1170     Args:
1171         symbol (str): the name to describe
1172
1173     Returns:
1174         str: paragraph or empty string
1175     """
1176     desc = ''
1177
1178     if False:   # NEW FEATURE: needs configurability
1179         sstr = uri_escape(symbol)
1180         mstr = uri_escape(MODULE)
1181         desc += '''<ulink role="extralinks" url="http://www.google.com/codesearch?q=%s">code search</ulink>
1182 <ulink role="extralinks" url="http://library.gnome.org/edit?module=%s&amp;symbol=%s">edit documentation</ulink>
1183 ''' % (sstr, mstr, sstr)
1184
1185     return desc
1186
1187
1188 def OutputSectionExtraLinks(symbol, docsymbol):
1189     desc = ''
1190
1191     if False:   # NEW FEATURE: needs configurability
1192         sstr = uri_escape(symbol)
1193         mstr = uri_escape(MODULE)
1194         dsstr = uri_escape(docsymbol)
1195         desc += '''<ulink role="extralinks" url="http://www.google.com/codesearch?q=%s">code search</ulink>
1196 <ulink role="extralinks" url="http://library.gnome.org/edit?module=%s&amp;symbol=%s">edit documentation</ulink>
1197 ''' % (sstr, mstr, dsstr)
1198     return desc
1199
1200
1201 def OutputMacro(symbol, declaration):
1202     """Returns the synopsis and detailed description of a macro.
1203
1204     Args:
1205         symbol (str): the macro name.
1206         declaration (str): the declaration of the macro.
1207
1208     Returns:
1209         str: the formated docs
1210     """
1211     sid = common.CreateValidSGMLID(symbol)
1212     condition = MakeConditionDescription(symbol)
1213     synop = "<row><entry role=\"define_keyword\">#define</entry><entry role=\"function_name\"><link linkend=\"%s\">%s</link>" % (
1214         sid, symbol)
1215
1216     fields = common.ParseMacroDeclaration(declaration, CreateValidSGML)
1217     title = symbol
1218     if len(fields) > 0:
1219         title += '()'
1220
1221     desc = '<refsect2 id="%s" role="macro"%s>\n<title>%s</title>\n' % (sid, condition, title)
1222     desc += MakeIndexterms(symbol, sid)
1223     desc += "\n"
1224     desc += OutputSymbolExtraLinks(symbol)
1225
1226     if len(fields) > 0:
1227         synop += "<phrase role=\"c_punctuation\">()</phrase>"
1228
1229     synop += "</entry></row>\n"
1230
1231     # Don't output the macro definition if is is a conditional macro or it
1232     # looks like a function, i.e. starts with "g_" or "_?gnome_", or it is
1233     # longer than 2 lines, otherwise we get lots of complicated macros like
1234     # g_assert.
1235     if symbol not in DeclarationConditional and not symbol.startswith('g_') \
1236             and not re.search(r'^_?gnome_', symbol) and declaration.count('\n') < 2:
1237         decl_out = CreateValidSGML(declaration)
1238         desc += "<programlisting language=\"C\">%s</programlisting>\n" % decl_out
1239     else:
1240         desc += "<programlisting language=\"C\">" + "#define".ljust(RETURN_TYPE_FIELD_WIDTH) + symbol
1241         m = re.search(r'^\s*#\s*define\s+\w+(\([^\)]*\))', declaration)
1242         if m:
1243             args = m.group(1)
1244             pad = ' ' * (RETURN_TYPE_FIELD_WIDTH - len("#define "))
1245             # Align each line so that if should all line up OK.
1246             args = args.replace('\n', '\n' + pad)
1247             desc += CreateValidSGML(args)
1248
1249         desc += "</programlisting>\n"
1250
1251     desc += MakeDeprecationNote(symbol)
1252
1253     parameters = OutputParamDescriptions("MACRO", symbol, fields)
1254
1255     if symbol in SymbolDocs:
1256         symbol_docs = ConvertMarkDown(symbol, SymbolDocs[symbol])
1257         desc += symbol_docs
1258
1259     desc += parameters
1260     desc += OutputSymbolTraits(symbol)
1261     desc += "</refsect2>\n"
1262     return (synop, desc)
1263
1264
1265 def OutputTypedef(symbol, declaration):
1266     """Returns the synopsis and detailed description of a typedef.
1267
1268     Args:
1269         symbol (str): the typedef.
1270         declaration (str): the declaration of the typedef,
1271                            e.g. 'typedef unsigned int guint;'
1272
1273     Returns:
1274         str: the formated docs
1275     """
1276     sid = common.CreateValidSGMLID(symbol)
1277     condition = MakeConditionDescription(symbol)
1278     desc = "<refsect2 id=\"%s\" role=\"typedef\"%s>\n<title>%s</title>\n" % (sid, condition, symbol)
1279     synop = "<row><entry role=\"typedef_keyword\">typedef</entry><entry role=\"function_name\"><link linkend=\"%s\">%s</link></entry></row>\n" % (
1280         sid, symbol)
1281
1282     desc += MakeIndexterms(symbol, sid)
1283     desc += "\n"
1284     desc += OutputSymbolExtraLinks(symbol)
1285
1286     if symbol not in DeclarationConditional:
1287         decl_out = CreateValidSGML(declaration)
1288         desc += "<programlisting language=\"C\">%s</programlisting>\n" % decl_out
1289
1290     desc += MakeDeprecationNote(symbol)
1291
1292     if symbol in SymbolDocs:
1293         desc += ConvertMarkDown(symbol, SymbolDocs[symbol])
1294
1295     desc += OutputSymbolTraits(symbol)
1296     desc += "</refsect2>\n"
1297     return (synop, desc)
1298
1299
1300 def OutputStruct(symbol, declaration):
1301     """Returns the synopsis and detailed description of a struct.
1302
1303     We check if it is a object struct, and if so we only output parts of it that
1304     are noted as public fields. We also use a different IDs for object structs,
1305     since the original ID is used for the entire RefEntry.
1306
1307     Args:
1308         symbol (str): the struct.
1309         declaration (str): the declaration of the struct.
1310
1311     Returns:
1312         str: the formated docs
1313     """
1314     is_gtype = False
1315     default_to_public = True
1316     if CheckIsObject(symbol):
1317         logging.info("Found struct gtype: %s", symbol)
1318         is_gtype = True
1319         default_to_public = ObjectRoots[symbol] == 'GBoxed'
1320
1321     sid = None
1322     condition = None
1323     if is_gtype:
1324         sid = common.CreateValidSGMLID(symbol + "_struct")
1325         condition = MakeConditionDescription(symbol + "_struct")
1326     else:
1327         sid = common.CreateValidSGMLID(symbol)
1328         condition = MakeConditionDescription(symbol)
1329
1330     # Determine if it is a simple struct or it also has a typedef.
1331     has_typedef = False
1332     if symbol in StructHasTypedef or re.search(r'^\s*typedef\s+', declaration):
1333         has_typedef = True
1334
1335     type_output = None
1336     desc = None
1337     if has_typedef:
1338         # For structs with typedefs we just output the struct name.
1339         type_output = ''
1340         desc = "<refsect2 id=\"%s\" role=\"struct\"%s>\n<title>%s</title>\n" % (sid, condition, symbol)
1341     else:
1342         type_output = "struct"
1343         desc = "<refsect2 id=\"%s\" role=\"struct\"%s>\n<title>struct %s</title>\n" % (sid, condition, symbol)
1344
1345     synop = "<row><entry role=\"datatype_keyword\">%s</entry><entry role=\"function_name\"><link linkend=\"%s\">%s</link></entry></row>\n" % (
1346         type_output, sid, symbol)
1347
1348     desc += MakeIndexterms(symbol, sid)
1349     desc += "\n"
1350     desc += OutputSymbolExtraLinks(symbol)
1351
1352     # Form a pretty-printed, private-data-removed form of the declaration
1353
1354     decl_out = ''
1355     if re.search(r'^\s*$', declaration):
1356         logging.info("Found opaque struct: %s", symbol)
1357         decl_out = "typedef struct _%s %s;" % (symbol, symbol)
1358     elif re.search(r'^\s*struct\s+\w+\s*;\s*$', declaration):
1359         logging.info("Found opaque struct: %s", symbol)
1360         decl_out = "struct %s;" % symbol
1361     else:
1362         m = re.search(
1363             r'^\s*(typedef\s+)?struct\s*\w*\s*(?:\/\*.*\*\/)?\s*{(.*)}\s*\w*\s*;\s*$', declaration, flags=re.S)
1364         if m:
1365             struct_contents = m.group(2)
1366
1367             public = default_to_public
1368             new_declaration = ''
1369
1370             for decl_line in struct_contents.splitlines():
1371                 logging.info("Struct line: %s", decl_line)
1372                 m2 = re.search(r'/\*\s*<\s*public\s*>\s*\*/', decl_line)
1373                 m3 = re.search(r'/\*\s*<\s*(private|protected)\s*>\s*\*/', decl_line)
1374                 if m2:
1375                     public = True
1376                 elif m3:
1377                     public = False
1378                 elif public:
1379                     new_declaration += decl_line + "\n"
1380
1381             if new_declaration:
1382                 # Strip any blank lines off the ends.
1383                 new_declaration = re.sub(r'^\s*\n', '', new_declaration)
1384                 new_declaration = re.sub(r'\n\s*$', r'\n', new_declaration)
1385
1386                 if has_typedef:
1387                     decl_out = "typedef struct {\n%s} %s;\n" % (new_declaration, symbol)
1388                 else:
1389                     decl_out = "struct %s {\n%s};\n" % (symbol, new_declaration)
1390
1391         else:
1392             common.LogWarning(GetSymbolSourceFile(symbol), GetSymbolSourceLine(symbol),
1393                               "Couldn't parse struct:\n%s" % declaration)
1394
1395         # If we couldn't parse the struct or it was all private, output an
1396         # empty struct declaration.
1397         if decl_out == '':
1398             if has_typedef:
1399                 decl_out = "typedef struct _%s %s;" % (symbol, symbol)
1400             else:
1401                 decl_out = "struct %s;" % symbol
1402
1403     decl_out = CreateValidSGML(decl_out)
1404     desc += "<programlisting language=\"C\">%s</programlisting>\n" % decl_out
1405
1406     desc += MakeDeprecationNote(symbol)
1407
1408     if symbol in SymbolDocs:
1409         desc += ConvertMarkDown(symbol, SymbolDocs[symbol])
1410
1411     # Create a table of fields and descriptions
1412
1413     # FIXME: Inserting &#160's into the produced type declarations here would
1414     #        improve the output in most situations ... except for function
1415     #        members of structs!
1416     def pfunc(*args):
1417         return '<structfield id="%s">%s</structfield>' % (common.CreateValidSGMLID(sid + '.' + args[0]), args[0])
1418     fields = common.ParseStructDeclaration(declaration, not default_to_public, 0, MakeXRef, pfunc)
1419     params = SymbolParams.get(symbol)
1420
1421     # If no parameters are filled in, we don't generate the description
1422     # table, for backwards compatibility.
1423     found = False
1424     if params:
1425         found = next((True for p in params.values() if p.strip() != ''), False)
1426
1427     if found:
1428         field_descrs = params
1429         missing_parameters = ''
1430         unused_parameters = ''
1431         sid = common.CreateValidSGMLID(symbol + ".members")
1432
1433         desc += '''<refsect3 id="%s" role="struct_members">\n<title>Members</title>
1434 <informaltable role="struct_members_table" pgwide="1" frame="none">
1435 <tgroup cols="3">
1436 <colspec colname="struct_members_name" colwidth="300px"/>
1437 <colspec colname="struct_members_description"/>
1438 <colspec colname="struct_members_annotations" colwidth="200px"/>
1439 <tbody>
1440 ''' % sid
1441
1442         for field_name, text in iteritems(fields):
1443             param_annotations = ''
1444
1445             desc += "<row role=\"member\"><entry role=\"struct_member_name\"><para>%s</para></entry>\n" % text
1446             if field_name in field_descrs:
1447                 (field_descr, param_annotations) = ExpandAnnotation(symbol, field_descrs[field_name])
1448                 field_descr = ConvertMarkDown(symbol, field_descr)
1449                 # trim
1450                 field_descr = re.sub(r'^(\s|\n)+', '', field_descr, flags=re.M | re.S)
1451                 field_descr = re.sub(r'(\s|\n)+$', '', field_descr, flags=re.M | re.S)
1452                 desc += "<entry role=\"struct_member_description\">%s</entry>\n<entry role=\"struct_member_annotations\">%s</entry>\n" % (
1453                     field_descr, param_annotations)
1454                 del field_descrs[field_name]
1455             else:
1456                 common.LogWarning(GetSymbolSourceFile(symbol), GetSymbolSourceLine(symbol),
1457                                   "Field description for %s::%s is missing in source code comment block." % (symbol, field_name))
1458                 if missing_parameters != '':
1459                     missing_parameters += ", " + field_name
1460                 else:
1461                     missing_parameters = field_name
1462
1463                 desc += "<entry /><entry />\n"
1464
1465             desc += "</row>\n"
1466
1467         desc += "</tbody></tgroup></informaltable>\n</refsect3>\n"
1468         for field_name in field_descrs:
1469             # Documenting those standard fields is not required anymore, but
1470             # we don't want to warn if they are documented anyway.
1471             m = re.search(r'(g_iface|parent_instance|parent_class)', field_name)
1472             if m:
1473                 continue
1474
1475             common.LogWarning(GetSymbolSourceFile(symbol), GetSymbolSourceLine(symbol),
1476                               "Field description for %s::%s is not used from source code comment block." % (symbol, field_name))
1477             if unused_parameters != '':
1478                 unused_parameters += ", " + field_name
1479             else:
1480                 unused_parameters = field_name
1481
1482         # remember missing/unused parameters (needed in tmpl-free build)
1483         if missing_parameters != '' and (symbol not in AllIncompleteSymbols):
1484             AllIncompleteSymbols[symbol] = missing_parameters
1485
1486         if unused_parameters != '' and (symbol not in AllUnusedSymbols):
1487             AllUnusedSymbols[symbol] = unused_parameters
1488     else:
1489         if fields:
1490             if symbol not in AllIncompleteSymbols:
1491                 AllIncompleteSymbols[symbol] = "<items>"
1492                 common.LogWarning(GetSymbolSourceFile(symbol), GetSymbolSourceLine(symbol),
1493                                   "Field descriptions for struct %s are missing in source code comment block." % symbol)
1494                 logging.info("Remaining structs fields: " + ':'.join(fields) + "\n")
1495
1496     desc += OutputSymbolTraits(symbol)
1497     desc += "</refsect2>\n"
1498     return (synop, desc)
1499
1500
1501 def OutputUnion(symbol, declaration):
1502     """Returns the synopsis and detailed description of a union.
1503
1504     Args:
1505         symbol (str): the union.
1506         declaration (str): the declaration of the union.
1507
1508     Returns:
1509         str: the formated docs
1510     """
1511     is_gtype = False
1512     if CheckIsObject(symbol):
1513         logging.info("Found union gtype: %s", symbol)
1514         is_gtype = True
1515
1516     sid = None
1517     condition = None
1518     if is_gtype:
1519         sid = common.CreateValidSGMLID(symbol + "_union")
1520         condition = MakeConditionDescription(symbol + "_union")
1521     else:
1522         sid = common.CreateValidSGMLID(symbol)
1523         condition = MakeConditionDescription(symbol)
1524
1525     # Determine if it is a simple struct or it also has a typedef.
1526     has_typedef = False
1527     if symbol in StructHasTypedef or re.search(r'^\s*typedef\s+', declaration):
1528         has_typedef = True
1529
1530     type_output = None
1531     desc = None
1532     if has_typedef:
1533         # For unions with typedefs we just output the union name.
1534         type_output = ''
1535         desc = "<refsect2 id=\"%s\" role=\"union\"%s>\n<title>%s</title>\n" % (sid, condition, symbol)
1536     else:
1537         type_output = "union"
1538         desc = "<refsect2 id=\"%s\" role=\"union\"%s>\n<title>union %s</title>\n" % (sid, condition, symbol)
1539
1540     synop = "<row><entry role=\"datatype_keyword\">%s</entry><entry role=\"function_name\"><link linkend=\"%s\">%s</link></entry></row>\n" % (
1541         type_output, sid, symbol)
1542
1543     desc += MakeIndexterms(symbol, sid)
1544     desc += "\n"
1545     desc += OutputSymbolExtraLinks(symbol)
1546     desc += MakeDeprecationNote(symbol)
1547
1548     if symbol in SymbolDocs:
1549         desc += ConvertMarkDown(symbol, SymbolDocs[symbol])
1550
1551     # Create a table of fields and descriptions
1552
1553     # FIXME: Inserting &#160's into the produced type declarations here would
1554     #        improve the output in most situations ... except for function
1555     #        members of structs!
1556     def pfunc(*args):
1557         return '<structfield id="%s">%s</structfield>' % (common.CreateValidSGMLID(sid + '.' + args[0]), args[0])
1558     fields = common.ParseStructDeclaration(declaration, 0, 0, MakeXRef, pfunc)
1559     params = SymbolParams.get(symbol)
1560
1561     # If no parameters are filled in, we don't generate the description
1562     # table, for backwards compatibility
1563     found = False
1564     if params:
1565         found = next((True for p in params.values() if p.strip() != ''), False)
1566
1567     logging.debug('Union %s has %d entries, found=%d, has_typedef=%d', symbol, len(fields), found, has_typedef)
1568
1569     if found:
1570         field_descrs = params
1571         missing_parameters = ''
1572         unused_parameters = ''
1573         sid = common.CreateValidSGMLID('%s.members' % symbol)
1574
1575         desc += '''<refsect3 id="%s" role="union_members">\n<title>Members</title>
1576 <informaltable role="union_members_table" pgwide="1" frame="none">
1577 <tgroup cols="3">
1578 <colspec colname="union_members_name" colwidth="300px"/>
1579 <colspec colname="union_members_description"/>
1580 <colspec colname="union_members_annotations" colwidth="200px"/>
1581 <tbody>
1582 ''' % sid
1583
1584         for field_name, text in iteritems(fields):
1585             param_annotations = ''
1586
1587             desc += "<row><entry role=\"union_member_name\"><para>%s</para></entry>\n" % text
1588             if field_name in field_descrs:
1589                 (field_descr, param_annotations) = ExpandAnnotation(symbol, field_descrs[field_name])
1590                 field_descr = ConvertMarkDown(symbol, field_descr)
1591
1592                 # trim
1593                 field_descr = re.sub(r'^(\s|\n)+', '', field_descr, flags=re.M | re.S)
1594                 field_descr = re.sub(r'(\s|\n)+$', '', field_descr, flags=re.M | re.S)
1595                 desc += "<entry role=\"union_member_description\">%s</entry>\n<entry role=\"union_member_annotations\">%s</entry>\n" % (
1596                     field_descr, param_annotations)
1597                 del field_descrs[field_name]
1598             else:
1599                 common.LogWarning(GetSymbolSourceFile(symbol), GetSymbolSourceLine(symbol),
1600                                   "Field description for %s::%s is missing in source code comment block." % (symbol, field_name))
1601                 if missing_parameters != '':
1602                     missing_parameters += ", " + field_name
1603                 else:
1604                     missing_parameters = field_name
1605
1606                 desc += "<entry /><entry />\n"
1607
1608             desc += "</row>\n"
1609
1610         desc += "</tbody></tgroup></informaltable>\n</refsect3>"
1611         for field_name in field_descrs:
1612             common.LogWarning(GetSymbolSourceFile(symbol), GetSymbolSourceLine(symbol),
1613                               "Field description for %s::%s is not used from source code comment block." % (symbol, field_name))
1614             if unused_parameters != '':
1615                 unused_parameters += ", " + field_name
1616             else:
1617                 unused_parameters = field_name
1618
1619         # remember missing/unused parameters (needed in tmpl-free build)
1620         if missing_parameters != '' and (symbol not in AllIncompleteSymbols):
1621             AllIncompleteSymbols[symbol] = missing_parameters
1622
1623         if unused_parameters != '' and (symbol not in AllUnusedSymbols):
1624             AllUnusedSymbols[symbol] = unused_parameters
1625     else:
1626         if len(fields) > 0:
1627             if symbol not in AllIncompleteSymbols:
1628                 AllIncompleteSymbols[symbol] = "<items>"
1629                 common.LogWarning(GetSymbolSourceFile(symbol), GetSymbolSourceLine(symbol),
1630                                   "Field descriptions for union %s are missing in source code comment block." % symbol)
1631                 logging.info("Remaining union fields: " + ':'.join(fields) + "\n")
1632
1633     desc += OutputSymbolTraits(symbol)
1634     desc += "</refsect2>\n"
1635     return (synop, desc)
1636
1637
1638 def OutputEnum(symbol, declaration):
1639     """Returns the synopsis and detailed description of a enum.
1640
1641     Args:
1642         symbol (str): the enum.
1643         declaration (str): the declaration of the enum.
1644
1645     Returns:
1646         str: the formated docs
1647     """
1648     is_gtype = False
1649     if CheckIsObject(symbol):
1650         logging.info("Found enum gtype: %s", symbol)
1651         is_gtype = True
1652
1653     sid = None
1654     condition = None
1655     if is_gtype:
1656         sid = common.CreateValidSGMLID(symbol + "_enum")
1657         condition = MakeConditionDescription(symbol + "_enum")
1658     else:
1659         sid = common.CreateValidSGMLID(symbol)
1660         condition = MakeConditionDescription(symbol)
1661
1662     synop = "<row><entry role=\"datatype_keyword\">enum</entry><entry role=\"function_name\"><link linkend=\"%s\">%s</link></entry></row>\n" % (
1663         sid, symbol)
1664     desc = "<refsect2 id=\"%s\" role=\"enum\"%s>\n<title>enum %s</title>\n" % (sid, condition, symbol)
1665
1666     desc += MakeIndexterms(symbol, sid)
1667     desc += "\n"
1668     desc += OutputSymbolExtraLinks(symbol)
1669     desc += MakeDeprecationNote(symbol)
1670
1671     if symbol in SymbolDocs:
1672         desc += ConvertMarkDown(symbol, SymbolDocs[symbol])
1673
1674     # Create a table of fields and descriptions
1675
1676     fields = common.ParseEnumDeclaration(declaration)
1677     params = SymbolParams.get(symbol)
1678
1679     # If nothing at all is documented log a single summary warning at the end.
1680     # Otherwise, warn about each undocumented item.
1681
1682     found = False
1683     if params:
1684         found = next((True for p in params.values() if p.strip() != ''), False)
1685         field_descrs = params
1686     else:
1687         field_descrs = {}
1688
1689     missing_parameters = ''
1690     unused_parameters = ''
1691
1692     sid = common.CreateValidSGMLID("%s.members" % symbol)
1693     desc += '''<refsect3 id="%s" role="enum_members">\n<title>Members</title>
1694 <informaltable role="enum_members_table" pgwide="1" frame="none">
1695 <tgroup cols="3">
1696 <colspec colname="enum_members_name" colwidth="300px"/>
1697 <colspec colname="enum_members_description"/>
1698 <colspec colname="enum_members_annotations" colwidth="200px"/>
1699 <tbody>
1700 ''' % sid
1701
1702     for field_name in fields:
1703         field_descr = field_descrs.get(field_name)
1704         param_annotations = ''
1705
1706         sid = common.CreateValidSGMLID(field_name)
1707         condition = MakeConditionDescription(field_name)
1708         desc += "<row role=\"constant\"><entry role=\"enum_member_name\"><para id=\"%s\">%s</para></entry>\n" % (
1709             sid, field_name)
1710         if field_descr:
1711             field_descr, param_annotations = ExpandAnnotation(symbol, field_descr)
1712             field_descr = ConvertMarkDown(symbol, field_descr)
1713             desc += "<entry role=\"enum_member_description\">%s</entry>\n<entry role=\"enum_member_annotations\">%s</entry>\n" % (
1714                 field_descr, param_annotations)
1715             del field_descrs[field_name]
1716         else:
1717             if found:
1718                 common.LogWarning(GetSymbolSourceFile(symbol), GetSymbolSourceLine(symbol),
1719                                   "Value description for %s::%s is missing in source code comment block." % (symbol, field_name))
1720                 if missing_parameters != '':
1721                     missing_parameters += ", " + field_name
1722                 else:
1723                     missing_parameters = field_name
1724             desc += "<entry /><entry />\n"
1725         desc += "</row>\n"
1726
1727     desc += "</tbody></tgroup></informaltable>\n</refsect3>"
1728     for field_name in field_descrs:
1729         common.LogWarning(GetSymbolSourceFile(symbol), GetSymbolSourceLine(symbol),
1730                           "Value description for %s::%s is not used from source code comment block." % (symbol, field_name))
1731         if unused_parameters != '':
1732             unused_parameters += ", " + field_name
1733         else:
1734             unused_parameters = field_name
1735
1736     # remember missing/unused parameters (needed in tmpl-free build)
1737     if missing_parameters != '' and (symbol not in AllIncompleteSymbols):
1738         AllIncompleteSymbols[symbol] = missing_parameters
1739
1740     if unused_parameters != '' and (symbol not in AllUnusedSymbols):
1741         AllUnusedSymbols[symbol] = unused_parameters
1742
1743     if not found:
1744         if len(fields) > 0:
1745             if symbol not in AllIncompleteSymbols:
1746                 AllIncompleteSymbols[symbol] = "<items>"
1747                 common.LogWarning(GetSymbolSourceFile(symbol), GetSymbolSourceLine(symbol),
1748                                   "Value descriptions for %s are missing in source code comment block." % symbol)
1749
1750     desc += OutputSymbolTraits(symbol)
1751     desc += "</refsect2>\n"
1752     return (synop, desc)
1753
1754
1755 def OutputVariable(symbol, declaration):
1756     """Returns the synopsis and detailed description of a variable.
1757
1758     Args:
1759         symbol (str): the extern'ed variable.
1760         declaration (str): the declaration of the variable.
1761
1762     Returns:
1763         str: the formated docs
1764     """
1765     sid = common.CreateValidSGMLID(symbol)
1766     condition = MakeConditionDescription(symbol)
1767
1768     logging.info("ouputing variable: '%s' '%s'", symbol, declaration)
1769
1770     type_output = None
1771     m1 = re.search(
1772         r'^\s*extern\s+((const\s+|signed\s+|unsigned\s+|long\s+|short\s+)*\w+)(\s+\*+|\*+|\s)(\s*)(const\s+)*([A-Za-z]\w*)\s*;', declaration)
1773     m2 = re.search(
1774         r'\s*((const\s+|signed\s+|unsigned\s+|long\s+|short\s+)*\w+)(\s+\*+|\*+|\s)(\s*)(const\s+)*([A-Za-z]\w*)\s*=', declaration)
1775     if m1:
1776         mod1 = m1.group(1) or ''
1777         ptr = m1.group(3) or ''
1778         space = m1.group(4) or ''
1779         mod2 = m1.group(5) or ''
1780         type_output = "extern %s%s%s%s" % (mod1, ptr, space, mod2)
1781     elif m2:
1782         mod1 = m2.group(1) or ''
1783         ptr = m2.group(3) or ''
1784         space = m2.group(4) or ''
1785         mod2 = m2.group(5) or ''
1786         type_output = '%s%s%s%s' % (mod1, ptr, space, mod2)
1787     else:
1788         type_output = "extern"
1789
1790     synop = "<row><entry role=\"variable_type\">%s</entry><entry role=\"function_name\"><link linkend=\"%s\">%s</link></entry></row>\n" % (
1791         type_output, sid, symbol)
1792
1793     desc = "<refsect2 id=\"%s\" role=\"variable\"%s>\n<title>%s</title>\n" % (sid, condition, symbol)
1794
1795     desc += MakeIndexterms(symbol, sid)
1796     desc += "\n"
1797     desc += OutputSymbolExtraLinks(symbol)
1798
1799     decl_out = CreateValidSGML(declaration)
1800     desc += "<programlisting language=\"C\">%s</programlisting>\n" % decl_out
1801
1802     desc += MakeDeprecationNote(symbol)
1803
1804     if symbol in SymbolDocs:
1805         desc += ConvertMarkDown(symbol, SymbolDocs[symbol])
1806
1807     if symbol in SymbolAnnotations:
1808         param_desc = SymbolAnnotations[symbol]
1809         param_annotations = ''
1810         (param_desc, param_annotations) = ExpandAnnotation(symbol, param_desc)
1811         if param_annotations != '':
1812             desc += "\n<para>%s</para>" % param_annotations
1813
1814     desc += OutputSymbolTraits(symbol)
1815     desc += "</refsect2>\n"
1816     return (synop, desc)
1817
1818
1819 def OutputFunction(symbol, declaration, symbol_type):
1820     """Returns the synopsis and detailed description of a function.
1821
1822     Args:
1823         symbol (str): the function.
1824         declaration (str): the declaration of the function.
1825
1826     Returns:
1827         str: the formated docs
1828     """
1829     sid = common.CreateValidSGMLID(symbol)
1830     condition = MakeConditionDescription(symbol)
1831
1832     # Take out the return type
1833     #                      $1                                                                                       $2   $3
1834     regex = r'<RETURNS>\s*((?:const\s+|G_CONST_RETURN\s+|signed\s+|unsigned\s+|long\s+|short\s+|struct\s+|enum\s+)*)(\w+)(\s*\**\s*(?:const|G_CONST_RETURN)?\s*\**\s*(?:restrict)?\s*)<\/RETURNS>\n'
1835     m = re.search(regex, declaration)
1836     declaration = re.sub(regex, '', declaration)
1837     type_modifier = m.group(1) or ''
1838     type = m.group(2)
1839     pointer = m.group(3)
1840     pointer = pointer.rstrip()
1841     xref = MakeXRef(type, tagify(type, "returnvalue"))
1842     start = ''
1843     # if (symbol_type == 'USER_FUNCTION')
1844     #    start = "typedef "
1845     #
1846
1847     # We output const rather than G_CONST_RETURN.
1848     type_modifier = re.sub(r'G_CONST_RETURN', 'const', type_modifier)
1849     pointer = re.sub(r'G_CONST_RETURN', 'const', pointer)
1850     pointer = re.sub(r'^\s+', '&#160;', pointer)
1851
1852     ret_type_output = "%s%s%s%s\n" % (start, type_modifier, xref, pointer)
1853
1854     indent_len = len(symbol) + 2
1855     char1 = char2 = char3 = ''
1856     if symbol_type == 'USER_FUNCTION':
1857         indent_len += 3
1858         char1 = "<phrase role=\"c_punctuation\">(</phrase>"
1859         char2 = "*"
1860         char3 = "<phrase role=\"c_punctuation\">)</phrase>"
1861
1862     symbol_output = "%s<link linkend=\"%s\">%s%s</link>%s" % (char1, sid, char2, symbol, char3)
1863     if indent_len < MAX_SYMBOL_FIELD_WIDTH:
1864         symbol_desc_output = "%s%s%s%s " % (char1, char2, symbol, char3)
1865     else:
1866         indent_len = MAX_SYMBOL_FIELD_WIDTH - 8
1867         symbol_desc_output = ('%s%s%s%s\n' % (char1, char2, symbol, char3)) + (' ' * (indent_len - 1))
1868
1869     synop = "<row><entry role=\"function_type\">%s</entry><entry role=\"function_name\">%s&#160;<phrase role=\"c_punctuation\">()</phrase></entry></row>\n" % (
1870         ret_type_output, symbol_output)
1871
1872     desc = "<refsect2 id=\"%s\" role=\"function\"%s>\n<title>%s&#160;()</title>\n" % (sid, condition, symbol)
1873
1874     desc += MakeIndexterms(symbol, sid)
1875     desc += "\n"
1876     desc += OutputSymbolExtraLinks(symbol)
1877
1878     desc += "<programlisting language=\"C\">%s%s(" % (ret_type_output, symbol_desc_output)
1879
1880     def tagfun(*args):
1881         return tagify(args[0], "parameter")
1882
1883     fields = common.ParseFunctionDeclaration(declaration, MakeXRef, tagfun)
1884
1885     first = True
1886     for field_name in fields.values():
1887         if first:
1888             desc += field_name
1889             first = False
1890         else:
1891             desc += ",\n" + (' ' * indent_len) + field_name
1892
1893     desc += ");</programlisting>\n"
1894
1895     desc += MakeDeprecationNote(symbol)
1896
1897     if symbol in SymbolDocs:
1898         desc += ConvertMarkDown(symbol, SymbolDocs[symbol])
1899
1900     if symbol in SymbolAnnotations:
1901         param_desc = SymbolAnnotations[symbol]
1902         (param_desc, param_annotations) = ExpandAnnotation(symbol, param_desc)
1903         if param_annotations != '':
1904             desc += "\n<para>%s</para>" % param_annotations
1905
1906     desc += OutputParamDescriptions("FUNCTION", symbol, iterkeys(fields))
1907     desc += OutputSymbolTraits(symbol)
1908     desc += "</refsect2>\n"
1909     return (synop, desc)
1910
1911
1912 def OutputParamDescriptions(symbol_type, symbol, fields):
1913     """Returns the DocBook output describing the parameters of a symbol.
1914
1915     This can be used for functions, macros or signal handlers.
1916
1917     Args:
1918         symbol_type (str): 'FUNCTION', 'MACRO' or 'SIGNAL'. Signal
1919                            handlers have an implicit user_data parameter last.
1920         symbol (str): the name of the symbol being described.
1921         fields (list): parsed fields from the declaration, used to determine
1922                        undocumented/unused entries
1923
1924     Returns:
1925         str: the formated parameter docs
1926     """
1927     output = ''
1928     num_params = 0
1929     field_descrs = None
1930
1931     if fields:
1932         field_descrs = [f for f in fields if f not in ['void', 'Returns']]
1933     else:
1934         field_descrs = []
1935
1936     params = SymbolParams.get(symbol)
1937     logging.info("param_desc(%s, %s) = %s", symbol_type, symbol, str(params))
1938     # This might be an empty dict, but for SIGNALS we append the user_data docs.
1939     # TODO(ensonic): maybe create that docstring in GetSignals()
1940     if params is not None:
1941         returns = ''
1942         params_desc = ''
1943         missing_parameters = ''
1944         unused_parameters = ''
1945
1946         for param_name, param_desc in iteritems(params):
1947             (param_desc, param_annotations) = ExpandAnnotation(symbol, param_desc)
1948             param_desc = ConvertMarkDown(symbol, param_desc)
1949             # trim
1950             param_desc = re.sub(r'^(\s|\n)+', '', param_desc, flags=re.M | re.S)
1951             param_desc = re.sub(r'(\s|\n)+$', '', param_desc, flags=re.M | re.S)
1952             if param_name == "Returns":
1953                 returns = param_desc
1954                 if param_annotations != '':
1955                     returns += "\n<para>%s</para>" % param_annotations
1956
1957                 elif param_name == "void":
1958                     # FIXME: &common.LogWarning()?
1959                     logging.info("!!!! void in params for %s?\n", symbol)
1960             else:
1961                 if fields:
1962                     if param_name not in field_descrs:
1963                         common.LogWarning(GetSymbolSourceFile(symbol), GetSymbolSourceLine(symbol),
1964                                           "Parameter description for %s::%s is not used from source code comment block." % (symbol, param_name))
1965                         if unused_parameters != '':
1966                             unused_parameters += ", " + param_name
1967                         else:
1968                             unused_parameters = param_name
1969                     else:
1970                         field_descrs.remove(param_name)
1971
1972                 if param_desc != '':
1973                     params_desc += "<row><entry role=\"parameter_name\"><para>%s</para></entry>\n<entry role=\"parameter_description\">%s</entry>\n<entry role=\"parameter_annotations\">%s</entry></row>\n" % (
1974                         param_name, param_desc, param_annotations)
1975                     num_params += 1
1976
1977         for param_name in field_descrs:
1978             common.LogWarning(GetSymbolSourceFile(symbol), GetSymbolSourceLine(symbol),
1979                               "Parameter description for %s::%s is missing in source code comment block." % (symbol, param_name))
1980             if missing_parameters != '':
1981                 missing_parameters += ", " + param_name
1982             else:
1983                 missing_parameters = param_name
1984
1985         # Signals have an implicit user_data parameter which we describe.
1986         if symbol_type == "SIGNAL":
1987             params_desc += "<row><entry role=\"parameter_name\"><simpara>user_data</simpara></entry>\n<entry role=\"parameter_description\"><simpara>user data set when the signal handler was connected.</simpara></entry>\n<entry role=\"parameter_annotations\"></entry></row>\n"
1988
1989         # Start a table if we need one.
1990         if params_desc != '':
1991             sid = common.CreateValidSGMLID("%s.parameters" % symbol)
1992
1993             output += '''<refsect3 id="%s" role="parameters">\n<title>Parameters</title>
1994 <informaltable role="parameters_table" pgwide="1" frame="none">
1995 <tgroup cols="3">
1996 <colspec colname="parameters_name" colwidth="150px"/>
1997 <colspec colname="parameters_description"/>
1998 <colspec colname="parameters_annotations" colwidth="200px"/>
1999 <tbody>
2000 ''' % sid
2001             output += params_desc
2002             output += "</tbody></tgroup></informaltable>\n</refsect3>"
2003
2004         # Output the returns info last
2005         if returns != '':
2006             sid = common.CreateValidSGMLID("%s.returns" % symbol)
2007
2008             output += '''<refsect3 id="%s" role=\"returns\">\n<title>Returns</title>
2009 ''' % sid
2010             output += returns
2011             output += "\n</refsect3>"
2012
2013         # remember missing/unused parameters (needed in tmpl-free build)
2014         if missing_parameters != '' and (symbol not in AllIncompleteSymbols):
2015             AllIncompleteSymbols[symbol] = missing_parameters
2016
2017         if unused_parameters != '' and (symbol not in AllUnusedSymbols):
2018             AllUnusedSymbols[symbol] = unused_parameters
2019
2020     if num_params == 0 and fields and field_descrs:
2021         if symbol not in AllIncompleteSymbols:
2022             AllIncompleteSymbols[symbol] = "<parameters>"
2023     return output
2024
2025
2026 def ParseStabilityLevel(stability, file, line, message):
2027     """Parses a stability level and outputs a warning if it isn't valid.
2028     Args:
2029         stability (str): the stability text.
2030         file, line: context for error message
2031         message: description of where the level is from, to use in any error message.
2032     Returns:
2033         str: the parsed stability level string.
2034     """
2035     sl = stability.strip().lower()
2036     if sl == 'stable':
2037         stability = "Stable"
2038     elif sl == 'unstable':
2039         stability = "Unstable"
2040     elif sl == 'private':
2041         stability = "Private"
2042     else:
2043         common.LogWarning(file, line, "%s is %s." % (message, stability) +
2044                           "It should be one of these: Stable, Unstable, or Private.")
2045     return stability
2046
2047
2048 def OutputDBFile(file, title, section_id, includes, functions_synop, other_synop, functions_details, other_details, signals_synop, signals_desc, args_synop, args_desc, hierarchy, interfaces, implementations, prerequisites, derived, file_objects):
2049     """Outputs the final DocBook file for one section.
2050
2051     Args:
2052         file (str): the name of the file.
2053         title (str): the title from the $MODULE-sections.txt file
2054         section_id (str): the id to use for the toplevel tag.
2055         includes (str): comma-separates list of include files added at top of
2056                    synopsis, with '<' '>' around them (if not already enclosed in '').
2057         functions_synop (str): the DocBook for the Functions Synopsis part.
2058         other_synop (str): the DocBook for the Types and Values Synopsis part.
2059         functions_details (str): the DocBook for the Functions Details part.
2060         other_details (str): the DocBook for the Types and Values Details part.
2061         signal_synop (str): the DocBook for the Signal Synopsis part
2062         signal_desc (str): the DocBook for the Signal Description part
2063         args_synop (str): the DocBook for the Arg Synopsis part
2064         args_desc (str): the DocBook for the Arg Description part
2065         hierarchy (str): the DocBook for the Object Hierarchy part
2066         interfaces (str): the DocBook for the Interfaces part
2067         implementations (str): the DocBook for the Known Implementations part
2068         prerequisites (str): the DocBook for the Prerequisites part
2069         derived (str): the DocBook for the Derived Interfaces part
2070         file_objects (list): objects in this file
2071
2072     Returns:
2073         bool: True if the docs where updated
2074     """
2075
2076     logging.info("Output docbook for file %s with title '%s'", file, title)
2077
2078     # The edited title overrides the one from the sections file.
2079     new_title = SymbolDocs.get(file + ":Title")
2080     if new_title and not new_title.strip() == '':
2081         title = new_title
2082         logging.info("Found title: %s", title)
2083
2084     short_desc = SymbolDocs.get(file + ":Short_Description")
2085     if not short_desc or short_desc.strip() == '':
2086         short_desc = ''
2087     else:
2088         # Don't use ConvertMarkDown here for now since we don't want blocks
2089         short_desc = ExpandAbbreviations(title + ":Short_description", short_desc)
2090         logging.info("Found short_desc: %s", short_desc)
2091
2092     long_desc = SymbolDocs.get(file + ":Long_Description")
2093     if not long_desc or long_desc.strip() == '':
2094         long_desc = ''
2095     else:
2096         long_desc = ConvertMarkDown(title + ":Long_description", long_desc)
2097         logging.info("Found long_desc: %s", long_desc)
2098
2099     see_also = SymbolDocs.get(file + ":See_Also")
2100     if not see_also or re.search(r'^\s*(<para>)?\s*(</para>)?\s*$', see_also):
2101         see_also = ''
2102     else:
2103         see_also = ConvertMarkDown(title + ":See_Also", see_also)
2104         logging.info("Found see_also: %s", see_also)
2105
2106     if see_also:
2107         see_also = "<refsect1 id=\"%s.see-also\">\n<title>See Also</title>\n%s\n</refsect1>\n" % (section_id, see_also)
2108
2109     stability = SymbolDocs.get(file + ":Stability_Level")
2110     if not stability or re.search(r'^\s*$', stability):
2111         stability = ''
2112     else:
2113         line_number = GetSymbolSourceLine(file + ":Stability_Level")
2114         stability = ParseStabilityLevel(stability, file, line_number, "Section stability level")
2115         logging.info("Found stability: %s", stability)
2116
2117     if stability:
2118         AnnotationsUsed[stability] = 1
2119         stability = "<refsect1 id=\"%s.stability-level\">\n<title>Stability Level</title>\n<acronym>%s</acronym>, unless otherwise indicated\n</refsect1>\n" % (
2120             section_id, stability)
2121     elif DEFAULT_STABILITY:
2122         AnnotationsUsed[DEFAULT_STABILITY] = 1
2123         stability = "<refsect1 id=\"%s.stability-level\">\n<title>Stability Level</title>\n<acronym>%s</acronym>, unless otherwise indicated\n</refsect1>\n" % (
2124             section_id, DEFAULT_STABILITY)
2125
2126     image = SymbolDocs.get(file + ":Image")
2127     if not image or re.search(r'^\s*$', image):
2128         image = ''
2129     else:
2130         image = image.strip()
2131
2132         format = None
2133
2134         il = image.lower()
2135         if re.search(r'jpe?g$', il):
2136             format = "format='JPEG'"
2137         elif il.endswith('png'):
2138             format = "format='PNG'"
2139         elif il.endswith('svg'):
2140             format = "format='SVG'"
2141         else:
2142             format = ''
2143
2144         image = "  <inlinegraphic fileref='%s' %s/>\n" % (image, format)
2145
2146     include_output = ''
2147     if includes:
2148         include_output += "<refsect1 id=\"%s.includes\"><title>Includes</title><synopsis>" % section_id
2149         for include in includes.split(','):
2150             if re.search(r'^\".+\"$', include):
2151                 include_output += "#include %s\n" % include
2152             else:
2153                 include = re.sub(r'^\s+|\s+$', '', include, flags=re.S)
2154                 include_output += "#include &lt;%s&gt;\n" % include
2155
2156         include_output += "</synopsis></refsect1>\n"
2157
2158     extralinks = OutputSectionExtraLinks(title, "Section:%s" % file)
2159
2160     old_db_file = os.path.join(DB_OUTPUT_DIR, file + '.xml')
2161     new_db_file = os.path.join(DB_OUTPUT_DIR, file + '.xml.new')
2162
2163     OUTPUT = common.open_text(new_db_file, 'w')
2164
2165     object_anchors = ''
2166     for fobject in file_objects:
2167         if fobject == section_id:
2168             continue
2169         sid = common.CreateValidSGMLID(fobject)
2170         logging.info("Adding anchor for %s\n", fobject)
2171         object_anchors += "<anchor id=\"%s\"/>" % sid
2172
2173     # Make sure we produce valid docbook
2174     if not functions_details:
2175         functions_details = "<para />"
2176
2177     # We used to output this, but is messes up our common.UpdateFileIfChanged code
2178     # since it changes every day (and it is only used in the man pages):
2179     # "<refentry id="$section_id" revision="$mday $month $year">"
2180
2181     OUTPUT.write(REFENTRY.substitute({
2182         'args_desc': args_desc,
2183         'args_synop': args_synop,
2184         'derived': derived,
2185         'extralinks': extralinks,
2186         'functions_details': functions_details,
2187         'functions_synop': functions_synop,
2188         'header': MakeDocHeader('refentry'),
2189         'hierarchy': hierarchy,
2190         'image': image,
2191         'include_output': include_output,
2192         'interfaces': interfaces,
2193         'implementations': implementations,
2194         'long_desc': long_desc,
2195         'object_anchors': object_anchors,
2196         'other_details': other_details,
2197         'other_synop': other_synop,
2198         'prerequisites': prerequisites,
2199         'section_id': section_id,
2200         'see_also': see_also,
2201         'signals_desc': signals_desc,
2202         'signals_synop': signals_synop,
2203         'short_desc': short_desc,
2204         'stability': stability,
2205         'title': title,
2206         'MODULE': MODULE.upper(),
2207     }))
2208     OUTPUT.close()
2209
2210     return common.UpdateFileIfChanged(old_db_file, new_db_file, 0)
2211
2212
2213 def OutputProgramDBFile(program, section_id):
2214     """Outputs the final DocBook file for one program.
2215
2216     Args:
2217         file (str): the name of the file.
2218         section_id (str): the id to use for the toplevel tag.
2219
2220     Returns:
2221         bool: True if the docs where updated
2222     """
2223     logging.info("Output program docbook for %s", program)
2224
2225     short_desc = SourceSymbolDocs.get(program + ":Short_Description")
2226     if not short_desc or short_desc.strip() == '':
2227         short_desc = ''
2228     else:
2229         # Don't use ConvertMarkDown here for now since we don't want blocks
2230         short_desc = ExpandAbbreviations(program, short_desc)
2231         logging.info("Found short_desc: %s", short_desc)
2232
2233     synopsis = SourceSymbolDocs.get(program + ":Synopsis")
2234     if synopsis and synopsis.strip() != '':
2235         items = synopsis.split(' ')
2236         for i in range(0, len(items)):
2237             parameter = items[i]
2238             choice = "plain"
2239             rep = ''
2240
2241             # first parameter is the command name
2242             if i == 0:
2243                 synopsis = "<command>%s</command>\n" % parameter
2244                 continue
2245
2246             # square brackets indicate optional parameters, curly brackets
2247             # indicate required parameters ("plain" parameters are also
2248             # mandatory, but do not get extra decoration)
2249             m1 = re.search(r'^\[(.+?)\]$', parameter)
2250             m2 = re.search(r'^\{(.+?)\}$', parameter)
2251             if m1:
2252                 choice = "opt"
2253                 parameter = m1.group(1)
2254             elif m2:
2255                 choice = "req"
2256                 parameter = m2.group(1)
2257
2258             # parameters ending in "..." are repeatable
2259             if parameter.endswith('...'):
2260                 rep = ' rep=\"repeat\"'
2261                 parameter = parameter[:-3]
2262
2263             # italic parameters are replaceable parameters
2264             parameter = re.sub(r'\*(.+?)\*', r'<replaceable>\1</replaceable>', parameter)
2265
2266             synopsis += "<arg choice=\"%s\"%s>" % (choice, rep)
2267             synopsis += parameter
2268             synopsis += "</arg>\n"
2269
2270         logging.info("Found synopsis: %s", synopsis)
2271     else:
2272         synopsis = "<command>%s</command>" % program
2273
2274     long_desc = SourceSymbolDocs.get(program + ":Long_Description")
2275     if not long_desc or long_desc.strip() == '':
2276         long_desc = ''
2277     else:
2278         long_desc = ConvertMarkDown("%s:Long_description" % program, long_desc)
2279         logging.info("Found long_desc: %s", long_desc)
2280
2281     options = ''
2282     o = program + ":Options"
2283     if o in SourceSymbolDocs:
2284         opts = SourceSymbolDocs[o].split('\t')
2285
2286         logging.info('options: %d, %s', len(opts), str(opts))
2287
2288         options = "<refsect1>\n<title>Options</title>\n<variablelist>\n"
2289         for k in range(0, len(opts), 2):
2290             opt_desc = opts[k + 1]
2291
2292             opt_desc = re.sub(r'\*(.+?)\*', r'<replaceable>\1</replaceable>', opt_desc)
2293
2294             options += "<varlistentry>\n<term>"
2295             opt_names = opts[k].split(',')
2296             for i in range(len(opt_names)):
2297                 prefix = ', ' if i > 0 else ''
2298                 # italic parameters are replaceable parameters
2299                 opt_name = re.sub(r'\*(.+?)\*', r'<replaceable>\1</replaceable>', opt_names[i])
2300
2301                 options += "%s<option>%s</option>\n" % (prefix, opt_name)
2302
2303             options += "</term>\n"
2304             options += "<listitem><para>%s</para></listitem>\n" % opt_desc
2305             options += "</varlistentry>\n"
2306
2307         options += "</variablelist></refsect1>\n"
2308
2309     exit_status = SourceSymbolDocs.get(program + ":Returns")
2310     if exit_status and exit_status != '':
2311         exit_status = ConvertMarkDown("%s:Returns" % program, exit_status)
2312         exit_status = "<refsect1 id=\"%s.exit-status\">\n<title>Exit Status</title>\n%s\n</refsect1>\n" % (
2313             section_id, exit_status)
2314     else:
2315         exit_status = ''
2316
2317     see_also = SourceSymbolDocs.get(program + ":See_Also")
2318     if not see_also or re.search(r'^\s*(<para>)?\s*(</para>)?\s*$', see_also):
2319         see_also = ''
2320     else:
2321         see_also = ConvertMarkDown("%s:See_Also" % program, see_also)
2322         logging.info("Found see_also: %s", see_also)
2323
2324     if see_also:
2325         see_also = "<refsect1 id=\"%s.see-also\">\n<title>See Also</title>\n%s\n</refsect1>\n" % (section_id, see_also)
2326
2327     old_db_file = os.path.join(DB_OUTPUT_DIR, program + ".xml")
2328     new_db_file = os.path.join(DB_OUTPUT_DIR, program + ".xml.new")
2329
2330     OUTPUT = common.open_text(new_db_file, 'w')
2331
2332     OUTPUT.write('''%s
2333 <refentry id="%s">
2334 <refmeta>
2335 <refentrytitle role="top_of_page" id="%s.top_of_page">%s</refentrytitle>
2336 <manvolnum>1</manvolnum>
2337 <refmiscinfo>User Commands</refmiscinfo>
2338 </refmeta>
2339 <refnamediv>
2340 <refname>%s</refname>
2341 <refpurpose>%s</refpurpose>
2342 </refnamediv>
2343 <refsynopsisdiv>
2344 <cmdsynopsis>%s</cmdsynopsis>
2345 </refsynopsisdiv>
2346 <refsect1 id="%s.description" role="desc">
2347 <title role="desc.title">Description</title>
2348 %s
2349 </refsect1>
2350 %s%s%s
2351 </refentry>
2352 ''' % (MakeDocHeader("refentry"), section_id, section_id, program, program, short_desc, synopsis, section_id, long_desc, options, exit_status, see_also))
2353     OUTPUT.close()
2354
2355     return common.UpdateFileIfChanged(old_db_file, new_db_file, 0)
2356
2357
2358 def OutputExtraFile(file):
2359     """Copies an "extra" DocBook file into the output directory, expanding abbreviations.
2360
2361     Args:
2362         file (str): the source file.
2363
2364     Returns:
2365         bool: True if the docs where updated
2366     """
2367
2368     basename = os.path.basename(file)
2369
2370     old_db_file = os.path.join(DB_OUTPUT_DIR, basename)
2371     new_db_file = os.path.join(DB_OUTPUT_DIR, basename + ".new")
2372
2373     contents = common.open_text(file).read()
2374
2375     OUTPUT = common.open_text(new_db_file, 'w')
2376     OUTPUT.write(ExpandAbbreviations(basename + " file", contents))
2377     OUTPUT.close()
2378
2379     return common.UpdateFileIfChanged(old_db_file, new_db_file, 0)
2380
2381
2382 def GetDocbookHeader(main_file):
2383     if os.path.exists(main_file):
2384         INPUT = common.open_text(main_file)
2385         header = ''
2386         for line in INPUT:
2387             if re.search(r'^\s*<(book|chapter|article)', line):
2388                 # check that the top-level tagSYSTEM or the doctype decl contain the xinclude namespace decl
2389                 if not re.search(r'http:\/\/www.w3.org\/200[13]\/XInclude', line) and \
2390                         not re.search(r'http:\/\/www.w3.org\/200[13]\/XInclude', header, flags=re.MULTILINE):
2391                     header = ''
2392                 break
2393
2394             # if there are SYSTEM ENTITIES here, we should prepend "../" to the path
2395             # FIXME: not sure if we can do this now, as people already work-around the problem
2396             # r'#<!ENTITY % ([a-zA-Z-]+) SYSTEM \"([^/][a-zA-Z./]+)\">', r'<!ENTITY % \1 SYSTEM \"../\2\">';
2397             line = re.sub(
2398                 r'<!ENTITY % gtkdocentities SYSTEM "([^"]*)">', r'<!ENTITY % gtkdocentities SYSTEM "../\1">', line)
2399             header += line
2400         INPUT.close()
2401         header = header.strip()
2402     else:
2403         header = '''<?xml version="1.0"?>
2404 <!DOCTYPE refentry PUBLIC "-//OASIS//DTD DocBook XML V4.3//EN"
2405                "http://www.oasis-open.org/docbook/xml/4.3/docbookx.dtd"
2406 [
2407   <!ENTITY % local.common.attrib "xmlns:xi  CDATA  #FIXED 'http://www.w3.org/2003/XInclude'">
2408   <!ENTITY % gtkdocentities SYSTEM "../xml/gtkdocentities.ent">
2409   %gtkdocentities;
2410 ]>'''
2411     return header
2412
2413
2414 def OutputBook(main_file, book_top, book_bottom):
2415     """Outputs the entities that need to be included into the main docbook file for the module.
2416
2417     Args:
2418         book_top (str): the declarations of the entities, which are added
2419                         at the top of the main docbook file.
2420         book_bottom (str): the entities, which are added in the main docbook
2421                            file at the desired position.
2422     """
2423
2424     old_file = os.path.join(DB_OUTPUT_DIR, MODULE + "-doc.top")
2425     new_file = os.path.join(DB_OUTPUT_DIR, MODULE + "-doc.top.new")
2426
2427     OUTPUT = common.open_text(new_file, 'w')
2428     OUTPUT.write(book_top)
2429     OUTPUT.close()
2430
2431     common.UpdateFileIfChanged(old_file, new_file, 0)
2432
2433     old_file = os.path.join(DB_OUTPUT_DIR, MODULE + "-doc.bottom")
2434     new_file = os.path.join(DB_OUTPUT_DIR, MODULE + "-doc.bottom.new")
2435
2436     OUTPUT = common.open_text(new_file, 'w')
2437     OUTPUT.write(book_bottom)
2438     OUTPUT.close()
2439
2440     common.UpdateFileIfChanged(old_file, new_file, 0)
2441
2442     # If the main docbook file hasn't been created yet, we create it here.
2443     # The user can tweak it later.
2444     if main_file and not os.path.exists(main_file):
2445         OUTPUT = common.open_text(main_file, 'w')
2446
2447         logging.info("no master doc, create default one at: " + main_file)
2448
2449         OUTPUT.write('''%s
2450 <book id="index">
2451   <bookinfo>
2452     <title>&package_name; Reference Manual</title>
2453     <releaseinfo>
2454       for &package_string;.
2455       The latest version of this documentation can be found on-line at
2456       <ulink role="online-location" url="http://[SERVER]/&package_name;/index.html">http://[SERVER]/&package_name;/</ulink>.
2457     </releaseinfo>
2458   </bookinfo>
2459
2460   <chapter>
2461     <title>[Insert title here]</title>
2462     %s
2463   </chapter>
2464 ''' % (MakeDocHeader("book"), book_bottom))
2465         if os.path.exists('xml/tree_index.sgml'):
2466             OUTPUT.write('''  <chapter id="object-tree">
2467     <title>Object Hierarchy</title>
2468     <xi:include href="xml/tree_index.sgml"/>
2469   </chapter>
2470 ''')
2471         else:
2472             OUTPUT.write('''  <!-- enable this when you use gobject types
2473   <chapter id="object-tree">
2474     <title>Object Hierarchy</title>
2475     <xi:include href="xml/tree_index.sgml"/>
2476   </chapter>
2477   -->
2478 ''')
2479
2480         OUTPUT.write('''  <index id="api-index-full">
2481     <title>API Index</title>
2482     <xi:include href="xml/api-index-full.xml"><xi:fallback /></xi:include>
2483   </index>
2484   <index id="deprecated-api-index" role="deprecated">
2485     <title>Index of deprecated API</title>
2486     <xi:include href="xml/api-index-deprecated.xml"><xi:fallback /></xi:include>
2487   </index>
2488 ''')
2489         for version in set(Since.values()):
2490             dash_version = version.replace('.', '-')
2491             OUTPUT.write('''  <index id="api-index-%s" role="%s">
2492     <title>Index of new API in %s</title>
2493     <xi:include href="xml/api-index-%s.xml"><xi:fallback /></xi:include>
2494   </index>
2495 ''' % (dash_version, version, version, version))
2496
2497         if AnnotationsUsed:
2498             OUTPUT.write('''  <xi:include href="xml/annotation-glossary.xml"><xi:fallback /></xi:include>
2499 ''')
2500         else:
2501             OUTPUT.write('''  <!-- enable this when you use gobject introspection annotations
2502   <xi:include href="xml/annotation-glossary.xml"><xi:fallback /></xi:include>
2503   -->
2504 ''')
2505
2506         OUTPUT.write('''</book>
2507 ''')
2508
2509         OUTPUT.close()
2510
2511
2512 def CreateValidSGML(text):
2513     """Turn any chars which are used in XML into entities.
2514
2515     e.g. '<' into '&lt;'
2516
2517     Args:
2518         text (str): the text to turn into proper XML.
2519
2520     Returns:
2521         str: escaped input
2522     """
2523
2524     text = text.replace('&', '&amp;')        # Do this first, or the others get messed up.
2525     text = text.replace('<', '&lt;')
2526     text = text.replace('>', '&gt;')
2527     # browsers render single tabs inconsistently
2528     text = re.sub(r'([^\s])\t([^\s])', r'\1&#160;\2', text)
2529     return text
2530
2531
2532 def ConvertSGMLChars(symbol, text):
2533     """Escape XML chars.
2534
2535     This is used for text in source code comment blocks, to turn
2536     chars which are used in XML into entities, e.g. '<' into
2537     &lt;'. Depending on INLINE_MARKUP_MODE, this is done
2538     unconditionally or only if the character doesn't seem to be
2539     part of an XML construct (tag or entity reference).
2540     Args:
2541         text (str): the text to turn into proper XML.
2542
2543     Returns:
2544         str: escaped input
2545     """
2546
2547     if INLINE_MARKUP_MODE:
2548         # For the XML/SGML mode only convert to entities outside CDATA sections.
2549         return ModifyXMLElements(text, symbol,
2550                                  "<!\\[CDATA\\[|<programlisting[^>]*>",
2551                                  ConvertSGMLCharsEndTag,
2552                                  ConvertSGMLCharsCallback)
2553     # For the simple non-sgml mode, convert to entities everywhere.
2554
2555     text = re.sub(r'&(?![a-zA-Z#]+;)', r'&amp;', text)        # Do this first, or the others get messed up.
2556     text = re.sub(r'<', r'&lt;', text)
2557     # Allow '>' at beginning of string for blockquote markdown
2558     text = re.sub(r'''(?<=[^\w\n"'\/-])>''', r'&gt;', text)
2559
2560     return text
2561
2562
2563 def ConvertSGMLCharsEndTag(start_tag):
2564     if start_tag == '<![CDATA[':
2565         return "]]>"
2566     return "</programlisting>"
2567
2568
2569 def ConvertSGMLCharsCallback(text, symbol, tag):
2570     if re.search(r'^<programlisting', tag):
2571         logging.debug('call modifyXML')
2572         # We can handle <programlisting> specially here.
2573         return ModifyXMLElements(text, symbol,
2574                                  "<!\\[CDATA\\[",
2575                                  ConvertSGMLCharsEndTag,
2576                                  ConvertSGMLCharsCallback2)
2577     elif tag == '':
2578         logging.debug('replace entities')
2579         # If we're not in CDATA convert to entities.
2580         text = re.sub(r'&(?![a-zA-Z#]+;)', r'&amp;', text)        # Do this first, or the others get messed up.
2581         text = re.sub(r'<(?![a-zA-Z\/!])', r'&lt;', text)
2582         # Allow '>' at beginning of string for blockquote markdown
2583         text = re.sub(r'''(?<=[^\w\n"'\/-])>''', r'&gt;', text)
2584
2585         # Handle "#include <xxxxx>"
2586         text = re.sub(r'#include(\s+)<([^>]+)>', r'#include\1&lt;\2&gt;', text)
2587
2588     return text
2589
2590
2591 def ConvertSGMLCharsCallback2(text, symbol, tag):
2592     # If we're not in CDATA convert to entities.
2593     # We could handle <programlisting> differently, though I'm not sure it helps.
2594     if tag == '':
2595         # replace only if its not a tag
2596         text = re.sub(r'&(?![a-zA-Z#]+;)', r'&amp;', text)        # Do this first, or the others get messed up.
2597         text = re.sub(r'<(?![a-zA-Z\/!])', r'&lt;', text)
2598         text = re.sub(r'''(?<![a-zA-Z0-9"'\/-])>''', r'&gt;', text)
2599         # Handle "#include <xxxxx>"
2600         text = re.sub(r'/#include(\s+)<([^>]+)>', r'#include\1&lt;\2&gt;', text)
2601
2602     return text
2603
2604
2605 def ExpandAnnotation(symbol, param_desc):
2606     """This turns annotations into acronym tags.
2607     Args:
2608         symbol (str): the symbol being documented, for error messages.
2609         param_desc (str): the text to expand.
2610
2611     Returns:
2612         str: the remaining param_desc
2613         str: the formatted annotations
2614     """
2615     param_annotations = ''
2616
2617     # look for annotations at the start of the comment part
2618     # function level annotations don't end with a colon ':'
2619     m = re.search(r'^\s*\((.*?)\)(:|$)', param_desc)
2620     if m:
2621         param_desc = param_desc[m.end():]
2622
2623         annotations = re.split(r'\)\s*\(', m.group(1))
2624         logging.info("annotations for %s: '%s'\n", symbol, m.group(1))
2625         for annotation in annotations:
2626             # need to search for the longest key-match in %AnnotationDefinition
2627             match_length = 0
2628             match_annotation = ''
2629
2630             for annotationdef in AnnotationDefinition:
2631                 if annotation.startswith(annotationdef):
2632                     if len(annotationdef) > match_length:
2633                         match_length = len(annotationdef)
2634                         match_annotation = annotationdef
2635
2636             annotation_extra = ''
2637             if match_annotation != '':
2638                 m = re.search(match_annotation + r'\s+(.*)', annotation)
2639                 if m:
2640                     annotation_extra = " " + m.group(1)
2641
2642                 AnnotationsUsed[match_annotation] = 1
2643                 param_annotations += "[<acronym>%s</acronym>%s]" % (match_annotation, annotation_extra)
2644             else:
2645                 common.LogWarning(GetSymbolSourceFile(symbol), GetSymbolSourceLine(symbol),
2646                                   "unknown annotation \"%s\" in documentation for %s." % (annotation, symbol))
2647                 param_annotations += "[%s]" % annotation
2648
2649         param_desc = param_desc.strip()
2650         m = re.search(r'^(.*?)\.*\s*$', param_desc, flags=re.S)
2651         param_desc = m.group(1) + '. '
2652
2653     if param_annotations != '':
2654         param_annotations = "<emphasis role=\"annotation\">%s</emphasis>" % param_annotations
2655
2656     return (param_desc, param_annotations)
2657
2658
2659 def ExpandAbbreviations(symbol, text):
2660     """Expand the shortcut notation for symbol references.
2661
2662     This turns the abbreviations function(), macro(), @param, %constant, and #symbol
2663     into appropriate DocBook markup. CDATA sections and <programlisting> parts
2664     are skipped.
2665
2666     Args:
2667         symbol (str): the symbol being documented, for error messages.
2668         text (str): the text to expand.
2669
2670     Returns:
2671         str: the expanded text
2672     """
2673     # Note: This is a fallback and normally done in the markdown parser
2674
2675     logging.debug('expand abbreviations for "%s", text: [%s]', symbol, text)
2676     m = re.search(r'\|\[[^\n]*\n(.*)\]\|', text, flags=re.M | re.S)
2677     if m:
2678         logging.debug('replaced entities in code block')
2679         text = text[:m.start(1)] + md_to_db.ReplaceEntities(m.group(1)) + text[m.end(1):]
2680
2681     # Convert "|[" and "]|" into the start and end of program listing examples.
2682     # Support \[<!-- language="C" --> modifiers
2683     text = re.sub(r'\|\[<!-- language="([^"]+)" -->', r'<informalexample><programlisting language="\1"><![CDATA[', text)
2684     text = re.sub(r'\|\[', r'<informalexample><programlisting><![CDATA[', text)
2685     text = re.sub(r'\]\|', r']]></programlisting></informalexample>', text)
2686
2687     # keep CDATA unmodified, preserve ulink tags (ideally we preseve all tags
2688     # as such)
2689     return ModifyXMLElements(text, symbol,
2690                              "<!\\[CDATA\\[|<ulink[^>]*>|<programlisting[^>]*>|<!DOCTYPE",
2691                              ExpandAbbreviationsEndTag,
2692                              ExpandAbbreviationsCallback)
2693
2694
2695 def ExpandAbbreviationsEndTag(start_tag):
2696     # Returns the end tag (as a regexp) corresponding to the given start tag.
2697     if start_tag == r'<!\[CDATA\[':
2698         return "]]>"
2699     if start_tag == "<!DOCTYPE":
2700         return '>'
2701     m = re.search(r'<(\w+)', start_tag)
2702     if m:
2703         return "</%s>" % m.group(1)
2704
2705     logging.warning('no end tag for "%s"', start_tag)
2706     return ''
2707
2708
2709 def ExpandAbbreviationsCallback(text, symbol, tag):
2710     # Called inside or outside each CDATA or <programlisting> section.
2711     if tag.startswith(r'^<programlisting'):
2712         # Handle any embedded CDATA sections.
2713         return ModifyXMLElements(text, symbol,
2714                                  "<!\\[CDATA\\[",
2715                                  ExpandAbbreviationsEndTag,
2716                                  ExpandAbbreviationsCallback2)
2717     elif tag == '':
2718         # NOTE: this is a fallback. It is normally done by the Markdown parser.
2719         # but is also used for OutputExtraFile
2720
2721         # We are outside any CDATA or <programlisting> sections, so we expand
2722         # any gtk-doc abbreviations.
2723
2724         # Convert '@param()'
2725         # FIXME: we could make those also links ($symbol.$2), but that would be less
2726         # useful as the link target is a few lines up or down
2727         text = re.sub(r'(\A|[^\\])\@(\w+((\.|->)\w+)*)\s*\(\)', r'\1<parameter>\2()</parameter>', text)
2728
2729         # Convert 'function()' or 'macro()'.
2730         # if there is abc_*_def() we don't want to make a link to _def()
2731         # FIXME: also handle abc(def(....)) : but that would need to be done recursively :/
2732         def f1(m):
2733             return m.group(1) + MakeXRef(m.group(2), tagify(m.group(2) + "()", "function"))
2734         text = re.sub(r'([^\*.\w])(\w+)\s*\(\)', f1, text)
2735         # handle #Object.func()
2736         text = re.sub(r'(\A|[^\\])#([\w\-:\.]+[\w]+)\s*\(\)', f1, text)
2737
2738         # Convert '@param', but not '\@param'.
2739         text = re.sub(r'(\A|[^\\])\@(\w+((\.|->)\w+)*)', r'\1<parameter>\2</parameter>', text)
2740         text = re.sub(r'/\\\@', r'\@', text)
2741
2742         # Convert '%constant', but not '\%constant'.
2743         # Also allow negative numbers, e.g. %-1.
2744         def f2(m):
2745             return m.group(1) + MakeXRef(m.group(2), tagify(m.group(2), "literal"))
2746         text = re.sub(r'(\A|[^\\])\%(-?\w+)', f2, text)
2747         text = re.sub(r'\\\%', r'\%', text)
2748
2749         # Convert '#symbol', but not '\#symbol'.
2750         def f3(m):
2751             return m.group(1) + MakeHashXRef(m.group(2), "type")
2752         text = re.sub(r'(\A|[^\\])#([\w\-:\.]+[\w]+)', f3, text)
2753         text = re.sub(r'\\#', '#', text)
2754
2755     return text
2756
2757
2758 def ExpandAbbreviationsCallback2(text, symbol, tag):
2759     # This is called inside a <programlisting>
2760     if tag == '':
2761         # We are inside a <programlisting> but outside any CDATA sections,
2762         # so we expand any gtk-doc abbreviations.
2763         # FIXME: why is this different from &ExpandAbbreviationsCallback(),
2764         #        why not just call it
2765         text = re.sub(r'#(\w+)', lambda m: '%s;' % MakeHashXRef(m.group(1), ''), text)
2766     elif tag == "<![CDATA[":
2767         # NOTE: this is a fallback. It is normally done by the Markdown parser.
2768         text = ReplaceEntities(text, symbol)
2769
2770     return text
2771
2772
2773 def MakeHashXRef(symbol, tag):
2774     text = symbol
2775
2776     # Check for things like '#include', '#define', and skip them.
2777     if symbol in PreProcessorDirectives:
2778         return "#%s" % symbol
2779
2780     # Get rid of special suffixes ('-struct','-enum').
2781     text = re.sub(r'-struct$', '', text)
2782     text = re.sub(r'-enum$', '', text)
2783
2784     # If the symbol is in the form "Object::signal", then change the symbol to
2785     # "Object-signal" and use "signal" as the text.
2786     if '::' in symbol:
2787         o, s = symbol.split('::', 1)
2788         symbol = '%s-%s' % (o, s)
2789         text = u'“' + s + u'”'
2790
2791     # If the symbol is in the form "Object:property", then change the symbol to
2792     # "Object--property" and use "property" as the text.
2793     if ':' in symbol:
2794         o, p = symbol.split(':', 1)
2795         symbol = '%s--%s' % (o, p)
2796         text = u'“' + p + u'”'
2797
2798     if tag != '':
2799         text = tagify(text, tag)
2800
2801     return MakeXRef(symbol, text)
2802
2803
2804 def ModifyXMLElements(text, symbol, start_tag_regexp, end_tag_func, callback):
2805     """Rewrite XML blocks.
2806
2807     Looks for given XML element tags within the text, and calls
2808     the callback on pieces of text inside & outside those elements.
2809     Used for special handling of text inside things like CDATA
2810     and <programlisting>.
2811
2812     Args:
2813         text (str): the text.
2814         symbol (str): the symbol currently being documented (only used for
2815                       error messages).
2816         start_tag_regexp (str): the regular expression to match start tags.
2817                                 e.g. "<!\\[CDATA\\[|<programlisting[^>]*>" to
2818                                 match CDATA sections or programlisting elements.
2819         end_tag_func (func): function which is passed the matched start tag
2820                              and should return the appropriate end tag string
2821                              regexp.
2822         callback - callback called with each part of the text. It is
2823                    called with a piece of text, the symbol being
2824                    documented, and the matched start tag or '' if the text
2825                    is outside the XML elements being matched.
2826
2827     Returns:
2828         str: modified text
2829     """
2830     before_tag = start_tag = end_tag_regexp = end_tag = None
2831     result = ''
2832
2833     logging.debug('modify xml for symbol: %s, regex: %s, text: [%s]', symbol, start_tag_regexp, text)
2834
2835     m = re.search(start_tag_regexp, text, flags=re.S)
2836     while m:
2837         before_tag = text[:m.start()]  # Prematch for last successful match string
2838         start_tag = m.group(0)         # Last successful match
2839         text = text[m.end():]          # Postmatch for last successful match string
2840         # get the matching end-tag for current tag
2841         end_tag_regexp = end_tag_func(start_tag)
2842
2843         logging.debug('symbol: %s matched start: %s, end_tag: %s, text: [%s]', symbol, start_tag, end_tag_regexp, text)
2844
2845         logging.debug('converting before tag: [%s]', before_tag)
2846         result += callback(before_tag, symbol, '')
2847         result += start_tag
2848
2849         m2 = re.search(end_tag_regexp, text, flags=re.S)
2850         if m2:
2851             before_tag = text[:m2.start()]
2852             end_tag = m2.group(0)
2853             text = text[m2.end():]
2854
2855             logging.debug('symbol: %s matched end %s: text: [%s]', symbol, end_tag, text)
2856
2857             result += callback(before_tag, symbol, start_tag)
2858             result += end_tag
2859         else:
2860             common.LogWarning(GetSymbolSourceFile(symbol), GetSymbolSourceLine(symbol),
2861                               "Can't find tag end: %s in docs for: %s." % (end_tag_regexp, symbol))
2862             # Just assume it is all inside the tag.
2863             result += callback(text, symbol, start_tag)
2864             text = ''
2865         m = re.search(start_tag_regexp, text, flags=re.S)
2866
2867     # Handle any remaining text outside the tags.
2868     logging.debug('converting after tag: [%s]', text)
2869     result += callback(text, symbol, '')
2870     logging.debug('results for symbol: %s, text: [%s]', symbol, result)
2871
2872     return result
2873
2874
2875 def tagify(text, elem):
2876     # Adds a tag around some text.
2877     # e.g tagify("Text", "literal") => "<literal>Text</literal>".
2878     return '<' + elem + '>' + text + '</' + elem + '>'
2879
2880
2881 def MakeDocHeader(tag):
2882     """Builds a docbook header for the given tag.
2883
2884     Args:
2885         tag (str): doctype tag
2886
2887     Returns:
2888         str: the docbook header
2889     """
2890     header = re.sub(r'<!DOCTYPE \w+', r'<!DOCTYPE ' + tag, doctype_header)
2891     # fix the path for book since this is one level up
2892     if tag == 'book':
2893         header = re.sub(
2894             r'<!ENTITY % gtkdocentities SYSTEM "../([a-zA-Z./]+)">', r'<!ENTITY % gtkdocentities SYSTEM "\1">', header)
2895     return header
2896
2897
2898 def MakeXRef(symbol, text=None):
2899     """This returns a cross-reference link to the given symbol.
2900
2901     Though it doesn't try to do this for a few standard C types that it knows
2902     won't be in the documentation.
2903
2904     Args:
2905         symbol (str): the symbol to try to create a XRef to.
2906         text (str): text to put inside the XRef, defaults to symbol
2907
2908     Returns:
2909         str: a docbook link
2910     """
2911     symbol = symbol.strip()
2912     if not text:
2913         text = symbol
2914
2915         # Get rid of special suffixes ('-struct','-enum').
2916         text = re.sub(r'-struct$', '', text)
2917         text = re.sub(r'-enum$', '', text)
2918
2919     if ' ' in symbol:
2920         return text
2921
2922     logging.info("Getting type link for %s -> %s", symbol, text)
2923
2924     symbol_id = common.CreateValidSGMLID(symbol)
2925     return "<link linkend=\"%s\">%s</link>" % (symbol_id, text)
2926
2927
2928 def MakeIndexterms(symbol, sid):
2929     """This returns a indexterm elements for the given symbol
2930
2931     Args:
2932         symbol (str): the symbol to create indexterms for
2933
2934     Returns:
2935         str: doxbook index terms
2936     """
2937     terms = ''
2938     sortas = ''
2939
2940     # make the index useful, by ommiting the namespace when sorting
2941     if NAME_SPACE != '':
2942         m = re.search(r'^%s\_?(.*)' % NAME_SPACE, symbol, flags=re.I)
2943         if m:
2944             sortas = ' sortas="%s"' % m.group(1)
2945
2946     if symbol in Deprecated:
2947         terms += "<indexterm zone=\"%s\" role=\"deprecated\"><primary%s>%s</primary></indexterm>" % (
2948             sid, sortas, symbol)
2949         IndexEntriesDeprecated[symbol] = sid
2950         IndexEntriesFull[symbol] = sid
2951     if symbol in Since:
2952         since = Since[symbol].strip()
2953         if since != '':
2954             terms += "<indexterm zone=\"%s\" role=\"%s\"><primary%s>%s</primary></indexterm>" % (
2955                 sid, since, sortas, symbol)
2956         IndexEntriesSince[symbol] = sid
2957         IndexEntriesFull[symbol] = sid
2958     if terms == '':
2959         terms += "<indexterm zone=\"%s\"><primary%s>%s</primary></indexterm>" % (sid, sortas, symbol)
2960         IndexEntriesFull[symbol] = sid
2961     return terms
2962
2963
2964 def MakeDeprecationNote(symbol):
2965     """This returns a deprecation warning for the given symbol.
2966
2967     Args:
2968         symbol (str): the symbol to try to create a warning for.
2969
2970     Returns:
2971         str: formatted warning or empty string if symbol is not deprecated
2972     """
2973     desc = ''
2974     if symbol in Deprecated:
2975         desc += "<warning><para><literal>%s</literal> " % symbol
2976         note = Deprecated[symbol]
2977
2978         m = re.search(r'^\s*([0-9\.]+)\s*:?', note)
2979         if m:
2980             desc += "has been deprecated since version %s and should not be used in newly-written code.</para>" % m.group(
2981                 1)
2982         else:
2983             desc += "is deprecated and should not be used in newly-written code.</para>"
2984
2985         note = re.sub(r'^\s*([0-9\.]+)\s*:?\s*', '', note)
2986         note = note.strip()
2987
2988         if note != '':
2989             note = ConvertMarkDown(symbol, note)
2990             desc += " " + note
2991
2992         desc += "</warning>\n"
2993
2994     return desc
2995
2996
2997 def MakeConditionDescription(symbol):
2998     """This returns a sumary of conditions for the given symbol.
2999
3000     Args:
3001         symbol (str): the symbol to create the sumary for.
3002
3003     Returns:
3004         str: formatted text or empty string if no special conditions apply.
3005     """
3006     desc = ''
3007     if symbol in Deprecated:
3008         if desc != '':
3009             desc += "|"
3010         m = re.search(r'^\s*(.*?)\s*$', Deprecated[symbol])
3011         if m:
3012             desc += "deprecated:%s" % m.group(1)
3013         else:
3014             desc += "deprecated"
3015
3016     if symbol in Since:
3017         if desc != '':
3018             desc += "|"
3019         m = re.search(r'^\s*(.*?)\s*$', Since[symbol])
3020         if m:
3021             desc += "since:%s" % m.group(1)
3022         else:
3023             desc += "since"
3024
3025     if symbol in StabilityLevel:
3026         if desc != '':
3027             desc += "|"
3028
3029         desc += "stability:" + StabilityLevel[symbol]
3030
3031     if desc != '':
3032         cond = re.sub(r'"', r'&quot;', desc)
3033         desc = ' condition=\"%s\"' % cond
3034         logging.info("condition for '%s' = '%s'", symbol, desc)
3035
3036     return desc
3037
3038
3039 def GetHierarchy(gobject, hierarchy):
3040     """Generate the object inheritance graph.
3041
3042     Returns the DocBook output describing the ancestors and
3043     immediate children of a GObject subclass. It uses the
3044     global Objects and ObjectLevels arrays to walk the tree.
3045
3046     Args:
3047         object (str): the GtkObject subclass.
3048         hierarchy (list) - previous hierarchy
3049
3050     Returns:
3051         list: lines of docbook describing the hierarchy
3052     """
3053     # Find object in the objects array.
3054     found = False
3055     children = []
3056     level = 0
3057     j = 0
3058     for i in range(len(Objects)):
3059         if found:
3060             if ObjectLevels[i] <= level:
3061                 break
3062
3063             elif ObjectLevels[i] == level + 1:
3064                 children.append(Objects[i])
3065
3066         elif Objects[i] == gobject:
3067             found = True
3068             j = i
3069             level = ObjectLevels[i]
3070
3071     if not found:
3072         return hierarchy
3073
3074     logging.info("=== Hierachy for: %s (%d existing entries) ===", gobject, len(hierarchy))
3075
3076     # Walk up the hierarchy, pushing ancestors onto the ancestors array.
3077     ancestors = [gobject]
3078     logging.info("Level: %s", level)
3079     while level > 1:
3080         j -= 1
3081         if ObjectLevels[j] < level:
3082             ancestors.append(Objects[j])
3083             level = ObjectLevels[j]
3084             logging.info("Level: %s", level)
3085
3086     # Output the ancestors, indented and with links.
3087     logging.info('%d ancestors', len(ancestors))
3088     last_index = 0
3089     level = 1
3090     for i in range(len(ancestors) - 1, -1, -1):
3091         ancestor = ancestors[i]
3092         ancestor_id = common.CreateValidSGMLID(ancestor)
3093         indent = ' ' * (level * 4)
3094         # Don't add a link to the current object, i.e. when i == 0.
3095         if i > 0:
3096             entry_text = indent + "<link linkend=\"%s\">%s</link>" % (ancestor_id, ancestor)
3097             alt_text = indent + ancestor
3098         else:
3099             entry_text = indent + ancestor
3100             alt_text = indent + "<link linkend=\"%s\">%s</link>" % (ancestor_id, ancestor)
3101
3102         logging.info("Checking for '%s' or '%s'", entry_text, alt_text)
3103         # Check if we already have this object
3104         index = -1
3105         for j in range(len(hierarchy)):
3106             if hierarchy[j] == entry_text or (hierarchy[j] == alt_text):
3107                 index = j
3108                 break
3109         if index == -1:
3110             # We have a new entry, find insert position in alphabetical order
3111             found = False
3112             for j in range(last_index, len(hierarchy)):
3113                 if not re.search(r'^' + indent, hierarchy[j]):
3114                     last_index = j
3115                     found = True
3116                     break
3117                 elif re.search(r'^%s[^ ]' % indent, hierarchy[j]):
3118                     stripped_text = hierarchy[j]
3119                     if r'<link linkend' not in entry_text:
3120                         stripped_text = re.sub(r'<link linkend="[A-Za-z]*">', '', stripped_text)
3121                         stripped_text = re.sub(r'</link>', '', stripped_text)
3122
3123                     if entry_text < stripped_text:
3124                         last_index = j
3125                         found = True
3126                         break
3127
3128             # Append to bottom
3129             if not found:
3130                 last_index = len(hierarchy)
3131
3132             logging.debug('insert at %d: %s', last_index, entry_text)
3133             hierarchy.insert(last_index, entry_text)
3134             last_index += 1
3135         else:
3136             # Already have this one, make sure we use the not linked version
3137             if r'<link linkend' not in entry_text:
3138                 hierarchy[j] = entry_text
3139
3140             # Remember index as base insert point
3141             last_index = index + 1
3142
3143         level += 1
3144
3145     # Output the children, indented and with links.
3146     logging.info('%d children', len(children))
3147     for i in range(len(children)):
3148         sid = common.CreateValidSGMLID(children[i])
3149         indented_text = ' ' * (level * 4) + "<link linkend=\"%s\">%s</link>" % (sid, children[i])
3150         logging.debug('insert at %d: %s', last_index, indented_text)
3151         hierarchy.insert(last_index, indented_text)
3152         last_index += 1
3153     return hierarchy
3154
3155
3156 def GetInterfaces(gobject):
3157     """Generate interface implementation graph.
3158
3159     Returns the DocBook output describing the interfaces
3160     implemented by a class. It uses the global Interfaces hash.
3161
3162     Args:
3163         object (str): the GObject subclass.
3164
3165     Returns:
3166         str: implemented interfaces
3167     """
3168     text = ''
3169     # Find object in the objects array.
3170     if gobject in Interfaces:
3171         ifaces = Interfaces[gobject].split()
3172         text = '''<para>
3173 %s implements
3174 ''' % gobject
3175         count = len(ifaces)
3176         for i in range(count):
3177             sid = common.CreateValidSGMLID(ifaces[i])
3178             text += " <link linkend=\"%s\">%s</link>" % (sid, ifaces[i])
3179             if i < count - 2:
3180                 text += ', '
3181             elif i < count - 1:
3182                 text += ' and '
3183             else:
3184                 text += '.'
3185         text += '</para>\n'
3186     return text
3187
3188
3189 def GetImplementations(gobject):
3190     """Generate interface usage graph.
3191
3192     Returns the DocBook output describing the implementations
3193     of an interface. It uses the global Interfaces hash.
3194
3195     Args:
3196         object (str): the GObject subclass.
3197
3198     Returns:
3199         str: interface implementations
3200     """
3201     text = ''
3202     impls = []
3203     for key in Interfaces:
3204         if re.search(r'\b%s\b' % gobject, Interfaces[key]):
3205             impls.append(key)
3206
3207     count = len(impls)
3208     if count > 0:
3209         impls.sort()
3210         text = '''<para>
3211 %s is implemented by
3212 ''' % gobject
3213         for i in range(count):
3214             sid = common.CreateValidSGMLID(impls[i])
3215             text += " <link linkend=\"%s\">%s</link>" % (sid, impls[i])
3216             if i < count - 2:
3217                 text += ', '
3218             elif i < count - 1:
3219                 text += ' and '
3220             else:
3221                 text += '.'
3222         text += '</para>\n'
3223     return text
3224
3225
3226 def GetPrerequisites(iface):
3227     """Generates interface requirements.
3228
3229     Returns the DocBook output describing the prerequisites
3230     of an interface. It uses the global Prerequisites hash.
3231     Args:
3232         iface (str): the interface.
3233
3234     Returns:
3235         str: required interfaces
3236     """
3237
3238     text = ''
3239     if iface in Prerequisites:
3240         text = '''<para>
3241 %s requires
3242 ''' % iface
3243         prereqs = Prerequisites[iface].split()
3244         count = len(prereqs)
3245         for i in range(count):
3246             sid = common.CreateValidSGMLID(prereqs[i])
3247             text += " <link linkend=\"%s\">%s</link>" % (sid, prereqs[i])
3248             if i < count - 2:
3249                 text += ', '
3250             elif i < count - 1:
3251                 text += ' and '
3252             else:
3253                 text += '.'
3254         text += '</para>\n'
3255     return text
3256
3257
3258 def GetDerived(iface):
3259     """
3260     Returns the DocBook output describing the derived interfaces
3261     of an interface. It uses the global %Prerequisites hash.
3262
3263     Args:
3264         iface (str): the interface.
3265
3266     Returns:
3267         str: derived interfaces
3268     """
3269     text = ''
3270     derived = []
3271     for key in Prerequisites:
3272         if re.search(r'\b%s\b' % iface, Prerequisites[key]):
3273             derived.append(key)
3274
3275     count = len(derived)
3276     if count > 0:
3277         derived.sort()
3278         text = '''<para>
3279 %s is required by
3280 ''' % iface
3281         for i in range(count):
3282             sid = common.CreateValidSGMLID(derived[i])
3283             text += " <link linkend=\"%s\">%s</link>" % (sid, derived[i])
3284             if i < count - 2:
3285                 text += ', '
3286             elif i < count - 1:
3287                 text += ' and '
3288             else:
3289                 text += '.'
3290         text += '</para>\n'
3291     return text
3292
3293
3294 def GetSignals(gobject):
3295     """Generate signal docs.
3296
3297     Returns the synopsis and detailed description DocBook output
3298     for the signal handlers of a given GObject subclass.
3299
3300     Args:
3301         object (str): the GObject subclass, e.g. 'GtkButton'.
3302
3303     Returns:
3304         str: signal docs
3305     """
3306     synop = ''
3307     desc = ''
3308
3309     for i in range(len(SignalObjects)):
3310         if SignalObjects[i] == gobject:
3311             logging.info("Found signal: %s", SignalNames[i])
3312             name = SignalNames[i]
3313             symbol = '%s::%s' % (gobject, name)
3314             sid = common.CreateValidSGMLID('%s-%s' % (gobject, name))
3315
3316             desc += u"<refsect2 id=\"%s\" role=\"signal\"><title>The <literal>“%s”</literal> signal</title>\n" % (
3317                 sid, name)
3318             desc += MakeIndexterms(symbol, sid)
3319             desc += "\n"
3320             desc += OutputSymbolExtraLinks(symbol)
3321
3322             desc += "<programlisting language=\"C\">"
3323
3324             m = re.search(r'\s*(const\s+)?(\w+)\s*(\**)', SignalReturns[i])
3325             type_modifier = m.group(1) or ''
3326             gtype = m.group(2)
3327             pointer = m.group(3)
3328             xref = MakeXRef(gtype, tagify(gtype, "returnvalue"))
3329
3330             ret_type_output = '%s%s%s' % (type_modifier, xref, pointer)
3331             callback_name = "user_function"
3332             desc += '%s\n%s (' % (ret_type_output, callback_name)
3333
3334             indentation = ' ' * (len(callback_name) + 2)
3335
3336             sourceparams = SourceSymbolParams.get(symbol)
3337             sourceparam_names = None
3338             if sourceparams:
3339                 sourceparam_names = list(sourceparams)  # keys as list
3340             params = SignalPrototypes[i].splitlines()
3341             type_len = len("gpointer")
3342             name_len = len("user_data")
3343             # do two passes, the first one is to calculate padding
3344             for l in range(2):
3345                 for j in range(len(params)):
3346                     param_name = None
3347                     # allow alphanumerics, '_', '[' & ']' in param names
3348                     m = re.search(r'^\s*(\w+)\s*(\**)\s*([\w\[\]]+)\s*$', params[j])
3349                     if m:
3350                         gtype = m.group(1)
3351                         pointer = m.group(2)
3352                         if sourceparam_names:
3353                             if j < len(sourceparam_names):
3354                                 param_name = sourceparam_names[j]
3355                                 logging.info('from sourceparams: "%s" (%d: %s)', param_name, j, params[j])
3356                             # we're mssing the docs for this param, don't warn here though
3357                         else:
3358                             param_name = m.group(3)
3359                             logging.info('from params: "%s" (%d: %s)', param_name, j, params[j])
3360
3361                         if not param_name:
3362                             param_name = "arg%d" % j
3363
3364                         if l == 0:
3365                             if len(gtype) + len(pointer) > type_len:
3366                                 type_len = len(gtype) + len(pointer)
3367                             if len(param_name) > name_len:
3368                                 name_len = len(param_name)
3369                         else:
3370                             logging.info("signal arg[%d]: '%s'", j, param_name)
3371                             xref = MakeXRef(gtype, tagify(gtype, "type"))
3372                             pad = ' ' * (type_len - len(gtype) - len(pointer))
3373                             desc += '%s%s %s%s,\n' % (xref, pad, pointer, param_name)
3374                             desc += indentation
3375
3376                     else:
3377                         common.LogWarning(GetSymbolSourceFile(symbol), GetSymbolSourceLine(symbol),
3378                                           "Can't parse arg: %s\nArgs:%s" % (params[j], SignalPrototypes[i]))
3379
3380             xref = MakeXRef("gpointer", tagify("gpointer", "type"))
3381             pad = ' ' * (type_len - len("gpointer"))
3382             desc += '%s%s user_data)' % (xref, pad)
3383             desc += "</programlisting>\n"
3384
3385             flags = SignalFlags[i]
3386             flags_string = ''
3387             if flags:
3388                 if 'f' in flags:
3389                     flags_string = "<link linkend=\"G-SIGNAL-RUN-FIRST:CAPS\">Run First</link>"
3390
3391                 elif 'l' in flags:
3392                     flags_string = "<link linkend=\"G-SIGNAL-RUN-LAST:CAPS\">Run Last</link>"
3393
3394                 elif 'c' in flags:
3395                     flags_string = "<link linkend=\"G-SIGNAL-RUN-CLEANUP:CAPS\">Cleanup</link>"
3396                     flags_string = "Cleanup"
3397
3398                 if 'r' in flags:
3399                     if flags_string:
3400                         flags_string += " / "
3401                     flags_string = "<link linkend=\"G-SIGNAL-NO-RECURSE:CAPS\">No Recursion</link>"
3402
3403                 if 'd' in flags:
3404                     if flags_string:
3405                         flags_string += " / "
3406                     flags_string = "<link linkend=\"G-SIGNAL-DETAILED:CAPS\">Has Details</link>"
3407
3408                 if 'a' in flags:
3409                     if flags_string:
3410                         flags_string += " / "
3411                     flags_string = "<link linkend=\"G-SIGNAL-ACTION:CAPS\">Action</link>"
3412
3413                 if 'h' in flags:
3414                     if flags_string:
3415                         flags_string += " / "
3416                     flags_string = "<link linkend=\"G-SIGNAL-NO-HOOKS:CAPS\">No Hooks</link>"
3417
3418             synop += "<row><entry role=\"signal_type\">%s</entry><entry role=\"signal_name\"><link linkend=\"%s\">%s</link></entry><entry role=\"signal_flags\">%s</entry></row>\n" % (
3419                 ret_type_output, sid, name, flags_string)
3420
3421             parameters = OutputParamDescriptions("SIGNAL", symbol, None)
3422             logging.info("formatted signal params: '%s' -> '%s'", symbol, parameters)
3423
3424             AllSymbols[symbol] = 1
3425             if symbol in SymbolDocs:
3426                 symbol_docs = ConvertMarkDown(symbol, SymbolDocs[symbol])
3427
3428                 desc += symbol_docs
3429
3430                 if not IsEmptyDoc(SymbolDocs[symbol]):
3431                     AllDocumentedSymbols[symbol] = 1
3432
3433             if symbol in SymbolAnnotations:
3434                 param_desc = SymbolAnnotations[symbol]
3435                 (param_desc, param_annotations) = ExpandAnnotation(symbol, param_desc)
3436                 if param_annotations != '':
3437                     desc += "\n<para>%s</para>" % param_annotations
3438
3439             desc += MakeDeprecationNote(symbol)
3440
3441             desc += parameters
3442             if flags_string:
3443                 desc += "<para>Flags: %s</para>\n" % flags_string
3444
3445             desc += OutputSymbolTraits(symbol)
3446             desc += "</refsect2>"
3447
3448     return (synop, desc)
3449
3450
3451 def GetArgs(gobject):
3452     """Generate property docs.
3453
3454     Returns the synopsis and detailed description DocBook output
3455     for the Args of a given GtkObject subclass.
3456
3457     Args:
3458         object (str): the GObject subclass, e.g. 'GtkButton'.
3459
3460     Returns:
3461         str: property docs
3462     """
3463     synop = ''
3464     desc = ''
3465     child_synop = ''
3466     child_desc = ''
3467     style_synop = ''
3468     style_desc = ''
3469
3470     for i in range(len(ArgObjects)):
3471         if ArgObjects[i] == gobject:
3472             logging.info("Found arg: %s", ArgNames[i])
3473             name = ArgNames[i]
3474             flags = ArgFlags[i]
3475             flags_string = ''
3476             kind = ''
3477             id_sep = ''
3478
3479             if 'c' in flags:
3480                 kind = "child property"
3481                 id_sep = "c-"
3482             elif 's' in flags:
3483                 kind = "style property"
3484                 id_sep = "s-"
3485             else:
3486                 kind = "property"
3487
3488             # Remember only one colon so we don't clash with signals.
3489             symbol = '%s:%s' % (gobject, name)
3490             # use two dashes and ev. an extra separator here for the same reason.
3491             sid = common.CreateValidSGMLID('%s--%s%s' % (gobject, id_sep, name))
3492
3493             atype = ArgTypes[i]
3494             type_output = None
3495             arange = ArgRanges[i]
3496             range_output = CreateValidSGML(arange)
3497             default = ArgDefaults[i]
3498             default_output = CreateValidSGML(default)
3499
3500             if atype == "GtkString":
3501                 atype = "char&#160;*"
3502
3503             if atype == "GtkSignal":
3504                 atype = "GtkSignalFunc, gpointer"
3505                 type_output = MakeXRef("GtkSignalFunc") + ", " + MakeXRef("gpointer")
3506             elif re.search(r'^(\w+)\*$', atype):
3507                 m = re.search(r'^(\w+)\*$', atype)
3508                 type_output = MakeXRef(m.group(1), tagify(m.group(1), "type")) + "&#160;*"
3509             else:
3510                 type_output = MakeXRef(atype, tagify(atype, "type"))
3511
3512             if 'r' in flags:
3513                 flags_string = "Read"
3514
3515             if 'w' in flags:
3516                 if flags_string:
3517                     flags_string += " / "
3518                 flags_string += "Write"
3519
3520             if 'x' in flags:
3521                 if flags_string:
3522                     flags_string += " / "
3523                 flags_string += "Construct"
3524
3525             if 'X' in flags:
3526                 if flags_string:
3527                     flags_string += " / "
3528                 flags_string += "Construct Only"
3529
3530             AllSymbols[symbol] = 1
3531             blurb = ''
3532             if symbol in SymbolDocs and not IsEmptyDoc(SymbolDocs[symbol]):
3533                 blurb = ConvertMarkDown(symbol, SymbolDocs[symbol])
3534                 logging.info(".. [%s][%s]", SymbolDocs[symbol], blurb)
3535                 AllDocumentedSymbols[symbol] = 1
3536
3537             else:
3538                 if ArgBlurbs[i] != '':
3539                     blurb = "<para>" + CreateValidSGML(ArgBlurbs[i]) + "</para>"
3540                     AllDocumentedSymbols[symbol] = 1
3541                 else:
3542                     # FIXME: print a warning?
3543                     logging.info(".. no description")
3544
3545             pad1 = ''
3546             if len(name) < 24:
3547                 pad1 = " " * (24 - len(name))
3548
3549             arg_synop = "<row><entry role=\"property_type\">%s</entry><entry role=\"property_name\"><link linkend=\"%s\">%s</link></entry><entry role=\"property_flags\">%s</entry></row>\n" % (
3550                 type_output, sid, name, flags_string)
3551             arg_desc = u"<refsect2 id=\"%s\" role=\"property\"><title>The <literal>“%s”</literal> %s</title>\n" % (
3552                 sid, name, kind)
3553             arg_desc += MakeIndexterms(symbol, sid)
3554             arg_desc += "\n"
3555             arg_desc += OutputSymbolExtraLinks(symbol)
3556
3557             arg_desc += u"<programlisting>  “%s”%s %s</programlisting>\n" % (name, pad1, type_output)
3558             arg_desc += blurb
3559             if symbol in SymbolAnnotations:
3560                 param_desc = SymbolAnnotations[symbol]
3561                 (param_desc, param_annotations) = ExpandAnnotation(symbol, param_desc)
3562                 if param_annotations != '':
3563                     arg_desc += "\n<para>%s</para>" % param_annotations
3564
3565             arg_desc += MakeDeprecationNote(symbol)
3566
3567             if flags_string:
3568                 arg_desc += "<para>Flags: %s</para>\n" % flags_string
3569
3570             if arange != '':
3571                 arg_desc += "<para>Allowed values: %s</para>\n" % range_output
3572
3573             if default != '':
3574                 arg_desc += "<para>Default value: %s</para>\n" % default_output
3575
3576             arg_desc += OutputSymbolTraits(symbol)
3577             arg_desc += "</refsect2>\n"
3578
3579             if 'c' in flags:
3580                 child_synop += arg_synop
3581                 child_desc += arg_desc
3582
3583             elif 's' in flags:
3584                 style_synop += arg_synop
3585                 style_desc += arg_desc
3586
3587             else:
3588                 synop += arg_synop
3589                 desc += arg_desc
3590
3591     return (synop, child_synop, style_synop, desc, child_desc, style_desc)
3592
3593
3594 def IgnorePath(path, source_dirs, ignore_files):
3595     for sdir in source_dirs:
3596         # Cut off base directory
3597         m1 = re.search(r'^%s/(.*)$' % re.escape(sdir), path)
3598         if m1:
3599             # Check if the filename is in the ignore list.
3600             m2 = re.search(r'(\s|^)%s(\s|$)' % re.escape(m1.group(1)), ignore_files)
3601             if m2:
3602                 logging.info("Skipping path: %s", path)
3603                 return True
3604             else:
3605                 logging.info("No match for: %s", m1.group(1))
3606         else:
3607             logging.info("No match for: %s", path)
3608     return False
3609
3610
3611 def ReadSourceDocumentation(source_dir, suffix_list, source_dirs, ignore_files):
3612     """Read the documentation embedded in comment blocks in the source code.
3613
3614     It recursively descends the source directory looking for source files and
3615     scans them looking for specially-formatted comment blocks.
3616
3617     Args:
3618         source_dir (str): the directory to scan.
3619         suffix_list (list): extensions to check
3620     """
3621     if IgnorePath(source_dir, source_dirs, ignore_files):
3622         return
3623
3624     logging.info("Scanning source directory: %s", source_dir)
3625
3626     # This array holds any subdirectories found.
3627     subdirs = []
3628
3629     for ifile in sorted(os.listdir(source_dir)):
3630         logging.debug("... : %s", ifile)
3631         if ifile.startswith('.'):
3632             continue
3633         fname = os.path.join(source_dir, ifile)
3634         if os.path.isdir(fname):
3635             subdirs.append(fname)
3636         else:
3637             for suffix in suffix_list:
3638                 if ifile.endswith(suffix):
3639                     if not IgnorePath(fname, source_dirs, ignore_files):
3640                         ScanSourceFile(fname, ignore_files)
3641                         break
3642
3643     # Now recursively scan the subdirectories.
3644     for sdir in subdirs:
3645         ReadSourceDocumentation(sdir, suffix_list, source_dirs, ignore_files)
3646
3647
3648 def ScanSourceFile(ifile, ignore_files):
3649     """Scans one source file looking for specially-formatted comment blocks.
3650
3651     Later MergeSourceDocumentation() is copying over the doc blobs that are not
3652     suppressed/ignored.
3653
3654     Args:
3655         file (str): the file to scan.
3656     """
3657     m = re.search(r'^.*[\/\\]([^\/\\]*)$', ifile)
3658     if m:
3659         basename = m.group(1)
3660     else:
3661         common.LogWarning(ifile, 1, "Can't find basename for this filename.")
3662         basename = ifile
3663
3664     # Check if the basename is in the list of files to ignore.
3665     if re.search(r'(\s|^)%s(\s|$)' % re.escape(basename), ignore_files):
3666         logging.info("Skipping source file: %s", ifile)
3667         return
3668
3669     logging.info("Scanning source file: %s", ifile)
3670
3671     SRCFILE = common.open_text(ifile)
3672     in_comment_block = False
3673     symbol = None
3674     in_part = ''
3675     description = ''
3676     return_desc = ''
3677     since_desc = stability_desc = deprecated_desc = ''
3678     params = OrderedDict()
3679     param_name = None
3680     line_number = 0
3681     for line in SRCFILE:
3682         line_number += 1
3683         # Look for the start of a comment block.
3684         if not in_comment_block:
3685             if re.search(r'^\s*/\*.*\*/', line):
3686                 # one-line comment - not gtkdoc
3687                 pass
3688             elif re.search(r'^\s*/\*\*\s', line):
3689                 logging.info("Found comment block start")
3690
3691                 in_comment_block = True
3692
3693                 # Reset all the symbol data.
3694                 symbol = ''
3695                 in_part = ''
3696                 description = ''
3697                 return_desc = ''
3698                 since_desc = ''
3699                 deprecated_desc = ''
3700                 stability_desc = ''
3701                 params = OrderedDict()
3702                 param_name = None
3703
3704             continue
3705
3706         # We're in a comment block. Check if we've found the end of it.
3707         if re.search(r'^\s*\*+/', line):
3708             if not symbol:
3709                 # maybe its not even meant to be a gtk-doc comment?
3710                 common.LogWarning(ifile, line_number, "Symbol name not found at the start of the comment block.")
3711             else:
3712                 # Add the return value description onto the end of the params.
3713                 if return_desc:
3714                     # TODO(ensonic): check for duplicated Return docs
3715                     # common.LogWarning(file, line_number, "Multiple Returns for %s." % symbol)
3716                     params['Returns'] = return_desc
3717
3718                 # Convert special characters
3719                 description = ConvertSGMLChars(symbol, description)
3720                 for (param_name, param_desc) in iteritems(params):
3721                     params[param_name] = ConvertSGMLChars(symbol, param_desc)
3722
3723                 # Handle Section docs
3724                 m = re.search(r'SECTION:\s*(.*)', symbol)
3725                 m2 = re.search(r'PROGRAM:\s*(.*)', symbol)
3726                 if m:
3727                     real_symbol = m.group(1)
3728                     long_descr = real_symbol + ":Long_Description"
3729
3730                     if long_descr not in KnownSymbols or KnownSymbols[long_descr] != 1:
3731                         common.LogWarning(
3732                             ifile, line_number, "Section %s is not defined in the %s-sections.txt file." % (real_symbol, MODULE))
3733
3734                     logging.info("SECTION DOCS found in source for : '%s'", real_symbol)
3735                     for param_name, param_desc in iteritems(params):
3736                         logging.info("   '" + param_name + "'")
3737                         param_name = param_name.lower()
3738                         key = None
3739                         if param_name == "short_description":
3740                             key = real_symbol + ":Short_Description"
3741                         elif param_name == "see_also":
3742                             key = real_symbol + ":See_Also"
3743                         elif param_name == "title":
3744                             key = real_symbol + ":Title"
3745                         elif param_name == "stability":
3746                             key = real_symbol + ":Stability_Level"
3747                         elif param_name == "section_id":
3748                             key = real_symbol + ":Section_Id"
3749                         elif param_name == "include":
3750                             key = real_symbol + ":Include"
3751                         elif param_name == "image":
3752                             key = real_symbol + ":Image"
3753
3754                         if key:
3755                             SourceSymbolDocs[key] = param_desc
3756                             SourceSymbolSourceFile[key] = ifile
3757                             SourceSymbolSourceLine[key] = line_number
3758
3759                     SourceSymbolDocs[long_descr] = description
3760                     SourceSymbolSourceFile[long_descr] = ifile
3761                     SourceSymbolSourceLine[long_descr] = line_number
3762                 elif m2:
3763                     real_symbol = m2.group(1)
3764                     key = None
3765                     section_id = None
3766
3767                     logging.info("PROGRAM DOCS found in source for '%s'", real_symbol)
3768                     for param_name, param_desc in iteritems(params):
3769                         logging.info("PROGRAM key %s: '%s'", real_symbol, param_name)
3770                         param_name = param_name.lower()
3771                         key = None
3772                         if param_name == "short_description":
3773                             key = real_symbol + ":Short_Description"
3774                         elif param_name == "see_also":
3775                             key = real_symbol + ":See_Also"
3776                         elif param_name == "section_id":
3777                             key = real_symbol + ":Section_Id"
3778                         elif param_name == "synopsis":
3779                             key = real_symbol + ":Synopsis"
3780                         elif param_name == "returns":
3781                             key = real_symbol + ":Returns"
3782                         elif re.search(r'^(-.*)', param_name):
3783                             logging.info("PROGRAM opts: '%s': '%s'", param_name, param_desc)
3784                             key = real_symbol + ":Options"
3785                             opts = []
3786                             opts_str = SourceSymbolDocs.get(key)
3787                             if opts_str:
3788                                 opts = opts_str.split('\t')
3789                             opts.append(param_name)
3790                             opts.append(param_desc)
3791
3792                             logging.info("Setting options for symbol: %s: '%s'", real_symbol, '\t'.join(opts))
3793                             SourceSymbolDocs[key] = '\t'.join(opts)
3794                             continue
3795
3796                         if key:
3797                             logging.info("PROGRAM value %s: '%s'", real_symbol, param_desc.rstrip())
3798                             SourceSymbolDocs[key] = param_desc.rstrip()
3799                             SourceSymbolSourceFile[key] = ifile
3800                             SourceSymbolSourceLine[key] = line_number
3801
3802                     long_descr = real_symbol + ":Long_Description"
3803                     SourceSymbolDocs[long_descr] = description
3804                     SourceSymbolSourceFile[long_descr] = ifile
3805                     SourceSymbolSourceLine[long_descr] = line_number
3806
3807                     section_id = SourceSymbolDocs.get(real_symbol + ":Section_Id")
3808                     if section_id and section_id.strip() != '':
3809                         # Remove trailing blanks and use as is
3810                         section_id = section_id.rstrip()
3811                     else:
3812                         section_id = common.CreateValidSGMLID('%s-%s' % (MODULE, real_symbol))
3813                     OutputProgramDBFile(real_symbol, section_id)
3814
3815                 else:
3816                     logging.info("SYMBOL DOCS found in source for : '%s'", symbol)
3817                     SourceSymbolDocs[symbol] = description
3818                     SourceSymbolParams[symbol] = params
3819                     SourceSymbolSourceFile[symbol] = ifile
3820                     SourceSymbolSourceLine[symbol] = line_number
3821
3822                 if since_desc:
3823                     arr = since_desc.splitlines()
3824                     since_desc = arr[0].strip()
3825                     extra_lines = arr[1:]
3826                     logging.info("Since(%s) : [%s]", symbol, since_desc)
3827                     Since[symbol] = ConvertSGMLChars(symbol, since_desc)
3828                     if len(extra_lines) > 1:
3829                         common.LogWarning(ifile, line_number, "multi-line since docs found")
3830
3831                 if stability_desc:
3832                     stability_desc = ParseStabilityLevel(
3833                         stability_desc, ifile, line_number, "Stability level for %s" % symbol)
3834                     StabilityLevel[symbol] = ConvertSGMLChars(symbol, stability_desc)
3835
3836                 if deprecated_desc:
3837                     if symbol not in Deprecated:
3838                         # don't warn for signals and properties
3839                         # if ($symbol !~ m/::?(.*)/)
3840                         if symbol in DeclarationTypes:
3841                             common.LogWarning(ifile, line_number,
3842                                               "%s is deprecated in the inline comments, but no deprecation guards were found around the declaration. (See the --deprecated-guards option for gtkdoc-scan.)" % symbol)
3843
3844                     Deprecated[symbol] = ConvertSGMLChars(symbol, deprecated_desc)
3845
3846             in_comment_block = False
3847             continue
3848
3849         # Get rid of ' * ' at start of every line in the comment block.
3850         line = re.sub(r'^\s*\*\s?', '', line)
3851         # But make sure we don't get rid of the newline at the end.
3852         if not line.endswith('\n'):
3853             line = line + "\n"
3854
3855         logging.info("scanning :%s", line.strip())
3856
3857         # If we haven't found the symbol name yet, look for it.
3858         if not symbol:
3859             m1 = re.search(r'^\s*(SECTION:\s*\S+)', line)
3860             m2 = re.search(r'^\s*(PROGRAM:\s*\S+)', line)
3861             m3 = re.search(r'^\s*([\w:-]*\w)\s*:?\s*(\(.+?\)\s*)*$', line)
3862             if m1:
3863                 symbol = m1.group(1)
3864                 logging.info("SECTION DOCS found in source for : '%s'", symbol)
3865             elif m2:
3866                 symbol = m2.group(1)
3867                 logging.info("PROGRAM DOCS found in source for : '%s'", symbol)
3868             elif m3:
3869                 symbol = m3.group(1)
3870                 annotation = m3.group(2)
3871                 logging.info("SYMBOL DOCS found in source for : '%s'", symbol)
3872                 if annotation:
3873                     annotation = annotation.strip()
3874                     if annotation != '':
3875                         SymbolAnnotations[symbol] = annotation
3876                         logging.info("remaining text for %s: '%s'", symbol, annotation)
3877
3878             continue
3879
3880         if in_part == "description":
3881             # Get rid of 'Description:'
3882             line = re.sub(r'^\s*Description:', '', line)
3883
3884         m1 = re.search(r'^\s*(returns|return\s+value):', line, flags=re.I)
3885         m2 = re.search(r'^\s*since:', line, flags=re.I)
3886         m3 = re.search(r'^\s*deprecated:', line, flags=re.I)
3887         m4 = re.search(r'^\s*stability:', line, flags=re.I)
3888
3889         if m1:
3890             # we're in param section and have not seen the blank line
3891             if in_part != '':
3892                 return_desc = line[m1.end():]
3893                 in_part = "return"
3894                 continue
3895
3896         if m2:
3897             # we're in param section and have not seen the blank line
3898             if in_part != "param":
3899                 since_desc = line[m2.end():]
3900                 in_part = "since"
3901                 continue
3902
3903         elif m3:
3904             # we're in param section and have not seen the blank line
3905             if in_part != "param":
3906                 deprecated_desc = line[m3.end():]
3907                 in_part = "deprecated"
3908                 continue
3909
3910         elif m4:
3911             stability_desc = line[m4.end():]
3912             in_part = "stability"
3913             continue
3914
3915         if in_part == "description":
3916             description += line
3917             continue
3918         elif in_part == "return":
3919             return_desc += line
3920             continue
3921         elif in_part == "since":
3922             since_desc += line
3923             continue
3924         elif in_part == "stability":
3925             stability_desc += line
3926             continue
3927         elif in_part == "deprecated":
3928             deprecated_desc += line
3929             continue
3930
3931         # We must be in the parameters. Check for the empty line below them.
3932         if re.search(r'^\s*$', line):
3933             in_part = "description"
3934             continue
3935
3936         # Look for a parameter name.
3937         m = re.search(r'^\s*@(.+?)\s*:\s*', line)
3938         if m:
3939             param_name = m.group(1)
3940             param_desc = line[m.end():]
3941
3942             logging.info("Found parameter: %s", param_name)
3943             # Allow varargs variations
3944             if re.search(r'^\.\.\.$', param_name):
3945                 param_name = "..."
3946
3947             logging.info("Found param for symbol %s : '%s'= '%s'", symbol, param_name, line)
3948
3949             params[param_name] = param_desc
3950             in_part = "param"
3951             continue
3952         elif in_part == '':
3953             logging.info("continuation for %s annotation '%s'", symbol, line)
3954             annotation = re.sub(r'^\s+|\s+$', '', line)
3955             if symbol in SymbolAnnotations:
3956                 SymbolAnnotations[symbol] += annotation
3957             else:
3958                 SymbolAnnotations[symbol] = annotation
3959             continue
3960
3961         # We must be in the middle of a parameter description, so add it on
3962         # to the last element in @params.
3963         if not param_name:
3964             common.LogWarning(file, line_number, "Parsing comment block file : parameter expected, but got '%s'" % line)
3965         else:
3966             params[param_name] += line
3967
3968     SRCFILE.close()
3969
3970
3971 def OutputMissingDocumentation():
3972     """Outputs report of documentation coverage to a file.
3973
3974     Returns:
3975         bool: True if the report was updated
3976     """
3977     old_undocumented_file = os.path.join(ROOT_DIR, MODULE + "-undocumented.txt")
3978     new_undocumented_file = os.path.join(ROOT_DIR, MODULE + "-undocumented.new")
3979
3980     n_documented = 0
3981     n_incomplete = 0
3982     total = 0
3983     symbol = None
3984     percent = None
3985     buffer = ''
3986     buffer_deprecated = ''
3987     buffer_descriptions = ''
3988
3989     UNDOCUMENTED = common.open_text(new_undocumented_file, 'w')
3990
3991     for symbol in sorted(iterkeys(AllSymbols)):
3992         # FIXME: should we print common.LogWarnings for undocumented stuff?
3993         # DEBUG
3994         # location = "defined at " + GetSymbolSourceFile(symbol) + ":" + GetSymbolSourceLine(symbol) + "\n"
3995         # DEBUG
3996         m = re.search(
3997             r':(Title|Long_Description|Short_Description|See_Also|Stability_Level|Include|Section_Id|Image)', symbol)
3998         m2 = re.search(r':(Long_Description|Short_Description)', symbol)
3999         if not m:
4000             total += 1
4001             if symbol in AllDocumentedSymbols:
4002                 n_documented += 1
4003                 if symbol in AllIncompleteSymbols:
4004                     n_incomplete += 1
4005                     buffer += symbol + " (" + AllIncompleteSymbols[symbol] + ")\n"
4006                     #$buffer += "\t0: ".$location
4007
4008             elif symbol in Deprecated:
4009                 if symbol in AllIncompleteSymbols:
4010                     n_incomplete += 1
4011                     buffer_deprecated += symbol + " (" + AllIncompleteSymbols[symbol] + ")\n"
4012                     #$buffer += "\t1a: ".$location
4013                 else:
4014                     buffer_deprecated += symbol + "\n"
4015                     #$buffer += "\t1b: ".$location
4016
4017             else:
4018                 if symbol in AllIncompleteSymbols:
4019                     n_incomplete += 1
4020                     buffer += symbol + " (" + AllIncompleteSymbols[symbol] + ")\n"
4021                     #$buffer += "\t2a: ".$location
4022                 else:
4023                     buffer += symbol + "\n"
4024                     #$buffer += "\t2b: ".$location
4025
4026         elif m2:
4027             total += 1
4028             if (symbol in SymbolDocs and len(SymbolDocs[symbol]) > 0)\
4029                or (symbol in AllDocumentedSymbols and AllDocumentedSymbols[symbol] > 0):
4030                 n_documented += 1
4031             else:
4032                 buffer_descriptions += symbol + "\n"
4033
4034     if total == 0:
4035         percent = 100
4036     else:
4037         percent = (n_documented / total) * 100.0
4038
4039     UNDOCUMENTED.write("%.0f%% symbol docs coverage.\n" % percent)
4040     UNDOCUMENTED.write("%s symbols documented.\n" % n_documented)
4041     UNDOCUMENTED.write("%s symbols incomplete.\n" % n_incomplete)
4042     UNDOCUMENTED.write("%d not documented.\n" % (total - n_documented))
4043
4044     if buffer_deprecated != '':
4045         buffer += "\n" + buffer_deprecated
4046
4047     if buffer_descriptions != '':
4048         buffer += "\n" + buffer_descriptions
4049
4050     if buffer != '':
4051         UNDOCUMENTED.write("\n\n" + buffer)
4052
4053     UNDOCUMENTED.close()
4054
4055     return common.UpdateFileIfChanged(old_undocumented_file, new_undocumented_file, 0)
4056
4057
4058 def OutputUndeclaredSymbols():
4059     """Reports undeclared symbols.
4060
4061     Outputs symbols that are listed in the section file, but have no declaration
4062     in the sources.
4063
4064     Returns:
4065         bool: True if the report was updated
4066     """
4067     old_undeclared_file = os.path.join(ROOT_DIR, MODULE + "-undeclared.txt")
4068     new_undeclared_file = os.path.join(ROOT_DIR, MODULE + "-undeclared.new")
4069
4070     UNDECLARED = common.open_text(new_undeclared_file, 'w')
4071
4072     if UndeclaredSymbols:
4073         UNDECLARED.write("\n".join(sorted(iterkeys(UndeclaredSymbols))))
4074         UNDECLARED.write("\n")
4075         print("See %s-undeclared.txt for the list of undeclared symbols." % MODULE)
4076
4077     UNDECLARED.close()
4078
4079     return common.UpdateFileIfChanged(old_undeclared_file, new_undeclared_file, 0)
4080
4081
4082 def OutputUnusedSymbols():
4083     """Reports unused documentation.
4084
4085     Outputs symbols that are documented in comments, but not declared in the
4086     sources.
4087
4088     Returns:
4089         bool: True if the report was updated
4090     """
4091     num_unused = 0
4092     old_unused_file = os.path.join(ROOT_DIR, MODULE + "-unused.txt")
4093     new_unused_file = os.path.join(ROOT_DIR, MODULE + "-unused.new")
4094
4095     UNUSED = common.open_text(new_unused_file, 'w')
4096
4097     for symbol in sorted(iterkeys(Declarations)):
4098         if not symbol in DeclarationOutput:
4099             UNUSED.write("%s\n" % symbol)
4100             num_unused += 1
4101
4102     for symbol in sorted(iterkeys(AllUnusedSymbols)):
4103         UNUSED.write(symbol + "(" + AllUnusedSymbols[symbol] + ")\n")
4104         num_unused += 1
4105
4106     UNUSED.close()
4107     if num_unused != 0:
4108         common.LogWarning(
4109             old_unused_file, 1, "%d unused declarations. They should be added to %s-sections.txt in the appropriate place." % (num_unused, MODULE))
4110
4111     return common.UpdateFileIfChanged(old_unused_file, new_unused_file, 0)
4112
4113
4114 def OutputAllSymbols():
4115     """Outputs list of all symbols to a file."""
4116     SYMBOLS = common.open_text(os.path.join(ROOT_DIR, MODULE + "-symbols.txt"), 'w')
4117
4118     for symbol in sorted(iterkeys(AllSymbols)):
4119         SYMBOLS.write(symbol + "\n")
4120     SYMBOLS.close()
4121
4122
4123 def OutputSymbolsWithoutSince():
4124     """Outputs list of all symbols without a since tag to a file."""
4125     SYMBOLS = common.open_text(os.path.join(ROOT_DIR, MODULE + "-nosince.txt"), 'w')
4126
4127     for symbol in sorted(iterkeys(SourceSymbolDocs)):
4128         if symbol in Since:
4129             SYMBOLS.write(symbol + "\n")
4130     SYMBOLS.close()
4131
4132
4133 def CheckParamsDocumented(symbol, params):
4134     stype = DeclarationTypes.get(symbol)
4135
4136     item = "Parameter"
4137     if stype:
4138         if stype == 'STRUCT':
4139             item = "Field"
4140         elif stype == 'ENUM':
4141             item = "Value"
4142         elif stype == 'UNION':
4143             item = "Field"
4144     else:
4145         stype = "SIGNAL"
4146     logging.info("Check param docs for %s, params: %s entries, type=%s", symbol, len(params), stype)
4147
4148     if len(params) > 0:
4149         logging.info("params: %s", str(params))
4150         for (param_name, param_desc) in iteritems(params):
4151             # Output a warning if the parameter is empty and remember for stats.
4152             if param_name != "void" and not re.search(r'\S', param_desc):
4153                 if symbol in AllIncompleteSymbols:
4154                     AllIncompleteSymbols[symbol] += ", " + param_name
4155                 else:
4156                     AllIncompleteSymbols[symbol] = param_name
4157
4158                 common.LogWarning(GetSymbolSourceFile(symbol), GetSymbolSourceLine(symbol),
4159                                   "%s description for %s::%s is missing in source code comment block." % (item, symbol, param_name))
4160
4161     elif len(params) == 0:
4162         AllIncompleteSymbols[symbol] = "<items>"
4163         common.LogWarning(GetSymbolSourceFile(symbol), GetSymbolSourceLine(symbol),
4164                           "%s descriptions for %s are missing in source code comment block." % (item, symbol))
4165
4166
4167 def MergeSourceDocumentation():
4168     """Merges documentation read from a source file.
4169
4170     Parameter descriptions override any in the template files.
4171     Function descriptions are placed before any description from
4172     the template files.
4173     """
4174
4175     # add whats found in the source
4176     symbols = set(iterkeys(SourceSymbolDocs))
4177
4178     # and add known symbols from -sections.txt
4179     for symbol in iterkeys(KnownSymbols):
4180         if KnownSymbols[symbol] == 1:
4181             symbols.add(symbol)
4182
4183     logging.info("num source entries: %d", len(symbols))
4184
4185     for symbol in symbols:
4186         AllSymbols[symbol] = 1
4187
4188         if symbol in SourceSymbolDocs:
4189             logging.info("merging [%s] from source", symbol)
4190
4191             # remove leading and training whitespaces
4192             src_docs = SourceSymbolDocs[symbol].strip()
4193             if src_docs != '':
4194                 AllDocumentedSymbols[symbol] = 1
4195
4196             SymbolDocs[symbol] = src_docs
4197
4198             # merge parameters
4199             if symbol in SourceSymbolParams:
4200                 param_docs = SourceSymbolParams[symbol]
4201                 SymbolParams[symbol] = param_docs
4202                 # if this symbol is documented, check if docs are complete
4203                 # remove all xml-tags and whitespaces
4204                 check_docs = re.sub(r'\s', '', re.sub(r'<.*?>', '', src_docs))
4205                 if check_docs != '' and param_docs:
4206                     CheckParamsDocumented(symbol, param_docs)
4207         else:
4208             logging.info("[%s] undocumented", symbol)
4209
4210     logging.info("num doc entries: %d", len(SymbolDocs))
4211
4212
4213 def IsEmptyDoc(doc):
4214     """Check if a doc-string is empty.
4215
4216     It is also regarded as empty if it only consist of whitespace or e.g. FIXME.
4217
4218     Args:
4219         doc (str): the doc-string
4220
4221     Returns:
4222         bool: True if empty
4223     """
4224     if re.search(r'^\s*$', doc):
4225         return True
4226     if re.search(r'^\s*<para>\s*(FIXME)?\s*<\/para>\s*$', doc):
4227         return True
4228     return False
4229
4230
4231 def ConvertMarkDown(symbol, text):
4232     md_to_db.Init()
4233     return md_to_db.MarkDownParse(text, symbol)
4234
4235
4236 def ReadDeclarationsFile(ifile, override):
4237     """Reads in a file containing the function/macro/enum etc. declarations.
4238
4239     Note that in some cases there are several declarations with
4240     the same name, e.g. for conditional macros. In this case we
4241     set a flag in the DeclarationConditional hash so the
4242     declaration is not shown in the docs.
4243
4244     If a macro and a function have the same name, e.g. for
4245     g_object_ref, the function declaration takes precedence.
4246
4247     Some opaque structs are just declared with 'typedef struct
4248     _name name;' in which case the declaration may be empty.
4249     The structure may have been found later in the header, so
4250     that overrides the empty declaration.
4251
4252     Args:
4253         file (str): the declarations file to read
4254         override (bool): if declarations in this file should override
4255                          any current declaration.
4256     """
4257     if override == 0:
4258         Declarations.clear()
4259         DeclarationTypes.clear()
4260         DeclarationConditional.clear()
4261         DeclarationOutput.clear()
4262
4263     INPUT = common.open_text(ifile)
4264     declaration_type = ''
4265     declaration_name = None
4266     declaration = None
4267     is_deprecated = 0
4268     line_number = 0
4269     for line in INPUT:
4270         line_number += 1
4271         # logging.debug("%s:%d: %s", ifile, line_number, line)
4272         if not declaration_type:
4273             m1 = re.search(r'^<([^>]+)>', line)
4274             if m1:
4275                 declaration_type = m1.group(1)
4276                 declaration_name = ''
4277                 logging.info("Found declaration: %s", declaration_type)
4278                 declaration = ''
4279         else:
4280             m2 = re.search(r'^<NAME>(.*)</NAME>', line)
4281             m3 = re.search(r'^<DEPRECATED/>', line)
4282             m4 = re.search(r'^</%s>' % declaration_type, line)
4283             if m2:
4284                 declaration_name = m2.group(1)
4285             elif m3:
4286                 is_deprecated = True
4287             elif m4:
4288                 logging.info("Found end of declaration: %s, %s", declaration_type, declaration_name)
4289                 # Check that the declaration has a name
4290                 if declaration_name == '':
4291                     common.LogWarning(ifile, line_number, declaration_type + " has no name.\n")
4292
4293                 # If the declaration is an empty typedef struct _XXX XXX
4294                 # set the flag to indicate the struct has a typedef.
4295                 if (declaration_type == 'STRUCT' or declaration_type == 'UNION') \
4296                         and re.search(r'^\s*$', declaration):
4297                     logging.info("Struct has typedef: %s", declaration_name)
4298                     StructHasTypedef[declaration_name] = 1
4299
4300                 # Check if the symbol is already defined.
4301                 if declaration_name in Declarations and override == 0:
4302                     # Function declarations take precedence.
4303                     if DeclarationTypes[declaration_name] == 'FUNCTION':
4304                         # Ignore it.
4305                         pass
4306                     elif declaration_type == 'FUNCTION':
4307                         if is_deprecated:
4308                             Deprecated[declaration_name] = ''
4309
4310                         Declarations[declaration_name] = declaration
4311                         DeclarationTypes[declaration_name] = declaration_type
4312                     elif DeclarationTypes[declaration_name] == declaration_type:
4313                         # If the existing declaration is empty, or is just a
4314                         # forward declaration of a struct, override it.
4315                         if declaration_type == 'STRUCT' or declaration_type == 'UNION':
4316                             if re.search(r'^\s*((struct|union)\s+\w+\s*;)?\s*$', Declarations[declaration_name]):
4317                                 if is_deprecated:
4318                                     Deprecated[declaration_name] = ''
4319                                 Declarations[declaration_name] = declaration
4320                             elif re.search(r'^\s*((struct|union)\s+\w+\s*;)?\s*$', declaration):
4321                                 # Ignore an empty or forward declaration.
4322                                 pass
4323                             else:
4324                                 common.LogWarning(
4325                                     ifile, line_number, "Structure %s has multiple definitions." % declaration_name)
4326
4327                         else:
4328                             # set flag in %DeclarationConditional hash for
4329                             # multiply defined macros/typedefs.
4330                             DeclarationConditional[declaration_name] = 1
4331
4332                     else:
4333                         common.LogWarning(ifile, line_number, declaration_name + " has multiple definitions.")
4334
4335                 else:
4336                     if is_deprecated:
4337                         Deprecated[declaration_name] = ''
4338
4339                     Declarations[declaration_name] = declaration
4340                     DeclarationTypes[declaration_name] = declaration_type
4341                     logging.debug("added declaration: %s, %s, [%s]", declaration_type, declaration_name, declaration)
4342
4343                 declaration_type = ''
4344                 is_deprecated = False
4345             else:
4346                 declaration += line
4347     INPUT.close()
4348
4349
4350 def ReadSignalsFile(ifile):
4351     """Reads information about object signals.
4352
4353     It creates the arrays @SignalNames and @SignalPrototypes containing details
4354     about the signals. The first line of the SignalPrototype is the return type
4355     of the signal handler. The remaining lines are the parameters passed to it.
4356     The last parameter, "gpointer user_data" is always the same so is not included.
4357
4358     Args:
4359        ifile (str): the file containing the signal handler prototype information.
4360     """
4361     in_signal = 0
4362     signal_object = None
4363     signal_name = None
4364     signal_returns = None
4365     signal_flags = None
4366     signal_prototype = None
4367
4368     # Reset the signal info.
4369     SignalObjects[:] = []
4370     SignalNames[:] = []
4371     SignalReturns[:] = []
4372     SignalFlags[:] = []
4373     SignalPrototypes[:] = []
4374
4375     if not os.path.isfile(ifile):
4376         return
4377
4378     INPUT = common.open_text(ifile)
4379     line_number = 0
4380     for line in INPUT:
4381         line_number += 1
4382         if not in_signal:
4383             if re.search(r'^<SIGNAL>', line):
4384                 in_signal = 1
4385                 signal_object = ''
4386                 signal_name = ''
4387                 signal_returns = ''
4388                 signal_prototype = ''
4389
4390         else:
4391             m = re.search(r'^<NAME>(.*)<\/NAME>', line)
4392             m2 = re.search(r'^<RETURNS>(.*)<\/RETURNS>', line)
4393             m3 = re.search(r'^<FLAGS>(.*)<\/FLAGS>', line)
4394             if m:
4395                 signal_name = m.group(1)
4396                 m1_2 = re.search(r'^(.*)::(.*)$', signal_name)
4397                 if m1_2:
4398                     signal_object = m1_2.group(1)
4399                     signal_name = m1_2.group(2).replace('_', '-')
4400                     logging.info("Found signal: %s", signal_name)
4401                 else:
4402                     common.LogWarning(ifile, line_number, "Invalid signal name: %s." % signal_name)
4403
4404             elif m2:
4405                 signal_returns = m2.group(1)
4406             elif m3:
4407                 signal_flags = m3.group(1)
4408             elif re.search(r'^</SIGNAL>', line):
4409                 logging.info("Found end of signal: %s::%s\nReturns: %s\n%s",
4410                              signal_object, signal_name, signal_returns, signal_prototype)
4411                 SignalObjects.append(signal_object)
4412                 SignalNames.append(signal_name)
4413                 SignalReturns.append(signal_returns)
4414                 SignalFlags.append(signal_flags)
4415                 SignalPrototypes.append(signal_prototype)
4416                 in_signal = False
4417             else:
4418                 signal_prototype += line
4419     INPUT.close()
4420
4421
4422 def ReadObjectHierarchy(ifile):
4423     """Reads the $MODULE-hierarchy.txt file.
4424
4425     This contains all the GObject subclasses described in this module (and their
4426     ancestors).
4427     It places them in the Objects array, and places their level
4428     in the object hierarchy in the ObjectLevels array, at the
4429     same index. GObject, the root object, has a level of 1.
4430
4431     This also generates tree_index.sgml as it goes along.
4432
4433     Args:
4434         ifile (str): the input filename.
4435     """
4436
4437     Objects[:] = []
4438     ObjectLevels[:] = []
4439
4440     if not os.path.isfile(ifile):
4441         logging.debug('no *-hierarchy.tx')
4442         return
4443
4444     INPUT = common.open_text(ifile)
4445
4446     # Only emit objects if they are supposed to be documented, or if
4447     # they have documented children. To implement this, we maintain a
4448     # stack of pending objects which will be emitted if a documented
4449     # child turns up.
4450     pending_objects = []
4451     pending_levels = []
4452     root = None
4453     tree = []
4454     for line in INPUT:
4455         m1 = re.search(r'\S+', line)
4456         if not m1:
4457             continue
4458
4459         gobject = m1.group(0)
4460         level = len(line[:m1.start()]) // 2 + 1
4461
4462         if level == 1:
4463             root = gobject
4464
4465         while pending_levels and pending_levels[-1] >= level:
4466             pending_objects.pop()
4467             pending_levels.pop()
4468
4469         pending_objects.append(gobject)
4470         pending_levels.append(level)
4471
4472         if gobject in KnownSymbols:
4473             while len(pending_levels) > 0:
4474                 gobject = pending_objects.pop(0)
4475                 level = pending_levels.pop(0)
4476                 xref = MakeXRef(gobject)
4477
4478                 tree.append(' ' * (level * 4) + xref)
4479                 Objects.append(gobject)
4480                 ObjectLevels.append(level)
4481                 ObjectRoots[gobject] = root
4482         # else
4483         #    common.LogWarning(ifile, line_number, "unknown type %s" % object)
4484         #
4485
4486     INPUT.close()
4487
4488     # FIXME: use xml
4489     # my $old_tree_index = "$DB_OUTPUT_DIR/tree_index.$xml"
4490     old_tree_index = os.path.join(DB_OUTPUT_DIR, "tree_index.sgml")
4491     new_tree_index = os.path.join(DB_OUTPUT_DIR, "tree_index.new")
4492
4493     logging.debug('got %d entries for hierarchy', len(tree))
4494
4495     OUTPUT = common.open_text(new_tree_index, 'w')
4496     OUTPUT.write(MakeDocHeader("screen") + "\n<screen>\n" + AddTreeLineArt(tree) + "\n</screen>\n")
4497     OUTPUT.close()
4498
4499     common.UpdateFileIfChanged(old_tree_index, new_tree_index, 0)
4500
4501     OutputObjectList()
4502
4503
4504 def ReadInterfaces(ifile):
4505     """Reads the $MODULE.interfaces file.
4506
4507     Args:
4508         ifile (str): the input filename.
4509     """
4510
4511     Interfaces.clear()
4512
4513     if not os.path.isfile(ifile):
4514         return
4515
4516     INPUT = common.open_text(ifile)
4517
4518     for line in INPUT:
4519         line = line.strip()
4520         ifaces = line.split()
4521         gobject = ifaces.pop(0)
4522         if gobject in KnownSymbols and KnownSymbols[gobject] == 1:
4523             knownIfaces = []
4524
4525             # filter out private interfaces, but leave foreign interfaces
4526             for iface in ifaces:
4527                 if iface not in KnownSymbols or KnownSymbols[iface] == 1:
4528                     knownIfaces.append(iface)
4529
4530             Interfaces[gobject] = ' '.join(knownIfaces)
4531             logging.info("Interfaces for %s: %s", gobject, Interfaces[gobject])
4532         else:
4533             logging.info("skipping interfaces for unknown symbol: %s", gobject)
4534
4535     INPUT.close()
4536
4537
4538 def ReadPrerequisites(ifile):
4539     """This reads in the $MODULE.prerequisites file.
4540
4541     Args:
4542         ifile (str): the input filename.
4543     """
4544     Prerequisites.clear()
4545
4546     if not os.path.isfile(ifile):
4547         return
4548
4549     INPUT = common.open_text(ifile)
4550
4551     for line in INPUT:
4552         line = line.strip()
4553         prereqs = line.split()
4554         iface = prereqs.pop(0)
4555         if iface in KnownSymbols and KnownSymbols[iface] == 1:
4556             knownPrereqs = []
4557
4558             # filter out private prerequisites, but leave foreign prerequisites
4559             for prereq in prereqs:
4560                 if prereq not in KnownSymbols or KnownSymbols[prereq] == 1:
4561                     knownPrereqs.append(prereq)
4562
4563             Prerequisites[iface] = ' '.join(knownPrereqs)
4564
4565     INPUT.close()
4566
4567
4568 def ReadArgsFile(ifile):
4569     """Reads information about object properties
4570
4571     It creates the arrays ArgObjects, ArgNames, ArgTypes, ArgFlags, ArgNicks and
4572     ArgBlurbs containing info on the args.
4573
4574     Args:
4575         ifile (str): the input filename.
4576     """
4577     in_arg = False
4578     arg_object = None
4579     arg_name = None
4580     arg_type = None
4581     arg_flags = None
4582     arg_nick = None
4583     arg_blurb = None
4584     arg_default = None
4585     arg_range = None
4586
4587     # Reset the args info.
4588     ArgObjects[:] = []
4589     ArgNames[:] = []
4590     ArgTypes[:] = []
4591     ArgFlags[:] = []
4592     ArgNicks[:] = []
4593     ArgBlurbs[:] = []
4594     ArgDefaults[:] = []
4595     ArgRanges[:] = []
4596
4597     if not os.path.isfile(ifile):
4598         return
4599
4600     INPUT = common.open_text(ifile)
4601     line_number = 0
4602     for line in INPUT:
4603         line_number += 1
4604         if not in_arg:
4605             if line.startswith('<ARG>'):
4606                 in_arg = True
4607                 arg_object = ''
4608                 arg_name = ''
4609                 arg_type = ''
4610                 arg_flags = ''
4611                 arg_nick = ''
4612                 arg_blurb = ''
4613                 arg_default = ''
4614                 arg_range = ''
4615
4616         else:
4617             m1 = re.search(r'^<NAME>(.*)</NAME>', line)
4618             m2 = re.search(r'^<TYPE>(.*)</TYPE>', line)
4619             m3 = re.search(r'^<RANGE>(.*)</RANGE>', line)
4620             m4 = re.search(r'^<FLAGS>(.*)</FLAGS>', line)
4621             m5 = re.search(r'^<NICK>(.*)</NICK>', line)
4622             m6 = re.search(r'^<BLURB>(.*)</BLURB>', line)
4623             m7 = re.search(r'^<DEFAULT>(.*)</DEFAULT>', line)
4624             if m1:
4625                 arg_name = m1.group(1)
4626                 m1_1 = re.search(r'^(.*)::(.*)$', arg_name)
4627                 if m1_1:
4628                     arg_object = m1_1.group(1)
4629                     arg_name = m1_1.group(2).replace('_', '-')
4630                     logging.info("Found arg: %s", arg_name)
4631                 else:
4632                     common.LogWarning(ifile, line_number, "Invalid argument name: " + arg_name)
4633
4634             elif m2:
4635                 arg_type = m2.group(1)
4636             elif m3:
4637                 arg_range = m3.group(1)
4638             elif m4:
4639                 arg_flags = m4.group(1)
4640             elif m5:
4641                 arg_nick = m5.group(1)
4642             elif m6:
4643                 arg_blurb = m6.group(1)
4644                 if arg_blurb == "(null)":
4645                     arg_blurb = ''
4646                     common.LogWarning(
4647                         ifile, line_number, "Property %s:%s has no documentation." % (arg_object, arg_name))
4648
4649             elif m7:
4650                 arg_default = m7.group(1)
4651             elif re.search(r'^</ARG>', line):
4652                 logging.info("Found end of arg: %s::%s\n%s : %s", arg_object, arg_name, arg_type, arg_flags)
4653                 ArgObjects.append(arg_object)
4654                 ArgNames.append(arg_name)
4655                 ArgTypes.append(arg_type)
4656                 ArgRanges.append(arg_range)
4657                 ArgFlags.append(arg_flags)
4658                 ArgNicks.append(arg_nick)
4659                 ArgBlurbs.append(arg_blurb)
4660                 ArgDefaults.append(arg_default)
4661                 in_arg = False
4662
4663     INPUT.close()
4664
4665
4666 def AddTreeLineArt(tree):
4667     """Generate a line art tree.
4668
4669     Add unicode lineart to a pre-indented string array and returns
4670     it as as multiline string.
4671
4672     Args:
4673         tree (list): of indented strings.
4674
4675     Returns:
4676         str: multiline string with tree line art
4677     """
4678     # iterate bottom up over the tree
4679     for i in range(len(tree) - 1, -1, -1):
4680         # count leading spaces
4681         m = re.search(r'^([^<A-Za-z]*)', tree[i])
4682         indent = len(m.group(1))
4683         # replace with ╰───, if place of ╰ is not space insert ├
4684         if indent > 4:
4685             if tree[i][indent - 4] == " ":
4686                 tree[i] = tree[i][:indent - 4] + "--- " + tree[i][indent:]
4687             else:
4688                 tree[i] = tree[i][:indent - 4] + "+-- " + tree[i][indent:]
4689
4690             # go lines up while space and insert |
4691             j = i - 1
4692             while j >= 0 and tree[j][indent - 4] == ' ':
4693                 tree[j] = tree[j][:indent - 4] + '|' + tree[j][indent - 3:]
4694                 j -= 1
4695
4696     res = "\n".join(tree)
4697     # unicode chars for: ╰──
4698     res = re.sub(r'---', '<phrase role=\"lineart\">&#9584;&#9472;&#9472;</phrase>', res)
4699     # unicde chars for: ├──
4700     res = re.sub(r'\+--', '<phrase role=\"lineart\">&#9500;&#9472;&#9472;</phrase>', res)
4701     # unicode char for: │
4702     res = re.sub(r'\|', '<phrase role=\"lineart\">&#9474;</phrase>', res)
4703
4704     return res
4705
4706
4707 def CheckIsObject(name):
4708     """Check if symbols is an object.
4709
4710     It uses the global Objects array. Note that the Objects array only contains
4711     classes in the current module and their ancestors - not all GObject classes.
4712
4713     Args:
4714         name (str): the object name to check.
4715
4716     Returns:
4717         bool: True if the given name is a GObject or a subclass.
4718     """
4719     root = ObjectRoots.get(name)
4720     # Let GBoxed pass as an object here to get -struct appended to the id
4721     # and prevent conflicts with sections.
4722     return root and root != 'GEnum' and root != 'GFlags'
4723
4724
4725 def GetSymbolSourceFile(symbol):
4726     """Get the filename where the symbol docs where taken from."""
4727     return SourceSymbolSourceFile.get(symbol, '')
4728
4729
4730 def GetSymbolSourceLine(symbol):
4731     """Get the file line where the symbol docs where taken from."""
4732     return SourceSymbolSourceLine.get(symbol, 0)