bin/SConsDoc.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2010 The SCons Foundation
   4 #
   5 # Permission is hereby granted, free of charge, to any person obtaining
   6 # a copy of this software and associated documentation files (the
   7 # "Software"), to deal in the Software without restriction, including
   8 # without limitation the rights to use, copy, modify, merge, publish,
   9 # distribute, sublicense, and/or sell copies of the Software, and to
  10 # permit persons to whom the Software is furnished to do so, subject to
  11 # the following conditions:
  12 #
  13 # The above copyright notice and this permission notice shall be included
  14 # in all copies or substantial portions of the Software.
  15 #
  16 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
  17 # KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
  18 # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  19 # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  20 # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  21 # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  22 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  23 #
  24 #
  25 # Module for handling SCons documentation processing.
  26 #
  27
  28 __doc__ = r"""
  29 This module parses home-brew XML files that document various things
  30 in SCons.  Right now, it handles Builders, functions, construction
  31 variables, and Tools, but we expect it to get extended in the future.
  32
  33 In general, you can use any DocBook tag in the input, and this module
  34 just adds processing various home-brew tags to try to make life a
  35 little easier.
  36
  37 Builder example:
  38
  39     <builder name="BUILDER">
  40     <summary>
  41     <para>This is the summary description of an SCons Builder.
  42     It will get placed in the man page,
  43     and in the appropriate User's Guide appendix.
  44     The name of this builder may be interpolated
  45     anywhere in the document by specifying the
  46     &b-BUILDER; element. A link to this definition may be
  47     interpolated by specifying the &b-link-BUILDER; element.
  48     </para>
  49
  50     Unlike normal XML, blank lines are significant in these
  51     descriptions and serve to separate paragraphs.
  52     They'll get replaced in DocBook output with appropriate tags
  53     to indicate a new paragraph.
  54
  55     <example>
  56     print("this is example code, it will be offset and indented")
  57     </example>
  58     </summary>
  59     </builder>
  60
  61 Function example:
  62
  63     <scons_function name="FUNCTION">
  64     <arguments signature="SIGTYPE">
  65     (arg1, arg2, key=value)
  66     </arguments>
  67     <summary>
  68     <para>This is the summary description of an SCons function.
  69     It will get placed in the man page,
  70     and in the appropriate User's Guide appendix.
  71     If the "signature" attribute is specified, SIGTYPE may be one
  72     of "global", "env" or "both" (the default if omitted is "both"),
  73     to indicate the signature applies to the global form or the
  74     environment form, or to generate both with the same signature
  75     (excepting the insertion of "env.").
  76     This allows for the cases of
  77     describing that only one signature should be generated,
  78     or both signatures should be generated and they differ,
  79     or both signatures should be generated and they are the same.
  80     The name of this function may be interpolated
  81     anywhere in the document by specifying the
  82     &f-FUNCTION; element or the &f-env-FUNCTION; element.
  83     Links to this definition may be interpolated by specifying
  84     the &f-link-FUNCTION: or &f-link-env-FUNCTION; element.
  85     </para>
  86
  87     <example>
  88     print("this is example code, it will be offset and indented")
  89     </example>
  90     </summary>
  91     </scons_function>
  92
  93 Construction variable example:
  94
  95     <cvar name="VARIABLE">
  96     <summary>
  97     <para>This is the summary description of a construction variable.
  98     It will get placed in the man page,
  99     and in the appropriate User's Guide appendix.
 100     The name of this construction variable may be interpolated
 101     anywhere in the document by specifying the
 102     &cv-VARIABLE; element. A link to this definition may be
 103     interpolated by specifying the &cv-link-VARIABLE; element.
 104     </para>
 105
 106     <example>
 107     print("this is example code, it will be offset and indented")
 108     </example>
 109     </summary>
 110     </cvar>
 111
 112 Tool example:
 113
 114     <tool name="TOOL">
 115     <summary>
 116     <para>This is the summary description of an SCons Tool.
 117     It will get placed in the man page,
 118     and in the appropriate User's Guide appendix.
 119     The name of this tool may be interpolated
 120     anywhere in the document by specifying the
 121     &t-TOOL; element. A link to this definition may be
 122     interpolated by specifying the &t-link-TOOL; element.
 123     </para>
 124
 125     <example>
 126     print("this is example code, it will be offset and indented")
 127     </example>
 128     </summary>
 129     </tool>
 130 """
 131
 132 import os.path
 133 import re
 134 import sys
 135 import copy
 136 import importlib
 137
 138 try:
 139     from lxml import etree
 140 except ImportError:
 141     try:
 142         import xml.etree.ElementTree as etree
 143     except ImportError:
 144         raise ImportError("Failed to import ElementTree from any known place")
 145
 146 # patterns to help trim XML passed in as strings
 147 re_entity = re.compile(r"&([^;]+);")
 148 re_entity_header = re.compile(r"<!DOCTYPE\s+sconsdoc\s+[^\]]+\]>")
 149
 150 # Namespace for the SCons Docbook XSD
 151 dbxsd = "http://www.scons.org/dbxsd/v1.0"
 152 # Namsespace pattern to help identify an scons-xml file read as  bytes
 153 dbxsdpat = b'xmlns="%s"' % dbxsd.encode('utf-8')
 154 # Namespace map identifier for the SCons Docbook XSD
 155 dbxid = "dbx"
 156 # Namespace for schema instances
 157 xsi = "http://www.w3.org/2001/XMLSchema-instance"
 158
 159 # Header comment with copyright
 160 copyright_comment = """
 161 __COPYRIGHT__
 162
 163 This file is processed by the bin/SConsDoc.py module.
 164 See its docstring for a discussion of the format.
 165 """
 166
 167 def isSConsXml(fpath):
 168     """ Check whether the given file is an SCons XML file.
 169
 170     It is SCons XML if it contains the default target namespace definition
 171     described by dbxsdpat
 172
 173     """
 174     try:
 175         with open(fpath, 'rb') as f:
 176             content = f.read()
 177         if content.find(dbxsdpat) >= 0:
 178             return True
 179     except Exception:
 180         pass
 181
 182     return False
 183
 184 def remove_entities(content):
 185     # Cut out entity inclusions
 186     content = re_entity_header.sub("", content, re.M)
 187     # Cut out entities themselves
 188     content = re_entity.sub(lambda match: match.group(1), content)
 189
 190     return content
 191
 192 default_xsd = os.path.join('doc', 'xsd', 'scons.xsd')
 193
 194 ARG = "dbscons"
 195
 196
 197 class Libxml2ValidityHandler:
 198
 199     def __init__(self):
 200         self.errors = []
 201         self.warnings = []
 202
 203     def error(self, msg, data):
 204         if data != ARG:
 205             raise Exception("Error handler did not receive correct argument")
 206         self.errors.append(msg)
 207
 208     def warning(self, msg, data):
 209         if data != ARG:
 210             raise Exception("Warning handler did not receive correct argument")
 211         self.warnings.append(msg)
 212
 213
 214 class DoctypeEntity:
 215     def __init__(self, name_, uri_):
 216         self.name = name_
 217         self.uri = uri_
 218
 219     def getEntityString(self):
 220         txt = """    <!ENTITY %(perc)s %(name)s SYSTEM "%(uri)s">
 221     %(perc)s%(name)s;
 222 """ % {'perc': perc, 'name': self.name, 'uri': self.uri}
 223
 224         return txt
 225
 226
 227 class DoctypeDeclaration:
 228     def __init__(self, name_=None):
 229         self.name = name_
 230         self.entries = []
 231         if self.name is None:
 232             # Add default entries
 233             self.name = "sconsdoc"
 234             self.addEntity("scons", "../scons.mod")
 235             self.addEntity("builders-mod", "builders.mod")
 236             self.addEntity("functions-mod", "functions.mod")
 237             self.addEntity("tools-mod", "tools.mod")
 238             self.addEntity("variables-mod", "variables.mod")
 239
 240     def addEntity(self, name, uri):
 241         self.entries.append(DoctypeEntity(name, uri))
 242
 243     def createDoctype(self):
 244         content = '<!DOCTYPE %s [\n' % self.name
 245         for e in self.entries:
 246             content += e.getEntityString()
 247         content += ']>\n'
 248
 249         return content
 250
 251 class TreeFactory:
 252     def __init__(self):
 253         pass
 254
 255     @staticmethod
 256     def newNode(tag, **kwargs):
 257         return etree.Element(tag, **kwargs)
 258
 259     @staticmethod
 260     def newSubNode(parent, tag, **kwargs):
 261         return etree.SubElement(parent, tag, **kwargs)
 262
 263     @staticmethod
 264     def newEtreeNode(tag, init_ns=False, **kwargs):
 265         if init_ns:
 266             NSMAP = {None: dbxsd,
 267                      'xsi' : xsi}
 268             return etree.Element(tag, nsmap=NSMAP, **kwargs)
 269
 270         return etree.Element(tag, **kwargs)
 271
 272     @staticmethod
 273     def copyNode(node):
 274         return copy.deepcopy(node)
 275
 276     @staticmethod
 277     def appendNode(parent, child):
 278         parent.append(child)
 279
 280     @staticmethod
 281     def hasAttribute(node, att):
 282         return att in node.attrib
 283
 284     @staticmethod
 285     def getAttribute(node, att):
 286         return node.attrib[att]
 287
 288     @staticmethod
 289     def setAttribute(node, att, value):
 290         node.attrib[att] = value
 291
 292     @staticmethod
 293     def getText(root):
 294         return root.text
 295
 296     @staticmethod
 297     def appendCvLink(root, key, lntail):
 298         linknode = etree.Entity('cv-link-' + key)
 299         linknode.tail = lntail
 300         root.append(linknode)
 301
 302     @staticmethod
 303     def setText(root, txt):
 304         root.text = txt
 305
 306     @staticmethod
 307     def getTail(root):
 308         return root.tail
 309
 310     @staticmethod
 311     def setTail(root, txt):
 312         root.tail = txt
 313
 314     @staticmethod
 315     def writeGenTree(root, fp):
 316         dt = DoctypeDeclaration()
 317         fp.write(etree.tostring(root, encoding="utf-8",
 318                                 pretty_print=True,
 319                                 doctype=dt.createDoctype()).decode('utf-8'))
 320
 321     @staticmethod
 322     def writeTree(root, fpath):
 323         with open(fpath, 'wb') as fp:
 324             fp.write(etree.tostring(root, encoding="utf-8",
 325                                     pretty_print=True))
 326
 327     @staticmethod
 328     def prettyPrintFile(fpath):
 329         with open(fpath,'rb') as fin:
 330             tree = etree.parse(fin)
 331             pretty_content = etree.tostring(tree, encoding="utf-8",
 332                                             pretty_print=True)
 333
 334         with open(fpath,'wb') as fout:
 335             fout.write(pretty_content)
 336
 337     @staticmethod
 338     def decorateWithHeader(root):
 339         root.attrib["{"+xsi+"}schemaLocation"] = "%s %s/scons.xsd" % (dbxsd, dbxsd)
 340         return root
 341
 342     def newXmlTree(self, root):
 343         """ Return a XML file tree with the correct namespaces set,
 344             the element root as top entry and the given header comment.
 345         """
 346         NSMAP = {None: dbxsd, 'xsi' : xsi}
 347         t = etree.Element(root, nsmap=NSMAP)
 348         return self.decorateWithHeader(t)
 349
 350     # singleton to cache parsed xmlschema..
 351     xmlschema = None
 352
 353     @staticmethod
 354     def validateXml(fpath, xmlschema_context):
 355
 356         if TreeFactory.xmlschema is None:
 357             TreeFactory.xmlschema = etree.XMLSchema(xmlschema_context)
 358         try:
 359             doc = etree.parse(fpath)
 360         except Exception as e:
 361             print("ERROR: %s fails to parse:"%fpath)
 362             print(e)
 363             return False
 364         doc.xinclude()
 365         try:
 366             TreeFactory.xmlschema.assertValid(doc)
 367         except etree.XMLSchemaValidateError as e:
 368             print("ERROR: %s fails to validate:" % fpath)
 369             print(e)
 370             print(e.error_log.last_error.message)
 371             print("In file: [%s]" % e.error_log.last_error.filename)
 372             print("Line   : %d" % e.error_log.last_error.line)
 373             return False
 374
 375         except Exception as e:
 376             print("ERROR: %s fails to validate:" % fpath)
 377             print(e)
 378
 379             return False
 380         return True
 381
 382     @staticmethod
 383     def findAll(root, tag, ns=None, xp_ctxt=None, nsmap=None):
 384         expression = ".//{%s}%s" % (nsmap[ns], tag)
 385         if not ns or not nsmap:
 386             expression = ".//%s" % tag
 387         return root.findall(expression)
 388
 389     @staticmethod
 390     def findAllChildrenOf(root, tag, ns=None, xp_ctxt=None, nsmap=None):
 391         expression = "./{%s}%s/*" % (nsmap[ns], tag)
 392         if not ns or not nsmap:
 393             expression = "./%s/*" % tag
 394         return root.findall(expression)
 395
 396     @staticmethod
 397     def convertElementTree(root):
 398         """ Convert the given tree of etree.Element
 399             entries to a list of tree nodes for the
 400             current XML toolkit.
 401         """
 402         return [root]
 403
 404 tf = TreeFactory()
 405
 406
 407 class SConsDocTree:
 408     def __init__(self):
 409         self.nsmap = {'dbx': dbxsd}
 410         self.doc = None
 411         self.root = None
 412         self.xpath_context = None
 413
 414     def parseContent(self, content, include_entities=True):
 415         """ Parses the given text content as XML
 416
 417         This is the setup portion, called from parseContent in
 418         an SConsDocHandler instance - see the notes there.
 419         """
 420         if not include_entities:
 421             content = remove_entities(content)
 422         # Create domtree from given content string
 423         self.root = etree.fromstring(content)
 424
 425     def parseXmlFile(self, fpath):
 426         # Create domtree from file
 427         parser = etree.XMLParser(load_dtd=True, resolve_entities=False)
 428         domtree = etree.parse(fpath, parser)
 429         self.root = domtree.getroot()
 430
 431     def __del__(self):
 432         if self.doc is not None:
 433             self.doc.freeDoc()
 434         if self.xpath_context is not None:
 435             self.xpath_context.xpathFreeContext()
 436
 437 perc = "%"
 438
 439 def validate_all_xml(dpaths, xsdfile=default_xsd):
 440     xmlschema_context = etree.parse(xsdfile)
 441
 442     fpaths = []
 443     for dp in dpaths:
 444         if dp.endswith('.xml') and isSConsXml(dp):
 445             path = '.'
 446             fpaths.append(dp)
 447         else:
 448             for path, dirs, files in os.walk(dp):
 449                 for f in files:
 450                     if f.endswith('.xml'):
 451                         fp = os.path.join(path, f)
 452                         if isSConsXml(fp):
 453                             fpaths.append(fp)
 454
 455     fails = []
 456     fpaths = sorted(fpaths)
 457     for idx, fp in enumerate(fpaths):
 458         fpath = os.path.join(path, fp)
 459         print("%.2f%s (%d/%d) %s" % (float(idx + 1) * 100.0 /float(len(fpaths)),
 460                                      perc, idx + 1, len(fpaths), fp))
 461
 462         if not tf.validateXml(fp, xmlschema_context):
 463             fails.append(fp)
 464             continue
 465
 466     if fails:
 467         return False
 468
 469     return True
 470
 471
 472 class Item:
 473     def __init__(self, name):
 474         self.name = name
 475         self.sort_name = name.lower()
 476         if self.sort_name[0] == '_':
 477             self.sort_name = self.sort_name[1:]
 478         self.sets = []
 479         self.uses = []
 480         self.summary = None
 481         self.arguments = None
 482     def cmp_name(self, name):
 483         if name[0] == '_':
 484             name = name[1:]
 485         return name.lower()
 486     def __eq__(self, other):
 487         return self.sort_name == other.sort_name
 488     def __lt__(self, other):
 489         return self.sort_name < other.sort_name
 490
 491
 492 class Builder(Item):
 493     pass
 494
 495
 496 class Function(Item):
 497     pass
 498
 499
 500 class Tool(Item):
 501     def __init__(self, name):
 502         super().__init__(name)
 503         self.entity = self.name.replace('+', 'X')
 504
 505
 506 class ConstructionVariable(Item):
 507     pass
 508
 509
 510 class Arguments:
 511     def __init__(self, signature, body=None):
 512         if not body:
 513             body = []
 514         self.body = body
 515         self.signature = signature
 516     def __str__(self):
 517         s = ''.join(self.body).strip()
 518         result = []
 519         for m in re.findall(r'([a-zA-Z/_]+|[^a-zA-Z/_]+)', s):
 520             if ' ' in m:
 521                 m = '"%s"' % m
 522             result.append(m)
 523         return ' '.join(result)
 524     def append(self, data):
 525         self.body.append(data)
 526
 527
 528 class SConsDocHandler:
 529     def __init__(self):
 530         self.builders = {}
 531         self.functions = {}
 532         self.tools = {}
 533         self.cvars = {}
 534
 535     def parseItems(self, domelem, xpath_context, nsmap):
 536         items = []
 537
 538         for i in tf.findAll(domelem, "item", dbxid, xpath_context, nsmap):
 539             txt = tf.getText(i)
 540             if txt is not None:
 541                 txt = txt.strip()
 542                 if len(txt):
 543                     items.append(txt.strip())
 544
 545         return items
 546
 547     def parseUsesSets(self, domelem, xpath_context, nsmap):
 548         uses = []
 549         sets = []
 550
 551         for u in tf.findAll(domelem, "uses", dbxid, xpath_context, nsmap):
 552             uses.extend(self.parseItems(u, xpath_context, nsmap))
 553         for s in tf.findAll(domelem, "sets", dbxid, xpath_context, nsmap):
 554             sets.extend(self.parseItems(s, xpath_context, nsmap))
 555
 556         return sorted(uses), sorted(sets)
 557
 558     def parseInstance(self, domelem, map, Class,
 559                         xpath_context, nsmap, include_entities=True):
 560         name = 'unknown'
 561         if tf.hasAttribute(domelem, 'name'):
 562             name = tf.getAttribute(domelem, 'name')
 563         try:
 564             instance = map[name]
 565         except KeyError:
 566             instance = Class(name)
 567             map[name] = instance
 568         uses, sets = self.parseUsesSets(domelem, xpath_context, nsmap)
 569         instance.uses.extend(uses)
 570         instance.sets.extend(sets)
 571         if include_entities:
 572             # Parse summary and function arguments
 573             for s in tf.findAllChildrenOf(domelem, "summary", dbxid, xpath_context, nsmap):
 574                 if instance.summary is None:
 575                     instance.summary = []
 576                 instance.summary.append(tf.copyNode(s))
 577             for a in tf.findAll(domelem, "arguments", dbxid, xpath_context, nsmap):
 578                 if instance.arguments is None:
 579                     instance.arguments = []
 580                 instance.arguments.append(tf.copyNode(a))
 581
 582     def parseDomtree(self, root, xpath_context=None, nsmap=None, include_entities=True):
 583         # Process Builders
 584         for b in tf.findAll(root, "builder", dbxid, xpath_context, nsmap):
 585             self.parseInstance(b, self.builders, Builder,
 586                                xpath_context, nsmap, include_entities)
 587         # Process Functions
 588         for f in tf.findAll(root, "scons_function", dbxid, xpath_context, nsmap):
 589             self.parseInstance(f, self.functions, Function,
 590                                xpath_context, nsmap, include_entities)
 591         # Process Tools
 592         for t in tf.findAll(root, "tool", dbxid, xpath_context, nsmap):
 593             self.parseInstance(t, self.tools, Tool,
 594                                xpath_context, nsmap, include_entities)
 595         # Process CVars
 596         for c in tf.findAll(root, "cvar", dbxid, xpath_context, nsmap):
 597             self.parseInstance(c, self.cvars, ConstructionVariable,
 598                                xpath_context, nsmap, include_entities)
 599
 600     def parseContent(self, content, include_entities=True):
 601         """Parse the given content as XML.
 602
 603         This method is used when we generate the basic lists of entities
 604         for the builders, tools and functions.  So we usually don't
 605         bother about namespaces and resolving entities here...
 606         this is handled in parseXmlFile below (step 2 of the overall process).
 607         """
 608         # Create doctree
 609         t = SConsDocTree()
 610         t.parseContent(content, include_entities)
 611         # Parse it
 612         self.parseDomtree(t.root, t.xpath_context, t.nsmap, include_entities)
 613
 614     def parseXmlFile(self, fpath):
 615         # Create doctree
 616         t = SConsDocTree()
 617         t.parseXmlFile(fpath)
 618         # Parse it
 619         self.parseDomtree(t.root, t.xpath_context, t.nsmap)
 620
 621 def importfile(path):
 622     """Import a Python source file or compiled file given its path."""
 623     from importlib.util import MAGIC_NUMBER
 624     with open(path, 'rb') as ifp:
 625         is_bytecode = MAGIC_NUMBER == ifp.read(len(MAGIC_NUMBER))
 626     filename = os.path.basename(path)
 627     name, ext = os.path.splitext(filename)
 628     if is_bytecode:
 629         loader = importlib._bootstrap_external.SourcelessFileLoader(name, path)
 630     else:
 631         loader = importlib._bootstrap_external.SourceFileLoader(name, path)
 632     # XXX We probably don't need to pass in the loader here.
 633     spec = importlib.util.spec_from_file_location(name, path, loader=loader)
 634     try:
 635         return importlib._bootstrap._load(spec)
 636     except ImportError:
 637         raise Exception(path, sys.exc_info())
 638
 639 # Local Variables:
 640 # tab-width:4
 641 # indent-tabs-mode:nil
 642 # End:
 643 # vim: set expandtab tabstop=4 shiftwidth=4: