xmlmerge.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 # Copyright 2008,2009  Felix Rabe  <public@felixrabe.net>
   5
   6
   7 # This file is part of XML Merge.
   8
   9 # XML Merge is free software: you can redistribute it and/or modify it
  10 # under the terms of the GNU Lesser General Public License as published by
  11 # the Free Software Foundation, either version 3 of the License, or (at
  12 # your option) any later version.
  13
  14 # XML Merge is distributed in the hope that it will be useful, but
  15 # WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU Lesser General Public License for more details.
  18
  19 # You should have received a copy of the GNU Lesser General Public License
  20 # along with XML Merge.  If not, see <http://www.gnu.org/licenses/>.
  21
  22
  23 # Developed (i.e. tested) using Python 2.6.4 and lxml 2.2.2.
  24
  25 # TODO: What if an attribute should include the '{' or '}' chars?
  26
  27 """
  28 The purpose of XML Merge is to preprocess any kind of XML file with great
  29 flexibility.
  30
  31 XML Merge performs (among other things) recursive XML file inclusion and
  32 XML element and attribute modification.
  33
  34 XML Merge is a Python module. It is normally invoked as a program from the
  35 command line, but can equally well be used from within another Python
  36 program or module.
  37 """
  38
  39 ## IMPORTS AND CONSTANTS
  40
  41 import copy
  42 import itertools
  43 import optparse
  44 import os
  45 import re
  46 import sys
  47 import textwrap
  48
  49 import lxml.etree as ET
  50
  51 # Namespace mapping (can be directly used for lxml nsmap arguments):
  52 xmns = {"xm":   "urn:felixrabe:xmlns:xmlmerge:preprocess",
  53         "xmt":  "urn:felixrabe:xmlns:xmlmerge:inctrace"}
  54
  55
  56 ## COMMAND LINE OPTION PARSING
  57
  58 class OptionParser(optparse.OptionParser):
  59
  60     def __init__(self, *a, **kw):
  61         optparse.OptionParser.__init__(self, *a, **kw)
  62         self.add_option("-i", "--input",
  63                         help=("(REQUIRED) input XML file"))
  64         self.add_option("-o", "--output",
  65                         help=("output XML file (.out.xml if not given)"))
  66         self.add_option("-s", "--xml-schema",
  67                         help=("XML Schema (.xsd) to validate output " +
  68                               "against"))
  69         self.add_option("-r", "--reference",
  70                         help=("reference XML file to compare output " +
  71                               "against"))
  72         self.add_option("-d", "--html-diff", action="store_true",
  73                         help=("only with -r; if output and reference " +
  74                               "differ, produce a HTML file showing the " +
  75                               "differences"))
  76         self.add_option("-t", "--trace-includes", action="store_true",
  77                         help=("add tracing information to included " +
  78                               "XML fragments"))
  79         self.add_option("-v", "--verbose", action="store_const",
  80                         dest="verbose", const=3,
  81                         help=("show debugging messages"))
  82         self.add_option("-q", "--quiet", action="store_const",
  83                         dest="verbose", const=1,
  84                         help=("only show error messages"))
  85         self.set_defaults(verbose=2)
  86
  87         # Explanation: levels of verbosity
  88         # --quiet   -> self.verbose == 1  # only show error messages
  89         #           -> self.verbose == 2  # no verbosity option given
  90         # --verbose -> self.verbose == 3  # show debugging messages
  91
  92
  93 def parse_command_line(argv):
  94     """
  95     parse_command_line(argv) -> optparse.Values
  96
  97     Parse argv and return an optparse.Values object containing the options.
  98
  99     This function performs all the necessary checks and conversions to make
 100     sure all necessary options are given, and that all options are
 101     available in a normalized format.
 102
 103     It also tries to create the containing directory for the output file if
 104     it does not exist already.
 105     """
 106     # Parse options using OptionParser:
 107     option_parser = OptionParser()
 108     options, args = option_parser.parse_args(argv[1:])
 109
 110     # Make sure only options, and no other arguments, are passed on the
 111     # command line:
 112     try:
 113         assert args == []
 114         assert options.input is not None
 115     except:
 116         option_parser.error("Error: invalid argument list")
 117
 118     # If the output option has been omitted, build the output filename from
 119     # the input filename, resulting in the file extension ".out.xml":
 120     if options.output is None:
 121         if options.input.lower().endswith(".xml"):
 122             options.output = options.input[:-4] + ".out.xml"
 123         else:
 124             options.output = options.input      + ".out.xml"
 125
 126     # Convert all filename options to normalized absolutized pathnames:
 127     for n in "input output reference".split():
 128         if getattr(options, n) is None: continue  # if "-r" was not given
 129         setattr(options, n, os.path.abspath(getattr(options, n)))
 130
 131     # When --verbose, print all filename options:
 132     if options.verbose >= 3:
 133         print "Input:     %s" % options.input
 134         print "Output:    %s" % options.output
 135         print "Reference: %s" % options.reference
 136
 137     # Make sure there is a directory where the output XML file should go:
 138     try:
 139         os.makedirs(os.path.dirname(options.output))
 140     except:
 141         pass  # fail later if there still is no output directory now
 142
 143     return options
 144
 145
 146 ## XML PROCESSING AND COMPARISON
 147
 148 def read_input_file(input_filename):
 149     """
 150     read_input_file(input_filename) -> ET._Element
 151
 152     Read the input file, and return the corresponding XML Element object,
 153     the element tree root.
 154     """
 155     input_xml = ET.parse(input_filename).getroot()
 156     return input_xml
 157
 158 def postprocess_xml(output_xml):
 159     """
 160     postprocess_xml(output_xml) -> ET._Element
 161
 162     Remove unnecessary namespace declarations and whitespace. Returns a
 163     modified copy of output_xml. The argument may be modified by calling
 164     this function.
 165     """
 166     # Remove unused namespace declarations:
 167     # (http://codespeak.net/pipermail/lxml-dev/2009-September/004888.html)
 168     ns_root = ET.Element("NS_ROOT", nsmap=xmns)
 169     ns_root.append(output_xml)
 170     ns_root.remove(output_xml)
 171     # If you don't perform this copy, each output_xml element's
 172     # getroottree() will report the temporary tree containing the empty
 173     # NS_ROOT element. This is not a hack, this is about how lxml works.
 174     output_xml = ET.ElementTree(copy.copy(output_xml)).getroot()
 175
 176     # Make pretty-printing work by removing unnecessary whitespace:
 177     for el in output_xml.iter():
 178         if el.text and not el.text.strip():
 179             el.text = None
 180         if el.tail and not el.tail.strip():
 181             el.tail = None
 182
 183     return output_xml
 184
 185 def write_output_file(output_xml, output_filename):
 186     """
 187     Write the output XML Element to the specified output filename.
 188     """
 189     output_xmltree = output_xml.getroottree()
 190     output_xmltree.write(output_filename, pretty_print=True,
 191                          xml_declaration=True, encoding="utf-8")
 192
 193 def read_xml_schema_file(xml_schema_filename):
 194     """
 195     read_xml_schema_file(xml_schema_filename) -> ET.XMLSchema
 196
 197     Read the XML Schema file, and return the corresponding XML Schema
 198     object.
 199     """
 200     xml_schema_xmltree = ET.parse(xml_schema_filename)
 201     xml_schema = ET.XMLSchema(xml_schema_xmltree)
 202     return xml_schema
 203
 204 def match_against_schema(options, output_xml, xml_schema):
 205     """
 206     match_against_schema(options, output_xml, xml_schema) -> bool
 207
 208     Validate output against XML Schema.
 209
 210     The result is True if the output XML Element (tree) matches the XML
 211     Schema, otherwise the result is False.
 212     """
 213     is_valid = xml_schema.validate(output_xml.getroottree())
 214     if options.verbose >= 2:
 215         if is_valid:
 216             print "Output matches XML Schema."
 217         else:
 218             print "Output invalid according to XML Schema."
 219             print xml_schema.error_log.last_error
 220     return is_valid
 221
 222 def match_against_reference(options, output_xml):
 223     """
 224     match_against_reference(options, output_xml) -> bool
 225
 226     Compare the output string (read from file options.output) to the
 227     reference string (read from options.reference). If they are not the
 228     same (bytewise), and if options.html_diff is True, create an HTML file
 229     showing the differences.
 230
 231     The result is True if output and reference are the same (bytewise),
 232     otherwise the result is False.
 233     """
 234     reference_filename = options.reference
 235     output_filename = options.output
 236     do_html_diff = options.html_diff
 237
 238     reference_str = file(reference_filename, "rb").read()
 239     output_str = file(output_filename, "rb").read()
 240     is_valid = (reference_str == output_str)
 241     if options.verbose >= 2:
 242         if is_valid:
 243             print "Output matches reference."
 244         elif not do_html_diff:
 245             print "Output and reference differ."
 246     if do_html_diff and not is_valid:
 247         html_filename = "%s.diff.html" % output_filename
 248         if options.verbose >= 2:
 249             print ("Output and reference differ - " +
 250                    "generating '%s'..." % html_filename)
 251         create_reference_diff_html(html_filename, reference_str,
 252                                    output_str)
 253     return is_valid
 254
 255 def create_reference_diff_html(html_filename, reference_str, output_str):
 256     """
 257     Create an HTML file (created at html_filename) showing the differrences
 258     between the reference string and the output string side-by-side.
 259     """
 260     reference_lines = reference_str.splitlines()
 261     output_lines    = output_str   .splitlines()
 262
 263     import difflib
 264     html_diff = difflib.HtmlDiff(wrapcolumn=75)
 265     html_str = html_diff.make_file(reference_lines, output_lines,
 266                                    "Reference",     "Output")
 267     file(html_filename, "w").write(html_str)
 268
 269
 270 ## XML PREPROCESS CLASS
 271
 272 class XMLPreprocess(object):
 273     """
 274     Use:
 275
 276     >>> proc = XMLPreprocess()
 277     >>> output_xml = proc(options, input_xml)  # input_xml may change
 278     """
 279
 280     def __init__(self):
 281         super(XMLPreprocess, self).__init__()
 282         self._namespace_stack = [{}]
 283
 284     def __call__(self, xml_element, namespace=None,
 285                  trace_includes=False, xml_filename=None):
 286         """
 287         XMLPreprocess()(...)
 288
 289         Preprocess the input XML Element, xml_element. The element tree of
 290         xml_element will be modified in-place.
 291
 292         The namespace given should be a dict that can be used as a Python
 293         namespace. This namespace will be used in XML attribute
 294         substitution.
 295
 296         If trace_includes is True, the output will contain tags that
 297         surround included sections of the file. The xml_filename argument
 298         is then required.
 299
 300         Processing tags will recursively call this method (__call__) for
 301         preprocessing the included file and for recursive inclusion.
 302         """
 303         print "Processing", xml_element.tag
 304         if namespace is not None:
 305             self._namespace_stack.append(namespace)
 306         self.namespace = self._namespace_stack[-1]
 307         self.trace_includes = trace_includes
 308         self.xml_filename = xml_filename
 309
 310         ns = "{%s}" % xmns["xm"]
 311         len_ns = len(ns)
 312
 313         # Evaluate Python expressions in the attributes of xml_element:
 314         for attr_name, attr_value in xml_element.items():  # attr map
 315             v = self._eval_substitution(attr_value, self.namespace)
 316             xml_element.set(attr_name, v)
 317
 318         # If xml_element has xmns["xm"] as its namespace, proceed with the
 319         # appropriate method of this class:
 320         if xml_element.nsmap.get(xml_element.prefix) == xmns["xm"]:
 321             tag = xml_element.tag[len_ns:]  # just the tag without namespc
 322             method = "_xm_" + tag.lower()  # tolerate any case
 323             if not hasattr(self, method):
 324                 raise Exception, "cannot process <xm:%s/>" % tag
 325             getattr(self, method)(xml_element)  # call the method
 326             xml_element.getparent().remove(xml_element)
 327
 328         # If not, recurse:
 329         else:
 330             self._recurse_into(xml_element)
 331
 332         return None
 333
 334     def _recurse_into(self, xml_element, namespace=None):
 335         if namespace is not None:
 336             self._namespace_stack.append(namespace)
 337         for xml_sub_element in xml_element.xpath("*"):
 338             self(xml_sub_element, None,
 339                  self.trace_includes, self.xml_filename)
 340         if namespace is not None:
 341             self.namespace = self._namespace_stack.pop()
 342
 343     _eval_substitution_regex = re.compile(r"\{(.*?)\}")
 344
 345     def _eval_substitution(self, attr_value, namespace):
 346         """
 347         Evaluate Python expressions within strings.
 348
 349         Internal method to perform substitution of Python expressions
 350         within attribute values, {x} -> str(eval(x)).  Example:
 351
 352         >>> self._attr_substitution("3 + 5 = {3 + 5} in Python", {})
 353         '3 + 5 = 8 in Python'
 354
 355         Multiple Python expressions in one string are supported as well.
 356         """
 357         new_a_value = []  # faster than always concatenating strings
 358         last_index = 0
 359         for match in self._eval_substitution_regex.finditer(attr_value):
 360             new_a_value.append(attr_value[last_index:match.start()])
 361             result = str(eval(match.group(1), namespace))
 362             new_a_value.append(result)
 363             last_index = match.end()
 364         new_a_value.append(attr_value[last_index:])
 365         return "".join(new_a_value)
 366
 367     def _xm_addelements(self, xml_element):
 368         """
 369         Add subelements to, before, or after the element selected by XPath
 370         (@to, @before or @after).
 371         """
 372         to = xml_element.get("to")
 373         before = xml_element.get("before")
 374         after = xml_element.get("after")
 375         assert sum((to is None, before is None, after is None)) == 2
 376         select = to or before or after
 377
 378     def _xm_block(self, xml_element):
 379         """
 380         Create a scope to contain visibility of newly assigned Python
 381         variables.  This works the same way that Python itself scopes
 382         variables, i.e. by creating a shallow copy of the Python namespace.
 383         E.g. assignments to list items will be visible to outside scopes!
 384         """
 385         self._recurse_into(xml_element, self.namespace.copy())
 386         for xml_sub_element in xml_element[::-1]:
 387             xml_element.addnext(xml_sub_element)
 388
 389     def _xm_comment(self, xml_element):
 390         """
 391         A comment that is removed by XML Merge.
 392         """
 393         pass  # that's it
 394
 395     def _xm_include(self, xml_element):
 396         """
 397         Include from the specified file (@file) the elements selected by
 398         XPath (@select).
 399         """
 400
 401     def _xm_loop(self, xml_element):
 402         """
 403         Loop over a range of integer values.
 404
 405         The first attribute is evaluated as the loop counter.  Example:
 406
 407             i="range(5, 9)"  =>  iterates with i being 5, 6, 7, 8
 408
 409         WARNING: The loop counter attribute, as well as all substitutions
 410         in subelement attributes (XPath ".//@*": "...{foo_bar}...") will
 411         (wholly or partially) be evaluated as Python expressions using
 412         eval().
 413         """
 414         # Get the loop counter name and list:
 415         loop_counter_name = xml_element.keys()[0]
 416         loop_counter_list = eval(xml_element.get(loop_counter_name),
 417                                  self.namespace)
 418
 419         # Loop:
 420         addnext_to_node = xml_element  # for new elements
 421         for loop_counter_value in loop_counter_list:
 422             pass
 423
 424     def _xm_pythonexec(self, xml_element):
 425         """
 426         Execute Python code in the current namespace.
 427
 428         'self' and 'xml_element' are supplied temporarily. They are added
 429         to the current namespace before the 'exec' statement, and removed
 430         again afterwards.
 431         """
 432         code = textwrap.dedent(xml_element.text).strip()
 433         self.namespace["self"] = self
 434         self.namespace["xml_element"] = xml_element
 435         exec code in self.namespace
 436         del self.namespace["self"], self.namespace["xml_element"]
 437
 438     def _xm_removeattribute(self, xml_element):
 439         """
 440         Remove the attribute (@name) from the element selected by XPath
 441         (@select).
 442         """
 443
 444     def _xm_removeelements(self, xml_element):
 445         """
 446         Remove (zero or more) elements selected by XPath (@select).
 447         """
 448
 449     def _xm_setattribute(self, xml_element):
 450         """
 451         Assign the value (@value) to the attribute (@name) of the element
 452         selected by XPath (@select).
 453
 454         Example:
 455             <Object index="0x1234"/>
 456             <xm:SetAttribute name="otherattr" value="hallo"/>
 457
 458         Leads to:
 459             <Object index="0x1234" otherattr="hello"/>
 460         """
 461
 462     def _xm_var(self, xml_element):
 463         """
 464         Set a variable.
 465         """
 466         ns = self.namespace
 467         for attr_name, attr_value in xml_element.items():  # attr map
 468             ns[attr_name] = eval(attr_value, ns, ns)
 469
 470
 471 ## MAIN FUNCTION
 472
 473 def main(argv):
 474     """
 475     main(argv) -> int
 476
 477     Process input to produce output according to the command line options.
 478
 479     After the XML Merge Manual, this is the first piece of the code a new
 480     developer will read. Keep this code as simple as possible if you change
 481     it in any way.
 482
 483     These are all possible exit status codes returned or raised (using
 484     SystemExit) by main or the functions it calls:
 485         - On success, and if all requested validations (-s, -r) match:
 486             return 0
 487         - On error, e.g. wrong options (see parse_command_line()):
 488             return 1
 489         - On mismatch (either XML Schema (-s) or reference (-r)):
 490             return mismatch_bitmap  # see end of main()
 491         - To aid understanding the bitmap: If N matching functions are
 492           provided, and all are requested and all fail to match the output
 493           file:
 494             return (2 ** N - 1) * 2  # mismatch_bitmap
 495     """
 496     # Parse command line to get options:
 497     options = parse_command_line(argv)
 498
 499     # Input file => preprocessing => output file:
 500     xml = read_input_file(options.input)
 501     proc = XMLPreprocess()
 502     proc(xml, trace_includes=options.trace_includes,
 503          xml_filename=options.input)
 504     xml = postprocess_xml(xml)
 505     write_output_file(xml, options.output)
 506
 507     # If -s: Compare output to XML Schema file:
 508     matches_schema = True  # False means: match requested and negative
 509     if options.xml_schema is not None:
 510         xml_schema = read_xml_schema_file(options.xml_schema)
 511         matches_schema = match_against_schema(options, xml, xml_schema)
 512
 513     # If -r: Compare output to reference:
 514     matches_reference = True  # False means: match requested and negative
 515     if options.reference is not None:
 516         matches_reference = match_against_reference(options, xml)
 517
 518     # Calculate and return the mismatch bitmap:
 519     mismatch_bitmap = 0
 520     mismatch_bitmap |= int(not matches_schema)    << 1  # 2 on mismatch
 521     mismatch_bitmap |= int(not matches_reference) << 2  # 4 on mismatch
 522     return mismatch_bitmap
 523
 524
 525 if __name__ == "__main__":
 526     sys.exit(main(sys.argv))