xmlmerge.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 # Copyright 2008,2009  Felix Rabe  <public@felixrabe.net>
   5
   6
   7 # This file is part of XML Merge.
   8
   9 # XML Merge is free software: you can redistribute it and/or modify it
  10 # under the terms of the GNU Lesser General Public License as published by
  11 # the Free Software Foundation, either version 3 of the License, or (at
  12 # your option) any later version.
  13
  14 # XML Merge is distributed in the hope that it will be useful, but
  15 # WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU Lesser General Public License for more details.
  18
  19 # You should have received a copy of the GNU Lesser General Public License
  20 # along with XML Merge.  If not, see <http://www.gnu.org/licenses/>.
  21
  22
  23 # Developed (i.e. tested) using Python 2.6.4 and lxml 2.2.2.
  24
  25 """
  26 The purpose of XML Merge is to preprocess any kind of XML file with great
  27 flexibility.
  28
  29 XML Merge performs (among other things) recursive XML file inclusion and
  30 XML element and attribute modification.
  31
  32 XML Merge is a Python module. It is normally invoked as a program from the
  33 command line, but can equally well be used from within another Python
  34 program or module.
  35 """
  36
  37 ## IMPORTS AND CONSTANTS
  38
  39 import copy
  40 import itertools
  41 import optparse
  42 import os
  43 import re
  44 import sys
  45
  46 import lxml.etree as ET
  47
  48 # Namespace mapping (can be directly used for lxml nsmap arguments):
  49 xmns = {"xm":   "urn:felixrabe:xmlns:xmlmerge:preprocess",
  50         "xmt":  "urn:felixrabe:xmlns:xmlmerge:inctrace"}
  51
  52
  53 ## COMMAND LINE OPTION PARSING
  54
  55 class OptionParser(optparse.OptionParser):
  56
  57     def __init__(self, *a, **kw):
  58         optparse.OptionParser.__init__(self, *a, **kw)
  59         self.add_option("-i", "--input",
  60                         help=("(REQUIRED) input XML file"))
  61         self.add_option("-o", "--output",
  62                         help=("output XML file (.out.xml if not given)"))
  63         self.add_option("-s", "--xml-schema",
  64                         help=("XML Schema (.xsd) to validate output " +
  65                               "against"))
  66         self.add_option("-r", "--reference",
  67                         help=("reference XML file to compare output " +
  68                               "against"))
  69         self.add_option("-d", "--html-diff", action="store_true",
  70                         help=("only with -r; if output and reference " +
  71                               "differ, produce a HTML file showing the " +
  72                               "differences"))
  73         self.add_option("-t", "--trace-includes", action="store_true",
  74                         help=("add tracing information to included " +
  75                               "XML fragments"))
  76         self.add_option("-v", "--verbose", action="store_const",
  77                         dest="verbose", const=3,
  78                         help=("show debugging messages"))
  79         self.add_option("-q", "--quiet", action="store_const",
  80                         dest="verbose", const=1,
  81                         help=("only show error messages"))
  82         self.set_defaults(verbose=2)
  83
  84         # Explanation: levels of verbosity
  85         # --quiet   -> self.verbose == 1  # only show error messages
  86         #           -> self.verbose == 2  # no verbosity option given
  87         # --verbose -> self.verbose == 3  # show debugging messages
  88
  89
  90 def parse_command_line(argv):
  91     """
  92     parse_command_line(argv) -> optparse.Values
  93
  94     Parse argv and return an optparse.Values object containing the options.
  95
  96     This function performs all the necessary checks and conversions to make
  97     sure all necessary options are given, and that all options are
  98     available in a normalized format.
  99
 100     It also tries to create the containing directory for the output file if
 101     it does not exist already.
 102     """
 103     # Parse options using OptionParser:
 104     option_parser = OptionParser()
 105     options, args = option_parser.parse_args(argv[1:])
 106
 107     # Make sure only options, and no other arguments, are passed on the
 108     # command line:
 109     try:
 110         assert args == []
 111         assert options.input is not None
 112     except:
 113         option_parser.error("Error: invalid argument list")
 114
 115     # If the output option has been omitted, build the output filename from
 116     # the input filename, resulting in the file extension ".out.xml":
 117     if options.output is None:
 118         if options.input.lower().endswith(".xml"):
 119             options.output = options.input[:-4] + ".out.xml"
 120         else:
 121             options.output = options.input      + ".out.xml"
 122
 123     # Convert all filename options to normalized absolutized pathnames:
 124     for n in "input output reference".split():
 125         if getattr(options, n) is None: continue  # if "-r" was not given
 126         setattr(options, n, os.path.abspath(getattr(options, n)))
 127
 128     # When --verbose, print all filename options:
 129     if options.verbose >= 3:
 130         print "Input:     %s" % options.input
 131         print "Output:    %s" % options.output
 132         print "Reference: %s" % options.reference
 133
 134     # Make sure there is a directory where the output XML file should go:
 135     try:
 136         os.makedirs(os.path.dirname(options.output))
 137     except:
 138         pass  # fail later if there still is no output directory now
 139
 140     return options
 141
 142
 143 ## XML PROCESSING AND COMPARISON
 144
 145 def read_input_file(input_filename):
 146     """
 147     read_input_file(input_filename) -> ET._Element
 148
 149     Read the input file, and return the corresponding XML Element object,
 150     the element tree root.
 151     """
 152     input_xml = ET.parse(input_filename).getroot()
 153     return input_xml
 154
 155 def postprocess_xml(output_xml):
 156     """
 157     postprocess_xml(output_xml) -> ET._Element
 158
 159     Remove unnecessary namespace declarations and whitespace. Returns a
 160     modified copy of output_xml. The argument may be modified by calling
 161     this function.
 162     """
 163     # Remove unused namespace declarations:
 164     # (http://codespeak.net/pipermail/lxml-dev/2009-September/004888.html)
 165     ns_root = ET.Element("NS_ROOT", nsmap=xmns)
 166     ns_root.append(output_xml)
 167     ns_root.remove(output_xml)
 168     # If you don't perform this copy, each output_xml element's
 169     # getroottree() will report the temporary tree containing the empty
 170     # NS_ROOT element. This is not a hack, this is about how lxml works.
 171     output_xml = ET.ElementTree(copy.copy(output_xml)).getroot()
 172
 173     # Make pretty-printing work by removing unnecessary whitespace:
 174     for el in output_xml.iter():
 175         if len(el) and el.text and not el.text.strip():
 176             el.text = None
 177         if el.tail and not el.tail.strip():
 178             el.tail = None
 179
 180     return output_xml
 181
 182 def write_output_file(output_xml, output_filename):
 183     """
 184     Write the output XML Element to the specified output filename.
 185     """
 186     output_xmltree = output_xml.getroottree()
 187     output_xmltree.write(output_filename, pretty_print=True,
 188                          xml_declaration=True, encoding="utf-8")
 189
 190 def read_xml_schema_file(xml_schema_filename):
 191     """
 192     read_xml_schema_file(xml_schema_filename) -> ET.XMLSchema
 193
 194     Read the XML Schema file, and return the corresponding XML Schema
 195     object.
 196     """
 197     xml_schema_xmltree = ET.parse(xml_schema_filename)
 198     xml_schema = ET.XMLSchema(xml_schema_xmltree)
 199     return xml_schema
 200
 201 def match_against_schema(options, output_xml, xml_schema):
 202     """
 203     match_against_schema(options, output_xml, xml_schema) -> bool
 204
 205     Validate output against XML Schema.
 206
 207     The result is True if the output XML Element (tree) matches the XML
 208     Schema, otherwise the result is False.
 209     """
 210     is_valid = xml_schema.validate(output_xml.getroottree())
 211     if options.verbose >= 2:
 212         if is_valid:
 213             print "Output matches XML Schema."
 214         else:
 215             print "Output invalid according to XML Schema."
 216             print xml_schema.error_log.last_error
 217     return is_valid
 218
 219 def match_against_reference(options, output_xml):
 220     """
 221     match_against_reference(options, output_xml) -> bool
 222
 223     Compare the output string (read from file options.output) to the
 224     reference string (read from options.reference). If they are not the
 225     same (bytewise), and if options.html_diff is True, create an HTML file
 226     showing the differences.
 227
 228     The result is True if output and reference are the same (bytewise),
 229     otherwise the result is False.
 230     """
 231     reference_filename = options.reference
 232     output_filename = options.output
 233     do_html_diff = options.html_diff
 234
 235     reference_str = file(reference_filename, "rb").read()
 236     output_str = file(output_filename, "rb").read()
 237     is_valid = (reference_str == output_str)
 238     if options.verbose >= 2:
 239         if is_valid:
 240             print "Output matches reference."
 241         elif not do_html_diff:
 242             print "Output and reference differ."
 243     if do_html_diff and not is_valid:
 244         html_filename = "%s.diff.html" % output_filename
 245         if options.verbose >= 2:
 246             print ("Output and reference differ - " +
 247                    "generating '%s'..." % html_filename)
 248         create_reference_diff_html(html_filename, reference_str,
 249                                    output_str)
 250     return is_valid
 251
 252 def create_reference_diff_html(html_filename, reference_str, output_str):
 253     """
 254     Create an HTML file (created at html_filename) showing the differrences
 255     between the reference string and the output string side-by-side.
 256     """
 257     reference_lines = reference_str.splitlines()
 258     output_lines    = output_str   .splitlines()
 259
 260     import difflib
 261     html_diff = difflib.HtmlDiff(wrapcolumn=75)
 262     html_str = html_diff.make_file(reference_lines, output_lines,
 263                                    "Reference",     "Output")
 264     file(html_filename, "w").write(html_str)
 265
 266
 267 ## XML PREPROCESS CLASS
 268
 269 class XMLPreprocess(object):
 270     """
 271     Use:
 272
 273     >>> proc = XMLPreprocess()
 274     >>> output_xml = proc(options, input_xml)  # input_xml may change
 275     """
 276
 277     def __call__(self, xml_element, namespace=None,
 278                  trace_includes=False, xml_filename=None):
 279         """
 280         XMLPreprocess()(...)
 281
 282         Preprocess the input XML Element, xml_element. The element tree of
 283         xml_element will be modified in-place.
 284
 285         The namespace given should be a dict that can be used as a Python
 286         namespace. This namespace will be used in XML attribute
 287         substitution, and modified by the xm:Loop and xm:Var elements.
 288
 289         If trace_includes is True, the output will contain tags that
 290         surround included sections of the file. The xml_filename argument
 291         is then required.
 292
 293         Inclusion will recursively call this method (__call__) for
 294         preprocessing the included file and for recursive inclusion.
 295         """
 296         self.namespace = namespace or {}
 297         self.trace_includes = trace_includes
 298         self.xml_filename = xml_filename
 299
 300         ns = "{%s}" % xmns["xm"]
 301         len_ns = len(ns)
 302
 303         # for el in doc.xpath("//xm:*", **n): print el.tag, (el.xpath("(.//xm:*|following::xm:*)[1]", **n) or [ET.Element("NULL")])[0].tag
 304
 305         # if False:
 306         #     child_list = loop_copy.xpath(".//*")
 307         #     lower_loop_child_set = set(loop_copy.xpath(".//xm:Loop/*",
 308         #                                                namespaces=xmns))
 309         #     interesting_set = set(child_list) - lower_loop_child_set
 310         #     child_list = sorted(interesting_set, key=child_list.index)
 311
 312         return xml_element
 313
 314     _eval_substitution_regex = re.compile(r"\{(.*?)\}")
 315
 316     def _eval_substitution(self, attr_value, namespace):
 317         """
 318         Evaluate Python expressions within strings.
 319
 320         Internal method to perform substitution of Python expressions
 321         within attribute values, {x} -> str(eval(x)).  Example:
 322
 323         >>> self._attr_substitution("3 + 5 = {3 + 5} in Python", {})
 324         '3 + 5 = 8 in Python'
 325
 326         Multiple Python expressions in one string are supported as well.
 327         """
 328         new_a_value = []  # faster than always concatenating strings
 329         last_index = 0
 330         for match in self._eval_substitution_regex.finditer(attr_value):
 331             new_a_value.append(attr_value[last_index:match.start()])
 332             result = str(eval(match.group(1), namespace, namespace))
 333             new_a_value.append(result)
 334             last_index = match.end()
 335         new_a_value.append(attr_value[last_index:])
 336         return "".join(new_a_value)
 337
 338     def Var(self, var_element):
 339         pass
 340
 341     def Loop(self, loop_element):
 342         """
 343         Loop over a range of integer values.
 344
 345         The first attribute is evaluated as the loop counter.  Example:
 346
 347             i="range(5, 9)"  =>  iterates with i being 5, 6, 7, 8
 348
 349         WARNING: The loop counter attribute, as well as all substitutions
 350         in subelement attributes (XPath ".//@*": "...{foo_bar}...") will
 351         (wholly or partially) be evaluated as Python expressions using
 352         eval().
 353         """
 354         # Get the loop counter name and list:
 355         loop_counter_name = loop_element.keys()[0]
 356         loop_counter_list = eval(loop_element.get(loop_counter_name))
 357
 358         # Loop:
 359         addnext_to_node = loop_element  # for new elements
 360         for loop_counter_value in loop_counter_list:
 361             pass
 362
 363     def Include(self, el):
 364         """
 365         Include from the specified file (@file) the elements selected by
 366         XPath (@select).
 367         """
 368
 369     def AddElements(self, el):
 370         """
 371         Add subelements to, before, or after the element selected by XPath
 372         (@to, @before or @after).
 373         """
 374         to = el.get("to")
 375         before = el.get("before")
 376         after = el.get("after")
 377         assert sum((to is None, before is None, after is None)) == 2
 378         select = to or before or after
 379
 380     def RemoveElements(self, el):
 381         """
 382         Remove elements selected by XPath (@select).
 383         """
 384
 385     def SetAttribute(self, el):
 386         """
 387         Assign the value (@value) to the attribute (@name) of the element
 388         selected by XPath (@select).
 389         """
 390
 391     def RemoveAttribute(self, el):
 392         """
 393         Remove the attribute (@name) from the element selected by XPath
 394         (@select).
 395         """
 396
 397     def PythonCode(self, el):
 398         """
 399         Execute Python code.
 400         """
 401
 402
 403 ## MAIN FUNCTION
 404
 405 def main(argv):
 406     """
 407     main(argv) -> int
 408
 409     Process input to produce output according to the command line options.
 410
 411     After the XML Merge Manual, this is the first piece of the code a new
 412     developer will read. Keep this code as simple as possible if you change
 413     it in any way.
 414
 415     These are all possible exit status codes returned or raised (using
 416     SystemExit) by main or the functions it calls:
 417         - On success, and if all requested validations (-s, -r) match:
 418             return 0
 419         - On error, e.g. wrong options (see parse_command_line()):
 420             return 1
 421         - On mismatch (either XML Schema (-s) or reference (-r)):
 422             return mismatch_bitmap  # see end of main()
 423         - To aid understanding the bitmap: If N matching functions are
 424           provided, and all are requested and all fail to match the output
 425           file:
 426             return (2 ** N - 1) * 2  # mismatch_bitmap
 427     """
 428     # Parse command line to get options:
 429     options = parse_command_line(argv)
 430
 431     # Input file => preprocessing => output file:
 432     xml = read_input_file(options.input)
 433     proc = XMLPreprocess()
 434     proc(xml, trace_includes=options.trace_includes,
 435          xml_filename=options.input)
 436     xml = postprocess_xml(xml)
 437     write_output_file(xml, options.output)
 438
 439     # If -s: Compare output to XML Schema file:
 440     matches_schema = True  # False means: match requested and negative
 441     if options.xml_schema is not None:
 442         xml_schema = read_xml_schema_file(options.xml_schema)
 443         matches_schema = match_against_schema(options, xml, xml_schema)
 444
 445     # If -r: Compare output to reference:
 446     matches_reference = True  # False means: match requested and negative
 447     if options.reference is not None:
 448         matches_reference = match_against_reference(options, xml)
 449
 450     # Calculate and return the mismatch bitmap:
 451     mismatch_bitmap = 0
 452     mismatch_bitmap |= int(not matches_schema)    << 1  # 2 on mismatch
 453     mismatch_bitmap |= int(not matches_reference) << 2  # 4 on mismatch
 454     return mismatch_bitmap
 455
 456
 457 if __name__ == "__main__":
 458     sys.exit(main(sys.argv))