PyCIM/RDFXMLReader.py

   1 # Copyright (C) 2010-2011 Richard Lincoln
   2 # Copyright (C) 2011 Stefan Scherfke
   3 # Copyright (C) 2015 Wouter Labeeuw
   4 # Copyright (C) 2016 Konstantin Krumov Gerasimov, KKG - kkgerasimov@gmail.com
   5 #
   6 # Permission is hereby granted, free of charge, to any person obtaining a copy
   7 # of this software and associated documentation files (the "Software"), to
   8 # deal in the Software without restriction, including without limitation the
   9 # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
  10 # sell copies of the Software, and to permit persons to whom the Software is
  11 # furnished to do so, subject to the following conditions:
  12 #
  13 # The above copyright notice and this permission notice shall be included in
  14 # all copies or substantial portions of the Software.
  15 #
  16 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  19 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  21 # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  22 # IN THE SOFTWARE.
  23
  24 from xml.etree.cElementTree import iterparse
  25 from time import time
  26
  27 import logging
  28 logger = logging.getLogger(__name__)
  29
  30
  31 def cimread(source, packageMap=None, nsURI=None, start_dict=None):
  32     """ CIM RDF/XML parser.
  33
  34     @type source: File-like object or a path to a file.
  35     @param source: CIM RDF/XML file.
  36     @type profile: dict
  37     @param packageMap: Map of class name to PyCIM package name. All CIM
  38     classes are under the one namespace, but are arranged into sub-packages
  39     so a map from class name to package name is required. Defaults to the
  40     latest CIM version, but may be set to a map from a profile to return
  41     a profile model.
  42     @type profile: string
  43     @param nsURI: CIM namespace URI used in the RDF/XML file. For example:
  44     http://iec.ch/TC57/2010/CIM-schema-cim15
  45     @rtype: dict
  46     @return: Map of UUID to CIM object.
  47
  48     @author: Richard Lincoln <r.w.lincoln@gmail.com>
  49     """
  50     # Start the clock.
  51     t0 = time()
  52
  53     #logger.info('##########################################################################')
  54     logger.info('START of parsing file \"%s\"', source)
  55     logger_errors_grouped = {}
  56
  57     # A map of uuids to CIM objects to be returned.
  58     d = start_dict if start_dict is not None else {}
  59
  60     # Obtain the namespaces from the input file
  61     namespaces = xmlns(source)
  62     ns_rdf = get_rdf_ns(namespaces)
  63     if bool(nsURI) != bool(packageMap):
  64         raise ValueError(
  65                 'Either pass "packageMap" AND "nsURI" or none of them.')
  66     elif (nsURI is None) and (packageMap is None):
  67         nsURI, packageMap = get_cim_ns(namespaces)
  68
  69     # CIM element tag base (e.g. {http://iec.ch/TC57/2009/CIM-schema-cim14#}).
  70     base = "{%s#}" % nsURI
  71     # Length of element tag base.
  72     m = len(base)
  73
  74     # First pass instantiates the classes.
  75     context = iterparse(source, ("start", "end"))
  76
  77     # Turn it into an iterator (required for cElementTree).
  78     context = iter(context)
  79
  80     # Get the root element ({http://www.w3.org/1999/02/22-rdf-syntax-ns#}RDF).
  81     _, root = next(context)
  82
  83     for event, elem in context:
  84         # Process 'end' elements in the CIM namespace.
  85         if event == "end" and elem.tag[:m] == base:
  86
  87             # Unique resource identifier for the CIM object.
  88             uuid = elem.get("{%s}ID" % ns_rdf)
  89             if uuid is not None: # class
  90                 # Element tag without namespace (e.g. VoltageLevel).
  91                 tag = elem.tag[m:]
  92                 try:
  93                     mname = packageMap[tag]
  94                 except KeyError:
  95                     logger.error("Unable to locate module for: %s (%s)",
  96                                  tag, uuid)
  97                     root.clear()
  98                     continue
  99                 # Import the module for the CIM object.
 100                 module = __import__(mname, globals(), locals(), [tag], 0)
 101                 # Get the CIM class from the module.
 102                 klass = getattr(module, tag)
 103
 104                 # Instantiate the class and map it to the uuid.
 105                 d[uuid] = klass(UUID=uuid)
 106
 107         # Clear children of the root element to minimise memory usage.
 108         root.clear()
 109
 110     # Reset stream
 111     if hasattr(source, "seek"):
 112         source.seek(0)
 113
 114     ## Second pass sets attributes and references.
 115     context = iter( iterparse(source, ("start", "end")) )
 116
 117     # Get the root element ({http://www.w3.org/1999/02/22-rdf-syntax-ns#}RDF).
 118     _, root = next(context)
 119
 120     for event, elem in context:
 121         # Process 'start' elements in the CIM namespace.
 122         if event == "start" and elem.tag[:m] == base:
 123             uuid = elem.get("{%s}ID" % ns_rdf)
 124             if uuid is None:
 125                 uuid = elem.get("{%s}about" % ns_rdf)
 126                 if uuid is not None:
 127                     uuid = uuid[1:]
 128             if uuid is not None:
 129                 # Locate the CIM object using the uuid.
 130                 try:
 131                     obj = d[uuid]
 132                 except KeyError:
 133                     logger.error("Missing '%s' object with uuid: %s",
 134                                  elem.tag[m:], uuid)
 135                     root.clear()
 136                     continue
 137
 138                 # Iterate over attributes/references.
 139                 for event, elem in context:
 140                     # Process end events with elements in the CIM namespace.
 141                     if event == "end" and elem.tag[:m] == base:
 142                         # Break if class closing element (e.g. </cim:Terminal>).
 143                         if elem.get("{%s}ID" % ns_rdf) is None and \
 144                                 elem.get("{%s}about" % ns_rdf) is None:
 145                             # Get the attribute/reference name.
 146                             attr = elem.tag[m:].rsplit(".")[-1]
 147
 148                             if not hasattr(obj, attr):
 149                                 error_msg = "'%s' has not attribute '%s'" %(obj.__class__.__name__, attr)
 150                                 try:
 151                                     logger_errors_grouped[error_msg] += 1
 152                                 except KeyError:
 153                                     logger_errors_grouped[error_msg] = 1
 154                                 # logger.error("'%s' has not attribute '%s'",
 155                                 #              obj.__class__.__name__, attr)
 156                                 continue
 157
 158                             # Use the rdf:resource attribute to distinguish
 159                             # between attributes and references/enums.
 160                             uuid2 = elem.get("{%s}resource" % ns_rdf)
 161
 162                             if uuid2 is None: # attribute
 163                                 # Convert value type using the default value.
 164                                 try:
 165                                     typ = type( getattr(obj, attr) )
 166                                     if typ == type(True): # KKG: Test if it is boolean value
 167                                         # KKG: NB: The function bool("false") returns True, because it is called upon non-empty string!
 168                                         # This means that it wrongly reads "false" value as boolean True and this is why this special case testing is necessary
 169                                         if str.title(elem.text) == 'True':
 170                                             setattr(obj, attr, True)
 171                                         else:
 172                                             setattr(obj, attr, False)
 173                                     else:
 174                                         setattr(obj, attr, typ(elem.text))
 175                                 except TypeError:
 176                                     pass
 177                             else:  # reference or enum
 178                                 # Use the '#' prefix to distinguish between
 179                                 # references and enumerations.
 180                                 if uuid2[0] == "#": # reference
 181                                     try:
 182                                         val = d[uuid2[1:]] # remove '#' prefix
 183                                     except KeyError:
 184                                         logger.error("Referenced '%s' [%s] "
 185                                                      "object missing.",
 186                                                      obj.__class__.__name__,
 187                                                      uuid2[1:])
 188                                         continue
 189
 190                                     default = getattr(obj, attr)
 191                                     if default == None: # 1..1 or 1..n
 192                                         # Rely on properties to set any
 193                                         # bi-directional references.
 194                                         setattr(obj, attr, val)
 195                                     elif isinstance(default, list): # many
 196                                         # Use 'add*' method to set reference.
 197                                         getattr(obj, ("add%s" % attr))(val)
 198 #                                    else:
 199 #                                        logger.error("Reference error [%s].",
 200 #                                                     default)
 201
 202                                 else: # enum
 203                                     val = uuid2.rsplit(".", 1)[1]
 204                                     setattr(obj, attr, val)
 205
 206                         else:
 207                             # Finished setting object attributes.
 208                             break
 209
 210         # Clear children of the root element to minimise memory usage.
 211         root.clear()
 212
 213     if logger_errors_grouped:
 214         for error, count in logger_errors_grouped.items():
 215             logging_message = '%s : %d times' %(error, count)
 216             logger.warn(logging_message)
 217
 218     # logging_message = 'Created totally %d CIM objects in %.2fs.' %(len(d), time() - t0)
 219     logger.info('Created totally %d CIM objects in %.2fs.' %(len(d), time() - t0))
 220     # logging_message = 'END of parsing file \"%s\"\n' % source
 221     logger.info('END of parsing file \"%s\"\n' % source)
 222
 223     return d
 224
 225
 226 def xmlns(source):
 227     """
 228     Returns a map of prefix to namespace for the given XML file.
 229
 230     """
 231     namespaces = {}
 232     events=("end", "start-ns", "end-ns")
 233     for (event, elem) in iterparse(source, events):
 234         if event == "start-ns":
 235             prefix, ns = elem
 236             namespaces[prefix] = ns
 237         elif event == "end":
 238             break
 239
 240     # Reset stream
 241     if hasattr(source, "seek"):
 242         source.seek(0)
 243
 244     return namespaces
 245
 246
 247 def get_rdf_ns(namespaces):
 248     try:
 249         ns = namespaces['rdf']
 250     except KeyError:
 251         ns = "http://www.w3.org/1999/02/22-rdf-syntax-ns#"
 252         logger.warn('No rdf namespace found. Using %s' % ns)
 253
 254     return ns
 255
 256
 257 def get_cim_ns(namespaces):
 258     """
 259     Tries to obtain the CIM version from the given map of namespaces and
 260     returns the appropriate *nsURI* and *packageMap*.
 261
 262     """
 263     try:
 264         ns = namespaces['cim']
 265         if ns.endswith('#'):
 266             ns = ns[:-1]
 267     except KeyError:
 268         ns = ''
 269         logger.error('No CIM namespace defined in input file.')
 270
 271     CIM16nsURI = 'http://iec.ch/TC57/2013/CIM-schema-cim16'
 272
 273     nsuri = ns
 274
 275     import CIM14, CIM15
 276     if ns == CIM14.nsURI:
 277         ns = 'CIM14'
 278     elif ns == CIM15.nsURI:
 279         ns = 'CIM15'
 280     elif ns == CIM16nsURI:
 281         ns  = 'CIM15'
 282     else:
 283         ns = 'CIM15'
 284         logger.warn('Could not detect CIM version. Using %s.' % ns)
 285
 286     cim = __import__(ns, globals(), locals(), ['nsURI', 'packageMap'])
 287
 288     return nsuri, cim.packageMap
 289
 290
 291 if __name__ == "__main__":
 292     logging.basicConfig(level=logging.INFO)
 293     cimread("Test/Data/EDF_AIGUE_v9_COMBINED.xml")