lib/yaml/constructor.py

   1
   2 __all__ = ['BaseConstructor', 'SafeConstructor', 'Constructor',
   3     'ConstructorError']
   4
   5 from error import *
   6 from nodes import *
   7 from composer import *
   8
   9 try:
  10     import datetime
  11     datetime_available = True
  12 except ImportError:
  13     datetime_available = False
  14
  15 try:
  16     set
  17 except NameError:
  18     from sets import Set as set
  19
  20 import binascii, re, sys
  21
  22 class ConstructorError(MarkedYAMLError):
  23     pass
  24
  25 class BaseConstructor(Composer):
  26
  27     yaml_constructors = {}
  28     yaml_multi_constructors = {}
  29
  30     def __init__(self):
  31         self.constructed_objects = {}
  32         self.recursive_objects = {}
  33
  34     def check_data(self):
  35         # If there are more documents available?
  36         return self.check_node()
  37
  38     def get_data(self):
  39         # Construct and return the next document.
  40         if self.check_node():
  41             return self.construct_document(self.get_node())
  42
  43     def __iter__(self):
  44         # Iterator protocol.
  45         while self.check_node():
  46             yield self.construct_document(self.get_node())
  47
  48     def construct_document(self, node):
  49         data = self.construct_object(node)
  50         self.constructed_objects = {}
  51         self.recursive_objects = {}
  52         return data
  53
  54     def construct_object(self, node):
  55         if node in self.constructed_objects:
  56             return self.constructed_objects[node]
  57         if node in self.recursive_objects:
  58             raise ConstructorError(None, None,
  59                     "found recursive node", node.start_mark)
  60         self.recursive_objects[node] = None
  61         constructor = None
  62         if node.tag in self.yaml_constructors:
  63             constructor = lambda node: self.yaml_constructors[node.tag](self, node)
  64         else:
  65             for tag_prefix in self.yaml_multi_constructors:
  66                 if node.tag.startswith(tag_prefix):
  67                     tag_suffix = node.tag[len(tag_prefix):]
  68                     constructor = lambda node:  \
  69                             self.yaml_multi_constructors[tag_prefix](self, tag_suffix, node)
  70                     break
  71             else:
  72                 if None in self.yaml_multi_constructors:
  73                     constructor = lambda node:  \
  74                             self.yaml_multi_constructors[None](self, node.tag, node)
  75                 elif None in self.yaml_constructors:
  76                     constructor = lambda node:  \
  77                             self.yaml_constructors[None](self, node)
  78                 elif isinstance(node, ScalarNode):
  79                     constructor = self.construct_scalar
  80                 elif isinstance(node, SequenceNode):
  81                     constructor = self.construct_sequence
  82                 elif isinstance(node, MappingNode):
  83                     constructor = self.construct_mapping
  84                 else:
  85                     print node.tag
  86         data = constructor(node)
  87         self.constructed_objects[node] = data
  88         del self.recursive_objects[node]
  89         return data
  90
  91     def construct_scalar(self, node):
  92         if not isinstance(node, ScalarNode):
  93             if isinstance(node, MappingNode):
  94                 for key_node in node.value:
  95                     if key_node.tag == u'tag:yaml.org,2002:value':
  96                         return self.construct_scalar(node.value[key_node])
  97             raise ConstructorError(None, None,
  98                     "expected a scalar node, but found %s" % node.id,
  99                     node.start_mark)
 100         return node.value
 101
 102     def construct_sequence(self, node):
 103         if not isinstance(node, SequenceNode):
 104             raise ConstructorError(None, None,
 105                     "expected a sequence node, but found %s" % node.id,
 106                     node.start_mark)
 107         return [self.construct_object(child) for child in node.value]
 108
 109     def construct_mapping(self, node):
 110         if not isinstance(node, MappingNode):
 111             raise ConstructorError(None, None,
 112                     "expected a mapping node, but found %s" % node.id,
 113                     node.start_mark)
 114         mapping = {}
 115         merge = None
 116         for key_node in node.value:
 117             if key_node.tag == u'tag:yaml.org,2002:merge':
 118                 if merge is not None:
 119                     raise ConstructorError("while constructing a mapping", node.start_mark,
 120                             "found duplicate merge key", key_node.start_mark)
 121                 value_node = node.value[key_node]
 122                 if isinstance(value_node, MappingNode):
 123                     merge = [self.construct_mapping(value_node)]
 124                 elif isinstance(value_node, SequenceNode):
 125                     merge = []
 126                     for subnode in value_node.value:
 127                         if not isinstance(subnode, MappingNode):
 128                             raise ConstructorError("while constructing a mapping",
 129                                     node.start_mark,
 130                                     "expected a mapping for merging, but found %s"
 131                                     % subnode.id, subnode.start_mark)
 132                         merge.append(self.construct_mapping(subnode))
 133                     merge.reverse()
 134                 else:
 135                     raise ConstructorError("while constructing a mapping", node.start_mark,
 136                             "expected a mapping or list of mappings for merging, but found %s"
 137                             % value_node.id, value_node.start_mark)
 138             elif key_node.tag == u'tag:yaml.org,2002:value':
 139                 if '=' in mapping:
 140                     raise ConstructorError("while construction a mapping", node.start_mark,
 141                             "found duplicate value key", key_node.start_mark)
 142                 value = self.construct_object(node.value[key_node])
 143                 mapping['='] = value
 144             else:
 145                 key = self.construct_object(key_node)
 146                 try:
 147                     duplicate_key = key in mapping
 148                 except TypeError, exc:
 149                     raise ConstructorError("while constructing a mapping", node.start_mark,
 150                             "found unacceptable key (%s)" % exc, key_node.start_mark)
 151                 if duplicate_key:
 152                     raise ConstructorError("while constructing a mapping", node.start_mark,
 153                             "found duplicate key", key_node.start_mark)
 154                 value = self.construct_object(node.value[key_node])
 155                 mapping[key] = value
 156         if merge is not None:
 157             merge.append(mapping)
 158             mapping = {}
 159             for submapping in merge:
 160                 mapping.update(submapping)
 161         return mapping
 162
 163     def construct_pairs(self, node):
 164         if not isinstance(node, MappingNode):
 165             raise ConstructorError(None, None,
 166                     "expected a mapping node, but found %s" % node.id,
 167                     node.start_mark)
 168         pairs = []
 169         for key_node in node.value:
 170             key = self.construct_object(key_node)
 171             value = self.construct_object(node.value[key_node])
 172             pairs.append((key, value))
 173         return pairs
 174
 175     def add_constructor(cls, tag, constructor):
 176         if not 'yaml_constructors' in cls.__dict__:
 177             cls.yaml_constructors = cls.yaml_constructors.copy()
 178         cls.yaml_constructors[tag] = constructor
 179     add_constructor = classmethod(add_constructor)
 180
 181     def add_multi_constructor(cls, tag_prefix, multi_constructor):
 182         if not 'yaml_multi_constructors' in cls.__dict__:
 183             cls.yaml_multi_constructors = cls.yaml_multi_constructors.copy()
 184         cls.yaml_multi_constructors[tag_prefix] = multi_constructor
 185     add_multi_constructor = classmethod(add_multi_constructor)
 186
 187 class SafeConstructor(BaseConstructor):
 188
 189     def construct_yaml_null(self, node):
 190         self.construct_scalar(node)
 191         return None
 192
 193     bool_values = {
 194         u'yes':     True,
 195         u'no':      False,
 196         u'true':    True,
 197         u'false':   False,
 198         u'on':      True,
 199         u'off':     False,
 200     }
 201
 202     def construct_yaml_bool(self, node):
 203         value = self.construct_scalar(node)
 204         return self.bool_values[value.lower()]
 205
 206     def construct_yaml_int(self, node):
 207         value = str(self.construct_scalar(node))
 208         value = value.replace('_', '')
 209         sign = +1
 210         if value[0] == '-':
 211             sign = -1
 212         if value[0] in '+-':
 213             value = value[1:]
 214         if value == '0':
 215             return 0
 216         elif value.startswith('0b'):
 217             return sign*int(value[2:], 2)
 218         elif value.startswith('0x'):
 219             return sign*int(value[2:], 16)
 220         elif value[0] == '0':
 221             return sign*int(value, 8)
 222         elif ':' in value:
 223             digits = [int(part) for part in value.split(':')]
 224             digits.reverse()
 225             base = 1
 226             value = 0
 227             for digit in digits:
 228                 value += digit*base
 229                 base *= 60
 230             return sign*value
 231         else:
 232             return sign*int(value)
 233
 234     inf_value = 1e300
 235     while repr(inf_value) != repr(inf_value*inf_value):
 236         inf_value *= inf_value
 237     nan_value = inf_value/inf_value
 238
 239     def construct_yaml_float(self, node):
 240         value = str(self.construct_scalar(node))
 241         value = value.replace('_', '')
 242         sign = +1
 243         if value[0] == '-':
 244             sign = -1
 245         if value[0] in '+-':
 246             value = value[1:]
 247         if value.lower() == '.inf':
 248             return sign*self.inf_value
 249         elif value.lower() == '.nan':
 250             return self.nan_value
 251         elif ':' in value:
 252             digits = [float(part) for part in value.split(':')]
 253             digits.reverse()
 254             base = 1
 255             value = 0.0
 256             for digit in digits:
 257                 value += digit*base
 258                 base *= 60
 259             return sign*value
 260         else:
 261             return sign*float(value)
 262
 263     def construct_yaml_binary(self, node):
 264         value = self.construct_scalar(node)
 265         try:
 266             return str(value).decode('base64')
 267         except (binascii.Error, UnicodeEncodeError), exc:
 268             raise ConstructorError(None, None,
 269                     "failed to decode base64 data: %s" % exc, node.start_mark)
 270
 271     timestamp_regexp = re.compile(
 272             ur'''^(?P<year>[0-9][0-9][0-9][0-9])
 273                 -(?P<month>[0-9][0-9]?)
 274                 -(?P<day>[0-9][0-9]?)
 275                 (?:(?:[Tt]|[ \t]+)
 276                 (?P<hour>[0-9][0-9]?)
 277                 :(?P<minute>[0-9][0-9])
 278                 :(?P<second>[0-9][0-9])
 279                 (?:\.(?P<fraction>[0-9]*))?
 280                 (?:[ \t]*(?:Z|(?P<tz_hour>[-+][0-9][0-9]?)
 281                 (?::(?P<tz_minute>[0-9][0-9])?)?))?)?$''', re.X)
 282
 283     def construct_yaml_timestamp(self, node):
 284         value = self.construct_scalar(node)
 285         match = self.timestamp_regexp.match(node.value)
 286         values = match.groupdict()
 287         for key in values:
 288             if values[key]:
 289                 values[key] = int(values[key])
 290             else:
 291                 values[key] = 0
 292         fraction = values['fraction']
 293         if fraction:
 294             while 10*fraction < 1000000:
 295                 fraction *= 10
 296             values['fraction'] = fraction
 297         stamp = datetime.datetime(values['year'], values['month'], values['day'],
 298                 values['hour'], values['minute'], values['second'], values['fraction'])
 299         diff = datetime.timedelta(hours=values['tz_hour'], minutes=values['tz_minute'])
 300         return stamp-diff
 301
 302     def construct_yaml_omap(self, node):
 303         # Note: we do not check for duplicate keys, because it's too
 304         # CPU-expensive.
 305         if not isinstance(node, SequenceNode):
 306             raise ConstructorError("while constructing an ordered map", node.start_mark,
 307                     "expected a sequence, but found %s" % node.id, node.start_mark)
 308         omap = []
 309         for subnode in node.value:
 310             if not isinstance(subnode, MappingNode):
 311                 raise ConstructorError("while constructing an ordered map", node.start_mark,
 312                         "expected a mapping of length 1, but found %s" % subnode.id,
 313                         subnode.start_mark)
 314             if len(subnode.value) != 1:
 315                 raise ConstructorError("while constructing an ordered map", node.start_mark,
 316                         "expected a single mapping item, but found %d items" % len(subnode.value),
 317                         subnode.start_mark)
 318             key_node = subnode.value.keys()[0]
 319             key = self.construct_object(key_node)
 320             value = self.construct_object(subnode.value[key_node])
 321             omap.append((key, value))
 322         return omap
 323
 324     def construct_yaml_pairs(self, node):
 325         # Note: the same code as `construct_yaml_omap`.
 326         if not isinstance(node, SequenceNode):
 327             raise ConstructorError("while constructing pairs", node.start_mark,
 328                     "expected a sequence, but found %s" % node.id, node.start_mark)
 329         pairs = []
 330         for subnode in node.value:
 331             if not isinstance(subnode, MappingNode):
 332                 raise ConstructorError("while constructing pairs", node.start_mark,
 333                         "expected a mapping of length 1, but found %s" % subnode.id,
 334                         subnode.start_mark)
 335             if len(subnode.value) != 1:
 336                 raise ConstructorError("while constructing pairs", node.start_mark,
 337                         "expected a single mapping item, but found %d items" % len(subnode.value),
 338                         subnode.start_mark)
 339             key_node = subnode.value.keys()[0]
 340             key = self.construct_object(key_node)
 341             value = self.construct_object(subnode.value[key_node])
 342             pairs.append((key, value))
 343         return pairs
 344
 345     def construct_yaml_set(self, node):
 346         value = self.construct_mapping(node)
 347         return set(value)
 348
 349     def construct_yaml_str(self, node):
 350         value = self.construct_scalar(node)
 351         try:
 352             return str(value)
 353         except UnicodeEncodeError:
 354             return value
 355
 356     def construct_yaml_seq(self, node):
 357         return self.construct_sequence(node)
 358
 359     def construct_yaml_map(self, node):
 360         return self.construct_mapping(node)
 361
 362     def construct_yaml_object(self, node, cls):
 363         state = self.construct_mapping(node)
 364         data = cls.__new__(cls)
 365         if hasattr(data, '__setstate__'):
 366             data.__setstate__(state)
 367         else:
 368             data.__dict__.update(state)
 369         return data
 370
 371     def construct_undefined(self, node):
 372         raise ConstructorError(None, None,
 373                 "could not determine a constructor for the tag %r" % node.tag.encode('utf-8'),
 374                 node.start_mark)
 375
 376 SafeConstructor.add_constructor(
 377         u'tag:yaml.org,2002:null',
 378         SafeConstructor.construct_yaml_null)
 379
 380 SafeConstructor.add_constructor(
 381         u'tag:yaml.org,2002:bool',
 382         SafeConstructor.construct_yaml_bool)
 383
 384 SafeConstructor.add_constructor(
 385         u'tag:yaml.org,2002:int',
 386         SafeConstructor.construct_yaml_int)
 387
 388 SafeConstructor.add_constructor(
 389         u'tag:yaml.org,2002:float',
 390         SafeConstructor.construct_yaml_float)
 391
 392 SafeConstructor.add_constructor(
 393         u'tag:yaml.org,2002:binary',
 394         SafeConstructor.construct_yaml_binary)
 395
 396 if datetime_available:
 397     SafeConstructor.add_constructor(
 398             u'tag:yaml.org,2002:timestamp',
 399             SafeConstructor.construct_yaml_timestamp)
 400
 401 SafeConstructor.add_constructor(
 402         u'tag:yaml.org,2002:omap',
 403         SafeConstructor.construct_yaml_omap)
 404
 405 SafeConstructor.add_constructor(
 406         u'tag:yaml.org,2002:pairs',
 407         SafeConstructor.construct_yaml_pairs)
 408
 409 SafeConstructor.add_constructor(
 410         u'tag:yaml.org,2002:set',
 411         SafeConstructor.construct_yaml_set)
 412
 413 SafeConstructor.add_constructor(
 414         u'tag:yaml.org,2002:str',
 415         SafeConstructor.construct_yaml_str)
 416
 417 SafeConstructor.add_constructor(
 418         u'tag:yaml.org,2002:seq',
 419         SafeConstructor.construct_yaml_seq)
 420
 421 SafeConstructor.add_constructor(
 422         u'tag:yaml.org,2002:map',
 423         SafeConstructor.construct_yaml_map)
 424
 425 SafeConstructor.add_constructor(None,
 426         SafeConstructor.construct_undefined)
 427
 428 class Constructor(SafeConstructor):
 429
 430     def construct_python_str(self, node):
 431         return self.construct_scalar(node).encode('utf-8')
 432
 433     def construct_python_unicode(self, node):
 434         return self.construct_scalar(node)
 435
 436     def construct_python_long(self, node):
 437         return long(self.construct_yaml_int(node))
 438
 439     def construct_python_complex(self, node):
 440        return complex(self.construct_scalar(node))
 441
 442     def construct_python_tuple(self, node):
 443         return tuple(self.construct_yaml_seq(node))
 444
 445     def find_python_module(self, name, mark):
 446         if not name:
 447             raise ConstructorError("while constructing a Python module", mark,
 448                     "expected non-empty name appended to the tag", mark)
 449         try:
 450             __import__(name)
 451         except ImportError, exc:
 452             raise ConstructorError("while constructing a Python module", mark,
 453                     "cannot find module %r (%s)" % (name.encode('utf-8'), exc), mark)
 454         return sys.modules[name]
 455
 456     def find_python_name(self, name, mark):
 457         if not name:
 458             raise ConstructorError("while constructing a Python object", mark,
 459                     "expected non-empty name appended to the tag", mark)
 460         if u'.' in name:
 461             # Python 2.4 only
 462             #module_name, object_name = name.rsplit('.', 1)
 463             items = name.split('.')
 464             object_name = items.pop()
 465             module_name = '.'.join(items)
 466         else:
 467             module_name = '__builtin__'
 468             object_name = name
 469         try:
 470             __import__(module_name)
 471         except ImportError, exc:
 472             raise ConstructorError("while constructing a Python object", mark,
 473                     "cannot find module %r (%s)" % (module_name.encode('utf-8'), exc), mark)
 474         module = sys.modules[module_name]
 475         if not hasattr(module, object_name):
 476             raise ConstructorError("while constructing a Python object", mark,
 477                     "cannot find %r in the module %r" % (object_name.encode('utf-8'),
 478                         module.__name__), mark)
 479         return getattr(module, object_name)
 480
 481     def construct_python_name(self, suffix, node):
 482         value = self.construct_scalar(node)
 483         if value:
 484             raise ConstructorError("while constructing a Python name", node.start_mark,
 485                     "expected the empty value, but found %r" % value.encode('utf-8'),
 486                     node.start_mark)
 487         return self.find_python_name(suffix, node.start_mark)
 488
 489     def construct_python_module(self, suffix, node):
 490         value = self.construct_scalar(node)
 491         if value:
 492             raise ConstructorError("while constructing a Python module", node.start_mark,
 493                     "expected the empty value, but found %r" % value.encode('utf-8'),
 494                     node.start_mark)
 495         return self.find_python_module(suffix, node.start_mark)
 496
 497     class classobj: pass
 498
 499     def make_python_instance(self, suffix, node,
 500             args=None, kwds=None, newobj=False):
 501         if not args:
 502             args = []
 503         if not kwds:
 504             kwds = {}
 505         cls = self.find_python_name(suffix, node.start_mark)
 506         if newobj and isinstance(cls, type(self.classobj))  \
 507                 and not args and not kwds:
 508             instance = self.classobj()
 509             instance.__class__ = cls
 510             return instance
 511         elif newobj and isinstance(cls, type):
 512             return cls.__new__(cls, *args, **kwds)
 513         else:
 514             return cls(*args, **kwds)
 515
 516     def set_python_instance_state(self, instance, state):
 517         if hasattr(instance, '__setstate__'):
 518             instance.__setstate__(state)
 519         else:
 520             slotstate = {}
 521             if isinstance(state, tuple) and len(state) == 2:
 522                 state, slotstate = state
 523             if hasattr(instance, '__dict__'):
 524                 instance.__dict__.update(state)
 525             elif state:
 526                 slotstate.update(state)
 527             for key, value in slotstate.items():
 528                 setattr(object, key, value)
 529
 530     def construct_python_object(self, suffix, node):
 531         # Format:
 532         #   !!python/object:module.name { ... state ... }
 533         instance = self.make_python_instance(suffix, node, newobj=True)
 534         state = self.construct_mapping(node)
 535         self.set_python_instance_state(instance, state)
 536         return instance
 537
 538     def construct_python_object_apply(self, suffix, node, newobj=False):
 539         # Format:
 540         #   !!python/object/apply       # (or !!python/object/new)
 541         #   args: [ ... arguments ... ]
 542         #   kwds: { ... keywords ... }
 543         #   state: ... state ...
 544         #   listitems: [ ... listitems ... ]
 545         #   dictitems: { ... dictitems ... }
 546         # or short format:
 547         #   !!python/object/apply [ ... arguments ... ]
 548         # The difference between !!python/object/apply and !!python/object/new
 549         # is how an object is created, check make_python_instance for details.
 550         if isinstance(node, SequenceNode):
 551             args = self.construct_sequence(node)
 552             kwds = {}
 553             state = {}
 554             listitems = []
 555             dictitems = {}
 556         else:
 557             value = self.construct_mapping(node)
 558             args = value.get('args', [])
 559             kwds = value.get('kwds', {})
 560             state = value.get('state', {})
 561             listitems = value.get('listitems', [])
 562             dictitems = value.get('dictitems', {})
 563         instance = self.make_python_instance(suffix, node, args, kwds, newobj)
 564         if state:
 565             self.set_python_instance_state(instance, state)
 566         if listitems:
 567             instance.extend(listitems)
 568         if dictitems:
 569             for key in dictitems:
 570                 instance[key] = dictitems[key]
 571         return instance
 572
 573     def construct_python_object_new(self, suffix, node):
 574         return self.construct_python_object_apply(suffix, node, newobj=True)
 575
 576
 577 Constructor.add_constructor(
 578     u'tag:yaml.org,2002:python/none',
 579     Constructor.construct_yaml_null)
 580
 581 Constructor.add_constructor(
 582     u'tag:yaml.org,2002:python/bool',
 583     Constructor.construct_yaml_bool)
 584
 585 Constructor.add_constructor(
 586     u'tag:yaml.org,2002:python/str',
 587     Constructor.construct_python_str)
 588
 589 Constructor.add_constructor(
 590     u'tag:yaml.org,2002:python/unicode',
 591     Constructor.construct_python_unicode)
 592
 593 Constructor.add_constructor(
 594     u'tag:yaml.org,2002:python/int',
 595     Constructor.construct_yaml_int)
 596
 597 Constructor.add_constructor(
 598     u'tag:yaml.org,2002:python/long',
 599     Constructor.construct_python_long)
 600
 601 Constructor.add_constructor(
 602     u'tag:yaml.org,2002:python/float',
 603     Constructor.construct_yaml_float)
 604
 605 Constructor.add_constructor(
 606     u'tag:yaml.org,2002:python/complex',
 607     Constructor.construct_python_complex)
 608
 609 Constructor.add_constructor(
 610     u'tag:yaml.org,2002:python/list',
 611     Constructor.construct_yaml_seq)
 612
 613 Constructor.add_constructor(
 614     u'tag:yaml.org,2002:python/tuple',
 615     Constructor.construct_python_tuple)
 616
 617 Constructor.add_constructor(
 618     u'tag:yaml.org,2002:python/dict',
 619     Constructor.construct_yaml_map)
 620
 621 Constructor.add_multi_constructor(
 622     u'tag:yaml.org,2002:python/name:',
 623     Constructor.construct_python_name)
 624
 625 Constructor.add_multi_constructor(
 626     u'tag:yaml.org,2002:python/module:',
 627     Constructor.construct_python_module)
 628
 629 Constructor.add_multi_constructor(
 630     u'tag:yaml.org,2002:python/object:',
 631     Constructor.construct_python_object)
 632
 633 Constructor.add_multi_constructor(
 634     u'tag:yaml.org,2002:python/object/apply:',
 635     Constructor.construct_python_object_apply)
 636
 637 Constructor.add_multi_constructor(
 638     u'tag:yaml.org,2002:python/object/new:',
 639     Constructor.construct_python_object_new)
 640