1 """create portable serialized representations of Python objects.
3 See module cPickle for a (much) faster implementation.
4 See module copy_reg for a mechanism for registering custom picklers.
14 dumps(object) -> string
16 loads(string) -> object
26 __version__
= "$Revision$" # Code version
29 from copy_reg
import dispatch_table
, safe_constructors
35 format_version
= "1.3" # File format version we write
36 compatible_formats
= ["1.0", "1.1", "1.2"] # Old format versions we can read
38 mdumps
= marshal
.dumps
39 mloads
= marshal
.loads
41 PicklingError
= "pickle.PicklingError"
42 UnpicklingError
= "pickle.UnpicklingError"
45 from org
.python
.core
import PyStringMap
91 def __init__(self
, file, bin
= 0):
92 self
.write
= file.write
96 def dump(self
, object):
106 return LONG_BINPUT
+ s
108 return PUT
+ `i`
+ '\n'
117 return LONG_BINGET
+ s
119 return GET
+ `i`
+ '\n'
121 def save(self
, object, pers_save
= 0):
125 pid
= self
.persistent_id(object)
126 if (pid
is not None):
134 if ((t
is TupleType
) and (len(object) == 0)):
136 self
.save_empty_tuple(object)
138 self
.save_tuple(object)
142 self
.write(self
.get(memo
[d
][0]))
148 pid
= self
.inst_persistent_id(object)
154 reduce = dispatch_table
[t
]
157 reduce = object.__reduce
__
158 except AttributeError:
159 raise PicklingError
, \
160 "can't pickle %s objects" % `t
.__name
__`
166 if type(tup
) is StringType
:
167 self
.save_global(object, tup
)
170 if (type(tup
) is not TupleType
):
171 raise PicklingError
, "Value returned by %s must be a " \
176 if ((l
!= 2) and (l
!= 3)):
177 raise PicklingError
, "tuple returned by %s must contain " \
178 "only two or three elements" % reduce
188 if type(arg_tup
) is not TupleType
and arg_tup
is not None:
189 raise PicklingError
, "Second element of tuple returned " \
190 "by %s must be a tuple" % reduce
192 self
.save_reduce(callable, arg_tup
, state
)
194 self
.write(self
.put(memo_len
))
195 memo
[d
] = (memo_len
, object)
200 def persistent_id(self
, object):
203 def inst_persistent_id(self
, object):
206 def save_pers(self
, pid
):
208 self
.write(PERSID
+ str(pid
) + '\n')
211 self
.write(BINPERSID
)
213 def save_reduce(self
, callable, arg_tup
, state
= None):
221 if (state
is not None):
227 def save_none(self
, object):
229 dispatch
[NoneType
] = save_none
231 def save_int(self
, object):
233 i
= mdumps(object)[1:]
234 if (i
[-2:] == '\000\000'):
235 if (i
[-3] == '\000'):
236 self
.write(BININT1
+ i
[:-3])
239 self
.write(BININT2
+ i
[:-2])
242 self
.write(BININT
+ i
)
244 self
.write(INT
+ `
object`
+ '\n')
245 dispatch
[IntType
] = save_int
247 def save_long(self
, object):
248 self
.write(LONG
+ `
object`
+ '\n')
249 dispatch
[LongType
] = save_long
251 def save_float(self
, object, pack
=struct
.pack
):
253 self
.write(BINFLOAT
+ pack('>d', object))
255 self
.write(FLOAT
+ `
object`
+ '\n')
256 dispatch
[FloatType
] = save_float
258 def save_string(self
, object):
266 self
.write(SHORT_BINSTRING
+ s
[0] + object)
268 self
.write(BINSTRING
+ s
+ object)
270 self
.write(STRING
+ `
object`
+ '\n')
273 self
.write(self
.put(memo_len
))
274 memo
[d
] = (memo_len
, object)
275 dispatch
[StringType
] = save_string
277 def save_tuple(self
, object):
287 for element
in object:
290 if (len(object) and memo
.has_key(d
)):
292 write(POP_MARK
+ self
.get(memo
[d
][0]))
295 write(POP
* (len(object) + 1) + self
.get(memo
[d
][0]))
299 self
.write(TUPLE
+ self
.put(memo_len
))
300 memo
[d
] = (memo_len
, object)
301 dispatch
[TupleType
] = save_tuple
303 def save_empty_tuple(self
, object):
304 self
.write(EMPTY_TUPLE
)
306 def save_list(self
, object):
319 write(self
.put(memo_len
))
320 memo
[d
] = (memo_len
, object)
322 using_appends
= (self
.bin
and (len(object) > 1))
327 for element
in object:
330 if (not using_appends
):
335 dispatch
[ListType
] = save_list
337 def save_dict(self
, object):
350 self
.write(self
.put(memo_len
))
351 memo
[d
] = (memo_len
, object)
353 using_setitems
= (self
.bin
and (len(object) > 1))
358 items
= object.items()
359 for key
, value
in items
:
363 if (not using_setitems
):
369 dispatch
[DictionaryType
] = save_dict
370 if not PyStringMap
is None:
371 dispatch
[PyStringMap
] = save_dict
373 def save_inst(self
, object):
375 cls
= object.__class
__
381 if hasattr(object, '__getinitargs__'):
382 args
= object.__getinitargs
__()
383 len(args
) # XXX Assert it's a sequence
384 _keep_alive(args
, memo
)
398 write(OBJ
+ self
.put(memo_len
))
400 write(INST
+ cls
.__module
__ + '\n' + cls
.__name
__ + '\n' +
403 memo
[d
] = (memo_len
, object)
406 getstate
= object.__getstate
__
407 except AttributeError:
408 stuff
= object.__dict
__
411 _keep_alive(stuff
, memo
)
414 dispatch
[InstanceType
] = save_inst
416 def save_global(self
, object, name
= None):
421 name
= object.__name
__
424 module
= object.__module
__
425 except AttributeError:
426 module
= whichmodule(object, name
)
429 write(GLOBAL
+ module
+ '\n' + name
+ '\n' +
431 memo
[id(object)] = (memo_len
, object)
432 dispatch
[ClassType
] = save_global
433 dispatch
[FunctionType
] = save_global
434 dispatch
[BuiltinFunctionType
] = save_global
437 def _keep_alive(x
, memo
):
438 """Keeps a reference to the object x in the memo.
440 Because we remember objects by their id, we have
441 to assure that possibly temporary objects are kept
442 alive by referencing them.
443 We store a reference at the id of the memo, which should
444 normally not be used unless someone tries to deepcopy
448 memo
[id(memo
)].append(x
)
450 # aha, this is the first one :-)
456 # This is no longer used to find classes, but still for functions
457 def whichmodule(cls
, clsname
):
458 """Figure out the module in which a class occurs.
460 Search sys.modules for the module.
462 Return a module name.
463 If the class cannot be found, return __main__.
465 if classmap
.has_key(cls
):
468 for name
, module
in sys
.modules
.items():
469 if name
!= '__main__' and \
470 hasattr(module
, clsname
) and \
471 getattr(module
, clsname
) is cls
:
481 def __init__(self
, file):
482 self
.readline
= file.readline
483 self
.read
= file.read
487 self
.mark
= ['spam'] # Any new unique object
489 self
.append
= self
.stack
.append
491 dispatch
= self
.dispatch
503 while stack
[k
] is not mark
: k
= k
-1
510 dispatch
[''] = load_eof
512 def load_persid(self
):
513 pid
= self
.readline()[:-1]
514 self
.append(self
.persistent_load(pid
))
515 dispatch
[PERSID
] = load_persid
517 def load_binpersid(self
):
523 self
.append(self
.persistent_load(pid
))
524 dispatch
[BINPERSID
] = load_binpersid
528 dispatch
[NONE
] = load_none
531 self
.append(string
.atoi(self
.readline()[:-1]))
532 dispatch
[INT
] = load_int
534 def load_binint(self
):
535 self
.append(mloads('i' + self
.read(4)))
536 dispatch
[BININT
] = load_binint
538 def load_binint1(self
):
539 self
.append(mloads('i' + self
.read(1) + '\000\000\000'))
540 dispatch
[BININT1
] = load_binint1
542 def load_binint2(self
):
543 self
.append(mloads('i' + self
.read(2) + '\000\000'))
544 dispatch
[BININT2
] = load_binint2
547 self
.append(string
.atol(self
.readline()[:-1], 0))
548 dispatch
[LONG
] = load_long
550 def load_float(self
):
551 self
.append(string
.atof(self
.readline()[:-1]))
552 dispatch
[FLOAT
] = load_float
554 def load_binfloat(self
, unpack
=struct
.unpack
):
555 self
.append(unpack('>d', self
.read(8))[0])
556 dispatch
[BINFLOAT
] = load_binfloat
558 def load_string(self
):
559 self
.append(eval(self
.readline()[:-1],
560 {'__builtins__': {}})) # Let's be careful
561 dispatch
[STRING
] = load_string
563 def load_binstring(self
):
564 len = mloads('i' + self
.read(4))
565 self
.append(self
.read(len))
566 dispatch
[BINSTRING
] = load_binstring
568 def load_short_binstring(self
):
569 len = mloads('i' + self
.read(1) + '\000\000\000')
570 self
.append(self
.read(len))
571 dispatch
[SHORT_BINSTRING
] = load_short_binstring
573 def load_tuple(self
):
575 self
.stack
[k
:] = [tuple(self
.stack
[k
+1:])]
576 dispatch
[TUPLE
] = load_tuple
578 def load_empty_tuple(self
):
579 self
.stack
.append(())
580 dispatch
[EMPTY_TUPLE
] = load_empty_tuple
582 def load_empty_list(self
):
583 self
.stack
.append([])
584 dispatch
[EMPTY_LIST
] = load_empty_list
586 def load_empty_dictionary(self
):
587 self
.stack
.append({})
588 dispatch
[EMPTY_DICT
] = load_empty_dictionary
592 self
.stack
[k
:] = [self
.stack
[k
+1:]]
593 dispatch
[LIST
] = load_list
598 items
= self
.stack
[k
+1:]
599 for i
in range(0, len(items
), 2):
604 dispatch
[DICT
] = load_dict
608 args
= tuple(self
.stack
[k
+1:])
610 module
= self
.readline()[:-1]
611 name
= self
.readline()[:-1]
612 klass
= self
.find_class(module
, name
)
614 if (not args
and type(klass
) is ClassType
and
615 not hasattr(klass
, "__getinitargs__")):
617 value
= _EmptyClass()
618 value
.__class
__ = klass
621 # In restricted execution, assignment to inst.__class__ is
626 value
= apply(klass
, args
)
627 except TypeError, err
:
628 raise TypeError, "in constructor for %s: %s" % (
629 klass
.__name
__, str(err
)), sys
.exc_info()[2]
631 dispatch
[INST
] = load_inst
638 args
= tuple(stack
[k
+ 1:])
641 if (not args
and type(klass
) is ClassType
and
642 not hasattr(klass
, "__getinitargs__")):
644 value
= _EmptyClass()
645 value
.__class
__ = klass
648 # In restricted execution, assignment to inst.__class__ is
652 value
= apply(klass
, args
)
654 dispatch
[OBJ
] = load_obj
656 def load_global(self
):
657 module
= self
.readline()[:-1]
658 name
= self
.readline()[:-1]
659 klass
= self
.find_class(module
, name
)
661 dispatch
[GLOBAL
] = load_global
663 def find_class(self
, module
, name
):
666 mod
= sys
.modules
[module
]
667 klass
= getattr(mod
, name
)
668 except (ImportError, KeyError, AttributeError):
670 "Failed to import class %s from module %s" % \
674 def load_reduce(self
):
681 if type(callable) is not ClassType
:
682 if not safe_constructors
.has_key(callable):
684 safe
= callable.__safe
_for
_unpickling
__
685 except AttributeError:
689 raise UnpicklingError
, "%s is not safe for " \
690 "unpickling" % callable
693 value
= callable.__basicnew
__()
695 value
= apply(callable, arg_tup
)
697 dispatch
[REDUCE
] = load_reduce
701 dispatch
[POP
] = load_pop
703 def load_pop_mark(self
):
706 dispatch
[POP_MARK
] = load_pop_mark
709 self
.append(self
.stack
[-1])
710 dispatch
[DUP
] = load_dup
713 self
.append(self
.memo
[self
.readline()[:-1]])
714 dispatch
[GET
] = load_get
716 def load_binget(self
):
717 i
= mloads('i' + self
.read(1) + '\000\000\000')
718 self
.append(self
.memo
[`i`
])
719 dispatch
[BINGET
] = load_binget
721 def load_long_binget(self
):
722 i
= mloads('i' + self
.read(4))
723 self
.append(self
.memo
[`i`
])
724 dispatch
[LONG_BINGET
] = load_long_binget
727 self
.memo
[self
.readline()[:-1]] = self
.stack
[-1]
728 dispatch
[PUT
] = load_put
730 def load_binput(self
):
731 i
= mloads('i' + self
.read(1) + '\000\000\000')
732 self
.memo
[`i`
] = self
.stack
[-1]
733 dispatch
[BINPUT
] = load_binput
735 def load_long_binput(self
):
736 i
= mloads('i' + self
.read(4))
737 self
.memo
[`i`
] = self
.stack
[-1]
738 dispatch
[LONG_BINPUT
] = load_long_binput
740 def load_append(self
):
746 dispatch
[APPEND
] = load_append
748 def load_appends(self
):
751 list = stack
[mark
- 1]
752 for i
in range(mark
+ 1, len(stack
)):
753 list.append(stack
[i
])
756 dispatch
[APPENDS
] = load_appends
758 def load_setitem(self
):
765 dispatch
[SETITEM
] = load_setitem
767 def load_setitems(self
):
770 dict = stack
[mark
- 1]
771 for i
in range(mark
+ 1, len(stack
), 2):
772 dict[stack
[i
]] = stack
[i
+ 1]
775 dispatch
[SETITEMS
] = load_setitems
777 def load_build(self
):
783 setstate
= inst
.__setstate
__
784 except AttributeError:
786 inst
.__dict
__.update(value
)
788 # XXX In restricted execution, the instance's __dict__ is not
789 # accessible. Use the old way of unpickling the instance
790 # variables. This is a semantic different when unpickling in
791 # restricted vs. unrestricted modes.
792 for k
, v
in value
.items():
796 dispatch
[BUILD
] = load_build
799 self
.append(self
.mark
)
800 dispatch
[MARK
] = load_mark
803 value
= self
.stack
[-1]
806 dispatch
[STOP
] = load_stop
808 # Helper class for load_inst/load_obj
815 from StringIO
import StringIO
817 def dump(object, file, bin
= 0):
818 Pickler(file, bin
).dump(object)
820 def dumps(object, bin
= 0):
822 Pickler(file, bin
).dump(object)
823 return file.getvalue()
826 return Unpickler(file).load()
830 return Unpickler(file).load()
833 # The rest is used for testing only
836 def __cmp__(self
, other
):
837 return cmp(self
.__dict
__, other
.__dict
__)
845 y
= ('abc', 'abc', c
, c
)
864 if __name__
== '__main__':