1 /***********************************************************
2 Copyright (c) 2000, BeOpen.com.
3 Copyright (c) 1995-2000, Corporation for National Research Initiatives.
4 Copyright (c) 1990-1995, Stichting Mathematisch Centrum.
7 See the file "Misc/COPYRIGHT" for information on usage and
8 redistribution of this file, and for a DISCLAIMER OF ALL WARRANTIES.
9 ******************************************************************/
11 /* Write Python objects to files and read them back.
12 This is intended for writing and reading compiled Python code only;
13 a true persistent storage facility would be much harder, since
14 it would have to take circular links and sharing into account. */
17 #include "longintrepr.h"
21 /* High water mark to determine when the marshalled object is dangerously deep
22 * and risks coring the interpreter. When the object stack gets this deep,
23 * raise an exception instead of continuing.
25 #define MAX_MARSHAL_STACK_DEPTH 5000
29 #define TYPE_ELLIPSIS '.'
31 #define TYPE_INT64 'I'
32 #define TYPE_FLOAT 'f'
33 #define TYPE_COMPLEX 'x'
35 #define TYPE_STRING 's'
36 #define TYPE_TUPLE '('
40 #define TYPE_UNICODE 'u'
41 #define TYPE_UNKNOWN '?'
47 /* If fp == NULL, the following are valid: */
53 #define w_byte(c, p) if (((p)->fp)) putc((c), (p)->fp); \
54 else if ((p)->ptr != (p)->end) *(p)->ptr++ = (c); \
58 w_more(int c
, WFILE
*p
)
62 return; /* An error already occurred */
63 size
= PyString_Size(p
->str
);
64 newsize
= size
+ 1024;
65 if (_PyString_Resize(&p
->str
, newsize
) != 0) {
66 p
->ptr
= p
->end
= NULL
;
69 p
->ptr
= PyString_AS_STRING((PyStringObject
*)p
->str
) + size
;
71 PyString_AS_STRING((PyStringObject
*)p
->str
) + newsize
;
72 *p
->ptr
++ = Py_SAFE_DOWNCAST(c
, int, char);
77 w_string(char *s
, int n
, WFILE
*p
)
80 fwrite(s
, 1, n
, p
->fp
);
91 w_short(int x
, WFILE
*p
)
94 w_byte((x
>> 8) & 0xff, p
);
98 w_long(long x
, WFILE
*p
)
100 w_byte((int)( x
& 0xff), p
);
101 w_byte((int)((x
>> 8) & 0xff), p
);
102 w_byte((int)((x
>>16) & 0xff), p
);
103 w_byte((int)((x
>>24) & 0xff), p
);
108 w_long64(long x
, WFILE
*p
)
116 w_object(PyObject
*v
, WFILE
*p
)
123 if (p
->depth
> MAX_MARSHAL_STACK_DEPTH
) {
126 else if (v
== NULL
) {
127 w_byte(TYPE_NULL
, p
);
129 else if (v
== Py_None
) {
130 w_byte(TYPE_NONE
, p
);
132 else if (v
== Py_Ellipsis
) {
133 w_byte(TYPE_ELLIPSIS
, p
);
135 else if (PyInt_Check(v
)) {
136 long x
= PyInt_AS_LONG((PyIntObject
*)v
);
140 w_byte(TYPE_INT64
, p
);
150 else if (PyLong_Check(v
)) {
151 PyLongObject
*ob
= (PyLongObject
*)v
;
152 w_byte(TYPE_LONG
, p
);
157 for (i
= 0; i
< n
; i
++)
158 w_short(ob
->ob_digit
[i
], p
);
160 else if (PyFloat_Check(v
)) {
161 extern void PyFloat_AsString(char *, PyFloatObject
*);
162 char buf
[256]; /* Plenty to format any double */
163 PyFloat_AsString(buf
, (PyFloatObject
*)v
);
165 w_byte(TYPE_FLOAT
, p
);
169 #ifndef WITHOUT_COMPLEX
170 else if (PyComplex_Check(v
)) {
171 extern void PyFloat_AsString(char *, PyFloatObject
*);
172 char buf
[256]; /* Plenty to format any double */
174 w_byte(TYPE_COMPLEX
, p
);
175 temp
= (PyFloatObject
*)PyFloat_FromDouble(
176 PyComplex_RealAsDouble(v
));
177 PyFloat_AsString(buf
, temp
);
182 temp
= (PyFloatObject
*)PyFloat_FromDouble(
183 PyComplex_ImagAsDouble(v
));
184 PyFloat_AsString(buf
, temp
);
191 else if (PyString_Check(v
)) {
192 w_byte(TYPE_STRING
, p
);
193 n
= PyString_GET_SIZE(v
);
195 w_string(PyString_AS_STRING(v
), n
, p
);
197 else if (PyUnicode_Check(v
)) {
199 utf8
= PyUnicode_AsUTF8String(v
);
205 w_byte(TYPE_UNICODE
, p
);
206 n
= PyString_GET_SIZE(utf8
);
208 w_string(PyString_AS_STRING(utf8
), n
, p
);
211 else if (PyTuple_Check(v
)) {
212 w_byte(TYPE_TUPLE
, p
);
215 for (i
= 0; i
< n
; i
++) {
216 w_object(PyTuple_GET_ITEM(v
, i
), p
);
219 else if (PyList_Check(v
)) {
220 w_byte(TYPE_LIST
, p
);
221 n
= PyList_GET_SIZE(v
);
223 for (i
= 0; i
< n
; i
++) {
224 w_object(PyList_GET_ITEM(v
, i
), p
);
227 else if (PyDict_Check(v
)) {
229 PyObject
*key
, *value
;
230 w_byte(TYPE_DICT
, p
);
231 /* This one is NULL object terminated! */
233 while (PyDict_Next(v
, &pos
, &key
, &value
)) {
237 w_object((PyObject
*)NULL
, p
);
239 else if (PyCode_Check(v
)) {
240 PyCodeObject
*co
= (PyCodeObject
*)v
;
241 w_byte(TYPE_CODE
, p
);
242 w_short(co
->co_argcount
, p
);
243 w_short(co
->co_nlocals
, p
);
244 w_short(co
->co_stacksize
, p
);
245 w_short(co
->co_flags
, p
);
246 w_object(co
->co_code
, p
);
247 w_object(co
->co_consts
, p
);
248 w_object(co
->co_names
, p
);
249 w_object(co
->co_varnames
, p
);
250 w_object(co
->co_filename
, p
);
251 w_object(co
->co_name
, p
);
252 w_short(co
->co_firstlineno
, p
);
253 w_object(co
->co_lnotab
, p
);
255 else if ((pb
= v
->ob_type
->tp_as_buffer
) != NULL
&&
256 pb
->bf_getsegcount
!= NULL
&&
257 pb
->bf_getreadbuffer
!= NULL
&&
258 (*pb
->bf_getsegcount
)(v
, NULL
) == 1)
260 /* Write unknown buffer-style objects as a string */
262 w_byte(TYPE_STRING
, p
);
263 n
= (*pb
->bf_getreadbuffer
)(v
, 0, (void **)&s
);
268 w_byte(TYPE_UNKNOWN
, p
);
276 PyMarshal_WriteLongToFile(long x
, FILE *fp
)
286 PyMarshal_WriteObjectToFile(PyObject
*x
, FILE *fp
)
295 typedef WFILE RFILE
; /* Same struct with different invariants */
297 #define rs_byte(p) (((p)->ptr != (p)->end) ? (unsigned char)*(p)->ptr++ : EOF)
299 #define r_byte(p) ((p)->fp ? getc((p)->fp) : rs_byte(p))
302 r_string(char *s
, int n
, RFILE
*p
)
305 return fread(s
, 1, n
, p
->fp
);
306 if (p
->end
- p
->ptr
< n
)
308 memcpy(s
, p
->ptr
, n
);
319 /* XXX If your short is > 16 bits, add sign-extension here!!! */
327 register FILE *fp
= p
->fp
;
330 x
|= (long)getc(fp
) << 8;
331 x
|= (long)getc(fp
) << 16;
332 x
|= (long)getc(fp
) << 24;
336 x
|= (long)rs_byte(p
) << 8;
337 x
|= (long)rs_byte(p
) << 16;
338 x
|= (long)rs_byte(p
) << 24;
341 /* Sign extension for 64-bit machines */
342 x
<<= (8*sizeof(long) - 32);
343 x
>>= (8*sizeof(long) - 32);
354 x
= (x
& 0xFFFFFFFF) | (r_long(p
) << 32);
356 if (r_long(p
) != 0) {
357 PyObject
*f
= PySys_GetObject("stderr");
359 (void) PyFile_WriteString(
360 "Warning: un-marshal 64-bit int in 32-bit mode\n",
372 int type
= r_byte(p
);
377 PyErr_SetString(PyExc_EOFError
,
378 "EOF read where object expected");
389 Py_INCREF(Py_Ellipsis
);
393 return PyInt_FromLong(r_long(p
));
396 return PyInt_FromLong(r_long64(p
));
404 ob
= _PyLong_New(size
);
408 for (i
= 0; i
< size
; i
++)
409 ob
->ob_digit
[i
] = r_short(p
);
410 return (PyObject
*)ob
;
415 extern double atof(const char *);
419 if (r_string(buf
, (int)n
, p
) != n
) {
420 PyErr_SetString(PyExc_EOFError
,
421 "EOF read where object expected");
425 PyFPE_START_PROTECT("atof", return 0)
427 PyFPE_END_PROTECT(dx
)
428 return PyFloat_FromDouble(dx
);
431 #ifndef WITHOUT_COMPLEX
434 extern double atof(const char *);
438 if (r_string(buf
, (int)n
, p
) != n
) {
439 PyErr_SetString(PyExc_EOFError
,
440 "EOF read where object expected");
444 PyFPE_START_PROTECT("atof", return 0)
448 if (r_string(buf
, (int)n
, p
) != n
) {
449 PyErr_SetString(PyExc_EOFError
,
450 "EOF read where object expected");
454 PyFPE_START_PROTECT("atof", return 0)
457 return PyComplex_FromCComplex(c
);
464 PyErr_SetString(PyExc_ValueError
, "bad marshal data");
467 v
= PyString_FromStringAndSize((char *)NULL
, n
);
469 if (r_string(PyString_AS_STRING(v
), (int)n
, p
) != n
) {
472 PyErr_SetString(PyExc_EOFError
,
473 "EOF read where object expected");
484 PyErr_SetString(PyExc_ValueError
, "bad marshal data");
487 buffer
= PyMem_NEW(char, n
);
489 return PyErr_NoMemory();
490 if (r_string(buffer
, (int)n
, p
) != n
) {
492 PyErr_SetString(PyExc_EOFError
,
493 "EOF read where object expected");
496 v
= PyUnicode_DecodeUTF8(buffer
, n
, NULL
);
504 PyErr_SetString(PyExc_ValueError
, "bad marshal data");
507 v
= PyTuple_New((int)n
);
510 for (i
= 0; i
< n
; i
++) {
517 PyTuple_SET_ITEM(v
, (int)i
, v2
);
524 PyErr_SetString(PyExc_ValueError
, "bad marshal data");
527 v
= PyList_New((int)n
);
530 for (i
= 0; i
< n
; i
++) {
537 PyList_SetItem(v
, (int)i
, v2
);
549 break; /* XXX Assume TYPE_NULL, not an error */
552 PyDict_SetItem(v
, key
, val
);
560 int argcount
= r_short(p
);
561 int nlocals
= r_short(p
);
562 int stacksize
= r_short(p
);
563 int flags
= r_short(p
);
564 PyObject
*code
= NULL
;
565 PyObject
*consts
= NULL
;
566 PyObject
*names
= NULL
;
567 PyObject
*varnames
= NULL
;
568 PyObject
*filename
= NULL
;
569 PyObject
*name
= NULL
;
571 PyObject
*lnotab
= NULL
;
574 if (code
) consts
= r_object(p
);
575 if (consts
) names
= r_object(p
);
576 if (names
) varnames
= r_object(p
);
577 if (varnames
) filename
= r_object(p
);
578 if (filename
) name
= r_object(p
);
580 firstlineno
= r_short(p
);
581 lnotab
= r_object(p
);
584 if (!PyErr_Occurred()) {
585 v
= (PyObject
*) PyCode_New(
586 argcount
, nlocals
, stacksize
, flags
,
587 code
, consts
, names
, varnames
,
588 filename
, name
, firstlineno
, lnotab
);
595 Py_XDECREF(varnames
);
596 Py_XDECREF(filename
);
604 /* Bogus data got written, which isn't ideal.
605 This will let you keep working and recover. */
606 PyErr_SetString(PyExc_ValueError
, "bad marshal data");
613 PyMarshal_ReadLongFromFile(FILE *fp
)
621 PyMarshal_ReadObjectFromFile(FILE *fp
)
624 if (PyErr_Occurred()) {
625 fprintf(stderr
, "XXX rd_object called with exception set\n");
629 return r_object(&rf
);
633 PyMarshal_ReadObjectFromString(char *str
, int len
)
636 if (PyErr_Occurred()) {
637 fprintf(stderr
, "XXX rds_object called with exception set\n");
644 return r_object(&rf
);
648 PyMarshal_WriteObjectToString(PyObject
*x
) /* wrs_object() */
652 wf
.str
= PyString_FromStringAndSize((char *)NULL
, 50);
655 wf
.ptr
= PyString_AS_STRING((PyStringObject
*)wf
.str
);
656 wf
.end
= wf
.ptr
+ PyString_Size(wf
.str
);
661 _PyString_Resize(&wf
.str
,
663 PyString_AS_STRING((PyStringObject
*)wf
.str
)));
666 PyErr_SetString(PyExc_ValueError
,
667 (wf
.error
==1)?"unmarshallable object"
668 :"object too deeply nested to marshal");
674 /* And an interface for Python programs... */
677 marshal_dump(PyObject
*self
, PyObject
*args
)
682 if (!PyArg_ParseTuple(args
, "OO:dump", &x
, &f
))
684 if (!PyFile_Check(f
)) {
685 PyErr_SetString(PyExc_TypeError
,
686 "marshal.dump() 2nd arg must be file");
689 wf
.fp
= PyFile_AsFile(f
);
691 wf
.ptr
= wf
.end
= NULL
;
696 PyErr_SetString(PyExc_ValueError
,
697 (wf
.error
==1)?"unmarshallable object"
698 :"object too deeply nested to marshal");
706 marshal_load(PyObject
*self
, PyObject
*args
)
711 if (!PyArg_ParseTuple(args
, "O:load", &f
))
713 if (!PyFile_Check(f
)) {
714 PyErr_SetString(PyExc_TypeError
,
715 "marshal.load() arg must be file");
718 rf
.fp
= PyFile_AsFile(f
);
720 rf
.ptr
= rf
.end
= NULL
;
723 if (PyErr_Occurred()) {
731 marshal_dumps(PyObject
*self
, PyObject
*args
)
734 if (!PyArg_ParseTuple(args
, "O:dumps", &x
))
736 return PyMarshal_WriteObjectToString(x
);
740 marshal_loads(PyObject
*self
, PyObject
*args
)
746 if (!PyArg_ParseTuple(args
, "s#:loads", &s
, &n
))
754 if (PyErr_Occurred()) {
761 static PyMethodDef marshal_methods
[] = {
762 {"dump", marshal_dump
, 1},
763 {"load", marshal_load
, 1},
764 {"dumps", marshal_dumps
, 1},
765 {"loads", marshal_loads
, 1},
766 {NULL
, NULL
} /* sentinel */
772 (void) Py_InitModule("marshal", marshal_methods
);