2 /* Write Python objects to files and read them back.
3 This is intended for writing and reading compiled Python code only;
4 a true persistent storage facility would be much harder, since
5 it would have to take circular links and sharing into account. */
8 #include "longintrepr.h"
12 /* High water mark to determine when the marshalled object is dangerously deep
13 * and risks coring the interpreter. When the object stack gets this deep,
14 * raise an exception instead of continuing.
16 #define MAX_MARSHAL_STACK_DEPTH 5000
20 #define TYPE_ELLIPSIS '.'
22 #define TYPE_INT64 'I'
23 #define TYPE_FLOAT 'f'
24 #define TYPE_COMPLEX 'x'
26 #define TYPE_STRING 's'
27 #define TYPE_TUPLE '('
31 #define TYPE_UNICODE 'u'
32 #define TYPE_UNKNOWN '?'
38 /* If fp == NULL, the following are valid: */
44 #define w_byte(c, p) if (((p)->fp)) putc((c), (p)->fp); \
45 else if ((p)->ptr != (p)->end) *(p)->ptr++ = (c); \
49 w_more(int c
, WFILE
*p
)
53 return; /* An error already occurred */
54 size
= PyString_Size(p
->str
);
55 newsize
= size
+ 1024;
56 if (_PyString_Resize(&p
->str
, newsize
) != 0) {
57 p
->ptr
= p
->end
= NULL
;
60 p
->ptr
= PyString_AS_STRING((PyStringObject
*)p
->str
) + size
;
62 PyString_AS_STRING((PyStringObject
*)p
->str
) + newsize
;
63 *p
->ptr
++ = Py_SAFE_DOWNCAST(c
, int, char);
68 w_string(char *s
, int n
, WFILE
*p
)
71 fwrite(s
, 1, n
, p
->fp
);
82 w_short(int x
, WFILE
*p
)
85 w_byte((x
>> 8) & 0xff, p
);
89 w_long(long x
, WFILE
*p
)
91 w_byte((int)( x
& 0xff), p
);
92 w_byte((int)((x
>> 8) & 0xff), p
);
93 w_byte((int)((x
>>16) & 0xff), p
);
94 w_byte((int)((x
>>24) & 0xff), p
);
99 w_long64(long x
, WFILE
*p
)
107 w_object(PyObject
*v
, WFILE
*p
)
114 if (p
->depth
> MAX_MARSHAL_STACK_DEPTH
) {
117 else if (v
== NULL
) {
118 w_byte(TYPE_NULL
, p
);
120 else if (v
== Py_None
) {
121 w_byte(TYPE_NONE
, p
);
123 else if (v
== Py_Ellipsis
) {
124 w_byte(TYPE_ELLIPSIS
, p
);
126 else if (PyInt_Check(v
)) {
127 long x
= PyInt_AS_LONG((PyIntObject
*)v
);
131 w_byte(TYPE_INT64
, p
);
141 else if (PyLong_Check(v
)) {
142 PyLongObject
*ob
= (PyLongObject
*)v
;
143 w_byte(TYPE_LONG
, p
);
148 for (i
= 0; i
< n
; i
++)
149 w_short(ob
->ob_digit
[i
], p
);
151 else if (PyFloat_Check(v
)) {
152 extern void PyFloat_AsString(char *, PyFloatObject
*);
153 char buf
[256]; /* Plenty to format any double */
154 PyFloat_AsString(buf
, (PyFloatObject
*)v
);
156 w_byte(TYPE_FLOAT
, p
);
160 #ifndef WITHOUT_COMPLEX
161 else if (PyComplex_Check(v
)) {
162 extern void PyFloat_AsString(char *, PyFloatObject
*);
163 char buf
[256]; /* Plenty to format any double */
165 w_byte(TYPE_COMPLEX
, p
);
166 temp
= (PyFloatObject
*)PyFloat_FromDouble(
167 PyComplex_RealAsDouble(v
));
168 PyFloat_AsString(buf
, temp
);
173 temp
= (PyFloatObject
*)PyFloat_FromDouble(
174 PyComplex_ImagAsDouble(v
));
175 PyFloat_AsString(buf
, temp
);
182 else if (PyString_Check(v
)) {
183 w_byte(TYPE_STRING
, p
);
184 n
= PyString_GET_SIZE(v
);
186 w_string(PyString_AS_STRING(v
), n
, p
);
188 else if (PyUnicode_Check(v
)) {
190 utf8
= PyUnicode_AsUTF8String(v
);
196 w_byte(TYPE_UNICODE
, p
);
197 n
= PyString_GET_SIZE(utf8
);
199 w_string(PyString_AS_STRING(utf8
), n
, p
);
202 else if (PyTuple_Check(v
)) {
203 w_byte(TYPE_TUPLE
, p
);
206 for (i
= 0; i
< n
; i
++) {
207 w_object(PyTuple_GET_ITEM(v
, i
), p
);
210 else if (PyList_Check(v
)) {
211 w_byte(TYPE_LIST
, p
);
212 n
= PyList_GET_SIZE(v
);
214 for (i
= 0; i
< n
; i
++) {
215 w_object(PyList_GET_ITEM(v
, i
), p
);
218 else if (PyDict_Check(v
)) {
220 PyObject
*key
, *value
;
221 w_byte(TYPE_DICT
, p
);
222 /* This one is NULL object terminated! */
224 while (PyDict_Next(v
, &pos
, &key
, &value
)) {
228 w_object((PyObject
*)NULL
, p
);
230 else if (PyCode_Check(v
)) {
231 PyCodeObject
*co
= (PyCodeObject
*)v
;
232 w_byte(TYPE_CODE
, p
);
233 w_short(co
->co_argcount
, p
);
234 w_short(co
->co_nlocals
, p
);
235 w_short(co
->co_stacksize
, p
);
236 w_short(co
->co_flags
, p
);
237 w_object(co
->co_code
, p
);
238 w_object(co
->co_consts
, p
);
239 w_object(co
->co_names
, p
);
240 w_object(co
->co_varnames
, p
);
241 w_object(co
->co_filename
, p
);
242 w_object(co
->co_name
, p
);
243 w_short(co
->co_firstlineno
, p
);
244 w_object(co
->co_lnotab
, p
);
246 else if ((pb
= v
->ob_type
->tp_as_buffer
) != NULL
&&
247 pb
->bf_getsegcount
!= NULL
&&
248 pb
->bf_getreadbuffer
!= NULL
&&
249 (*pb
->bf_getsegcount
)(v
, NULL
) == 1)
251 /* Write unknown buffer-style objects as a string */
253 w_byte(TYPE_STRING
, p
);
254 n
= (*pb
->bf_getreadbuffer
)(v
, 0, (void **)&s
);
259 w_byte(TYPE_UNKNOWN
, p
);
267 PyMarshal_WriteLongToFile(long x
, FILE *fp
)
277 PyMarshal_WriteObjectToFile(PyObject
*x
, FILE *fp
)
286 typedef WFILE RFILE
; /* Same struct with different invariants */
288 #define rs_byte(p) (((p)->ptr != (p)->end) ? (unsigned char)*(p)->ptr++ : EOF)
290 #define r_byte(p) ((p)->fp ? getc((p)->fp) : rs_byte(p))
293 r_string(char *s
, int n
, RFILE
*p
)
296 return fread(s
, 1, n
, p
->fp
);
297 if (p
->end
- p
->ptr
< n
)
299 memcpy(s
, p
->ptr
, n
);
310 /* Sign-extension, in case short greater than 16 bits */
319 register FILE *fp
= p
->fp
;
322 x
|= (long)getc(fp
) << 8;
323 x
|= (long)getc(fp
) << 16;
324 x
|= (long)getc(fp
) << 24;
328 x
|= (long)rs_byte(p
) << 8;
329 x
|= (long)rs_byte(p
) << 16;
330 x
|= (long)rs_byte(p
) << 24;
333 /* Sign extension for 64-bit machines */
334 x
|= -(x
& 0x80000000L
);
345 x
= (x
& 0xFFFFFFFFL
) | (r_long(p
) << 32);
347 if (r_long(p
) != 0) {
348 PyObject
*f
= PySys_GetObject("stderr");
350 (void) PyFile_WriteString(
351 "Warning: un-marshal 64-bit int in 32-bit mode\n",
363 int type
= r_byte(p
);
368 PyErr_SetString(PyExc_EOFError
,
369 "EOF read where object expected");
380 Py_INCREF(Py_Ellipsis
);
384 return PyInt_FromLong(r_long(p
));
387 return PyInt_FromLong(r_long64(p
));
395 ob
= _PyLong_New(size
);
399 for (i
= 0; i
< size
; i
++)
400 ob
->ob_digit
[i
] = r_short(p
);
401 return (PyObject
*)ob
;
406 extern double atof(const char *);
410 if (r_string(buf
, (int)n
, p
) != n
) {
411 PyErr_SetString(PyExc_EOFError
,
412 "EOF read where object expected");
416 PyFPE_START_PROTECT("atof", return 0)
418 PyFPE_END_PROTECT(dx
)
419 return PyFloat_FromDouble(dx
);
422 #ifndef WITHOUT_COMPLEX
425 extern double atof(const char *);
429 if (r_string(buf
, (int)n
, p
) != n
) {
430 PyErr_SetString(PyExc_EOFError
,
431 "EOF read where object expected");
435 PyFPE_START_PROTECT("atof", return 0)
439 if (r_string(buf
, (int)n
, p
) != n
) {
440 PyErr_SetString(PyExc_EOFError
,
441 "EOF read where object expected");
445 PyFPE_START_PROTECT("atof", return 0)
448 return PyComplex_FromCComplex(c
);
455 PyErr_SetString(PyExc_ValueError
, "bad marshal data");
458 v
= PyString_FromStringAndSize((char *)NULL
, n
);
460 if (r_string(PyString_AS_STRING(v
), (int)n
, p
) != n
) {
463 PyErr_SetString(PyExc_EOFError
,
464 "EOF read where object expected");
475 PyErr_SetString(PyExc_ValueError
, "bad marshal data");
478 buffer
= PyMem_NEW(char, n
);
480 return PyErr_NoMemory();
481 if (r_string(buffer
, (int)n
, p
) != n
) {
483 PyErr_SetString(PyExc_EOFError
,
484 "EOF read where object expected");
487 v
= PyUnicode_DecodeUTF8(buffer
, n
, NULL
);
495 PyErr_SetString(PyExc_ValueError
, "bad marshal data");
498 v
= PyTuple_New((int)n
);
501 for (i
= 0; i
< n
; i
++) {
508 PyTuple_SET_ITEM(v
, (int)i
, v2
);
515 PyErr_SetString(PyExc_ValueError
, "bad marshal data");
518 v
= PyList_New((int)n
);
521 for (i
= 0; i
< n
; i
++) {
528 PyList_SetItem(v
, (int)i
, v2
);
540 break; /* XXX Assume TYPE_NULL, not an error */
543 PyDict_SetItem(v
, key
, val
);
551 int argcount
= r_short(p
);
552 int nlocals
= r_short(p
);
553 int stacksize
= r_short(p
);
554 int flags
= r_short(p
);
555 PyObject
*code
= NULL
;
556 PyObject
*consts
= NULL
;
557 PyObject
*names
= NULL
;
558 PyObject
*varnames
= NULL
;
559 PyObject
*filename
= NULL
;
560 PyObject
*name
= NULL
;
562 PyObject
*lnotab
= NULL
;
565 if (code
) consts
= r_object(p
);
566 if (consts
) names
= r_object(p
);
567 if (names
) varnames
= r_object(p
);
568 if (varnames
) filename
= r_object(p
);
569 if (filename
) name
= r_object(p
);
571 firstlineno
= r_short(p
);
572 lnotab
= r_object(p
);
575 if (!PyErr_Occurred()) {
576 v
= (PyObject
*) PyCode_New(
577 argcount
, nlocals
, stacksize
, flags
,
578 code
, consts
, names
, varnames
,
579 filename
, name
, firstlineno
, lnotab
);
586 Py_XDECREF(varnames
);
587 Py_XDECREF(filename
);
595 /* Bogus data got written, which isn't ideal.
596 This will let you keep working and recover. */
597 PyErr_SetString(PyExc_ValueError
, "bad marshal data");
604 PyMarshal_ReadLongFromFile(FILE *fp
)
612 PyMarshal_ReadObjectFromFile(FILE *fp
)
615 if (PyErr_Occurred()) {
616 fprintf(stderr
, "XXX rd_object called with exception set\n");
620 return r_object(&rf
);
624 PyMarshal_ReadObjectFromString(char *str
, int len
)
627 if (PyErr_Occurred()) {
628 fprintf(stderr
, "XXX rds_object called with exception set\n");
635 return r_object(&rf
);
639 PyMarshal_WriteObjectToString(PyObject
*x
) /* wrs_object() */
643 wf
.str
= PyString_FromStringAndSize((char *)NULL
, 50);
646 wf
.ptr
= PyString_AS_STRING((PyStringObject
*)wf
.str
);
647 wf
.end
= wf
.ptr
+ PyString_Size(wf
.str
);
652 _PyString_Resize(&wf
.str
,
654 PyString_AS_STRING((PyStringObject
*)wf
.str
)));
657 PyErr_SetString(PyExc_ValueError
,
658 (wf
.error
==1)?"unmarshallable object"
659 :"object too deeply nested to marshal");
665 /* And an interface for Python programs... */
668 marshal_dump(PyObject
*self
, PyObject
*args
)
673 if (!PyArg_ParseTuple(args
, "OO:dump", &x
, &f
))
675 if (!PyFile_Check(f
)) {
676 PyErr_SetString(PyExc_TypeError
,
677 "marshal.dump() 2nd arg must be file");
680 wf
.fp
= PyFile_AsFile(f
);
682 wf
.ptr
= wf
.end
= NULL
;
687 PyErr_SetString(PyExc_ValueError
,
688 (wf
.error
==1)?"unmarshallable object"
689 :"object too deeply nested to marshal");
697 marshal_load(PyObject
*self
, PyObject
*args
)
702 if (!PyArg_ParseTuple(args
, "O:load", &f
))
704 if (!PyFile_Check(f
)) {
705 PyErr_SetString(PyExc_TypeError
,
706 "marshal.load() arg must be file");
709 rf
.fp
= PyFile_AsFile(f
);
711 rf
.ptr
= rf
.end
= NULL
;
714 if (PyErr_Occurred()) {
722 marshal_dumps(PyObject
*self
, PyObject
*args
)
725 if (!PyArg_ParseTuple(args
, "O:dumps", &x
))
727 return PyMarshal_WriteObjectToString(x
);
731 marshal_loads(PyObject
*self
, PyObject
*args
)
737 if (!PyArg_ParseTuple(args
, "s#:loads", &s
, &n
))
745 if (PyErr_Occurred()) {
752 static PyMethodDef marshal_methods
[] = {
753 {"dump", marshal_dump
, 1},
754 {"load", marshal_load
, 1},
755 {"dumps", marshal_dumps
, 1},
756 {"loads", marshal_loads
, 1},
757 {NULL
, NULL
} /* sentinel */
763 (void) Py_InitModule("marshal", marshal_methods
);