Bump version to 0.9.1.
[python/dscho.git] / Python / marshal.c
blobf6447131d0edf47dd1e09b8a4e7d3e2341fce9ae
1 /***********************************************************
2 Copyright (c) 2000, BeOpen.com.
3 Copyright (c) 1995-2000, Corporation for National Research Initiatives.
4 Copyright (c) 1990-1995, Stichting Mathematisch Centrum.
5 All rights reserved.
7 See the file "Misc/COPYRIGHT" for information on usage and
8 redistribution of this file, and for a DISCLAIMER OF ALL WARRANTIES.
9 ******************************************************************/
11 /* Write Python objects to files and read them back.
12 This is intended for writing and reading compiled Python code only;
13 a true persistent storage facility would be much harder, since
14 it would have to take circular links and sharing into account. */
16 #include "Python.h"
17 #include "longintrepr.h"
18 #include "compile.h"
19 #include "marshal.h"
21 /* High water mark to determine when the marshalled object is dangerously deep
22 * and risks coring the interpreter. When the object stack gets this deep,
23 * raise an exception instead of continuing.
25 #define MAX_MARSHAL_STACK_DEPTH 5000
27 #define TYPE_NULL '0'
28 #define TYPE_NONE 'N'
29 #define TYPE_ELLIPSIS '.'
30 #define TYPE_INT 'i'
31 #define TYPE_INT64 'I'
32 #define TYPE_FLOAT 'f'
33 #define TYPE_COMPLEX 'x'
34 #define TYPE_LONG 'l'
35 #define TYPE_STRING 's'
36 #define TYPE_TUPLE '('
37 #define TYPE_LIST '['
38 #define TYPE_DICT '{'
39 #define TYPE_CODE 'c'
40 #define TYPE_UNICODE 'u'
41 #define TYPE_UNKNOWN '?'
43 typedef struct {
44 FILE *fp;
45 int error;
46 int depth;
47 /* If fp == NULL, the following are valid: */
48 PyObject *str;
49 char *ptr;
50 char *end;
51 } WFILE;
53 #define w_byte(c, p) if (((p)->fp)) putc((c), (p)->fp); \
54 else if ((p)->ptr != (p)->end) *(p)->ptr++ = (c); \
55 else w_more(c, p)
57 static void
58 w_more(int c, WFILE *p)
60 int size, newsize;
61 if (p->str == NULL)
62 return; /* An error already occurred */
63 size = PyString_Size(p->str);
64 newsize = size + 1024;
65 if (_PyString_Resize(&p->str, newsize) != 0) {
66 p->ptr = p->end = NULL;
68 else {
69 p->ptr = PyString_AS_STRING((PyStringObject *)p->str) + size;
70 p->end =
71 PyString_AS_STRING((PyStringObject *)p->str) + newsize;
72 *p->ptr++ = Py_SAFE_DOWNCAST(c, int, char);
76 static void
77 w_string(char *s, int n, WFILE *p)
79 if (p->fp != NULL) {
80 fwrite(s, 1, n, p->fp);
82 else {
83 while (--n >= 0) {
84 w_byte(*s, p);
85 s++;
90 static void
91 w_short(int x, WFILE *p)
93 w_byte( x & 0xff, p);
94 w_byte((x>> 8) & 0xff, p);
97 static void
98 w_long(long x, WFILE *p)
100 w_byte((int)( x & 0xff), p);
101 w_byte((int)((x>> 8) & 0xff), p);
102 w_byte((int)((x>>16) & 0xff), p);
103 w_byte((int)((x>>24) & 0xff), p);
106 #if SIZEOF_LONG > 4
107 static void
108 w_long64(long x, WFILE *p)
110 w_long(x, p);
111 w_long(x>>32, p);
113 #endif
115 static void
116 w_object(PyObject *v, WFILE *p)
118 int i, n;
119 PyBufferProcs *pb;
121 p->depth++;
123 if (p->depth > MAX_MARSHAL_STACK_DEPTH) {
124 p->error = 2;
126 else if (v == NULL) {
127 w_byte(TYPE_NULL, p);
129 else if (v == Py_None) {
130 w_byte(TYPE_NONE, p);
132 else if (v == Py_Ellipsis) {
133 w_byte(TYPE_ELLIPSIS, p);
135 else if (PyInt_Check(v)) {
136 long x = PyInt_AS_LONG((PyIntObject *)v);
137 #if SIZEOF_LONG > 4
138 long y = x>>31;
139 if (y && y != -1) {
140 w_byte(TYPE_INT64, p);
141 w_long64(x, p);
143 else
144 #endif
146 w_byte(TYPE_INT, p);
147 w_long(x, p);
150 else if (PyLong_Check(v)) {
151 PyLongObject *ob = (PyLongObject *)v;
152 w_byte(TYPE_LONG, p);
153 n = ob->ob_size;
154 w_long((long)n, p);
155 if (n < 0)
156 n = -n;
157 for (i = 0; i < n; i++)
158 w_short(ob->ob_digit[i], p);
160 else if (PyFloat_Check(v)) {
161 extern void PyFloat_AsString(char *, PyFloatObject *);
162 char buf[256]; /* Plenty to format any double */
163 PyFloat_AsString(buf, (PyFloatObject *)v);
164 n = strlen(buf);
165 w_byte(TYPE_FLOAT, p);
166 w_byte(n, p);
167 w_string(buf, n, p);
169 #ifndef WITHOUT_COMPLEX
170 else if (PyComplex_Check(v)) {
171 extern void PyFloat_AsString(char *, PyFloatObject *);
172 char buf[256]; /* Plenty to format any double */
173 PyFloatObject *temp;
174 w_byte(TYPE_COMPLEX, p);
175 temp = (PyFloatObject*)PyFloat_FromDouble(
176 PyComplex_RealAsDouble(v));
177 PyFloat_AsString(buf, temp);
178 Py_DECREF(temp);
179 n = strlen(buf);
180 w_byte(n, p);
181 w_string(buf, n, p);
182 temp = (PyFloatObject*)PyFloat_FromDouble(
183 PyComplex_ImagAsDouble(v));
184 PyFloat_AsString(buf, temp);
185 Py_DECREF(temp);
186 n = strlen(buf);
187 w_byte(n, p);
188 w_string(buf, n, p);
190 #endif
191 else if (PyString_Check(v)) {
192 w_byte(TYPE_STRING, p);
193 n = PyString_GET_SIZE(v);
194 w_long((long)n, p);
195 w_string(PyString_AS_STRING(v), n, p);
197 else if (PyUnicode_Check(v)) {
198 PyObject *utf8;
199 utf8 = PyUnicode_AsUTF8String(v);
200 if (utf8 == NULL) {
201 p->depth--;
202 p->error = 1;
203 return;
205 w_byte(TYPE_UNICODE, p);
206 n = PyString_GET_SIZE(utf8);
207 w_long((long)n, p);
208 w_string(PyString_AS_STRING(utf8), n, p);
209 Py_DECREF(utf8);
211 else if (PyTuple_Check(v)) {
212 w_byte(TYPE_TUPLE, p);
213 n = PyTuple_Size(v);
214 w_long((long)n, p);
215 for (i = 0; i < n; i++) {
216 w_object(PyTuple_GET_ITEM(v, i), p);
219 else if (PyList_Check(v)) {
220 w_byte(TYPE_LIST, p);
221 n = PyList_GET_SIZE(v);
222 w_long((long)n, p);
223 for (i = 0; i < n; i++) {
224 w_object(PyList_GET_ITEM(v, i), p);
227 else if (PyDict_Check(v)) {
228 int pos;
229 PyObject *key, *value;
230 w_byte(TYPE_DICT, p);
231 /* This one is NULL object terminated! */
232 pos = 0;
233 while (PyDict_Next(v, &pos, &key, &value)) {
234 w_object(key, p);
235 w_object(value, p);
237 w_object((PyObject *)NULL, p);
239 else if (PyCode_Check(v)) {
240 PyCodeObject *co = (PyCodeObject *)v;
241 w_byte(TYPE_CODE, p);
242 w_short(co->co_argcount, p);
243 w_short(co->co_nlocals, p);
244 w_short(co->co_stacksize, p);
245 w_short(co->co_flags, p);
246 w_object(co->co_code, p);
247 w_object(co->co_consts, p);
248 w_object(co->co_names, p);
249 w_object(co->co_varnames, p);
250 w_object(co->co_filename, p);
251 w_object(co->co_name, p);
252 w_short(co->co_firstlineno, p);
253 w_object(co->co_lnotab, p);
255 else if ((pb = v->ob_type->tp_as_buffer) != NULL &&
256 pb->bf_getsegcount != NULL &&
257 pb->bf_getreadbuffer != NULL &&
258 (*pb->bf_getsegcount)(v, NULL) == 1)
260 /* Write unknown buffer-style objects as a string */
261 char *s;
262 w_byte(TYPE_STRING, p);
263 n = (*pb->bf_getreadbuffer)(v, 0, (void **)&s);
264 w_long((long)n, p);
265 w_string(s, n, p);
267 else {
268 w_byte(TYPE_UNKNOWN, p);
269 p->error = 1;
272 p->depth--;
275 void
276 PyMarshal_WriteLongToFile(long x, FILE *fp)
278 WFILE wf;
279 wf.fp = fp;
280 wf.error = 0;
281 wf.depth = 0;
282 w_long(x, &wf);
285 void
286 PyMarshal_WriteObjectToFile(PyObject *x, FILE *fp)
288 WFILE wf;
289 wf.fp = fp;
290 wf.error = 0;
291 wf.depth = 0;
292 w_object(x, &wf);
295 typedef WFILE RFILE; /* Same struct with different invariants */
297 #define rs_byte(p) (((p)->ptr != (p)->end) ? (unsigned char)*(p)->ptr++ : EOF)
299 #define r_byte(p) ((p)->fp ? getc((p)->fp) : rs_byte(p))
301 static int
302 r_string(char *s, int n, RFILE *p)
304 if (p->fp != NULL)
305 return fread(s, 1, n, p->fp);
306 if (p->end - p->ptr < n)
307 n = p->end - p->ptr;
308 memcpy(s, p->ptr, n);
309 p->ptr += n;
310 return n;
313 static int
314 r_short(RFILE *p)
316 register short x;
317 x = r_byte(p);
318 x |= r_byte(p) << 8;
319 /* XXX If your short is > 16 bits, add sign-extension here!!! */
320 return x;
323 static long
324 r_long(RFILE *p)
326 register long x;
327 register FILE *fp = p->fp;
328 if (fp) {
329 x = getc(fp);
330 x |= (long)getc(fp) << 8;
331 x |= (long)getc(fp) << 16;
332 x |= (long)getc(fp) << 24;
334 else {
335 x = rs_byte(p);
336 x |= (long)rs_byte(p) << 8;
337 x |= (long)rs_byte(p) << 16;
338 x |= (long)rs_byte(p) << 24;
340 #if SIZEOF_LONG > 4
341 /* Sign extension for 64-bit machines */
342 x <<= (8*sizeof(long) - 32);
343 x >>= (8*sizeof(long) - 32);
344 #endif
345 return x;
348 static long
349 r_long64(RFILE *p)
351 register long x;
352 x = r_long(p);
353 #if SIZEOF_LONG > 4
354 x = (x & 0xFFFFFFFF) | (r_long(p) << 32);
355 #else
356 if (r_long(p) != 0) {
357 PyObject *f = PySys_GetObject("stderr");
358 if (f != NULL)
359 (void) PyFile_WriteString(
360 "Warning: un-marshal 64-bit int in 32-bit mode\n",
363 #endif
364 return x;
367 static PyObject *
368 r_object(RFILE *p)
370 PyObject *v, *v2;
371 long i, n;
372 int type = r_byte(p);
374 switch (type) {
376 case EOF:
377 PyErr_SetString(PyExc_EOFError,
378 "EOF read where object expected");
379 return NULL;
381 case TYPE_NULL:
382 return NULL;
384 case TYPE_NONE:
385 Py_INCREF(Py_None);
386 return Py_None;
388 case TYPE_ELLIPSIS:
389 Py_INCREF(Py_Ellipsis);
390 return Py_Ellipsis;
392 case TYPE_INT:
393 return PyInt_FromLong(r_long(p));
395 case TYPE_INT64:
396 return PyInt_FromLong(r_long64(p));
398 case TYPE_LONG:
400 int size;
401 PyLongObject *ob;
402 n = r_long(p);
403 size = n<0 ? -n : n;
404 ob = _PyLong_New(size);
405 if (ob == NULL)
406 return NULL;
407 ob->ob_size = n;
408 for (i = 0; i < size; i++)
409 ob->ob_digit[i] = r_short(p);
410 return (PyObject *)ob;
413 case TYPE_FLOAT:
415 extern double atof(const char *);
416 char buf[256];
417 double dx;
418 n = r_byte(p);
419 if (r_string(buf, (int)n, p) != n) {
420 PyErr_SetString(PyExc_EOFError,
421 "EOF read where object expected");
422 return NULL;
424 buf[n] = '\0';
425 PyFPE_START_PROTECT("atof", return 0)
426 dx = atof(buf);
427 PyFPE_END_PROTECT(dx)
428 return PyFloat_FromDouble(dx);
431 #ifndef WITHOUT_COMPLEX
432 case TYPE_COMPLEX:
434 extern double atof(const char *);
435 char buf[256];
436 Py_complex c;
437 n = r_byte(p);
438 if (r_string(buf, (int)n, p) != n) {
439 PyErr_SetString(PyExc_EOFError,
440 "EOF read where object expected");
441 return NULL;
443 buf[n] = '\0';
444 PyFPE_START_PROTECT("atof", return 0)
445 c.real = atof(buf);
446 PyFPE_END_PROTECT(c)
447 n = r_byte(p);
448 if (r_string(buf, (int)n, p) != n) {
449 PyErr_SetString(PyExc_EOFError,
450 "EOF read where object expected");
451 return NULL;
453 buf[n] = '\0';
454 PyFPE_START_PROTECT("atof", return 0)
455 c.imag = atof(buf);
456 PyFPE_END_PROTECT(c)
457 return PyComplex_FromCComplex(c);
459 #endif
461 case TYPE_STRING:
462 n = r_long(p);
463 if (n < 0) {
464 PyErr_SetString(PyExc_ValueError, "bad marshal data");
465 return NULL;
467 v = PyString_FromStringAndSize((char *)NULL, n);
468 if (v != NULL) {
469 if (r_string(PyString_AS_STRING(v), (int)n, p) != n) {
470 Py_DECREF(v);
471 v = NULL;
472 PyErr_SetString(PyExc_EOFError,
473 "EOF read where object expected");
476 return v;
478 case TYPE_UNICODE:
480 char *buffer;
482 n = r_long(p);
483 if (n < 0) {
484 PyErr_SetString(PyExc_ValueError, "bad marshal data");
485 return NULL;
487 buffer = PyMem_NEW(char, n);
488 if (buffer == NULL)
489 return PyErr_NoMemory();
490 if (r_string(buffer, (int)n, p) != n) {
491 PyMem_DEL(buffer);
492 PyErr_SetString(PyExc_EOFError,
493 "EOF read where object expected");
494 return NULL;
496 v = PyUnicode_DecodeUTF8(buffer, n, NULL);
497 PyMem_DEL(buffer);
498 return v;
501 case TYPE_TUPLE:
502 n = r_long(p);
503 if (n < 0) {
504 PyErr_SetString(PyExc_ValueError, "bad marshal data");
505 return NULL;
507 v = PyTuple_New((int)n);
508 if (v == NULL)
509 return v;
510 for (i = 0; i < n; i++) {
511 v2 = r_object(p);
512 if ( v2 == NULL ) {
513 Py_DECREF(v);
514 v = NULL;
515 break;
517 PyTuple_SET_ITEM(v, (int)i, v2);
519 return v;
521 case TYPE_LIST:
522 n = r_long(p);
523 if (n < 0) {
524 PyErr_SetString(PyExc_ValueError, "bad marshal data");
525 return NULL;
527 v = PyList_New((int)n);
528 if (v == NULL)
529 return v;
530 for (i = 0; i < n; i++) {
531 v2 = r_object(p);
532 if ( v2 == NULL ) {
533 Py_DECREF(v);
534 v = NULL;
535 break;
537 PyList_SetItem(v, (int)i, v2);
539 return v;
541 case TYPE_DICT:
542 v = PyDict_New();
543 if (v == NULL)
544 return NULL;
545 for (;;) {
546 PyObject *key, *val;
547 key = r_object(p);
548 if (key == NULL)
549 break; /* XXX Assume TYPE_NULL, not an error */
550 val = r_object(p);
551 if (val != NULL)
552 PyDict_SetItem(v, key, val);
553 Py_DECREF(key);
554 Py_XDECREF(val);
556 return v;
558 case TYPE_CODE:
560 int argcount = r_short(p);
561 int nlocals = r_short(p);
562 int stacksize = r_short(p);
563 int flags = r_short(p);
564 PyObject *code = NULL;
565 PyObject *consts = NULL;
566 PyObject *names = NULL;
567 PyObject *varnames = NULL;
568 PyObject *filename = NULL;
569 PyObject *name = NULL;
570 int firstlineno = 0;
571 PyObject *lnotab = NULL;
573 code = r_object(p);
574 if (code) consts = r_object(p);
575 if (consts) names = r_object(p);
576 if (names) varnames = r_object(p);
577 if (varnames) filename = r_object(p);
578 if (filename) name = r_object(p);
579 if (name) {
580 firstlineno = r_short(p);
581 lnotab = r_object(p);
584 if (!PyErr_Occurred()) {
585 v = (PyObject *) PyCode_New(
586 argcount, nlocals, stacksize, flags,
587 code, consts, names, varnames,
588 filename, name, firstlineno, lnotab);
590 else
591 v = NULL;
592 Py_XDECREF(code);
593 Py_XDECREF(consts);
594 Py_XDECREF(names);
595 Py_XDECREF(varnames);
596 Py_XDECREF(filename);
597 Py_XDECREF(name);
598 Py_XDECREF(lnotab);
601 return v;
603 default:
604 /* Bogus data got written, which isn't ideal.
605 This will let you keep working and recover. */
606 PyErr_SetString(PyExc_ValueError, "bad marshal data");
607 return NULL;
612 long
613 PyMarshal_ReadLongFromFile(FILE *fp)
615 RFILE rf;
616 rf.fp = fp;
617 return r_long(&rf);
620 PyObject *
621 PyMarshal_ReadObjectFromFile(FILE *fp)
623 RFILE rf;
624 if (PyErr_Occurred()) {
625 fprintf(stderr, "XXX rd_object called with exception set\n");
626 return NULL;
628 rf.fp = fp;
629 return r_object(&rf);
632 PyObject *
633 PyMarshal_ReadObjectFromString(char *str, int len)
635 RFILE rf;
636 if (PyErr_Occurred()) {
637 fprintf(stderr, "XXX rds_object called with exception set\n");
638 return NULL;
640 rf.fp = NULL;
641 rf.str = NULL;
642 rf.ptr = str;
643 rf.end = str + len;
644 return r_object(&rf);
647 PyObject *
648 PyMarshal_WriteObjectToString(PyObject *x) /* wrs_object() */
650 WFILE wf;
651 wf.fp = NULL;
652 wf.str = PyString_FromStringAndSize((char *)NULL, 50);
653 if (wf.str == NULL)
654 return NULL;
655 wf.ptr = PyString_AS_STRING((PyStringObject *)wf.str);
656 wf.end = wf.ptr + PyString_Size(wf.str);
657 wf.error = 0;
658 wf.depth = 0;
659 w_object(x, &wf);
660 if (wf.str != NULL)
661 _PyString_Resize(&wf.str,
662 (int) (wf.ptr -
663 PyString_AS_STRING((PyStringObject *)wf.str)));
664 if (wf.error) {
665 Py_XDECREF(wf.str);
666 PyErr_SetString(PyExc_ValueError,
667 (wf.error==1)?"unmarshallable object"
668 :"object too deeply nested to marshal");
669 return NULL;
671 return wf.str;
674 /* And an interface for Python programs... */
676 static PyObject *
677 marshal_dump(PyObject *self, PyObject *args)
679 WFILE wf;
680 PyObject *x;
681 PyObject *f;
682 if (!PyArg_ParseTuple(args, "OO:dump", &x, &f))
683 return NULL;
684 if (!PyFile_Check(f)) {
685 PyErr_SetString(PyExc_TypeError,
686 "marshal.dump() 2nd arg must be file");
687 return NULL;
689 wf.fp = PyFile_AsFile(f);
690 wf.str = NULL;
691 wf.ptr = wf.end = NULL;
692 wf.error = 0;
693 wf.depth = 0;
694 w_object(x, &wf);
695 if (wf.error) {
696 PyErr_SetString(PyExc_ValueError,
697 (wf.error==1)?"unmarshallable object"
698 :"object too deeply nested to marshal");
699 return NULL;
701 Py_INCREF(Py_None);
702 return Py_None;
705 static PyObject *
706 marshal_load(PyObject *self, PyObject *args)
708 RFILE rf;
709 PyObject *f;
710 PyObject *v;
711 if (!PyArg_ParseTuple(args, "O:load", &f))
712 return NULL;
713 if (!PyFile_Check(f)) {
714 PyErr_SetString(PyExc_TypeError,
715 "marshal.load() arg must be file");
716 return NULL;
718 rf.fp = PyFile_AsFile(f);
719 rf.str = NULL;
720 rf.ptr = rf.end = NULL;
721 PyErr_Clear();
722 v = r_object(&rf);
723 if (PyErr_Occurred()) {
724 Py_XDECREF(v);
725 v = NULL;
727 return v;
730 static PyObject *
731 marshal_dumps(PyObject *self, PyObject *args)
733 PyObject *x;
734 if (!PyArg_ParseTuple(args, "O:dumps", &x))
735 return NULL;
736 return PyMarshal_WriteObjectToString(x);
739 static PyObject *
740 marshal_loads(PyObject *self, PyObject *args)
742 RFILE rf;
743 PyObject *v;
744 char *s;
745 int n;
746 if (!PyArg_ParseTuple(args, "s#:loads", &s, &n))
747 return NULL;
748 rf.fp = NULL;
749 rf.str = args;
750 rf.ptr = s;
751 rf.end = s + n;
752 PyErr_Clear();
753 v = r_object(&rf);
754 if (PyErr_Occurred()) {
755 Py_XDECREF(v);
756 v = NULL;
758 return v;
761 static PyMethodDef marshal_methods[] = {
762 {"dump", marshal_dump, 1},
763 {"load", marshal_load, 1},
764 {"dumps", marshal_dumps, 1},
765 {"loads", marshal_loads, 1},
766 {NULL, NULL} /* sentinel */
769 void
770 PyMarshal_Init(void)
772 (void) Py_InitModule("marshal", marshal_methods);