Remove a ?? in the description of Mac OS support.
[python/dscho.git] / Python / marshal.c
blobf8953ceae4c96df5594e6dcb04c62822991f6774
2 /* Write Python objects to files and read them back.
3 This is intended for writing and reading compiled Python code only;
4 a true persistent storage facility would be much harder, since
5 it would have to take circular links and sharing into account. */
7 #include "Python.h"
8 #include "longintrepr.h"
9 #include "compile.h"
10 #include "marshal.h"
12 /* High water mark to determine when the marshalled object is dangerously deep
13 * and risks coring the interpreter. When the object stack gets this deep,
14 * raise an exception instead of continuing.
16 #define MAX_MARSHAL_STACK_DEPTH 5000
18 #define TYPE_NULL '0'
19 #define TYPE_NONE 'N'
20 #define TYPE_ELLIPSIS '.'
21 #define TYPE_INT 'i'
22 #define TYPE_INT64 'I'
23 #define TYPE_FLOAT 'f'
24 #define TYPE_COMPLEX 'x'
25 #define TYPE_LONG 'l'
26 #define TYPE_STRING 's'
27 #define TYPE_TUPLE '('
28 #define TYPE_LIST '['
29 #define TYPE_DICT '{'
30 #define TYPE_CODE 'c'
31 #define TYPE_UNICODE 'u'
32 #define TYPE_UNKNOWN '?'
34 typedef struct {
35 FILE *fp;
36 int error;
37 int depth;
38 /* If fp == NULL, the following are valid: */
39 PyObject *str;
40 char *ptr;
41 char *end;
42 } WFILE;
44 #define w_byte(c, p) if (((p)->fp)) putc((c), (p)->fp); \
45 else if ((p)->ptr != (p)->end) *(p)->ptr++ = (c); \
46 else w_more(c, p)
48 static void
49 w_more(int c, WFILE *p)
51 int size, newsize;
52 if (p->str == NULL)
53 return; /* An error already occurred */
54 size = PyString_Size(p->str);
55 newsize = size + 1024;
56 if (_PyString_Resize(&p->str, newsize) != 0) {
57 p->ptr = p->end = NULL;
59 else {
60 p->ptr = PyString_AS_STRING((PyStringObject *)p->str) + size;
61 p->end =
62 PyString_AS_STRING((PyStringObject *)p->str) + newsize;
63 *p->ptr++ = Py_SAFE_DOWNCAST(c, int, char);
67 static void
68 w_string(char *s, int n, WFILE *p)
70 if (p->fp != NULL) {
71 fwrite(s, 1, n, p->fp);
73 else {
74 while (--n >= 0) {
75 w_byte(*s, p);
76 s++;
81 static void
82 w_short(int x, WFILE *p)
84 w_byte( x & 0xff, p);
85 w_byte((x>> 8) & 0xff, p);
88 static void
89 w_long(long x, WFILE *p)
91 w_byte((int)( x & 0xff), p);
92 w_byte((int)((x>> 8) & 0xff), p);
93 w_byte((int)((x>>16) & 0xff), p);
94 w_byte((int)((x>>24) & 0xff), p);
97 #if SIZEOF_LONG > 4
98 static void
99 w_long64(long x, WFILE *p)
101 w_long(x, p);
102 w_long(x>>32, p);
104 #endif
106 static void
107 w_object(PyObject *v, WFILE *p)
109 int i, n;
110 PyBufferProcs *pb;
112 p->depth++;
114 if (p->depth > MAX_MARSHAL_STACK_DEPTH) {
115 p->error = 2;
117 else if (v == NULL) {
118 w_byte(TYPE_NULL, p);
120 else if (v == Py_None) {
121 w_byte(TYPE_NONE, p);
123 else if (v == Py_Ellipsis) {
124 w_byte(TYPE_ELLIPSIS, p);
126 else if (PyInt_Check(v)) {
127 long x = PyInt_AS_LONG((PyIntObject *)v);
128 #if SIZEOF_LONG > 4
129 long y = x>>31;
130 if (y && y != -1) {
131 w_byte(TYPE_INT64, p);
132 w_long64(x, p);
134 else
135 #endif
137 w_byte(TYPE_INT, p);
138 w_long(x, p);
141 else if (PyLong_Check(v)) {
142 PyLongObject *ob = (PyLongObject *)v;
143 w_byte(TYPE_LONG, p);
144 n = ob->ob_size;
145 w_long((long)n, p);
146 if (n < 0)
147 n = -n;
148 for (i = 0; i < n; i++)
149 w_short(ob->ob_digit[i], p);
151 else if (PyFloat_Check(v)) {
152 extern void PyFloat_AsString(char *, PyFloatObject *);
153 char buf[256]; /* Plenty to format any double */
154 PyFloat_AsString(buf, (PyFloatObject *)v);
155 n = strlen(buf);
156 w_byte(TYPE_FLOAT, p);
157 w_byte(n, p);
158 w_string(buf, n, p);
160 #ifndef WITHOUT_COMPLEX
161 else if (PyComplex_Check(v)) {
162 extern void PyFloat_AsString(char *, PyFloatObject *);
163 char buf[256]; /* Plenty to format any double */
164 PyFloatObject *temp;
165 w_byte(TYPE_COMPLEX, p);
166 temp = (PyFloatObject*)PyFloat_FromDouble(
167 PyComplex_RealAsDouble(v));
168 PyFloat_AsString(buf, temp);
169 Py_DECREF(temp);
170 n = strlen(buf);
171 w_byte(n, p);
172 w_string(buf, n, p);
173 temp = (PyFloatObject*)PyFloat_FromDouble(
174 PyComplex_ImagAsDouble(v));
175 PyFloat_AsString(buf, temp);
176 Py_DECREF(temp);
177 n = strlen(buf);
178 w_byte(n, p);
179 w_string(buf, n, p);
181 #endif
182 else if (PyString_Check(v)) {
183 w_byte(TYPE_STRING, p);
184 n = PyString_GET_SIZE(v);
185 w_long((long)n, p);
186 w_string(PyString_AS_STRING(v), n, p);
188 else if (PyUnicode_Check(v)) {
189 PyObject *utf8;
190 utf8 = PyUnicode_AsUTF8String(v);
191 if (utf8 == NULL) {
192 p->depth--;
193 p->error = 1;
194 return;
196 w_byte(TYPE_UNICODE, p);
197 n = PyString_GET_SIZE(utf8);
198 w_long((long)n, p);
199 w_string(PyString_AS_STRING(utf8), n, p);
200 Py_DECREF(utf8);
202 else if (PyTuple_Check(v)) {
203 w_byte(TYPE_TUPLE, p);
204 n = PyTuple_Size(v);
205 w_long((long)n, p);
206 for (i = 0; i < n; i++) {
207 w_object(PyTuple_GET_ITEM(v, i), p);
210 else if (PyList_Check(v)) {
211 w_byte(TYPE_LIST, p);
212 n = PyList_GET_SIZE(v);
213 w_long((long)n, p);
214 for (i = 0; i < n; i++) {
215 w_object(PyList_GET_ITEM(v, i), p);
218 else if (PyDict_Check(v)) {
219 int pos;
220 PyObject *key, *value;
221 w_byte(TYPE_DICT, p);
222 /* This one is NULL object terminated! */
223 pos = 0;
224 while (PyDict_Next(v, &pos, &key, &value)) {
225 w_object(key, p);
226 w_object(value, p);
228 w_object((PyObject *)NULL, p);
230 else if (PyCode_Check(v)) {
231 PyCodeObject *co = (PyCodeObject *)v;
232 w_byte(TYPE_CODE, p);
233 w_short(co->co_argcount, p);
234 w_short(co->co_nlocals, p);
235 w_short(co->co_stacksize, p);
236 w_short(co->co_flags, p);
237 w_object(co->co_code, p);
238 w_object(co->co_consts, p);
239 w_object(co->co_names, p);
240 w_object(co->co_varnames, p);
241 w_object(co->co_filename, p);
242 w_object(co->co_name, p);
243 w_short(co->co_firstlineno, p);
244 w_object(co->co_lnotab, p);
246 else if ((pb = v->ob_type->tp_as_buffer) != NULL &&
247 pb->bf_getsegcount != NULL &&
248 pb->bf_getreadbuffer != NULL &&
249 (*pb->bf_getsegcount)(v, NULL) == 1)
251 /* Write unknown buffer-style objects as a string */
252 char *s;
253 w_byte(TYPE_STRING, p);
254 n = (*pb->bf_getreadbuffer)(v, 0, (void **)&s);
255 w_long((long)n, p);
256 w_string(s, n, p);
258 else {
259 w_byte(TYPE_UNKNOWN, p);
260 p->error = 1;
263 p->depth--;
266 void
267 PyMarshal_WriteLongToFile(long x, FILE *fp)
269 WFILE wf;
270 wf.fp = fp;
271 wf.error = 0;
272 wf.depth = 0;
273 w_long(x, &wf);
276 void
277 PyMarshal_WriteObjectToFile(PyObject *x, FILE *fp)
279 WFILE wf;
280 wf.fp = fp;
281 wf.error = 0;
282 wf.depth = 0;
283 w_object(x, &wf);
286 typedef WFILE RFILE; /* Same struct with different invariants */
288 #define rs_byte(p) (((p)->ptr != (p)->end) ? (unsigned char)*(p)->ptr++ : EOF)
290 #define r_byte(p) ((p)->fp ? getc((p)->fp) : rs_byte(p))
292 static int
293 r_string(char *s, int n, RFILE *p)
295 if (p->fp != NULL)
296 return fread(s, 1, n, p->fp);
297 if (p->end - p->ptr < n)
298 n = p->end - p->ptr;
299 memcpy(s, p->ptr, n);
300 p->ptr += n;
301 return n;
304 static int
305 r_short(RFILE *p)
307 register short x;
308 x = r_byte(p);
309 x |= r_byte(p) << 8;
310 /* Sign-extension, in case short greater than 16 bits */
311 x |= -(x & 0x8000);
312 return x;
315 static long
316 r_long(RFILE *p)
318 register long x;
319 register FILE *fp = p->fp;
320 if (fp) {
321 x = getc(fp);
322 x |= (long)getc(fp) << 8;
323 x |= (long)getc(fp) << 16;
324 x |= (long)getc(fp) << 24;
326 else {
327 x = rs_byte(p);
328 x |= (long)rs_byte(p) << 8;
329 x |= (long)rs_byte(p) << 16;
330 x |= (long)rs_byte(p) << 24;
332 #if SIZEOF_LONG > 4
333 /* Sign extension for 64-bit machines */
334 x |= -(x & 0x80000000L);
335 #endif
336 return x;
339 static long
340 r_long64(RFILE *p)
342 register long x;
343 x = r_long(p);
344 #if SIZEOF_LONG > 4
345 x = (x & 0xFFFFFFFFL) | (r_long(p) << 32);
346 #else
347 if (r_long(p) != 0) {
348 PyObject *f = PySys_GetObject("stderr");
349 if (f != NULL)
350 (void) PyFile_WriteString(
351 "Warning: un-marshal 64-bit int in 32-bit mode\n",
354 #endif
355 return x;
358 static PyObject *
359 r_object(RFILE *p)
361 PyObject *v, *v2;
362 long i, n;
363 int type = r_byte(p);
365 switch (type) {
367 case EOF:
368 PyErr_SetString(PyExc_EOFError,
369 "EOF read where object expected");
370 return NULL;
372 case TYPE_NULL:
373 return NULL;
375 case TYPE_NONE:
376 Py_INCREF(Py_None);
377 return Py_None;
379 case TYPE_ELLIPSIS:
380 Py_INCREF(Py_Ellipsis);
381 return Py_Ellipsis;
383 case TYPE_INT:
384 return PyInt_FromLong(r_long(p));
386 case TYPE_INT64:
387 return PyInt_FromLong(r_long64(p));
389 case TYPE_LONG:
391 int size;
392 PyLongObject *ob;
393 n = r_long(p);
394 size = n<0 ? -n : n;
395 ob = _PyLong_New(size);
396 if (ob == NULL)
397 return NULL;
398 ob->ob_size = n;
399 for (i = 0; i < size; i++)
400 ob->ob_digit[i] = r_short(p);
401 return (PyObject *)ob;
404 case TYPE_FLOAT:
406 extern double atof(const char *);
407 char buf[256];
408 double dx;
409 n = r_byte(p);
410 if (r_string(buf, (int)n, p) != n) {
411 PyErr_SetString(PyExc_EOFError,
412 "EOF read where object expected");
413 return NULL;
415 buf[n] = '\0';
416 PyFPE_START_PROTECT("atof", return 0)
417 dx = atof(buf);
418 PyFPE_END_PROTECT(dx)
419 return PyFloat_FromDouble(dx);
422 #ifndef WITHOUT_COMPLEX
423 case TYPE_COMPLEX:
425 extern double atof(const char *);
426 char buf[256];
427 Py_complex c;
428 n = r_byte(p);
429 if (r_string(buf, (int)n, p) != n) {
430 PyErr_SetString(PyExc_EOFError,
431 "EOF read where object expected");
432 return NULL;
434 buf[n] = '\0';
435 PyFPE_START_PROTECT("atof", return 0)
436 c.real = atof(buf);
437 PyFPE_END_PROTECT(c)
438 n = r_byte(p);
439 if (r_string(buf, (int)n, p) != n) {
440 PyErr_SetString(PyExc_EOFError,
441 "EOF read where object expected");
442 return NULL;
444 buf[n] = '\0';
445 PyFPE_START_PROTECT("atof", return 0)
446 c.imag = atof(buf);
447 PyFPE_END_PROTECT(c)
448 return PyComplex_FromCComplex(c);
450 #endif
452 case TYPE_STRING:
453 n = r_long(p);
454 if (n < 0) {
455 PyErr_SetString(PyExc_ValueError, "bad marshal data");
456 return NULL;
458 v = PyString_FromStringAndSize((char *)NULL, n);
459 if (v != NULL) {
460 if (r_string(PyString_AS_STRING(v), (int)n, p) != n) {
461 Py_DECREF(v);
462 v = NULL;
463 PyErr_SetString(PyExc_EOFError,
464 "EOF read where object expected");
467 return v;
469 case TYPE_UNICODE:
471 char *buffer;
473 n = r_long(p);
474 if (n < 0) {
475 PyErr_SetString(PyExc_ValueError, "bad marshal data");
476 return NULL;
478 buffer = PyMem_NEW(char, n);
479 if (buffer == NULL)
480 return PyErr_NoMemory();
481 if (r_string(buffer, (int)n, p) != n) {
482 PyMem_DEL(buffer);
483 PyErr_SetString(PyExc_EOFError,
484 "EOF read where object expected");
485 return NULL;
487 v = PyUnicode_DecodeUTF8(buffer, n, NULL);
488 PyMem_DEL(buffer);
489 return v;
492 case TYPE_TUPLE:
493 n = r_long(p);
494 if (n < 0) {
495 PyErr_SetString(PyExc_ValueError, "bad marshal data");
496 return NULL;
498 v = PyTuple_New((int)n);
499 if (v == NULL)
500 return v;
501 for (i = 0; i < n; i++) {
502 v2 = r_object(p);
503 if ( v2 == NULL ) {
504 Py_DECREF(v);
505 v = NULL;
506 break;
508 PyTuple_SET_ITEM(v, (int)i, v2);
510 return v;
512 case TYPE_LIST:
513 n = r_long(p);
514 if (n < 0) {
515 PyErr_SetString(PyExc_ValueError, "bad marshal data");
516 return NULL;
518 v = PyList_New((int)n);
519 if (v == NULL)
520 return v;
521 for (i = 0; i < n; i++) {
522 v2 = r_object(p);
523 if ( v2 == NULL ) {
524 Py_DECREF(v);
525 v = NULL;
526 break;
528 PyList_SetItem(v, (int)i, v2);
530 return v;
532 case TYPE_DICT:
533 v = PyDict_New();
534 if (v == NULL)
535 return NULL;
536 for (;;) {
537 PyObject *key, *val;
538 key = r_object(p);
539 if (key == NULL)
540 break; /* XXX Assume TYPE_NULL, not an error */
541 val = r_object(p);
542 if (val != NULL)
543 PyDict_SetItem(v, key, val);
544 Py_DECREF(key);
545 Py_XDECREF(val);
547 return v;
549 case TYPE_CODE:
551 int argcount = r_short(p);
552 int nlocals = r_short(p);
553 int stacksize = r_short(p);
554 int flags = r_short(p);
555 PyObject *code = NULL;
556 PyObject *consts = NULL;
557 PyObject *names = NULL;
558 PyObject *varnames = NULL;
559 PyObject *filename = NULL;
560 PyObject *name = NULL;
561 int firstlineno = 0;
562 PyObject *lnotab = NULL;
564 code = r_object(p);
565 if (code) consts = r_object(p);
566 if (consts) names = r_object(p);
567 if (names) varnames = r_object(p);
568 if (varnames) filename = r_object(p);
569 if (filename) name = r_object(p);
570 if (name) {
571 firstlineno = r_short(p);
572 lnotab = r_object(p);
575 if (!PyErr_Occurred()) {
576 v = (PyObject *) PyCode_New(
577 argcount, nlocals, stacksize, flags,
578 code, consts, names, varnames,
579 filename, name, firstlineno, lnotab);
581 else
582 v = NULL;
583 Py_XDECREF(code);
584 Py_XDECREF(consts);
585 Py_XDECREF(names);
586 Py_XDECREF(varnames);
587 Py_XDECREF(filename);
588 Py_XDECREF(name);
589 Py_XDECREF(lnotab);
592 return v;
594 default:
595 /* Bogus data got written, which isn't ideal.
596 This will let you keep working and recover. */
597 PyErr_SetString(PyExc_ValueError, "bad marshal data");
598 return NULL;
603 long
604 PyMarshal_ReadLongFromFile(FILE *fp)
606 RFILE rf;
607 rf.fp = fp;
608 return r_long(&rf);
611 PyObject *
612 PyMarshal_ReadObjectFromFile(FILE *fp)
614 RFILE rf;
615 if (PyErr_Occurred()) {
616 fprintf(stderr, "XXX rd_object called with exception set\n");
617 return NULL;
619 rf.fp = fp;
620 return r_object(&rf);
623 PyObject *
624 PyMarshal_ReadObjectFromString(char *str, int len)
626 RFILE rf;
627 if (PyErr_Occurred()) {
628 fprintf(stderr, "XXX rds_object called with exception set\n");
629 return NULL;
631 rf.fp = NULL;
632 rf.str = NULL;
633 rf.ptr = str;
634 rf.end = str + len;
635 return r_object(&rf);
638 PyObject *
639 PyMarshal_WriteObjectToString(PyObject *x) /* wrs_object() */
641 WFILE wf;
642 wf.fp = NULL;
643 wf.str = PyString_FromStringAndSize((char *)NULL, 50);
644 if (wf.str == NULL)
645 return NULL;
646 wf.ptr = PyString_AS_STRING((PyStringObject *)wf.str);
647 wf.end = wf.ptr + PyString_Size(wf.str);
648 wf.error = 0;
649 wf.depth = 0;
650 w_object(x, &wf);
651 if (wf.str != NULL)
652 _PyString_Resize(&wf.str,
653 (int) (wf.ptr -
654 PyString_AS_STRING((PyStringObject *)wf.str)));
655 if (wf.error) {
656 Py_XDECREF(wf.str);
657 PyErr_SetString(PyExc_ValueError,
658 (wf.error==1)?"unmarshallable object"
659 :"object too deeply nested to marshal");
660 return NULL;
662 return wf.str;
665 /* And an interface for Python programs... */
667 static PyObject *
668 marshal_dump(PyObject *self, PyObject *args)
670 WFILE wf;
671 PyObject *x;
672 PyObject *f;
673 if (!PyArg_ParseTuple(args, "OO:dump", &x, &f))
674 return NULL;
675 if (!PyFile_Check(f)) {
676 PyErr_SetString(PyExc_TypeError,
677 "marshal.dump() 2nd arg must be file");
678 return NULL;
680 wf.fp = PyFile_AsFile(f);
681 wf.str = NULL;
682 wf.ptr = wf.end = NULL;
683 wf.error = 0;
684 wf.depth = 0;
685 w_object(x, &wf);
686 if (wf.error) {
687 PyErr_SetString(PyExc_ValueError,
688 (wf.error==1)?"unmarshallable object"
689 :"object too deeply nested to marshal");
690 return NULL;
692 Py_INCREF(Py_None);
693 return Py_None;
696 static PyObject *
697 marshal_load(PyObject *self, PyObject *args)
699 RFILE rf;
700 PyObject *f;
701 PyObject *v;
702 if (!PyArg_ParseTuple(args, "O:load", &f))
703 return NULL;
704 if (!PyFile_Check(f)) {
705 PyErr_SetString(PyExc_TypeError,
706 "marshal.load() arg must be file");
707 return NULL;
709 rf.fp = PyFile_AsFile(f);
710 rf.str = NULL;
711 rf.ptr = rf.end = NULL;
712 PyErr_Clear();
713 v = r_object(&rf);
714 if (PyErr_Occurred()) {
715 Py_XDECREF(v);
716 v = NULL;
718 return v;
721 static PyObject *
722 marshal_dumps(PyObject *self, PyObject *args)
724 PyObject *x;
725 if (!PyArg_ParseTuple(args, "O:dumps", &x))
726 return NULL;
727 return PyMarshal_WriteObjectToString(x);
730 static PyObject *
731 marshal_loads(PyObject *self, PyObject *args)
733 RFILE rf;
734 PyObject *v;
735 char *s;
736 int n;
737 if (!PyArg_ParseTuple(args, "s#:loads", &s, &n))
738 return NULL;
739 rf.fp = NULL;
740 rf.str = args;
741 rf.ptr = s;
742 rf.end = s + n;
743 PyErr_Clear();
744 v = r_object(&rf);
745 if (PyErr_Occurred()) {
746 Py_XDECREF(v);
747 v = NULL;
749 return v;
752 static PyMethodDef marshal_methods[] = {
753 {"dump", marshal_dump, 1},
754 {"load", marshal_load, 1},
755 {"dumps", marshal_dumps, 1},
756 {"loads", marshal_loads, 1},
757 {NULL, NULL} /* sentinel */
760 void
761 PyMarshal_Init(void)
763 (void) Py_InitModule("marshal", marshal_methods);