Python/marshal.c

   1 /***********************************************************
   2 Copyright 1991-1995 by Stichting Mathematisch Centrum, Amsterdam,
   3 The Netherlands.
   4
   5                         All Rights Reserved
   6
   7 Permission to use, copy, modify, and distribute this software and its
   8 documentation for any purpose and without fee is hereby granted,
   9 provided that the above copyright notice appear in all copies and that
  10 both that copyright notice and this permission notice appear in
  11 supporting documentation, and that the names of Stichting Mathematisch
  12 Centrum or CWI not be used in advertising or publicity pertaining to
  13 distribution of the software without specific, written prior permission.
  14
  15 STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO
  16 THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
  17 FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE
  18 FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  19 WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  20 ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
  21 OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  22
  23 ******************************************************************/
  24
  25 /* Write Python objects to files and read them back.
  26    This is intended for writing and reading compiled Python code only;
  27    a true persistent storage facility would be much harder, since
  28    it would have to take circular links and sharing into account. */
  29
  30 #include "allobjects.h"
  31 #include "modsupport.h"
  32 #include "longintrepr.h"
  33 #include "compile.h"
  34 #include "marshal.h"
  35
  36 #include <errno.h>
  37
  38 #define TYPE_NULL       '0'
  39 #define TYPE_NONE       'N'
  40 #define TYPE_INT        'i'
  41 #define TYPE_FLOAT      'f'
  42 #define TYPE_COMPLEX    'x'
  43 #define TYPE_LONG       'l'
  44 #define TYPE_STRING     's'
  45 #define TYPE_TUPLE      '('
  46 #define TYPE_LIST       '['
  47 #define TYPE_DICT       '{'
  48 #define TYPE_CODE       'c'
  49 #define TYPE_UNKNOWN    '?'
  50
  51 typedef struct {
  52         FILE *fp;
  53         /* If fp == NULL, the following are valid: */
  54         object *str;
  55         char *ptr;
  56         char *end;
  57 } WFILE;
  58
  59 #define w_byte(c, p) if (((p)->fp)) putc((c), (p)->fp); \
  60                       else if ((p)->ptr != (p)->end) *(p)->ptr++ = (c); \
  61                            else w_more(c, p)
  62
  63 static void
  64 w_more(c, p)
  65         char c;
  66         WFILE *p;
  67 {
  68         int size, newsize;
  69         if (p->str == NULL)
  70                 return; /* An error already occurred */
  71         size = getstringsize(p->str);
  72         newsize = size + 1024;
  73         if (resizestring(&p->str, newsize) != 0) {
  74                 p->ptr = p->end = NULL;
  75         }
  76         else {
  77                 p->ptr = GETSTRINGVALUE((stringobject *)p->str) + size;
  78                 p->end = GETSTRINGVALUE((stringobject *)p->str) + newsize;
  79                 *p->ptr++ = c;
  80         }
  81 }
  82
  83 static void
  84 w_string(s, n, p)
  85         char *s;
  86         int n;
  87         WFILE *p;
  88 {
  89         if (p->fp != NULL) {
  90                 fwrite(s, 1, n, p->fp);
  91         }
  92         else {
  93                 while (--n >= 0) {
  94                         w_byte(*s, p);
  95                         s++;
  96                 }
  97         }
  98 }
  99
 100 static void
 101 w_short(x, p)
 102         int x;
 103         WFILE *p;
 104 {
 105         w_byte( x      & 0xff, p);
 106         w_byte((x>> 8) & 0xff, p);
 107 }
 108
 109 static void
 110 w_long(x, p)
 111         long x;
 112         WFILE *p;
 113 {
 114         w_byte((int)( x      & 0xff), p);
 115         w_byte((int)((x>> 8) & 0xff), p);
 116         w_byte((int)((x>>16) & 0xff), p);
 117         w_byte((int)((x>>24) & 0xff), p);
 118 }
 119
 120 static void
 121 w_object(v, p)
 122         object *v;
 123         WFILE *p;
 124 {
 125         int i, n;
 126
 127         if (v == NULL)
 128                 w_byte(TYPE_NULL, p);
 129         else if (v == None)
 130                 w_byte(TYPE_NONE, p);
 131         else if (is_intobject(v)) {
 132                 w_byte(TYPE_INT, p);
 133                 w_long(getintvalue(v), p);
 134         }
 135         else if (is_longobject(v)) {
 136                 longobject *ob = (longobject *)v;
 137                 w_byte(TYPE_LONG, p);
 138                 n = ob->ob_size;
 139                 w_long((long)n, p);
 140                 if (n < 0)
 141                         n = -n;
 142                 for (i = 0; i < n; i++)
 143                         w_short(ob->ob_digit[i], p);
 144         }
 145         else if (is_floatobject(v)) {
 146                 extern void float_buf_repr PROTO((char *, floatobject *));
 147                 char buf[256]; /* Plenty to format any double */
 148                 float_buf_repr(buf, (floatobject *)v);
 149                 n = strlen(buf);
 150                 w_byte(TYPE_FLOAT, p);
 151                 w_byte(n, p);
 152                 w_string(buf, n, p);
 153         }
 154 #ifndef WITHOUT_COMPLEX
 155         else if (is_complexobject(v)) {
 156                 extern void float_buf_repr PROTO((char *, floatobject *));
 157                 char buf[256]; /* Plenty to format any double */
 158                 floatobject *temp;
 159                 w_byte(TYPE_COMPLEX, p);
 160                 temp = (floatobject*)newfloatobject(PyComplex_RealAsDouble(v));
 161                 float_buf_repr(buf, temp);
 162                 DECREF(temp);
 163                 n = strlen(buf);
 164                 w_byte(n, p);
 165                 w_string(buf, n, p);
 166                 temp = (floatobject*)newfloatobject(PyComplex_ImagAsDouble(v));
 167                 float_buf_repr(buf, temp);
 168                 DECREF(temp);
 169                 n = strlen(buf);
 170                 w_byte(n, p);
 171                 w_string(buf, n, p);
 172         }
 173 #endif
 174         else if (is_stringobject(v)) {
 175                 w_byte(TYPE_STRING, p);
 176                 n = getstringsize(v);
 177                 w_long((long)n, p);
 178                 w_string(getstringvalue(v), n, p);
 179         }
 180         else if (is_tupleobject(v)) {
 181                 w_byte(TYPE_TUPLE, p);
 182                 n = gettuplesize(v);
 183                 w_long((long)n, p);
 184                 for (i = 0; i < n; i++) {
 185                         w_object(GETTUPLEITEM(v, i), p);
 186                 }
 187         }
 188         else if (is_listobject(v)) {
 189                 w_byte(TYPE_LIST, p);
 190                 n = getlistsize(v);
 191                 w_long((long)n, p);
 192                 for (i = 0; i < n; i++) {
 193                         w_object(getlistitem(v, i), p);
 194                 }
 195         }
 196         else if (is_dictobject(v)) {
 197                 int pos;
 198                 object *key, *value;
 199                 w_byte(TYPE_DICT, p);
 200                 /* This one is NULL object terminated! */
 201                 pos = 0;
 202                 while (mappinggetnext(v, &pos, &key, &value)) {
 203                         w_object(key, p);
 204                         w_object(value, p);
 205                 }
 206                 w_object((object *)NULL, p);
 207         }
 208         else if (is_codeobject(v)) {
 209                 codeobject *co = (codeobject *)v;
 210                 w_byte(TYPE_CODE, p);
 211                 w_short(co->co_argcount, p);
 212                 w_short(co->co_nlocals, p);
 213                 w_short(co->co_flags, p);
 214                 w_object((object *)co->co_code, p);
 215                 w_object(co->co_consts, p);
 216                 w_object(co->co_names, p);
 217                 w_object(co->co_varnames, p);
 218                 w_object(co->co_filename, p);
 219                 w_object(co->co_name, p);
 220         }
 221         else {
 222                 w_byte(TYPE_UNKNOWN, p);
 223         }
 224 }
 225
 226 void
 227 wr_long(x, fp)
 228         long x;
 229         FILE *fp;
 230 {
 231         WFILE wf;
 232         wf.fp = fp;
 233         w_long(x, &wf);
 234 }
 235
 236 void
 237 wr_object(x, fp)
 238         object *x;
 239         FILE *fp;
 240 {
 241         WFILE wf;
 242         wf.fp = fp;
 243         w_object(x, &wf);
 244 }
 245
 246 typedef WFILE RFILE; /* Same struct with different invariants */
 247
 248 #define rs_byte(p) (((p)->ptr != (p)->end) ? (unsigned char)*(p)->ptr++ : EOF)
 249
 250 #define r_byte(p) ((p)->fp ? getc((p)->fp) : rs_byte(p))
 251
 252 static int
 253 r_string(s, n, p)
 254         char *s;
 255         int n;
 256         RFILE *p;
 257 {
 258         if (p->fp != NULL)
 259                 return fread(s, 1, n, p->fp);
 260         if (p->end - p->ptr < n)
 261                 n = p->end - p->ptr;
 262         memcpy(s, p->ptr, n);
 263         p->ptr += n;
 264         return n;
 265 }
 266
 267 static int
 268 r_short(p)
 269         RFILE *p;
 270 {
 271         register short x;
 272         x = r_byte(p);
 273         x |= r_byte(p) << 8;
 274         /* XXX If your short is > 16 bits, add sign-extension here!!! */
 275         return x;
 276 }
 277
 278 static long
 279 r_long(p)
 280         RFILE *p;
 281 {
 282         register long x;
 283         register FILE *fp = p->fp;
 284         if (fp) {
 285                 x = getc(fp);
 286                 x |= (long)getc(fp) << 8;
 287                 x |= (long)getc(fp) << 16;
 288                 x |= (long)getc(fp) << 24;
 289         }
 290         else {
 291                 x = rs_byte(p);
 292                 x |= (long)rs_byte(p) << 8;
 293                 x |= (long)rs_byte(p) << 16;
 294                 x |= (long)rs_byte(p) << 24;
 295         }
 296         /* XXX If your long is > 32 bits, add sign-extension here!!! */
 297         return x;
 298 }
 299
 300 static object *
 301 r_object(p)
 302         RFILE *p;
 303 {
 304         object *v, *v2;
 305         long i, n;
 306         int type = r_byte(p);
 307
 308         switch (type) {
 309
 310         case EOF:
 311                 err_setstr(EOFError, "EOF read where object expected");
 312                 return NULL;
 313
 314         case TYPE_NULL:
 315                 return NULL;
 316
 317         case TYPE_NONE:
 318                 INCREF(None);
 319                 return None;
 320
 321         case TYPE_INT:
 322                 return newintobject(r_long(p));
 323
 324         case TYPE_LONG:
 325                 {
 326                         int size;
 327                         longobject *ob;
 328                         n = r_long(p);
 329                         size = n<0 ? -n : n;
 330                         ob = alloclongobject(size);
 331                         if (ob == NULL)
 332                                 return NULL;
 333                         ob->ob_size = n;
 334                         for (i = 0; i < size; i++)
 335                                 ob->ob_digit[i] = r_short(p);
 336                         return (object *)ob;
 337                 }
 338
 339         case TYPE_FLOAT:
 340                 {
 341                         extern double atof PROTO((const char *));
 342                         char buf[256];
 343                         n = r_byte(p);
 344                         if (r_string(buf, (int)n, p) != n) {
 345                                 err_setstr(EOFError,
 346                                         "EOF read where object expected");
 347                                 return NULL;
 348                         }
 349                         buf[n] = '\0';
 350                         return newfloatobject(atof(buf));
 351                 }
 352
 353 #ifndef WITHOUT_COMPLEX
 354         case TYPE_COMPLEX:
 355                 {
 356                         extern double atof PROTO((const char *));
 357                         char buf[256];
 358                         complex c;
 359                         n = r_byte(p);
 360                         if (r_string(buf, (int)n, p) != n) {
 361                                 err_setstr(EOFError,
 362                                         "EOF read where object expected");
 363                                 return NULL;
 364                         }
 365                         buf[n] = '\0';
 366                         c.real = atof(buf);
 367                         n = r_byte(p);
 368                         if (r_string(buf, (int)n, p) != n) {
 369                                 err_setstr(EOFError,
 370                                         "EOF read where object expected");
 371                                 return NULL;
 372                         }
 373                         buf[n] = '\0';
 374                         c.imag = atof(buf);
 375                         return newcomplexobject(c);
 376                 }
 377 #endif
 378
 379         case TYPE_STRING:
 380                 n = r_long(p);
 381                 v = newsizedstringobject((char *)NULL, n);
 382                 if (v != NULL) {
 383                         if (r_string(getstringvalue(v), (int)n, p) != n) {
 384                                 DECREF(v);
 385                                 v = NULL;
 386                                 err_setstr(EOFError,
 387                                         "EOF read where object expected");
 388                         }
 389                 }
 390                 return v;
 391
 392         case TYPE_TUPLE:
 393                 n = r_long(p);
 394                 v = newtupleobject((int)n);
 395                 if (v == NULL)
 396                         return v;
 397                 for (i = 0; i < n; i++) {
 398                         v2 = r_object(p);
 399                         if ( v2 == NULL ) {
 400                                 DECREF(v);
 401                                 v = NULL;
 402                                 break;
 403                         }
 404                         SETTUPLEITEM(v, (int)i, v2);
 405                 }
 406                 return v;
 407
 408         case TYPE_LIST:
 409                 n = r_long(p);
 410                 v = newlistobject((int)n);
 411                 if (v == NULL)
 412                         return v;
 413                 for (i = 0; i < n; i++) {
 414                         v2 = r_object(p);
 415                         if ( v2 == NULL ) {
 416                                 DECREF(v);
 417                                 v = NULL;
 418                                 break;
 419                         }
 420                         setlistitem(v, (int)i, v2);
 421                 }
 422                 return v;
 423
 424         case TYPE_DICT:
 425                 v = newdictobject();
 426                 if (v == NULL)
 427                         return NULL;
 428                 for (;;) {
 429                         object *key, *val;
 430                         key = r_object(p);
 431                         if (key == NULL)
 432                                 break; /* XXXX and how about memory errors? */
 433                         val = r_object(p);
 434                         /* XXXX error check? */
 435                         dict2insert(v, key, val);
 436                         DECREF(key);
 437                         XDECREF(val);
 438                 }
 439                 return v;
 440
 441         case TYPE_CODE:
 442                 {
 443                         int argcount = r_short(p);
 444                         int nlocals = r_short(p);
 445                         int flags = r_short(p);
 446                         object *code = NULL;
 447                         object *consts = NULL;
 448                         object *names = NULL;
 449                         object *varnames = NULL;
 450                         object *filename = NULL;
 451                         object *name = NULL;
 452
 453                         code = r_object(p);
 454                         if (code) consts = r_object(p);
 455                         if (consts) names = r_object(p);
 456                         if (names) varnames = r_object(p);
 457                         if (varnames) filename = r_object(p);
 458                         if (filename) name = r_object(p);
 459
 460                         if (!err_occurred()) {
 461                                 v = (object *) newcodeobject(
 462                                         argcount, nlocals, flags,
 463                                         code, consts, names, varnames,
 464                                         filename, name);
 465                         }
 466                         else
 467                                 v = NULL;
 468                         XDECREF(code);
 469                         XDECREF(consts);
 470                         XDECREF(names);
 471                         XDECREF(varnames);
 472                         XDECREF(filename);
 473                         XDECREF(name);
 474
 475                 }
 476                 return v;
 477
 478         default:
 479                 err_setstr(TypeError, "read unknown object");
 480                 return NULL;
 481
 482         }
 483 }
 484
 485 long
 486 rd_long(fp)
 487         FILE *fp;
 488 {
 489         RFILE rf;
 490         rf.fp = fp;
 491         return r_long(&rf);
 492 }
 493
 494 object *
 495 rd_object(fp)
 496         FILE *fp;
 497 {
 498         RFILE rf;
 499         if (err_occurred()) {
 500                 fatal("XXX rd_object called with exception set"); /* tmp */
 501                 fprintf(stderr, "XXX rd_object called with exception set\n");
 502                 return NULL;
 503         }
 504         rf.fp = fp;
 505         return r_object(&rf);
 506 }
 507
 508 object *
 509 rds_object(str, len)
 510         char *str;
 511         int len;
 512 {
 513         RFILE rf;
 514         if (err_occurred()) {
 515                 fprintf(stderr, "XXX rds_object called with exception set\n");
 516                 return NULL;
 517         }
 518         rf.fp = NULL;
 519         rf.str = NULL;
 520         rf.ptr = str;
 521         rf.end = str + len;
 522         return r_object(&rf);
 523 }
 524
 525 /* And an interface for Python programs... */
 526
 527 static object *
 528 marshal_dump(self, args)
 529         object *self;
 530         object *args;
 531 {
 532         WFILE wf;
 533         object *x;
 534         object *f;
 535         if (!getargs(args, "(OO)", &x, &f))
 536                 return NULL;
 537         if (!is_fileobject(f)) {
 538                 err_setstr(TypeError, "marshal.dump() 2nd arg must be file");
 539                 return NULL;
 540         }
 541         wf.fp = getfilefile(f);
 542         wf.str = NULL;
 543         wf.ptr = wf.end = NULL;
 544         w_object(x, &wf);
 545         INCREF(None);
 546         return None;
 547 }
 548
 549 static object *
 550 marshal_load(self, args)
 551         object *self;
 552         object *args;
 553 {
 554         RFILE rf;
 555         object *f;
 556         object *v;
 557         if (!getargs(args, "O", &f))
 558                 return NULL;
 559         if (!is_fileobject(f)) {
 560                 err_setstr(TypeError, "marshal.load() arg must be file");
 561                 return NULL;
 562         }
 563         rf.fp = getfilefile(f);
 564         rf.str = NULL;
 565         rf.ptr = rf.end = NULL;
 566         err_clear();
 567         v = r_object(&rf);
 568         if (err_occurred()) {
 569                 XDECREF(v);
 570                 v = NULL;
 571         }
 572         return v;
 573 }
 574
 575 static object *
 576 marshal_dumps(self, args)
 577         object *self;
 578         object *args;
 579 {
 580         WFILE wf;
 581         object *x;
 582         if (!getargs(args, "O", &x))
 583                 return NULL;
 584         wf.fp = NULL;
 585         wf.str = newsizedstringobject((char *)NULL, 50);
 586         if (wf.str == NULL)
 587                 return NULL;
 588         wf.ptr = GETSTRINGVALUE((stringobject *)wf.str);
 589         wf.end = wf.ptr + getstringsize(wf.str);
 590         w_object(x, &wf);
 591         if (wf.str != NULL)
 592                 resizestring(&wf.str,
 593                     (int) (wf.ptr - GETSTRINGVALUE((stringobject *)wf.str)));
 594         return wf.str;
 595 }
 596
 597 static object *
 598 marshal_loads(self, args)
 599         object *self;
 600         object *args;
 601 {
 602         RFILE rf;
 603         object *v;
 604         char *s;
 605         int n;
 606         if (!getargs(args, "s#", &s, &n))
 607                 return NULL;
 608         rf.fp = NULL;
 609         rf.str = args;
 610         rf.ptr = s;
 611         rf.end = s + n;
 612         err_clear();
 613         v = r_object(&rf);
 614         if (err_occurred()) {
 615                 XDECREF(v);
 616                 v = NULL;
 617         }
 618         return v;
 619 }
 620
 621 static struct methodlist marshal_methods[] = {
 622         {"dump",        marshal_dump},
 623         {"load",        marshal_load},
 624         {"dumps",       marshal_dumps},
 625         {"loads",       marshal_loads},
 626         {NULL,          NULL}           /* sentinel */
 627 };
 628
 629 void
 630 initmarshal()
 631 {
 632         (void) initmodule("marshal", marshal_methods);
 633 }