shotgun/lib/cpu_marshal.c

   1 #include <string.h>
   2 #include <stdlib.h>
   3 #include <sys/types.h>
   4 #include <sys/mman.h>
   5 #include <sys/stat.h>
   6 #include <fcntl.h>
   7 #include <unistd.h>
   8
   9 #include "shotgun/lib/shotgun.h"
  10 #include "shotgun/lib/cpu.h"
  11 #include "shotgun/lib/string.h"
  12 #include "shotgun/lib/bytearray.h"
  13 #include "shotgun/lib/symbol.h"
  14 #include "shotgun/lib/tuple.h"
  15 #include "shotgun/lib/bignum.h"
  16 #include "shotgun/lib/float.h"
  17 #include "shotgun/lib/sha1.h"
  18 #include "shotgun/lib/sendsite.h"
  19
  20 #include "shotgun/lib/primitive_util.h"
  21
  22 struct marshal_state {
  23   int consumed;
  24   ptr_array objects;
  25   uint8_t *buf;
  26 };
  27
  28 static int _find_object(OBJECT obj, struct marshal_state *ms) {
  29   int i;
  30   return -1;
  31   for(i = 0; i < ptr_array_length(ms->objects); i++) {
  32     if(obj == (OBJECT)ptr_array_get_index(ms->objects, i)) {
  33       return i;
  34     }
  35   }
  36
  37   return -1;
  38 }
  39
  40 static void _add_object(OBJECT obj, struct marshal_state *ms) {
  41   ptr_array_append(ms->objects, (xpointer)obj);
  42 }
  43
  44 static OBJECT unmarshal(STATE, struct marshal_state *ms);
  45 static void   marshal(STATE, OBJECT obj, bstring buf, struct marshal_state *ms);
  46
  47 static void int2be(unsigned int i, unsigned char bytes[4]) {
  48   bytes[0] = ( i >> 24 ) & 0xff;
  49   bytes[1] = ( i >> 16 ) & 0xff;
  50   bytes[2] = ( i >> 8  ) & 0xff;
  51   bytes[3] = i & 0xff;
  52 }
  53
  54 #define append_c(ch) bconchar(buf, ch)
  55 static void _append_sz(bstring buf, unsigned int i) {
  56   unsigned char bytes[4];
  57   int2be(i, bytes);
  58   bcatblk(buf, bytes, 4);
  59 }
  60 #define append_sz(sz) _append_sz(buf, (unsigned int)sz)
  61 #define append_str(str, sz) bcatblk(buf, str, sz)
  62
  63 static int read_int(uint8_t *str) {
  64   return (int)( (str[0] << 24)
  65               | (str[1] << 16)
  66               | (str[2] << 8 )
  67               |  str[3]      );
  68 }
  69
  70 static OBJECT _nth_object(STATE, struct marshal_state *ms) {
  71   int ref;
  72
  73   ms->consumed += 5;
  74   ref = read_int(ms->buf + 1);
  75   return (OBJECT)ptr_array_get_index(ms->objects, ref);
  76 }
  77
  78 static OBJECT unmarshal_int(STATE, struct marshal_state *ms) {
  79   int i;
  80   ms->consumed += 6;
  81   i = read_int(ms->buf + 2);
  82   if(ms->buf[1] == 'n') {
  83     i = -i;
  84   }
  85   return I2N(i);
  86 }
  87
  88 static void marshal_str(STATE, OBJECT obj, bstring buf) {
  89   int i;
  90   i = N2I(string_get_bytes(obj));
  91   append_c('s');
  92   append_sz(i);
  93   append_str(rbx_string_as_cstr(state, obj), i);
  94 }
  95
  96 static OBJECT unmarshal_str(STATE, struct marshal_state *ms) {
  97   int sz;
  98   sz = read_int(ms->buf + 1);
  99   ms->consumed += 5;
 100   ms->consumed += sz;
 101   return string_new2(state, (char *) ms->buf + 5, sz);
 102 }
 103
 104 static void marshal_sym(STATE, OBJECT obj, bstring buf) {
 105   OBJECT str;
 106   int i;
 107   str = symtbl_find_string(state, state->global->symbols, obj);
 108   i = N2I(string_get_bytes(str));
 109   append_c('x');
 110   append_sz(i);
 111   append_str(rbx_string_as_cstr(state, str), i);
 112 }
 113
 114 static OBJECT unmarshal_sym(STATE, struct marshal_state *ms) {
 115   int sz;
 116
 117   sz = read_int(ms->buf + 1);
 118   ms->consumed += 5;
 119   ms->consumed += sz;
 120
 121   return symtbl_lookup_str_with_size(state, state->global->symbols,
 122                                      (char *) ms->buf + 5, sz);
 123 }
 124
 125 static void marshal_sendsite(STATE, OBJECT obj, bstring buf) {
 126   OBJECT str;
 127   int i;
 128   str = symtbl_find_string(state, state->global->symbols, SENDSITE(obj)->name);
 129   i = N2I(string_get_bytes(str));
 130   append_c('S');
 131   append_sz(i);
 132   append_str(rbx_string_as_cstr(state, str), i);
 133 }
 134
 135 static OBJECT unmarshal_sendsite(STATE, struct marshal_state *ms) {
 136   int sz;
 137   OBJECT sym;
 138
 139   sz = read_int(ms->buf + 1);
 140   ms->consumed += 5;
 141   ms->consumed += sz;
 142
 143   sym = symtbl_lookup_str_with_size(state, state->global->symbols,
 144                                      (char *) ms->buf + 5, sz);
 145
 146   return send_site_create(state, sym);
 147 }
 148
 149 static void marshal_fields_as(STATE, OBJECT obj, bstring buf, char type, struct marshal_state *ms) {
 150   int sz, i;
 151   sz = NUM_FIELDS(obj);
 152   append_c(type);
 153   append_sz(sz);
 154   for(i = 0; i < sz; i++) {
 155     marshal(state, NTH_FIELD(obj, i), buf, ms);
 156   }
 157 }
 158
 159 static OBJECT unmarshal_into_fields(STATE, int sz, OBJECT tup, struct marshal_state *ms) {
 160   int i, j, cur;
 161   OBJECT o;
 162   cur = ms->consumed;
 163   for(i = 0; i < sz; i++) {
 164     uint8_t *old = ms->buf;
 165
 166     o = unmarshal(state, ms);
 167     j = ms->consumed - cur;
 168     ms->buf = old + j;
 169     cur = ms->consumed;
 170     SET_FIELD(tup, i, o);
 171   }
 172
 173   return tup;
 174 }
 175
 176 static int unmarshal_num_fields(struct marshal_state *ms) {
 177   int i;
 178   i = read_int(ms->buf + 1);
 179
 180   ms->consumed += 5;
 181   ms->buf += 5;
 182
 183   return i;
 184 }
 185
 186 static void marshal_ary(STATE, OBJECT obj, bstring buf, struct marshal_state *ms) {
 187   int sz, i;
 188   sz = N2I(array_get_total(obj));
 189   append_c('A');
 190   append_sz(sz);
 191   for(i = 0; i < sz; i++) {
 192     marshal(state, array_get(state, obj, i), buf, ms);
 193   }
 194 }
 195
 196 static OBJECT unmarshal_ary(STATE, struct marshal_state *ms) {
 197   int i, j, cur;
 198   OBJECT o;
 199   int sz = unmarshal_num_fields(ms);
 200   OBJECT ary = array_new(state, sz);
 201
 202   cur = ms->consumed;
 203   for(i = 0; i < sz; i++) {
 204     uint8_t *old = ms->buf;
 205
 206     o = unmarshal(state, ms);
 207     j = ms->consumed - cur;
 208     ms->buf = old + j;
 209     cur = ms->consumed;
 210     array_set(state, ary, i, o);
 211   }
 212
 213   return ary;
 214 }
 215
 216 static void marshal_tup(STATE, OBJECT obj, bstring buf, struct marshal_state *ms) {
 217   return marshal_fields_as(state, obj, buf, 'p', ms);
 218 }
 219
 220 static OBJECT unmarshal_tup(STATE, struct marshal_state *ms) {
 221   int sz;
 222   OBJECT tup;
 223   sz = unmarshal_num_fields(ms);
 224   tup = tuple_new(state, sz);
 225   unmarshal_into_fields(state, sz, tup, ms);
 226   return tup;
 227 }
 228
 229 static void marshal_bignum(STATE, OBJECT obj, bstring buf) {
 230   int i;
 231   char buffer[1024];
 232   bignum_into_string(state, obj, 10, buffer, 1024);
 233   append_c('B');
 234   i = strlen(buffer);
 235   append_sz(i);
 236   bcatblk(buf, buffer, i);
 237   append_c(0);                  /* zero byte */
 238 }
 239
 240 static void marshal_fixnum(STATE, OBJECT obj, bstring buf) {
 241   char buffer[1024];
 242   int i;
 243
 244   i = snprintf(buffer, 1023, "%ld", (long int)N2I(obj));
 245
 246   append_c('B');
 247   append_sz(i);
 248   bcatblk(buf, buffer, i);
 249   append_c(0);
 250 }
 251
 252 static OBJECT unmarshal_bignum(STATE, struct marshal_state *ms) {
 253   int sz;
 254   sz = read_int(ms->buf + 1);
 255   ms->consumed += 5;
 256   ms->consumed += sz;
 257   ms->consumed++;               /* zero byte */
 258   return bignum_from_string(state, (char *) ms->buf + 5, 10);
 259 }
 260
 261 static void marshal_floatpoint(STATE, OBJECT obj, bstring buf) {
 262   int i;
 263   char buffer[26];
 264
 265   float_into_string(state, obj, buffer, 26);
 266   append_c('d');
 267   i = strlen(buffer);
 268   append_sz(i);
 269   bcatblk(buf, buffer, i);
 270   append_c(0);               /* zero byte */
 271 }
 272
 273 static OBJECT unmarshal_floatpoint(STATE, struct marshal_state *ms) {
 274   int sz;
 275   sz = read_int(ms->buf + 1);
 276   ms->consumed += 5;
 277   ms->consumed += sz;
 278   ms->consumed++;               /* zero byte */
 279   return float_from_string(state, (char *) ms->buf + 5);
 280 }
 281
 282 static void marshal_bytes(STATE, OBJECT obj, bstring buf) {
 283   int i;
 284   i = SIZE_OF_BODY(obj);
 285   append_c('b');
 286   append_sz(i);
 287   append_str(bytearray_byte_address(state, obj), i);
 288 }
 289
 290 static OBJECT unmarshal_bytes(STATE, struct marshal_state *ms) {
 291   int sz;
 292   OBJECT obj;
 293   sz = read_int(ms->buf + 1);
 294
 295   ms->consumed += 5;
 296   ms->consumed += sz;
 297   obj = bytearray_new(state, sz);
 298
 299   memcpy(bytearray_byte_address(state, obj), ms->buf + 5, sz);
 300
 301   // printf("Unmarshaled bytes: %p / %d / %d\n", bytearray_byte_address(state, obj), sz, bytearray_bytes(state, obj));
 302
 303   return obj;
 304 }
 305
 306 static void marshal_iseq(STATE, OBJECT obj, bstring buf) {
 307   int i;
 308   append_c('I');
 309   append_c('b');
 310
 311   i = SIZE_OF_BODY(obj);
 312   append_sz(i);
 313   append_str(bytearray_byte_address(state, obj), i);
 314 }
 315
 316 static OBJECT unmarshal_iseq(STATE, struct marshal_state *ms) {
 317   int sz;
 318   char endian;
 319   OBJECT obj;
 320   endian = ms->buf[1];
 321
 322   sz = read_int(ms->buf + 2);
 323
 324   ms->consumed += 6;
 325   ms->consumed += sz;
 326
 327   obj = iseq_new(state, sz);
 328
 329   memcpy(bytearray_byte_address(state, obj), ms->buf + 6, sz);
 330
 331   /* We only support iseq's stored big endian currently. */
 332   sassert(endian == 'b');
 333
 334   return obj;
 335 }
 336
 337 static OBJECT unmarshal_cmethod(STATE, struct marshal_state *ms) {
 338   int sz;
 339   OBJECT cm, prim;
 340   sz = unmarshal_num_fields(ms);
 341   cm = cmethod_allocate(state);
 342   unmarshal_into_fields(state, sz, cm, ms);
 343
 344   /* fixups */
 345   prim = cmethod_get_primitive(cm);
 346   if(SYMBOL_P(prim)) {
 347     int idx = calc_primitive_index(state, symbol_to_string(state, prim));
 348     sassert(idx >= 0);
 349     cmethod_set_primitive(cm, I2N(idx));
 350   } else if(NIL_P(prim)) {
 351     cmethod_set_primitive(cm, I2N(-1));
 352   }
 353   return cm;
 354 }
 355
 356 static void marshal_cmethod2(STATE, OBJECT obj, bstring buf, struct marshal_state *ms) {
 357
 358   int i;
 359   append_c('M');
 360   /* rather than a size, we use a version id */
 361   append_sz(1);
 362
 363   for(i = 0; i < 16; i++) {
 364     marshal(state, NTH_FIELD(obj, i), buf, ms);
 365   }
 366 }
 367
 368
 369 static OBJECT unmarshal_cmethod2(STATE, struct marshal_state *ms) {
 370   int ver, i;
 371   OBJECT cm, prim, o, l;
 372
 373   ver = unmarshal_num_fields(ms);
 374   cm = cmethod_allocate(state);
 375
 376   unmarshal_into_fields(state, 16, cm, ms);
 377
 378   /* fixups */
 379   prim = cmethod_get_primitive(cm);
 380   if(SYMBOL_P(prim)) {
 381     int idx = calc_primitive_index(state, symbol_to_string(state, prim));
 382     sassert(idx >= 0);
 383     cmethod_set_primitive(cm, I2N(idx));
 384   } else if(NIL_P(prim)) {
 385     cmethod_set_primitive(cm, I2N(-1));
 386   }
 387
 388   /* Set the compiled method field on each SendSite */
 389   l = cmethod_get_literals(cm);
 390   if(TUPLE_P(l)) {
 391     int sz = tuple_fields(state, l);
 392     for(i = 0; i < sz; i++) {
 393       o = tuple_at(state, l, i);
 394       if(SENDSITE_P(o)) {
 395         send_site_set_sender(state, o, cm);
 396       }
 397     }
 398   }
 399
 400   return cm;
 401 }
 402
 403 static OBJECT unmarshal(STATE, struct marshal_state *ms) {
 404   uint8_t tag = *ms->buf;
 405   OBJECT o;
 406
 407 //  printf("%c\n", tag);
 408   switch(tag) {
 409     case 'i':
 410       o = unmarshal_int(state, ms);
 411       break;
 412     case 's':
 413       o = unmarshal_str(state, ms);
 414       _add_object(o, ms);
 415       break;
 416     case 'S':
 417       o = unmarshal_sendsite(state, ms);
 418       break;
 419     case 'x':
 420       o = unmarshal_sym(state, ms);
 421       break;
 422     case 'p':
 423       o = unmarshal_tup(state, ms);
 424       _add_object(o, ms);
 425       break;
 426     case 'A':
 427       o = unmarshal_ary(state, ms);
 428       _add_object(o, ms);
 429       break;
 430     case 'b':
 431       o = unmarshal_bytes(state, ms);
 432       _add_object(o, ms);
 433       break;
 434     case 'I':
 435       o = unmarshal_iseq(state, ms);
 436       _add_object(o, ms);
 437       break;
 438     case 'm':
 439       o = unmarshal_cmethod(state, ms);
 440       _add_object(o, ms);
 441       break;
 442     case 'M':
 443       o = unmarshal_cmethod2(state, ms);
 444       _add_object(o, ms);
 445       break;
 446     case 'B':
 447       o = unmarshal_bignum(state, ms);
 448       _add_object(o, ms);
 449       break;
 450     case 'd':
 451       o = unmarshal_floatpoint(state, ms);
 452       _add_object(o, ms);
 453       break;
 454     case 'r':
 455       o = _nth_object(state, ms);
 456       break;
 457     case 'n':
 458       ms->consumed += 1;
 459       o = Qnil;
 460       break;
 461     case 't':
 462       ms->consumed += 1;
 463       o = Qtrue;
 464       break;
 465     case 'f':
 466       ms->consumed += 1;
 467       o = Qfalse;
 468       break;
 469     default:
 470       o = Qnil;
 471       printf("Unknown marshal type '0x%x' at %d!\n", tag, ms->consumed);
 472       sassert(0);
 473   }
 474   return o;
 475 }
 476
 477 static void marshal(STATE, OBJECT obj, bstring buf, struct marshal_state *ms) {
 478   OBJECT kls;
 479   int ref;
 480
 481   if(FIXNUM_P(obj)) {
 482     marshal_fixnum(state, obj, buf);
 483   } else if(SYMBOL_P(obj)) {
 484     marshal_sym(state, obj, buf);
 485   } else if(obj == Qnil) {
 486     append_c('n');
 487   } else if(obj == Qtrue) {
 488     append_c('t');
 489   } else if(obj == Qfalse) {
 490     append_c('f');
 491   } else if(REFERENCE_P(obj)) {
 492     if((ref = _find_object(obj, ms)) > 0) {
 493       append_c('r');
 494       append_sz(ref);
 495     } else {
 496       _add_object(obj, ms);
 497       kls = object_class(state, obj);
 498       if(kls == state->global->string) {
 499         marshal_str(state, obj, buf);
 500       } else if(kls == state->global->tuple) {
 501         marshal_tup(state, obj, buf, ms);
 502       } else if(kls == state->global->array) {
 503         marshal_ary(state, obj, buf, ms);
 504       } else if(kls == state->global->cmethod) {
 505         marshal_cmethod2(state, obj, buf, ms);
 506       } else if(kls == state->global->bytearray) {
 507         marshal_bytes(state, obj, buf);
 508       } else if(kls == state->global->iseq) {
 509         marshal_iseq(state, obj, buf);
 510       } else if(kls == BASIC_CLASS(bignum)) {
 511         marshal_bignum(state, obj, buf);
 512       } else if(kls == BASIC_CLASS(floatpoint)) {
 513         marshal_floatpoint(state, obj, buf);
 514       } else if(SENDSITE_P(obj)) {
 515         marshal_sendsite(state, obj, buf);
 516       } else {
 517         printf("Unable to marshal class %p = %s!\n", (void *)kls, rbs_inspect(state, kls));
 518       }
 519     }
 520   }
 521 }
 522
 523 OBJECT cpu_marshal(STATE, OBJECT obj, int version) {
 524   bstring buf;
 525   OBJECT ret;
 526
 527   buf = cpu_marshal_to_bstring(state, obj, version);
 528   ret = string_newfrombstr(state, buf);
 529   bdestroy(buf);
 530   return ret;
 531 }
 532
 533 bstring cpu_marshal_to_bstring(STATE, OBJECT obj, int version) {
 534   bstring buf, stream;
 535   struct marshal_state ms;
 536   unsigned char cur_digest[20];
 537
 538   ms.consumed = 0;
 539   ms.objects = ptr_array_new(8);
 540
 541   stream = cstr2bstr("");
 542   marshal(state, obj, stream, &ms);
 543   sha1_hash_string((unsigned char*)bdata(stream), blength(stream), cur_digest);
 544
 545   buf = cstr2bstr("RBIX");
 546
 547   _append_sz(buf, version);
 548   bcatblk(buf, (void*)cur_digest, 20);
 549   bconcat(buf, stream);
 550   bdestroy(stream);
 551
 552   ptr_array_free(ms.objects);
 553   return buf;
 554 }
 555
 556 OBJECT cpu_marshal_to_file(STATE, OBJECT obj, char *path, int version) {
 557   bstring buf;
 558   FILE *f;
 559   struct marshal_state ms;
 560   unsigned char cur_digest[20];
 561   unsigned char bytes[4];
 562
 563   f = fopen(path, "wb");
 564   if(!f) {
 565     return Qfalse;
 566   }
 567
 568   ms.consumed = 0;
 569   ms.objects = ptr_array_new(8);
 570
 571   buf = cstr2bstr("");
 572
 573   marshal(state, obj, buf, &ms);
 574   sha1_hash_string((unsigned char*)bdata(buf), blength(buf), cur_digest);
 575
 576   /* TODO do error chceking here */
 577   fwrite("RBIX", 1, 4, f);
 578
 579   int2be(version, bytes);
 580   fwrite(bytes, 1, 4, f);
 581
 582   fwrite(cur_digest, 1, 20, f);
 583
 584   fwrite(bdatae(buf,""), 1, blength(buf), f);
 585   fclose(f);
 586
 587   bdestroy(buf);
 588   ptr_array_free(ms.objects);
 589   return Qtrue;
 590 }
 591
 592 OBJECT cpu_unmarshal(STATE, uint8_t *str, int len, int version) {
 593   struct marshal_state ms;
 594   OBJECT ret;
 595   int in_version;
 596   int offset = 4;
 597   unsigned char cur_digest[20];
 598
 599   if(!memcmp(str, "RBIS", 4)) {
 600     version = -1;
 601   } else if(!memcmp(str, "RBIX", 4)) {
 602     in_version = read_int(str + 4);
 603     if(in_version < version) {
 604       /* file is out of date. */
 605       return Qnil;
 606     }
 607
 608     offset += 4;
 609     offset += 20;
 610
 611     sha1_hash_string((unsigned char*)(str + offset), len - offset, cur_digest);
 612
 613     /* Check if the calculate one is the one in the stream. */
 614     if(memcmp(str + offset - 20, cur_digest, 20)) {
 615       return Qnil;
 616     }
 617   } else {
 618     printf("Invalid compiled file.\n");
 619     return Qnil;
 620   }
 621   ms.consumed = 0;
 622   ms.objects = ptr_array_new(8);
 623   ms.buf = str + offset;
 624
 625   ret = unmarshal(state, &ms);
 626   ptr_array_free(ms.objects);
 627   return ret;
 628 }
 629
 630 OBJECT cpu_unmarshal_file(STATE, const char *path, int version) {
 631   OBJECT obj;
 632   void *map;
 633   struct stat st;
 634   int fd;
 635
 636   fd = open(path, O_RDONLY);
 637   if (fd < 0) {
 638     return Qnil;
 639   }
 640
 641   if (fstat(fd, &st) || !st.st_size) {
 642     close(fd);
 643     return Qnil;
 644   }
 645
 646   map = mmap(NULL, (size_t) st.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
 647   close(fd);
 648
 649   obj = cpu_unmarshal(state, map, (int)st.st_size, version);
 650   munmap(map, st.st_size);
 651
 652   return obj;
 653 }
 654