token.c

   1 /*
   2  * Routines used by the file-transfer code.
   3  *
   4  * Copyright (C) 1996 Andrew Tridgell
   5  * Copyright (C) 1996 Paul Mackerras
   6  * Copyright (C) 2003-2008 Wayne Davison
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License as published by
  10  * the Free Software Foundation; either version 3 of the License, or
  11  * (at your option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License along
  19  * with this program; if not, visit the http://fsf.org website.
  20  */
  21
  22 #include "rsync.h"
  23 #include "ifuncs.h"
  24 #include "zlib/zlib.h"
  25
  26 extern int do_compression;
  27 extern int module_id;
  28 extern int def_compress_level;
  29 extern char *skip_compress;
  30
  31 static int compression_level, per_file_default_level;
  32
  33 struct suffix_tree {
  34         struct suffix_tree *sibling;
  35         struct suffix_tree *child;
  36         char letter, word_end;
  37 };
  38
  39 static char *match_list;
  40 static struct suffix_tree *suftree;
  41
  42 static void add_suffix(struct suffix_tree **prior, char ltr, const char *str)
  43 {
  44         struct suffix_tree *node, *newnode;
  45
  46         if (ltr == '[') {
  47                 const char *after = strchr(str, ']');
  48                 /* Just skip bogus character classes. */
  49                 if (!after++)
  50                         return;
  51                 while ((ltr = *str++) != ']')
  52                         add_suffix(prior, ltr, after);
  53                 return;
  54         }
  55
  56         for (node = *prior; node; prior = &node->sibling, node = node->sibling) {
  57                 if (node->letter == ltr) {
  58                         if (*str)
  59                                 add_suffix(&node->child, *str, str+1);
  60                         else
  61                                 node->word_end = 1;
  62                         return;
  63                 }
  64                 if (node->letter > ltr)
  65                         break;
  66         }
  67         if (!(newnode = new(struct suffix_tree)))
  68                 out_of_memory("add_suffix");
  69         newnode->sibling = node;
  70         newnode->child = NULL;
  71         newnode->letter = ltr;
  72         *prior = newnode;
  73         if (*str) {
  74                 add_suffix(&newnode->child, *str, str+1);
  75                 newnode->word_end = 0;
  76         } else
  77                 newnode->word_end = 1;
  78 }
  79
  80 static void add_nocompress_suffixes(const char *str)
  81 {
  82         char *buf, *t;
  83         const char *f = str;
  84
  85         if (!(buf = new_array(char, strlen(f) + 1)))
  86                 out_of_memory("add_nocompress_suffixes");
  87
  88         while (*f) {
  89                 if (*f == '/') {
  90                         f++;
  91                         continue;
  92                 }
  93
  94                 t = buf;
  95                 do {
  96                         if (isUpper(f))
  97                                 *t++ = toLower(f);
  98                         else
  99                                 *t++ = *f;
 100                 } while (*++f != '/' && *f);
 101                 *t++ = '\0';
 102
 103                 fprintf(stderr, "adding `%s'\n", buf);
 104                 add_suffix(&suftree, *buf, buf+1);
 105         }
 106
 107         free(buf);
 108 }
 109
 110 static void init_set_compression(void)
 111 {
 112         const char *f;
 113         char *t, *start;
 114
 115         if (skip_compress)
 116                 add_nocompress_suffixes(skip_compress);
 117
 118         /* A non-daemon transfer skips the default suffix list if the
 119          * user specified --skip-compress. */
 120         if (skip_compress && module_id < 0)
 121                 f = "";
 122         else
 123                 f = lp_dont_compress(module_id);
 124
 125         if (!(match_list = t = new_array(char, strlen(f) + 2)))
 126                 out_of_memory("set_compression");
 127
 128         per_file_default_level = def_compress_level;
 129
 130         while (*f) {
 131                 if (*f == ' ') {
 132                         f++;
 133                         continue;
 134                 }
 135
 136                 start = t;
 137                 do {
 138                         if (isUpper(f))
 139                                 *t++ = toLower(f);
 140                         else
 141                                 *t++ = *f;
 142                 } while (*++f != ' ' && *f);
 143                 *t++ = '\0';
 144
 145                 if (t - start == 1+1 && *start == '*') {
 146                         /* Optimize a match-string of "*". */
 147                         *match_list = '\0';
 148                         suftree = NULL;
 149                         per_file_default_level = 0;
 150                         break;
 151                 }
 152
 153                 /* Move *.foo items into the stuffix tree. */
 154                 if (*start == '*' && start[1] == '.' && start[2]
 155                  && !strpbrk(start+2, ".?*")) {
 156                         add_suffix(&suftree, start[2], start+3);
 157                         t = start;
 158                 }
 159         }
 160         *t++ = '\0';
 161 }
 162
 163 /* determine the compression level based on a wildcard filename list */
 164 void set_compression(const char *fname)
 165 {
 166         const struct suffix_tree *node;
 167         const char *s;
 168         char ltr;
 169
 170         if (!do_compression)
 171                 return;
 172
 173         if (!match_list)
 174                 init_set_compression();
 175
 176         compression_level = per_file_default_level;
 177
 178         if (!*match_list && !suftree)
 179                 return;
 180
 181         if ((s = strrchr(fname, '/')) != NULL)
 182                 fname = s + 1;
 183
 184         for (s = match_list; *s; s += strlen(s) + 1) {
 185                 if (iwildmatch(s, fname)) {
 186                         compression_level = 0;
 187                         return;
 188                 }
 189         }
 190
 191         if (!(node = suftree) || !(s = strrchr(fname, '.'))
 192          || s == fname || !(ltr = *++s))
 193                 return;
 194
 195         while (1) {
 196                 while (node->letter != ltr) {
 197                         if (node->letter > ltr)
 198                                 return;
 199                         if (!(node = node->sibling))
 200                                 return;
 201                 }
 202                 if ((ltr = *++s) == '\0') {
 203                         if (node->word_end)
 204                                 compression_level = 0;
 205                         return;
 206                 }
 207                 if (!(node = node->child))
 208                         return;
 209         }
 210 }
 211
 212 /* non-compressing recv token */
 213 static int32 simple_recv_token(int f, char **data)
 214 {
 215         static int32 residue;
 216         static char *buf;
 217         int32 n;
 218
 219         if (!buf) {
 220                 buf = new_array(char, CHUNK_SIZE);
 221                 if (!buf)
 222                         out_of_memory("simple_recv_token");
 223         }
 224
 225         if (residue == 0) {
 226                 int32 i = read_int(f);
 227                 if (i <= 0)
 228                         return i;
 229                 residue = i;
 230         }
 231
 232         *data = buf;
 233         n = MIN(CHUNK_SIZE,residue);
 234         residue -= n;
 235         read_buf(f,buf,n);
 236         return n;
 237 }
 238
 239 /* non-compressing send token */
 240 static void simple_send_token(int f, int32 token, struct map_struct *buf,
 241                               OFF_T offset, int32 n)
 242 {
 243         if (n > 0) {
 244                 int32 len = 0;
 245                 while (len < n) {
 246                         int32 n1 = MIN(CHUNK_SIZE, n-len);
 247                         write_int(f, n1);
 248                         write_buf(f, map_ptr(buf, offset+len, n1), n1);
 249                         len += n1;
 250                 }
 251         }
 252         /* a -2 token means to send data only and no token */
 253         if (token != -2)
 254                 write_int(f, -(token+1));
 255 }
 256
 257 /* Flag bytes in compressed stream are encoded as follows: */
 258 #define END_FLAG        0       /* that's all folks */
 259 #define TOKEN_LONG      0x20    /* followed by 32-bit token number */
 260 #define TOKENRUN_LONG   0x21    /* ditto with 16-bit run count */
 261 #define DEFLATED_DATA   0x40    /* + 6-bit high len, then low len byte */
 262 #define TOKEN_REL       0x80    /* + 6-bit relative token number */
 263 #define TOKENRUN_REL    0xc0    /* ditto with 16-bit run count */
 264
 265 #define MAX_DATA_COUNT  16383   /* fit 14 bit count into 2 bytes with flags */
 266
 267 /* zlib.h says that if we want to be able to compress something in a single
 268  * call, avail_out must be at least 0.1% larger than avail_in plus 12 bytes.
 269  * We'll add in 0.1%+16, just to be safe (and we'll avoid floating point,
 270  * to ensure that this is a compile-time value). */
 271 #define AVAIL_OUT_SIZE(avail_in_size) ((avail_in_size)*1001/1000+16)
 272
 273 /* For coding runs of tokens */
 274 static int32 last_token = -1;
 275 static int32 run_start;
 276 static int32 last_run_end;
 277
 278 /* Deflation state */
 279 static z_stream tx_strm;
 280
 281 /* Output buffer */
 282 static char *obuf;
 283
 284 /* We want obuf to be able to hold both MAX_DATA_COUNT+2 bytes as well as
 285  * AVAIL_OUT_SIZE(CHUNK_SIZE) bytes, so make sure that it's large enough. */
 286 #if MAX_DATA_COUNT+2 > AVAIL_OUT_SIZE(CHUNK_SIZE)
 287 #define OBUF_SIZE       (MAX_DATA_COUNT+2)
 288 #else
 289 #define OBUF_SIZE       AVAIL_OUT_SIZE(CHUNK_SIZE)
 290 #endif
 291
 292 /* Send a deflated token */
 293 static void
 294 send_deflated_token(int f, int32 token, struct map_struct *buf, OFF_T offset,
 295                     int32 nb, int32 toklen)
 296 {
 297         int32 n, r;
 298         static int init_done, flush_pending;
 299
 300         if (last_token == -1) {
 301                 /* initialization */
 302                 if (!init_done) {
 303                         tx_strm.next_in = NULL;
 304                         tx_strm.zalloc = NULL;
 305                         tx_strm.zfree = NULL;
 306                         if (deflateInit2(&tx_strm, compression_level,
 307                                          Z_DEFLATED, -15, 8,
 308                                          Z_DEFAULT_STRATEGY) != Z_OK) {
 309                                 rprintf(FERROR, "compression init failed\n");
 310                                 exit_cleanup(RERR_STREAMIO);
 311                         }
 312                         if ((obuf = new_array(char, OBUF_SIZE)) == NULL)
 313                                 out_of_memory("send_deflated_token");
 314                         init_done = 1;
 315                 } else
 316                         deflateReset(&tx_strm);
 317                 last_run_end = 0;
 318                 run_start = token;
 319                 flush_pending = 0;
 320         } else if (last_token == -2) {
 321                 run_start = token;
 322         } else if (nb != 0 || token != last_token + 1
 323                    || token >= run_start + 65536) {
 324                 /* output previous run */
 325                 r = run_start - last_run_end;
 326                 n = last_token - run_start;
 327                 if (r >= 0 && r <= 63) {
 328                         write_byte(f, (n==0? TOKEN_REL: TOKENRUN_REL) + r);
 329                 } else {
 330                         write_byte(f, (n==0? TOKEN_LONG: TOKENRUN_LONG));
 331                         write_int(f, run_start);
 332                 }
 333                 if (n != 0) {
 334                         write_byte(f, n);
 335                         write_byte(f, n >> 8);
 336                 }
 337                 last_run_end = last_token;
 338                 run_start = token;
 339         }
 340
 341         last_token = token;
 342
 343         if (nb != 0 || flush_pending) {
 344                 /* deflate the data starting at offset */
 345                 int flush = Z_NO_FLUSH;
 346                 tx_strm.avail_in = 0;
 347                 tx_strm.avail_out = 0;
 348                 do {
 349                         if (tx_strm.avail_in == 0 && nb != 0) {
 350                                 /* give it some more input */
 351                                 n = MIN(nb, CHUNK_SIZE);
 352                                 tx_strm.next_in = (Bytef *)
 353                                         map_ptr(buf, offset, n);
 354                                 tx_strm.avail_in = n;
 355                                 nb -= n;
 356                                 offset += n;
 357                         }
 358                         if (tx_strm.avail_out == 0) {
 359                                 tx_strm.next_out = (Bytef *)(obuf + 2);
 360                                 tx_strm.avail_out = MAX_DATA_COUNT;
 361                                 if (flush != Z_NO_FLUSH) {
 362                                         /*
 363                                          * We left the last 4 bytes in the
 364                                          * buffer, in case they are the
 365                                          * last 4.  Move them to the front.
 366                                          */
 367                                         memcpy(tx_strm.next_out,
 368                                                obuf+MAX_DATA_COUNT-2, 4);
 369                                         tx_strm.next_out += 4;
 370                                         tx_strm.avail_out -= 4;
 371                                 }
 372                         }
 373                         if (nb == 0 && token != -2)
 374                                 flush = Z_SYNC_FLUSH;
 375                         r = deflate(&tx_strm, flush);
 376                         if (r != Z_OK) {
 377                                 rprintf(FERROR, "deflate returned %d\n", r);
 378                                 exit_cleanup(RERR_STREAMIO);
 379                         }
 380                         if (nb == 0 || tx_strm.avail_out == 0) {
 381                                 n = MAX_DATA_COUNT - tx_strm.avail_out;
 382                                 if (flush != Z_NO_FLUSH) {
 383                                         /*
 384                                          * We have to trim off the last 4
 385                                          * bytes of output when flushing
 386                                          * (they are just 0, 0, ff, ff).
 387                                          */
 388                                         n -= 4;
 389                                 }
 390                                 if (n > 0) {
 391                                         obuf[0] = DEFLATED_DATA + (n >> 8);
 392                                         obuf[1] = n;
 393                                         write_buf(f, obuf, n+2);
 394                                 }
 395                         }
 396                 } while (nb != 0 || tx_strm.avail_out == 0);
 397                 flush_pending = token == -2;
 398         }
 399
 400         if (token == -1) {
 401                 /* end of file - clean up */
 402                 write_byte(f, END_FLAG);
 403         } else if (token != -2) {
 404                 /* Add the data in the current block to the compressor's
 405                  * history and hash table. */
 406                 do {
 407                         /* Break up long sections in the same way that
 408                          * see_deflate_token() does. */
 409                         int32 n1 = toklen > 0xffff ? 0xffff : toklen;
 410                         toklen -= n1;
 411                         tx_strm.next_in = (Bytef *)map_ptr(buf, offset, n1);
 412                         tx_strm.avail_in = n1;
 413                         tx_strm.next_out = (Bytef *) obuf;
 414                         tx_strm.avail_out = AVAIL_OUT_SIZE(CHUNK_SIZE);
 415                         r = deflate(&tx_strm, Z_INSERT_ONLY);
 416                         if (r != Z_OK || tx_strm.avail_in != 0) {
 417                                 rprintf(FERROR, "deflate on token returned %d (%d bytes left)\n",
 418                                         r, tx_strm.avail_in);
 419                                 exit_cleanup(RERR_STREAMIO);
 420                         }
 421                 } while (toklen > 0);
 422         }
 423 }
 424
 425 /* tells us what the receiver is in the middle of doing */
 426 static enum { r_init, r_idle, r_running, r_inflating, r_inflated } recv_state;
 427
 428 /* for inflating stuff */
 429 static z_stream rx_strm;
 430 static char *cbuf;
 431 static char *dbuf;
 432
 433 /* for decoding runs of tokens */
 434 static int32 rx_token;
 435 static int32 rx_run;
 436
 437 /* Receive a deflated token and inflate it */
 438 static int32 recv_deflated_token(int f, char **data)
 439 {
 440         static int init_done;
 441         static int32 saved_flag;
 442         int32 n, flag;
 443         int r;
 444
 445         for (;;) {
 446                 switch (recv_state) {
 447                 case r_init:
 448                         if (!init_done) {
 449                                 rx_strm.next_out = NULL;
 450                                 rx_strm.zalloc = NULL;
 451                                 rx_strm.zfree = NULL;
 452                                 if (inflateInit2(&rx_strm, -15) != Z_OK) {
 453                                         rprintf(FERROR, "inflate init failed\n");
 454                                         exit_cleanup(RERR_STREAMIO);
 455                                 }
 456                                 if (!(cbuf = new_array(char, MAX_DATA_COUNT))
 457                                     || !(dbuf = new_array(char, AVAIL_OUT_SIZE(CHUNK_SIZE))))
 458                                         out_of_memory("recv_deflated_token");
 459                                 init_done = 1;
 460                         } else {
 461                                 inflateReset(&rx_strm);
 462                         }
 463                         recv_state = r_idle;
 464                         rx_token = 0;
 465                         break;
 466
 467                 case r_idle:
 468                 case r_inflated:
 469                         if (saved_flag) {
 470                                 flag = saved_flag & 0xff;
 471                                 saved_flag = 0;
 472                         } else
 473                                 flag = read_byte(f);
 474                         if ((flag & 0xC0) == DEFLATED_DATA) {
 475                                 n = ((flag & 0x3f) << 8) + read_byte(f);
 476                                 read_buf(f, cbuf, n);
 477                                 rx_strm.next_in = (Bytef *)cbuf;
 478                                 rx_strm.avail_in = n;
 479                                 recv_state = r_inflating;
 480                                 break;
 481                         }
 482                         if (recv_state == r_inflated) {
 483                                 /* check previous inflated stuff ended correctly */
 484                                 rx_strm.avail_in = 0;
 485                                 rx_strm.next_out = (Bytef *)dbuf;
 486                                 rx_strm.avail_out = AVAIL_OUT_SIZE(CHUNK_SIZE);
 487                                 r = inflate(&rx_strm, Z_SYNC_FLUSH);
 488                                 n = AVAIL_OUT_SIZE(CHUNK_SIZE) - rx_strm.avail_out;
 489                                 /*
 490                                  * Z_BUF_ERROR just means no progress was
 491                                  * made, i.e. the decompressor didn't have
 492                                  * any pending output for us.
 493                                  */
 494                                 if (r != Z_OK && r != Z_BUF_ERROR) {
 495                                         rprintf(FERROR, "inflate flush returned %d (%d bytes)\n",
 496                                                 r, n);
 497                                         exit_cleanup(RERR_STREAMIO);
 498                                 }
 499                                 if (n != 0 && r != Z_BUF_ERROR) {
 500                                         /* have to return some more data and
 501                                            save the flag for later. */
 502                                         saved_flag = flag + 0x10000;
 503                                         *data = dbuf;
 504                                         return n;
 505                                 }
 506                                 /*
 507                                  * At this point the decompressor should
 508                                  * be expecting to see the 0, 0, ff, ff bytes.
 509                                  */
 510                                 if (!inflateSyncPoint(&rx_strm)) {
 511                                         rprintf(FERROR, "decompressor lost sync!\n");
 512                                         exit_cleanup(RERR_STREAMIO);
 513                                 }
 514                                 rx_strm.avail_in = 4;
 515                                 rx_strm.next_in = (Bytef *)cbuf;
 516                                 cbuf[0] = cbuf[1] = 0;
 517                                 cbuf[2] = cbuf[3] = 0xff;
 518                                 inflate(&rx_strm, Z_SYNC_FLUSH);
 519                                 recv_state = r_idle;
 520                         }
 521                         if (flag == END_FLAG) {
 522                                 /* that's all folks */
 523                                 recv_state = r_init;
 524                                 return 0;
 525                         }
 526
 527                         /* here we have a token of some kind */
 528                         if (flag & TOKEN_REL) {
 529                                 rx_token += flag & 0x3f;
 530                                 flag >>= 6;
 531                         } else
 532                                 rx_token = read_int(f);
 533                         if (flag & 1) {
 534                                 rx_run = read_byte(f);
 535                                 rx_run += read_byte(f) << 8;
 536                                 recv_state = r_running;
 537                         }
 538                         return -1 - rx_token;
 539
 540                 case r_inflating:
 541                         rx_strm.next_out = (Bytef *)dbuf;
 542                         rx_strm.avail_out = AVAIL_OUT_SIZE(CHUNK_SIZE);
 543                         r = inflate(&rx_strm, Z_NO_FLUSH);
 544                         n = AVAIL_OUT_SIZE(CHUNK_SIZE) - rx_strm.avail_out;
 545                         if (r != Z_OK) {
 546                                 rprintf(FERROR, "inflate returned %d (%d bytes)\n", r, n);
 547                                 exit_cleanup(RERR_STREAMIO);
 548                         }
 549                         if (rx_strm.avail_in == 0)
 550                                 recv_state = r_inflated;
 551                         if (n != 0) {
 552                                 *data = dbuf;
 553                                 return n;
 554                         }
 555                         break;
 556
 557                 case r_running:
 558                         ++rx_token;
 559                         if (--rx_run == 0)
 560                                 recv_state = r_idle;
 561                         return -1 - rx_token;
 562                 }
 563         }
 564 }
 565
 566 /*
 567  * put the data corresponding to a token that we've just returned
 568  * from recv_deflated_token into the decompressor's history buffer.
 569  */
 570 static void see_deflate_token(char *buf, int32 len)
 571 {
 572         int r;
 573         int32 blklen;
 574         unsigned char hdr[5];
 575
 576         rx_strm.avail_in = 0;
 577         blklen = 0;
 578         hdr[0] = 0;
 579         do {
 580                 if (rx_strm.avail_in == 0 && len != 0) {
 581                         if (blklen == 0) {
 582                                 /* Give it a fake stored-block header. */
 583                                 rx_strm.next_in = (Bytef *)hdr;
 584                                 rx_strm.avail_in = 5;
 585                                 blklen = len;
 586                                 if (blklen > 0xffff)
 587                                         blklen = 0xffff;
 588                                 hdr[1] = blklen;
 589                                 hdr[2] = blklen >> 8;
 590                                 hdr[3] = ~hdr[1];
 591                                 hdr[4] = ~hdr[2];
 592                         } else {
 593                                 rx_strm.next_in = (Bytef *)buf;
 594                                 rx_strm.avail_in = blklen;
 595                                 len -= blklen;
 596                                 blklen = 0;
 597                         }
 598                 }
 599                 rx_strm.next_out = (Bytef *)dbuf;
 600                 rx_strm.avail_out = AVAIL_OUT_SIZE(CHUNK_SIZE);
 601                 r = inflate(&rx_strm, Z_SYNC_FLUSH);
 602                 if (r != Z_OK) {
 603                         rprintf(FERROR, "inflate (token) returned %d\n", r);
 604                         exit_cleanup(RERR_STREAMIO);
 605                 }
 606         } while (len || rx_strm.avail_out == 0);
 607 }
 608
 609 /**
 610  * Transmit a verbatim buffer of length @p n followed by a token.
 611  * If token == -1 then we have reached EOF
 612  * If n == 0 then don't send a buffer
 613  */
 614 void send_token(int f, int32 token, struct map_struct *buf, OFF_T offset,
 615                 int32 n, int32 toklen)
 616 {
 617         if (!do_compression)
 618                 simple_send_token(f, token, buf, offset, n);
 619         else
 620                 send_deflated_token(f, token, buf, offset, n, toklen);
 621 }
 622
 623 /*
 624  * receive a token or buffer from the other end. If the reurn value is >0 then
 625  * it is a data buffer of that length, and *data will point at the data.
 626  * if the return value is -i then it represents token i-1
 627  * if the return value is 0 then the end has been reached
 628  */
 629 int32 recv_token(int f, char **data)
 630 {
 631         int tok;
 632
 633         if (!do_compression) {
 634                 tok = simple_recv_token(f,data);
 635         } else {
 636                 tok = recv_deflated_token(f, data);
 637         }
 638         return tok;
 639 }
 640
 641 /*
 642  * look at the data corresponding to a token, if necessary
 643  */
 644 void see_token(char *data, int32 toklen)
 645 {
 646         if (do_compression)
 647                 see_deflate_token(data, toklen);
 648 }