lib/inflate.c

   1 /*
   2  * Copyright (c) 2018 Stefan Sperling <stsp@openbsd.org>
   3  *
   4  * Permission to use, copy, modify, and distribute this software for any
   5  * purpose with or without fee is hereby granted, provided that the above
   6  * copyright notice and this permission notice appear in all copies.
   7  *
   8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
   9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
  11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  15  */
  16
  17 #include "got_compat.h"
  18
  19 #include <sys/queue.h>
  20
  21 #include <errno.h>
  22 #include <stdio.h>
  23 #include <stdlib.h>
  24 #include <string.h>
  25 #include <poll.h>
  26 #include <unistd.h>
  27 #include <zlib.h>
  28 #include <time.h>
  29
  30 #include "got_error.h"
  31 #include "got_object.h"
  32 #include "got_path.h"
  33
  34 #include "got_lib_hash.h"
  35 #include "got_lib_inflate.h"
  36 #include "got_lib_poll.h"
  37
  38 #ifndef MIN
  39 #define MIN(_a,_b) ((_a) < (_b) ? (_a) : (_b))
  40 #endif
  41
  42 static const struct got_error *
  43 wrap_inflate_error(int zerr, const char *prefix)
  44 {
  45         if  (zerr == Z_ERRNO)
  46                 return got_error_from_errno(prefix);
  47         if  (zerr == Z_MEM_ERROR)
  48                 return got_error_set_errno(ENOMEM, prefix);
  49
  50         return got_error(GOT_ERR_DECOMPRESSION);
  51 }
  52
  53 const struct got_error *
  54 got_inflate_init(struct got_inflate_buf *zb, uint8_t *outbuf, size_t bufsize,
  55     struct got_inflate_checksum *csum)
  56 {
  57         const struct got_error *err = NULL;
  58         int zerr;
  59
  60         memset(zb, 0, sizeof(*zb));
  61
  62         zb->z.zalloc = Z_NULL;
  63         zb->z.zfree = Z_NULL;
  64         zerr = inflateInit(&zb->z);
  65         if (zerr != Z_OK)
  66                 return wrap_inflate_error(zerr, "inflateInit");
  67
  68         zb->inlen = zb->outlen = bufsize;
  69
  70         zb->inbuf = calloc(1, zb->inlen);
  71         if (zb->inbuf == NULL) {
  72                 err = got_error_from_errno("calloc");
  73                 goto done;
  74         }
  75
  76         zb->flags = 0;
  77         if (outbuf == NULL) {
  78                 zb->outbuf = calloc(1, zb->outlen);
  79                 if (zb->outbuf == NULL) {
  80                         err = got_error_from_errno("calloc");
  81                         goto done;
  82                 }
  83                 zb->flags |= GOT_INFLATE_F_OWN_OUTBUF;
  84         } else
  85                 zb->outbuf = outbuf;
  86
  87         zb->csum = csum;
  88 done:
  89         if (err)
  90                 got_inflate_end(zb);
  91         return err;
  92 }
  93
  94 static void
  95 csum_input(struct got_inflate_checksum *csum, const uint8_t *buf, size_t len)
  96 {
  97         if (csum->input_crc)
  98                 *csum->input_crc = crc32(*csum->input_crc, buf, len);
  99
 100         if (csum->input_ctx)
 101                 got_hash_update(csum->input_ctx, buf, len);
 102 }
 103
 104 static void
 105 csum_output(struct got_inflate_checksum *csum, const uint8_t *buf, size_t len)
 106 {
 107         if (csum->output_crc)
 108                 *csum->output_crc = crc32(*csum->output_crc, buf, len);
 109
 110         if (csum->output_ctx)
 111                 got_hash_update(csum->output_ctx, buf, len);
 112 }
 113
 114 const struct got_error *
 115 got_inflate_read(struct got_inflate_buf *zb, FILE *f, size_t *outlenp,
 116     size_t *consumed)
 117 {
 118         size_t last_total_out = zb->z.total_out;
 119         size_t last_total_in = zb->z.total_in;
 120         z_stream *z = &zb->z;
 121         int ret = Z_ERRNO;
 122
 123         z->next_out = zb->outbuf;
 124         z->avail_out = zb->outlen;
 125
 126         *outlenp = 0;
 127         if (consumed)
 128                 *consumed = 0;
 129         do {
 130                 uint8_t *csum_in = NULL, *csum_out = NULL;
 131                 size_t csum_avail_in = 0, csum_avail_out = 0;
 132
 133                 if (z->avail_in == 0) {
 134                         size_t n = fread(zb->inbuf, 1, zb->inlen, f);
 135                         if (n == 0) {
 136                                 if (ferror(f))
 137                                         return got_ferror(f, GOT_ERR_IO);
 138                                 /* EOF */
 139                                 ret = Z_STREAM_END;
 140                                 break;
 141                         }
 142                         z->next_in = zb->inbuf;
 143                         z->avail_in = n;
 144                 }
 145                 if (zb->csum) {
 146                         csum_in = z->next_in;
 147                         csum_avail_in = z->avail_in;
 148                         csum_out = z->next_out;
 149                         csum_avail_out = z->avail_out;
 150                 }
 151                 ret = inflate(z, Z_SYNC_FLUSH);
 152                 if (zb->csum) {
 153                         csum_input(zb->csum, csum_in,
 154                             csum_avail_in - z->avail_in);
 155                         csum_output(zb->csum, csum_out,
 156                             csum_avail_out - z->avail_out);
 157                 }
 158         } while (ret == Z_OK && z->avail_out > 0);
 159
 160         if (ret == Z_OK || ret == Z_BUF_ERROR) {
 161                 zb->flags |= GOT_INFLATE_F_HAVE_MORE;
 162         } else {
 163                 if (ret != Z_STREAM_END)
 164                         return wrap_inflate_error(ret, "inflate");
 165                 zb->flags &= ~GOT_INFLATE_F_HAVE_MORE;
 166         }
 167
 168         *outlenp = z->total_out - last_total_out;
 169         if (consumed)
 170                 *consumed += z->total_in - last_total_in;
 171         return NULL;
 172 }
 173
 174 const struct got_error *
 175 got_inflate_read_fd(struct got_inflate_buf *zb, int fd, size_t *outlenp,
 176     size_t *consumed)
 177 {
 178         const struct got_error *err = NULL;
 179         size_t last_total_out = zb->z.total_out;
 180         size_t last_total_in = zb->z.total_in;
 181         z_stream *z = &zb->z;
 182         int ret = Z_ERRNO;
 183
 184         z->next_out = zb->outbuf;
 185         z->avail_out = zb->outlen;
 186
 187         *outlenp = 0;
 188         if (consumed)
 189                 *consumed = 0;
 190         do {
 191                 uint8_t *csum_in = NULL, *csum_out = NULL;
 192                 size_t csum_avail_in = 0, csum_avail_out = 0;
 193
 194                 if (z->avail_in == 0) {
 195                         ssize_t n;
 196                         err = got_poll_fd(fd, POLLIN, INFTIM);
 197                         if (err) {
 198                                 if (err->code == GOT_ERR_EOF) {
 199                                         ret = Z_STREAM_END;
 200                                         break;
 201                                 }
 202                                 return err;
 203                         }
 204                         n = read(fd, zb->inbuf, zb->inlen);
 205                         if (n < 0)
 206                                 return got_error_from_errno("read");
 207                         else if (n == 0) {
 208                                 /* EOF */
 209                                 ret = Z_STREAM_END;
 210                                 break;
 211                         }
 212                         z->next_in = zb->inbuf;
 213                         z->avail_in = n;
 214                 }
 215                 if (zb->csum) {
 216                         csum_in = z->next_in;
 217                         csum_avail_in = z->avail_in;
 218                         csum_out = z->next_out;
 219                         csum_avail_out = z->avail_out;
 220                 }
 221                 ret = inflate(z, Z_SYNC_FLUSH);
 222                 if (zb->csum) {
 223                         csum_input(zb->csum, csum_in,
 224                             csum_avail_in - z->avail_in);
 225                         csum_output(zb->csum, csum_out,
 226                             csum_avail_out - z->avail_out);
 227                 }
 228         } while (ret == Z_OK && z->avail_out > 0);
 229
 230         if (ret == Z_OK || ret == Z_BUF_ERROR) {
 231                 zb->flags |= GOT_INFLATE_F_HAVE_MORE;
 232         } else {
 233                 if (ret != Z_STREAM_END)
 234                         return wrap_inflate_error(ret, "inflate");
 235                 zb->flags &= ~GOT_INFLATE_F_HAVE_MORE;
 236         }
 237
 238         *outlenp = z->total_out - last_total_out;
 239         if (consumed)
 240                 *consumed += z->total_in - last_total_in;
 241         return NULL;
 242 }
 243
 244 const struct got_error *
 245 got_inflate_read_mmap(struct got_inflate_buf *zb, uint8_t *map, size_t offset,
 246     size_t len, size_t *outlenp, size_t *consumed)
 247 {
 248         size_t last_total_out = zb->z.total_out;
 249         z_stream *z = &zb->z;
 250         int ret = Z_ERRNO;
 251
 252         z->next_out = zb->outbuf;
 253         z->avail_out = zb->outlen;
 254
 255         *outlenp = 0;
 256         *consumed = 0;
 257
 258         do {
 259                 uint8_t *csum_in = NULL, *csum_out = NULL;
 260                 size_t csum_avail_in = 0, csum_avail_out = 0;
 261                 size_t last_total_in = zb->z.total_in;
 262
 263                 if (z->avail_in == 0) {
 264                         if (len == 0) {
 265                                 /* EOF */
 266                                 ret = Z_STREAM_END;
 267                                 break;
 268                         }
 269                         z->next_in = map + offset + *consumed;
 270                         if (len - *consumed > UINT_MAX)
 271                                 z->avail_in = UINT_MAX;
 272                         else
 273                                 z->avail_in = len - *consumed;
 274                 }
 275                 if (zb->csum) {
 276                         csum_in = z->next_in;
 277                         csum_avail_in = z->avail_in;
 278                         csum_out = z->next_out;
 279                         csum_avail_out = z->avail_out;
 280                 }
 281                 ret = inflate(z, Z_SYNC_FLUSH);
 282                 if (zb->csum) {
 283                         csum_input(zb->csum, csum_in,
 284                             csum_avail_in - z->avail_in);
 285                         csum_output(zb->csum, csum_out,
 286                             csum_avail_out - z->avail_out);
 287                 }
 288                 *consumed += z->total_in - last_total_in;
 289         } while (ret == Z_OK && z->avail_out > 0);
 290
 291         if (ret == Z_OK || ret == Z_BUF_ERROR) {
 292                 zb->flags |= GOT_INFLATE_F_HAVE_MORE;
 293         } else {
 294                 if (ret != Z_STREAM_END)
 295                         return wrap_inflate_error(ret, "inflate");
 296                 zb->flags &= ~GOT_INFLATE_F_HAVE_MORE;
 297         }
 298
 299         *outlenp = z->total_out - last_total_out;
 300         return NULL;
 301 }
 302
 303 void
 304 got_inflate_end(struct got_inflate_buf *zb)
 305 {
 306         free(zb->inbuf);
 307         if (zb->flags & GOT_INFLATE_F_OWN_OUTBUF)
 308                 free(zb->outbuf);
 309         inflateEnd(&zb->z);
 310 }
 311
 312 const struct got_error *
 313 got_inflate_to_mem(uint8_t **outbuf, size_t *outlen,
 314     size_t *consumed_total, struct got_inflate_checksum *csum, FILE *f)
 315 {
 316         const struct got_error *err;
 317         size_t avail, consumed;
 318         struct got_inflate_buf zb;
 319         void *newbuf;
 320         int nbuf = 1;
 321
 322         if (outbuf) {
 323                 *outbuf = malloc(GOT_INFLATE_BUFSIZE);
 324                 if (*outbuf == NULL)
 325                         return got_error_from_errno("malloc");
 326                 err = got_inflate_init(&zb, *outbuf, GOT_INFLATE_BUFSIZE, csum);
 327         } else
 328                 err = got_inflate_init(&zb, NULL, GOT_INFLATE_BUFSIZE, csum);
 329         if (err)
 330                 return err;
 331
 332         *outlen = 0;
 333         if (consumed_total)
 334                 *consumed_total = 0;
 335
 336         do {
 337                 err = got_inflate_read(&zb, f, &avail, &consumed);
 338                 if (err)
 339                         goto done;
 340                 *outlen += avail;
 341                 if (consumed_total)
 342                         *consumed_total += consumed;
 343                 if (zb.flags & GOT_INFLATE_F_HAVE_MORE) {
 344                         if (outbuf == NULL)
 345                                 continue;
 346                         newbuf = reallocarray(*outbuf, ++nbuf,
 347                             GOT_INFLATE_BUFSIZE);
 348                         if (newbuf == NULL) {
 349                                 err = got_error_from_errno("reallocarray");
 350                                 free(*outbuf);
 351                                 *outbuf = NULL;
 352                                 *outlen = 0;
 353                                 goto done;
 354                         }
 355                         *outbuf = newbuf;
 356                         zb.outbuf = newbuf + *outlen;
 357                         zb.outlen = (nbuf * GOT_INFLATE_BUFSIZE) - *outlen;
 358                 }
 359         } while (zb.flags & GOT_INFLATE_F_HAVE_MORE);
 360
 361 done:
 362         got_inflate_end(&zb);
 363         return err;
 364 }
 365
 366 const struct got_error *
 367 got_inflate_to_mem_fd(uint8_t **outbuf, size_t *outlen,
 368     size_t *consumed_total, struct got_inflate_checksum *csum,
 369     size_t expected_size, int infd)
 370 {
 371         const struct got_error *err;
 372         size_t avail, consumed;
 373         struct got_inflate_buf zb;
 374         void *newbuf;
 375         int nbuf = 1;
 376         size_t bufsize = GOT_INFLATE_BUFSIZE;
 377
 378         /* Optimize buffer size in case short reads should suffice. */
 379         if (expected_size > 0 && expected_size < bufsize)
 380                 bufsize = expected_size;
 381
 382         if (outbuf) {
 383                 *outbuf = malloc(bufsize);
 384                 if (*outbuf == NULL)
 385                         return got_error_from_errno("malloc");
 386                 err = got_inflate_init(&zb, *outbuf, GOT_INFLATE_BUFSIZE, csum);
 387         } else
 388                 err = got_inflate_init(&zb, NULL, bufsize, csum);
 389         if (err)
 390                 goto done;
 391
 392         *outlen = 0;
 393         if (consumed_total)
 394                 *consumed_total = 0;
 395
 396         do {
 397                 err = got_inflate_read_fd(&zb, infd, &avail, &consumed);
 398                 if (err)
 399                         goto done;
 400                 *outlen += avail;
 401                 if (consumed_total)
 402                         *consumed_total += consumed;
 403                 if (zb.flags & GOT_INFLATE_F_HAVE_MORE) {
 404                         if (outbuf == NULL)
 405                                 continue;
 406                         newbuf = reallocarray(*outbuf, ++nbuf,
 407                             GOT_INFLATE_BUFSIZE);
 408                         if (newbuf == NULL) {
 409                                 err = got_error_from_errno("reallocarray");
 410                                 free(*outbuf);
 411                                 *outbuf = NULL;
 412                                 *outlen = 0;
 413                                 goto done;
 414                         }
 415                         *outbuf = newbuf;
 416                         zb.outbuf = newbuf + *outlen;
 417                         zb.outlen = (nbuf * GOT_INFLATE_BUFSIZE) - *outlen;
 418                 }
 419         } while (zb.flags & GOT_INFLATE_F_HAVE_MORE);
 420
 421 done:
 422         got_inflate_end(&zb);
 423         return err;
 424 }
 425
 426 const struct got_error *
 427 got_inflate_to_mem_mmap(uint8_t **outbuf, size_t *outlen,
 428     size_t *consumed_total, struct got_inflate_checksum *csum, uint8_t *map,
 429     size_t offset, size_t len)
 430 {
 431         const struct got_error *err;
 432         size_t avail, consumed;
 433         struct got_inflate_buf zb;
 434         void *newbuf;
 435         int nbuf = 1;
 436
 437         if (outbuf) {
 438                 *outbuf = malloc(GOT_INFLATE_BUFSIZE);
 439                 if (*outbuf == NULL)
 440                         return got_error_from_errno("malloc");
 441                 err = got_inflate_init(&zb, *outbuf, GOT_INFLATE_BUFSIZE, csum);
 442                 if (err) {
 443                         free(*outbuf);
 444                         *outbuf = NULL;
 445                         return err;
 446                 }
 447         } else {
 448                 err = got_inflate_init(&zb, NULL, GOT_INFLATE_BUFSIZE, csum);
 449                 if (err)
 450                         return err;
 451         }
 452
 453         *outlen = 0;
 454         if (consumed_total)
 455                 *consumed_total = 0;
 456         do {
 457                 err = got_inflate_read_mmap(&zb, map, offset, len, &avail,
 458                     &consumed);
 459                 if (err)
 460                         goto done;
 461                 offset += consumed;
 462                 if (consumed_total)
 463                         *consumed_total += consumed;
 464                 len -= consumed;
 465                 *outlen += avail;
 466                 if (len == 0)
 467                         break;
 468                 if (zb.flags & GOT_INFLATE_F_HAVE_MORE) {
 469                         if (outbuf == NULL)
 470                                 continue;
 471                         newbuf = reallocarray(*outbuf, ++nbuf,
 472                             GOT_INFLATE_BUFSIZE);
 473                         if (newbuf == NULL) {
 474                                 err = got_error_from_errno("reallocarray");
 475                                 free(*outbuf);
 476                                 *outbuf = NULL;
 477                                 *outlen = 0;
 478                                 goto done;
 479                         }
 480                         *outbuf = newbuf;
 481                         zb.outbuf = newbuf + *outlen;
 482                         zb.outlen = (nbuf * GOT_INFLATE_BUFSIZE) - *outlen;
 483                 }
 484         } while (zb.flags & GOT_INFLATE_F_HAVE_MORE);
 485 done:
 486         got_inflate_end(&zb);
 487         return err;
 488 }
 489
 490 const struct got_error *
 491 got_inflate_to_fd(size_t *outlen, FILE *infile,
 492     struct got_inflate_checksum *csum, int outfd)
 493 {
 494         const struct got_error *err = NULL;
 495         size_t avail;
 496         struct got_inflate_buf zb;
 497
 498         err = got_inflate_init(&zb, NULL, GOT_INFLATE_BUFSIZE, csum);
 499         if (err)
 500                 goto done;
 501
 502         *outlen = 0;
 503
 504         do {
 505                 err = got_inflate_read(&zb, infile, &avail, NULL);
 506                 if (err)
 507                         goto done;
 508                 if (avail > 0) {
 509                         ssize_t n;
 510                         n = write(outfd, zb.outbuf, avail);
 511                         if (n != avail) {
 512                                 err = got_error_from_errno("write");
 513                                 goto done;
 514                         }
 515                         *outlen += avail;
 516                 }
 517         } while (zb.flags & GOT_INFLATE_F_HAVE_MORE);
 518
 519 done:
 520         if (err == NULL) {
 521                 if (lseek(outfd, SEEK_SET, 0) == -1)
 522                         err = got_error_from_errno("lseek");
 523         }
 524         got_inflate_end(&zb);
 525         return err;
 526 }
 527
 528 const struct got_error *
 529 got_inflate_to_file(size_t *outlen, FILE *infile,
 530     struct got_inflate_checksum *csum, FILE *outfile)
 531 {
 532         const struct got_error *err;
 533         size_t avail;
 534         struct got_inflate_buf zb;
 535
 536         err = got_inflate_init(&zb, NULL, GOT_INFLATE_BUFSIZE, csum);
 537         if (err)
 538                 goto done;
 539
 540         *outlen = 0;
 541
 542         do {
 543                 err = got_inflate_read(&zb, infile, &avail, NULL);
 544                 if (err)
 545                         goto done;
 546                 if (avail > 0) {
 547                         size_t n;
 548                         n = fwrite(zb.outbuf, avail, 1, outfile);
 549                         if (n != 1) {
 550                                 err = got_ferror(outfile, GOT_ERR_IO);
 551                                 goto done;
 552                         }
 553                         *outlen += avail;
 554                 }
 555         } while (zb.flags & GOT_INFLATE_F_HAVE_MORE);
 556
 557 done:
 558         if (err == NULL)
 559                 rewind(outfile);
 560         got_inflate_end(&zb);
 561         return err;
 562 }
 563
 564 const struct got_error *
 565 got_inflate_to_file_fd(size_t *outlen, size_t *consumed_total,
 566     struct got_inflate_checksum *csum, int infd, FILE *outfile)
 567 {
 568         const struct got_error *err;
 569         size_t avail, consumed;
 570         struct got_inflate_buf zb;
 571
 572         err = got_inflate_init(&zb, NULL, GOT_INFLATE_BUFSIZE, csum);
 573         if (err)
 574                 goto done;
 575
 576         *outlen = 0;
 577         if (consumed_total)
 578                 *consumed_total = 0;
 579         do {
 580                 err = got_inflate_read_fd(&zb, infd, &avail, &consumed);
 581                 if (err)
 582                         goto done;
 583                 if (avail > 0) {
 584                         size_t n;
 585                         n = fwrite(zb.outbuf, avail, 1, outfile);
 586                         if (n != 1) {
 587                                 err = got_ferror(outfile, GOT_ERR_IO);
 588                                 goto done;
 589                         }
 590                         *outlen += avail;
 591                         if (consumed_total)
 592                                 *consumed_total += consumed;
 593                 }
 594         } while (zb.flags & GOT_INFLATE_F_HAVE_MORE);
 595
 596 done:
 597         if (err == NULL)
 598                 rewind(outfile);
 599         got_inflate_end(&zb);
 600         return err;
 601 }
 602
 603 const struct got_error *
 604 got_inflate_to_file_mmap(size_t *outlen, size_t *consumed_total,
 605     struct got_inflate_checksum *csum, uint8_t *map, size_t offset,
 606     size_t len, FILE *outfile)
 607 {
 608         const struct got_error *err;
 609         size_t avail, consumed;
 610         struct got_inflate_buf zb;
 611
 612         err = got_inflate_init(&zb, NULL, GOT_INFLATE_BUFSIZE, csum);
 613         if (err)
 614                 goto done;
 615
 616         *outlen = 0;
 617         if (consumed_total)
 618                 *consumed_total = 0;
 619         do {
 620                 err = got_inflate_read_mmap(&zb, map, offset, len, &avail,
 621                     &consumed);
 622                 if (err)
 623                         goto done;
 624                 offset += consumed;
 625                 if (consumed_total)
 626                         *consumed_total += consumed;
 627                 len -= consumed;
 628                 if (avail > 0) {
 629                         size_t n;
 630                         n = fwrite(zb.outbuf, avail, 1, outfile);
 631                         if (n != 1) {
 632                                 err = got_ferror(outfile, GOT_ERR_IO);
 633                                 goto done;
 634                         }
 635                         *outlen += avail;
 636                 }
 637         } while (zb.flags & GOT_INFLATE_F_HAVE_MORE);
 638
 639 done:
 640         if (err == NULL)
 641                 rewind(outfile);
 642         got_inflate_end(&zb);
 643         return err;
 644 }