lib/inflate.c

   1 /*
   2  * Copyright (c) 2018 Stefan Sperling <stsp@openbsd.org>
   3  *
   4  * Permission to use, copy, modify, and distribute this software for any
   5  * purpose with or without fee is hereby granted, provided that the above
   6  * copyright notice and this permission notice appear in all copies.
   7  *
   8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
   9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
  11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  15  */
  16
  17 #include "got_compat.h"
  18
  19 #include <sys/queue.h>
  20
  21 #include <errno.h>
  22 #include <stdio.h>
  23 #include <stdlib.h>
  24 #include <string.h>
  25 #include <poll.h>
  26 #include <unistd.h>
  27 #include <zlib.h>
  28 #include <time.h>
  29
  30 #include "got_error.h"
  31 #include "got_object.h"
  32 #include "got_path.h"
  33
  34 #include "got_lib_hash.h"
  35 #include "got_lib_inflate.h"
  36 #include "got_lib_poll.h"
  37
  38 #ifndef MIN
  39 #define MIN(_a,_b) ((_a) < (_b) ? (_a) : (_b))
  40 #endif
  41
  42 const struct got_error *
  43 got_inflate_init(struct got_inflate_buf *zb, uint8_t *outbuf, size_t bufsize,
  44     struct got_inflate_checksum *csum)
  45 {
  46         const struct got_error *err = NULL;
  47         int zerr;
  48
  49         memset(zb, 0, sizeof(*zb));
  50
  51         zb->z.zalloc = Z_NULL;
  52         zb->z.zfree = Z_NULL;
  53         zerr = inflateInit(&zb->z);
  54         if (zerr != Z_OK) {
  55                 if  (zerr == Z_ERRNO)
  56                         return got_error_from_errno("inflateInit");
  57                 if  (zerr == Z_MEM_ERROR) {
  58                         errno = ENOMEM;
  59                         return got_error_from_errno("inflateInit");
  60                 }
  61                 return got_error(GOT_ERR_DECOMPRESSION);
  62         }
  63
  64         zb->inlen = zb->outlen = bufsize;
  65
  66         zb->inbuf = calloc(1, zb->inlen);
  67         if (zb->inbuf == NULL) {
  68                 err = got_error_from_errno("calloc");
  69                 goto done;
  70         }
  71
  72         zb->flags = 0;
  73         if (outbuf == NULL) {
  74                 zb->outbuf = calloc(1, zb->outlen);
  75                 if (zb->outbuf == NULL) {
  76                         err = got_error_from_errno("calloc");
  77                         goto done;
  78                 }
  79                 zb->flags |= GOT_INFLATE_F_OWN_OUTBUF;
  80         } else
  81                 zb->outbuf = outbuf;
  82
  83         zb->csum = csum;
  84 done:
  85         if (err)
  86                 got_inflate_end(zb);
  87         return err;
  88 }
  89
  90 static void
  91 csum_input(struct got_inflate_checksum *csum, const uint8_t *buf, size_t len)
  92 {
  93         if (csum->input_crc)
  94                 *csum->input_crc = crc32(*csum->input_crc, buf, len);
  95
  96         if (csum->input_ctx)
  97                 got_hash_update(csum->input_ctx, buf, len);
  98 }
  99
 100 static void
 101 csum_output(struct got_inflate_checksum *csum, const uint8_t *buf, size_t len)
 102 {
 103         if (csum->output_crc)
 104                 *csum->output_crc = crc32(*csum->output_crc, buf, len);
 105
 106         if (csum->output_ctx)
 107                 got_hash_update(csum->output_ctx, buf, len);
 108 }
 109
 110 const struct got_error *
 111 got_inflate_read(struct got_inflate_buf *zb, FILE *f, size_t *outlenp,
 112     size_t *consumed)
 113 {
 114         size_t last_total_out = zb->z.total_out;
 115         size_t last_total_in = zb->z.total_in;
 116         z_stream *z = &zb->z;
 117         int ret = Z_ERRNO;
 118
 119         z->next_out = zb->outbuf;
 120         z->avail_out = zb->outlen;
 121
 122         *outlenp = 0;
 123         if (consumed)
 124                 *consumed = 0;
 125         do {
 126                 uint8_t *csum_in = NULL, *csum_out = NULL;
 127                 size_t csum_avail_in = 0, csum_avail_out = 0;
 128
 129                 if (z->avail_in == 0) {
 130                         size_t n = fread(zb->inbuf, 1, zb->inlen, f);
 131                         if (n == 0) {
 132                                 if (ferror(f))
 133                                         return got_ferror(f, GOT_ERR_IO);
 134                                 /* EOF */
 135                                 ret = Z_STREAM_END;
 136                                 break;
 137                         }
 138                         z->next_in = zb->inbuf;
 139                         z->avail_in = n;
 140                 }
 141                 if (zb->csum) {
 142                         csum_in = z->next_in;
 143                         csum_avail_in = z->avail_in;
 144                         csum_out = z->next_out;
 145                         csum_avail_out = z->avail_out;
 146                 }
 147                 ret = inflate(z, Z_SYNC_FLUSH);
 148                 if (zb->csum) {
 149                         csum_input(zb->csum, csum_in,
 150                             csum_avail_in - z->avail_in);
 151                         csum_output(zb->csum, csum_out,
 152                             csum_avail_out - z->avail_out);
 153                 }
 154         } while (ret == Z_OK && z->avail_out > 0);
 155
 156         if (ret == Z_OK || ret == Z_BUF_ERROR) {
 157                 zb->flags |= GOT_INFLATE_F_HAVE_MORE;
 158         } else {
 159                 if (ret != Z_STREAM_END)
 160                         return got_error(GOT_ERR_DECOMPRESSION);
 161                 zb->flags &= ~GOT_INFLATE_F_HAVE_MORE;
 162         }
 163
 164         *outlenp = z->total_out - last_total_out;
 165         if (consumed)
 166                 *consumed += z->total_in - last_total_in;
 167         return NULL;
 168 }
 169
 170 const struct got_error *
 171 got_inflate_read_fd(struct got_inflate_buf *zb, int fd, size_t *outlenp,
 172     size_t *consumed)
 173 {
 174         const struct got_error *err = NULL;
 175         size_t last_total_out = zb->z.total_out;
 176         size_t last_total_in = zb->z.total_in;
 177         z_stream *z = &zb->z;
 178         int ret = Z_ERRNO;
 179
 180         z->next_out = zb->outbuf;
 181         z->avail_out = zb->outlen;
 182
 183         *outlenp = 0;
 184         if (consumed)
 185                 *consumed = 0;
 186         do {
 187                 uint8_t *csum_in = NULL, *csum_out = NULL;
 188                 size_t csum_avail_in = 0, csum_avail_out = 0;
 189
 190                 if (z->avail_in == 0) {
 191                         ssize_t n;
 192                         err = got_poll_fd(fd, POLLIN, INFTIM);
 193                         if (err) {
 194                                 if (err->code == GOT_ERR_EOF) {
 195                                         ret = Z_STREAM_END;
 196                                         break;
 197                                 }
 198                                 return err;
 199                         }
 200                         n = read(fd, zb->inbuf, zb->inlen);
 201                         if (n < 0)
 202                                 return got_error_from_errno("read");
 203                         else if (n == 0) {
 204                                 /* EOF */
 205                                 ret = Z_STREAM_END;
 206                                 break;
 207                         }
 208                         z->next_in = zb->inbuf;
 209                         z->avail_in = n;
 210                 }
 211                 if (zb->csum) {
 212                         csum_in = z->next_in;
 213                         csum_avail_in = z->avail_in;
 214                         csum_out = z->next_out;
 215                         csum_avail_out = z->avail_out;
 216                 }
 217                 ret = inflate(z, Z_SYNC_FLUSH);
 218                 if (zb->csum) {
 219                         csum_input(zb->csum, csum_in,
 220                             csum_avail_in - z->avail_in);
 221                         csum_output(zb->csum, csum_out,
 222                             csum_avail_out - z->avail_out);
 223                 }
 224         } while (ret == Z_OK && z->avail_out > 0);
 225
 226         if (ret == Z_OK || ret == Z_BUF_ERROR) {
 227                 zb->flags |= GOT_INFLATE_F_HAVE_MORE;
 228         } else {
 229                 if (ret != Z_STREAM_END)
 230                         return got_error(GOT_ERR_DECOMPRESSION);
 231                 zb->flags &= ~GOT_INFLATE_F_HAVE_MORE;
 232         }
 233
 234         *outlenp = z->total_out - last_total_out;
 235         if (consumed)
 236                 *consumed += z->total_in - last_total_in;
 237         return NULL;
 238 }
 239
 240 const struct got_error *
 241 got_inflate_read_mmap(struct got_inflate_buf *zb, uint8_t *map, size_t offset,
 242     size_t len, size_t *outlenp, size_t *consumed)
 243 {
 244         size_t last_total_out = zb->z.total_out;
 245         z_stream *z = &zb->z;
 246         int ret = Z_ERRNO;
 247
 248         z->next_out = zb->outbuf;
 249         z->avail_out = zb->outlen;
 250
 251         *outlenp = 0;
 252         *consumed = 0;
 253
 254         do {
 255                 uint8_t *csum_in = NULL, *csum_out = NULL;
 256                 size_t csum_avail_in = 0, csum_avail_out = 0;
 257                 size_t last_total_in = zb->z.total_in;
 258
 259                 if (z->avail_in == 0) {
 260                         if (len == 0) {
 261                                 /* EOF */
 262                                 ret = Z_STREAM_END;
 263                                 break;
 264                         }
 265                         z->next_in = map + offset + *consumed;
 266                         if (len - *consumed > UINT_MAX)
 267                                 z->avail_in = UINT_MAX;
 268                         else
 269                                 z->avail_in = len - *consumed;
 270                 }
 271                 if (zb->csum) {
 272                         csum_in = z->next_in;
 273                         csum_avail_in = z->avail_in;
 274                         csum_out = z->next_out;
 275                         csum_avail_out = z->avail_out;
 276                 }
 277                 ret = inflate(z, Z_SYNC_FLUSH);
 278                 if (zb->csum) {
 279                         csum_input(zb->csum, csum_in,
 280                             csum_avail_in - z->avail_in);
 281                         csum_output(zb->csum, csum_out,
 282                             csum_avail_out - z->avail_out);
 283                 }
 284                 *consumed += z->total_in - last_total_in;
 285         } while (ret == Z_OK && z->avail_out > 0);
 286
 287         if (ret == Z_OK || ret == Z_BUF_ERROR) {
 288                 zb->flags |= GOT_INFLATE_F_HAVE_MORE;
 289         } else {
 290                 if (ret != Z_STREAM_END)
 291                         return got_error(GOT_ERR_DECOMPRESSION);
 292                 zb->flags &= ~GOT_INFLATE_F_HAVE_MORE;
 293         }
 294
 295         *outlenp = z->total_out - last_total_out;
 296         return NULL;
 297 }
 298
 299 void
 300 got_inflate_end(struct got_inflate_buf *zb)
 301 {
 302         free(zb->inbuf);
 303         if (zb->flags & GOT_INFLATE_F_OWN_OUTBUF)
 304                 free(zb->outbuf);
 305         inflateEnd(&zb->z);
 306 }
 307
 308 const struct got_error *
 309 got_inflate_to_mem(uint8_t **outbuf, size_t *outlen,
 310     size_t *consumed_total, struct got_inflate_checksum *csum, FILE *f)
 311 {
 312         const struct got_error *err;
 313         size_t avail, consumed;
 314         struct got_inflate_buf zb;
 315         void *newbuf;
 316         int nbuf = 1;
 317
 318         if (outbuf) {
 319                 *outbuf = malloc(GOT_INFLATE_BUFSIZE);
 320                 if (*outbuf == NULL)
 321                         return got_error_from_errno("malloc");
 322                 err = got_inflate_init(&zb, *outbuf, GOT_INFLATE_BUFSIZE, csum);
 323         } else
 324                 err = got_inflate_init(&zb, NULL, GOT_INFLATE_BUFSIZE, csum);
 325         if (err)
 326                 return err;
 327
 328         *outlen = 0;
 329         if (consumed_total)
 330                 *consumed_total = 0;
 331
 332         do {
 333                 err = got_inflate_read(&zb, f, &avail, &consumed);
 334                 if (err)
 335                         goto done;
 336                 *outlen += avail;
 337                 if (consumed_total)
 338                         *consumed_total += consumed;
 339                 if (zb.flags & GOT_INFLATE_F_HAVE_MORE) {
 340                         if (outbuf == NULL)
 341                                 continue;
 342                         newbuf = reallocarray(*outbuf, ++nbuf,
 343                             GOT_INFLATE_BUFSIZE);
 344                         if (newbuf == NULL) {
 345                                 err = got_error_from_errno("reallocarray");
 346                                 free(*outbuf);
 347                                 *outbuf = NULL;
 348                                 *outlen = 0;
 349                                 goto done;
 350                         }
 351                         *outbuf = newbuf;
 352                         zb.outbuf = newbuf + *outlen;
 353                         zb.outlen = (nbuf * GOT_INFLATE_BUFSIZE) - *outlen;
 354                 }
 355         } while (zb.flags & GOT_INFLATE_F_HAVE_MORE);
 356
 357 done:
 358         got_inflate_end(&zb);
 359         return err;
 360 }
 361
 362 const struct got_error *
 363 got_inflate_to_mem_fd(uint8_t **outbuf, size_t *outlen,
 364     size_t *consumed_total, struct got_inflate_checksum *csum,
 365     size_t expected_size, int infd)
 366 {
 367         const struct got_error *err;
 368         size_t avail, consumed;
 369         struct got_inflate_buf zb;
 370         void *newbuf;
 371         int nbuf = 1;
 372         size_t bufsize = GOT_INFLATE_BUFSIZE;
 373
 374         /* Optimize buffer size in case short reads should suffice. */
 375         if (expected_size > 0 && expected_size < bufsize)
 376                 bufsize = expected_size;
 377
 378         if (outbuf) {
 379                 *outbuf = malloc(bufsize);
 380                 if (*outbuf == NULL)
 381                         return got_error_from_errno("malloc");
 382                 err = got_inflate_init(&zb, *outbuf, GOT_INFLATE_BUFSIZE, csum);
 383         } else
 384                 err = got_inflate_init(&zb, NULL, bufsize, csum);
 385         if (err)
 386                 goto done;
 387
 388         *outlen = 0;
 389         if (consumed_total)
 390                 *consumed_total = 0;
 391
 392         do {
 393                 err = got_inflate_read_fd(&zb, infd, &avail, &consumed);
 394                 if (err)
 395                         goto done;
 396                 *outlen += avail;
 397                 if (consumed_total)
 398                         *consumed_total += consumed;
 399                 if (zb.flags & GOT_INFLATE_F_HAVE_MORE) {
 400                         if (outbuf == NULL)
 401                                 continue;
 402                         newbuf = reallocarray(*outbuf, ++nbuf,
 403                             GOT_INFLATE_BUFSIZE);
 404                         if (newbuf == NULL) {
 405                                 err = got_error_from_errno("reallocarray");
 406                                 free(*outbuf);
 407                                 *outbuf = NULL;
 408                                 *outlen = 0;
 409                                 goto done;
 410                         }
 411                         *outbuf = newbuf;
 412                         zb.outbuf = newbuf + *outlen;
 413                         zb.outlen = (nbuf * GOT_INFLATE_BUFSIZE) - *outlen;
 414                 }
 415         } while (zb.flags & GOT_INFLATE_F_HAVE_MORE);
 416
 417 done:
 418         got_inflate_end(&zb);
 419         return err;
 420 }
 421
 422 const struct got_error *
 423 got_inflate_to_mem_mmap(uint8_t **outbuf, size_t *outlen,
 424     size_t *consumed_total, struct got_inflate_checksum *csum, uint8_t *map,
 425     size_t offset, size_t len)
 426 {
 427         const struct got_error *err;
 428         size_t avail, consumed;
 429         struct got_inflate_buf zb;
 430         void *newbuf;
 431         int nbuf = 1;
 432
 433         if (outbuf) {
 434                 *outbuf = malloc(GOT_INFLATE_BUFSIZE);
 435                 if (*outbuf == NULL)
 436                         return got_error_from_errno("malloc");
 437                 err = got_inflate_init(&zb, *outbuf, GOT_INFLATE_BUFSIZE, csum);
 438                 if (err) {
 439                         free(*outbuf);
 440                         *outbuf = NULL;
 441                         return err;
 442                 }
 443         } else {
 444                 err = got_inflate_init(&zb, NULL, GOT_INFLATE_BUFSIZE, csum);
 445                 if (err)
 446                         return err;
 447         }
 448
 449         *outlen = 0;
 450         if (consumed_total)
 451                 *consumed_total = 0;
 452         do {
 453                 err = got_inflate_read_mmap(&zb, map, offset, len, &avail,
 454                     &consumed);
 455                 if (err)
 456                         goto done;
 457                 offset += consumed;
 458                 if (consumed_total)
 459                         *consumed_total += consumed;
 460                 len -= consumed;
 461                 *outlen += avail;
 462                 if (len == 0)
 463                         break;
 464                 if (zb.flags & GOT_INFLATE_F_HAVE_MORE) {
 465                         if (outbuf == NULL)
 466                                 continue;
 467                         newbuf = reallocarray(*outbuf, ++nbuf,
 468                             GOT_INFLATE_BUFSIZE);
 469                         if (newbuf == NULL) {
 470                                 err = got_error_from_errno("reallocarray");
 471                                 free(*outbuf);
 472                                 *outbuf = NULL;
 473                                 *outlen = 0;
 474                                 goto done;
 475                         }
 476                         *outbuf = newbuf;
 477                         zb.outbuf = newbuf + *outlen;
 478                         zb.outlen = (nbuf * GOT_INFLATE_BUFSIZE) - *outlen;
 479                 }
 480         } while (zb.flags & GOT_INFLATE_F_HAVE_MORE);
 481 done:
 482         got_inflate_end(&zb);
 483         return err;
 484 }
 485
 486 const struct got_error *
 487 got_inflate_to_fd(size_t *outlen, FILE *infile,
 488     struct got_inflate_checksum *csum, int outfd)
 489 {
 490         const struct got_error *err = NULL;
 491         size_t avail;
 492         struct got_inflate_buf zb;
 493
 494         err = got_inflate_init(&zb, NULL, GOT_INFLATE_BUFSIZE, csum);
 495         if (err)
 496                 goto done;
 497
 498         *outlen = 0;
 499
 500         do {
 501                 err = got_inflate_read(&zb, infile, &avail, NULL);
 502                 if (err)
 503                         goto done;
 504                 if (avail > 0) {
 505                         ssize_t n;
 506                         n = write(outfd, zb.outbuf, avail);
 507                         if (n != avail) {
 508                                 err = got_error_from_errno("write");
 509                                 goto done;
 510                         }
 511                         *outlen += avail;
 512                 }
 513         } while (zb.flags & GOT_INFLATE_F_HAVE_MORE);
 514
 515 done:
 516         if (err == NULL) {
 517                 if (lseek(outfd, SEEK_SET, 0) == -1)
 518                         err = got_error_from_errno("lseek");
 519         }
 520         got_inflate_end(&zb);
 521         return err;
 522 }
 523
 524 const struct got_error *
 525 got_inflate_to_file(size_t *outlen, FILE *infile,
 526     struct got_inflate_checksum *csum, FILE *outfile)
 527 {
 528         const struct got_error *err;
 529         size_t avail;
 530         struct got_inflate_buf zb;
 531
 532         err = got_inflate_init(&zb, NULL, GOT_INFLATE_BUFSIZE, csum);
 533         if (err)
 534                 goto done;
 535
 536         *outlen = 0;
 537
 538         do {
 539                 err = got_inflate_read(&zb, infile, &avail, NULL);
 540                 if (err)
 541                         goto done;
 542                 if (avail > 0) {
 543                         size_t n;
 544                         n = fwrite(zb.outbuf, avail, 1, outfile);
 545                         if (n != 1) {
 546                                 err = got_ferror(outfile, GOT_ERR_IO);
 547                                 goto done;
 548                         }
 549                         *outlen += avail;
 550                 }
 551         } while (zb.flags & GOT_INFLATE_F_HAVE_MORE);
 552
 553 done:
 554         if (err == NULL)
 555                 rewind(outfile);
 556         got_inflate_end(&zb);
 557         return err;
 558 }
 559
 560 const struct got_error *
 561 got_inflate_to_file_fd(size_t *outlen, size_t *consumed_total,
 562     struct got_inflate_checksum *csum, int infd, FILE *outfile)
 563 {
 564         const struct got_error *err;
 565         size_t avail, consumed;
 566         struct got_inflate_buf zb;
 567
 568         err = got_inflate_init(&zb, NULL, GOT_INFLATE_BUFSIZE, csum);
 569         if (err)
 570                 goto done;
 571
 572         *outlen = 0;
 573         if (consumed_total)
 574                 *consumed_total = 0;
 575         do {
 576                 err = got_inflate_read_fd(&zb, infd, &avail, &consumed);
 577                 if (err)
 578                         goto done;
 579                 if (avail > 0) {
 580                         size_t n;
 581                         n = fwrite(zb.outbuf, avail, 1, outfile);
 582                         if (n != 1) {
 583                                 err = got_ferror(outfile, GOT_ERR_IO);
 584                                 goto done;
 585                         }
 586                         *outlen += avail;
 587                         if (consumed_total)
 588                                 *consumed_total += consumed;
 589                 }
 590         } while (zb.flags & GOT_INFLATE_F_HAVE_MORE);
 591
 592 done:
 593         if (err == NULL)
 594                 rewind(outfile);
 595         got_inflate_end(&zb);
 596         return err;
 597 }
 598
 599 const struct got_error *
 600 got_inflate_to_file_mmap(size_t *outlen, size_t *consumed_total,
 601     struct got_inflate_checksum *csum, uint8_t *map, size_t offset,
 602     size_t len, FILE *outfile)
 603 {
 604         const struct got_error *err;
 605         size_t avail, consumed;
 606         struct got_inflate_buf zb;
 607
 608         err = got_inflate_init(&zb, NULL, GOT_INFLATE_BUFSIZE, csum);
 609         if (err)
 610                 goto done;
 611
 612         *outlen = 0;
 613         if (consumed_total)
 614                 *consumed_total = 0;
 615         do {
 616                 err = got_inflate_read_mmap(&zb, map, offset, len, &avail,
 617                     &consumed);
 618                 if (err)
 619                         goto done;
 620                 offset += consumed;
 621                 if (consumed_total)
 622                         *consumed_total += consumed;
 623                 len -= consumed;
 624                 if (avail > 0) {
 625                         size_t n;
 626                         n = fwrite(zb.outbuf, avail, 1, outfile);
 627                         if (n != 1) {
 628                                 err = got_ferror(outfile, GOT_ERR_IO);
 629                                 goto done;
 630                         }
 631                         *outlen += avail;
 632                 }
 633         } while (zb.flags & GOT_INFLATE_F_HAVE_MORE);
 634
 635 done:
 636         if (err == NULL)
 637                 rewind(outfile);
 638         got_inflate_end(&zb);
 639         return err;
 640 }