lib/deflate.c

   1 /*
   2  * Copyright (c) 2019 Stefan Sperling <stsp@openbsd.org>
   3  *
   4  * Permission to use, copy, modify, and distribute this software for any
   5  * purpose with or without fee is hereby granted, provided that the above
   6  * copyright notice and this permission notice appear in all copies.
   7  *
   8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
   9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
  11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  15  */
  16
  17 #include "got_compat.h"
  18
  19 #include <sys/queue.h>
  20 #include <sys/tree.h>
  21
  22 #include <errno.h>
  23 #include <stdio.h>
  24 #include <stdlib.h>
  25 #include <string.h>
  26 #include <zlib.h>
  27 #include <time.h>
  28
  29 #include "got_error.h"
  30 #include "got_object.h"
  31 #include "got_path.h"
  32
  33 #include "got_lib_deflate.h"
  34 #include "got_lib_hash.h"
  35 #include "got_lib_poll.h"
  36
  37 #ifndef MIN
  38 #define MIN(_a,_b) ((_a) < (_b) ? (_a) : (_b))
  39 #endif
  40
  41 static const struct got_error *
  42 wrap_deflate_error(int zerr, const char *prefix)
  43 {
  44         if  (zerr == Z_ERRNO)
  45                 return got_error_from_errno(prefix);
  46         if  (zerr == Z_MEM_ERROR)
  47                 return got_error_set_errno(ENOMEM, prefix);
  48
  49         return got_error(GOT_ERR_COMPRESSION);
  50 }
  51
  52 const struct got_error *
  53 got_deflate_init(struct got_deflate_buf *zb, uint8_t *outbuf, size_t bufsize)
  54 {
  55         const struct got_error *err = NULL;
  56         int zerr;
  57
  58         memset(zb, 0, sizeof(*zb));
  59
  60         zb->z.zalloc = Z_NULL;
  61         zb->z.zfree = Z_NULL;
  62         zerr = deflateInit(&zb->z, Z_DEFAULT_COMPRESSION);
  63         if (zerr != Z_OK)
  64                 return wrap_deflate_error(zerr, "deflateInit");
  65
  66         zb->inlen = zb->outlen = bufsize;
  67
  68         zb->inbuf = calloc(1, zb->inlen);
  69         if (zb->inbuf == NULL) {
  70                 err = got_error_from_errno("calloc");
  71                 goto done;
  72         }
  73
  74         zb->flags = 0;
  75         if (outbuf == NULL) {
  76                 zb->outbuf = calloc(1, zb->outlen);
  77                 if (zb->outbuf == NULL) {
  78                         err = got_error_from_errno("calloc");
  79                         goto done;
  80                 }
  81                 zb->flags |= GOT_DEFLATE_F_OWN_OUTBUF;
  82         } else
  83                 zb->outbuf = outbuf;
  84 done:
  85         if (err)
  86                 got_deflate_end(zb);
  87         return err;
  88 }
  89
  90 static void
  91 csum_output(struct got_deflate_checksum *csum, const uint8_t *buf, size_t len)
  92 {
  93         if (csum->output_crc)
  94                 *csum->output_crc = crc32(*csum->output_crc, buf, len);
  95
  96         if (csum->output_ctx)
  97                 got_hash_update(csum->output_ctx, buf, len);
  98 }
  99
 100 const struct got_error *
 101 got_deflate_read(struct got_deflate_buf *zb, FILE *f, off_t len,
 102     size_t *outlenp, off_t *consumed)
 103 {
 104         size_t last_total_out = zb->z.total_out;
 105         z_stream *z = &zb->z;
 106         int ret = Z_ERRNO;
 107
 108         z->next_out = zb->outbuf;
 109         z->avail_out = zb->outlen;
 110
 111         *outlenp = 0;
 112         *consumed = 0;
 113         do {
 114                 size_t last_total_in = z->total_in;
 115                 if (z->avail_in == 0) {
 116                         size_t n = 0;
 117                         if (*consumed < len) {
 118                                 n = fread(zb->inbuf, 1,
 119                                     MIN(zb->inlen, len - *consumed), f);
 120                         }
 121                         if (n == 0) {
 122                                 if (ferror(f))
 123                                         return got_ferror(f, GOT_ERR_IO);
 124                                 /* EOF */
 125                                 ret = deflate(z, Z_FINISH);
 126                                 break;
 127                         }
 128                         z->next_in = zb->inbuf;
 129                         z->avail_in = n;
 130                 }
 131                 ret = deflate(z, Z_NO_FLUSH);
 132                 *consumed += z->total_in - last_total_in;
 133         } while (ret == Z_OK && z->avail_out > 0);
 134
 135         if (ret == Z_OK) {
 136                 zb->flags |= GOT_DEFLATE_F_HAVE_MORE;
 137         } else {
 138                 if (ret != Z_STREAM_END)
 139                         return wrap_deflate_error(ret, "deflate");
 140                 zb->flags &= ~GOT_DEFLATE_F_HAVE_MORE;
 141         }
 142
 143         *outlenp = z->total_out - last_total_out;
 144         return NULL;
 145 }
 146
 147 static const struct got_error *
 148 deflate_read_mmap(struct got_deflate_buf *zb, uint8_t *map, size_t offset,
 149     size_t len, size_t *outlenp, size_t *consumed, int flush_on_eof)
 150 {
 151         z_stream *z = &zb->z;
 152         size_t last_total_out = z->total_out;
 153         int ret = Z_ERRNO;
 154
 155         z->next_out = zb->outbuf;
 156         z->avail_out = zb->outlen;
 157
 158         *outlenp = 0;
 159         *consumed = 0;
 160         do {
 161                 size_t last_total_in = z->total_in;
 162                 if (z->avail_in == 0) {
 163                         z->next_in = map + offset + *consumed;
 164                         if (len - *consumed > UINT_MAX)
 165                                 z->avail_in = UINT_MAX;
 166                         else
 167                                 z->avail_in = len - *consumed;
 168                         if (z->avail_in == 0) {
 169                                 /* EOF */
 170                                 if (flush_on_eof)
 171                                         ret = deflate(z, Z_FINISH);
 172                                 break;
 173                         }
 174                 }
 175                 ret = deflate(z, Z_NO_FLUSH);
 176                 *consumed += z->total_in - last_total_in;
 177         } while (ret == Z_OK && z->avail_out > 0);
 178
 179         if (ret == Z_OK) {
 180                 zb->flags |= GOT_DEFLATE_F_HAVE_MORE;
 181         } else {
 182                 if (ret != Z_STREAM_END)
 183                         return wrap_deflate_error(ret, "deflate");
 184                 zb->flags &= ~GOT_DEFLATE_F_HAVE_MORE;
 185         }
 186
 187         *outlenp = z->total_out - last_total_out;
 188         return NULL;
 189 }
 190
 191 const struct got_error *
 192 got_deflate_read_mmap(struct got_deflate_buf *zb, uint8_t *map, size_t offset,
 193     size_t len, size_t *outlenp, size_t *consumed)
 194 {
 195         return deflate_read_mmap(zb, map, offset, len, outlenp, consumed, 1);
 196 }
 197
 198 const struct got_error *
 199 got_deflate_flush(struct got_deflate_buf *zb, FILE *outfile,
 200     struct got_deflate_checksum *csum, off_t *outlenp)
 201 {
 202         int ret;
 203         size_t n;
 204         z_stream *z = &zb->z;
 205
 206         if (z->avail_in != 0)
 207                 return got_error_msg(GOT_ERR_COMPRESSION,
 208                     "cannot flush zb with pending input data");
 209
 210         do {
 211                 size_t avail, last_total_out = zb->z.total_out;
 212
 213                 z->next_out = zb->outbuf;
 214                 z->avail_out = zb->outlen;
 215
 216                 ret = deflate(z, Z_FINISH);
 217                 if (ret != Z_STREAM_END && ret != Z_OK)
 218                         return wrap_deflate_error(ret, "deflate");
 219
 220                 avail = z->total_out - last_total_out;
 221                 if (avail > 0) {
 222                         n = fwrite(zb->outbuf, avail, 1, outfile);
 223                         if (n != 1)
 224                                 return got_ferror(outfile, GOT_ERR_IO);
 225                         if (csum)
 226                                 csum_output(csum, zb->outbuf, avail);
 227                         if (outlenp)
 228                                 *outlenp += avail;
 229                 }
 230         } while (ret != Z_STREAM_END);
 231
 232         zb->flags &= ~GOT_DEFLATE_F_HAVE_MORE;
 233         return NULL;
 234 }
 235
 236 void
 237 got_deflate_end(struct got_deflate_buf *zb)
 238 {
 239         free(zb->inbuf);
 240         if (zb->flags & GOT_DEFLATE_F_OWN_OUTBUF)
 241                 free(zb->outbuf);
 242         deflateEnd(&zb->z);
 243 }
 244
 245 const struct got_error *
 246 got_deflate_to_fd(off_t *outlen, FILE *infile, off_t len, int outfd,
 247     struct got_deflate_checksum *csum)
 248 {
 249         const struct got_error *err;
 250         size_t avail;
 251         off_t consumed;
 252         struct got_deflate_buf zb;
 253
 254         err = got_deflate_init(&zb, NULL, GOT_DEFLATE_BUFSIZE);
 255         if (err)
 256                 goto done;
 257
 258         *outlen = 0;
 259
 260         do {
 261                 err = got_deflate_read(&zb, infile, len, &avail, &consumed);
 262                 if (err)
 263                         goto done;
 264                 len -= consumed;
 265                 if (avail > 0) {
 266                         err = got_poll_write_full(outfd, zb.outbuf, avail);
 267                         if (err)
 268                                 goto done;
 269                         if (csum)
 270                                 csum_output(csum, zb.outbuf, avail);
 271                         *outlen += avail;
 272                 }
 273         } while (zb.flags & GOT_DEFLATE_F_HAVE_MORE);
 274
 275 done:
 276         got_deflate_end(&zb);
 277         return err;
 278 }
 279
 280 const struct got_error *
 281 got_deflate_to_fd_mmap(off_t *outlen, uint8_t *map, size_t offset,
 282     size_t len, int outfd, struct got_deflate_checksum *csum)
 283 {
 284         const struct got_error *err;
 285         size_t avail, consumed;
 286         struct got_deflate_buf zb;
 287
 288         err = got_deflate_init(&zb, NULL, GOT_DEFLATE_BUFSIZE);
 289         if (err)
 290                 goto done;
 291
 292         *outlen = 0;
 293         do {
 294                 err = got_deflate_read_mmap(&zb, map, offset, len, &avail,
 295                     &consumed);
 296                 if (err)
 297                         goto done;
 298                 offset += consumed;
 299                 len -= consumed;
 300                 if (avail > 0) {
 301                         err = got_poll_write_full(outfd, zb.outbuf, avail);
 302                         if (err)
 303                                 goto done;
 304                         if (csum)
 305                                 csum_output(csum, zb.outbuf, avail);
 306                         *outlen += avail;
 307                 }
 308         } while (zb.flags & GOT_DEFLATE_F_HAVE_MORE);
 309
 310 done:
 311         got_deflate_end(&zb);
 312         return err;
 313 }
 314
 315 const struct got_error *
 316 got_deflate_to_file(off_t *outlen, FILE *infile, off_t len,
 317     FILE *outfile, struct got_deflate_checksum *csum)
 318 {
 319         const struct got_error *err;
 320         size_t avail;
 321         off_t consumed;
 322         struct got_deflate_buf zb;
 323
 324         err = got_deflate_init(&zb, NULL, GOT_DEFLATE_BUFSIZE);
 325         if (err)
 326                 goto done;
 327
 328         *outlen = 0;
 329
 330         do {
 331                 err = got_deflate_read(&zb, infile, len, &avail, &consumed);
 332                 if (err)
 333                         goto done;
 334                 len -= consumed;
 335                 if (avail > 0) {
 336                         size_t n;
 337                         n = fwrite(zb.outbuf, avail, 1, outfile);
 338                         if (n != 1) {
 339                                 err = got_ferror(outfile, GOT_ERR_IO);
 340                                 goto done;
 341                         }
 342                         if (csum)
 343                                 csum_output(csum, zb.outbuf, avail);
 344                         *outlen += avail;
 345                 }
 346         } while (zb.flags & GOT_DEFLATE_F_HAVE_MORE);
 347
 348 done:
 349         got_deflate_end(&zb);
 350         return err;
 351 }
 352
 353 const struct got_error *
 354 got_deflate_to_file_mmap(off_t *outlen, uint8_t *map, size_t offset,
 355     size_t len, FILE *outfile, struct got_deflate_checksum *csum)
 356 {
 357         const struct got_error *err;
 358         size_t avail, consumed;
 359         struct got_deflate_buf zb;
 360
 361         err = got_deflate_init(&zb, NULL, GOT_DEFLATE_BUFSIZE);
 362         if (err)
 363                 goto done;
 364
 365         *outlen = 0;
 366         do {
 367                 err = got_deflate_read_mmap(&zb, map, offset, len, &avail,
 368                     &consumed);
 369                 if (err)
 370                         goto done;
 371                 offset += consumed;
 372                 len -= consumed;
 373                 if (avail > 0) {
 374                         size_t n;
 375                         n = fwrite(zb.outbuf, avail, 1, outfile);
 376                         if (n != 1) {
 377                                 err = got_ferror(outfile, GOT_ERR_IO);
 378                                 goto done;
 379                         }
 380                         if (csum)
 381                                 csum_output(csum, zb.outbuf, avail);
 382                         *outlen += avail;
 383                 }
 384         } while (zb.flags & GOT_DEFLATE_F_HAVE_MORE);
 385
 386 done:
 387         got_deflate_end(&zb);
 388         return err;
 389 }
 390
 391 const struct got_error *
 392 got_deflate_append_to_file_mmap(struct got_deflate_buf *zb, off_t *outlen,
 393     uint8_t *map, size_t offset, size_t len, FILE *outfile,
 394     struct got_deflate_checksum *csum)
 395 {
 396         const struct got_error *err;
 397         size_t avail, consumed;
 398
 399         do {
 400                 err = deflate_read_mmap(zb, map, offset, len, &avail,
 401                     &consumed, 0);
 402                 if (err)
 403                         break;
 404                 offset += consumed;
 405                 len -= consumed;
 406                 if (avail > 0) {
 407                         size_t n;
 408                         n = fwrite(zb->outbuf, avail, 1, outfile);
 409                         if (n != 1) {
 410                                 err = got_ferror(outfile, GOT_ERR_IO);
 411                                 break;
 412                         }
 413                         if (csum)
 414                                 csum_output(csum, zb->outbuf, avail);
 415                         if (outlen)
 416                                 *outlen += avail;
 417                 }
 418         } while ((zb->flags & GOT_DEFLATE_F_HAVE_MORE) && len > 0);
 419
 420         return err;
 421 }
 422
 423 const struct got_error *
 424 got_deflate_to_mem_mmap(uint8_t **outbuf, size_t *outlen,
 425     size_t *consumed_total, struct got_deflate_checksum *csum, uint8_t *map,
 426     size_t offset, size_t len)
 427 {
 428         const struct got_error *err;
 429         size_t avail, consumed;
 430         struct got_deflate_buf zb;
 431         void *newbuf;
 432         size_t nbuf = 1;
 433
 434         if (outbuf) {
 435                 *outbuf = malloc(GOT_DEFLATE_BUFSIZE);
 436                 if (*outbuf == NULL)
 437                         return got_error_from_errno("malloc");
 438                 err = got_deflate_init(&zb, *outbuf, GOT_DEFLATE_BUFSIZE);
 439                 if (err) {
 440                         free(*outbuf);
 441                         *outbuf = NULL;
 442                         return err;
 443                 }
 444         } else {
 445                 err = got_deflate_init(&zb, NULL, GOT_DEFLATE_BUFSIZE);
 446                 if (err)
 447                         return err;
 448         }
 449
 450         *outlen = 0;
 451         if (consumed_total)
 452                 *consumed_total = 0;
 453         do {
 454                 err = got_deflate_read_mmap(&zb, map, offset, len, &avail,
 455                     &consumed);
 456                 if (err)
 457                         goto done;
 458                 offset += consumed;
 459                 if (consumed_total)
 460                         *consumed_total += consumed;
 461                 len -= consumed;
 462                 if (avail > 0 && csum)
 463                         csum_output(csum, zb.outbuf, avail);
 464                 *outlen += avail;
 465                 if ((zb.flags & GOT_DEFLATE_F_HAVE_MORE) && outbuf != NULL) {
 466                         newbuf = reallocarray(*outbuf, ++nbuf,
 467                             GOT_DEFLATE_BUFSIZE);
 468                         if (newbuf == NULL) {
 469                                 err = got_error_from_errno("reallocarray");
 470                                 free(*outbuf);
 471                                 *outbuf = NULL;
 472                                 *outlen = 0;
 473                                 goto done;
 474                         }
 475                         *outbuf = newbuf;
 476                         zb.outbuf = newbuf + *outlen;
 477                         zb.outlen = (nbuf * GOT_DEFLATE_BUFSIZE) - *outlen;
 478                 }
 479         } while (zb.flags & GOT_DEFLATE_F_HAVE_MORE);
 480 done:
 481         got_deflate_end(&zb);
 482         return err;
 483 }