lib/deflate.c

   1 /*
   2  * Copyright (c) 2019 Stefan Sperling <stsp@openbsd.org>
   3  *
   4  * Permission to use, copy, modify, and distribute this software for any
   5  * purpose with or without fee is hereby granted, provided that the above
   6  * copyright notice and this permission notice appear in all copies.
   7  *
   8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
   9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
  11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  15  */
  16
  17 #include "got_compat.h"
  18
  19 #include <sys/queue.h>
  20
  21 #include <errno.h>
  22 #include <stdio.h>
  23 #include <stdlib.h>
  24 #include <string.h>
  25 #include <zlib.h>
  26 #include <time.h>
  27
  28 #include "got_error.h"
  29 #include "got_object.h"
  30 #include "got_path.h"
  31
  32 #include "got_lib_deflate.h"
  33 #include "got_lib_hash.h"
  34 #include "got_lib_poll.h"
  35
  36 #ifndef MIN
  37 #define MIN(_a,_b) ((_a) < (_b) ? (_a) : (_b))
  38 #endif
  39
  40 static const struct got_error *
  41 wrap_deflate_error(int zerr, const char *prefix)
  42 {
  43         if  (zerr == Z_ERRNO)
  44                 return got_error_from_errno(prefix);
  45         if  (zerr == Z_MEM_ERROR)
  46                 return got_error_set_errno(ENOMEM, prefix);
  47
  48         return got_error(GOT_ERR_COMPRESSION);
  49 }
  50
  51 const struct got_error *
  52 got_deflate_init(struct got_deflate_buf *zb, uint8_t *outbuf, size_t bufsize)
  53 {
  54         const struct got_error *err = NULL;
  55         int zerr;
  56
  57         memset(zb, 0, sizeof(*zb));
  58
  59         zb->z.zalloc = Z_NULL;
  60         zb->z.zfree = Z_NULL;
  61         zerr = deflateInit(&zb->z, Z_DEFAULT_COMPRESSION);
  62         if (zerr != Z_OK)
  63                 return wrap_deflate_error(zerr, "deflateInit");
  64
  65         zb->inlen = zb->outlen = bufsize;
  66
  67         zb->inbuf = calloc(1, zb->inlen);
  68         if (zb->inbuf == NULL) {
  69                 err = got_error_from_errno("calloc");
  70                 goto done;
  71         }
  72
  73         zb->flags = 0;
  74         if (outbuf == NULL) {
  75                 zb->outbuf = calloc(1, zb->outlen);
  76                 if (zb->outbuf == NULL) {
  77                         err = got_error_from_errno("calloc");
  78                         goto done;
  79                 }
  80                 zb->flags |= GOT_DEFLATE_F_OWN_OUTBUF;
  81         } else
  82                 zb->outbuf = outbuf;
  83 done:
  84         if (err)
  85                 got_deflate_end(zb);
  86         return err;
  87 }
  88
  89 static void
  90 csum_output(struct got_deflate_checksum *csum, const uint8_t *buf, size_t len)
  91 {
  92         if (csum->output_crc)
  93                 *csum->output_crc = crc32(*csum->output_crc, buf, len);
  94
  95         if (csum->output_ctx)
  96                 got_hash_update(csum->output_ctx, buf, len);
  97 }
  98
  99 const struct got_error *
 100 got_deflate_read(struct got_deflate_buf *zb, FILE *f, off_t len,
 101     size_t *outlenp, off_t *consumed)
 102 {
 103         size_t last_total_out = zb->z.total_out;
 104         z_stream *z = &zb->z;
 105         int ret = Z_ERRNO;
 106
 107         z->next_out = zb->outbuf;
 108         z->avail_out = zb->outlen;
 109
 110         *outlenp = 0;
 111         *consumed = 0;
 112         do {
 113                 size_t last_total_in = z->total_in;
 114                 if (z->avail_in == 0) {
 115                         size_t n = 0;
 116                         if (*consumed < len) {
 117                                 n = fread(zb->inbuf, 1,
 118                                     MIN(zb->inlen, len - *consumed), f);
 119                         }
 120                         if (n == 0) {
 121                                 if (ferror(f))
 122                                         return got_ferror(f, GOT_ERR_IO);
 123                                 /* EOF */
 124                                 ret = deflate(z, Z_FINISH);
 125                                 break;
 126                         }
 127                         z->next_in = zb->inbuf;
 128                         z->avail_in = n;
 129                 }
 130                 ret = deflate(z, Z_NO_FLUSH);
 131                 *consumed += z->total_in - last_total_in;
 132         } while (ret == Z_OK && z->avail_out > 0);
 133
 134         if (ret == Z_OK) {
 135                 zb->flags |= GOT_DEFLATE_F_HAVE_MORE;
 136         } else {
 137                 if (ret != Z_STREAM_END)
 138                         return wrap_deflate_error(ret, "deflate");
 139                 zb->flags &= ~GOT_DEFLATE_F_HAVE_MORE;
 140         }
 141
 142         *outlenp = z->total_out - last_total_out;
 143         return NULL;
 144 }
 145
 146 static const struct got_error *
 147 deflate_read_mmap(struct got_deflate_buf *zb, uint8_t *map, size_t offset,
 148     size_t len, size_t *outlenp, size_t *consumed, int flush_on_eof)
 149 {
 150         z_stream *z = &zb->z;
 151         size_t last_total_out = z->total_out;
 152         int ret = Z_ERRNO;
 153
 154         z->next_out = zb->outbuf;
 155         z->avail_out = zb->outlen;
 156
 157         *outlenp = 0;
 158         *consumed = 0;
 159         do {
 160                 size_t last_total_in = z->total_in;
 161                 if (z->avail_in == 0) {
 162                         z->next_in = map + offset + *consumed;
 163                         if (len - *consumed > UINT_MAX)
 164                                 z->avail_in = UINT_MAX;
 165                         else
 166                                 z->avail_in = len - *consumed;
 167                         if (z->avail_in == 0) {
 168                                 /* EOF */
 169                                 if (flush_on_eof)
 170                                         ret = deflate(z, Z_FINISH);
 171                                 break;
 172                         }
 173                 }
 174                 ret = deflate(z, Z_NO_FLUSH);
 175                 *consumed += z->total_in - last_total_in;
 176         } while (ret == Z_OK && z->avail_out > 0);
 177
 178         if (ret == Z_OK) {
 179                 zb->flags |= GOT_DEFLATE_F_HAVE_MORE;
 180         } else {
 181                 if (ret != Z_STREAM_END)
 182                         return wrap_deflate_error(ret, "deflate");
 183                 zb->flags &= ~GOT_DEFLATE_F_HAVE_MORE;
 184         }
 185
 186         *outlenp = z->total_out - last_total_out;
 187         return NULL;
 188 }
 189
 190 const struct got_error *
 191 got_deflate_read_mmap(struct got_deflate_buf *zb, uint8_t *map, size_t offset,
 192     size_t len, size_t *outlenp, size_t *consumed)
 193 {
 194         return deflate_read_mmap(zb, map, offset, len, outlenp, consumed, 1);
 195 }
 196
 197 const struct got_error *
 198 got_deflate_flush(struct got_deflate_buf *zb, FILE *outfile,
 199     struct got_deflate_checksum *csum, off_t *outlenp)
 200 {
 201         int ret;
 202         size_t n;
 203         z_stream *z = &zb->z;
 204
 205         if (z->avail_in != 0)
 206                 return got_error_msg(GOT_ERR_COMPRESSION,
 207                     "cannot flush zb with pending input data");
 208
 209         do {
 210                 size_t avail, last_total_out = zb->z.total_out;
 211
 212                 z->next_out = zb->outbuf;
 213                 z->avail_out = zb->outlen;
 214
 215                 ret = deflate(z, Z_FINISH);
 216                 if (ret != Z_STREAM_END && ret != Z_OK)
 217                         return wrap_deflate_error(ret, "deflate");
 218
 219                 avail = z->total_out - last_total_out;
 220                 if (avail > 0) {
 221                         n = fwrite(zb->outbuf, avail, 1, outfile);
 222                         if (n != 1)
 223                                 return got_ferror(outfile, GOT_ERR_IO);
 224                         if (csum)
 225                                 csum_output(csum, zb->outbuf, avail);
 226                         if (outlenp)
 227                                 *outlenp += avail;
 228                 }
 229         } while (ret != Z_STREAM_END);
 230
 231         zb->flags &= ~GOT_DEFLATE_F_HAVE_MORE;
 232         return NULL;
 233 }
 234
 235 void
 236 got_deflate_end(struct got_deflate_buf *zb)
 237 {
 238         free(zb->inbuf);
 239         if (zb->flags & GOT_DEFLATE_F_OWN_OUTBUF)
 240                 free(zb->outbuf);
 241         deflateEnd(&zb->z);
 242 }
 243
 244 const struct got_error *
 245 got_deflate_to_fd(off_t *outlen, FILE *infile, off_t len, int outfd,
 246     struct got_deflate_checksum *csum)
 247 {
 248         const struct got_error *err;
 249         size_t avail;
 250         off_t consumed;
 251         struct got_deflate_buf zb;
 252
 253         err = got_deflate_init(&zb, NULL, GOT_DEFLATE_BUFSIZE);
 254         if (err)
 255                 goto done;
 256
 257         *outlen = 0;
 258
 259         do {
 260                 err = got_deflate_read(&zb, infile, len, &avail, &consumed);
 261                 if (err)
 262                         goto done;
 263                 len -= consumed;
 264                 if (avail > 0) {
 265                         err = got_poll_write_full(outfd, zb.outbuf, avail);
 266                         if (err)
 267                                 goto done;
 268                         if (csum)
 269                                 csum_output(csum, zb.outbuf, avail);
 270                         *outlen += avail;
 271                 }
 272         } while (zb.flags & GOT_DEFLATE_F_HAVE_MORE);
 273
 274 done:
 275         got_deflate_end(&zb);
 276         return err;
 277 }
 278
 279 const struct got_error *
 280 got_deflate_to_fd_mmap(off_t *outlen, uint8_t *map, size_t offset,
 281     size_t len, int outfd, struct got_deflate_checksum *csum)
 282 {
 283         const struct got_error *err;
 284         size_t avail, consumed;
 285         struct got_deflate_buf zb;
 286
 287         err = got_deflate_init(&zb, NULL, GOT_DEFLATE_BUFSIZE);
 288         if (err)
 289                 goto done;
 290
 291         *outlen = 0;
 292         do {
 293                 err = got_deflate_read_mmap(&zb, map, offset, len, &avail,
 294                     &consumed);
 295                 if (err)
 296                         goto done;
 297                 offset += consumed;
 298                 len -= consumed;
 299                 if (avail > 0) {
 300                         err = got_poll_write_full(outfd, zb.outbuf, avail);
 301                         if (err)
 302                                 goto done;
 303                         if (csum)
 304                                 csum_output(csum, zb.outbuf, avail);
 305                         *outlen += avail;
 306                 }
 307         } while (zb.flags & GOT_DEFLATE_F_HAVE_MORE);
 308
 309 done:
 310         got_deflate_end(&zb);
 311         return err;
 312 }
 313
 314 const struct got_error *
 315 got_deflate_to_file(off_t *outlen, FILE *infile, off_t len,
 316     FILE *outfile, struct got_deflate_checksum *csum)
 317 {
 318         const struct got_error *err;
 319         size_t avail;
 320         off_t consumed;
 321         struct got_deflate_buf zb;
 322
 323         err = got_deflate_init(&zb, NULL, GOT_DEFLATE_BUFSIZE);
 324         if (err)
 325                 goto done;
 326
 327         *outlen = 0;
 328
 329         do {
 330                 err = got_deflate_read(&zb, infile, len, &avail, &consumed);
 331                 if (err)
 332                         goto done;
 333                 len -= consumed;
 334                 if (avail > 0) {
 335                         size_t n;
 336                         n = fwrite(zb.outbuf, avail, 1, outfile);
 337                         if (n != 1) {
 338                                 err = got_ferror(outfile, GOT_ERR_IO);
 339                                 goto done;
 340                         }
 341                         if (csum)
 342                                 csum_output(csum, zb.outbuf, avail);
 343                         *outlen += avail;
 344                 }
 345         } while (zb.flags & GOT_DEFLATE_F_HAVE_MORE);
 346
 347 done:
 348         got_deflate_end(&zb);
 349         return err;
 350 }
 351
 352 const struct got_error *
 353 got_deflate_to_file_mmap(off_t *outlen, uint8_t *map, size_t offset,
 354     size_t len, FILE *outfile, struct got_deflate_checksum *csum)
 355 {
 356         const struct got_error *err;
 357         size_t avail, consumed;
 358         struct got_deflate_buf zb;
 359
 360         err = got_deflate_init(&zb, NULL, GOT_DEFLATE_BUFSIZE);
 361         if (err)
 362                 goto done;
 363
 364         *outlen = 0;
 365         do {
 366                 err = got_deflate_read_mmap(&zb, map, offset, len, &avail,
 367                     &consumed);
 368                 if (err)
 369                         goto done;
 370                 offset += consumed;
 371                 len -= consumed;
 372                 if (avail > 0) {
 373                         size_t n;
 374                         n = fwrite(zb.outbuf, avail, 1, outfile);
 375                         if (n != 1) {
 376                                 err = got_ferror(outfile, GOT_ERR_IO);
 377                                 goto done;
 378                         }
 379                         if (csum)
 380                                 csum_output(csum, zb.outbuf, avail);
 381                         *outlen += avail;
 382                 }
 383         } while (zb.flags & GOT_DEFLATE_F_HAVE_MORE);
 384
 385 done:
 386         got_deflate_end(&zb);
 387         return err;
 388 }
 389
 390 const struct got_error *
 391 got_deflate_append_to_file_mmap(struct got_deflate_buf *zb, off_t *outlen,
 392     uint8_t *map, size_t offset, size_t len, FILE *outfile,
 393     struct got_deflate_checksum *csum)
 394 {
 395         const struct got_error *err;
 396         size_t avail, consumed;
 397
 398         do {
 399                 err = deflate_read_mmap(zb, map, offset, len, &avail,
 400                     &consumed, 0);
 401                 if (err)
 402                         break;
 403                 offset += consumed;
 404                 len -= consumed;
 405                 if (avail > 0) {
 406                         size_t n;
 407                         n = fwrite(zb->outbuf, avail, 1, outfile);
 408                         if (n != 1) {
 409                                 err = got_ferror(outfile, GOT_ERR_IO);
 410                                 break;
 411                         }
 412                         if (csum)
 413                                 csum_output(csum, zb->outbuf, avail);
 414                         if (outlen)
 415                                 *outlen += avail;
 416                 }
 417         } while ((zb->flags & GOT_DEFLATE_F_HAVE_MORE) && len > 0);
 418
 419         return err;
 420 }
 421
 422 const struct got_error *
 423 got_deflate_to_mem_mmap(uint8_t **outbuf, size_t *outlen,
 424     size_t *consumed_total, struct got_deflate_checksum *csum, uint8_t *map,
 425     size_t offset, size_t len)
 426 {
 427         const struct got_error *err;
 428         size_t avail, consumed;
 429         struct got_deflate_buf zb;
 430         void *newbuf;
 431         size_t nbuf = 1;
 432
 433         if (outbuf) {
 434                 *outbuf = malloc(GOT_DEFLATE_BUFSIZE);
 435                 if (*outbuf == NULL)
 436                         return got_error_from_errno("malloc");
 437                 err = got_deflate_init(&zb, *outbuf, GOT_DEFLATE_BUFSIZE);
 438                 if (err) {
 439                         free(*outbuf);
 440                         *outbuf = NULL;
 441                         return err;
 442                 }
 443         } else {
 444                 err = got_deflate_init(&zb, NULL, GOT_DEFLATE_BUFSIZE);
 445                 if (err)
 446                         return err;
 447         }
 448
 449         *outlen = 0;
 450         if (consumed_total)
 451                 *consumed_total = 0;
 452         do {
 453                 err = got_deflate_read_mmap(&zb, map, offset, len, &avail,
 454                     &consumed);
 455                 if (err)
 456                         goto done;
 457                 offset += consumed;
 458                 if (consumed_total)
 459                         *consumed_total += consumed;
 460                 len -= consumed;
 461                 if (avail > 0 && csum)
 462                         csum_output(csum, zb.outbuf, avail);
 463                 *outlen += avail;
 464                 if ((zb.flags & GOT_DEFLATE_F_HAVE_MORE) && outbuf != NULL) {
 465                         newbuf = reallocarray(*outbuf, ++nbuf,
 466                             GOT_DEFLATE_BUFSIZE);
 467                         if (newbuf == NULL) {
 468                                 err = got_error_from_errno("reallocarray");
 469                                 free(*outbuf);
 470                                 *outbuf = NULL;
 471                                 *outlen = 0;
 472                                 goto done;
 473                         }
 474                         *outbuf = newbuf;
 475                         zb.outbuf = newbuf + *outlen;
 476                         zb.outlen = (nbuf * GOT_DEFLATE_BUFSIZE) - *outlen;
 477                 }
 478         } while (zb.flags & GOT_DEFLATE_F_HAVE_MORE);
 479 done:
 480         got_deflate_end(&zb);
 481         return err;
 482 }