src/fmtutil-cmpr.c

   1 // This file is part of Deark.
   2 // Copyright (C) 2018 Jason Summers
   3 // See the file COPYING for terms of use.
   4
   5 // Decompression, etc.
   6
   7 #define DE_NOT_IN_MODULE
   8 #include "deark-config.h"
   9 #include "deark-private.h"
  10 #include "deark-fmtutil.h"
  11
  12 // Returns a message that is valid until the next operation on dres.
  13 const char *de_dfilter_get_errmsg(deark *c, struct de_dfilter_results *dres)
  14 {
  15         if(dres->errcode==0) {
  16                 return "No error";
  17         }
  18         if(dres->errmsg[0]) {
  19                 return dres->errmsg;
  20         }
  21         return "Unspecified error";
  22 }
  23
  24 // Initialize or reset a dfilter results struct
  25 void de_dfilter_results_clear(deark *c, struct de_dfilter_results *dres)
  26 {
  27         de_zeromem(dres, sizeof(struct de_dfilter_results));
  28 }
  29
  30 // Note: It is also okay to init these objects by zeroing out their bytes.
  31 void de_dfilter_init_objects(deark *c, struct de_dfilter_in_params *dcmpri,
  32         struct de_dfilter_out_params *dcmpro, struct de_dfilter_results *dres)
  33 {
  34         if(dcmpri)
  35                 de_zeromem(dcmpri, sizeof(struct de_dfilter_in_params));
  36         if(dcmpro)
  37                 de_zeromem(dcmpro, sizeof(struct de_dfilter_out_params));
  38         if(dres)
  39                 de_dfilter_results_clear(c, dres);
  40 }
  41
  42 void de_dfilter_set_errorf(deark *c, struct de_dfilter_results *dres, const char *modname,
  43         const char *fmt, ...)
  44 {
  45         va_list ap;
  46
  47         if(dres->errcode != 0) return; // Only record the first error
  48         dres->errcode = 1;
  49
  50         va_start(ap, fmt);
  51         if(modname) {
  52                 char tmpbuf[80];
  53
  54                 de_vsnprintf(tmpbuf, sizeof(tmpbuf), fmt, ap);
  55                 de_snprintf(dres->errmsg, sizeof(dres->errmsg), "[%s] %s", modname, tmpbuf);
  56         }
  57         else {
  58                 de_vsnprintf(dres->errmsg, sizeof(dres->errmsg), fmt, ap);
  59         }
  60         va_end(ap);
  61 }
  62
  63 void de_dfilter_set_generic_error(deark *c, struct de_dfilter_results *dres, const char *modname)
  64 {
  65         if(dres->errcode != 0) return;
  66         de_dfilter_set_errorf(c, dres, modname, "Unspecified error");
  67 }
  68
  69 // This is a decompression API that uses a "push" input model. The client
  70 // sends data to the codec as the data becomes available.
  71 // (The client must still be able to consume any amount of output data
  72 // immediately.)
  73 // This model makes it easier to chain multiple codecs together, and to handle
  74 // input data that is not contiguous.
  75
  76 struct de_dfilter_ctx *de_dfilter_create(deark *c,
  77         dfilter_codec_type codec_init_fn, void *codec_private_params,
  78         struct de_dfilter_out_params *dcmpro, struct de_dfilter_results *dres)
  79 {
  80         struct de_dfilter_ctx *dfctx = NULL;
  81
  82         dfctx = de_malloc(c, sizeof(struct de_dfilter_ctx));
  83         dfctx->c = c;
  84         dfctx->dres = dres;
  85         dfctx->dcmpro = dcmpro;
  86
  87         if(codec_init_fn) {
  88                 codec_init_fn(dfctx, codec_private_params);
  89         }
  90         // TODO: How should we handle failure to initialize a codec?
  91
  92         return dfctx;
  93 }
  94
  95 void de_dfilter_addbuf(struct de_dfilter_ctx *dfctx,
  96         const u8 *buf, i64 buf_len)
  97 {
  98         if(dfctx->codec_addbuf_fn && (buf_len>0)) {
  99                 dfctx->codec_addbuf_fn(dfctx, buf, buf_len);
 100         }
 101 }
 102
 103 void de_dfilter_finish(struct de_dfilter_ctx *dfctx)
 104 {
 105         if(dfctx->codec_finish_fn) {
 106                 dfctx->codec_finish_fn(dfctx);
 107         }
 108 }
 109
 110 void de_dfilter_destroy(struct de_dfilter_ctx *dfctx)
 111 {
 112         deark *c;
 113
 114         if(!dfctx) return;
 115         c = dfctx->c;
 116         if(dfctx->codec_destroy_fn) {
 117                 dfctx->codec_destroy_fn(dfctx);
 118         }
 119
 120         de_free(c, dfctx);
 121 }
 122
 123 static int my_dfilter_oneshot_buffered_read_cbfn(struct de_bufferedreadctx *brctx, const u8 *buf,
 124         i64 buf_len)
 125 {
 126         struct de_dfilter_ctx *dfctx = (struct de_dfilter_ctx *)brctx->userdata;
 127
 128         de_dfilter_addbuf(dfctx, buf, buf_len);
 129         if(dfctx->finished_flag) return 0;
 130         return 1;
 131 }
 132
 133 // Use a "pushable" codec in a non-pushable way.
 134 void de_dfilter_decompress_oneshot(deark *c,
 135         dfilter_codec_type codec_init_fn, void *codec_private_params,
 136         struct de_dfilter_in_params *dcmpri, struct de_dfilter_out_params *dcmpro,
 137         struct de_dfilter_results *dres)
 138 {
 139         struct de_dfilter_ctx *dfctx = NULL;
 140
 141         dfctx = de_dfilter_create(c, codec_init_fn, codec_private_params,
 142                 dcmpro, dres);
 143         dbuf_buffered_read(dcmpri->f, dcmpri->pos, dcmpri->len,
 144                 my_dfilter_oneshot_buffered_read_cbfn, (void*)dfctx);
 145         de_dfilter_finish(dfctx);
 146         de_dfilter_destroy(dfctx);
 147 }
 148
 149 // Trivial "decompression" of uncompressed data.
 150 void fmtutil_decompress_uncompressed(deark *c, struct de_dfilter_in_params *dcmpri,
 151         struct de_dfilter_out_params *dcmpro, struct de_dfilter_results *dres, UI flags)
 152 {
 153         i64 len;
 154         i64 nbytes_avail;
 155
 156         nbytes_avail = de_min_int(dcmpri->len, dcmpri->f->len - dcmpri->pos);
 157
 158         if(dcmpro->len_known) {
 159                 len = dcmpro->expected_len;
 160         }
 161         else {
 162                 len = dcmpri->len;
 163         }
 164
 165         if(len>nbytes_avail) len = nbytes_avail;
 166         if(len<0) len = 0;
 167
 168         dbuf_copy(dcmpri->f, dcmpri->pos, len, dcmpro->f);
 169         dres->bytes_consumed = len;
 170         dres->bytes_consumed_valid = 1;
 171 }
 172
 173 void fmtutil_decompress_packbits_ex(deark *c, struct de_dfilter_in_params *dcmpri,
 174         struct de_dfilter_out_params *dcmpro, struct de_dfilter_results *dres)
 175 {
 176         i64 pos;
 177         u8 b, b2;
 178         i64 count;
 179         i64 endpos;
 180         i64 outf_len_limit = 0;
 181         dbuf *f = dcmpri->f;
 182         dbuf *unc_pixels = dcmpro->f;
 183
 184         pos = dcmpri->pos;
 185         endpos = dcmpri->pos + dcmpri->len;
 186
 187         if(dcmpro->len_known) {
 188                 outf_len_limit = unc_pixels->len + dcmpro->expected_len;
 189         }
 190
 191         while(1) {
 192                 if(dcmpro->len_known && unc_pixels->len >= outf_len_limit) {
 193                         break; // Decompressed the requested amount of dst data.
 194                 }
 195
 196                 if(pos>=endpos) {
 197                         break; // Reached the end of source data
 198                 }
 199                 b = dbuf_getbyte(f, pos++);
 200
 201                 if(b>128) { // A compressed run
 202                         count = 257 - (i64)b;
 203                         b2 = dbuf_getbyte(f, pos++);
 204                         dbuf_write_run(unc_pixels, b2, count);
 205                 }
 206                 else if(b<128) { // An uncompressed run
 207                         count = 1 + (i64)b;
 208                         dbuf_copy(f, pos, count, unc_pixels);
 209                         pos += count;
 210                 }
 211                 // Else b==128. No-op.
 212                 // TODO: Some (but not most) ILBM specs say that code 128 is used to
 213                 // mark the end of compressed data, so maybe there should be options to
 214                 // tell us what to do when code 128 is encountered.
 215         }
 216
 217         dres->bytes_consumed = pos - dcmpri->pos;
 218         dres->bytes_consumed_valid = 1;
 219 }
 220
 221 // Returns 0 on failure (currently impossible).
 222 int fmtutil_decompress_packbits(dbuf *f, i64 pos1, i64 len,
 223         dbuf *unc_pixels, i64 *cmpr_bytes_consumed)
 224 {
 225         struct de_dfilter_results dres;
 226         struct de_dfilter_in_params dcmpri;
 227         struct de_dfilter_out_params dcmpro;
 228
 229         if(cmpr_bytes_consumed) *cmpr_bytes_consumed = 0;
 230         de_dfilter_init_objects(f->c, &dcmpri, &dcmpro, &dres);
 231
 232         dcmpri.f = f;
 233         dcmpri.pos = pos1;
 234         dcmpri.len = len;
 235         dcmpro.f = unc_pixels;
 236         if(unc_pixels->has_len_limit) {
 237                 dcmpro.len_known = 1;
 238                 dcmpro.expected_len = unc_pixels->len_limit - unc_pixels->len;
 239         }
 240
 241         fmtutil_decompress_packbits_ex(f->c, &dcmpri, &dcmpro, &dres);
 242
 243         if(cmpr_bytes_consumed && dres.bytes_consumed_valid) {
 244                 *cmpr_bytes_consumed = dres.bytes_consumed;
 245         }
 246         if(dres.errcode != 0) return 0;
 247         return 1;
 248 }
 249
 250 // A 16-bit variant of de_fmtutil_uncompress_packbits().
 251 void fmtutil_decompress_packbits16_ex(deark *c, struct de_dfilter_in_params *dcmpri,
 252         struct de_dfilter_out_params *dcmpro, struct de_dfilter_results *dres)
 253 {
 254         i64 pos;
 255         u8 b, b1, b2;
 256         i64 k;
 257         i64 count;
 258         i64 endpos;
 259         i64 outf_len_limit = 0;
 260         dbuf *f = dcmpri->f;
 261         dbuf *unc_pixels = dcmpro->f;
 262
 263         pos = dcmpri->pos;
 264         endpos = dcmpri->pos + dcmpri->len;
 265
 266         if(dcmpro->len_known) {
 267                 outf_len_limit = unc_pixels->len + dcmpro->expected_len;
 268         }
 269
 270         while(1) {
 271                 if(dcmpro->len_known && unc_pixels->len >= outf_len_limit) {
 272                         break; // Decompressed the requested amount of dst data.
 273                 }
 274
 275                 if(pos>=endpos) {
 276                         break; // Reached the end of source data
 277                 }
 278                 b = dbuf_getbyte(f, pos++);
 279
 280                 if(b>128) { // A compressed run
 281                         count = 257 - (i64)b;
 282                         b1 = dbuf_getbyte(f, pos++);
 283                         b2 = dbuf_getbyte(f, pos++);
 284                         for(k=0; k<count; k++) {
 285                                 dbuf_writebyte(unc_pixels, b1);
 286                                 dbuf_writebyte(unc_pixels, b2);
 287                         }
 288                 }
 289                 else if(b<128) { // An uncompressed run
 290                         count = 1 + (i64)b;
 291                         dbuf_copy(f, pos, count*2, unc_pixels);
 292                         pos += count*2;
 293                 }
 294                 // Else b==128. No-op.
 295         }
 296
 297         dres->bytes_consumed = pos - dcmpri->pos;
 298         dres->bytes_consumed_valid = 1;
 299 }
 300
 301 int fmtutil_decompress_packbits16(dbuf *f, i64 pos1, i64 len,
 302         dbuf *unc_pixels, i64 *cmpr_bytes_consumed)
 303 {
 304         struct de_dfilter_results dres;
 305         struct de_dfilter_in_params dcmpri;
 306         struct de_dfilter_out_params dcmpro;
 307
 308         if(cmpr_bytes_consumed) *cmpr_bytes_consumed = 0;
 309         de_dfilter_init_objects(f->c, &dcmpri, &dcmpro, &dres);
 310
 311         dcmpri.f = f;
 312         dcmpri.pos = pos1;
 313         dcmpri.len = len;
 314         dcmpro.f = unc_pixels;
 315         if(unc_pixels->has_len_limit) {
 316                 dcmpro.len_known = 1;
 317                 dcmpro.expected_len = unc_pixels->len_limit - unc_pixels->len;
 318         }
 319
 320         fmtutil_decompress_packbits16_ex(f->c, &dcmpri, &dcmpro, &dres);
 321
 322         if(cmpr_bytes_consumed && dres.bytes_consumed_valid) {
 323                 *cmpr_bytes_consumed = dres.bytes_consumed;
 324         }
 325         if(dres.errcode != 0) return 0;
 326         return 1;
 327 }
 328
 329 void fmtutil_decompress_rle90_ex(deark *c, struct de_dfilter_in_params *dcmpri,
 330         struct de_dfilter_out_params *dcmpro, struct de_dfilter_results *dres,
 331         unsigned int flags)
 332 {
 333         de_dfilter_decompress_oneshot(c, dfilter_rle90_codec, NULL,
 334                 dcmpri, dcmpro, dres);
 335 }
 336
 337 struct rle90ctx {
 338         i64 total_nbytes_processed;
 339         i64 nbytes_written;
 340         u8 last_output_byte;
 341         int countcode_pending;
 342 };
 343
 344 static void my_rle90_codec_addbuf(struct de_dfilter_ctx *dfctx,
 345         const u8 *buf, i64 buf_len)
 346 {
 347         int i;
 348         u8 b;
 349         struct rle90ctx *rctx = (struct rle90ctx*)dfctx->codec_private;
 350
 351         if(!rctx) return;
 352
 353         for(i=0; i<buf_len; i++) {
 354                 if(dfctx->dcmpro->len_known &&
 355                         (rctx->nbytes_written >= dfctx->dcmpro->expected_len))
 356                 {
 357                         dfctx->finished_flag = 1;
 358                         break;
 359                 }
 360
 361                 b = buf[i];
 362                 rctx->total_nbytes_processed++;
 363
 364                 if(rctx->countcode_pending && b==0) {
 365                         // Not RLE, just an escaped 0x90 byte.
 366                         dbuf_writebyte(dfctx->dcmpro->f, 0x90);
 367                         rctx->nbytes_written++;
 368                         rctx->last_output_byte = 0x90;
 369                         rctx->countcode_pending = 0;
 370                 }
 371                 else if(rctx->countcode_pending) {
 372                         i64 count;
 373
 374                         // RLE. We already emitted one byte (because the byte to repeat
 375                         // comes before the repeat count), so write countcode-1 bytes.
 376                         count = (i64)(b-1);
 377                         if(dfctx->dcmpro->len_known &&
 378                                 (rctx->nbytes_written+count > dfctx->dcmpro->expected_len))
 379                         {
 380                                 count = dfctx->dcmpro->expected_len - rctx->nbytes_written;
 381                         }
 382                         dbuf_write_run(dfctx->dcmpro->f, rctx->last_output_byte, count);
 383                         rctx->nbytes_written += count;
 384
 385                         rctx->countcode_pending = 0;
 386                 }
 387                 else if(b==0x90) {
 388                         rctx->countcode_pending = 1;
 389                 }
 390                 else {
 391                         dbuf_writebyte(dfctx->dcmpro->f, b);
 392                         rctx->nbytes_written++;
 393                         rctx->last_output_byte = b;
 394                 }
 395         }
 396 }
 397
 398 static void my_rle90_codec_finish(struct de_dfilter_ctx *dfctx)
 399 {
 400         struct rle90ctx *rctx = (struct rle90ctx*)dfctx->codec_private;
 401
 402         if(!rctx) return;
 403         dfctx->dres->bytes_consumed = rctx->total_nbytes_processed;
 404         dfctx->dres->bytes_consumed_valid = 1;
 405 }
 406
 407 static void my_rle90_codec_destroy(struct de_dfilter_ctx *dfctx)
 408 {
 409         struct rle90ctx *rctx = (struct rle90ctx*)dfctx->codec_private;
 410
 411         if(rctx) {
 412                 de_free(dfctx->c, rctx);
 413         }
 414         dfctx->codec_private = NULL;
 415 }
 416
 417 // RLE algorithm occasionally called "RLE90". Variants of this are used by
 418 // BinHex, ARC, StuffIt, and others.
 419 // codec_private_params: Unused, must be NULL.
 420 void dfilter_rle90_codec(struct de_dfilter_ctx *dfctx, void *codec_private_params)
 421 {
 422         struct rle90ctx *rctx = NULL;
 423
 424         rctx = de_malloc(dfctx->c, sizeof(struct rle90ctx));
 425         dfctx->codec_private = (void*)rctx;
 426         dfctx->codec_addbuf_fn = my_rle90_codec_addbuf;
 427         dfctx->codec_finish_fn = my_rle90_codec_finish;
 428         dfctx->codec_destroy_fn = my_rle90_codec_destroy;
 429 }
 430
 431 struct szdd_ctx {
 432         i64 nbytes_written;
 433         int stop_flag;
 434         struct de_dfilter_out_params *dcmpro;
 435         struct de_lz77buffer *ringbuf;
 436 };
 437
 438 static void szdd_lz77buf_writebytecb(struct de_lz77buffer *rb, const u8 n)
 439 {
 440         struct szdd_ctx *sctx = (struct szdd_ctx*)rb->userdata;
 441
 442         if(sctx->stop_flag) return;
 443         if(sctx->dcmpro->len_known) {
 444                 if(sctx->nbytes_written >= sctx->dcmpro->expected_len) {
 445                         sctx->stop_flag = 1;
 446                         return;
 447                 }
 448         }
 449
 450         dbuf_writebyte(sctx->dcmpro->f, n);
 451         sctx->nbytes_written++;
 452 }
 453
 454 static void szdd_init_window_default(struct de_lz77buffer *ringbuf)
 455 {
 456         de_lz77buffer_clear(ringbuf, 0x20);
 457         ringbuf->curpos = 4096 - 16;
 458 }
 459
 460 static void szdd_init_window_lz5(struct de_lz77buffer *ringbuf)
 461 {
 462         size_t wpos;
 463         int i;
 464
 465         de_zeromem(ringbuf->buf, 4096);
 466         wpos = 13;
 467         for(i=1; i<256; i++) {
 468                 de_memset(&ringbuf->buf[wpos], i, 13);
 469                 wpos += 13;
 470         }
 471         for(i=0; i<256; i++) {
 472                 ringbuf->buf[wpos++] = i;
 473         }
 474         for(i=255; i>=0; i--) {
 475                 ringbuf->buf[wpos++] = i;
 476         }
 477         wpos += 128;
 478         de_memset(&ringbuf->buf[wpos], 0x20, 110);
 479         wpos += 110;
 480         ringbuf->curpos = (UI)wpos;
 481 }
 482
 483 // Partially based on the libmspack's format documentation at
 484 // <https://www.cabextract.org.uk/libmspack/doc/szdd_kwaj_format.html>
 485 // flags:
 486 //   0x1: LArc lz5 mode
 487 void fmtutil_decompress_szdd(deark *c, struct de_dfilter_in_params *dcmpri,
 488         struct de_dfilter_out_params *dcmpro, struct de_dfilter_results *dres, unsigned int flags)
 489 {
 490         i64 pos = dcmpri->pos;
 491         i64 endpos = dcmpri->pos + dcmpri->len;
 492         struct szdd_ctx *sctx = NULL;
 493
 494         sctx = de_malloc(c, sizeof(struct szdd_ctx));
 495         sctx->dcmpro = dcmpro;
 496         sctx->ringbuf = de_lz77buffer_create(c, 4096);
 497         sctx->ringbuf->writebyte_cb = szdd_lz77buf_writebytecb;
 498         sctx->ringbuf->userdata = (void*)sctx;
 499
 500         if(flags & 0x1) {
 501                 szdd_init_window_lz5(sctx->ringbuf);
 502         }
 503         else {
 504                 szdd_init_window_default(sctx->ringbuf);
 505         }
 506
 507         while(1) {
 508                 UI control;
 509                 UI cbit;
 510
 511                 if(pos+1 > endpos) goto unc_done; // Out of input data
 512                 control = (UI)dbuf_getbyte(dcmpri->f, pos++);
 513
 514                 for(cbit=0x01; cbit<=0x80; cbit<<=1) {
 515                         if(control & cbit) { // literal
 516                                 u8 b;
 517
 518                                 if(pos+1 > endpos) goto unc_done;
 519                                 b = dbuf_getbyte(dcmpri->f, pos++);
 520                                 de_lz77buffer_add_literal_byte(sctx->ringbuf, b);
 521                                 if(sctx->stop_flag) goto unc_done;
 522                         }
 523                         else { // match
 524                                 UI x0, x1;
 525                                 UI matchpos;
 526                                 UI matchlen;
 527
 528                                 if(pos+2 > endpos) goto unc_done;
 529                                 x0 = (UI)dbuf_getbyte_p(dcmpri->f, &pos);
 530                                 x1 = (UI)dbuf_getbyte_p(dcmpri->f, &pos);
 531                                 matchpos = ((x1 & 0xf0) << 4) | x0;
 532                                 matchlen = (x1 & 0x0f) + 3;
 533                                 de_lz77buffer_copy_from_hist(sctx->ringbuf, matchpos, matchlen);
 534                                 if(sctx->stop_flag) goto unc_done;
 535                         }
 536                 }
 537         }
 538
 539 unc_done:
 540         dres->bytes_consumed_valid = 1;
 541         dres->bytes_consumed = pos - dcmpri->pos;
 542         if(sctx) {
 543                 de_lz77buffer_destroy(c, sctx->ringbuf);
 544                 de_free(c, sctx);
 545         }
 546 }
 547
 548 //======================= hlp_lz77 =======================
 549
 550 struct hlplz77ctx {
 551         i64 nbytes_written;
 552         int stop_flag;
 553         struct de_dfilter_out_params *dcmpro;
 554         struct de_lz77buffer *ringbuf;
 555 };
 556
 557 static void hlplz77_lz77buf_writebytecb(struct de_lz77buffer *rb, const u8 n)
 558 {
 559         struct hlplz77ctx *sctx = (struct hlplz77ctx*)rb->userdata;
 560
 561         if(sctx->stop_flag) return;
 562         if(sctx->dcmpro->len_known) {
 563                 if(sctx->nbytes_written >= sctx->dcmpro->expected_len) {
 564                         sctx->stop_flag = 1;
 565                         return;
 566                 }
 567         }
 568
 569         dbuf_writebyte(sctx->dcmpro->f, n);
 570         sctx->nbytes_written++;
 571 }
 572
 573 // This is very similar to the mscompress SZDD algorithm, but
 574 // gratuitously different.
 575 void fmtutil_hlp_lz77_codectype1(deark *c, struct de_dfilter_in_params *dcmpri,
 576         struct de_dfilter_out_params *dcmpro, struct de_dfilter_results *dres,
 577         void *codec_private_params)
 578 {
 579         i64 pos = dcmpri->pos;
 580         i64 endpos = dcmpri->pos + dcmpri->len;
 581         struct hlplz77ctx *sctx = NULL;
 582
 583         sctx = de_malloc(c, sizeof(struct hlplz77ctx));
 584         sctx->dcmpro = dcmpro;
 585         sctx->ringbuf = de_lz77buffer_create(c, 4096);
 586         sctx->ringbuf->writebyte_cb = hlplz77_lz77buf_writebytecb;
 587         sctx->ringbuf->userdata = (void*)sctx;
 588         de_lz77buffer_clear(sctx->ringbuf, 0x20);
 589
 590         while(1) {
 591                 UI control;
 592                 UI cbit;
 593
 594                 if(pos+1 > endpos) goto unc_done; // Out of input data
 595                 control = (UI)dbuf_getbyte(dcmpri->f, pos++);
 596
 597                 for(cbit=0x01; cbit<=0x80; cbit<<=1) {
 598                         if((control & cbit)==0) { // literal
 599                                 u8 b;
 600
 601                                 if(pos+1 > endpos) goto unc_done;
 602                                 b = dbuf_getbyte(dcmpri->f, pos++);
 603                                 de_lz77buffer_add_literal_byte(sctx->ringbuf, b);
 604                                 if(sctx->stop_flag) goto unc_done;
 605                         }
 606                         else { // match
 607                                 UI x;
 608                                 UI matchpos;
 609                                 UI matchlen;
 610
 611                                 if(pos+2 > endpos) goto unc_done;
 612                                 x = (UI)dbuf_getu16le_p(dcmpri->f, &pos);
 613                                 matchlen = (x>>12) + 3;
 614                                 matchpos = sctx->ringbuf->curpos - ((x & 0x0fff)+1);
 615                                 de_lz77buffer_copy_from_hist(sctx->ringbuf, matchpos, matchlen);
 616                                 if(sctx->stop_flag) goto unc_done;
 617                         }
 618                 }
 619         }
 620
 621 unc_done:
 622         dres->bytes_consumed_valid = 1;
 623         dres->bytes_consumed = pos - dcmpri->pos;
 624         if(sctx) {
 625                 de_lz77buffer_destroy(c, sctx->ringbuf);
 626                 de_free(c, sctx);
 627         }
 628 }
 629
 630 //========================================================
 631
 632 struct my_2layer_userdata {
 633         struct de_dfilter_ctx *dfctx_codec2;
 634         i64 intermediate_nbytes;
 635 };
 636
 637 static void my_2layer_write_cb(dbuf *f, void *userdata,
 638         const u8 *buf, i64 size)
 639 {
 640         struct my_2layer_userdata *u = (struct my_2layer_userdata*)userdata;
 641
 642         de_dfilter_addbuf(u->dfctx_codec2, buf, size);
 643         u->intermediate_nbytes += size;
 644 }
 645
 646 static void dres_transfer_error(deark *c, struct de_dfilter_results *src,
 647         struct de_dfilter_results *dst)
 648 {
 649         if(src->errcode) {
 650                 dst->errcode = src->errcode;
 651                 de_strlcpy(dst->errmsg, src->errmsg, sizeof(dst->errmsg));
 652         }
 653 }
 654
 655 // Decompress an arbitrary two-layer compressed format.
 656 // tlp->codec1* is the first one that will be used during decompression (i.e. the second
 657 // method used when during *compression*).
 658 void de_dfilter_decompress_two_layer(deark *c, struct de_dcmpr_two_layer_params *tlp)
 659 {
 660         dbuf *outf_codec1 = NULL;
 661         struct de_dfilter_out_params dcmpro_codec1;
 662         struct de_dfilter_results dres_codec2;
 663         struct my_2layer_userdata u;
 664         struct de_dfilter_ctx *dfctx_codec2 = NULL;
 665
 666         de_dfilter_init_objects(c, NULL, &dcmpro_codec1, NULL);
 667         de_dfilter_init_objects(c, NULL, NULL, &dres_codec2);
 668         de_zeromem(&u, sizeof(struct my_2layer_userdata));
 669
 670         // Make a custom dbuf. The output from the first decompressor will be written
 671         // to it, and it will relay that output to the second decompressor.
 672         outf_codec1 = dbuf_create_custom_dbuf(c, 0, 0);
 673         outf_codec1->userdata_for_customwrite = (void*)&u;
 674         outf_codec1->customwrite_fn = my_2layer_write_cb;
 675
 676         dcmpro_codec1.f = outf_codec1;
 677         if(tlp->intermed_len_known) {
 678                 dcmpro_codec1.len_known = 1;
 679                 dcmpro_codec1.expected_len = tlp->intermed_expected_len;
 680         }
 681         else {
 682                 dcmpro_codec1.len_known = 0;
 683                 dcmpro_codec1.expected_len = 0;
 684         }
 685
 686         dfctx_codec2 = de_dfilter_create(c, tlp->codec2, tlp->codec2_private_params, tlp->dcmpro, &dres_codec2);
 687         u.dfctx_codec2 = dfctx_codec2;
 688
 689         // The first codec in the chain does not need the advanced (de_dfilter_create) API.
 690         if(tlp->codec1_type1) {
 691                 tlp->codec1_type1(c, tlp->dcmpri, &dcmpro_codec1, tlp->dres, tlp->codec1_private_params);
 692         }
 693         else {
 694                 de_dfilter_decompress_oneshot(c, tlp->codec1_pushable, tlp->codec1_private_params,
 695                         tlp->dcmpri, &dcmpro_codec1, tlp->dres);
 696         }
 697         de_dfilter_finish(dfctx_codec2);
 698
 699         if(tlp->dres->errcode) goto done;
 700         de_dbg2(c, "size after intermediate decompression: %"I64_FMT, u.intermediate_nbytes);
 701
 702         if(dres_codec2.errcode) {
 703                 // An error occurred in codec2, and not in codec1.
 704                 // Copy the error info to the dres that will be returned to the caller.
 705                 // TODO: Make a cleaner way to do this.
 706                 dres_transfer_error(c, &dres_codec2, tlp->dres);
 707                 goto done;
 708         }
 709
 710 done:
 711         de_dfilter_destroy(dfctx_codec2);
 712         dbuf_close(outf_codec1);
 713 }
 714
 715 // TODO: Retire this function.
 716 void de_dfilter_decompress_two_layer_type2(deark *c,
 717         dfilter_codec_type codec1, void *codec1_private_params,
 718         dfilter_codec_type codec2, void *codec2_private_params,
 719         struct de_dfilter_in_params *dcmpri, struct de_dfilter_out_params *dcmpro,
 720         struct de_dfilter_results *dres)
 721 {
 722         struct de_dcmpr_two_layer_params tlp;
 723
 724         de_zeromem(&tlp, sizeof(struct de_dcmpr_two_layer_params));
 725         tlp.codec1_pushable = codec1;
 726         tlp.codec1_private_params = codec1_private_params;
 727         tlp.codec2 = codec2;
 728         tlp.codec2_private_params = codec2_private_params;
 729         tlp.dcmpri = dcmpri;
 730         tlp.dcmpro = dcmpro;
 731         tlp.dres = dres;
 732         de_dfilter_decompress_two_layer(c, &tlp);
 733 }
 734
 735  struct de_lz77buffer *de_lz77buffer_create(deark *c, UI bufsize)
 736 {
 737         struct de_lz77buffer *rb;
 738
 739         rb = de_malloc(c, sizeof(struct de_lz77buffer));
 740         rb->buf = de_malloc(c, (i64)bufsize);
 741         rb->bufsize = bufsize;
 742         rb->mask = bufsize - 1;
 743         return rb;
 744 }
 745
 746 void de_lz77buffer_destroy(deark *c, struct de_lz77buffer *rb)
 747 {
 748         if(!rb) return;
 749         de_free(c, rb->buf);
 750         de_free(c, rb);
 751 }
 752
 753 // Set all bytes to the same value, and reset the current position to 0.
 754 void de_lz77buffer_clear(struct de_lz77buffer *rb, UI val)
 755 {
 756         de_memset(rb->buf, val, rb->bufsize);
 757         rb->curpos = 0;
 758 }
 759
 760 void de_lz77buffer_set_curpos(struct de_lz77buffer *rb, UI newpos)
 761 {
 762         rb->curpos = newpos & rb->mask;
 763 }
 764
 765 void de_lz77buffer_add_literal_byte(struct de_lz77buffer *rb, u8 b)
 766 {
 767         rb->writebyte_cb(rb, b);
 768         rb->buf[rb->curpos] = b;
 769         rb->curpos = (rb->curpos+1) & rb->mask;
 770 }
 771
 772 void de_lz77buffer_copy_from_hist(struct de_lz77buffer *rb,
 773         UI startpos, UI count)
 774 {
 775         UI frompos;
 776         UI i;
 777
 778         frompos = startpos & rb->mask;
 779         for(i=0; i<count; i++) {
 780                 de_lz77buffer_add_literal_byte(rb, rb->buf[frompos]);
 781                 frompos = (frompos+1) & rb->mask;
 782         }
 783 }