src/liblzma/common/lzip_decoder.c

   1 ///////////////////////////////////////////////////////////////////////////////
   2 //
   3 /// \file       lzip_decoder.c
   4 /// \brief      Decodes .lz (lzip) files
   5 //
   6 //  Author:     Michał Górny
   7 //              Lasse Collin
   8 //
   9 //  This file has been put into the public domain.
  10 //  You can do whatever you want with this file.
  11 //
  12 ///////////////////////////////////////////////////////////////////////////////
  13
  14 #include "lzip_decoder.h"
  15 #include "lzma_decoder.h"
  16 #include "check.h"
  17
  18
  19 // .lz format version 0 lacks the 64-bit Member size field in the footer.
  20 #define LZIP_V0_FOOTER_SIZE 12
  21 #define LZIP_V1_FOOTER_SIZE 20
  22 #define LZIP_FOOTER_SIZE_MAX LZIP_V1_FOOTER_SIZE
  23
  24 // lc/lp/pb are hardcoded in the .lz format.
  25 #define LZIP_LC 3
  26 #define LZIP_LP 0
  27 #define LZIP_PB 2
  28
  29
  30 typedef struct {
  31         enum {
  32                 SEQ_ID_STRING,
  33                 SEQ_VERSION,
  34                 SEQ_DICT_SIZE,
  35                 SEQ_CODER_INIT,
  36                 SEQ_LZMA_STREAM,
  37                 SEQ_MEMBER_FOOTER,
  38         } sequence;
  39
  40         /// .lz member format version
  41         uint32_t version;
  42
  43         /// CRC32 of the uncompressed data in the .lz member
  44         uint32_t crc32;
  45
  46         /// Uncompressed size of the .lz member
  47         uint64_t uncompressed_size;
  48
  49         /// Compressed size of the .lz member
  50         uint64_t member_size;
  51
  52         /// Memory usage limit
  53         uint64_t memlimit;
  54
  55         /// Amount of memory actually needed
  56         uint64_t memusage;
  57
  58         /// If true, LZMA_GET_CHECK is returned after decoding the header
  59         /// fields. As all files use CRC32 this is redundant but it's
  60         /// implemented anyway since the initialization functions supports
  61         /// all other flags in addition to LZMA_TELL_ANY_CHECK.
  62         bool tell_any_check;
  63
  64         /// If true, we won't calculate or verify the CRC32 of
  65         /// the uncompressed data.
  66         bool ignore_check;
  67
  68         /// If true, we will decode concatenated .lz members and stop if
  69         /// non-.lz data is seen after at least one member has been
  70         /// successfully decoded.
  71         bool concatenated;
  72
  73         /// When decoding concatenated .lz members, this is true as long as
  74         /// we are decoding the first .lz member. This is needed to avoid
  75         /// incorrect LZMA_FORMAT_ERROR in case there is non-.lz data at
  76         /// the end of the file.
  77         bool first_member;
  78
  79         /// Reading position in the header and footer fields
  80         size_t pos;
  81
  82         /// Buffer to hold the .lz footer fields
  83         uint8_t buffer[LZIP_FOOTER_SIZE_MAX];
  84
  85         /// Options decoded from the .lz header that needed to initialize
  86         /// the LZMA1 decoder.
  87         lzma_options_lzma options;
  88
  89         /// LZMA1 decoder
  90         lzma_next_coder lzma_decoder;
  91
  92 } lzma_lzip_coder;
  93
  94
  95 static lzma_ret
  96 lzip_decode(void *coder_ptr, const lzma_allocator *allocator,
  97                 const uint8_t *restrict in, size_t *restrict in_pos,
  98                 size_t in_size, uint8_t *restrict out,
  99                 size_t *restrict out_pos, size_t out_size, lzma_action action)
 100 {
 101         lzma_lzip_coder *coder = coder_ptr;
 102
 103         while (true)
 104         switch (coder->sequence) {
 105         case SEQ_ID_STRING: {
 106                 // The "ID string" or magic bytes are "LZIP" in US-ASCII.
 107                 const uint8_t lzip_id_string[4] = { 0x4C, 0x5A, 0x49, 0x50 };
 108
 109                 while (coder->pos < sizeof(lzip_id_string)) {
 110                         if (*in_pos >= in_size) {
 111                                 // If we are on the 2nd+ concatenated member
 112                                 // and the input ends before we can read
 113                                 // the magic bytes, we discard the bytes that
 114                                 // were already read (up to 3) and finish.
 115                                 // See the reasoning below.
 116                                 return !coder->first_member
 117                                                 && action == LZMA_FINISH
 118                                         ? LZMA_STREAM_END : LZMA_OK;
 119                         }
 120
 121                         if (in[*in_pos] != lzip_id_string[coder->pos]) {
 122                                 // The .lz format allows putting non-.lz data
 123                                 // at the end of the file. If we have seen
 124                                 // at least one valid .lz member already,
 125                                 // then we won't consume the byte at *in_pos
 126                                 // and will return LZMA_STREAM_END. This way
 127                                 // apps can easily locate and read the non-.lz
 128                                 // data after the .lz member(s).
 129                                 //
 130                                 // NOTE: If the first 1-3 bytes of the non-.lz
 131                                 // data match the .lz ID string then the first
 132                                 // 1-3 bytes of the junk will get ignored by
 133                                 // us. If apps want to properly locate the
 134                                 // trailing data they must ensure that the
 135                                 // first byte of their custom data isn't the
 136                                 // same as the first byte of .lz ID string.
 137                                 // With the liblzma API we cannot rewind the
 138                                 // input position across calls to lzma_code().
 139                                 return !coder->first_member
 140                                         ? LZMA_STREAM_END : LZMA_FORMAT_ERROR;
 141                         }
 142
 143                         ++*in_pos;
 144                         ++coder->pos;
 145                 }
 146
 147                 coder->pos = 0;
 148
 149                 coder->crc32 = 0;
 150                 coder->uncompressed_size = 0;
 151                 coder->member_size = sizeof(lzip_id_string);
 152
 153                 coder->sequence = SEQ_VERSION;
 154         }
 155
 156         // Fall through
 157
 158         case SEQ_VERSION:
 159                 if (*in_pos >= in_size)
 160                         return LZMA_OK;
 161
 162                 coder->version = in[(*in_pos)++];
 163
 164                 // We support version 0 and unextended version 1.
 165                 if (coder->version > 1)
 166                         return LZMA_OPTIONS_ERROR;
 167
 168                 ++coder->member_size;
 169                 coder->sequence = SEQ_DICT_SIZE;
 170
 171                 // .lz versions 0 and 1 use CRC32 as the integrity check
 172                 // so if the application wanted to know that
 173                 // (LZMA_TELL_ANY_CHECK) we can tell it now.
 174                 if (coder->tell_any_check)
 175                         return LZMA_GET_CHECK;
 176
 177         // Fall through
 178
 179         case SEQ_DICT_SIZE: {
 180                 if (*in_pos >= in_size)
 181                         return LZMA_OK;
 182
 183                 const uint32_t ds = in[(*in_pos)++];
 184                 ++coder->member_size;
 185
 186                 // The five lowest bits are for the base-2 logarithm of
 187                 // the dictionary size and the highest three bits are
 188                 // the fractional part (0/16 to 7/16) that will be
 189                 // substracted to get the final value.
 190                 //
 191                 // For example, with 0xB5:
 192                 //     b2log = 21
 193                 //     fracnum = 5
 194                 //     dict_size = 2^21 - 2^21 * 5 / 16 = 1408 KiB
 195                 const uint32_t b2log = ds & 0x1F;
 196                 const uint32_t fracnum = ds >> 5;
 197
 198                 // The format versions 0 and 1 allow dictionary size in the
 199                 // range [4 KiB, 512 MiB].
 200                 if (b2log < 12 || b2log > 29 || (b2log == 12 && fracnum > 0))
 201                         return LZMA_DATA_ERROR;
 202
 203                 //   2^[b2log] - 2^[b2log] * [fracnum] / 16
 204                 // = 2^[b2log] - [fracnum] * 2^([b2log] - 4)
 205                 coder->options.dict_size = (UINT32_C(1) << b2log)
 206                                 - (fracnum << (b2log - 4));
 207
 208                 assert(coder->options.dict_size >= 4096);
 209                 assert(coder->options.dict_size <= (UINT32_C(512) << 20));
 210
 211                 coder->options.preset_dict = NULL;
 212                 coder->options.lc = LZIP_LC;
 213                 coder->options.lp = LZIP_LP;
 214                 coder->options.pb = LZIP_PB;
 215
 216                 // Calculate the memory usage.
 217                 coder->memusage = lzma_lzma_decoder_memusage(&coder->options)
 218                                 + LZMA_MEMUSAGE_BASE;
 219
 220                 // Initialization is a separate step because if we return
 221                 // LZMA_MEMLIMIT_ERROR we need to be able to restart after
 222                 // the memlimit has been increased.
 223                 coder->sequence = SEQ_CODER_INIT;
 224         }
 225
 226         // Fall through
 227
 228         case SEQ_CODER_INIT: {
 229                 if (coder->memusage > coder->memlimit)
 230                         return LZMA_MEMLIMIT_ERROR;
 231
 232                 const lzma_filter_info filters[2] = {
 233                         {
 234                                 .id = LZMA_FILTER_LZMA1,
 235                                 .init = &lzma_lzma_decoder_init,
 236                                 .options = &coder->options,
 237                         }, {
 238                                 .init = NULL,
 239                         }
 240                 };
 241
 242                 return_if_error(lzma_next_filter_init(&coder->lzma_decoder,
 243                                 allocator, filters));
 244
 245                 coder->crc32 = 0;
 246                 coder->sequence = SEQ_LZMA_STREAM;
 247         }
 248
 249         // Fall through
 250
 251         case SEQ_LZMA_STREAM: {
 252                 const size_t in_start = *in_pos;
 253                 const size_t out_start = *out_pos;
 254
 255                 const lzma_ret ret = coder->lzma_decoder.code(
 256                                 coder->lzma_decoder.coder, allocator,
 257                                 in, in_pos, in_size, out, out_pos, out_size,
 258                                 action);
 259
 260                 const size_t out_used = *out_pos - out_start;
 261
 262                 coder->member_size += *in_pos - in_start;
 263                 coder->uncompressed_size += out_used;
 264
 265                 // Don't update the CRC32 if the integrity check will be
 266                 // ignored or if there was no new output. The latter is
 267                 // important in case out == NULL to avoid null pointer + 0
 268                 // which is undefined behavior.
 269                 if (!coder->ignore_check && out_used > 0)
 270                         coder->crc32 = lzma_crc32(out + out_start, out_used,
 271                                         coder->crc32);
 272
 273                 if (ret != LZMA_STREAM_END)
 274                         return ret;
 275
 276                 coder->sequence = SEQ_MEMBER_FOOTER;
 277         }
 278
 279         // Fall through
 280
 281         case SEQ_MEMBER_FOOTER: {
 282                 // The footer of .lz version 0 lacks the Member size field.
 283                 // This is the only difference between version 0 and
 284                 // unextended version 1 formats.
 285                 const size_t footer_size = coder->version == 0
 286                                 ? LZIP_V0_FOOTER_SIZE
 287                                 : LZIP_V1_FOOTER_SIZE;
 288
 289                 // Copy the CRC32, Data size, and Member size fields to
 290                 // the internal buffer.
 291                 lzma_bufcpy(in, in_pos, in_size, coder->buffer, &coder->pos,
 292                                 footer_size);
 293
 294                 // Return if we didn't get the whole footer yet.
 295                 if (coder->pos < footer_size)
 296                         return LZMA_OK;
 297
 298                 coder->pos = 0;
 299                 coder->member_size += footer_size;
 300
 301                 // Check that the footer fields match the observed data.
 302                 if (!coder->ignore_check
 303                                 && coder->crc32 != read32le(&coder->buffer[0]))
 304                         return LZMA_DATA_ERROR;
 305
 306                 if (coder->uncompressed_size != read64le(&coder->buffer[4]))
 307                         return LZMA_DATA_ERROR;
 308
 309                 if (coder->version > 0) {
 310                         // .lz version 0 has no Member size field.
 311                         if (coder->member_size != read64le(&coder->buffer[12]))
 312                                 return LZMA_DATA_ERROR;
 313                 }
 314
 315                 // Decoding is finished if we weren't requested to decode
 316                 // more than one .lz member.
 317                 if (!coder->concatenated)
 318                         return LZMA_STREAM_END;
 319
 320                 coder->first_member = false;
 321                 coder->sequence = SEQ_ID_STRING;
 322                 break;
 323         }
 324
 325         default:
 326                 assert(0);
 327                 return LZMA_PROG_ERROR;
 328         }
 329
 330         // Never reached
 331 }
 332
 333
 334 static void
 335 lzip_decoder_end(void *coder_ptr, const lzma_allocator *allocator)
 336 {
 337         lzma_lzip_coder *coder = coder_ptr;
 338         lzma_next_end(&coder->lzma_decoder, allocator);
 339         lzma_free(coder, allocator);
 340         return;
 341 }
 342
 343
 344 static lzma_check
 345 lzip_decoder_get_check(const void *coder_ptr lzma_attribute((__unused__)))
 346 {
 347         return LZMA_CHECK_CRC32;
 348 }
 349
 350
 351 static lzma_ret
 352 lzip_decoder_memconfig(void *coder_ptr, uint64_t *memusage,
 353                 uint64_t *old_memlimit, uint64_t new_memlimit)
 354 {
 355         lzma_lzip_coder *coder = coder_ptr;
 356
 357         *memusage = coder->memusage;
 358         *old_memlimit = coder->memlimit;
 359
 360         if (new_memlimit != 0) {
 361                 if (new_memlimit < coder->memusage)
 362                         return LZMA_MEMLIMIT_ERROR;
 363
 364                 coder->memlimit = new_memlimit;
 365         }
 366
 367         return LZMA_OK;
 368 }
 369
 370
 371 extern lzma_ret
 372 lzma_lzip_decoder_init(
 373                 lzma_next_coder *next, const lzma_allocator *allocator,
 374                 uint64_t memlimit, uint32_t flags)
 375 {
 376         lzma_next_coder_init(&lzma_lzip_decoder_init, next, allocator);
 377
 378         if (flags & ~LZMA_SUPPORTED_FLAGS)
 379                 return LZMA_OPTIONS_ERROR;
 380
 381         lzma_lzip_coder *coder = next->coder;
 382         if (coder == NULL) {
 383                 coder = lzma_alloc(sizeof(lzma_lzip_coder), allocator);
 384                 if (coder == NULL)
 385                         return LZMA_MEM_ERROR;
 386
 387                 next->coder = coder;
 388                 next->code = &lzip_decode;
 389                 next->end = &lzip_decoder_end;
 390                 next->get_check = &lzip_decoder_get_check;
 391                 next->memconfig = &lzip_decoder_memconfig;
 392
 393                 coder->lzma_decoder = LZMA_NEXT_CODER_INIT;
 394         }
 395
 396         coder->sequence = SEQ_ID_STRING;
 397         coder->memlimit = my_max(1, memlimit);
 398         coder->memusage = LZMA_MEMUSAGE_BASE;
 399         coder->tell_any_check = (flags & LZMA_TELL_ANY_CHECK) != 0;
 400         coder->ignore_check = (flags & LZMA_IGNORE_CHECK) != 0;
 401         coder->concatenated = (flags & LZMA_CONCATENATED) != 0;
 402         coder->first_member = true;
 403         coder->pos = 0;
 404
 405         return LZMA_OK;
 406 }
 407
 408
 409 extern LZMA_API(lzma_ret)
 410 lzma_lzip_decoder(lzma_stream *strm, uint64_t memlimit, uint32_t flags)
 411 {
 412         lzma_next_strm_init(lzma_lzip_decoder_init, strm, memlimit, flags);
 413
 414         strm->internal->supported_actions[LZMA_RUN] = true;
 415         strm->internal->supported_actions[LZMA_FINISH] = true;
 416
 417         return LZMA_OK;
 418 }