1 // SPDX-License-Identifier: 0BSD
6 * Author: Lasse Collin <lasse.collin@tukaani.org>
9 #include "xz_private.h"
10 #include "xz_stream.h"
12 /* Hash used to validate the Index field */
15 vli_type uncompressed
;
20 /* Position in dec_main() */
34 /* Position in variable-length integers and Check fields */
37 /* Variable-length integer decoded by dec_vli() */
40 /* Saved in_pos and out_pos */
44 /* CRC32 value in Block or Index */
47 /* Type of the integrity check calculated from uncompressed data */
48 enum xz_check check_type
;
54 * True if the next call to xz_dec_run() is allowed to return
59 /* Information stored in Block Header */
62 * Value stored in the Compressed Size field, or
63 * VLI_UNKNOWN if Compressed Size is not present.
68 * Value stored in the Uncompressed Size field, or
69 * VLI_UNKNOWN if Uncompressed Size is not present.
71 vli_type uncompressed
;
73 /* Size of the Block Header field */
77 /* Information collected when decoding Blocks */
79 /* Observed compressed size of the current Block */
82 /* Observed uncompressed size of the current Block */
83 vli_type uncompressed
;
85 /* Number of Blocks decoded so far */
89 * Hash calculated from the Block sizes. This is used to
90 * validate the Index field.
92 struct xz_dec_hash hash
;
95 /* Variables needed when verifying the Index field */
97 /* Position in dec_index() */
101 SEQ_INDEX_UNCOMPRESSED
104 /* Size of the Index in bytes */
107 /* Number of Records (matches block.count in valid files) */
111 * Hash calculated from the Records (matches block.hash in
114 struct xz_dec_hash hash
;
118 * Temporary buffer needed to hold Stream Header, Block Header,
119 * and Stream Footer. The Block Header is the biggest (1 KiB)
120 * so we reserve space according to that. buf[] has to be aligned
121 * to a multiple of four bytes; the size_t variables before it
122 * should guarantee this.
130 struct xz_dec_lzma2
*lzma2
;
133 struct xz_dec_bcj
*bcj
;
138 #ifdef XZ_DEC_ANY_CHECK
139 /* Sizes of the Check field with different Check IDs */
140 static const uint8_t check_sizes
[16] = {
151 * Fill s->temp by copying data starting from b->in[b->in_pos]. Caller
152 * must have set s->temp.pos to indicate how much data we are supposed
153 * to copy into s->temp.buf. Return true once s->temp.pos has reached
156 static bool fill_temp(struct xz_dec
*s
, struct xz_buf
*b
)
158 size_t copy_size
= min_t(size_t,
159 b
->in_size
- b
->in_pos
, s
->temp
.size
- s
->temp
.pos
);
161 memcpy(s
->temp
.buf
+ s
->temp
.pos
, b
->in
+ b
->in_pos
, copy_size
);
162 b
->in_pos
+= copy_size
;
163 s
->temp
.pos
+= copy_size
;
165 if (s
->temp
.pos
== s
->temp
.size
) {
173 /* Decode a variable-length integer (little-endian base-128 encoding) */
174 static enum xz_ret
dec_vli(struct xz_dec
*s
, const uint8_t *in
,
175 size_t *in_pos
, size_t in_size
)
182 while (*in_pos
< in_size
) {
186 s
->vli
|= (vli_type
)(byte
& 0x7F) << s
->pos
;
188 if ((byte
& 0x80) == 0) {
189 /* Don't allow non-minimal encodings. */
190 if (byte
== 0 && s
->pos
!= 0)
191 return XZ_DATA_ERROR
;
194 return XZ_STREAM_END
;
198 if (s
->pos
== 7 * VLI_BYTES_MAX
)
199 return XZ_DATA_ERROR
;
206 * Decode the Compressed Data field from a Block. Update and validate
207 * the observed compressed and uncompressed sizes of the Block so that
208 * they don't exceed the values possibly stored in the Block Header
209 * (validation assumes that no integer overflow occurs, since vli_type
210 * is normally uint64_t). Update the CRC32 if presence of the CRC32
211 * field was indicated in Stream Header.
213 * Once the decoding is finished, validate that the observed sizes match
214 * the sizes possibly stored in the Block Header. Update the hash and
215 * Block count, which are later used to validate the Index field.
217 static enum xz_ret
dec_block(struct xz_dec
*s
, struct xz_buf
*b
)
221 s
->in_start
= b
->in_pos
;
222 s
->out_start
= b
->out_pos
;
226 ret
= xz_dec_bcj_run(s
->bcj
, s
->lzma2
, b
);
229 ret
= xz_dec_lzma2_run(s
->lzma2
, b
);
231 s
->block
.compressed
+= b
->in_pos
- s
->in_start
;
232 s
->block
.uncompressed
+= b
->out_pos
- s
->out_start
;
235 * There is no need to separately check for VLI_UNKNOWN, since
236 * the observed sizes are always smaller than VLI_UNKNOWN.
238 if (s
->block
.compressed
> s
->block_header
.compressed
239 || s
->block
.uncompressed
240 > s
->block_header
.uncompressed
)
241 return XZ_DATA_ERROR
;
243 if (s
->check_type
== XZ_CHECK_CRC32
)
244 s
->crc32
= xz_crc32(b
->out
+ s
->out_start
,
245 b
->out_pos
- s
->out_start
, s
->crc32
);
247 if (ret
== XZ_STREAM_END
) {
248 if (s
->block_header
.compressed
!= VLI_UNKNOWN
249 && s
->block_header
.compressed
250 != s
->block
.compressed
)
251 return XZ_DATA_ERROR
;
253 if (s
->block_header
.uncompressed
!= VLI_UNKNOWN
254 && s
->block_header
.uncompressed
255 != s
->block
.uncompressed
)
256 return XZ_DATA_ERROR
;
258 s
->block
.hash
.unpadded
+= s
->block_header
.size
259 + s
->block
.compressed
;
261 #ifdef XZ_DEC_ANY_CHECK
262 s
->block
.hash
.unpadded
+= check_sizes
[s
->check_type
];
264 if (s
->check_type
== XZ_CHECK_CRC32
)
265 s
->block
.hash
.unpadded
+= 4;
268 s
->block
.hash
.uncompressed
+= s
->block
.uncompressed
;
269 s
->block
.hash
.crc32
= xz_crc32(
270 (const uint8_t *)&s
->block
.hash
,
271 sizeof(s
->block
.hash
), s
->block
.hash
.crc32
);
279 /* Update the Index size and the CRC32 value. */
280 static void index_update(struct xz_dec
*s
, const struct xz_buf
*b
)
282 size_t in_used
= b
->in_pos
- s
->in_start
;
283 s
->index
.size
+= in_used
;
284 s
->crc32
= xz_crc32(b
->in
+ s
->in_start
, in_used
, s
->crc32
);
288 * Decode the Number of Records, Unpadded Size, and Uncompressed Size
289 * fields from the Index field. That is, Index Padding and CRC32 are not
290 * decoded by this function.
292 * This can return XZ_OK (more input needed), XZ_STREAM_END (everything
293 * successfully decoded), or XZ_DATA_ERROR (input is corrupt).
295 static enum xz_ret
dec_index(struct xz_dec
*s
, struct xz_buf
*b
)
300 ret
= dec_vli(s
, b
->in
, &b
->in_pos
, b
->in_size
);
301 if (ret
!= XZ_STREAM_END
) {
306 switch (s
->index
.sequence
) {
307 case SEQ_INDEX_COUNT
:
308 s
->index
.count
= s
->vli
;
311 * Validate that the Number of Records field
312 * indicates the same number of Records as
313 * there were Blocks in the Stream.
315 if (s
->index
.count
!= s
->block
.count
)
316 return XZ_DATA_ERROR
;
318 s
->index
.sequence
= SEQ_INDEX_UNPADDED
;
321 case SEQ_INDEX_UNPADDED
:
322 s
->index
.hash
.unpadded
+= s
->vli
;
323 s
->index
.sequence
= SEQ_INDEX_UNCOMPRESSED
;
326 case SEQ_INDEX_UNCOMPRESSED
:
327 s
->index
.hash
.uncompressed
+= s
->vli
;
328 s
->index
.hash
.crc32
= xz_crc32(
329 (const uint8_t *)&s
->index
.hash
,
330 sizeof(s
->index
.hash
),
331 s
->index
.hash
.crc32
);
333 s
->index
.sequence
= SEQ_INDEX_UNPADDED
;
336 } while (s
->index
.count
> 0);
338 return XZ_STREAM_END
;
342 * Validate that the next four input bytes match the value of s->crc32.
343 * s->pos must be zero when starting to validate the first byte.
345 static enum xz_ret
crc32_validate(struct xz_dec
*s
, struct xz_buf
*b
)
348 if (b
->in_pos
== b
->in_size
)
351 if (((s
->crc32
>> s
->pos
) & 0xFF) != b
->in
[b
->in_pos
++])
352 return XZ_DATA_ERROR
;
356 } while (s
->pos
< 32);
361 return XZ_STREAM_END
;
364 #ifdef XZ_DEC_ANY_CHECK
366 * Skip over the Check field when the Check ID is not supported.
367 * Returns true once the whole Check field has been skipped over.
369 static bool check_skip(struct xz_dec
*s
, struct xz_buf
*b
)
371 while (s
->pos
< check_sizes
[s
->check_type
]) {
372 if (b
->in_pos
== b
->in_size
)
385 /* Decode the Stream Header field (the first 12 bytes of the .xz Stream). */
386 static enum xz_ret
dec_stream_header(struct xz_dec
*s
)
388 if (!memeq(s
->temp
.buf
, HEADER_MAGIC
, HEADER_MAGIC_SIZE
))
389 return XZ_FORMAT_ERROR
;
391 if (xz_crc32(s
->temp
.buf
+ HEADER_MAGIC_SIZE
, 2, 0)
392 != get_le32(s
->temp
.buf
+ HEADER_MAGIC_SIZE
+ 2))
393 return XZ_DATA_ERROR
;
395 if (s
->temp
.buf
[HEADER_MAGIC_SIZE
] != 0)
396 return XZ_OPTIONS_ERROR
;
399 * Of integrity checks, we support only none (Check ID = 0) and
400 * CRC32 (Check ID = 1). However, if XZ_DEC_ANY_CHECK is defined,
401 * we will accept other check types too, but then the check won't
402 * be verified and a warning (XZ_UNSUPPORTED_CHECK) will be given.
404 if (s
->temp
.buf
[HEADER_MAGIC_SIZE
+ 1] > XZ_CHECK_MAX
)
405 return XZ_OPTIONS_ERROR
;
407 s
->check_type
= s
->temp
.buf
[HEADER_MAGIC_SIZE
+ 1];
409 #ifdef XZ_DEC_ANY_CHECK
410 if (s
->check_type
> XZ_CHECK_CRC32
)
411 return XZ_UNSUPPORTED_CHECK
;
413 if (s
->check_type
> XZ_CHECK_CRC32
)
414 return XZ_OPTIONS_ERROR
;
420 /* Decode the Stream Footer field (the last 12 bytes of the .xz Stream) */
421 static enum xz_ret
dec_stream_footer(struct xz_dec
*s
)
423 if (!memeq(s
->temp
.buf
+ 10, FOOTER_MAGIC
, FOOTER_MAGIC_SIZE
))
424 return XZ_DATA_ERROR
;
426 if (xz_crc32(s
->temp
.buf
+ 4, 6, 0) != get_le32(s
->temp
.buf
))
427 return XZ_DATA_ERROR
;
430 * Validate Backward Size. Note that we never added the size of the
431 * Index CRC32 field to s->index.size, thus we use s->index.size / 4
432 * instead of s->index.size / 4 - 1.
434 if ((s
->index
.size
>> 2) != get_le32(s
->temp
.buf
+ 4))
435 return XZ_DATA_ERROR
;
437 if (s
->temp
.buf
[8] != 0 || s
->temp
.buf
[9] != s
->check_type
)
438 return XZ_DATA_ERROR
;
441 * Use XZ_STREAM_END instead of XZ_OK to be more convenient
444 return XZ_STREAM_END
;
447 /* Decode the Block Header and initialize the filter chain. */
448 static enum xz_ret
dec_block_header(struct xz_dec
*s
)
453 * Validate the CRC32. We know that the temp buffer is at least
454 * eight bytes so this is safe.
457 if (xz_crc32(s
->temp
.buf
, s
->temp
.size
, 0)
458 != get_le32(s
->temp
.buf
+ s
->temp
.size
))
459 return XZ_DATA_ERROR
;
464 * Catch unsupported Block Flags. We support only one or two filters
465 * in the chain, so we catch that with the same test.
468 if (s
->temp
.buf
[1] & 0x3E)
470 if (s
->temp
.buf
[1] & 0x3F)
472 return XZ_OPTIONS_ERROR
;
474 /* Compressed Size */
475 if (s
->temp
.buf
[1] & 0x40) {
476 if (dec_vli(s
, s
->temp
.buf
, &s
->temp
.pos
, s
->temp
.size
)
478 return XZ_DATA_ERROR
;
480 s
->block_header
.compressed
= s
->vli
;
482 s
->block_header
.compressed
= VLI_UNKNOWN
;
485 /* Uncompressed Size */
486 if (s
->temp
.buf
[1] & 0x80) {
487 if (dec_vli(s
, s
->temp
.buf
, &s
->temp
.pos
, s
->temp
.size
)
489 return XZ_DATA_ERROR
;
491 s
->block_header
.uncompressed
= s
->vli
;
493 s
->block_header
.uncompressed
= VLI_UNKNOWN
;
497 /* If there are two filters, the first one must be a BCJ filter. */
498 s
->bcj_active
= s
->temp
.buf
[1] & 0x01;
500 if (s
->temp
.size
- s
->temp
.pos
< 2)
501 return XZ_OPTIONS_ERROR
;
503 ret
= xz_dec_bcj_reset(s
->bcj
, s
->temp
.buf
[s
->temp
.pos
++]);
508 * We don't support custom start offset,
509 * so Size of Properties must be zero.
511 if (s
->temp
.buf
[s
->temp
.pos
++] != 0x00)
512 return XZ_OPTIONS_ERROR
;
516 /* Valid Filter Flags always take at least two bytes. */
517 if (s
->temp
.size
- s
->temp
.pos
< 2)
518 return XZ_DATA_ERROR
;
520 /* Filter ID = LZMA2 */
521 if (s
->temp
.buf
[s
->temp
.pos
++] != 0x21)
522 return XZ_OPTIONS_ERROR
;
524 /* Size of Properties = 1-byte Filter Properties */
525 if (s
->temp
.buf
[s
->temp
.pos
++] != 0x01)
526 return XZ_OPTIONS_ERROR
;
528 /* Filter Properties contains LZMA2 dictionary size. */
529 if (s
->temp
.size
- s
->temp
.pos
< 1)
530 return XZ_DATA_ERROR
;
532 ret
= xz_dec_lzma2_reset(s
->lzma2
, s
->temp
.buf
[s
->temp
.pos
++]);
536 /* The rest must be Header Padding. */
537 while (s
->temp
.pos
< s
->temp
.size
)
538 if (s
->temp
.buf
[s
->temp
.pos
++] != 0x00)
539 return XZ_OPTIONS_ERROR
;
542 s
->block
.compressed
= 0;
543 s
->block
.uncompressed
= 0;
548 static enum xz_ret
dec_main(struct xz_dec
*s
, struct xz_buf
*b
)
553 * Store the start position for the case when we are in the middle
554 * of the Index field.
556 s
->in_start
= b
->in_pos
;
559 switch (s
->sequence
) {
560 case SEQ_STREAM_HEADER
:
562 * Stream Header is copied to s->temp, and then
563 * decoded from there. This way if the caller
564 * gives us only little input at a time, we can
565 * still keep the Stream Header decoding code
566 * simple. Similar approach is used in many places
569 if (!fill_temp(s
, b
))
573 * If dec_stream_header() returns
574 * XZ_UNSUPPORTED_CHECK, it is still possible
575 * to continue decoding if working in multi-call
576 * mode. Thus, update s->sequence before calling
577 * dec_stream_header().
579 s
->sequence
= SEQ_BLOCK_START
;
581 ret
= dec_stream_header(s
);
587 case SEQ_BLOCK_START
:
588 /* We need one byte of input to continue. */
589 if (b
->in_pos
== b
->in_size
)
592 /* See if this is the beginning of the Index field. */
593 if (b
->in
[b
->in_pos
] == 0) {
594 s
->in_start
= b
->in_pos
++;
595 s
->sequence
= SEQ_INDEX
;
600 * Calculate the size of the Block Header and
601 * prepare to decode it.
604 = ((uint32_t)b
->in
[b
->in_pos
] + 1) * 4;
606 s
->temp
.size
= s
->block_header
.size
;
608 s
->sequence
= SEQ_BLOCK_HEADER
;
612 case SEQ_BLOCK_HEADER
:
613 if (!fill_temp(s
, b
))
616 ret
= dec_block_header(s
);
620 s
->sequence
= SEQ_BLOCK_UNCOMPRESS
;
624 case SEQ_BLOCK_UNCOMPRESS
:
625 ret
= dec_block(s
, b
);
626 if (ret
!= XZ_STREAM_END
)
629 s
->sequence
= SEQ_BLOCK_PADDING
;
633 case SEQ_BLOCK_PADDING
:
635 * Size of Compressed Data + Block Padding
636 * must be a multiple of four. We don't need
637 * s->block.compressed for anything else
638 * anymore, so we use it here to test the size
639 * of the Block Padding field.
641 while (s
->block
.compressed
& 3) {
642 if (b
->in_pos
== b
->in_size
)
645 if (b
->in
[b
->in_pos
++] != 0)
646 return XZ_DATA_ERROR
;
648 ++s
->block
.compressed
;
651 s
->sequence
= SEQ_BLOCK_CHECK
;
655 case SEQ_BLOCK_CHECK
:
656 if (s
->check_type
== XZ_CHECK_CRC32
) {
657 ret
= crc32_validate(s
, b
);
658 if (ret
!= XZ_STREAM_END
)
661 #ifdef XZ_DEC_ANY_CHECK
662 else if (!check_skip(s
, b
)) {
667 s
->sequence
= SEQ_BLOCK_START
;
671 ret
= dec_index(s
, b
);
672 if (ret
!= XZ_STREAM_END
)
675 s
->sequence
= SEQ_INDEX_PADDING
;
679 case SEQ_INDEX_PADDING
:
680 while ((s
->index
.size
+ (b
->in_pos
- s
->in_start
))
682 if (b
->in_pos
== b
->in_size
) {
687 if (b
->in
[b
->in_pos
++] != 0)
688 return XZ_DATA_ERROR
;
691 /* Finish the CRC32 value and Index size. */
694 /* Compare the hashes to validate the Index field. */
695 if (!memeq(&s
->block
.hash
, &s
->index
.hash
,
696 sizeof(s
->block
.hash
)))
697 return XZ_DATA_ERROR
;
699 s
->sequence
= SEQ_INDEX_CRC32
;
703 case SEQ_INDEX_CRC32
:
704 ret
= crc32_validate(s
, b
);
705 if (ret
!= XZ_STREAM_END
)
708 s
->temp
.size
= STREAM_HEADER_SIZE
;
709 s
->sequence
= SEQ_STREAM_FOOTER
;
713 case SEQ_STREAM_FOOTER
:
714 if (!fill_temp(s
, b
))
717 return dec_stream_footer(s
);
725 * xz_dec_run() is a wrapper for dec_main() to handle some special cases in
726 * multi-call and single-call decoding.
728 * In multi-call mode, we must return XZ_BUF_ERROR when it seems clear that we
729 * are not going to make any progress anymore. This is to prevent the caller
730 * from calling us infinitely when the input file is truncated or otherwise
731 * corrupt. Since zlib-style API allows that the caller fills the input buffer
732 * only when the decoder doesn't produce any new output, we have to be careful
733 * to avoid returning XZ_BUF_ERROR too easily: XZ_BUF_ERROR is returned only
734 * after the second consecutive call to xz_dec_run() that makes no progress.
736 * In single-call mode, if we couldn't decode everything and no error
737 * occurred, either the input is truncated or the output buffer is too small.
738 * Since we know that the last input byte never produces any output, we know
739 * that if all the input was consumed and decoding wasn't finished, the file
740 * must be corrupt. Otherwise the output buffer has to be too small or the
741 * file is corrupt in a way that decoding it produces too big output.
743 * If single-call decoding fails, we reset b->in_pos and b->out_pos back to
744 * their original values. This is because with some filter chains there won't
745 * be any valid uncompressed data in the output buffer unless the decoding
746 * actually succeeds (that's the price to pay of using the output buffer as
749 enum xz_ret
xz_dec_run(struct xz_dec
*s
, struct xz_buf
*b
)
755 if (DEC_IS_SINGLE(s
->mode
))
758 in_start
= b
->in_pos
;
759 out_start
= b
->out_pos
;
760 ret
= dec_main(s
, b
);
762 if (DEC_IS_SINGLE(s
->mode
)) {
764 ret
= b
->in_pos
== b
->in_size
765 ? XZ_DATA_ERROR
: XZ_BUF_ERROR
;
767 if (ret
!= XZ_STREAM_END
) {
768 b
->in_pos
= in_start
;
769 b
->out_pos
= out_start
;
772 } else if (ret
== XZ_OK
&& in_start
== b
->in_pos
773 && out_start
== b
->out_pos
) {
774 if (s
->allow_buf_error
)
777 s
->allow_buf_error
= true;
779 s
->allow_buf_error
= false;
785 struct xz_dec
*xz_dec_init(enum xz_mode mode
, uint32_t dict_max
)
787 struct xz_dec
*s
= kmalloc(sizeof(*s
), GFP_KERNEL
);
794 s
->bcj
= xz_dec_bcj_create(DEC_IS_SINGLE(mode
));
799 s
->lzma2
= xz_dec_lzma2_create(mode
, dict_max
);
800 if (s
->lzma2
== NULL
)
808 xz_dec_bcj_end(s
->bcj
);
815 void xz_dec_reset(struct xz_dec
*s
)
817 s
->sequence
= SEQ_STREAM_HEADER
;
818 s
->allow_buf_error
= false;
821 memzero(&s
->block
, sizeof(s
->block
));
822 memzero(&s
->index
, sizeof(s
->index
));
824 s
->temp
.size
= STREAM_HEADER_SIZE
;
827 void xz_dec_end(struct xz_dec
*s
)
830 xz_dec_lzma2_end(s
->lzma2
);
832 xz_dec_bcj_end(s
->bcj
);