1 /* XzEnc.c -- Xz Encode
2 2019-02-02 : Igor Pavlov : Public domain */
26 #define MTCODER__THREADS_MAX 1
27 #define MTCODER__BLOCKS_MAX 1
30 #define XZ_GET_PAD_SIZE(dataSize) ((4 - ((unsigned)(dataSize) & 3)) & 3)
32 /* max pack size for LZMA2 block + check-64bytrs: */
33 #define XZ_GET_MAX_BLOCK_PACK_SIZE(unpackSize) ((unpackSize) + ((unpackSize) >> 10) + 16 + 64)
35 #define XZ_GET_ESTIMATED_BLOCK_TOTAL_PACK_SIZE(unpackSize) (XZ_BLOCK_HEADER_SIZE_MAX + XZ_GET_MAX_BLOCK_PACK_SIZE(unpackSize))
38 #define XzBlock_ClearFlags(p) (p)->flags = 0;
39 #define XzBlock_SetNumFilters(p, n) (p)->flags |= ((n) - 1);
40 #define XzBlock_SetHasPackSize(p) (p)->flags |= XZ_BF_PACK_SIZE;
41 #define XzBlock_SetHasUnpackSize(p) (p)->flags |= XZ_BF_UNPACK_SIZE;
44 static SRes
WriteBytes(ISeqOutStream
*s
, const void *buf
, size_t size
)
46 return (ISeqOutStream_Write(s
, buf
, size
) == size
) ? SZ_OK
: SZ_ERROR_WRITE
;
49 static SRes
WriteBytesUpdateCrc(ISeqOutStream
*s
, const void *buf
, size_t size
, UInt32
*crc
)
51 *crc
= CrcUpdate(*crc
, buf
, size
);
52 return WriteBytes(s
, buf
, size
);
56 static SRes
Xz_WriteHeader(CXzStreamFlags f
, ISeqOutStream
*s
)
59 Byte header
[XZ_STREAM_HEADER_SIZE
];
60 memcpy(header
, XZ_SIG
, XZ_SIG_SIZE
);
61 header
[XZ_SIG_SIZE
] = (Byte
)(f
>> 8);
62 header
[XZ_SIG_SIZE
+ 1] = (Byte
)(f
& 0xFF);
63 crc
= CrcCalc(header
+ XZ_SIG_SIZE
, XZ_STREAM_FLAGS_SIZE
);
64 SetUi32(header
+ XZ_SIG_SIZE
+ XZ_STREAM_FLAGS_SIZE
, crc
);
65 return WriteBytes(s
, header
, XZ_STREAM_HEADER_SIZE
);
69 static SRes
XzBlock_WriteHeader(const CXzBlock
*p
, ISeqOutStream
*s
)
71 Byte header
[XZ_BLOCK_HEADER_SIZE_MAX
];
74 unsigned numFilters
, i
;
75 header
[pos
++] = p
->flags
;
77 if (XzBlock_HasPackSize(p
)) pos
+= Xz_WriteVarInt(header
+ pos
, p
->packSize
);
78 if (XzBlock_HasUnpackSize(p
)) pos
+= Xz_WriteVarInt(header
+ pos
, p
->unpackSize
);
79 numFilters
= XzBlock_GetNumFilters(p
);
81 for (i
= 0; i
< numFilters
; i
++)
83 const CXzFilter
*f
= &p
->filters
[i
];
84 pos
+= Xz_WriteVarInt(header
+ pos
, f
->id
);
85 pos
+= Xz_WriteVarInt(header
+ pos
, f
->propsSize
);
86 memcpy(header
+ pos
, f
->props
, f
->propsSize
);
90 while ((pos
& 3) != 0)
93 header
[0] = (Byte
)(pos
>> 2);
94 SetUi32(header
+ pos
, CrcCalc(header
, pos
));
95 return WriteBytes(s
, header
, pos
+ 4);
110 static void XzEncIndex_Construct(CXzEncIndex
*p
)
118 static void XzEncIndex_Init(CXzEncIndex
*p
)
124 static void XzEncIndex_Free(CXzEncIndex
*p
, ISzAllocPtr alloc
)
128 ISzAlloc_Free(alloc
, p
->blocks
);
137 static SRes
XzEncIndex_ReAlloc(CXzEncIndex
*p
, size_t newSize
, ISzAllocPtr alloc
)
139 Byte
*blocks
= (Byte
*)ISzAlloc_Alloc(alloc
, newSize
);
143 memcpy(blocks
, p
->blocks
, p
->size
);
145 ISzAlloc_Free(alloc
, p
->blocks
);
147 p
->allocated
= newSize
;
152 static SRes
XzEncIndex_PreAlloc(CXzEncIndex
*p
, UInt64 numBlocks
, UInt64 unpackSize
, UInt64 totalSize
, ISzAllocPtr alloc
)
157 unsigned pos2
= Xz_WriteVarInt(buf
, totalSize
);
158 pos2
+= Xz_WriteVarInt(buf
+ pos2
, unpackSize
);
159 pos
= numBlocks
* pos2
;
162 if (pos
<= p
->allocated
- p
->size
)
165 UInt64 newSize64
= p
->size
+ pos
;
166 size_t newSize
= (size_t)newSize64
;
167 if (newSize
!= newSize64
)
169 return XzEncIndex_ReAlloc(p
, newSize
, alloc
);
174 static SRes
XzEncIndex_AddIndexRecord(CXzEncIndex
*p
, UInt64 unpackSize
, UInt64 totalSize
, ISzAllocPtr alloc
)
177 unsigned pos
= Xz_WriteVarInt(buf
, totalSize
);
178 pos
+= Xz_WriteVarInt(buf
+ pos
, unpackSize
);
180 if (pos
> p
->allocated
- p
->size
)
182 size_t newSize
= p
->allocated
* 2 + 16 * 2;
183 if (newSize
< p
->size
+ pos
)
185 RINOK(XzEncIndex_ReAlloc(p
, newSize
, alloc
));
187 memcpy(p
->blocks
+ p
->size
, buf
, pos
);
194 static SRes
XzEncIndex_WriteFooter(const CXzEncIndex
*p
, CXzStreamFlags flags
, ISeqOutStream
*s
)
198 UInt32 crc
= CRC_INIT_VAL
;
199 unsigned pos
= 1 + Xz_WriteVarInt(buf
+ 1, p
->numBlocks
);
203 RINOK(WriteBytesUpdateCrc(s
, buf
, pos
, &crc
));
204 RINOK(WriteBytesUpdateCrc(s
, p
->blocks
, p
->size
, &crc
));
205 globalPos
+= p
->size
;
207 pos
= XZ_GET_PAD_SIZE(globalPos
);
213 crc
= CrcUpdate(crc
, buf
+ 4 - pos
, pos
);
214 SetUi32(buf
+ 4, CRC_GET_DIGEST(crc
));
216 SetUi32(buf
+ 8 + 4, (UInt32
)(globalPos
>> 2));
217 buf
[8 + 8] = (Byte
)(flags
>> 8);
218 buf
[8 + 9] = (Byte
)(flags
& 0xFF);
219 SetUi32(buf
+ 8, CrcCalc(buf
+ 8 + 4, 6));
220 buf
[8 + 10] = XZ_FOOTER_SIG_0
;
221 buf
[8 + 11] = XZ_FOOTER_SIG_1
;
223 return WriteBytes(s
, buf
+ 4 - pos
, pos
+ 4 + 12);
228 /* ---------- CSeqCheckInStream ---------- */
233 ISeqInStream
*realStream
;
237 int realStreamFinished
;
241 static void SeqCheckInStream_Init(CSeqCheckInStream
*p
, unsigned checkMode
)
243 p
->limit
= (UInt64
)(Int64
)-1;
245 p
->realStreamFinished
= 0;
246 XzCheck_Init(&p
->check
, checkMode
);
249 static void SeqCheckInStream_GetDigest(CSeqCheckInStream
*p
, Byte
*digest
)
251 XzCheck_Final(&p
->check
, digest
);
254 static SRes
SeqCheckInStream_Read(const ISeqInStream
*pp
, void *data
, size_t *size
)
256 CSeqCheckInStream
*p
= CONTAINER_FROM_VTBL(pp
, CSeqCheckInStream
, vt
);
257 size_t size2
= *size
;
260 if (p
->limit
!= (UInt64
)(Int64
)-1)
262 UInt64 rem
= p
->limit
- p
->processed
;
270 res
= ISeqInStream_Read(p
->realStream
, data
, &size2
);
271 p
->realStreamFinished
= (size2
== 0) ? 1 : 0;
274 memcpy(data
, p
->data
+ (size_t)p
->processed
, size2
);
275 XzCheck_Update(&p
->check
, data
, size2
);
276 p
->processed
+= size2
;
283 /* ---------- CSeqSizeOutStream ---------- */
288 ISeqOutStream
*realStream
;
294 static size_t SeqSizeOutStream_Write(const ISeqOutStream
*pp
, const void *data
, size_t size
)
296 CSeqSizeOutStream
*p
= CONTAINER_FROM_VTBL(pp
, CSeqSizeOutStream
, vt
);
298 size
= ISeqOutStream_Write(p
->realStream
, data
, size
);
301 if (size
> p
->outBufLimit
- (size_t)p
->processed
)
303 memcpy(p
->outBuf
+ (size_t)p
->processed
, data
, size
);
305 p
->processed
+= size
;
310 /* ---------- CSeqInFilter ---------- */
312 #define FILTER_BUF_SIZE (1 << 20)
317 ISeqInStream
*realStream
;
318 IStateCoder StateCoder
;
326 SRes
BraState_SetFromMethod(IStateCoder
*p
, UInt64 id
, int encodeMode
, ISzAllocPtr alloc
);
328 static SRes
SeqInFilter_Init(CSeqInFilter
*p
, const CXzFilter
*props
, ISzAllocPtr alloc
)
332 p
->buf
= (Byte
*)ISzAlloc_Alloc(alloc
, FILTER_BUF_SIZE
);
336 p
->curPos
= p
->endPos
= 0;
337 p
->srcWasFinished
= 0;
338 RINOK(BraState_SetFromMethod(&p
->StateCoder
, props
->id
, 1, alloc
));
339 RINOK(p
->StateCoder
.SetProps(p
->StateCoder
.p
, props
->props
, props
->propsSize
, alloc
));
340 p
->StateCoder
.Init(p
->StateCoder
.p
);
345 static SRes
SeqInFilter_Read(const ISeqInStream
*pp
, void *data
, size_t *size
)
347 CSeqInFilter
*p
= CONTAINER_FROM_VTBL(pp
, CSeqInFilter
, p
);
348 size_t sizeOriginal
= *size
;
349 if (sizeOriginal
== 0)
355 if (!p
->srcWasFinished
&& p
->curPos
== p
->endPos
)
358 p
->endPos
= FILTER_BUF_SIZE
;
359 RINOK(ISeqInStream_Read(p
->realStream
, p
->buf
, &p
->endPos
));
361 p
->srcWasFinished
= 1;
364 SizeT srcLen
= p
->endPos
- p
->curPos
;
367 *size
= sizeOriginal
;
368 res
= p
->StateCoder
.Code2(p
->StateCoder
.p
,
370 p
->buf
+ p
->curPos
, &srcLen
,
371 p
->srcWasFinished
, CODER_FINISH_ANY
,
374 if (*size
!= 0 || srcLen
== 0 || res
!= SZ_OK
)
380 static void SeqInFilter_Construct(CSeqInFilter
*p
)
383 p
->StateCoder
.p
= NULL
;
384 p
->p
.Read
= SeqInFilter_Read
;
387 static void SeqInFilter_Free(CSeqInFilter
*p
, ISzAllocPtr alloc
)
391 p
->StateCoder
.Free(p
->StateCoder
.p
, alloc
);
392 p
->StateCoder
.p
= NULL
;
396 ISzAlloc_Free(alloc
, p
->buf
);
402 /* ---------- CSbEncInStream ---------- */
409 ISeqInStream
*inStream
;
413 static SRes
SbEncInStream_Read(const ISeqInStream
*pp
, void *data
, size_t *size
)
415 CSbEncInStream
*p
= CONTAINER_FROM_VTBL(pp
, CSbEncInStream
, vt
);
416 size_t sizeOriginal
= *size
;
417 if (sizeOriginal
== 0)
422 if (p
->enc
.needRead
&& !p
->enc
.readWasFinished
)
424 size_t processed
= p
->enc
.needReadSizeMax
;
425 RINOK(p
->inStream
->Read(p
->inStream
, p
->enc
.buf
+ p
->enc
.readPos
, &processed
));
426 p
->enc
.readPos
+= processed
;
429 p
->enc
.readWasFinished
= True
;
430 p
->enc
.isFinalFinished
= True
;
432 p
->enc
.needRead
= False
;
435 *size
= sizeOriginal
;
436 RINOK(SbEnc_Read(&p
->enc
, data
, size
));
437 if (*size
!= 0 || !p
->enc
.needRead
)
442 void SbEncInStream_Construct(CSbEncInStream
*p
, ISzAllocPtr alloc
)
444 SbEnc_Construct(&p
->enc
, alloc
);
445 p
->vt
.Read
= SbEncInStream_Read
;
448 SRes
SbEncInStream_Init(CSbEncInStream
*p
)
450 return SbEnc_Init(&p
->enc
);
453 void SbEncInStream_Free(CSbEncInStream
*p
)
462 /* ---------- CXzProps ---------- */
465 void XzFilterProps_Init(CXzFilterProps
*p
)
470 p
->ipDefined
= False
;
473 void XzProps_Init(CXzProps
*p
)
475 p
->checkId
= XZ_CHECK_CRC32
;
476 p
->blockSize
= XZ_PROPS__BLOCK_SIZE__AUTO
;
477 p
->numBlockThreads_Reduced
= -1;
478 p
->numBlockThreads_Max
= -1;
479 p
->numTotalThreads
= -1;
480 p
->reduceSize
= (UInt64
)(Int64
)-1;
481 p
->forceWriteSizesInHeader
= 0;
482 // p->forceWriteSizesInHeader = 1;
484 XzFilterProps_Init(&p
->filterProps
);
485 Lzma2EncProps_Init(&p
->lzma2Props
);
489 static void XzEncProps_Normalize_Fixed(CXzProps
*p
)
492 int t1
, t1n
, t2
, t2r
, t3
;
494 CLzma2EncProps tp
= p
->lzma2Props
;
495 if (tp
.numTotalThreads
<= 0)
496 tp
.numTotalThreads
= p
->numTotalThreads
;
497 Lzma2EncProps_Normalize(&tp
);
498 t1n
= tp
.numTotalThreads
;
501 t1
= p
->lzma2Props
.numTotalThreads
;
502 t2
= p
->numBlockThreads_Max
;
503 t3
= p
->numTotalThreads
;
505 if (t2
> MTCODER__THREADS_MAX
)
506 t2
= MTCODER__THREADS_MAX
;
522 if (t2
> MTCODER__THREADS_MAX
)
523 t2
= MTCODER__THREADS_MAX
;
534 p
->lzma2Props
.numTotalThreads
= t1
;
538 fileSize
= p
->reduceSize
;
540 if ((p
->blockSize
< fileSize
|| fileSize
== (UInt64
)(Int64
)-1))
541 p
->lzma2Props
.lzmaProps
.reduceSize
= p
->blockSize
;
543 Lzma2EncProps_Normalize(&p
->lzma2Props
);
545 t1
= p
->lzma2Props
.numTotalThreads
;
548 if (t2
> 1 && fileSize
!= (UInt64
)(Int64
)-1)
550 UInt64 numBlocks
= fileSize
/ p
->blockSize
;
551 if (numBlocks
* p
->blockSize
!= fileSize
)
553 if (numBlocks
< (unsigned)t2
)
555 t2r
= (unsigned)numBlocks
;
563 p
->numBlockThreads_Max
= t2
;
564 p
->numBlockThreads_Reduced
= t2r
;
565 p
->numTotalThreads
= t3
;
569 static void XzProps_Normalize(CXzProps
*p
)
571 /* we normalize xzProps properties, but we normalize only some of CXzProps::lzma2Props properties.
572 Lzma2Enc_SetProps() will normalize lzma2Props later. */
574 if (p
->blockSize
== XZ_PROPS__BLOCK_SIZE__SOLID
)
576 p
->lzma2Props
.lzmaProps
.reduceSize
= p
->reduceSize
;
577 p
->numBlockThreads_Reduced
= 1;
578 p
->numBlockThreads_Max
= 1;
579 if (p
->lzma2Props
.numTotalThreads
<= 0)
580 p
->lzma2Props
.numTotalThreads
= p
->numTotalThreads
;
585 CLzma2EncProps
*lzma2
= &p
->lzma2Props
;
586 if (p
->blockSize
== LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO
)
589 p
->lzma2Props
.lzmaProps
.reduceSize
= p
->reduceSize
;
591 if (lzma2
->blockSize
== LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID
)
593 // if (xz-auto && lzma2-solid) - we use solid for both
594 p
->blockSize
= XZ_PROPS__BLOCK_SIZE__SOLID
;
595 p
->numBlockThreads_Reduced
= 1;
596 p
->numBlockThreads_Max
= 1;
597 if (p
->lzma2Props
.numTotalThreads
<= 0)
598 p
->lzma2Props
.numTotalThreads
= p
->numTotalThreads
;
602 // if (xz-auto && (lzma2-auto || lzma2-fixed_)
603 // we calculate block size for lzma2 and use that block size for xz, lzma2 uses single-chunk per block
604 CLzma2EncProps tp
= p
->lzma2Props
;
605 if (tp
.numTotalThreads
<= 0)
606 tp
.numTotalThreads
= p
->numTotalThreads
;
608 Lzma2EncProps_Normalize(&tp
);
610 p
->blockSize
= tp
.blockSize
; // fixed or solid
611 p
->numBlockThreads_Reduced
= tp
.numBlockThreads_Reduced
;
612 p
->numBlockThreads_Max
= tp
.numBlockThreads_Max
;
613 if (lzma2
->blockSize
== LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO
)
614 lzma2
->blockSize
= tp
.blockSize
; // fixed or solid, LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID
615 if (lzma2
->lzmaProps
.reduceSize
> tp
.blockSize
&& tp
.blockSize
!= LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID
)
616 lzma2
->lzmaProps
.reduceSize
= tp
.blockSize
;
617 lzma2
->numBlockThreads_Reduced
= 1;
618 lzma2
->numBlockThreads_Max
= 1;
625 // we can use xz::reduceSize or xz::blockSize as base for lzmaProps::reduceSize
627 p
->lzma2Props
.lzmaProps
.reduceSize
= p
->reduceSize
;
629 UInt64 r
= p
->reduceSize
;
630 if (r
> p
->blockSize
|| r
== (UInt64
)(Int64
)-1)
632 lzma2
->lzmaProps
.reduceSize
= r
;
634 if (lzma2
->blockSize
== LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO
)
635 lzma2
->blockSize
= LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID
;
636 else if (lzma2
->blockSize
> p
->blockSize
&& lzma2
->blockSize
!= LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID
)
637 lzma2
->blockSize
= p
->blockSize
;
639 XzEncProps_Normalize_Fixed(p
);
645 /* ---------- CLzma2WithFilters ---------- */
649 CLzma2EncHandle lzma2
;
658 static void Lzma2WithFilters_Construct(CLzma2WithFilters
*p
)
661 SeqInFilter_Construct(&p
->filter
);
664 SbEncInStream_Construct(&p
->sb
, alloc
);
669 static SRes
Lzma2WithFilters_Create(CLzma2WithFilters
*p
, ISzAllocPtr alloc
, ISzAllocPtr bigAlloc
)
673 p
->lzma2
= Lzma2Enc_Create(alloc
, bigAlloc
);
681 static void Lzma2WithFilters_Free(CLzma2WithFilters
*p
, ISzAllocPtr alloc
)
684 SbEncInStream_Free(&p
->sb
);
687 SeqInFilter_Free(&p
->filter
, alloc
);
690 Lzma2Enc_Destroy(p
->lzma2
);
704 static SRes
Xz_CompressBlock(
705 CLzma2WithFilters
*lzmaf
,
707 ISeqOutStream
*outStream
,
709 Byte
*outBufData
, size_t outBufDataLimit
,
711 ISeqInStream
*inStream
,
712 // UInt64 expectedSize,
713 const Byte
*inBuf
, // used if (!inStream)
714 size_t inBufSize
, // used if (!inStream), it's block size, props->blockSize is ignored
716 const CXzProps
*props
,
717 ICompressProgress
*progress
,
718 int *inStreamFinished
, /* only for inStream version */
719 CXzEncBlockInfo
*blockSizes
,
721 ISzAllocPtr allocBig
)
723 CSeqCheckInStream checkInStream
;
724 CSeqSizeOutStream seqSizeOutStream
;
726 unsigned filterIndex
= 0;
727 CXzFilter
*filter
= NULL
;
728 const CXzFilterProps
*fp
= &props
->filterProps
;
732 *inStreamFinished
= False
;
734 RINOK(Lzma2WithFilters_Create(lzmaf
, alloc
, allocBig
));
736 RINOK(Lzma2Enc_SetProps(lzmaf
->lzma2
, &props
->lzma2Props
));
738 XzBlock_ClearFlags(&block
);
739 XzBlock_SetNumFilters(&block
, 1 + (fp
? 1 : 0));
743 filter
= &block
.filters
[filterIndex
++];
745 filter
->propsSize
= 0;
747 if (fp
->id
== XZ_ID_Delta
)
749 filter
->props
[0] = (Byte
)(fp
->delta
- 1);
750 filter
->propsSize
= 1;
752 else if (fp
->ipDefined
)
754 SetUi32(filter
->props
, fp
->ip
);
755 filter
->propsSize
= 4;
760 CXzFilter
*f
= &block
.filters
[filterIndex
++];
763 f
->props
[0] = Lzma2Enc_WriteProperties(lzmaf
->lzma2
);
766 seqSizeOutStream
.vt
.Write
= SeqSizeOutStream_Write
;
767 seqSizeOutStream
.realStream
= outStream
;
768 seqSizeOutStream
.outBuf
= outBufData
;
769 seqSizeOutStream
.outBufLimit
= outBufDataLimit
;
770 seqSizeOutStream
.processed
= 0;
773 if (expectedSize != (UInt64)(Int64)-1)
775 block.unpackSize = expectedSize;
776 if (props->blockSize != (UInt64)(Int64)-1)
777 if (expectedSize > props->blockSize)
778 block.unpackSize = props->blockSize;
779 XzBlock_SetHasUnpackSize(&block);
785 RINOK(XzBlock_WriteHeader(&block
, &seqSizeOutStream
.vt
));
788 checkInStream
.vt
.Read
= SeqCheckInStream_Read
;
789 SeqCheckInStream_Init(&checkInStream
, props
->checkId
);
791 checkInStream
.realStream
= inStream
;
792 checkInStream
.data
= inBuf
;
793 checkInStream
.limit
= props
->blockSize
;
795 checkInStream
.limit
= inBufSize
;
800 if (fp
->id
== XZ_ID_Subblock
)
802 lzmaf
->sb
.inStream
= &checkInStream
.vt
;
803 RINOK(SbEncInStream_Init(&lzmaf
->sb
));
808 lzmaf
->filter
.realStream
= &checkInStream
.vt
;
809 RINOK(SeqInFilter_Init(&lzmaf
->filter
, filter
, alloc
));
817 BoolInt useStream
= (fp
|| inStream
);
822 XzCheck_Update(&checkInStream
.check
, inBuf
, inBufSize
);
823 checkInStream
.processed
= inBufSize
;
828 outBuf
= seqSizeOutStream
.outBuf
; // + (size_t)seqSizeOutStream.processed;
829 outSize
= seqSizeOutStream
.outBufLimit
; // - (size_t)seqSizeOutStream.processed;
832 res
= Lzma2Enc_Encode2(lzmaf
->lzma2
,
833 outBuf
? NULL
: &seqSizeOutStream
.vt
,
835 outBuf
? &outSize
: NULL
,
841 (fp
->id
== XZ_ID_Subblock
) ? &lzmaf
->sb
.vt
:
844 &checkInStream
.vt
) : NULL
,
846 useStream
? NULL
: inBuf
,
847 useStream
? 0 : inBufSize
,
852 seqSizeOutStream
.processed
+= outSize
;
855 blockSizes
->unpackSize
= checkInStream
.processed
;
859 unsigned padSize
= XZ_GET_PAD_SIZE(seqSizeOutStream
.processed
);
860 UInt64 packSize
= seqSizeOutStream
.processed
;
867 SeqCheckInStream_GetDigest(&checkInStream
, buf
+ 4);
868 RINOK(WriteBytes(&seqSizeOutStream
.vt
, buf
+ (4 - padSize
), padSize
+ XzFlags_GetCheckSize((CXzStreamFlags
)props
->checkId
)));
870 blockSizes
->totalSize
= seqSizeOutStream
.processed
- padSize
;
874 seqSizeOutStream
.outBuf
= outBufHeader
;
875 seqSizeOutStream
.outBufLimit
= XZ_BLOCK_HEADER_SIZE_MAX
;
876 seqSizeOutStream
.processed
= 0;
878 block
.unpackSize
= blockSizes
->unpackSize
;
879 XzBlock_SetHasUnpackSize(&block
);
881 block
.packSize
= packSize
;
882 XzBlock_SetHasPackSize(&block
);
884 RINOK(XzBlock_WriteHeader(&block
, &seqSizeOutStream
.vt
));
886 blockSizes
->headerSize
= (size_t)seqSizeOutStream
.processed
;
887 blockSizes
->totalSize
+= seqSizeOutStream
.processed
;
892 *inStreamFinished
= checkInStream
.realStreamFinished
;
895 *inStreamFinished
= False
;
896 if (checkInStream
.processed
!= inBufSize
)
897 return SZ_ERROR_FAIL
;
907 ICompressProgress vt
;
908 ICompressProgress
*progress
;
911 } CCompressProgress_XzEncOffset
;
914 static SRes
CompressProgress_XzEncOffset_Progress(const ICompressProgress
*pp
, UInt64 inSize
, UInt64 outSize
)
916 const CCompressProgress_XzEncOffset
*p
= CONTAINER_FROM_VTBL(pp
, CCompressProgress_XzEncOffset
, vt
);
917 inSize
+= p
->inOffset
;
918 outSize
+= p
->outOffset
;
919 return ICompressProgress_Progress(p
->progress
, inSize
, outSize
);
928 ISzAllocPtr allocBig
;
931 UInt64 expectedDataSize
;
935 CLzma2WithFilters lzmaf_Items
[MTCODER__THREADS_MAX
];
937 size_t outBufSize
; /* size of allocated outBufs[i] */
938 Byte
*outBufs
[MTCODER__BLOCKS_MAX
];
942 ISeqOutStream
*outStream
;
943 BoolInt mtCoder_WasConstructed
;
945 CXzEncBlockInfo EncBlocks
[MTCODER__BLOCKS_MAX
];
951 static void XzEnc_Construct(CXzEnc
*p
)
955 XzEncIndex_Construct(&p
->xzIndex
);
957 for (i
= 0; i
< MTCODER__THREADS_MAX
; i
++)
958 Lzma2WithFilters_Construct(&p
->lzmaf_Items
[i
]);
961 p
->mtCoder_WasConstructed
= False
;
963 for (i
= 0; i
< MTCODER__BLOCKS_MAX
; i
++)
964 p
->outBufs
[i
] = NULL
;
971 static void XzEnc_FreeOutBufs(CXzEnc
*p
)
974 for (i
= 0; i
< MTCODER__BLOCKS_MAX
; i
++)
977 ISzAlloc_Free(p
->alloc
, p
->outBufs
[i
]);
978 p
->outBufs
[i
] = NULL
;
984 static void XzEnc_Free(CXzEnc
*p
, ISzAllocPtr alloc
)
988 XzEncIndex_Free(&p
->xzIndex
, alloc
);
990 for (i
= 0; i
< MTCODER__THREADS_MAX
; i
++)
991 Lzma2WithFilters_Free(&p
->lzmaf_Items
[i
], alloc
);
994 if (p
->mtCoder_WasConstructed
)
996 MtCoder_Destruct(&p
->mtCoder
);
997 p
->mtCoder_WasConstructed
= False
;
999 XzEnc_FreeOutBufs(p
);
1004 CXzEncHandle
XzEnc_Create(ISzAllocPtr alloc
, ISzAllocPtr allocBig
)
1006 CXzEnc
*p
= (CXzEnc
*)ISzAlloc_Alloc(alloc
, sizeof(CXzEnc
));
1010 XzProps_Init(&p
->xzProps
);
1011 XzProps_Normalize(&p
->xzProps
);
1012 p
->expectedDataSize
= (UInt64
)(Int64
)-1;
1014 p
->allocBig
= allocBig
;
1019 void XzEnc_Destroy(CXzEncHandle pp
)
1021 CXzEnc
*p
= (CXzEnc
*)pp
;
1022 XzEnc_Free(p
, p
->alloc
);
1023 ISzAlloc_Free(p
->alloc
, p
);
1027 SRes
XzEnc_SetProps(CXzEncHandle pp
, const CXzProps
*props
)
1029 CXzEnc
*p
= (CXzEnc
*)pp
;
1030 p
->xzProps
= *props
;
1031 XzProps_Normalize(&p
->xzProps
);
1036 void XzEnc_SetDataSize(CXzEncHandle pp
, UInt64 expectedDataSiize
)
1038 CXzEnc
*p
= (CXzEnc
*)pp
;
1039 p
->expectedDataSize
= expectedDataSiize
;
1047 static SRes
XzEnc_MtCallback_Code(void *pp
, unsigned coderIndex
, unsigned outBufIndex
,
1048 const Byte
*src
, size_t srcSize
, int finished
)
1050 CXzEnc
*me
= (CXzEnc
*)pp
;
1052 CMtProgressThunk progressThunk
;
1054 Byte
*dest
= me
->outBufs
[outBufIndex
];
1056 UNUSED_VAR(finished
)
1059 CXzEncBlockInfo
*bInfo
= &me
->EncBlocks
[outBufIndex
];
1060 bInfo
->totalSize
= 0;
1061 bInfo
->unpackSize
= 0;
1062 bInfo
->headerSize
= 0;
1067 dest
= (Byte
*)ISzAlloc_Alloc(me
->alloc
, me
->outBufSize
);
1069 return SZ_ERROR_MEM
;
1070 me
->outBufs
[outBufIndex
] = dest
;
1073 MtProgressThunk_CreateVTable(&progressThunk
);
1074 progressThunk
.mtProgress
= &me
->mtCoder
.mtProgress
;
1075 MtProgressThunk_Init(&progressThunk
);
1078 CXzEncBlockInfo blockSizes
;
1079 int inStreamFinished
;
1081 res
= Xz_CompressBlock(
1082 &me
->lzmaf_Items
[coderIndex
],
1086 dest
+ XZ_BLOCK_HEADER_SIZE_MAX
, me
->outBufSize
- XZ_BLOCK_HEADER_SIZE_MAX
,
1089 // srcSize, // expectedSize
1100 me
->EncBlocks
[outBufIndex
] = blockSizes
;
1107 static SRes
XzEnc_MtCallback_Write(void *pp
, unsigned outBufIndex
)
1109 CXzEnc
*me
= (CXzEnc
*)pp
;
1111 const CXzEncBlockInfo
*bInfo
= &me
->EncBlocks
[outBufIndex
];
1112 const Byte
*data
= me
->outBufs
[outBufIndex
];
1114 RINOK(WriteBytes(me
->outStream
, data
, bInfo
->headerSize
));
1117 UInt64 totalPackFull
= bInfo
->totalSize
+ XZ_GET_PAD_SIZE(bInfo
->totalSize
);
1118 RINOK(WriteBytes(me
->outStream
, data
+ XZ_BLOCK_HEADER_SIZE_MAX
, (size_t)totalPackFull
- bInfo
->headerSize
));
1121 return XzEncIndex_AddIndexRecord(&me
->xzIndex
, bInfo
->unpackSize
, bInfo
->totalSize
, me
->alloc
);
1128 SRes
XzEnc_Encode(CXzEncHandle pp
, ISeqOutStream
*outStream
, ISeqInStream
*inStream
, ICompressProgress
*progress
)
1130 CXzEnc
*p
= (CXzEnc
*)pp
;
1132 const CXzProps
*props
= &p
->xzProps
;
1134 XzEncIndex_Init(&p
->xzIndex
);
1136 UInt64 numBlocks
= 1;
1137 UInt64 blockSize
= props
->blockSize
;
1139 if (blockSize
!= XZ_PROPS__BLOCK_SIZE__SOLID
1140 && props
->reduceSize
!= (UInt64
)(Int64
)-1)
1142 numBlocks
= props
->reduceSize
/ blockSize
;
1143 if (numBlocks
* blockSize
!= props
->reduceSize
)
1147 blockSize
= (UInt64
)1 << 62;
1149 RINOK(XzEncIndex_PreAlloc(&p
->xzIndex
, numBlocks
, blockSize
, XZ_GET_ESTIMATED_BLOCK_TOTAL_PACK_SIZE(blockSize
), p
->alloc
));
1152 RINOK(Xz_WriteHeader((CXzStreamFlags
)props
->checkId
, outStream
));
1156 if (props
->numBlockThreads_Reduced
> 1)
1158 IMtCoderCallback2 vt
;
1160 if (!p
->mtCoder_WasConstructed
)
1162 p
->mtCoder_WasConstructed
= True
;
1163 MtCoder_Construct(&p
->mtCoder
);
1166 vt
.Code
= XzEnc_MtCallback_Code
;
1167 vt
.Write
= XzEnc_MtCallback_Write
;
1169 p
->checkType
= props
->checkId
;
1170 p
->xzProps
= *props
;
1172 p
->outStream
= outStream
;
1174 p
->mtCoder
.allocBig
= p
->allocBig
;
1175 p
->mtCoder
.progress
= progress
;
1176 p
->mtCoder
.inStream
= inStream
;
1177 p
->mtCoder
.inData
= NULL
;
1178 p
->mtCoder
.inDataSize
= 0;
1179 p
->mtCoder
.mtCallback
= &vt
;
1180 p
->mtCoder
.mtCallbackObject
= p
;
1182 if ( props
->blockSize
== XZ_PROPS__BLOCK_SIZE__SOLID
1183 || props
->blockSize
== XZ_PROPS__BLOCK_SIZE__AUTO
)
1184 return SZ_ERROR_FAIL
;
1186 p
->mtCoder
.blockSize
= (size_t)props
->blockSize
;
1187 if (p
->mtCoder
.blockSize
!= props
->blockSize
)
1188 return SZ_ERROR_PARAM
; /* SZ_ERROR_MEM */
1191 size_t destBlockSize
= XZ_BLOCK_HEADER_SIZE_MAX
+ XZ_GET_MAX_BLOCK_PACK_SIZE(p
->mtCoder
.blockSize
);
1192 if (destBlockSize
< p
->mtCoder
.blockSize
)
1193 return SZ_ERROR_PARAM
;
1194 if (p
->outBufSize
!= destBlockSize
)
1195 XzEnc_FreeOutBufs(p
);
1196 p
->outBufSize
= destBlockSize
;
1199 p
->mtCoder
.numThreadsMax
= props
->numBlockThreads_Max
;
1200 p
->mtCoder
.expectedDataSize
= p
->expectedDataSize
;
1202 RINOK(MtCoder_Code(&p
->mtCoder
));
1207 int writeStartSizes
;
1208 CCompressProgress_XzEncOffset progress2
;
1209 Byte
*bufData
= NULL
;
1212 progress2
.vt
.Progress
= CompressProgress_XzEncOffset_Progress
;
1213 progress2
.inOffset
= 0;
1214 progress2
.outOffset
= 0;
1215 progress2
.progress
= progress
;
1217 writeStartSizes
= 0;
1219 if (props
->blockSize
!= XZ_PROPS__BLOCK_SIZE__SOLID
)
1221 writeStartSizes
= (props
->forceWriteSizesInHeader
> 0);
1223 if (writeStartSizes
)
1226 size_t t
= (size_t)props
->blockSize
;
1227 if (t
!= props
->blockSize
)
1228 return SZ_ERROR_PARAM
;
1229 t
= XZ_GET_MAX_BLOCK_PACK_SIZE(t
);
1230 if (t
< props
->blockSize
)
1231 return SZ_ERROR_PARAM
;
1232 t2
= XZ_BLOCK_HEADER_SIZE_MAX
+ t
;
1233 if (!p
->outBufs
[0] || t2
!= p
->outBufSize
)
1235 XzEnc_FreeOutBufs(p
);
1236 p
->outBufs
[0] = (Byte
*)ISzAlloc_Alloc(p
->alloc
, t2
);
1238 return SZ_ERROR_MEM
;
1241 bufData
= p
->outBufs
[0] + XZ_BLOCK_HEADER_SIZE_MAX
;
1248 CXzEncBlockInfo blockSizes
;
1249 int inStreamFinished
;
1252 UInt64 rem = (UInt64)(Int64)-1;
1253 if (props->reduceSize != (UInt64)(Int64)-1
1254 && props->reduceSize >= progress2.inOffset)
1255 rem = props->reduceSize - progress2.inOffset;
1258 blockSizes
.headerSize
= 0; // for GCC
1260 RINOK(Xz_CompressBlock(
1263 writeStartSizes
? NULL
: outStream
,
1264 writeStartSizes
? p
->outBufs
[0] : NULL
,
1272 progress
? &progress2
.vt
: NULL
,
1279 UInt64 totalPackFull
= blockSizes
.totalSize
+ XZ_GET_PAD_SIZE(blockSizes
.totalSize
);
1281 if (writeStartSizes
)
1283 RINOK(WriteBytes(outStream
, p
->outBufs
[0], blockSizes
.headerSize
));
1284 RINOK(WriteBytes(outStream
, bufData
, (size_t)totalPackFull
- blockSizes
.headerSize
));
1287 RINOK(XzEncIndex_AddIndexRecord(&p
->xzIndex
, blockSizes
.unpackSize
, blockSizes
.totalSize
, p
->alloc
));
1289 progress2
.inOffset
+= blockSizes
.unpackSize
;
1290 progress2
.outOffset
+= totalPackFull
;
1293 if (inStreamFinished
)
1298 return XzEncIndex_WriteFooter(&p
->xzIndex
, (CXzStreamFlags
)props
->checkId
, outStream
);
1304 SRes
Xz_Encode(ISeqOutStream
*outStream
, ISeqInStream
*inStream
,
1305 const CXzProps
*props
, ICompressProgress
*progress
)
1308 CXzEncHandle xz
= XzEnc_Create(&g_Alloc
, &g_BigAlloc
);
1310 return SZ_ERROR_MEM
;
1311 res
= XzEnc_SetProps(xz
, props
);
1313 res
= XzEnc_Encode(xz
, outStream
, inStream
, progress
);
1319 SRes
Xz_EncodeEmpty(ISeqOutStream
*outStream
)
1322 CXzEncIndex xzIndex
;
1323 XzEncIndex_Construct(&xzIndex
);
1324 res
= Xz_WriteHeader((CXzStreamFlags
)0, outStream
);
1326 res
= XzEncIndex_WriteFooter(&xzIndex
, (CXzStreamFlags
)0, outStream
);
1327 XzEncIndex_Free(&xzIndex
, NULL
); // g_Alloc