1 /* trees.c -- output deflated data using Huffman coding
2 * Copyright (C) 1995-2009 Jean-loup Gailly
3 * detect_data_type() function provided freely by Cosmin Truta, 2006
4 * For conditions of distribution and use, see copyright notice in zlib.h
10 * The "deflation" process uses several Huffman trees. The more
11 * common source values are represented by shorter bit sequences.
13 * Each code tree is stored in a compressed form which is itself
14 * a Huffman encoding of the lengths of all the code strings (in
15 * ascending order by source values). The actual code strings are
16 * reconstructed from the lengths in the inflate process, as described
17 * in the deflate specification.
21 * Deutsch, L.P.,"'Deflate' Compressed Data Format Specification".
22 * Available in ftp.uu.net:/pub/archiving/zip/doc/deflate-1.1.doc
25 * Data Compression: Methods and Theory, pp. 49-50.
26 * Computer Science Press, 1988. ISBN 0-7167-8156-5.
30 * Addison-Wesley, 1983. ISBN 0-201-06672-6.
35 /* #define GEN_TREES_H */
43 /* ===========================================================================
48 /* Bit length codes must not exceed MAX_BL_BITS bits */
51 /* end of block literal code */
54 /* repeat previous bit length 3-6 times (2 bits of repeat count) */
57 /* repeat a zero length 3-10 times (3 bits of repeat count) */
59 #define REPZ_11_138 18
60 /* repeat a zero length 11-138 times (7 bits of repeat count) */
62 local
const int extra_lbits
[LENGTH_CODES
] /* extra bits for each length code */
63 = {0,0,0,0,0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,4,4,4,4,5,5,5,5,0};
65 local
const int extra_dbits
[D_CODES
] /* extra bits for each distance code */
66 = {0,0,0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13,13};
68 local
const int extra_blbits
[BL_CODES
]/* extra bits for each bit length code */
69 = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,3,7};
71 local
const uch bl_order
[BL_CODES
]
72 = {16,17,18,0,8,7,9,6,10,5,11,4,12,3,13,2,14,1,15};
73 /* The lengths of the bit length codes are sent in order of decreasing
74 * probability, to avoid transmitting the lengths for unused bit length codes.
77 #define Buf_size (8 * 2*sizeof(char))
78 /* Number of bits used within bi_buf. (bi_buf might be implemented on
79 * more than 16 bits on some systems.)
82 /* ===========================================================================
83 * Local data. These are initialized only once.
86 #define DIST_CODE_LEN 512 /* see definition of array dist_code below */
88 #if defined(GEN_TREES_H) || !defined(STDC)
89 /* non ANSI compilers may not accept trees.h */
91 local ct_data static_ltree
[L_CODES
+2];
92 /* The static literal tree. Since the bit lengths are imposed, there is no
93 * need for the L_CODES extra codes used during heap construction. However
94 * The codes 286 and 287 are needed to build a canonical tree (see _tr_init
98 local ct_data static_dtree
[D_CODES
];
99 /* The static distance tree. (Actually a trivial tree since all codes use
103 uch _dist_code
[DIST_CODE_LEN
];
104 /* Distance codes. The first 256 values correspond to the distances
105 * 3 .. 258, the last 256 values correspond to the top 8 bits of
106 * the 15 bit distances.
109 uch _length_code
[MAX_MATCH
-MIN_MATCH
+1];
110 /* length code for each normalized match length (0 == MIN_MATCH) */
112 local
int base_length
[LENGTH_CODES
];
113 /* First normalized length for each code (0 = MIN_MATCH) */
115 local
int base_dist
[D_CODES
];
116 /* First normalized distance for each code (0 = distance of 1) */
120 #endif /* GEN_TREES_H */
122 struct static_tree_desc_s
{
123 const ct_data
*static_tree
; /* static tree or NULL */
124 const intf
*extra_bits
; /* extra bits for each code or NULL */
125 int extra_base
; /* base index for extra_bits */
126 int elems
; /* max number of elements in the tree */
127 int max_length
; /* max bit length for the codes */
130 local static_tree_desc static_l_desc
=
131 {static_ltree
, extra_lbits
, LITERALS
+1, L_CODES
, MAX_BITS
};
133 local static_tree_desc static_d_desc
=
134 {static_dtree
, extra_dbits
, 0, D_CODES
, MAX_BITS
};
136 local static_tree_desc static_bl_desc
=
137 {(const ct_data
*)0, extra_blbits
, 0, BL_CODES
, MAX_BL_BITS
};
139 /* ===========================================================================
140 * Local (static) routines in this file.
143 local
void tr_static_init
OF((void));
144 local
void init_block
OF((deflate_state
*s
));
145 local
void pqdownheap
OF((deflate_state
*s
, ct_data
*tree
, int k
));
146 local
void gen_bitlen
OF((deflate_state
*s
, tree_desc
*desc
));
147 local
void gen_codes
OF((ct_data
*tree
, int max_code
, ushf
*bl_count
));
148 local
void build_tree
OF((deflate_state
*s
, tree_desc
*desc
));
149 local
void scan_tree
OF((deflate_state
*s
, ct_data
*tree
, int max_code
));
150 local
void send_tree
OF((deflate_state
*s
, ct_data
*tree
, int max_code
));
151 local
int build_bl_tree
OF((deflate_state
*s
));
152 local
void send_all_trees
OF((deflate_state
*s
, int lcodes
, int dcodes
,
154 local
void compress_block
OF((deflate_state
*s
, ct_data
*ltree
,
156 local
int detect_data_type
OF((deflate_state
*s
));
157 local
unsigned bi_reverse
OF((unsigned value
, int length
));
158 local
void bi_windup
OF((deflate_state
*s
));
159 local
void bi_flush
OF((deflate_state
*s
));
160 local
void copy_block
OF((deflate_state
*s
, charf
*buf
, unsigned len
,
164 local
void gen_trees_header
OF((void));
168 # define send_code(s, c, tree) send_bits(s, tree[c].Code, tree[c].Len)
169 /* Send a code of the given tree. c and tree must not have side effects */
172 # define send_code(s, c, tree) \
173 { if (z_verbose>2) fprintf(stderr,"\ncd %3d ",(c)); \
174 send_bits(s, tree[c].Code, tree[c].Len); }
177 /* ===========================================================================
178 * Output a short LSB first on the stream.
179 * IN assertion: there is enough room in pendingBuf.
181 #define put_short(s, w) { \
182 put_byte(s, (uch)((w) & 0xff)); \
183 put_byte(s, (uch)((ush)(w) >> 8)); \
186 /* ===========================================================================
187 * Send a value on a given number of bits.
188 * IN assertion: length <= 16 and value fits in length bits.
191 local
void send_bits
OF((deflate_state
*s
, int value
, int length
));
193 local
void send_bits(s
, value
, length
)
195 int value
; /* value to send */
196 int length
; /* number of bits */
198 Tracevv((stderr
," l %2d v %4x ", length
, value
));
199 Assert(length
> 0 && length
<= 15, "invalid length");
200 s
->bits_sent
+= (ulg
)length
;
202 /* If not enough room in bi_buf, use (valid) bits from bi_buf and
203 * (16 - bi_valid) bits from value, leaving (width - (16-bi_valid))
204 * unused bits in value.
206 if (s
->bi_valid
> (int)Buf_size
- length
) {
207 s
->bi_buf
|= (ush
)value
<< s
->bi_valid
;
208 put_short(s
, s
->bi_buf
);
209 s
->bi_buf
= (ush
)value
>> (Buf_size
- s
->bi_valid
);
210 s
->bi_valid
+= length
- Buf_size
;
212 s
->bi_buf
|= (ush
)value
<< s
->bi_valid
;
213 s
->bi_valid
+= length
;
218 #define send_bits(s, value, length) \
220 if (s->bi_valid > (int)Buf_size - len) {\
222 s->bi_buf |= (ush)val << s->bi_valid;\
223 put_short(s, s->bi_buf);\
224 s->bi_buf = (ush)val >> (Buf_size - s->bi_valid);\
225 s->bi_valid += len - Buf_size;\
227 s->bi_buf |= (ush)(value) << s->bi_valid;\
234 /* the arguments must not have side effects */
236 /* ===========================================================================
237 * Initialize the various 'constant' tables.
239 local
void tr_static_init()
241 #if defined(GEN_TREES_H) || !defined(STDC)
242 static int static_init_done
= 0;
243 int n
; /* iterates over tree elements */
244 int bits
; /* bit counter */
245 int length
; /* length value */
246 int code
; /* code value */
247 int dist
; /* distance index */
248 ush bl_count
[MAX_BITS
+1];
249 /* number of codes at each bit length for an optimal tree */
251 if (static_init_done
) return;
253 /* For some embedded targets, global variables are not initialized: */
254 #ifdef NO_INIT_GLOBAL_POINTERS
255 static_l_desc
.static_tree
= static_ltree
;
256 static_l_desc
.extra_bits
= extra_lbits
;
257 static_d_desc
.static_tree
= static_dtree
;
258 static_d_desc
.extra_bits
= extra_dbits
;
259 static_bl_desc
.extra_bits
= extra_blbits
;
262 /* Initialize the mapping length (0..255) -> length code (0..28) */
264 for (code
= 0; code
< LENGTH_CODES
-1; code
++) {
265 base_length
[code
] = length
;
266 for (n
= 0; n
< (1<<extra_lbits
[code
]); n
++) {
267 _length_code
[length
++] = (uch
)code
;
270 Assert (length
== 256, "tr_static_init: length != 256");
271 /* Note that the length 255 (match length 258) can be represented
272 * in two different ways: code 284 + 5 bits or code 285, so we
273 * overwrite length_code[255] to use the best encoding:
275 _length_code
[length
-1] = (uch
)code
;
277 /* Initialize the mapping dist (0..32K) -> dist code (0..29) */
279 for (code
= 0 ; code
< 16; code
++) {
280 base_dist
[code
] = dist
;
281 for (n
= 0; n
< (1<<extra_dbits
[code
]); n
++) {
282 _dist_code
[dist
++] = (uch
)code
;
285 Assert (dist
== 256, "tr_static_init: dist != 256");
286 dist
>>= 7; /* from now on, all distances are divided by 128 */
287 for ( ; code
< D_CODES
; code
++) {
288 base_dist
[code
] = dist
<< 7;
289 for (n
= 0; n
< (1<<(extra_dbits
[code
]-7)); n
++) {
290 _dist_code
[256 + dist
++] = (uch
)code
;
293 Assert (dist
== 256, "tr_static_init: 256+dist != 512");
295 /* Construct the codes of the static literal tree */
296 for (bits
= 0; bits
<= MAX_BITS
; bits
++) bl_count
[bits
] = 0;
298 while (n
<= 143) static_ltree
[n
++].Len
= 8, bl_count
[8]++;
299 while (n
<= 255) static_ltree
[n
++].Len
= 9, bl_count
[9]++;
300 while (n
<= 279) static_ltree
[n
++].Len
= 7, bl_count
[7]++;
301 while (n
<= 287) static_ltree
[n
++].Len
= 8, bl_count
[8]++;
302 /* Codes 286 and 287 do not exist, but we must include them in the
303 * tree construction to get a canonical Huffman tree (longest code
306 gen_codes((ct_data
*)static_ltree
, L_CODES
+1, bl_count
);
308 /* The static distance tree is trivial: */
309 for (n
= 0; n
< D_CODES
; n
++) {
310 static_dtree
[n
].Len
= 5;
311 static_dtree
[n
].Code
= bi_reverse((unsigned)n
, 5);
313 static_init_done
= 1;
318 #endif /* defined(GEN_TREES_H) || !defined(STDC) */
321 /* ===========================================================================
322 * Genererate the file trees.h describing the static trees.
329 # define SEPARATOR(i, last, width) \
330 ((i) == (last)? "\n};\n\n" : \
331 ((i) % (width) == (width)-1 ? ",\n" : ", "))
333 void gen_trees_header()
335 FILE *header
= fopen("trees.h", "w");
338 Assert (header
!= NULL
, "Can't open trees.h");
340 "/* header created automatically with -DGEN_TREES_H */\n\n");
342 fprintf(header
, "local const ct_data static_ltree[L_CODES+2] = {\n");
343 for (i
= 0; i
< L_CODES
+2; i
++) {
344 fprintf(header
, "{{%3u},{%3u}}%s", static_ltree
[i
].Code
,
345 static_ltree
[i
].Len
, SEPARATOR(i
, L_CODES
+1, 5));
348 fprintf(header
, "local const ct_data static_dtree[D_CODES] = {\n");
349 for (i
= 0; i
< D_CODES
; i
++) {
350 fprintf(header
, "{{%2u},{%2u}}%s", static_dtree
[i
].Code
,
351 static_dtree
[i
].Len
, SEPARATOR(i
, D_CODES
-1, 5));
354 fprintf(header
, "const uch _dist_code[DIST_CODE_LEN] = {\n");
355 for (i
= 0; i
< DIST_CODE_LEN
; i
++) {
356 fprintf(header
, "%2u%s", _dist_code
[i
],
357 SEPARATOR(i
, DIST_CODE_LEN
-1, 20));
360 fprintf(header
, "const uch _length_code[MAX_MATCH-MIN_MATCH+1]= {\n");
361 for (i
= 0; i
< MAX_MATCH
-MIN_MATCH
+1; i
++) {
362 fprintf(header
, "%2u%s", _length_code
[i
],
363 SEPARATOR(i
, MAX_MATCH
-MIN_MATCH
, 20));
366 fprintf(header
, "local const int base_length[LENGTH_CODES] = {\n");
367 for (i
= 0; i
< LENGTH_CODES
; i
++) {
368 fprintf(header
, "%1u%s", base_length
[i
],
369 SEPARATOR(i
, LENGTH_CODES
-1, 20));
372 fprintf(header
, "local const int base_dist[D_CODES] = {\n");
373 for (i
= 0; i
< D_CODES
; i
++) {
374 fprintf(header
, "%5u%s", base_dist
[i
],
375 SEPARATOR(i
, D_CODES
-1, 10));
380 #endif /* GEN_TREES_H */
382 /* ===========================================================================
383 * Initialize the tree data structures for a new zlib stream.
390 s
->l_desc
.dyn_tree
= s
->dyn_ltree
;
391 s
->l_desc
.stat_desc
= &static_l_desc
;
393 s
->d_desc
.dyn_tree
= s
->dyn_dtree
;
394 s
->d_desc
.stat_desc
= &static_d_desc
;
396 s
->bl_desc
.dyn_tree
= s
->bl_tree
;
397 s
->bl_desc
.stat_desc
= &static_bl_desc
;
401 s
->last_eob_len
= 8; /* enough lookahead for inflate */
403 s
->compressed_len
= 0L;
407 /* Initialize the first block of the first file: */
411 /* ===========================================================================
412 * Initialize a new block.
414 local
void init_block(s
)
417 int n
; /* iterates over tree elements */
419 /* Initialize the trees. */
420 for (n
= 0; n
< L_CODES
; n
++) s
->dyn_ltree
[n
].Freq
= 0;
421 for (n
= 0; n
< D_CODES
; n
++) s
->dyn_dtree
[n
].Freq
= 0;
422 for (n
= 0; n
< BL_CODES
; n
++) s
->bl_tree
[n
].Freq
= 0;
424 s
->dyn_ltree
[END_BLOCK
].Freq
= 1;
425 s
->opt_len
= s
->static_len
= 0L;
426 s
->last_lit
= s
->matches
= 0;
430 /* Index within the heap array of least frequent node in the Huffman tree */
433 /* ===========================================================================
434 * Remove the smallest element from the heap and recreate the heap with
435 * one less element. Updates heap and heap_len.
437 #define pqremove(s, tree, top) \
439 top = s->heap[SMALLEST]; \
440 s->heap[SMALLEST] = s->heap[s->heap_len--]; \
441 pqdownheap(s, tree, SMALLEST); \
444 /* ===========================================================================
445 * Compares to subtrees, using the tree depth as tie breaker when
446 * the subtrees have equal frequency. This minimizes the worst case length.
448 #define smaller(tree, n, m, depth) \
449 (tree[n].Freq < tree[m].Freq || \
450 (tree[n].Freq == tree[m].Freq && depth[n] <= depth[m]))
452 /* ===========================================================================
453 * Restore the heap property by moving down the tree starting at node k,
454 * exchanging a node with the smallest of its two sons if necessary, stopping
455 * when the heap property is re-established (each father smaller than its
458 local
void pqdownheap(s
, tree
, k
)
460 ct_data
*tree
; /* the tree to restore */
461 int k
; /* node to move down */
464 int j
= k
<< 1; /* left son of k */
465 while (j
<= s
->heap_len
) {
466 /* Set j to the smallest of the two sons: */
467 if (j
< s
->heap_len
&&
468 smaller(tree
, s
->heap
[j
+1], s
->heap
[j
], s
->depth
)) {
471 /* Exit if v is smaller than both sons */
472 if (smaller(tree
, v
, s
->heap
[j
], s
->depth
)) break;
474 /* Exchange v with the smallest son */
475 s
->heap
[k
] = s
->heap
[j
]; k
= j
;
477 /* And continue down the tree, setting j to the left son of k */
483 /* ===========================================================================
484 * Compute the optimal bit lengths for a tree and update the total bit length
485 * for the current block.
486 * IN assertion: the fields freq and dad are set, heap[heap_max] and
487 * above are the tree nodes sorted by increasing frequency.
488 * OUT assertions: the field len is set to the optimal bit length, the
489 * array bl_count contains the frequencies for each bit length.
490 * The length opt_len is updated; static_len is also updated if stree is
493 local
void gen_bitlen(s
, desc
)
495 tree_desc
*desc
; /* the tree descriptor */
497 ct_data
*tree
= desc
->dyn_tree
;
498 int max_code
= desc
->max_code
;
499 const ct_data
*stree
= desc
->stat_desc
->static_tree
;
500 const intf
*extra
= desc
->stat_desc
->extra_bits
;
501 int base
= desc
->stat_desc
->extra_base
;
502 int max_length
= desc
->stat_desc
->max_length
;
503 int h
; /* heap index */
504 int n
, m
; /* iterate over the tree elements */
505 int bits
; /* bit length */
506 int xbits
; /* extra bits */
507 ush f
; /* frequency */
508 int overflow
= 0; /* number of elements with bit length too large */
510 for (bits
= 0; bits
<= MAX_BITS
; bits
++) s
->bl_count
[bits
] = 0;
512 /* In a first pass, compute the optimal bit lengths (which may
513 * overflow in the case of the bit length tree).
515 tree
[s
->heap
[s
->heap_max
]].Len
= 0; /* root of the heap */
517 for (h
= s
->heap_max
+1; h
< HEAP_SIZE
; h
++) {
519 bits
= tree
[tree
[n
].Dad
].Len
+ 1;
520 if (bits
> max_length
) bits
= max_length
, overflow
++;
521 tree
[n
].Len
= (ush
)bits
;
522 /* We overwrite tree[n].Dad which is no longer needed */
524 if (n
> max_code
) continue; /* not a leaf node */
528 if (n
>= base
) xbits
= extra
[n
-base
];
530 s
->opt_len
+= (ulg
)f
* (bits
+ xbits
);
531 if (stree
) s
->static_len
+= (ulg
)f
* (stree
[n
].Len
+ xbits
);
533 if (overflow
== 0) return;
535 Trace((stderr
,"\nbit length overflow\n"));
536 /* This happens for example on obj2 and pic of the Calgary corpus */
538 /* Find the first bit length which could increase: */
541 while (s
->bl_count
[bits
] == 0) bits
--;
542 s
->bl_count
[bits
]--; /* move one leaf down the tree */
543 s
->bl_count
[bits
+1] += 2; /* move one overflow item as its brother */
544 s
->bl_count
[max_length
]--;
545 /* The brother of the overflow item also moves one step up,
546 * but this does not affect bl_count[max_length]
549 } while (overflow
> 0);
551 /* Now recompute all bit lengths, scanning in increasing frequency.
552 * h is still equal to HEAP_SIZE. (It is simpler to reconstruct all
553 * lengths instead of fixing only the wrong ones. This idea is taken
554 * from 'ar' written by Haruhiko Okumura.)
556 for (bits
= max_length
; bits
!= 0; bits
--) {
557 n
= s
->bl_count
[bits
];
560 if (m
> max_code
) continue;
561 if ((unsigned) tree
[m
].Len
!= (unsigned) bits
) {
562 Trace((stderr
,"code %d bits %d->%d\n", m
, tree
[m
].Len
, bits
));
563 s
->opt_len
+= ((long)bits
- (long)tree
[m
].Len
)
565 tree
[m
].Len
= (ush
)bits
;
572 /* ===========================================================================
573 * Generate the codes for a given tree and bit counts (which need not be
575 * IN assertion: the array bl_count contains the bit length statistics for
576 * the given tree and the field len is set for all tree elements.
577 * OUT assertion: the field code is set for all tree elements of non
580 local
void gen_codes (tree
, max_code
, bl_count
)
581 ct_data
*tree
; /* the tree to decorate */
582 int max_code
; /* largest code with non zero frequency */
583 ushf
*bl_count
; /* number of codes at each bit length */
585 ush next_code
[MAX_BITS
+1]; /* next code value for each bit length */
586 ush code
= 0; /* running code value */
587 int bits
; /* bit index */
588 int n
; /* code index */
590 /* The distribution counts are first used to generate the code values
591 * without bit reversal.
593 for (bits
= 1; bits
<= MAX_BITS
; bits
++) {
594 next_code
[bits
] = code
= (code
+ bl_count
[bits
-1]) << 1;
596 /* Check that the bit counts in bl_count are consistent. The last code
599 Assert (code
+ bl_count
[MAX_BITS
]-1 == (1<<MAX_BITS
)-1,
600 "inconsistent bit counts");
601 Tracev((stderr
,"\ngen_codes: max_code %d ", max_code
));
603 for (n
= 0; n
<= max_code
; n
++) {
604 int len
= tree
[n
].Len
;
605 if (len
== 0) continue;
606 /* Now reverse the bits */
607 tree
[n
].Code
= bi_reverse(next_code
[len
]++, len
);
609 Tracecv(tree
!= static_ltree
, (stderr
,"\nn %3d %c l %2d c %4x (%x) ",
610 n
, (isgraph(n
) ? n
: ' '), len
, tree
[n
].Code
, next_code
[len
]-1));
614 /* ===========================================================================
615 * Construct one Huffman tree and assigns the code bit strings and lengths.
616 * Update the total bit length for the current block.
617 * IN assertion: the field freq is set for all tree elements.
618 * OUT assertions: the fields len and code are set to the optimal bit length
619 * and corresponding code. The length opt_len is updated; static_len is
620 * also updated if stree is not null. The field max_code is set.
622 local
void build_tree(s
, desc
)
624 tree_desc
*desc
; /* the tree descriptor */
626 ct_data
*tree
= desc
->dyn_tree
;
627 const ct_data
*stree
= desc
->stat_desc
->static_tree
;
628 int elems
= desc
->stat_desc
->elems
;
629 int n
, m
; /* iterate over heap elements */
630 int max_code
= -1; /* largest code with non zero frequency */
631 int node
; /* new node being created */
633 /* Construct the initial heap, with least frequent element in
634 * heap[SMALLEST]. The sons of heap[n] are heap[2*n] and heap[2*n+1].
635 * heap[0] is not used.
637 s
->heap_len
= 0, s
->heap_max
= HEAP_SIZE
;
639 for (n
= 0; n
< elems
; n
++) {
640 if (tree
[n
].Freq
!= 0) {
641 s
->heap
[++(s
->heap_len
)] = max_code
= n
;
648 /* The pkzip format requires that at least one distance code exists,
649 * and that at least one bit should be sent even if there is only one
650 * possible code. So to avoid special checks later on we force at least
651 * two codes of non zero frequency.
653 while (s
->heap_len
< 2) {
654 node
= s
->heap
[++(s
->heap_len
)] = (max_code
< 2 ? ++max_code
: 0);
657 s
->opt_len
--; if (stree
) s
->static_len
-= stree
[node
].Len
;
658 /* node is 0 or 1 so it does not have extra bits */
660 desc
->max_code
= max_code
;
662 /* The elements heap[heap_len/2+1 .. heap_len] are leaves of the tree,
663 * establish sub-heaps of increasing lengths:
665 for (n
= s
->heap_len
/2; n
>= 1; n
--) pqdownheap(s
, tree
, n
);
667 /* Construct the Huffman tree by repeatedly combining the least two
670 node
= elems
; /* next internal node of the tree */
672 pqremove(s
, tree
, n
); /* n = node of least frequency */
673 m
= s
->heap
[SMALLEST
]; /* m = node of next least frequency */
675 s
->heap
[--(s
->heap_max
)] = n
; /* keep the nodes sorted by frequency */
676 s
->heap
[--(s
->heap_max
)] = m
;
678 /* Create a new node father of n and m */
679 tree
[node
].Freq
= tree
[n
].Freq
+ tree
[m
].Freq
;
680 s
->depth
[node
] = (uch
)((s
->depth
[n
] >= s
->depth
[m
] ?
681 s
->depth
[n
] : s
->depth
[m
]) + 1);
682 tree
[n
].Dad
= tree
[m
].Dad
= (ush
)node
;
684 if (tree
== s
->bl_tree
) {
685 fprintf(stderr
,"\nnode %d(%d), sons %d(%d) %d(%d)",
686 node
, tree
[node
].Freq
, n
, tree
[n
].Freq
, m
, tree
[m
].Freq
);
689 /* and insert the new node in the heap */
690 s
->heap
[SMALLEST
] = node
++;
691 pqdownheap(s
, tree
, SMALLEST
);
693 } while (s
->heap_len
>= 2);
695 s
->heap
[--(s
->heap_max
)] = s
->heap
[SMALLEST
];
697 /* At this point, the fields freq and dad are set. We can now
698 * generate the bit lengths.
700 gen_bitlen(s
, (tree_desc
*)desc
);
702 /* The field len is now set, we can generate the bit codes */
703 gen_codes ((ct_data
*)tree
, max_code
, s
->bl_count
);
706 /* ===========================================================================
707 * Scan a literal or distance tree to determine the frequencies of the codes
708 * in the bit length tree.
710 local
void scan_tree (s
, tree
, max_code
)
712 ct_data
*tree
; /* the tree to be scanned */
713 int max_code
; /* and its largest code of non zero frequency */
715 int n
; /* iterates over all tree elements */
716 int prevlen
= -1; /* last emitted length */
717 int curlen
; /* length of current code */
718 int nextlen
= tree
[0].Len
; /* length of next code */
719 int count
= 0; /* repeat count of the current code */
720 int max_count
= 7; /* max repeat count */
721 int min_count
= 4; /* min repeat count */
723 if (nextlen
== 0) max_count
= 138, min_count
= 3;
724 tree
[max_code
+1].Len
= (ush
)0xffff; /* guard */
726 for (n
= 0; n
<= max_code
; n
++) {
727 curlen
= nextlen
; nextlen
= tree
[n
+1].Len
;
728 if (++count
< max_count
&& curlen
== nextlen
) {
730 } else if (count
< min_count
) {
731 s
->bl_tree
[curlen
].Freq
+= count
;
732 } else if (curlen
!= 0) {
733 if (curlen
!= prevlen
) s
->bl_tree
[curlen
].Freq
++;
734 s
->bl_tree
[REP_3_6
].Freq
++;
735 } else if (count
<= 10) {
736 s
->bl_tree
[REPZ_3_10
].Freq
++;
738 s
->bl_tree
[REPZ_11_138
].Freq
++;
740 count
= 0; prevlen
= curlen
;
742 max_count
= 138, min_count
= 3;
743 } else if (curlen
== nextlen
) {
744 max_count
= 6, min_count
= 3;
746 max_count
= 7, min_count
= 4;
751 /* ===========================================================================
752 * Send a literal or distance tree in compressed form, using the codes in
755 local
void send_tree (s
, tree
, max_code
)
757 ct_data
*tree
; /* the tree to be scanned */
758 int max_code
; /* and its largest code of non zero frequency */
760 int n
; /* iterates over all tree elements */
761 int prevlen
= -1; /* last emitted length */
762 int curlen
; /* length of current code */
763 int nextlen
= tree
[0].Len
; /* length of next code */
764 int count
= 0; /* repeat count of the current code */
765 int max_count
= 7; /* max repeat count */
766 int min_count
= 4; /* min repeat count */
768 /* tree[max_code+1].Len = -1; */ /* guard already set */
769 if (nextlen
== 0) max_count
= 138, min_count
= 3;
771 for (n
= 0; n
<= max_code
; n
++) {
772 curlen
= nextlen
; nextlen
= tree
[n
+1].Len
;
773 if (++count
< max_count
&& curlen
== nextlen
) {
775 } else if (count
< min_count
) {
776 do { send_code(s
, curlen
, s
->bl_tree
); } while (--count
!= 0);
778 } else if (curlen
!= 0) {
779 if (curlen
!= prevlen
) {
780 send_code(s
, curlen
, s
->bl_tree
); count
--;
782 Assert(count
>= 3 && count
<= 6, " 3_6?");
783 send_code(s
, REP_3_6
, s
->bl_tree
); send_bits(s
, count
-3, 2);
785 } else if (count
<= 10) {
786 send_code(s
, REPZ_3_10
, s
->bl_tree
); send_bits(s
, count
-3, 3);
789 send_code(s
, REPZ_11_138
, s
->bl_tree
); send_bits(s
, count
-11, 7);
791 count
= 0; prevlen
= curlen
;
793 max_count
= 138, min_count
= 3;
794 } else if (curlen
== nextlen
) {
795 max_count
= 6, min_count
= 3;
797 max_count
= 7, min_count
= 4;
802 /* ===========================================================================
803 * Construct the Huffman tree for the bit lengths and return the index in
804 * bl_order of the last bit length code to send.
806 local
int build_bl_tree(s
)
809 int max_blindex
; /* index of last bit length code of non zero freq */
811 /* Determine the bit length frequencies for literal and distance trees */
812 scan_tree(s
, (ct_data
*)s
->dyn_ltree
, s
->l_desc
.max_code
);
813 scan_tree(s
, (ct_data
*)s
->dyn_dtree
, s
->d_desc
.max_code
);
815 /* Build the bit length tree: */
816 build_tree(s
, (tree_desc
*)(&(s
->bl_desc
)));
817 /* opt_len now includes the length of the tree representations, except
818 * the lengths of the bit lengths codes and the 5+5+4 bits for the counts.
821 /* Determine the number of bit length codes to send. The pkzip format
822 * requires that at least 4 bit length codes be sent. (appnote.txt says
823 * 3 but the actual value used is 4.)
825 for (max_blindex
= BL_CODES
-1; max_blindex
>= 3; max_blindex
--) {
826 if (s
->bl_tree
[bl_order
[max_blindex
]].Len
!= 0) break;
828 /* Update opt_len to include the bit length tree and counts */
829 s
->opt_len
+= 3*(max_blindex
+1) + 5+5+4;
830 Tracev((stderr
, "\ndyn trees: dyn %ld, stat %ld",
831 s
->opt_len
, s
->static_len
));
836 /* ===========================================================================
837 * Send the header for a block using dynamic Huffman trees: the counts, the
838 * lengths of the bit length codes, the literal tree and the distance tree.
839 * IN assertion: lcodes >= 257, dcodes >= 1, blcodes >= 4.
841 local
void send_all_trees(s
, lcodes
, dcodes
, blcodes
)
843 int lcodes
, dcodes
, blcodes
; /* number of codes for each tree */
845 int rank
; /* index in bl_order */
847 Assert (lcodes
>= 257 && dcodes
>= 1 && blcodes
>= 4, "not enough codes");
848 Assert (lcodes
<= L_CODES
&& dcodes
<= D_CODES
&& blcodes
<= BL_CODES
,
850 Tracev((stderr
, "\nbl counts: "));
851 send_bits(s
, lcodes
-257, 5); /* not +255 as stated in appnote.txt */
852 send_bits(s
, dcodes
-1, 5);
853 send_bits(s
, blcodes
-4, 4); /* not -3 as stated in appnote.txt */
854 for (rank
= 0; rank
< blcodes
; rank
++) {
855 Tracev((stderr
, "\nbl code %2d ", bl_order
[rank
]));
856 send_bits(s
, s
->bl_tree
[bl_order
[rank
]].Len
, 3);
858 Tracev((stderr
, "\nbl tree: sent %ld", s
->bits_sent
));
860 send_tree(s
, (ct_data
*)s
->dyn_ltree
, lcodes
-1); /* literal tree */
861 Tracev((stderr
, "\nlit tree: sent %ld", s
->bits_sent
));
863 send_tree(s
, (ct_data
*)s
->dyn_dtree
, dcodes
-1); /* distance tree */
864 Tracev((stderr
, "\ndist tree: sent %ld", s
->bits_sent
));
867 /* ===========================================================================
868 * Send a stored block
870 void _tr_stored_block(s
, buf
, stored_len
, last
)
872 charf
*buf
; /* input block */
873 ulg stored_len
; /* length of input block */
874 int last
; /* one if this is the last block for a file */
876 send_bits(s
, (STORED_BLOCK
<<1)+last
, 3); /* send block type */
878 s
->compressed_len
= (s
->compressed_len
+ 3 + 7) & (ulg
)~7L;
879 s
->compressed_len
+= (stored_len
+ 4) << 3;
881 copy_block(s
, buf
, (unsigned)stored_len
, 1); /* with header */
884 /* ===========================================================================
885 * Send one empty static block to give enough lookahead for inflate.
886 * This takes 10 bits, of which 7 may remain in the bit buffer.
887 * The current inflate code requires 9 bits of lookahead. If the
888 * last two codes for the previous block (real code plus EOB) were coded
889 * on 5 bits or less, inflate may have only 5+3 bits of lookahead to decode
890 * the last real code. In this case we send two empty static blocks instead
891 * of one. (There are no problems if the previous block is stored or fixed.)
892 * To simplify the code, we assume the worst case of last real code encoded
898 send_bits(s
, STATIC_TREES
<<1, 3);
899 send_code(s
, END_BLOCK
, static_ltree
);
901 s
->compressed_len
+= 10L; /* 3 for block type, 7 for EOB */
904 /* Of the 10 bits for the empty block, we have already sent
905 * (10 - bi_valid) bits. The lookahead for the last real code (before
906 * the EOB of the previous block) was thus at least one plus the length
907 * of the EOB plus what we have just sent of the empty static block.
909 if (1 + s
->last_eob_len
+ 10 - s
->bi_valid
< 9) {
910 send_bits(s
, STATIC_TREES
<<1, 3);
911 send_code(s
, END_BLOCK
, static_ltree
);
913 s
->compressed_len
+= 10L;
920 /* ===========================================================================
921 * Determine the best encoding for the current block: dynamic trees, static
922 * trees or store, and output the encoded block to the zip file.
924 void _tr_flush_block(s
, buf
, stored_len
, last
)
926 charf
*buf
; /* input block, or NULL if too old */
927 ulg stored_len
; /* length of input block */
928 int last
; /* one if this is the last block for a file */
930 ulg opt_lenb
, static_lenb
; /* opt_len and static_len in bytes */
931 int max_blindex
= 0; /* index of last bit length code of non zero freq */
933 /* Build the Huffman trees unless a stored block is forced */
936 /* Check if the file is binary or text */
937 if (s
->strm
->data_type
== Z_UNKNOWN
)
938 s
->strm
->data_type
= detect_data_type(s
);
940 /* Construct the literal and distance trees */
941 build_tree(s
, (tree_desc
*)(&(s
->l_desc
)));
942 Tracev((stderr
, "\nlit data: dyn %ld, stat %ld", s
->opt_len
,
945 build_tree(s
, (tree_desc
*)(&(s
->d_desc
)));
946 Tracev((stderr
, "\ndist data: dyn %ld, stat %ld", s
->opt_len
,
948 /* At this point, opt_len and static_len are the total bit lengths of
949 * the compressed block data, excluding the tree representations.
952 /* Build the bit length tree for the above two trees, and get the index
953 * in bl_order of the last bit length code to send.
955 max_blindex
= build_bl_tree(s
);
957 /* Determine the best encoding. Compute the block lengths in bytes. */
958 opt_lenb
= (s
->opt_len
+3+7)>>3;
959 static_lenb
= (s
->static_len
+3+7)>>3;
961 Tracev((stderr
, "\nopt %lu(%lu) stat %lu(%lu) stored %lu lit %u ",
962 opt_lenb
, s
->opt_len
, static_lenb
, s
->static_len
, stored_len
,
965 if (static_lenb
<= opt_lenb
) opt_lenb
= static_lenb
;
968 Assert(buf
!= (char*)0, "lost buf");
969 opt_lenb
= static_lenb
= stored_len
+ 5; /* force a stored block */
973 if (buf
!= (char*)0) { /* force stored block */
975 if (stored_len
+4 <= opt_lenb
&& buf
!= (char*)0) {
976 /* 4: two words for the lengths */
978 /* The test buf != NULL is only necessary if LIT_BUFSIZE > WSIZE.
979 * Otherwise we can't have processed more than WSIZE input bytes since
980 * the last block flush, because compression would have been
981 * successful. If LIT_BUFSIZE <= WSIZE, it is never too late to
982 * transform a block into a stored block.
984 _tr_stored_block(s
, buf
, stored_len
, last
);
987 } else if (static_lenb
>= 0) { /* force static trees */
989 } else if (s
->strategy
== Z_FIXED
|| static_lenb
== opt_lenb
) {
991 send_bits(s
, (STATIC_TREES
<<1)+last
, 3);
992 compress_block(s
, (ct_data
*)static_ltree
, (ct_data
*)static_dtree
);
994 s
->compressed_len
+= 3 + s
->static_len
;
997 send_bits(s
, (DYN_TREES
<<1)+last
, 3);
998 send_all_trees(s
, s
->l_desc
.max_code
+1, s
->d_desc
.max_code
+1,
1000 compress_block(s
, (ct_data
*)s
->dyn_ltree
, (ct_data
*)s
->dyn_dtree
);
1002 s
->compressed_len
+= 3 + s
->opt_len
;
1005 Assert (s
->compressed_len
== s
->bits_sent
, "bad compressed size");
1006 /* The above check is made mod 2^32, for files larger than 512 MB
1007 * and uLong implemented on 32 bits.
1014 s
->compressed_len
+= 7; /* align on byte boundary */
1017 Tracev((stderr
,"\ncomprlen %lu(%lu) ", s
->compressed_len
>>3,
1018 s
->compressed_len
-7*last
));
1021 /* ===========================================================================
1022 * Save the match info and tally the frequency counts. Return true if
1023 * the current block must be flushed.
1025 int _tr_tally (s
, dist
, lc
)
1027 unsigned dist
; /* distance of matched string */
1028 unsigned lc
; /* match length-MIN_MATCH or unmatched char (if dist==0) */
1030 s
->d_buf
[s
->last_lit
] = (ush
)dist
;
1031 s
->l_buf
[s
->last_lit
++] = (uch
)lc
;
1033 /* lc is the unmatched char */
1034 s
->dyn_ltree
[lc
].Freq
++;
1037 /* Here, lc is the match length - MIN_MATCH */
1038 dist
--; /* dist = match distance - 1 */
1039 Assert((ush
)dist
< (ush
)MAX_DIST(s
) &&
1040 (ush
)lc
<= (ush
)(MAX_MATCH
-MIN_MATCH
) &&
1041 (ush
)d_code(dist
) < (ush
)D_CODES
, "_tr_tally: bad match");
1043 s
->dyn_ltree
[_length_code
[lc
]+LITERALS
+1].Freq
++;
1044 s
->dyn_dtree
[d_code(dist
)].Freq
++;
1047 #ifdef TRUNCATE_BLOCK
1048 /* Try to guess if it is profitable to stop the current block here */
1049 if ((s
->last_lit
& 0x1fff) == 0 && s
->level
> 2) {
1050 /* Compute an upper bound for the compressed length */
1051 ulg out_length
= (ulg
)s
->last_lit
*8L;
1052 ulg in_length
= (ulg
)((long)s
->strstart
- s
->block_start
);
1054 for (dcode
= 0; dcode
< D_CODES
; dcode
++) {
1055 out_length
+= (ulg
)s
->dyn_dtree
[dcode
].Freq
*
1056 (5L+extra_dbits
[dcode
]);
1059 Tracev((stderr
,"\nlast_lit %u, in %ld, out ~%ld(%ld%%) ",
1060 s
->last_lit
, in_length
, out_length
,
1061 100L - out_length
*100L/in_length
));
1062 if (s
->matches
< s
->last_lit
/2 && out_length
< in_length
/2) return 1;
1065 return (s
->last_lit
== s
->lit_bufsize
-1);
1066 /* We avoid equality with lit_bufsize because of wraparound at 64K
1067 * on 16 bit machines and because stored blocks are restricted to
1072 /* ===========================================================================
1073 * Send the block data compressed using the given Huffman trees
1075 local
void compress_block(s
, ltree
, dtree
)
1077 ct_data
*ltree
; /* literal tree */
1078 ct_data
*dtree
; /* distance tree */
1080 unsigned dist
; /* distance of matched string */
1081 int lc
; /* match length or unmatched char (if dist == 0) */
1082 unsigned lx
= 0; /* running index in l_buf */
1083 unsigned code
; /* the code to send */
1084 int extra
; /* number of extra bits to send */
1086 if (s
->last_lit
!= 0) do {
1087 dist
= s
->d_buf
[lx
];
1088 lc
= s
->l_buf
[lx
++];
1090 send_code(s
, lc
, ltree
); /* send a literal byte */
1091 Tracecv(isgraph(lc
), (stderr
," '%c' ", lc
));
1093 /* Here, lc is the match length - MIN_MATCH */
1094 code
= _length_code
[lc
];
1095 send_code(s
, code
+LITERALS
+1, ltree
); /* send the length code */
1096 extra
= extra_lbits
[code
];
1098 lc
-= base_length
[code
];
1099 send_bits(s
, lc
, extra
); /* send the extra length bits */
1101 dist
--; /* dist is now the match distance - 1 */
1102 code
= d_code(dist
);
1103 Assert (code
< D_CODES
, "bad d_code");
1105 send_code(s
, code
, dtree
); /* send the distance code */
1106 extra
= extra_dbits
[code
];
1108 dist
-= base_dist
[code
];
1109 send_bits(s
, dist
, extra
); /* send the extra distance bits */
1111 } /* literal or match pair ? */
1113 /* Check that the overlay between pending_buf and d_buf+l_buf is ok: */
1114 Assert((uInt
)(s
->pending
) < s
->lit_bufsize
+ 2*lx
,
1115 "pendingBuf overflow");
1117 } while (lx
< s
->last_lit
);
1119 send_code(s
, END_BLOCK
, ltree
);
1120 s
->last_eob_len
= ltree
[END_BLOCK
].Len
;
1123 /* ===========================================================================
1124 * Check if the data type is TEXT or BINARY, using the following algorithm:
1125 * - TEXT if the two conditions below are satisfied:
1126 * a) There are no non-portable control characters belonging to the
1127 * "black list" (0..6, 14..25, 28..31).
1128 * b) There is at least one printable character belonging to the
1129 * "white list" (9 {TAB}, 10 {LF}, 13 {CR}, 32..255).
1130 * - BINARY otherwise.
1131 * - The following partially-portable control characters form a
1132 * "gray list" that is ignored in this detection algorithm:
1133 * (7 {BEL}, 8 {BS}, 11 {VT}, 12 {FF}, 26 {SUB}, 27 {ESC}).
1134 * IN assertion: the fields Freq of dyn_ltree are set.
1136 local
int detect_data_type(s
)
1139 /* black_mask is the bit mask of black-listed bytes
1140 * set bits 0..6, 14..25, and 28..31
1141 * 0xf3ffc07f = binary 11110011111111111100000001111111
1143 unsigned long black_mask
= 0xf3ffc07fUL
;
1146 /* Check for non-textual ("black-listed") bytes. */
1147 for (n
= 0; n
<= 31; n
++, black_mask
>>= 1)
1148 if ((black_mask
& 1) && (s
->dyn_ltree
[n
].Freq
!= 0))
1151 /* Check for textual ("white-listed") bytes. */
1152 if (s
->dyn_ltree
[9].Freq
!= 0 || s
->dyn_ltree
[10].Freq
!= 0
1153 || s
->dyn_ltree
[13].Freq
!= 0)
1155 for (n
= 32; n
< LITERALS
; n
++)
1156 if (s
->dyn_ltree
[n
].Freq
!= 0)
1159 /* There are no "black-listed" or "white-listed" bytes:
1160 * this stream either is empty or has tolerated ("gray-listed") bytes only.
1165 /* ===========================================================================
1166 * Reverse the first len bits of a code, using straightforward code (a faster
1167 * method would use a table)
1168 * IN assertion: 1 <= len <= 15
1170 local
unsigned bi_reverse(code
, len
)
1171 unsigned code
; /* the value to invert */
1172 int len
; /* its bit length */
1174 register unsigned res
= 0;
1177 code
>>= 1, res
<<= 1;
1178 } while (--len
> 0);
1182 /* ===========================================================================
1183 * Flush the bit buffer, keeping at most 7 bits in it.
1185 local
void bi_flush(s
)
1188 if (s
->bi_valid
== 16) {
1189 put_short(s
, s
->bi_buf
);
1192 } else if (s
->bi_valid
>= 8) {
1193 put_byte(s
, (Byte
)s
->bi_buf
);
1199 /* ===========================================================================
1200 * Flush the bit buffer and align the output on a byte boundary
1202 local
void bi_windup(s
)
1205 if (s
->bi_valid
> 8) {
1206 put_short(s
, s
->bi_buf
);
1207 } else if (s
->bi_valid
> 0) {
1208 put_byte(s
, (Byte
)s
->bi_buf
);
1213 s
->bits_sent
= (s
->bits_sent
+7) & ~7;
1217 /* ===========================================================================
1218 * Copy a stored block, storing first the length and its
1219 * one's complement if requested.
1221 local
void copy_block(s
, buf
, len
, header
)
1223 charf
*buf
; /* the input data */
1224 unsigned len
; /* its length */
1225 int header
; /* true if block header must be written */
1227 bi_windup(s
); /* align on byte boundary */
1228 s
->last_eob_len
= 8; /* enough lookahead for inflate */
1231 put_short(s
, (ush
)len
);
1232 put_short(s
, (ush
)~len
);
1234 s
->bits_sent
+= 2*16;
1238 s
->bits_sent
+= (ulg
)len
<<3;
1241 put_byte(s
, *buf
++);