1 /*@ S-nail - a mail user agent derived from Berkeley Mail.
2 *@ LZW file compression.
4 * Copyright (c) 2000-2004 Gunnar Ritter, Freiburg i. Br., Germany.
5 * Copyright (c) 2012 - 2020 Steffen (Daode) Nurpmeso <sdaoden@users.sf.net>.
6 * SPDX-License-Identifier: BSD-4-Clause
9 * Copyright (c) 1985, 1986, 1992, 1993
10 * The Regents of the University of California. All rights reserved.
12 * This code is derived from software contributed to Berkeley by
13 * Diomidis Spinellis and James A. Woods, derived from original
14 * work by Spencer Thomas and Joseph Orost.
16 * Redistribution and use in source and binary forms, with or without
17 * modification, are permitted provided that the following conditions
19 * 1. Redistributions of source code must retain the above copyright
20 * notice, this list of conditions and the following disclaimer.
21 * 2. Redistributions in binary form must reproduce the above copyright
22 * notice, this list of conditions and the following disclaimer in the
23 * documentation and/or other materials provided with the distribution.
24 * 4. Neither the name of the University nor the names of its contributors
25 * may be used to endorse or promote products derived from this software
26 * without specific prior written permission.
28 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
29 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
30 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
31 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
32 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
33 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
34 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
35 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
36 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
37 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
41 /* from zopen.c 8.1 (Berkeley) 6/27/93 */
42 /* from FreeBSD: /repoman/r/ncvs/src/usr.bin/compress/zopen.c,v
43 * 1.5.6.1 2002/07/16 00:52:08 tjr Exp */
44 /* from FreeBSD: git://git.freebsd.org/freebsd,
45 * master:usr.bin/compress/zopen.c,
46 * (Fix handling of corrupt compress(1)ed data. [11:04], 2011-09-28) */
49 * lzw.c - File compression ala IEEE Computer, June 1984.
52 * Spencer W. Thomas (decvax!utah-cs!thomas)
53 * Jim McKie (decvax!mcvax!jim)
54 * Steve Davies (decvax!vax135!petsd!peora!srd)
55 * Ken Turkowski (decvax!decwrl!turtlevax!ken)
56 * James A. Woods (decvax!ihnp4!ames!jaw)
57 * Joe Orost (decvax!vax135!petsd!joe)
59 * Cleaned up and converted to library returning I/O streams by
60 * Diomidis Spinellis <dds@doc.ic.ac.uk>.
62 * Adopted for Heirloom mailx by Gunnar Ritter.
65 #define su_FILE obs_lzw
68 #ifndef mx_HAVE_AMALGAMATION
77 #include "su/code-in.h"
79 /* Minimize differences to FreeBSDs usr.bin/compress/zopen.c */
81 #define u_int unsigned int
83 #define u_short unsigned short
85 #define u_char unsigned char
88 #define BITS 16 /* Default bits. */
89 #define HSIZE 69001 /* 95% occupancy */
91 /* A code_int must be able to hold 2**BITS values of type int, and also -1. */
92 typedef long code_int
;
93 typedef long count_int
;
95 typedef u_char char_type
;
96 static char_type magic_header
[] = {0x1F, 0x9D}; /* \037, \235 */
98 #define BIT_MASK 0x1f /* Defines for third byte of header. */
99 #define BLOCK_MASK 0x80
102 * Masks 0x40 and 0x20 are free. I think 0x20 should mean that there is
103 * a fourth header byte (for expansion).
105 #define INIT_BITS 9 /* Initial number of bits/code. */
107 #define MAXCODE(n_bits) ((1 << (n_bits)) - 1)
110 FILE *zs_fp
; /* File stream for I/O */
111 char zs_mode
; /* r or w */
113 S_START
, S_MIDDLE
, S_EOF
114 } zs_state
; /* State of computation */
115 u_int zs_n_bits
; /* Number of bits/code. */
116 u_int zs_maxbits
; /* User settable max # bits/code. */
117 code_int zs_maxcode
; /* Maximum code, given n_bits. */
118 code_int zs_maxmaxcode
; /* Should NEVER generate this code. */
119 count_int zs_htab
[HSIZE
];
120 u_short zs_codetab
[HSIZE
];
121 code_int zs_hsize
; /* For dynamic table sizing. */
122 code_int zs_free_ent
; /* First unused entry. */
124 * Block compression parameters -- after all codes are used up,
125 * and compression rate changes, start over.
127 int zs_block_compress
;
130 count_int zs_checkpoint
;
132 long zs_in_count
; /* Length of input. */
133 long zs_bytes_out
; /* Length of compressed output. */
134 long zs_out_count
; /* # of codes output (for debugging). */
135 char_type zs_buf
[BITS
];
140 code_int zs_hsize_reg
;
142 } w
; /* Write parameters */
144 char_type
*zs_stackp
;
146 code_int zs_code
, zs_oldcode
, zs_incode
;
147 int zs_roffset
, zs_size
;
148 char_type zs_gbuf
[BITS
];
149 } r
; /* Read parameters */
153 /* Definitions to retain old variable names */
155 #define zmode zs->zs_mode
156 #define state zs->zs_state
157 #define n_bits zs->zs_n_bits
158 #define maxbits zs->zs_maxbits
159 #define maxcode zs->zs_maxcode
160 #define maxmaxcode zs->zs_maxmaxcode
161 #define htab zs->zs_htab
162 #define codetab zs->zs_codetab
163 #define hsize zs->zs_hsize
164 #define free_ent zs->zs_free_ent
165 #define block_compress zs->zs_block_compress
166 #define clear_flg zs->zs_clear_flg
167 #define ratio zs->zs_ratio
168 #define checkpoint zs->zs_checkpoint
169 #define offset zs->zs_offset
170 #define in_count zs->zs_in_count
171 #define bytes_out zs->zs_bytes_out
172 #define out_count zs->zs_out_count
173 #define buf zs->zs_buf
174 #define fcode zs->u.w.zs_fcode
175 #define hsize_reg zs->u.w.zs_hsize_reg
176 #define ent zs->u.w.zs_ent
177 #define hshift zs->u.w.zs_hshift
178 #define stackp zs->u.r.zs_stackp
179 #define finchar zs->u.r.zs_finchar
180 #define code zs->u.r.zs_code
181 #define oldcode zs->u.r.zs_oldcode
182 #define incode zs->u.r.zs_incode
183 #define roffset zs->u.r.zs_roffset
184 #define size zs->u.r.zs_size
185 #define gbuf zs->u.r.zs_gbuf
188 * To save much memory, we overlay the table used by compress() with those
189 * used by decompress(). The tab_prefix table is the same size and type as
190 * the codetab. The tab_suffix table needs 2**BITS characters. We get this
191 * from the beginning of htab. The output stack uses the rest of htab, and
192 * contains characters. There is plenty of room for any possible stack
193 * (stack used to be 8000 characters).
196 #define htabof(i) htab[i]
197 #define codetabof(i) codetab[i]
199 #define tab_prefixof(i) codetabof(i)
200 #define tab_suffixof(i) ((char_type *)(htab))[i]
201 #define de_stack ((char_type *)&tab_suffixof(1 << BITS))
203 #define CHECK_GAP 10000 /* Ratio check interval. */
206 * the next two codes should not be changed lightly, as they must not
207 * lie within the contiguous general code space.
209 #define FIRST 257 /* First free entry. */
210 #define CLEAR 256 /* Table clear output code. */
212 static int cl_block(struct s_zstate
*);
213 static void cl_hash(struct s_zstate
*, count_int
);
214 static code_int
getcode(struct s_zstate
*);
215 static int output(struct s_zstate
*, code_int
);
218 * Algorithm from "A Technique for High Performance Data Compression",
219 * Terry A. Welch, IEEE Computer Vol 17, No 6 (June 1984), pp 8-19.
222 * Modified Lempel-Ziv method (LZW). Basically finds common
223 * substrings and replaces them with a variable size code. This is
224 * deterministic, and can be done on the fly. Thus, the decompression
225 * procedure needs no input table, but tracks the way the table was built.
231 * Algorithm: use open addressing double hashing (no chaining) on the
232 * prefix code / next character combination. We do a variant of Knuth's
233 * algorithm D (vol. 3, sec. 6.4) along with G. Knott's relatively-prime
234 * secondary probe. Here, the modular division first probe is gives way
235 * to a faster exclusive-or manipulation. Also do block compression with
236 * an adaptive reset, whereby the code table is cleared when the compression
237 * ratio decreases, but after the table fills. The variable-length output
238 * codes are re-sized at this point, and a special CLEAR code is generated
239 * for the decompressor. Late addition: construct the table according to
240 * file size for noticeable speed improvement on small files. Please direct
241 * questions about this implementation to ames!jaw.
244 zwrite(void *cookie
, const char *wbp
, int num
)
259 bp
= (const u_char
*)wbp
;
260 if (state
== S_MIDDLE
)
264 maxmaxcode
= 1L << maxbits
;
265 if (fwrite(magic_header
,
266 sizeof(char), sizeof(magic_header
), fp
) != sizeof(magic_header
))
268 tmp
= (u_char
)((maxbits
) | block_compress
);
269 if (fwrite(&tmp
, sizeof(char), sizeof(tmp
), fp
) != sizeof(tmp
))
273 bytes_out
= 3; /* Includes 3-byte header mojo. */
278 checkpoint
= CHECK_GAP
;
279 maxcode
= MAXCODE(n_bits
= INIT_BITS
);
280 free_ent
= ((block_compress
) ? FIRST
: 256);
286 for (fcode
= (long)hsize
; fcode
< 65536L; fcode
*= 2L)
288 hshift
= 8 - hshift
; /* Set hash code range bound. */
291 cl_hash(zs
, (count_int
)hsize_reg
); /* Clear hash table. */
293 middle
: for (i
= 0; count
--;) {
296 fcode
= (long)(((long)c
<< maxbits
) + ent
);
297 i
= ((c
<< hshift
) ^ ent
); /* Xor hashing. */
299 if (htabof(i
) == fcode
) {
302 } else if ((long)htabof(i
) < 0) /* Empty slot. */
304 disp
= hsize_reg
- i
; /* Secondary hash (after G. Knott). */
307 probe
: if ((i
-= disp
) < 0)
310 if (htabof(i
) == fcode
) {
314 if ((long)htabof(i
) >= 0)
316 nomatch
: if (output(zs
, (code_int
) ent
) == -1)
320 if (free_ent
< maxmaxcode
) {
321 codetabof(i
) = free_ent
++; /* code -> hashtable */
323 } else if ((count_int
)in_count
>=
324 checkpoint
&& block_compress
) {
325 if (cl_block(zs
) == -1)
338 if (zmode
== 'w') { /* Put out the final code. */
339 if (output(zs
, (code_int
) ent
) == -1) {
344 if (output(zs
, (code_int
) - 1) == -1) {
354 * Output the given code.
356 * code: A n_bits-bit integer. If == -1, then EOF. This assumes
357 * that n_bits =< (long)wordsize - 1.
359 * Outputs code to the file.
361 * Chars are 8 bits long.
363 * Maintain a BITS character long buffer (so that 8 codes will
364 * fit in it exactly). Use the VAX insv instruction to insert each
365 * code in turn. When the buffer fills up empty it and start over.
368 static char_type lmask
[9] =
369 {0xff, 0xfe, 0xfc, 0xf8, 0xf0, 0xe0, 0xc0, 0x80, 0x00};
370 static char_type rmask
[9] =
371 {0x00, 0x01, 0x03, 0x07, 0x0f, 0x1f, 0x3f, 0x7f, 0xff};
374 output(struct s_zstate
*zs
, code_int ocode
)
384 /* Get to the first byte. */
388 * Since ocode is always >= 8 bits, only need to mask the first
391 *bp
= (*bp
& rmask
[r_off
]) | ((ocode
<< r_off
) & lmask
[r_off
]);
395 /* Get any 8 bit parts in the middle (<=1 for up to 16 bits). */
405 if (offset
== (n_bits
<< 3)) {
409 if (fwrite(bp
, sizeof(char), bits
, fp
) != bits
)
416 * If the next entry is going to be too big for the ocode size,
417 * then increase it, if possible.
419 if (free_ent
> maxcode
|| (clear_flg
> 0)) {
421 * Write the whole buffer, because the input side won't
422 * discover the size increase until after it has read it.
425 if (fwrite(buf
, 1, n_bits
, fp
) != n_bits
)
432 maxcode
= MAXCODE(n_bits
= INIT_BITS
);
436 if (n_bits
== maxbits
)
437 maxcode
= maxmaxcode
;
439 maxcode
= MAXCODE(n_bits
);
443 /* At EOF, write the rest of the buffer. */
445 offset
= (offset
+ 7) / 8;
446 if (fwrite(buf
, 1, offset
, fp
) != offset
)
456 * Decompress read. This routine adapts to the codes in the file building
457 * the "string" table on-the-fly; requiring no table to be stored in the
458 * compressed file. The tables used herein are shared with those of the
459 * compress() routine. See the definitions above.
462 zread(void *cookie
, char *rbp
, int num
)
466 u_char
*bp
, header
[3];
484 /* Check the magic number */
486 sizeof(char), sizeof(header
), fp
) != sizeof(header
) ||
487 su_mem_cmp(header
, magic_header
, sizeof(magic_header
)) != 0) {
490 maxbits
= header
[2]; /* Set -b from file. */
491 block_compress
= maxbits
& BLOCK_MASK
;
493 maxmaxcode
= 1L << maxbits
;
494 if (maxbits
> BITS
|| maxbits
< 12) {
497 /* As above, initialize the first 256 entries in the table. */
498 maxcode
= MAXCODE(n_bits
= INIT_BITS
);
499 for (code
= 255; code
>= 0; code
--) {
500 tab_prefixof(code
) = 0;
501 tab_suffixof(code
) = (char_type
) code
;
503 free_ent
= block_compress
? FIRST
: 256;
505 finchar
= oldcode
= getcode(zs
);
506 if (oldcode
== -1) /* EOF already? */
507 return (0); /* Get out of here */
509 /* First code must be 8 bits = char. */
510 *bp
++ = (u_char
)finchar
;
514 while ((code
= getcode(zs
)) > -1) {
516 if ((code
== CLEAR
) && block_compress
) {
517 for (code
= 255; code
>= 0; code
--)
518 tab_prefixof(code
) = 0;
526 /* Special case for kWkWk string. */
527 if (code
>= free_ent
) {
528 if (code
> free_ent
|| oldcode
== -1) {
535 * The above condition ensures that code < free_ent.
536 * The construction of tab_prefixof in turn guarantees that
537 * each iteration decreases code and therefore stack usage is
538 * bound by 1 << BITS - 256.
541 /* Generate output characters in reverse order. */
542 while (code
>= 256) {
543 *stackp
++ = tab_suffixof(code
);
544 code
= tab_prefixof(code
);
546 *stackp
++ = finchar
= tab_suffixof(code
);
548 /* And put them out in forward order. */
553 } while (stackp
> de_stack
);
555 /* Generate the new entry. */
556 if ((code
= free_ent
) < maxmaxcode
&& oldcode
!= -1) {
557 tab_prefixof(code
) = (u_short
) oldcode
;
558 tab_suffixof(code
) = finchar
;
562 /* Remember previous code. */
566 eof
: return (num
- count
);
570 * Read one code from the standard input. If EOF, return -1.
574 * code or -1 is returned.
577 getcode(struct s_zstate
*zs
)
584 if (clear_flg
> 0 || roffset
>= size
|| free_ent
> maxcode
) {
586 * If the next entry will be too big for the current gcode
587 * size, then we must increase the size. This implies reading
588 * a new buffer full, too.
590 if (free_ent
> maxcode
) {
592 if (n_bits
== maxbits
) /* Won't get any bigger now. */
593 maxcode
= maxmaxcode
;
595 maxcode
= MAXCODE(n_bits
);
598 maxcode
= MAXCODE(n_bits
= INIT_BITS
);
601 size
= fread(gbuf
, 1, n_bits
, fp
);
602 if (size
<= 0) /* End of file. */
605 /* Round size down to integral number of codes. */
606 size
= (size
<< 3) - (n_bits
- 1);
611 /* Get to the first byte. */
615 /* Get first part (low order bits). */
616 gcode
= (*bp
++ >> r_off
);
618 r_off
= 8 - r_off
; /* Now, roffset into gcode word. */
620 /* Get any 8 bit parts in the middle (<=1 for up to 16 bits). */
622 gcode
|= *bp
++ << r_off
;
627 /* High order bits. */
628 gcode
|= (*bp
& rmask
[bits
]) << r_off
;
635 cl_block(struct s_zstate
*zs
) /* Table clear for block compress. */
639 checkpoint
= in_count
+ CHECK_GAP
;
641 if (in_count
> 0x007fffff) { /* Shift will overflow. */
642 rat
= bytes_out
>> 8;
643 if (rat
== 0) /* Don't divide by zero. */
646 rat
= in_count
/ rat
;
648 rat
= (in_count
<< 8) / bytes_out
; /* 8 fractional bits. */
653 cl_hash(zs
, (count_int
) hsize
);
656 if (output(zs
, (code_int
) CLEAR
) == -1)
663 cl_hash(struct s_zstate
*zs
, count_int cl_hsize
) /* Reset code table. */
669 htab_p
= htab
+ cl_hsize
;
671 do { /* Might use Sys V su_mem_set(3) here. */
689 } while ((i
-= 16) >= 0);
690 for (i
+= 16; i
> 0; i
--)
701 zs
= n_calloc(1, sizeof *zs
);
702 maxbits
= bits
? bits
: BITS
; /* User settable max # bits/code. */
703 maxmaxcode
= 1L << maxbits
; /* Should NEVER generate this code. */
704 hsize
= HSIZE
; /* For dynamic table sizing. */
705 free_ent
= 0; /* First unused entry. */
706 block_compress
= BLOCK_MASK
;
709 checkpoint
= CHECK_GAP
;
710 in_count
= 1; /* Length of input. */
711 out_count
= 0; /* # of codes output (for debugging). */
740 #undef block_compress
762 #include "su/code-ou.h"
763 #endif /* ndef mx_HAVE_IMAP */