1 /* lzo_swd.ch -- sliding window dictionary
3 This file is part of the LZO real-time data compression library.
5 Copyright (C) 2008 Markus Franz Xaver Johannes Oberhumer
6 Copyright (C) 2007 Markus Franz Xaver Johannes Oberhumer
7 Copyright (C) 2006 Markus Franz Xaver Johannes Oberhumer
8 Copyright (C) 2005 Markus Franz Xaver Johannes Oberhumer
9 Copyright (C) 2004 Markus Franz Xaver Johannes Oberhumer
10 Copyright (C) 2003 Markus Franz Xaver Johannes Oberhumer
11 Copyright (C) 2002 Markus Franz Xaver Johannes Oberhumer
12 Copyright (C) 2001 Markus Franz Xaver Johannes Oberhumer
13 Copyright (C) 2000 Markus Franz Xaver Johannes Oberhumer
14 Copyright (C) 1999 Markus Franz Xaver Johannes Oberhumer
15 Copyright (C) 1998 Markus Franz Xaver Johannes Oberhumer
16 Copyright (C) 1997 Markus Franz Xaver Johannes Oberhumer
17 Copyright (C) 1996 Markus Franz Xaver Johannes Oberhumer
20 The LZO library is free software; you can redistribute it and/or
21 modify it under the terms of the GNU General Public License as
22 published by the Free Software Foundation; either version 2 of
23 the License, or (at your option) any later version.
25 The LZO library is distributed in the hope that it will be useful,
26 but WITHOUT ANY WARRANTY; without even the implied warranty of
27 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
28 GNU General Public License for more details.
30 You should have received a copy of the GNU General Public License
31 along with the LZO library; see the file COPYING.
32 If not, write to the Free Software Foundation, Inc.,
33 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
35 Markus F.X.J. Oberhumer
36 <markus@oberhumer.com>
37 http://www.oberhumer.com/opensource/lzo/
41 #if (LZO_UINT_MAX < LZO_0xffffffffL)
42 # error "LZO_UINT_MAX"
46 /***********************************************************************
48 ************************************************************************/
57 # define SWD_THRESHOLD THRESHOLD
60 /* unsigned type for dictionary access - don't waste memory here */
61 #if (0UL + SWD_N + SWD_F + SWD_F < 0UL + USHRT_MAX)
62 typedef unsigned short swd_uint;
63 # define SWD_UINT_MAX USHRT_MAX
64 #elif (0UL + SWD_N + SWD_F + SWD_F < 0UL + UINT_MAX)
65 typedef unsigned swd_uint;
66 # define SWD_UINT_MAX UINT_MAX
68 typedef lzo_uint swd_uint;
69 # define SWD_UINT_MAX LZO_UINT_MAX
71 #define swd_uintp swd_uint __LZO_MMODEL *
72 #define SWD_UINT(x) ((swd_uint)(x))
76 # define SWD_HSIZE 16384
79 # define SWD_MAX_CHAIN 2048
85 ((DMUL(0x9f5f,(((((lzo_xint)b[p]<<5)^b[p+1])<<5)^b[p+2]))>>5) & (SWD_HSIZE-1))
88 ((DMUL(0x9f5f,(((((lzo_xint)b[p+2]<<5)^b[p+1])<<5)^b[p]))>>5) & (SWD_HSIZE-1))
92 #if (SWD_THRESHOLD == 1) && !defined(HEAD2)
93 # if 1 && defined(LZO_UNALIGNED_OK_2)
94 # define HEAD2(b,p) (* (lzo_ushortp) &(b[p]))
96 # define HEAD2(b,p) (b[p] ^ ((unsigned)b[p+1]<<8))
98 # define NIL2 SWD_UINT_MAX
104 /* public - "built-in" */
109 /* public - configuration */
111 lzo_uint nice_length;
112 lzo_bool use_best_off;
113 lzo_uint lazy_insert;
115 /* public - output */
120 #if defined(SWD_BEST_OFF)
121 lzo_uint best_off[ SWD_BEST_OFF ];
127 #if defined(SWD_BEST_OFF)
128 lzo_uint best_pos[ SWD_BEST_OFF ];
132 const lzo_bytep dict;
133 const lzo_bytep dict_end;
137 lzo_uint ip; /* input pointer (lookahead) */
138 lzo_uint bp; /* buffer pointer */
139 lzo_uint rp; /* remove pointer */
147 #if defined(__LZO_MMODEL_HUGE)
148 # define A(type, n) ((((n) * sizeof(type)) + 3UL) &~ 3UL)
151 # define O_head3(s) (O_b(s) + A(char, 0UL + SWD_N + SWD_F + SWD_F))
152 # define O_succ3(s) (O_head3(s) + A(swd_uint, 0UL + SWD_HSIZE))
153 # define O_best3(s) (O_succ3(s) + A(swd_uint, 0UL + SWD_N + SWD_F))
154 # define O_llen3(s) (O_best3(s) + A(swd_uint, 0UL + SWD_N + SWD_F))
156 # define O_head2(s) (O_llen3(s) + A(swd_uint, 0UL + SWD_HSIZE))
157 # define O_END(s) (O_head2(s) + A(swd_uint, 0UL + 65536L))
159 # define O_END(s) (O_llen3(s) + A(swd_uint, 0UL + SWD_HSIZE))
162 # define S_DEF(s,type,off) ((type) ((lzo_bytep)s + 0L + sizeof(*s) + off))
163 # define s_b(s) S_DEF(s, lzo_bytep, O_b(s))
164 # define s_head3(s) S_DEF(s, swd_uintp, O_head3(s))
165 # define s_succ3(s) S_DEF(s, swd_uintp, O_succ3(s))
166 # define s_best3(s) S_DEF(s, swd_uintp, O_best3(s))
167 # define s_llen3(s) S_DEF(s, swd_uintp, O_llen3(s))
169 # define s_head2(s) S_DEF(s, swd_uintp, O_head2(s))
172 #elif defined(__LZO_CHECKER)
173 /* malloc arrays of the exact size to detect any overrun */
184 unsigned char b [ SWD_N + SWD_F + SWD_F ];
185 swd_uint head3 [ SWD_HSIZE ];
186 swd_uint succ3 [ SWD_N + SWD_F ];
187 swd_uint best3 [ SWD_N + SWD_F ];
188 swd_uint llen3 [ SWD_HSIZE ];
190 swd_uint head2 [ 65536L ];
195 #define lzo_swd_p lzo_swd_t __LZO_MMODEL *
198 #if defined(__LZO_MMODEL_HUGE)
199 #define SIZEOF_LZO_SWD_T O_END(0)
202 #define s_head3(s) s->head3
203 #define s_succ3(s) s->succ3
204 #define s_best3(s) s->best3
205 #define s_llen3(s) s->llen3
207 #define s_head2(s) s->head2
209 #define SIZEOF_LZO_SWD_T (sizeof(lzo_swd_t))
213 /* Access macro for head3.
214 * head3[key] may be uninitialized, but then its value will never be used.
216 #if defined(__LZO_CHECKER)
217 # define s_get_head3(s,key) \
218 ((s->llen3[key] == 0) ? SWD_UINT_MAX : s_head3(s)[key])
220 # define s_get_head3(s,key) s_head3(s)[key]
224 /***********************************************************************
226 ************************************************************************/
229 void swd_initdict(lzo_swd_p s, const lzo_bytep dict, lzo_uint dict_len)
231 s->dict = s->dict_end = NULL;
234 if (!dict || dict_len <= 0)
238 dict += dict_len - s->n;
243 s->dict_len = dict_len;
244 s->dict_end = dict + dict_len;
245 lzo_memcpy(s_b(s),dict,dict_len);
251 void swd_insertdict(lzo_swd_p s, lzo_uint node, lzo_uint len)
255 s->node_count = s->n - len;
260 key = HEAD3(s_b(s),node);
261 s_succ3(s)[node] = s_get_head3(s,key);
262 s_head3(s)[key] = SWD_UINT(node);
263 s_best3(s)[node] = SWD_UINT(s->f + 1);
265 assert(s_llen3(s)[key] <= SWD_N);
268 key = HEAD2(s_b(s),node);
269 s_head2(s)[key] = SWD_UINT(node);
277 /***********************************************************************
279 ************************************************************************/
282 int swd_init(lzo_swd_p s, const lzo_bytep dict, lzo_uint dict_len)
287 #if defined(__LZO_CHECKER)
288 s->b = malloc(SWD_N + SWD_F + SWD_F);
289 s->head3 = malloc(sizeof(swd_uint) * SWD_HSIZE);
290 s->succ3 = malloc(sizeof(swd_uint) * (SWD_N + SWD_F));
291 s->best3 = malloc(sizeof(swd_uint) * (SWD_N + SWD_F));
292 s->llen3 = malloc(sizeof(swd_uint) * SWD_HSIZE);
294 s->head2 = malloc(sizeof(swd_uint) * 65536L);
300 s->threshold = SWD_THRESHOLD;
303 s->max_chain = SWD_MAX_CHAIN;
304 s->nice_length = SWD_F;
308 s->b_size = s->n + s->f;
310 if (2 * s->f >= s->n || s->b_size + s->f >= SWD_UINT_MAX)
313 LZO_COMPILE_TIME_ASSERT(!(0ul + 2 * SWD_F >= SWD_N))
314 LZO_COMPILE_TIME_ASSERT(!(0ul + SWD_N + SWD_F + SWD_F >= SWD_UINT_MAX))
316 s->b_wrap = s_b(s) + s->b_size;
317 s->node_count = s->n;
319 lzo_memset(s_llen3(s), 0, sizeof(s_llen3(s)[0]) * (lzo_uint)SWD_HSIZE);
322 lzo_memset(s_head2(s), 0xff, sizeof(s_head2(s)[0]) * 65536L);
323 assert(s_head2(s)[0] == NIL2);
325 for (i = 0; i < 65536L; i++)
326 s_head2(s)[i] = NIL2;
331 swd_initdict(s,dict,dict_len);
335 assert(s->ip + s->f <= s->b_size);
337 s->look = (lzo_uint) (s->c->in_end - s->c->ip);
342 lzo_memcpy(&s_b(s)[s->ip],s->c->ip,s->look);
348 while (s->look < s->f)
350 if ((c = getbyte(*(s->c))) < 0)
352 s_b(s)[s->ip] = LZO_BYTE(c);
357 if (s->ip == s->b_size)
360 if (s->look >= 2 && s->dict_len > 0)
361 swd_insertdict(s,0,s->dict_len);
364 if (s->rp >= s->node_count)
365 s->rp -= s->node_count;
367 s->rp += s->b_size - s->node_count;
369 #if defined(__LZO_CHECKER)
370 /* initialize memory for the first few HEAD3 (if s->ip is not far
371 * enough ahead to do this job for us). The value doesn't matter. */
373 lzo_memset(&s_b(s)[s->bp+s->look],0,3);
383 void swd_exit(lzo_swd_p s)
385 #if defined(__LZO_CHECKER)
386 /* free in reverse order of allocations */
388 free(s->head2); s->head2 = NULL;
390 free(s->llen3); s->llen3 = NULL;
391 free(s->best3); s->best3 = NULL;
392 free(s->succ3); s->succ3 = NULL;
393 free(s->head3); s->head3 = NULL;
394 free(s->b); s->b = NULL;
401 #define swd_pos2off(s,pos) \
402 (s->bp > (pos) ? s->bp - (pos) : s->b_size - ((pos) - s->bp))
405 /***********************************************************************
407 ************************************************************************/
410 void swd_getbyte(lzo_swd_p s)
414 if ((c = getbyte(*(s->c))) < 0)
418 #if defined(__LZO_CHECKER)
419 /* initialize memory - value doesn't matter */
422 s->b_wrap[s->ip] = 0;
427 s_b(s)[s->ip] = LZO_BYTE(c);
429 s->b_wrap[s->ip] = LZO_BYTE(c);
431 if (++s->ip == s->b_size)
433 if (++s->bp == s->b_size)
435 if (++s->rp == s->b_size)
440 /***********************************************************************
441 // remove node from lists
442 ************************************************************************/
445 void swd_remove_node(lzo_swd_p s, lzo_uint node)
447 if (s->node_count == 0)
452 if (s->first_rp != LZO_UINT_MAX)
454 if (node != s->first_rp)
455 printf("Remove %5u: %5u %5u %5u %5u %6u %6u\n",
456 node, s->rp, s->ip, s->bp, s->first_rp,
457 s->ip - node, s->ip - s->bp);
458 assert(node == s->first_rp);
459 s->first_rp = LZO_UINT_MAX;
463 key = HEAD3(s_b(s),node);
464 assert(s_llen3(s)[key] > 0);
468 key = HEAD2(s_b(s),node);
469 assert(s_head2(s)[key] != NIL2);
470 if ((lzo_uint) s_head2(s)[key] == node)
471 s_head2(s)[key] = NIL2;
479 /***********************************************************************
481 ************************************************************************/
484 void swd_accept(lzo_swd_p s, lzo_uint n)
486 assert(n <= s->look);
492 swd_remove_node(s,s->rp);
494 /* add bp into HEAD3 */
495 key = HEAD3(s_b(s),s->bp);
496 s_succ3(s)[s->bp] = s_get_head3(s,key);
497 s_head3(s)[key] = SWD_UINT(s->bp);
498 s_best3(s)[s->bp] = SWD_UINT(s->f + 1);
500 assert(s_llen3(s)[key] <= SWD_N);
503 /* add bp into HEAD2 */
504 key = HEAD2(s_b(s),s->bp);
505 s_head2(s)[key] = SWD_UINT(s->bp);
513 /***********************************************************************
515 ************************************************************************/
518 void swd_search(lzo_swd_p s, lzo_uint node, lzo_uint cnt)
523 lzo_uint m_len = s->m_len;
524 const lzo_bytep b = s_b(s);
525 const lzo_bytep bp = s_b(s) + s->bp;
526 const lzo_bytep bx = s_b(s) + s->bp + s->look;
527 unsigned char scan_end1;
529 assert(s->m_len > 0);
531 scan_end1 = bp[m_len - 1];
532 for ( ; cnt-- > 0; node = s_succ3(s)[node])
538 assert(m_len < s->look);
542 p2[m_len - 1] == scan_end1 &&
543 p2[m_len] == p1[m_len] &&
549 assert(lzo_memcmp(bp,&b[node],3) == 0);
551 #if 0 && defined(LZO_UNALIGNED_OK_4)
553 while (p1 < px && * (const lzo_uint32p) p1 == * (const lzo_uint32p) p2)
555 while (p1 < px && *p1 == *p2)
559 do {} while (++p1 < px && *p1 == *++p2);
564 if (lzo_memcmp(bp,&b[node],i) != 0)
565 printf("%5ld %5ld %02x%02x %02x%02x\n",
566 (long)s->bp, (long) node,
567 bp[0], bp[1], b[node], b[node+1]);
569 assert(lzo_memcmp(bp,&b[node],i) == 0);
571 #if defined(SWD_BEST_OFF)
572 if (i < SWD_BEST_OFF)
574 if (s->best_pos[i] == 0)
575 s->best_pos[i] = node + 1;
580 s->m_len = m_len = i;
582 if (m_len == s->look)
584 if (m_len >= s->nice_length)
586 if (m_len > (lzo_uint) s_best3(s)[node])
588 scan_end1 = bp[m_len - 1];
595 /***********************************************************************
597 ************************************************************************/
602 lzo_bool swd_search2(lzo_swd_p s)
606 assert(s->look >= 2);
607 assert(s->m_len > 0);
609 key = s_head2(s)[ HEAD2(s_b(s),s->bp) ];
613 if (lzo_memcmp(&s_b(s)[s->bp],&s_b(s)[key],2) != 0)
614 printf("%5ld %5ld %02x%02x %02x%02x\n", (long)s->bp, (long)key,
615 s_b(s)[s->bp], s_b(s)[s->bp+1], s_b(s)[key], s_b(s)[key+1]);
617 assert(lzo_memcmp(&s_b(s)[s->bp],&s_b(s)[key],2) == 0);
618 #if defined(SWD_BEST_OFF)
619 if (s->best_pos[2] == 0)
620 s->best_pos[2] = key + 1;
634 /***********************************************************************
636 ************************************************************************/
639 void swd_findbest(lzo_swd_p s)
645 assert(s->m_len > 0);
647 /* get current head, add bp into HEAD3 */
648 key = HEAD3(s_b(s),s->bp);
649 node = s_succ3(s)[s->bp] = s_get_head3(s,key);
650 cnt = s_llen3(s)[key]++;
651 assert(s_llen3(s)[key] <= SWD_N + SWD_F);
652 if (cnt > s->max_chain && s->max_chain > 0)
654 s_head3(s)[key] = SWD_UINT(s->bp);
656 s->b_char = s_b(s)[s->bp];
658 if (s->m_len >= s->look)
663 s_best3(s)[s->bp] = SWD_UINT(s->f + 1);
671 swd_search(s,node,cnt);
673 s->m_off = swd_pos2off(s,s->m_pos);
674 s_best3(s)[s->bp] = SWD_UINT(s->m_len);
676 #if defined(SWD_BEST_OFF)
680 for (i = 2; i < SWD_BEST_OFF; i++)
681 if (s->best_pos[i] > 0)
682 s->best_off[i] = swd_pos2off(s,s->best_pos[i]-1);
689 swd_remove_node(s,s->rp);
692 /* add bp into HEAD2 */
693 key = HEAD2(s_b(s),s->bp);
694 s_head2(s)[key] = SWD_UINT(s->bp);