lzo/prepcore.c

   1 /* ----------------------------------------------------------------------- *
   2  *
   3  *   Copyright 2009 Intel Corporation; author: H. Peter Anvin
   4  *
   5  *   This program is free software; you can redistribute it and/or modify
   6  *   it under the terms of the GNU General Public License as published by
   7  *   the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
   8  *   Boston MA 02110-1301, USA; either version 2 of the License, or
   9  *   (at your option) any later version; incorporated herein by reference.
  10  *
  11  * ----------------------------------------------------------------------- */
  12
  13 /*
  14    This file is based in part on:
  15
  16    precomp2.c -- example program: how to generate pre-compressed data
  17
  18    This file is part of the LZO real-time data compression library.
  19
  20    Copyright (C) 2008 Markus Franz Xaver Johannes Oberhumer
  21    Copyright (C) 2007 Markus Franz Xaver Johannes Oberhumer
  22    Copyright (C) 2006 Markus Franz Xaver Johannes Oberhumer
  23    Copyright (C) 2005 Markus Franz Xaver Johannes Oberhumer
  24    Copyright (C) 2004 Markus Franz Xaver Johannes Oberhumer
  25    Copyright (C) 2003 Markus Franz Xaver Johannes Oberhumer
  26    Copyright (C) 2002 Markus Franz Xaver Johannes Oberhumer
  27    Copyright (C) 2001 Markus Franz Xaver Johannes Oberhumer
  28    Copyright (C) 2000 Markus Franz Xaver Johannes Oberhumer
  29    Copyright (C) 1999 Markus Franz Xaver Johannes Oberhumer
  30    Copyright (C) 1998 Markus Franz Xaver Johannes Oberhumer
  31    Copyright (C) 1997 Markus Franz Xaver Johannes Oberhumer
  32    Copyright (C) 1996 Markus Franz Xaver Johannes Oberhumer
  33    All Rights Reserved.
  34
  35    The LZO library is free software; you can redistribute it and/or
  36    modify it under the terms of the GNU General Public License as
  37    published by the Free Software Foundation; either version 2 of
  38    the License, or (at your option) any later version.
  39
  40    The LZO library is distributed in the hope that it will be useful,
  41    but WITHOUT ANY WARRANTY; without even the implied warranty of
  42    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  43    GNU General Public License for more details.
  44
  45    You should have received a copy of the GNU General Public License
  46    along with the LZO library; see the file COPYING.
  47    If not, write to the Free Software Foundation, Inc.,
  48    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  49
  50    Markus F.X.J. Oberhumer
  51    <markus@oberhumer.com>
  52    http://www.oberhumer.com/opensource/lzo/
  53  */
  54
  55 #include "lzo/lzoconf.h"
  56 #include "lzo/lzo1x.h"
  57
  58 LZO_EXTERN(int)
  59 lzo1x_999_compress_internal(const lzo_bytep in, lzo_uint in_len,
  60                             lzo_bytep out, lzo_uintp out_len,
  61                             lzo_voidp wrkmem,
  62                             const lzo_bytep dict, lzo_uint dict_len,
  63                             lzo_callback_p cb,
  64                             int try_lazy,
  65                             lzo_uint good_length,
  66                             lzo_uint max_lazy,
  67                             lzo_uint nice_length,
  68                             lzo_uint max_chain, lzo_uint32 flags);
  69
  70 LZO_EXTERN(int)
  71 lzo1y_999_compress_internal(const lzo_bytep in, lzo_uint in_len,
  72                             lzo_bytep out, lzo_uintp out_len,
  73                             lzo_voidp wrkmem,
  74                             const lzo_bytep dict, lzo_uint dict_len,
  75                             lzo_callback_p cb,
  76                             int try_lazy,
  77                             lzo_uint good_length,
  78                             lzo_uint max_lazy,
  79                             lzo_uint nice_length,
  80                             lzo_uint max_chain, lzo_uint32 flags);
  81
  82 #define PARANOID 1
  83
  84 #include <assert.h>
  85 #include <ctype.h>
  86 #include <errno.h>
  87 #include <inttypes.h>
  88 #include <stdarg.h>
  89 #include <stddef.h>
  90 #include <stdlib.h>
  91 #include <stdio.h>
  92 #include <string.h>
  93 #include <time.h>
  94
  95 #ifdef __GNUC__
  96 # define noreturn void __attribute__((noreturn))
  97 #else
  98 # define noreturn void
  99 #endif
 100
 101 struct prefix {
 102     uint32_t pfx_start;
 103     uint32_t pfx_compressed;
 104     uint32_t pfx_cdatalen;
 105     uint32_t pfx_checksum;
 106     uint32_t pfx_maxlma;
 107 };
 108
 109 static inline uint32_t get_32(const uint32_t * p)
 110 {
 111 #if defined(__i386__) || defined(__x86_64__)
 112     /* Littleendian and unaligned-capable */
 113     return *p;
 114 #else
 115     const uint8_t *pp = (const uint8_t *)p;
 116     return (uint32_t) pp[0] + ((uint32_t) pp[1] << 8) +
 117         ((uint32_t) pp[2] << 16) + ((uint32_t) pp[3] << 24);
 118 #endif
 119 }
 120
 121 static inline void set_32(uint32_t * p, uint32_t v)
 122 {
 123 #if defined(__i386__) || defined(__x86_64__)
 124     /* Littleendian and unaligned-capable */
 125     *p = v;
 126 #else
 127     uint8_t *pp = (uint8_t *) p;
 128     pp[0] = (v & 0xff);
 129     pp[1] = ((v >> 8) & 0xff);
 130     pp[2] = ((v >> 16) & 0xff);
 131     pp[3] = ((v >> 24) & 0xff);
 132 #endif
 133 }
 134
 135 const char *progname = NULL;
 136 const char *in_name  = NULL;
 137 const char *out_name = NULL;
 138
 139 static noreturn error(const char *fmt, ...)
 140 {
 141     va_list ap;
 142
 143     va_start(ap, fmt);
 144     fprintf(stderr, "%s: ", progname);
 145     if (in_name)
 146         fprintf(stderr, "%s: ", in_name);
 147     vfprintf(stderr, fmt, ap);
 148     fputc('\n', stderr);
 149     va_end(ap);
 150
 151     exit(1);
 152 }
 153
 154 static void *xzalloc(size_t n)
 155 {
 156     void *p = calloc(n, 1);
 157     if (!p)
 158         error("out of memory");
 159     return p;
 160 }
 161
 162 /*************************************************************************
 163 //
 164 **************************************************************************/
 165
 166 int main(int argc, char *argv[])
 167 {
 168     int r;
 169     int lazy;
 170     const int max_try_lazy = 5;
 171     const lzo_uint big = 65536L;        /* can result in very slow compression */
 172     const lzo_uint32 flags = 0x1;
 173
 174     lzo_bytep in;
 175     lzo_bytep infile;
 176     lzo_uint in_len, infile_len, start, offset, soff;
 177
 178     lzo_bytep out;
 179     lzo_uint out_bufsize;
 180     lzo_uint out_len = 0;
 181     lzo_uint outfile_len;
 182
 183     lzo_bytep test;
 184
 185     lzo_byte wrkmem[LZO1X_999_MEM_COMPRESS];
 186
 187     lzo_uint best_len;
 188     int best_lazy = -1;
 189
 190     lzo_uint orig_len;
 191     lzo_uint32 uncompressed_checksum;
 192     lzo_uint32 compressed_checksum;
 193
 194     FILE *f;
 195     long l;
 196
 197     struct prefix *prefix;
 198
 199     progname = argv[0];
 200     if (argc != 3) {
 201         fprintf(stderr, "Usage: %s file output-file\n", progname);
 202         exit(1);
 203     }
 204     in_name = argv[1];
 205     if (argc > 2)
 206         out_name = argv[2];
 207
 208 /*
 209  * Step 1: initialize the LZO library
 210  */
 211     if (lzo_init() != LZO_E_OK)
 212         error("internal error - lzo_init() failed!");
 213
 214 /*
 215  * Step 3: open the input file
 216  */
 217     f = fopen(in_name, "rb");
 218     if (!f)
 219         error("cannot open file: %s", strerror(errno));
 220
 221     fseek(f, 0, SEEK_END);
 222     l = ftell(f);
 223     fseek(f, 0, SEEK_SET);
 224     if (l <= 0) {
 225         error("empty file", progname, in_name);
 226         fclose(f);
 227         exit(1);
 228     }
 229     infile_len = (lzo_uint) l;
 230     out_bufsize = infile_len + infile_len / 16 + 64 + 3 + 2048;
 231
 232 /*
 233  * Step 4: allocate compression buffers and read the file
 234  */
 235     infile = xzalloc(infile_len);
 236     out = xzalloc(out_bufsize);
 237     infile_len = fread(infile, 1, infile_len, f);
 238     fclose(f);
 239
 240 /*
 241  * Select the portion which is for compression...
 242  */
 243     prefix = (struct prefix *)infile;
 244     start = get_32(&prefix->pfx_start);
 245     offset = get_32(&prefix->pfx_compressed);
 246     in = infile + offset;
 247     in_len = infile_len - offset;
 248     best_len = in_len;
 249
 250 /*
 251  * Step 5: compute a checksum of the uncompressed data
 252  */
 253     uncompressed_checksum = lzo_adler32(0, NULL, 0);
 254     uncompressed_checksum = lzo_adler32(uncompressed_checksum, in, in_len);
 255
 256 /*
 257  * Step 6a: compress from `in' to `out' with LZO1X-999
 258  */
 259     for (lazy = 0; lazy <= max_try_lazy; lazy++) {
 260         out_len = out_bufsize;
 261         r = lzo1x_999_compress_internal(in, in_len, out, &out_len, wrkmem,
 262                                         NULL, 0, 0,
 263                                         lazy, big, big, big, big, flags);
 264         if (r != LZO_E_OK)
 265             /* this should NEVER happen */
 266             error("internal error - compression failed: %d", r);
 267
 268         if (out_len < best_len) {
 269             best_len = out_len;
 270             best_lazy = lazy;
 271         }
 272     }
 273
 274 /*
 275  * Step 7: check if compressible
 276  */
 277     if (best_len >= in_len) {
 278         fprintf(stderr, "%s: %s: this file contains incompressible data.",
 279                 progname, in_name);
 280         /* do it anyway */
 281     }
 282
 283 /*
 284  * Step 8: compress data again using the best compressor found
 285  */
 286     out_len = out_bufsize;
 287     r = lzo1x_999_compress_internal(in, in_len, out, &out_len, wrkmem,
 288                                     NULL, 0, 0,
 289                                     best_lazy, big, big, big, big, flags);
 290     assert(r == LZO_E_OK);
 291     assert(out_len == best_len);
 292
 293 /*
 294  * Step 9: optimize compressed data (compressed data is in `out' buffer)
 295  */
 296 #if 1
 297     /* Optimization does not require any data in the buffer that will
 298      * hold the uncompressed data. To prove this, we clear the buffer.
 299      */
 300     memset(in, 0, in_len);
 301 #endif
 302
 303     orig_len = in_len;
 304     r = lzo1x_optimize(out, out_len, in, &orig_len, NULL);
 305     if (r != LZO_E_OK || orig_len != in_len) {
 306         /* this should NEVER happen */
 307         error("internal error - optimization failed: %d", r);
 308     }
 309
 310 /*
 311  * Step 10: compute a checksum of the compressed data
 312  */
 313     compressed_checksum = lzo_adler32(0, NULL, 0);
 314     compressed_checksum = lzo_adler32(compressed_checksum, out, out_len);
 315
 316 /*
 317  * Step 11: write compressed data to a file
 318  */
 319     /* Make sure we have up to 2048 bytes of zero after the output */
 320     memset(out + out_len, 0, 2048);
 321
 322     outfile_len = out_len;
 323
 324     soff = get_32(&prefix->pfx_cdatalen);
 325     set_32((uint32_t *) (infile + soff), out_len);
 326
 327     soff = get_32(&prefix->pfx_checksum);
 328     if (soff) {
 329         /* ISOLINUX padding and checksumming */
 330         uint32_t csum = 0;
 331         unsigned int ptr;
 332         outfile_len =
 333             ((offset - start + out_len + 2047) & ~2047) - (offset - start);
 334         for (ptr = 64; ptr < offset; ptr += 4)
 335             csum += get_32((uint32_t *) (infile + ptr));
 336         for (ptr = 0; ptr < outfile_len; ptr += 4)
 337             csum += get_32((uint32_t *) (out + ptr));
 338
 339         set_32((uint32_t *) (infile + soff), offset - start + outfile_len);
 340         set_32((uint32_t *) (infile + soff + 4), csum);
 341     }
 342
 343     if (offset+outfile_len > get_32(&prefix->pfx_maxlma))
 344         error("output too big (%lu, max %lu)",
 345               (unsigned long)offset+outfile_len,
 346               (unsigned long)get_32(&prefix->pfx_maxlma));
 347
 348     f = fopen(out_name, "wb");
 349     if (!f)
 350         error("cannot open output file %s: %s", out_name, strerror(errno));
 351
 352     if (fwrite(infile + start, 1, offset - start, f) != offset - start ||
 353         fwrite(out, 1, outfile_len, f) != outfile_len || fclose(f))
 354         error("write error");
 355
 356 /*
 357  * Step 12: verify decompression
 358  */
 359 #ifdef PARANOID
 360     orig_len = in_len * 2;
 361     test = xzalloc(orig_len);
 362     r = lzo1x_decompress_safe(out, out_len, test, &orig_len, NULL);
 363
 364     if (r != LZO_E_OK || orig_len != in_len) {
 365         /* this should NEVER happen */
 366         error("internal error - decompression failed: %d", r);
 367     }
 368
 369     if (memcmp(test, in, in_len)) {
 370         /* this should NEVER happen */
 371         error("internal error - decompression data error");
 372     }
 373
 374     /* Now you could also verify decompression under similar conditions as in
 375      * your application, e.g. overlapping assembler decompression etc.
 376      */
 377
 378     free(test);
 379 #endif
 380
 381     free(infile);
 382     free(out);
 383
 384     return 0;
 385 }