_obsolete_dont_use/zlib.d

   1 /* Invisible Vector Library
   2  * coded by Ketmar // Invisible Vector <ketmar@ketmar.no-ip.org>
   3  * Understanding is not required. Only obedience.
   4  *
   5  * This program is free software: you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation, version 3 of the License ONLY.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  16  */
  17 // severely outdated ZLib streams
  18 // use iv.vfs instead
  19 module iv.zlib /*is aliced*/;
  20 import iv.alice;
  21
  22
  23 // ////////////////////////////////////////////////////////////////////////// //
  24 //version=iv_zlib_malloc_compress;
  25 //version=iv_zlib_malloc_decompress;
  26
  27
  28 // ////////////////////////////////////////////////////////////////////////// //
  29 enum ZlibMax = 666; // special constant for compression level
  30
  31
  32 // ////////////////////////////////////////////////////////////////////////// //
  33 /**
  34  * Errors throw a ZlibException.
  35  */
  36 class ZlibException : Exception {
  37   this (int errnum, string file=__FILE__, usize line=__LINE__, Throwable next=null) @safe pure {
  38     import etc.c.zlib;
  39     string msg;
  40     switch (errnum) {
  41       case Z_OK: msg = "no error"; break;
  42       case Z_STREAM_END: msg = "stream end"; break;
  43       case Z_NEED_DICT: msg = "need dict"; break;
  44       case Z_ERRNO: msg = "errno"; break;
  45       case Z_STREAM_ERROR: msg = "stream error"; break;
  46       case Z_DATA_ERROR: msg = "data error"; break;
  47       case Z_MEM_ERROR: msg = "mem error"; break;
  48       case Z_BUF_ERROR: msg = "buf error"; break;
  49       case Z_VERSION_ERROR: msg = "version error"; break;
  50       default:
  51         // üÈ, ÔÁÞÁÎËÁ-ÒÏÓÔÏ×ÞÁÎËÁ,
  52         // îÁÛÁ ÇÏÒÄÏÓÔØ É ËÒÁÓÁ,
  53         // ëÏÎÁÒÍÅÊÓËÁÑ ÔÁÞÁÎËÁ,
  54         // ÷ÓÅ ÞÅÔÙÒÅ ËÏÌÅÓÁ.
  55         import std.conv : to;
  56         msg = "unknown error with code "~to!string(errnum);
  57         break;
  58     }
  59     super(msg, file, line, next);
  60   }
  61 }
  62
  63
  64 // ////////////////////////////////////////////////////////////////////////// //
  65 /**
  66  * Compute the Adler32 checksum of the data in buf[]. adler is the starting
  67  * value when computing a cumulative checksum.
  68  */
  69 uint adler32 (const(void)[] buf, uint prevadler=0) {
  70   import etc.c.zlib : adler32;
  71   while (buf.length > 0) {
  72     uint len = (buf.length > 0xffff_0000u ? 0xffff_0000u : cast(uint)buf.length);
  73     prevadler = /*etc.c.zlib.*/adler32(prevadler, cast(ubyte*)buf.ptr, len);
  74     buf = buf[len..$];
  75   }
  76   return prevadler;
  77 }
  78
  79
  80 /**
  81  * Compute the CRC32 checksum of the data in buf[]. crc is the starting value
  82  * when computing a cumulative checksum.
  83  */
  84 uint crc32 (const(void)[] buf, uint prevcrc=0) {
  85   import etc.c.zlib : crc32;
  86   while (buf.length > 0) {
  87     uint len = (buf.length > 0xffff_0000u ? 0xffff_0000u : cast(uint)buf.length);
  88     prevcrc = /*etc.c.zlib.*/crc32(prevcrc, cast(ubyte*)buf.ptr, len);
  89     buf = buf[len..$];
  90   }
  91   return prevcrc;
  92 }
  93
  94
  95 // ////////////////////////////////////////////////////////////////////////// //
  96 import std.range : isInputRange, isOutputRange;
  97
  98
  99 /// the header format the compressed stream is wrapped in
 100 enum ZHeader {
 101   deflate, /// a standard zlib header
 102   gzip, /// a gzip file format header
 103   detect /// used when decompressing: try to automatically detect the stream format by looking at the data
 104 }
 105
 106
 107 /**
 108  * Compresses the data from `ri` to `ro`.
 109  *
 110  * Params:
 111  *   ri = finite input range with byte-sized elements
 112  *   ro = output range that can accept ubyte or ubyte[]
 113  *   level = compression level; -1: default, ZlibMax: maximum, [0..9]
 114  *   header = compressed stream header (deflate or gzip)
 115  */
 116 void zcompress(RI, RO) (auto ref RI ri, auto ref RO ro, int level=ZlibMax, ZHeader header=ZHeader.deflate)
 117 if (isInputRange!RI && (isOutputRange!(RO, ubyte) || isOutputRange!(RO, ubyte[])))
 118 in {
 119   import std.range : isInfinite, ElementType;
 120   // check range types
 121   static assert(!isInfinite!RI, "ri should be finite range");
 122   static assert((ElementType!RI).sizeof == 1, "ri should be byte range");
 123   // check args
 124   assert(level == ZlibMax || (-1 <= level && level <= 9));
 125   assert(header == ZHeader.deflate || header == ZHeader.gzip);
 126 }
 127 body {
 128   import etc.c.zlib;
 129   version(test_zlib_log) import core.stdc.stdio : printf;
 130
 131   if (ri.empty) return; // nothing to do
 132
 133   z_stream zs;
 134   version(iv_zlib_malloc_compress) {
 135     enum ibuflen = 64*1024;
 136     enum obuflen = 64*1024;
 137     ubyte* ibuf = null, obuf = null;
 138     enum ibufptr = "ibuf";
 139     enum obufptr = "obuf";
 140   } else {
 141     enum ibuflen = 8*1024;
 142     enum obuflen = 8*1024;
 143     ubyte[ibuflen] ibuf = void;
 144     ubyte[obuflen] obuf = void;
 145     enum ibufptr = "ibuf.ptr";
 146     enum obufptr = "obuf.ptr";
 147   }
 148
 149   void prepareOBuf() () {
 150     zs.next_out = cast(typeof(zs.next_out))mixin(obufptr);
 151     zs.avail_out = cast(uint)obuflen;
 152   }
 153
 154   void writeOBuf() () {
 155     if (zs.avail_out < obuflen) {
 156       version(test_zlib_log) printf("writing %u packed bytes\n", cast(uint)(obuflen-zs.avail_out));
 157       static if (is(typeof((inout int=0) {
 158         auto r = RO.init;
 159         ubyte[2] b;
 160         r.put(b);
 161       }))) {
 162         // can put slices
 163         ro.put(obuf[0..obuflen-zs.avail_out]);
 164       } else {
 165         foreach (immutable pos; 0..obuflen-zs.avail_out) ro.put(obuf[pos]);
 166       }
 167     } else {
 168       version(test_zlib_log) printf("nothing to write\n");
 169     }
 170   }
 171
 172   // init zlib stream
 173   if (level < 0) level = 6;
 174   else if (level == 0) level = 1;
 175
 176   int err = deflateInit2(&zs,
 177               (level < 9 ? level : 9),
 178               Z_DEFLATED,
 179               15+(header == ZHeader.gzip ? 16 : 0),
 180               (level > 9 ? 9 : 8),
 181               Z_DEFAULT_STRATEGY);
 182   if (err) throw new ZlibException(err);
 183   scope(exit) deflateEnd(&zs);
 184
 185   // allocate buffers
 186   version(iv_zlib_malloc_compress) {
 187     import core.exception : onOutOfMemoryError;
 188     import core.stdc.stdlib : malloc, free;
 189     if ((ibuf = cast(ubyte*)malloc(ibuflen)) is null) onOutOfMemoryError();
 190     if ((obuf = cast(ubyte*)malloc(obuflen)) is null) { free(ibuf); onOutOfMemoryError(); }
 191     scope(exit) { free(obuf); free(ibuf); }
 192   }
 193
 194   // compress stream
 195   while (!ri.empty) {
 196     // fill input buffer
 197     zs.avail_in = 0;
 198     zs.next_in = cast(typeof(zs.next_in))mixin(ibufptr);
 199     while (zs.avail_in < cast(uint)ibuflen && !ri.empty) {
 200       // use `.ptr` to avoid range checking on array
 201       mixin(ibufptr)[zs.avail_in++] = cast(ubyte)ri.front;
 202       ri.popFront();
 203     }
 204     version(test_zlib_log) printf("read %u unpacked bytes\n", cast(uint)zs.avail_in);
 205     // process all data in input buffer
 206     while (zs.avail_in > 0) {
 207       prepareOBuf();
 208       err = deflate(&zs, Z_NO_FLUSH);
 209       if (err != Z_STREAM_END && err != Z_OK) throw new ZlibException(err);
 210       if (zs.avail_out == cast(uint)obuflen) {
 211         if (zs.avail_in != 0) throw new ZlibException(Z_BUF_ERROR); // something went wrong here
 212         break;
 213       }
 214       version(test_zlib_log) printf("got %u packed bytes; %u unpacked bytes left\n", cast(uint)(obuflen-zs.avail_out), cast(uint)zs.avail_in);
 215       writeOBuf();
 216     }
 217   }
 218
 219   // stream compressed, flush zstream
 220   do {
 221     zs.avail_in = 0;
 222     prepareOBuf();
 223     err = deflate(&zs, Z_FINISH);
 224     if (err == Z_OK) {
 225       version(test_zlib_log) printf("Z_OK: got %u packed bytes\n", cast(uint)(obuflen-zs.avail_out));
 226       if (zs.avail_out == cast(uint)obuflen) throw new ZlibException(Z_BUF_ERROR); // something went wrong here
 227       writeOBuf();
 228     }
 229   } while (err == Z_OK);
 230   // succesfully flushed?
 231   if (err != Z_STREAM_END) {
 232     if (err == Z_OK) err = Z_BUF_ERROR; // out of output space; this is fatal for now
 233     throw new ZlibException(err);
 234   }
 235   writeOBuf();
 236 }
 237
 238
 239 version(test_zlib)
 240 unittest {
 241   import iv.stream;
 242   import std.stdio;
 243   auto fi = File("iv.zlib.d", "r");
 244   auto fo = File("ztmp.bin.gz", "w");
 245   writeln("compressing...");
 246   zcompress(streamAsRange!"r"(fi), streamAsRange(fo), ZlibMax, ZHeader.gzip);
 247   writeln("done: ", fi.size, " -> ", fo.size);
 248 }
 249
 250
 251 // ////////////////////////////////////////////////////////////////////////// //
 252 /**
 253  * Decompresses the data from `ri` to `ro`.
 254  *
 255  * Params:
 256  *   ri = finite input range with byte-sized elements
 257  *   ro = output range that can accept ubyte or ubyte[]
 258  *   format = compressed stream format (deflate or gzip)
 259  */
 260 void zdecompress(RI, RO) (auto ref RI ri, auto ref RO ro, ZHeader format=ZHeader.detect)
 261 if (isInputRange!RI && (isOutputRange!(RO, ubyte) || isOutputRange!(RO, ubyte[])))
 262 in {
 263   import std.range : isInfinite, ElementType;
 264   // check range types
 265   static assert(!isInfinite!RI, "ri should be finite range");
 266   static assert((ElementType!RI).sizeof == 1, "ri should be byte range");
 267   // check args
 268   assert(format == ZHeader.deflate || format == ZHeader.gzip || format == ZHeader.detect);
 269 }
 270 body {
 271   import etc.c.zlib;
 272   version(test_zlib_log) import core.stdc.stdio : printf;
 273
 274   if (ri.empty) return;
 275
 276   z_stream zs;
 277   version(iv_zlib_malloc_decompress) {
 278     enum ibuflen = 64*1024;
 279     enum obuflen = 64*1024;
 280     ubyte* ibuf = null, obuf = null;
 281     enum ibufptr = "ibuf";
 282     enum obufptr = "obuf";
 283   } else {
 284     enum ibuflen = 8*1024;
 285     enum obuflen = 8*1024;
 286     ubyte[ibuflen] ibuf = void;
 287     ubyte[obuflen] obuf = void;
 288     enum ibufptr = "ibuf.ptr";
 289     enum obufptr = "obuf.ptr";
 290   }
 291
 292   void prepareOBuf() () {
 293     zs.next_out = cast(typeof(zs.next_out))mixin(obufptr);
 294     zs.avail_out = cast(uint)obuflen;
 295   }
 296
 297   void writeOBuf() () {
 298     if (zs.avail_out < obuflen) {
 299       version(test_zlib_log) printf("writing %u packed bytes\n", cast(uint)(obuflen-zs.avail_out));
 300       static if (is(typeof((inout int=0) {
 301         auto r = RO.init;
 302         ubyte[2] b;
 303         r.put(b);
 304       }))) {
 305         // can put slices
 306         ro.put(obuf[0..obuflen-zs.avail_out]);
 307       } else {
 308         foreach (immutable pos; 0..obuflen-zs.avail_out) ro.put(obuf[pos]);
 309       }
 310     } else {
 311       version(test_zlib_log) printf("nothing to write\n");
 312     }
 313   }
 314
 315   // init zlib stream
 316   int windowBits = 15;
 317   switch (format) with (ZHeader) {
 318     case gzip: windowBits += 16; break;
 319     case detect: windowBits += 32; break;
 320     default:
 321   }
 322
 323   int err = inflateInit2(&zs, windowBits);
 324   if (err) throw new ZlibException(err);
 325   scope(exit) inflateEnd(&zs);
 326
 327   // allocate buffers
 328   version(iv_zlib_malloc_decompress) {
 329     import core.exception : onOutOfMemoryError;
 330     import core.stdc.stdlib : malloc, free;
 331     if ((ibuf = cast(ubyte*)malloc(ibuflen)) is null) onOutOfMemoryError();
 332     if ((obuf = cast(ubyte*)malloc(obuflen)) is null) { free(ibuf); onOutOfMemoryError(); }
 333     scope(exit) { free(obuf); free(ibuf); }
 334   }
 335
 336   // decompress stream
 337   bool streamComplete = false;
 338   while (!streamComplete && !ri.empty) {
 339     // fill input buffer
 340     zs.avail_in = 0;
 341     zs.next_in = cast(typeof(zs.next_in))mixin(ibufptr);
 342     while (zs.avail_in < cast(uint)ibuflen && !ri.empty) {
 343       // use `.ptr` to avoid range checking on array
 344       mixin(ibufptr)[zs.avail_in++] = cast(ubyte)ri.front;
 345       ri.popFront();
 346     }
 347     version(test_zlib_log) printf("read %u packed bytes\n", cast(uint)zs.avail_in);
 348     // process all data in input buffer
 349     while (zs.avail_in > 0) {
 350       prepareOBuf();
 351       //err = inflate(&zs, (lastChunk ? Z_NO_FLUSH : Z_FINISH));
 352       err = inflate(&zs, Z_NO_FLUSH);
 353       if (err != Z_STREAM_END && err != Z_OK) throw new ZlibException(err);
 354       if (err == Z_STREAM_END) {
 355         assert(zs.avail_in == 0);
 356         streamComplete = true;
 357       }
 358       if (zs.avail_out == cast(uint)obuflen) {
 359         if (zs.avail_in != 0) throw new ZlibException(Z_BUF_ERROR); // something went wrong here
 360         break;
 361       }
 362       version(test_zlib_log) printf("got %u unpacked bytes; %u packed bytes left\n", cast(uint)(obuflen-zs.avail_out), cast(uint)zs.avail_in);
 363       writeOBuf();
 364     }
 365   }
 366
 367   // finish him!
 368   if (!streamComplete) {
 369     zs.avail_in = 0;
 370     prepareOBuf();
 371     err = inflate(&zs, Z_FINISH);
 372     // succesfully flushed?
 373     if (err != Z_STREAM_END) {
 374       if (err == Z_OK) err = Z_BUF_ERROR; // out of output space; this is fatal for now
 375       throw new ZlibException(err);
 376     }
 377     writeOBuf();
 378   }
 379 }
 380
 381
 382 version(test_zlib)
 383 unittest {
 384   import iv.stream;
 385   import std.stdio;
 386   auto fi = File("ztmp.bin.gz", "r");
 387   auto fo = File("ztmp.bin", "w");
 388   writeln("decompressing...");
 389   zdecompress(streamAsRange!"r"(fi), streamAsRange!"w"(fo));
 390   writeln("done: ", fi.size, " -> ", fo.size);
 391 }