dlzma/test/dlzmatest_vanilla.d

   1 /* converted by Ketmar // Invisible Vector <ketmar@ketmar.no-ip.org>
   2  * Understanding is not required. Only obedience.
   3  *
   4  * This program is free software. It comes without any warranty, to
   5  * the extent permitted by applicable law. You can redistribute it
   6  * and/or modify it under the terms of the Do What The Fuck You Want
   7  * To Public License, Version 2, as published by Sam Hocevar. See
   8  * http://www.wtfpl.net/txt/copying/ for more details.
   9  */
  10 module dlzmatest_vanilla;
  11
  12 import std.digest.ripemd;
  13 import std.stdio;
  14
  15 import iv.dlzma;
  16
  17
  18 // ////////////////////////////////////////////////////////////////////////// //
  19 /// convert integral number to number with commas
  20 char[] intWithCommas(T) (char[] dest, T nn, char comma=',') if (__traits(isIntegral, T)) {
  21   static if (__traits(isUnsigned, T)) {
  22     enum neg = false;
  23     //alias n = nn;
  24     static if (T.sizeof < 8) {
  25       uint n = nn;
  26     } else {
  27       ulong n = nn;
  28     }
  29   } else {
  30     bool neg = (nn < 0);
  31     static if (T.sizeof < 8) {
  32       long n = nn;
  33       if (neg) n = -n;
  34       if (n < 0) n = T.max;
  35     } else {
  36       //alias n = nn;
  37       long n = nn;
  38       if (neg) n = -n;
  39       if (n < 0) n = T.max; //FIXME
  40     }
  41   }
  42   char[256] buf = void;
  43   int bpos = cast(int)buf.length;
  44   int leftToComma = 3;
  45   do {
  46     if (leftToComma-- == 0) { buf[--bpos] = comma; leftToComma = 2; }
  47     buf[--bpos] = cast(char)('0'+n%10);
  48   } while ((n /= 10) != 0);
  49   if (neg) buf[--bpos] = '-';
  50   auto len = buf.length-bpos;
  51   if (dest is null) dest = new char[](len);
  52   if (len > dest.length) len = dest.length;
  53   dest[0..len] = buf[bpos..bpos+len];
  54   return dest[0..len];
  55 }
  56
  57 char[] intWithCommas(T) (T nn, char comma=',') if (__traits(isIntegral, T)) { return intWithCommas(null, nn, comma); }
  58
  59
  60 void rawWriteExact (ref File fo, const(void)[] buf) {
  61   if (buf.length == 0) return;
  62   fo.rawWrite(buf);
  63 }
  64
  65 void rawReadExact (ref File fo, void[] buf) {
  66   if (buf.length == 0) return;
  67   if (fo.rawRead(buf).length != buf.length) throw new Exception("read error");
  68 }
  69
  70 void writeNum(T) (ref File st, T n) if (__traits(isIntegral, T)) {
  71   static assert(T.sizeof <= 8); // just in case
  72   st.rawWriteExact((&n)[0..1]);
  73 }
  74
  75 T readNum(T) (ref File st) if (__traits(isIntegral, T)) {
  76   static assert(T.sizeof <= 8); // just in case
  77   T n;
  78   st.rawReadExact((&n)[0..1]);
  79   return n;
  80 }
  81
  82
  83 // ////////////////////////////////////////////////////////////////////////// //
  84 void compressFile (ref File fi, ref File fo) {
  85   fi.seek(0);
  86   ulong insize = fi.size;
  87   //fi.rawReadExact(inbuf[0..insize]);
  88   //fi.close();
  89
  90   CLzmaEncProps props;
  91   LzmaEncProps_Init(&props);
  92   props.level = 9;
  93   //props.dictSize = 1;
  94   //while (props.dictSize < insize) props.dictSize <<= 1;
  95   props.dictSize = 1<<27; //128MB
  96   //props.dictSize = 1<<22; //4MB
  97   props.reduceSize = insize;
  98
  99   ubyte[LZMA_PROPS_SIZE+8] header;
 100   uint headerSize = cast(uint)header.sizeof;
 101
 102   CLzmaEncHandle enc = LzmaEnc_Create(&lzmaDefAllocator);
 103   scope(exit) LzmaEnc_Destroy(enc, &lzmaDefAllocator, &lzmaDefAllocator);
 104
 105   if (LzmaEnc_SetProps(enc, &props) != SZ_OK) throw new Exception("cannot set encoder properties");
 106   LzmaEnc_SetDataSize(enc, insize); // just in case
 107
 108   if (LzmaEnc_WriteProperties(enc, header.ptr, &headerSize) != SZ_OK) throw new Exception("cannot encode encoder properties");
 109   assert(headerSize > 0 && headerSize < 256);
 110
 111   writeln("compressing...");
 112   fo.writeNum!ushort(cast(ushort)1); // version and endianness check
 113   fo.writeNum!ulong(insize); // unpacked file size
 114   fo.writeNum!ubyte(cast(ubyte)headerSize); // properties size
 115   fo.rawWriteExact(header[0..headerSize]);
 116
 117   ISeqInStream inStream;
 118   ISeqOutStream outStream;
 119   ICompressProgress progress;
 120
 121   immutable origInSize = insize;
 122   ulong destSize = 0;
 123
 124   auto csum = makeDigest!RIPEMD160;
 125
 126   /* if (input(*size) != 0 && output(*size) == 0) means end_of_stream.
 127      (output(*size) < input(*size)) is allowed */
 128   inStream.Read = delegate SRes (ISeqInStream* p, void* buf, usize* size) nothrow {
 129     if (*size > insize) *size = cast(usize)insize;
 130     if (*size) {
 131       try {
 132         fi.rawReadExact(buf[0..*size]);
 133       } catch (Exception e) {
 134         return SZ_ERROR_READ;
 135       }
 136       csum.put((cast(const(ubyte)*)buf)[0..*size]);
 137       insize -= *size;
 138     }
 139     return SZ_OK;
 140   };
 141
 142   /* Returns: result - the number of actually written bytes.
 143      (result < size) means error */
 144   outStream.Write = delegate usize (ISeqOutStream* p, const(void)* buf, usize size) nothrow {
 145     try {
 146       fo.rawWriteExact(buf[0..size]);
 147       destSize += size;
 148     } catch (Exception e) {
 149       return 0;
 150     }
 151     return size;
 152   };
 153
 154   uint prevPrc = uint.max;
 155
 156   progress.Progress = delegate SRes (ICompressProgress* p, ulong inSize, ulong outSize) nothrow {
 157     if (origInSize == 0) return SZ_OK; // just in case
 158     immutable uint prc = cast(uint)(inSize*100U/origInSize);
 159     if (prc == prevPrc && inSize != 0) return SZ_OK;
 160     prevPrc = prc;
 161     char[128] i0 = void;
 162     char[128] i1 = void;
 163     auto num0 = intWithCommas(i0[], inSize);
 164     auto num1 = intWithCommas(i1[], origInSize);
 165     try {
 166       stdout.write(" [", num0[], "/", num1[], "]  ", prc, "%\x1b[K\r");
 167       stdout.flush();
 168     } catch (Exception) {}
 169     return SZ_OK;
 170   };
 171
 172   progress.Progress(&progress, 0, 0);
 173
 174   SRes res = LzmaEnc_Encode(enc, &outStream, &inStream, &progress, &lzmaDefAllocator, &lzmaDefAllocator);
 175
 176   switch (res) {
 177     case SZ_OK: break;
 178     case SZ_ERROR_MEM: throw new Exception("FUCK: memory");
 179     case SZ_ERROR_PARAM: throw new Exception("FUCK: param");
 180     case SZ_ERROR_OUTPUT_EOF: throw new Exception("FUCK: compressed is bigger");
 181     default: throw new Exception("FUCK: something else");
 182   }
 183
 184   ubyte[20] hash = csum.finish()[];
 185   fo.rawWriteExact(hash[]);
 186
 187   writeln("\rcompressed ", intWithCommas(origInSize), " to ", intWithCommas(destSize), "; ratio: ", destSize*100U/(origInSize ? origInSize : 1), "%\x1b[K");
 188 }
 189
 190
 191 // ////////////////////////////////////////////////////////////////////////// //
 192 void decompressFile (ref File fi, ref File fo) {
 193   ubyte[LZMA_PROPS_SIZE+8] header;
 194
 195   fi.seek(0);
 196   ulong pksize = fi.size;
 197   if (fi.readNum!ushort != 1) throw new Exception("invalid archive version");
 198   ulong unsize = fi.readNum!ulong; // unpacked size
 199   ubyte hdrSize = fi.readNum!ubyte;
 200   if (hdrSize == 0 || hdrSize > header.sizeof) throw new Exception("invalid properties size");
 201   fi.rawReadExact(header[0..hdrSize]);
 202   pksize -= fi.tell;
 203
 204   if (pksize < 20) throw new Exception("invalid archive size");
 205   if (pksize == 0) {
 206     if (unsize != 0) throw new Exception("invalid archive size");
 207     return; // nothing to do
 208   }
 209   pksize -= 20; // digest size
 210
 211   auto csum = makeDigest!RIPEMD160;
 212
 213   enum InBufSize = 1024*1024;
 214   enum OutBufSize = 1024*1024;
 215
 216   ubyte *inbuf = cast(ubyte*)ISzAlloc_Alloc(&lzmaDefAllocator, InBufSize);
 217   ubyte *outbuf = cast(ubyte*)ISzAlloc_Alloc(&lzmaDefAllocator, OutBufSize);
 218   scope(exit) {
 219     ISzAlloc_Free(&lzmaDefAllocator, inbuf);
 220     ISzAlloc_Free(&lzmaDefAllocator, outbuf);
 221   }
 222
 223   CLzmaDec dec;
 224   LzmaDec_Init(&dec);
 225
 226   SRes res = LzmaDec_Allocate(&dec, header.ptr, hdrSize, &lzmaDefAllocator);
 227   if (res != SZ_OK) throw new Exception("cannot initialize decoder");
 228   scope(exit) LzmaDec_Free(&dec, &lzmaDefAllocator);
 229
 230   ulong unpackedTotal = 0;
 231   ulong readleft = pksize;
 232   usize inused = 0;
 233
 234   uint prevPrc = uint.max;
 235
 236   void showProgress () nothrow {
 237     immutable rds = unpackedTotal; //pksize-readleft;
 238     immutable uint prc = cast(uint)(rds*100U/unsize/*pksize*/);
 239     if (prc == prevPrc && unpackedTotal != unsize) return;
 240     prevPrc = prc;
 241     char[128] i0 = void;
 242     char[128] i1 = void;
 243     auto num0 = intWithCommas(i0[], rds);
 244     auto num1 = intWithCommas(i1[], unsize/*pksize*/);
 245     try {
 246       stdout.write(" [", num0[], "/", num1[], "]  ", prc, "%\x1b[K\r");
 247       stdout.flush();
 248     } catch (Exception) {}
 249   }
 250
 251   while (readleft || inused) {
 252     // read more
 253     if (readleft && inused < InBufSize) {
 254       uint rd = InBufSize-cast(uint)inused;
 255       if (rd > readleft) rd = cast(uint)readleft;
 256       fi.rawReadExact(inbuf[inused..inused+rd]);
 257       inused += rd;
 258       readleft -= rd;
 259       //showProgress();
 260     }
 261     usize outSize = OutBufSize;
 262     usize inSize = inused;
 263     ELzmaStatus status;
 264     // as we don't have a proper EOF mark, make sure to not unpack extra data
 265     if (unsize-unpackedTotal < outSize) outSize = cast(usize)(unsize-unpackedTotal);
 266     //writeln("\nunsize=", unsize, "; unpackedTotal=", unpackedTotal, "; outSize=", outSize);
 267     res = LzmaDec_DecodeToBuf(&dec, outbuf, &outSize, inbuf, &inSize, LZMA_FINISH_ANY, &status);
 268     if (res != SZ_OK) {
 269       writeln;
 270       writeln("ERROR: readleft=", readleft, "; inused=", inused, "; written=", unpackedTotal, " of ", unsize);
 271       switch (res) {
 272         case SZ_ERROR_DATA: throw new Exception("corrupted data");
 273         case SZ_ERROR_MEM: throw new Exception("out of memory");
 274         case SZ_ERROR_UNSUPPORTED: throw new Exception("unsupported properties");
 275         case SZ_ERROR_INPUT_EOF: throw new Exception("need bigger input buffer, but we don't have any");
 276         default: throw new Exception("some other error");
 277       }
 278     }
 279     if (outSize) {
 280       fo.rawWriteExact(outbuf[0..outSize]);
 281       unpackedTotal += outSize;
 282       csum.put((cast(const(ubyte)*)outbuf)[0..outSize]);
 283       showProgress();
 284       if (unpackedTotal == unsize) break; // we're done (we don't have EOF mark, so...)
 285     }
 286     if (inSize < inused) {
 287       import core.stdc.string : memmove;
 288       memmove(inbuf, inbuf+inSize, inused-inSize);
 289     }
 290     inused -= inSize;
 291     switch (status) {
 292       case LZMA_STATUS_FINISHED_WITH_MARK: throw new Exception("found EOF mark, but there should not be one");
 293       case LZMA_STATUS_NOT_FINISHED: break; // it is ok
 294       case LZMA_STATUS_NEEDS_MORE_INPUT: break; // it is ok
 295       case LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK: break; // ok ;-)
 296       default: break; // ignore others
 297     }
 298   }
 299
 300   if (unpackedTotal != unsize) {
 301     //import std.conv : to;
 302     throw new Exception("invalid unpacked file size; expected "~intWithCommas(unsize).idup~" but got "~intWithCommas(unpackedTotal).idup);
 303   }
 304
 305   // check hash
 306   ubyte[20] origcsum;
 307   fi.rawReadExact(origcsum[]);
 308   ubyte[20] hash = csum.finish()[];
 309   if (origcsum[] != hash[]) throw new Exception("invalid unpacked file hash");
 310
 311   writeln("\rsuccesfully unpacked ", intWithCommas(unpackedTotal), " bytes.\x1b[K");
 312 }
 313
 314
 315 // ////////////////////////////////////////////////////////////////////////// //
 316 void main (string[] args) {
 317   if (args.length < 2) {
 318     writeln("usage: fuckme <c|x> infile outfile");
 319     return;
 320   }
 321
 322   if (args[1] == "c") {
 323     if (args.length < 4) throw new Exception("out of args");
 324     if (args[2] == args[3]) throw new Exception("cannot compress in-place");
 325     auto fi = File(args[2]);
 326     auto fo = File(args[3], "w");
 327     compressFile(fi, fo);
 328   } else if (args[1] == "x") {
 329     if (args.length < 4) throw new Exception("out of args");
 330     if (args[2] == args[3]) throw new Exception("cannot decompress in-place");
 331     auto fi = File(args[2]);
 332     auto fo = File(args[3], "w");
 333     decompressFile(fi, fo);
 334   } else if (args[1] == "t") {
 335     if (args.length < 3) throw new Exception("out of args");
 336     auto fi = File(args[2]);
 337     auto fo = File("/dev/null", "w");
 338     decompressFile(fi, fo);
 339   }
 340 }