_obsolete_dont_use/ziparc.d

   1 /* Invisible Vector Library
   2  * coded by Ketmar // Invisible Vector <ketmar@ketmar.no-ip.org>
   3  * Understanding is not required. Only obedience.
   4  *
   5  * This program is free software: you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation, version 3 of the License ONLY.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  16  */
  17 // severely outdated ZIP archive interface
  18 // use iv.vfs instead
  19 module iv.ziparc /*is aliced*/;
  20
  21
  22 // ////////////////////////////////////////////////////////////////////////// //
  23 final class ZipArchive {
  24 public:
  25   import std.stdio : File;
  26
  27 private:
  28   align(1) static struct ZipFileHeader {
  29   align(1):
  30     char[4] sign; // "PK\x03\x04"
  31     ushort extrver; // version needed to extract
  32     ushort gflags; // general purpose bit flag
  33     ushort method; // compression method
  34     ushort mtime; // last mod file time
  35     ushort mdate; // last mod file date
  36     uint crc32;
  37     uint pksize; // compressed size
  38     uint size; // uncompressed size
  39     ushort namelen; // file name length
  40     ushort extlen; // extra field length
  41   }
  42
  43   align(1) static struct CDFileHeader {
  44   align(1):
  45     //char[4] sign; // "PK\x01\x02"
  46     ushort madebyver; // version made by
  47     ushort extrver; // version needed to extract
  48     ushort gflags; // general purpose bit flag
  49     ushort method; // compression method
  50     ushort mtime; // last mod file time
  51     ushort mdate; // last mod file date
  52     uint crc32;
  53     uint pksize; // compressed size
  54     uint size; // uncompressed size
  55     ushort namelen; // file name length
  56     ushort extlen; // extra field length
  57     ushort cmtlen; // file comment length
  58     ushort disk; // disk number start
  59     ushort iattr; // internal file attributes
  60     uint attr; // external file attributes
  61     uint hdrofs; // relative offset of local header
  62
  63   @property pure const nothrow @safe @nogc:
  64     ubyte hour () { return (mtime>>11); }
  65     ubyte min () { return (mtime>>5)&0x3f; }
  66     ubyte sec () { return (mtime&0x1f)*2; }
  67
  68     ushort year () { return cast(ushort)((mdate>>9)+1980); }
  69     ubyte month () { return (mdate>>5)&0x0f; }
  70     ubyte day () { return (mdate&0x1f); }
  71   }
  72
  73   align(1) static struct EOCDHeader {
  74   align(1):
  75     char[4] sign; // "PK\x05\x06"
  76     ushort diskno; // number of this disk
  77     ushort diskcd; // number of the disk with the start of the central directory
  78     ushort diskfileno; // total number of entries in the central directory on this disk
  79     ushort fileno; // total number of entries in the central directory
  80     uint cdsize; // size of the central directory
  81     uint cdofs; // offset of start of central directory with respect to the starting disk number
  82     ushort cmtsize; // .ZIP file comment length
  83   }
  84
  85   align(1) static struct EOCD64Header {
  86   align(1):
  87     char[4] sign; // "PK\x06\x06"
  88     ulong eocdsize; // size of zip64 end of central directory record
  89     ushort madebyver; // version made by
  90     ushort extrver; // version needed to extract
  91     uint diskno; // number of this disk
  92     uint diskcd; // number of the disk with the start of the central directory
  93     ulong diskfileno; // total number of entries in the central directory
  94     ulong fileno; // total number of entries in the central directory
  95     ulong cdsize; // size of the central directory
  96     ulong cdofs; // offset of start of central directory with respect to the starting disk number
  97   }
  98
  99   align(1) static struct Z64Locator {
 100   align(1):
 101     char[4] sign; // "PK\x06\x07"
 102     uint diskcd; // number of the disk with the start of the zip64 end of central directory
 103     long ecd64ofs; // relative offset of the zip64 end of central directory record
 104     uint diskno; // total number of disks
 105   }
 106
 107   align(1) static struct Z64Extra {
 108   align(1):
 109     ulong size;
 110     ulong pksize;
 111     ulong hdrofs;
 112     uint disk; // number of the disk on which this file starts
 113   }
 114
 115   static struct FileInfo {
 116     bool packed; // only "store" and "deflate" are supported
 117     ulong pksize;
 118     ulong size;
 119     ulong hdrofs;
 120     string path;
 121     string name;
 122   }
 123
 124   // for dir range
 125   public static struct DirEntry {
 126     string path;
 127     string name;
 128     ulong size;
 129   }
 130
 131 private:
 132   File zfl;
 133   FileInfo[] dir;
 134   bool mNormNames; // true: convert names to lower case, do case-insensitive comparison (ASCII only)
 135
 136 public:
 137   this (string fname, bool normNames=true) {
 138     import std.stdio : File;
 139     mNormNames = normNames;
 140     initLock();
 141     zfl = File(fname);
 142     open(zfl);
 143     scope(failure) { zfl.close; zfl = zfl.init; }
 144   }
 145
 146   // it now owns the file (if no exception was thrown)
 147   this (File fl, bool normNames=true) {
 148     mNormNames = normNames;
 149     initLock();
 150     open(fl);
 151     scope(success) zfl = fl;
 152   }
 153
 154   @property auto files () {
 155     static struct Range {
 156     private:
 157       ZipArchive me;
 158       ulong curindex;
 159
 160     nothrow @safe @nogc:
 161       this (ZipArchive ame, ulong aidx=0) { me = ame; curindex = aidx; }
 162
 163     public:
 164       @property bool empty () const { return (curindex >= me.dir.length); }
 165       @property DirEntry front () const {
 166         return DirEntry(
 167           (curindex < me.dir.length ? me.dir[cast(usize)curindex].path : null),
 168           (curindex < me.dir.length ? me.dir[cast(usize)curindex].name : null),
 169           (curindex < me.dir.length ? me.dir[cast(usize)curindex].size : 0));
 170       }
 171       @property Range save () { return Range(me, curindex); }
 172       void popFront () { if (curindex < me.dir.length) ++curindex; }
 173       @property ulong length () const { return me.dir.length; }
 174       @property ulong position () const { return curindex; } // current position
 175       @property void position (ulong np) { curindex = np; }
 176       void rewind () { curindex = 0; }
 177     }
 178     return Range(this);
 179   }
 180
 181   File fopen (ref in DirEntry de) {
 182     static bool strequ() (const(char)[] s0, const(char)[] s1) {
 183       if (s0.length != s1.length) return false;
 184       foreach (immutable idx, char ch; s0) {
 185         char c1 = s1[idx];
 186         if (ch >= 'A' && ch <= 'Z') ch += 32; // poor man's `toLower()`
 187         if (c1 >= 'A' && c1 <= 'Z') c1 += 32; // poor man's `toLower()`
 188         if (ch != c1) return false;
 189       }
 190       return true;
 191     }
 192
 193     foreach (immutable idx, ref fi; dir) {
 194       if (mNormNames) {
 195         if (strequ(fi.path, de.path) && strequ(fi.name, de.name)) return openDirEntry(idx, fi.name);
 196       } else {
 197         if (fi.path == de.path && fi.name == de.name) return openDirEntry(idx, fi.name);
 198       }
 199     }
 200
 201     throw new NamedException!"ZipArchive"("file not found");
 202   }
 203
 204   File fopen (const(char)[] fname) {
 205     DirEntry de;
 206     auto pos = fname.length;
 207     while (pos > 0 && fname[pos-1] != '/') --pos;
 208     if (pos) {
 209       de.path = cast(string)fname[0..pos]; // it's safe here
 210       de.name = cast(string)fname[pos..$]; // it's safe here
 211     } else {
 212       de.name = cast(string)fname; // it's safe here
 213     }
 214     return fopen(de);
 215   }
 216
 217 private:
 218   void cleanup () {
 219     dir.length = 0;
 220   }
 221
 222   void open (File fl) {
 223     import core.stdc.stdio : SEEK_CUR, SEEK_END;
 224     debug import std.stdio : writeln, writefln;
 225     scope(failure) cleanup();
 226
 227     ushort readU16 () {
 228       ubyte[2] data;
 229       if (fl.rawRead(data[]).length != data.length) throw new NamedException!"ZipArchive"("reading error");
 230       return cast(ushort)(data[0]+0x100*data[1]);
 231     }
 232
 233     if (fl.size > 0xffff_ffffu) throw new NamedException!"ZipArchive"("file too big");
 234     ulong flsize = fl.size;
 235     if (flsize < EOCDHeader.sizeof) throw new NamedException!"ZipArchive"("file too small");
 236
 237     // search for "end of central dir"
 238     auto cdbuf = xalloc!ubyte(65536+EOCDHeader.sizeof+Z64Locator.sizeof);
 239     scope(exit) xfree(cdbuf);
 240     ubyte[] buf;
 241     ulong ubufpos;
 242     if (flsize < cdbuf.length) {
 243       fl.seek(0);
 244       buf = fl.rawRead(cdbuf[0..cast(usize)flsize]);
 245       if (buf.length != flsize) throw new NamedException!"ZipArchive"("reading error");
 246     } else {
 247       fl.seek(-cast(ulong)cdbuf.length, SEEK_END);
 248       ubufpos = fl.tell;
 249       buf = fl.rawRead(cdbuf[]);
 250       if (buf.length != cdbuf.length) throw new NamedException!"ZipArchive"("reading error");
 251     }
 252     int pos;
 253     for (pos = cast(int)(buf.length-EOCDHeader.sizeof); pos >= 0; --pos) {
 254       if (buf[pos] == 'P' && buf[pos+1] == 'K' && buf[pos+2] == 5 && buf[pos+3] == 6) break;
 255     }
 256     if (pos < 0) throw new NamedException!"ZipArchive"("no central dir end marker found");
 257     auto eocd = cast(EOCDHeader*)&buf[pos];
 258     debug {
 259       writeln("=== EOCD ===");
 260       writeln("diskno: ", eocd.diskno);
 261       writeln("diskcd: ", eocd.diskcd);
 262       writeln("diskfileno: ", eocd.diskfileno);
 263       writeln("fileno: ", eocd.fileno);
 264       writeln("cdsize: ", eocd.cdsize);
 265       writefln("cdofs: %s (0x%08x)", eocd.cdofs, eocd.cdofs);
 266       writeln("cmtsize: ", eocd.cmtsize);
 267     }
 268     long cdofs = -1, cdsize = -1;
 269     bool zip64 = false;
 270     // zip64?
 271     if (eocd.cdofs == 0xffff_ffffu) {
 272       zip64 = true;
 273       if (pos < Z64Locator.sizeof) throw new NamedException!"ZipArchive"("corrupted archive");
 274       auto lt64 = cast(Z64Locator*)&buf[pos-Z64Locator.sizeof];
 275       if (lt64.sign != "PK\x06\x07") throw new NamedException!"ZipArchive"("corrupted archive");
 276       if (lt64.diskcd != 0 || lt64.diskno > 1) throw new NamedException!"ZipArchive"("multidisk archive");
 277       debug writeln("ecd64ofs=", lt64.ecd64ofs);
 278       if (lt64.ecd64ofs < 0 || lt64.ecd64ofs+EOCD64Header.sizeof > ubufpos+pos-Z64Locator.sizeof) throw new NamedException!"ZipArchive"("corrupted archive");
 279       EOCD64Header e64 = void;
 280       fl.seek(lt64.ecd64ofs);
 281       if (fl.rawRead((&e64)[0..1]).length != 1) throw new NamedException!"ZipArchive"("reading error");
 282       if (e64.sign != "PK\x06\x06") throw new NamedException!"ZipArchive"("corrupted archive");
 283       if (e64.diskno != 0 || e64.diskcd != 0) throw new NamedException!"ZipArchive"("multidisk archive");
 284       if (e64.diskfileno != e64.fileno) throw new NamedException!"ZipArchive"("corrupted archive");
 285       if (e64.cdsize >= lt64.ecd64ofs) throw new NamedException!"ZipArchive"("corrupted archive");
 286       if (e64.cdofs >= lt64.ecd64ofs || e64.cdofs+e64.cdsize > lt64.ecd64ofs) throw new NamedException!"ZipArchive"("corrupted archive");
 287       cdofs = e64.cdofs;
 288       cdsize = e64.cdsize;
 289     } else {
 290       if (eocd.diskno != 0 || eocd.diskcd != 0) throw new NamedException!"ZipArchive"("multidisk archive");
 291       if (eocd.diskfileno != eocd.fileno || ubufpos+pos+EOCDHeader.sizeof+eocd.cmtsize != flsize) throw new NamedException!"ZipArchive"("corrupted archive");
 292       cdofs = eocd.cdofs;
 293       cdsize = eocd.cdsize;
 294       if (cdofs >= ubufpos+pos || flsize-cdofs < cdsize) throw new NamedException!"ZipArchive"("corrupted archive");
 295     }
 296
 297     // now read central directory
 298     auto namebuf = xalloc!char(0x10000);
 299     scope(exit) xfree(namebuf);
 300
 301     uint[string] knownNames; // value is dir index
 302     scope(exit) knownNames.destroy;
 303     cleanup();
 304     auto bleft = cdsize;
 305     fl.seek(cdofs);
 306     CDFileHeader cdfh = void;
 307     char[4] sign;
 308     dir.assumeSafeAppend; // yep
 309     while (bleft > 0) {
 310       if (bleft < 4) break;
 311       if (fl.rawRead(sign[]).length != sign.length) throw new NamedException!"ZipArchive"("reading error");
 312       bleft -= 4;
 313       if (sign[0] != 'P' || sign[1] != 'K') throw new NamedException!"ZipArchive"("invalid central directory entry");
 314       // digital signature?
 315       if (sign[2] == 5 && sign[3] == 5) {
 316         // yes, skip it
 317         if (bleft < 2) throw new NamedException!"ZipArchive"("reading error");
 318         auto sz = readU16();
 319         if (sz > bleft) throw new NamedException!"ZipArchive"("invalid central directory entry");
 320         fl.seek(sz, SEEK_CUR);
 321         bleft -= sz;
 322         continue;
 323       }
 324       // file item?
 325       if (sign[2] == 1 && sign[3] == 2) {
 326         if (bleft < cdfh.sizeof) throw new NamedException!"ZipArchive"("reading error");
 327         if (fl.rawRead((&cdfh)[0..1]).length != 1) throw new NamedException!"ZipArchive"("reading error");
 328         bleft -= cdfh.sizeof;
 329         if (cdfh.disk != 0) throw new NamedException!"ZipArchive"("invalid central directory entry (disk number)");
 330         if (bleft < cdfh.namelen+cdfh.extlen+cdfh.cmtlen) throw new NamedException!"ZipArchive"("invalid central directory entry");
 331         // skip bad files
 332         if ((cdfh.method != 0 && cdfh.method != 8) || cdfh.namelen == 0 || (cdfh.gflags&0b10_0000_0110_0001) != 0 || (cdfh.attr&0x58) != 0 ||
 333             cast(long)cdfh.hdrofs+(cdfh.method ? cdfh.pksize : cdfh.size) >= ubufpos+pos)
 334         {
 335           // ignore this
 336           fl.seek(cdfh.namelen+cdfh.extlen+cdfh.cmtlen, SEEK_CUR);
 337           bleft -= cdfh.namelen+cdfh.extlen+cdfh.cmtlen;
 338           continue;
 339         }
 340         FileInfo fi;
 341         fi.packed = (cdfh.method != 0);
 342         fi.pksize = cdfh.pksize;
 343         fi.size = cdfh.size;
 344         fi.hdrofs = cdfh.hdrofs;
 345         if (!fi.packed) fi.pksize = fi.size;
 346         // now, this is valid file, so read it's name
 347         if (fl.rawRead(namebuf[0..cdfh.namelen]).length != cdfh.namelen) throw new NamedException!"ZipArchive"("reading error");
 348         auto nb = new char[](cdfh.namelen);
 349         uint nbpos = 0;
 350         uint lastSlash = 0;
 351         foreach (ref char ch; namebuf[0..cdfh.namelen]) {
 352           if (ch == '\\') ch = '/'; // just in case
 353           if (ch == '/' && (nbpos == 0 || (nbpos > 0 && nb[nbpos-1] == '/'))) continue;
 354           if (ch == '/') lastSlash = nbpos+1;
 355           if (mNormNames && ch >= 'A' && ch <= 'Z') ch += 32; // poor man's `toLower()`
 356           nb[nbpos++] = ch;
 357         }
 358         bool doSkip = false;
 359         // should we parse extra field?
 360         debug writefln("size=0x%08x; pksize=0x%08x; packed=%s", fi.size, fi.pksize, (fi.packed ? "tan" : "ona"));
 361         if (zip64 && (fi.size == 0xffff_ffffu || fi.pksize == 0xffff_ffffu || fi.hdrofs == 0xffff_ffffu)) {
 362           // yep, do it
 363           bool found = false;
 364           //Z64Extra z64e = void;
 365           debug writeln("extlen=", cdfh.extlen);
 366           while (cdfh.extlen >= 4) {
 367             auto eid = readU16();
 368             auto esize = readU16();
 369             debug writefln("0x%04x %s", eid, esize);
 370             cdfh.extlen -= 4;
 371             bleft -= 4;
 372             if (cdfh.extlen < esize) break;
 373             cdfh.extlen -= esize;
 374             bleft -= esize;
 375             // skip unknown info
 376             if (eid != 1 || esize < /*Z64Extra.sizeof*/8) {
 377               fl.seek(esize, SEEK_CUR);
 378             } else {
 379               // wow, Zip64 info
 380               found = true;
 381               if (fi.size == 0xffff_ffffu) {
 382                 if (fl.rawRead((&fi.size)[0..1]).length != 1) throw new NamedException!"ZipArchive"("reading error");
 383                 esize -= 8;
 384                 //debug writeln(" size=", fi.size);
 385               }
 386               if (fi.pksize == 0xffff_ffffu) {
 387                 if (esize == 0) {
 388                   //fi.pksize = ulong.max; // this means "get from local header"
 389                   // read local file header; it's slow, but i don't care
 390                   /*
 391                   if (fi.hdrofs == 0xffff_ffffu) throw new NamedException!"ZipArchive"("invalid zip64 archive (3)");
 392                   CDFileHeader lfh = void;
 393                   auto oldpos = fl.tell;
 394                   fl.seek(fi.hdrofs);
 395                   if (fl.rawRead((&lfh)[0..1]).length != 1) throw new NamedException!"ZipArchive"("reading error");
 396                   assert(0);
 397                   */
 398                   throw new NamedException!"ZipArchive"("invalid zip64 archive (4)");
 399                 } else {
 400                   if (esize < 8) throw new NamedException!"ZipArchive"("invalid zip64 archive (1)");
 401                   if (fl.rawRead((&fi.pksize)[0..1]).length != 1) throw new NamedException!"ZipArchive"("reading error");
 402                   esize -= 8;
 403                 }
 404               }
 405               if (fi.hdrofs == 0xffff_ffffu) {
 406                 if (esize < 8) throw new NamedException!"ZipArchive"("invalid zip64 archive (2)");
 407                 if (fl.rawRead((&fi.hdrofs)[0..1]).length != 1) throw new NamedException!"ZipArchive"("reading error");
 408                 esize -= 8;
 409               }
 410               if (esize > 0) fl.seek(esize, SEEK_CUR); // skip possible extra data
 411               //if (z64e.disk != 0) throw new NamedException!"ZipArchive"("invalid central directory entry (disk number)");
 412               break;
 413             }
 414           }
 415           if (!found) {
 416             debug writeln("required zip64 record not found");
 417             //throw new NamedException!"ZipArchive"("required zip64 record not found");
 418             //fi.size = fi.pksize = 0x1_0000_0000Lu; // hack: skip it
 419             doSkip = true;
 420           }
 421         }
 422         if (!doSkip && nbpos > 0 && nb[nbpos-1] != '/') {
 423           if (auto idx = nb[0..nbpos] in knownNames) {
 424             // replace
 425             auto fip = &dir[*idx];
 426             fip.packed = fi.packed;
 427             fip.pksize = fi.pksize;
 428             fip.size = fi.size;
 429             fip.hdrofs = fi.hdrofs;
 430           } else {
 431             // add new
 432             if (dir.length == uint.max) throw new NamedException!"ZipArchive"("directory too long");
 433             if (lastSlash) {
 434               fi.path = cast(string)nb[0..lastSlash]; // this is safe
 435               fi.name = cast(string)nb[lastSlash..nbpos]; // this is safe
 436             } else {
 437               fi.path = "";
 438               fi.name = cast(string)nb[0..nbpos]; // this is safe
 439             }
 440             knownNames[fi.name] = cast(uint)dir.length;
 441             dir ~= fi;
 442           }
 443           //debug writefln("%10s %10s %s %04s/%02s/%02s %02s:%02s:%02s %s", fi.pksize, fi.size, (fi.packed ? "P" : "."), cdfh.year, cdfh.month, cdfh.day, cdfh.hour, cdfh.min, cdfh.sec, fi.name);
 444         }
 445         // skip extra and comments
 446         fl.seek(cdfh.extlen+cdfh.cmtlen, SEEK_CUR);
 447         bleft -= cdfh.namelen+cdfh.extlen+cdfh.cmtlen;
 448         continue;
 449       }
 450       // wtf?!
 451       throw new NamedException!"ZipArchive"("unknown central directory entry");
 452     }
 453     debug writeln(dir.length, " files found");
 454   }
 455
 456
 457   // ////////////////////////////////////////////////////////////////////// //
 458   static import core.sync.mutex;
 459
 460   core.sync.mutex.Mutex lock;
 461
 462   void initLock () {
 463     lock = new core.sync.mutex.Mutex;
 464   }
 465
 466   auto openDirEntry (uint idx, string filename) {
 467     import core.sys.linux.stdio : fopencookie;
 468     import core.stdc.stdio : FILE;
 469     import core.stdc.stdio : fopen, fclose;
 470     import core.stdc.stdlib : calloc, free;
 471     import etc.c.zlib;
 472     import std.internal.cstring : tempCString;
 473     import core.memory : GC;
 474
 475     if (!zfl.isOpen) throw new NamedException!"ZipArchive"("archive wasn't opened");
 476     if (zfl.name.length == 0) throw new NamedException!"ZipArchive"("archive has no name");
 477     if (idx >= dir.length) if (!zfl.isOpen) throw new NamedException!"ZipArchive"("invalid dir index");
 478     ulong stofs;
 479     {
 480       lock.lock();
 481       scope(exit) lock.unlock();
 482       // read file header
 483       ZipFileHeader zfh = void;
 484       zfl.seek(dir[idx].hdrofs);
 485       if (zfl.rawRead((&zfh)[0..1]).length != 1) throw new NamedException!"ZipArchive"("reading error");
 486       if (zfh.sign != "PK\x03\x04") throw new NamedException!"ZipArchive"("invalid archive entry");
 487       // skip name and extra
 488       stofs = zfl.tell+zfh.namelen+zfh.extlen;
 489     }
 490
 491     // create cookied `FILE*`
 492     auto fc = cast(InnerFileCookied*)calloc(1, InnerFileCookied.sizeof);
 493     scope(exit) if (fc !is null) free(fc);
 494     if (fc is null) {
 495       import core.exception : onOutOfMemoryErrorNoGC;
 496       onOutOfMemoryErrorNoGC();
 497     }
 498     (*fc) = InnerFileCookied.init;
 499     (*fc).stpos = stofs;
 500     (*fc).size = cast(uint)dir[idx].size; //FIXME
 501     (*fc).pksize = cast(uint)dir[idx].pksize; //FIXME
 502     (*fc).mode = (dir[idx].packed ? InnerFileCookied.Mode.Zip : InnerFileCookied.Mode.Raw);
 503     (*fc).lock = lock;
 504     GC.addRange(fc, InnerFileCookied.sizeof);
 505     // open DAT file
 506     //(*fc).fl = //fopen(zfl.name.tempCString!char(), "r");
 507     //if ((*fc).fl is null) throw new NamedException!"ZipArchive"("can't open archive file");
 508     (*fc).xfl = zfl;
 509     // open `cooked` file
 510     FILE* fres = fopencookie(cast(void*)fc, "r", fcdatpkCallbacks);
 511     if (fres is null) {
 512       // alas
 513       if ((*fc).fl !is null) fclose((*fc).fl);
 514       try { (*fc).xfl.detach(); } catch (Exception) {}
 515       throw new NamedException!"ZipArchive"("can't open cookied file");
 516     }
 517     // ok
 518     (*fc).initialize();
 519     fc = null;
 520     return File(fres, filename);
 521   }
 522
 523
 524   // ////////////////////////////////////////////////////////////////////// //
 525   // "inner" file processor; processes both packed and unpacked files
 526   // can be used as normal disk file processor too
 527   static struct InnerFileCookied {
 528     private import etc.c.zlib;
 529     private import core.sys.posix.sys.types : off64_t = off_t;
 530     private import core.stdc.stdio : FILE;
 531
 532     enum ibsize = 32768;
 533
 534     enum Mode { Raw, ZLib, Zip }
 535
 536     core.sync.mutex.Mutex lock;
 537     // note that either one of `fl` or `xfl` must be opened and operational
 538     FILE* fl; // disk file, can be `null`
 539     File xfl; // disk file, can be closed
 540     Mode mode;
 541     long stpos; // starting position
 542     uint size; // unpacked size
 543     uint pksize; // packed size
 544     uint pos; // current file position
 545     uint prpos; // previous file position
 546     uint pkpos; // current position in DAT
 547     ubyte[] pkb; // packed data
 548     z_stream zs;
 549     bool eoz;
 550
 551     @disable this (this);
 552
 553   nothrow:
 554     ~this () { close(); }
 555
 556     @property bool isOpen () @safe /*@nogc*/ { return (fl !is null || xfl.isOpen); }
 557
 558     void initialize () {
 559       /*
 560       import core.memory : GC;
 561       lock = new core.sync.mutex.Mutex;
 562       GC.addRoot(*cast(void**)&lock);
 563       */
 564     }
 565
 566     void close () {
 567       import core.memory : GC;
 568       import core.stdc.stdlib : free;
 569       {
 570         //if (lock !is null) { import iv.writer; writeln("CLOSING!"); }
 571         if (lock !is null) lock.lock();
 572         scope(exit) if (lock !is null) lock.unlock();
 573         if (pkb.length) {
 574           inflateEnd(&zs);
 575           free(pkb.ptr);
 576           pkb = null;
 577         }
 578         if (fl !is null) {
 579           import core.stdc.stdio : fclose;
 580           fclose(fl);
 581           fl = null;
 582         }
 583         try { xfl.detach(); } catch (Exception) {} // it's safe to detach closed File
 584       }
 585       eoz = true;
 586       /*
 587       if (lock !is null) {
 588         GC.removeRoot(*cast(void**)&lock);
 589         delete lock;
 590         lock = null;
 591       }
 592       */
 593     }
 594
 595     private bool initZStream () {
 596       import core.stdc.stdlib : malloc, free;
 597       if (mode == Mode.Raw || pkb.ptr !is null) return true;
 598       // allocate buffer for packed data
 599       auto pb = cast(ubyte*)malloc(ibsize);
 600       if (pb is null) return false;
 601       pkb = pb[0..ibsize];
 602       zs.avail_in = 0;
 603       zs.avail_out = 0;
 604       // initialize unpacker
 605       // -15 is a magic value used to decompress zip files:
 606       // it has the effect of not requiring the 2 byte header and 4 byte trailer
 607       if (inflateInit2(&zs, (mode == Mode.Zip ? -15 : 15)) != Z_OK) {
 608         free(pb);
 609         pkb = null;
 610         return false;
 611       }
 612       // we are ready
 613       return true;
 614     }
 615
 616     private bool readPackedChunk () {
 617       import core.stdc.stdio : fread;
 618       import core.sys.posix.stdio : fseeko;
 619       if (zs.avail_in > 0) return true;
 620       if (pkpos >= pksize) return false;
 621       zs.next_in = cast(typeof(zs.next_in))pkb.ptr;
 622       zs.avail_in = cast(uint)(pksize-pkpos > ibsize ? ibsize : pksize-pkpos);
 623       if (fl !is null) {
 624         // `FILE*`
 625         if (fseeko(fl, stpos+pkpos, 0) < 0) return false;
 626         if (fread(pkb.ptr, zs.avail_in, 1, fl) != 1) return false;
 627       } else {
 628         // std.stdio.File
 629         try {
 630           xfl.seek(stpos+pkpos, 0);
 631           auto rd = xfl.rawRead(pkb[0..zs.avail_in]);
 632           if (rd.length != zs.avail_in) return false;
 633         } catch (Exception) { return false; } //BAD DOGGY!
 634       }
 635       pkpos += zs.avail_in;
 636       return true;
 637     }
 638
 639     private bool unpackNextChunk () {
 640       while (zs.avail_out > 0) {
 641         if (eoz) return false;
 642         if (!readPackedChunk()) return false;
 643         auto err = inflate(&zs, Z_SYNC_FLUSH);
 644         //if (err == Z_BUF_ERROR) { import iv.writer; writeln("*** OUT OF BUFFER!"); }
 645         if (err != Z_STREAM_END && err != Z_OK) return false;
 646         if (err == Z_STREAM_END) eoz = true;
 647       }
 648       return true;
 649     }
 650
 651
 652     ssize read (void* buf, size_t count) {
 653       if (buf is null) return -1;
 654       if (count == 0 || size == 0) return 0;
 655       lock.lock();
 656       scope(exit) lock.unlock();
 657       if (!isOpen) return -1; // read error
 658       if (pos >= size) return 0; // EOF
 659       if (mode == Mode.Raw) {
 660         import core.stdc.stdio : ferror, fread;
 661         import core.sys.posix.stdio : fseeko;
 662         if (size-pos < count) count = cast(size_t)(size-pos);
 663         if (fl !is null) {
 664           // `FILE*`
 665           if (fseeko(fl, stpos+pos, 0) < 0) return -1;
 666           auto rd = fread(buf, 1, count, fl);
 667           if (rd != count && (rd < 0 || ferror(fl))) rd = -1;
 668           if (rd > 0) pos += rd;
 669           return rd;
 670         } else {
 671           // std.stdio.File
 672           try {
 673             xfl.seek(stpos+pos, 0);
 674             auto rd = xfl.rawRead(buf[0..count]);
 675             pos += rd.length;
 676             return  (rd.length == count ? rd.length : -1);
 677           } catch (Exception) {} //BAD DOGGY!
 678           return -1;
 679         }
 680       } else {
 681         if (pkb.ptr is null && !initZStream()) return -1;
 682         // do we want to seek backward?
 683         if (prpos > pos) {
 684           // yes, rewind
 685           inflateEnd(&zs);
 686           zs = zs.init;
 687           pkpos = 0;
 688           if (!initZStream()) return -1;
 689           prpos = 0;
 690         }
 691         // do we need to seek forward?
 692         if (prpos < pos) {
 693           // yes, skip data
 694           ubyte[1024] tbuf;
 695           uint skp = pos-prpos;
 696           while (skp > 0) {
 697             uint rd = cast(uint)(skp > tbuf.length ? tbuf.length : skp);
 698             zs.next_out = cast(typeof(zs.next_out))tbuf.ptr;
 699             zs.avail_out = rd;
 700             if (!unpackNextChunk()) return -1;
 701             skp -= rd;
 702           }
 703           prpos = pos;
 704         }
 705         // unpack data
 706         if (size-pos < count) count = cast(size_t)(size-pos);
 707         zs.next_out = cast(typeof(zs.next_out))buf;
 708         zs.avail_out = cast(uint)count;
 709         if (!unpackNextChunk()) return -1;
 710         prpos = (pos += count);
 711         return count;
 712       }
 713     }
 714
 715     long seek (long ofs, int whence) {
 716       lock.lock();
 717       scope(exit) lock.unlock();
 718       if (!isOpen) return -1;
 719       //TODO: overflow checks
 720       switch (whence) {
 721         case 0: // SEEK_SET
 722           break;
 723         case 1: // SEEK_CUR
 724           ofs += pos;
 725           break;
 726         case 2: // SEEK_END
 727           if (ofs > 0) ofs = 0;
 728           ofs += size;
 729           break;
 730         default:
 731           return -1;
 732       }
 733       if (ofs < 0) return -1;
 734       if (ofs > size) ofs = size;
 735       pos = cast(uint)ofs;
 736       return ofs;
 737     }
 738   }
 739
 740
 741 static:
 742   //import iv.writer;
 743   // ////////////////////////////////////////////////////////////////////// //
 744   extern(C) nothrow {
 745     import core.sys.linux.stdio : cookie_io_functions_t;
 746     import core.sys.posix.sys.types : ssize, off64_t = off_t;
 747
 748     ssize fcdatpkRead (void* cookie, char* buf, size_t count) {
 749       //{ import iv.writer; writeln("reading ", count, " bytes"); }
 750       import core.stdc.errno;
 751       auto fc = cast(InnerFileCookied*)cookie;
 752       auto res = fc.read(buf, count);
 753       if (res < 0) { errno = EIO; return -1; }
 754       return res;
 755     }
 756
 757     ssize fcdatpkWrite (void* cookie, const(char)* buf, size_t count) {
 758       //{ import iv.writer; writeln("writing ", count, " bytes"); }
 759       import core.stdc.errno;
 760       errno = EIO; //FIXME: find better code
 761       return 0; // error; write should not return `-1`
 762     }
 763
 764     int fcdatpkSeek (void* cookie, off64_t* offset, int whence) {
 765       //{ import iv.writer; writeln("seeking ", *offset, " bytes, whence=", whence); }
 766       import core.stdc.errno;
 767       auto fc = cast(InnerFileCookied*)cookie;
 768       auto res = fc.seek(*offset, whence);
 769       if (res < 0) { errno = EIO; return -1; }
 770       *offset = cast(off64_t)res;
 771       return 0;
 772     }
 773
 774     int fcdatpkClose (void* cookie) {
 775       import core.memory : GC;
 776       import core.stdc.stdlib : free;
 777       //{ import iv.writer; writeln("closing"); }
 778       auto fc = cast(InnerFileCookied*)cookie;
 779       //fc.close();
 780       GC.removeRange(cookie);
 781       try { fc.__dtor(); } catch (Exception) {}
 782       // no need to run finalizers, we SHOULD NOT have any
 783       //try { GC.runFinalizers(cookie[0..InnerFileCookied.sizeof]); } catch (Exception) {}
 784       //fc.xfl.__dtor();
 785       free(cookie);
 786       //{ import iv.writer; writeln("closed"); }
 787       return 0;
 788     }
 789   }
 790
 791   __gshared cookie_io_functions_t fcdatpkCallbacks = cookie_io_functions_t(
 792     /*.read =*/ &fcdatpkRead,
 793     /*.write =*/ &fcdatpkWrite,
 794     /*.seek =*/ &fcdatpkSeek,
 795     /*.close =*/ &fcdatpkClose,
 796   );
 797
 798 static:
 799   T[] xalloc(T) (size_t len) {
 800     import core.stdc.stdlib : malloc;
 801     if (len < 1) return null;
 802     auto res = cast(T*)malloc(len*T.sizeof);
 803     if (res is null) {
 804       import core.exception : onOutOfMemoryErrorNoGC;
 805       onOutOfMemoryErrorNoGC();
 806     }
 807     res[0..len] = T.init;
 808     return res[0..len];
 809   }
 810
 811   void xfree(T) (ref T[] slc) {
 812     if (slc.ptr !is null) {
 813       import core.stdc.stdlib : free;
 814       free(slc.ptr);
 815     }
 816     slc = null;
 817   }
 818 }