vfs/samples/xzippack.d

   1 #!/usr/bin/env rdmdopt
   2 module xzippack is aliced;
   3
   4 import iv.cmdcon;
   5 import iv.glob;
   6 import iv.strex;
   7 import iv.unarray;
   8 import iv.vfs;
   9 import iv.vfs.util;
  10 import iv.vfs.writers.zip;
  11
  12
  13 bool ignoreKnownExts = false;
  14
  15
  16 // ////////////////////////////////////////////////////////////////////////// //
  17 struct FileInfo {
  18   enum Type { Normal, Dir }
  19
  20   string name;
  21   ulong size;
  22   Type type;
  23   uint modtime;
  24   uint stmode;
  25   ulong inode;
  26
  27   this (const(char)[] aname) {
  28     import core.sys.posix.sys.stat;
  29     import std.internal.cstring : tempCString;
  30     stat_t st;
  31     if (stat(aname.tempCString, &st) != 0) throw new Exception("cannot stat '"~aname.idup~"'");
  32     if (st.st_mode.S_ISDIR) {
  33       type = Type.Dir;
  34     } else if (st.st_mode.S_ISREG) {
  35       type = Type.Normal;
  36       size = st.st_size;
  37     } else if (st.st_mode.S_ISLNK) {
  38       throw new Exception("don't know what to do with symlink '"~aname.idup~"'");
  39     } else {
  40       throw new Exception("don't know what to do with special file '"~aname.idup~"'");
  41     }
  42     name = aname.idup;
  43     if (type == Type.Dir && name[$-1] != '/') name ~= '/';
  44     modtime = st.st_mtime;
  45     inode = st.st_ino;
  46     stmode = st.st_mode;
  47   }
  48
  49   @property ushort unixmode () const pure nothrow @safe @nogc { pragma(inline, true); return (stmode&ushort.max); }
  50
  51   @property string baseName () const pure nothrow @safe @nogc {
  52     if (name.length == 0) return null;
  53     string res = name;
  54     if (type == Type.Dir) {
  55       if (res[$-1] != '/') assert(0, "internal error");
  56       if (res == "/") return res;
  57       res = res[0..$-1];
  58     }
  59     if (res.length == 0) assert(0, "internal error");
  60     foreach (immutable pos, char ch; res; reverse) {
  61       if (ch == '/') {
  62         if (pos+1 == res.length) assert(0, "internal error");
  63         return res[pos+1..$];
  64       }
  65     }
  66     return res;
  67   }
  68
  69   @property string dirName () const pure nothrow @safe @nogc {
  70     if (name.length == 0) return null;
  71     string res = name;
  72     if (type == Type.Dir) {
  73       if (res[$-1] != '/') assert(0, "internal error");
  74       if (res == "/") return res;
  75       res = res[0..$-1];
  76     }
  77     if (res.length == 0) assert(0, "internal error");
  78     foreach (immutable pos, char ch; res; reverse) {
  79       if (ch == '/') {
  80         if (pos == 0) return "/";
  81         return res[0..pos];
  82       }
  83     }
  84     return ".";
  85   }
  86
  87   @property bool valid () const pure nothrow @safe @nogc { pragma(inline, true); return (name.length != 0); }
  88   @property bool isDir () const pure nothrow @safe @nogc { pragma(inline, true); return (type == Type.Dir); }
  89   @property bool isFile () const pure nothrow @safe @nogc { pragma(inline, true); return (type == Type.Normal); }
  90
  91   bool opEquals() (in auto ref FileInfo fi) const pure nothrow @safe @nogc {
  92     pragma(inline, true);
  93     return (name == fi.name);
  94   }
  95
  96   int opCmp() (in auto ref FileInfo fi) const pure nothrow @safe @nogc {
  97     pragma(inline, true);
  98     if (name == fi.name) {
  99       if (type != fi.type) return (type == Type.Dir ? -1 : 1); // dirs are always first
 100       return 0;
 101     } else {
 102       return (name < fi.name ? -1 : name > fi.name ? 1 : 0);
 103     }
 104   }
 105 }
 106
 107
 108 // ////////////////////////////////////////////////////////////////////////// //
 109 __gshared FileInfo[] diskFileList;
 110 __gshared bool[ulong] diskFilesSeen; // by inode
 111 __gshared ulong totalDiskSize;
 112
 113
 114 void scanDisk (string nameorpath) {
 115   if (nameorpath.length == 0) return;
 116
 117   auto cfi = FileInfo(nameorpath);
 118   //conwriteln("nameorpath: [", nameorpath, "] : [", cfi.name, "]; inode=", cfi.inode, "; mode=", cfi.stmode);
 119
 120   if (cfi.inode in diskFilesSeen) return;
 121   diskFilesSeen[cfi.inode] = true;
 122   if (!cfi.isDir) {
 123     totalDiskSize += cfi.size;
 124     diskFileList.unsafeArrayAppend(cfi);
 125     //conwriteln("  FILE!");
 126     return;
 127   }
 128
 129   string[] dirs;
 130   scope(exit) delete dirs;
 131
 132   assert(cfi.name[$-1] == '/');
 133   foreach (Glob.Item it; Glob(cfi.name~"*", GLOB_NOSORT|GLOB_PERIOD|GLOB_TILDE_CHECK|GLOB_MARK)) {
 134     auto fi = FileInfo(it.name);
 135     //conwriteln(it.index, ": [", it.name, "] : basename=[", fi.baseName, "]; dirname=[", fi.dirName, "]");
 136     if (fi.inode in diskFilesSeen) continue;
 137     if (fi.isDir) {
 138       auto bname = fi.baseName;
 139       if (bname == "." || bname == "..") continue;
 140       dirs.unsafeArrayAppend(fi.name);
 141       // dir will be marked as visited later
 142     } else {
 143       diskFilesSeen[fi.inode] = true;
 144       totalDiskSize += fi.size;
 145     }
 146     if (fi.name.length == 2 && fi.name == "./") assert(0, "internal error");
 147     if (fi.name.length > 2 && fi.name[0..2] == "./") fi.name = fi.name[2..$];
 148     diskFileList.unsafeArrayAppend(fi);
 149   }
 150
 151   // recurse dirs
 152   foreach (string dname; dirs) {
 153     if (dname == "/") continue;
 154     if (dname[$-1] == '/') dname = dname[0..$-1];
 155     if (dname.length == 0 || dname == "." || dname == "..") continue;
 156     scanDisk(dname);
 157   }
 158 }
 159
 160
 161 void finalizeDiskScan () {
 162   import std.algorithm : sort;
 163   diskFileList.sort;
 164 }
 165
 166
 167 // ////////////////////////////////////////////////////////////////////////// //
 168 string n2s (ulong n) {
 169   string res;
 170   int left = 3;
 171   do {
 172     if (left == 0) { res = ","~res; left = 3; }
 173     res = cast(char)('0'+n%10)~res;
 174     --left;
 175   } while ((n /= 10) != 0);
 176   return res;
 177 }
 178
 179
 180 // ////////////////////////////////////////////////////////////////////////// //
 181 // returns zip file size
 182 ulong packZip (ConString outfname, FileInfo[] flist, ZipWriter.Method pmt) {
 183   import core.time;
 184   import std.string : format;
 185   import std.exception : collectException;
 186   import std.file : chdir, getcwd, mkdirRecurse, remove, rmdirRecurse;
 187   import std.path : absolutePath, expandTilde;
 188   import std.process;
 189
 190   collectException(outfname.remove());
 191   auto fo = VFile(outfname.idup.expandTilde.absolutePath, "w");
 192   auto zw = new ZipWriter(fo);
 193   scope(failure) {
 194     if (zw.isOpen) zw.abort();
 195     fo.close();
 196     collectException(outfname.remove());
 197   }
 198   bool[string] fileseen;
 199   foreach (immutable flistidx, const ref de; flist) {
 200     if (de.name in fileseen) continue;
 201     fileseen[de.name] = true;
 202     conwrite("  [", n2s(flistidx+1), "/", n2s(flist.length), "] ", de.name, " ... ");
 203     uint zipidx;
 204     try {
 205       if (de.isDir) {
 206         zipidx = zw.appendDir(de.name, ZipFileTime(de.modtime));
 207         conwriteln("OK");
 208       } else {
 209         ZipWriter.Method rmt = pmt;
 210         if (!ignoreKnownExts && ZipWriter.IsKnownUncompressableByExt(de.name)) rmt = ZipWriter.Method.Store;
 211         ulong origsz = de.size;
 212         conwrite("  0%");
 213         int oldprc = 0;
 214         MonoTime lastProgressTime = MonoTime.currTime;
 215         // don't ignore case
 216         zipidx = zw.pack(VFile(de.name, "IZ"), de.name, ZipFileTime(de.modtime), rmt, de.size, delegate (ulong curpos) {
 217           int prc = (curpos > 0 ? cast(int)(cast(ulong)100*curpos/origsz) : 0);
 218           if (prc != oldprc) {
 219             auto stt = MonoTime.currTime;
 220             if ((stt-lastProgressTime).total!"msecs" >= 1000) {
 221               lastProgressTime = stt;
 222               if (prc < 0) prc = 0; else if (prc > 100) prc = 100;
 223               conwritef!"\x08\x08\x08\x08%3u%%"(cast(uint)prc);
 224               oldprc = prc;
 225             } else {
 226               //conwriteln(curpos, " : ", origsz);
 227             }
 228           }
 229         });
 230         conwritefln!"\x08\x08\x08\x08[%s] %s -> %s (%s%%)"(zw.files[zipidx].methodName, n2s(de.size), n2s(zw.files[zipidx].pksize),
 231           (de.size > 0 ? cast(uint)(100UL*zw.files[zipidx].pksize/de.size) : 0), // left from unpacked
 232         );
 233         //if (zw.files[zidx].crc != de.stat("crc32").get!uint) throw new Exception("crc error!");
 234       }
 235       zw.files[zipidx].unixmode = de.unixmode;
 236     } catch (Exception e) {
 237       conwriteln("ERROR: ", e.msg);
 238       throw e;
 239     }
 240   }
 241   zw.finish();
 242   return fo.size;
 243 }
 244
 245
 246 // ////////////////////////////////////////////////////////////////////////// //
 247 void main (string[] args) {
 248   auto method = ZipWriter.Method.Lzma;
 249
 250   for (usize idx = 1; idx < args.length;) {
 251     string arg = args[idx];
 252     if (arg == "--") {
 253       import std.algorithm : remove;
 254       args = args.remove(idx);
 255       break;
 256     }
 257     if (arg.length == 0) {
 258       import std.algorithm : remove;
 259       args = args.remove(idx);
 260       continue;
 261     }
 262     if (arg[0] == '-') {
 263       switch (arg) {
 264         case "--lzma": method = ZipWriter.Method.Lzma; break;
 265         case "--store": method = ZipWriter.Method.Store; break;
 266         case "--deflate": method = ZipWriter.Method.Deflate; break;
 267         case "--force": ignoreKnownExts = true; break;
 268         default: conwriteln("invalid argument: '", arg, "'"); throw new Exception("boom");
 269       }
 270       import std.algorithm : remove;
 271       args = args.remove(idx);
 272       continue;
 273     }
 274     ++idx;
 275   }
 276
 277   conwriteln("using '", method, "' method...");
 278
 279   if (args.length < 3) assert(0, "arcname?");
 280
 281   string outfname = args[1];
 282   if (!outfname.endsWithCI(".zip") && !outfname.endsWithCI(".pk3")) outfname ~= ".zip";
 283
 284   conwriteln("scanning...");
 285   foreach (string dpath; args[2..$]) scanDisk(dpath);
 286   finalizeDiskScan();
 287   if (diskFileList.length == 0) assert(0, "no files!");
 288   conwriteln(diskFileList.length, " file", (diskFileList.length != 1 ? "s" : ""), " found, ", n2s(totalDiskSize), " bytes.");
 289
 290   /*
 291   foreach (const ref fi; list) {
 292     import core.stdc.time : localtime, strftime;
 293     char[1024] buf = void;
 294     auto tmx = localtime(cast(int*)&fi.modtime);
 295     auto len = strftime(buf.ptr, buf.length, "%Y/%m/%d %H:%M:%S", tmx);
 296     conwriteln(fi.name, " [", fi.type, "]  ", fi.size, "  ", buf[0..len]);
 297   }
 298   */
 299
 300   conwriteln("creating '", outfname, "'...");
 301   ulong fsize = packZip(outfname, diskFileList, method);
 302   conwritefln!"DONE, TOTALS: %s -> %s %3u%%"(n2s(totalDiskSize), n2s(fsize), (totalDiskSize ? cast(uint)(100*fsize/totalDiskSize) : 100));
 303 }