chibackend/package.d

   1 /* E-Mail Client
   2  * coded by Ketmar // Invisible Vector <ketmar@ketmar.no-ip.org>
   3  * Understanding is not required. Only obedience.
   4  *
   5  * This program is free software: you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation, version 3 of the License ONLY.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  16  */
  17 module chibackend;
  18
  19 public import chibackend.sqbase;
  20 public import chibackend.decode;
  21 public import chibackend.parse;
  22 public import chibackend.mfilter;
  23 public import chibackend.mbuilder;
  24
  25 private import iv.strex;
  26 private import iv.vfs.util;
  27
  28
  29 // ////////////////////////////////////////////////////////////////////////// //
  30 // some utilities
  31
  32 // this marks the buffer as "opaque", so GC won't scan its contents
  33 // this is to avoid false positives in GC
  34 /+
  35 public void markGCOpaque (const(void)[] buf) /*nothrow @trusted*/ {
  36   import core.memory : GC;
  37   if (buf !is null && buf.length != 0 && GC.addrOf(buf.ptr)) {
  38     { import core.stdc.stdio : stderr, fprintf; fprintf(stderr, "X000!\n"); }
  39     GC.setAttr(buf.ptr, GC.BlkAttr.NO_SCAN/*|GC.BlkAttr.NO_INTERIOR*/);
  40     { import core.stdc.stdio : stderr, fprintf; fprintf(stderr, "X001!\n"); }
  41   }
  42 }
  43 +/
  44
  45
  46 public __gshared string chiroCLIMailPath = "~/Mail";
  47
  48 public void chiroParseCommonCLIArgs (ref string[] args) {
  49   for (usize idx = 1; idx < args.length; ) {
  50     string arg = args[idx];
  51     if (arg.length == 0) { ++idx; continue; }
  52     if (arg == "--") break;
  53
  54     usize argcount = 1;
  55     if (arg == "--mailpath") {
  56       argcount = 2;
  57       if (idx+1 >= args.length) throw new Exception("\"--mailpath\" expects argument");
  58       chiroCLIMailPath = args[idx+1];
  59     } else if (arg.strEquCI("--nocompress")) {
  60       ChiroCompressionLevel = 0;
  61     } else if (arg.strEquCI("--max")) {
  62       ChiroCompressionLevel = 666;
  63     } else if (arg.length == 2 && arg[0] == '-' && arg[1] >= '0' && arg[1] <= '9') {
  64       ChiroCompressionLevel = arg[1]-'0';
  65     } else {
  66       ++idx;
  67       continue;
  68     }
  69     foreach (usize c; idx+argcount..args.length) args[c-argcount] = args[c];
  70     args.length -= argcount;
  71   }
  72
  73   if (chiroCLIMailPath.length == 0) {
  74     chiroCLIMailPath = "./";
  75   } else if (chiroCLIMailPath[0] == '~') {
  76     char[] dpath = new char[chiroCLIMailPath.length+4096];
  77     dpath = expandTilde(dpath, chiroCLIMailPath);
  78     while (dpath.length > 1 && dpath[$-1] == '/') dpath = dpath[0..$-1];
  79     dpath ~= '/';
  80     chiroCLIMailPath = cast(string)dpath; // it is safe to cast here
  81   } else if (chiroCLIMailPath[$-1] == '/') {
  82     chiroCLIMailPath ~= '/';
  83   }
  84 }
  85
  86
  87 // ////////////////////////////////////////////////////////////////////////// //
  88 public string SysTimeToRFCString() (in auto ref Imp!"std.datetime".SysTime tm) {
  89   import std.datetime;
  90
  91   //Sun, 7 Dec 2014 16:04:04 +0200
  92   immutable string[7] daysOfWeekNames = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"];
  93   immutable string[12] monthNames = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
  94
  95   string tzstr (Duration utcOffset) {
  96     import std.format : format;
  97     immutable absOffset = abs(utcOffset);
  98     int hours;
  99     int minutes;
 100     absOffset.split!("hours", "minutes")(hours, minutes);
 101     return format(utcOffset < Duration.zero ? "-%02d%02d" : "+%02d%02d", hours, minutes);
 102   }
 103
 104   try {
 105     import std.format : format;
 106     auto dateTime = cast(DateTime)tm;
 107     return "%s, %d %s %d %02d:%02d:%02d %s".format(
 108       daysOfWeekNames[dateTime.dayOfWeek],
 109       dateTime.day,
 110       monthNames[dateTime.month-1],
 111       dateTime.year,
 112       dateTime.hour,
 113       dateTime.minute,
 114       dateTime.second,
 115       tzstr(tm.utcOffset),
 116     );
 117   } catch (Exception e) {
 118     assert(0, "format() threw.");
 119   }
 120 }
 121
 122
 123 // ////////////////////////////////////////////////////////////////////////// //
 124 public struct DynStr {
 125 package:
 126   static struct Data {
 127     usize udata = void;
 128     uint rc = void;
 129     usize used = void;
 130     usize alloted = void;
 131
 132     inout(char)* ptr () inout pure nothrow @trusted @nogc { pragma(inline, true); return cast(char*)udata; }
 133     void ptr (void *v) pure nothrow @trusted @nogc { pragma(inline, true); udata = cast(usize)v; }
 134   }
 135
 136 package:
 137   usize udata = 0;
 138
 139 package:
 140   inout(Data)* datap () inout pure nothrow @trusted @nogc { pragma(inline, true); return cast(Data*)udata; }
 141   void datap (Data *v) pure nothrow @trusted @nogc { pragma(inline, true); udata = cast(usize)v; }
 142
 143   void incRef () nothrow @trusted @nogc {
 144     pragma(inline, true);
 145     if (udata) ++((cast(Data*)udata).rc);
 146   }
 147
 148   void decRef () nothrow @trusted @nogc {
 149     pragma(inline, true);
 150     if (udata) {
 151       if (--((cast(Data*)udata).rc) == 0) {
 152         import core.stdc.stdlib : free;
 153         if ((cast(Data*)udata).udata) free((cast(Data*)udata).ptr);
 154       }
 155       udata = 0;
 156     }
 157   }
 158
 159 package:
 160   bool isMyData (const(void)* ptr) pure nothrow @trusted @nogc {
 161     //pragma(inline, true);
 162     if (!ptr || !udata || !datap.udata) return false;
 163     immutable usize cc = cast(usize)ptr;
 164     return (cc >= datap.udata && cc < datap.udata+datap.alloted);
 165   }
 166
 167 public:
 168   alias getData this;
 169
 170 public:
 171 nothrow @trusted @nogc:
 172   this (DynStr s) { pragma(inline, true); udata = s.udata; incRef(); }
 173
 174   this (const(char)[] s) {
 175     if (s.length) {
 176       ensure!false(s.length);
 177       datap.ptr[0..s.length] = s[];
 178       datap.used = s.length;
 179     }
 180   }
 181
 182   this (this) { pragma(inline, true); incRef(); }
 183   ~this () { pragma(inline, true); decRef(); }
 184
 185   void clear () { pragma(inline, true); decRef(); }
 186
 187   void compact () nothrow @trusted @nogc {
 188     if (!udata) return;
 189     Data* dp = datap;
 190     if (!dp.udata) return;
 191     if ((dp.used|0x1fU)+1U > dp.alloted) {
 192       immutable usize newsz = (dp.used|0x1fU)+1U;
 193       import core.stdc.stdlib : realloc;
 194       void *np = realloc(dp.ptr, newsz);
 195       if (np is null) return;
 196       dp.ptr = np;
 197       dp.alloted = newsz;
 198     }
 199   }
 200
 201   void ensure(bool overalloc) (usize sz) nothrow @trusted @nogc {
 202     if (sz == 0) return;
 203     static if (overalloc) {
 204            if (sz <= 256) sz = (sz|0x1fU)+1U;
 205       else if (sz <= 1024) sz = (sz|0x3fU)+1U;
 206       else if (sz <= 8192) sz = (sz|0x1ffU)+1U;
 207       else if (sz <= 16384) sz = (sz|0x3ffU)+1U;
 208       else if (sz <= 32768) sz = (sz|0xfffU)+1U;
 209       else sz = (sz|0x1fffU)+1U;
 210     } else {
 211       sz = (sz|0x1fU)+1U;
 212     }
 213     Data* dp = datap;
 214     if (dp is null) {
 215       import core.stdc.stdlib : calloc;
 216       dp = cast(Data*)calloc(1, Data.sizeof);
 217       if (dp is null) { import core.exception : onOutOfMemoryErrorNoGC; onOutOfMemoryErrorNoGC(); }
 218       datap = dp;
 219       dp.rc = 1U;
 220     }
 221     if (sz > dp.alloted) {
 222       import core.stdc.stdlib : realloc;
 223       void *np = realloc(dp.ptr, sz);
 224       if (np is null) { import core.exception : onOutOfMemoryErrorNoGC; onOutOfMemoryErrorNoGC(); }
 225       dp.ptr = np;
 226       dp.alloted = sz;
 227     }
 228   }
 229
 230   void makeUnique () nothrow @trusted @nogc {
 231     if (!udata || datap.rc == 1) return;
 232     immutable usize oldudata = udata;
 233     immutable usize oldused = datap.used;
 234     udata = 0;
 235     ensure!false(oldused);
 236     immutable usize xudata = udata;
 237     udata = oldudata;
 238     decRef();
 239     udata = xudata;
 240   }
 241
 242   void reserve(bool overalloc=true) (usize newsz) { pragma(inline, true); ensure!overalloc(newsz); }
 243
 244   void append(bool overalloc=true) (const(void)[] buf) {
 245     if (buf.length == 0) return;
 246     if (buf.length >= cast(usize)0x2fff_ffff) { import core.exception : onOutOfMemoryErrorNoGC; onOutOfMemoryErrorNoGC(); }
 247     if (udata && cast(usize)0x3000_0000-datap.used < buf.length) { import core.exception : onOutOfMemoryErrorNoGC; onOutOfMemoryErrorNoGC(); }
 248     DynStr tmpstr;
 249     if (isMyData(buf.ptr)) {
 250       tmpstr.set(buf[]);
 251       tmpstr.makeUnique();
 252       buf = cast(const(void)[])tmpstr.getData();
 253     }
 254     Data* data = void;
 255     if (udata) {
 256       data = datap;
 257       if (data.used+buf.length > data.alloted) {
 258         ensure!overalloc(data.used+buf.length);
 259         data = datap;
 260       }
 261     } else {
 262       ensure!overalloc(buf.length);
 263       data = datap;
 264     }
 265     data.ptr[data.used..data.used+buf.length] = cast(const(char)[])buf;
 266     data.used += buf.length;
 267   }
 268   void append (const char ch) { pragma(inline, true); append((&ch)[0..1]); }
 269
 270   void set (const(void)[] buf) {
 271     pragma(inline, true);
 272     if (buf.length == 0) {
 273       decRef();
 274     } else if (udata) {
 275       if (isMyData(buf.ptr)) {
 276         DynStr tmpstr;
 277         tmpstr.set(buf);
 278         makeUnique();
 279         append!false(tmpstr.getData);
 280       } else {
 281         makeUnique();
 282         datap.used = 0;
 283         append!false(buf);
 284       }
 285       compact();
 286     } else {
 287       append!false(buf);
 288     }
 289   }
 290   void set (const char ch) { pragma(inline, true); set((&ch)[0..1]); }
 291
 292   uint length () const pure { pragma(inline, true); return (udata ? cast(uint)datap.used : 0U); }
 293
 294   const(char)[] getData () const pure { pragma(inline, true); return (udata && datap.used ? datap.ptr[0..datap.used] : null); }
 295   const(ubyte)[] getBytes () const pure { pragma(inline, true); return cast(const(ubyte)[])(udata && datap.used ? datap.ptr[0..datap.used] : null); }
 296
 297   void opAssign() (const(char)[] s) { pragma(inline, true); set(s); }
 298   void opAssign() (const char ch) { pragma(inline, true); set(ch); }
 299   void opAssign() (DynStr s) { pragma(inline, true); if (s.udata != udata) set(s); }
 300
 301   void opOpAssign(string op:"~") (const(char)[] s) { pragma(inline, true); append(s); }
 302   void opOpAssign(string op:"~") (const char ch) { pragma(inline, true); append(ch); }
 303   void opOpAssign(string op:"~") (DynStr s) { pragma(inline, true); append(s); }
 304
 305   const(char)[] opCast(T:const(char)[]) () const pure { pragma(inline, true); return getData(); }
 306   const(ubyte)[] opCast(T:const(ubyte)[]) () const pure { pragma(inline, true); return getBytes(); }
 307
 308   bool opEqual (const(char)[] other) const pure { pragma(inline, true); return (other[] == getBytes()[]); }
 309   bool opEqual (const ref DynStr other) const pure { pragma(inline, true); return (other.getBytes()[] == getBytes()[]); }
 310
 311   int opCmp (const(char)[] other) const pure {
 312     if (!udata || datap.used == 0) return (other.length ? -1 : 0);
 313     if (other.length == 0) return 1;
 314     if (datap.ptr == other.ptr && datap.used == other.length) return 0;
 315     import core.stdc.string : memcmp;
 316     immutable int cres = memcmp(datap.ptr, other.ptr, (datap.used < other.length ? datap.used : other.length));
 317     if (cres != 0) return (cres < 0 ? -1 : +1);
 318     return (datap.used < other.length ? -1 : datap.used > other.length ? +1 : 0);
 319   }
 320
 321   int opCmp (const ref DynStr other) const { pragma(inline, true); return opCmp(other.getData); }
 322
 323   void appendQEncoded (const(void)[] ss) {
 324     static bool isSpecial (immutable char ch) pure nothrow @safe @nogc {
 325       return
 326         ch < ' ' ||
 327         ch >= 127 ||
 328         ch == '\'' ||
 329         ch == '`' ||
 330         ch == '"' ||
 331         ch == '\\' ||
 332         ch == '@';
 333     }
 334
 335     if (ss.length == 0) return;
 336     const(char)[] s = cast(const(char)[])ss;
 337
 338     static immutable string hexd = "0123456789abcdef";
 339
 340     bool needWork = (s[0] == '=' || s[0] == '?');
 341     if (!needWork) foreach (char ch; s) if (isSpecial(ch)) { needWork = true; break; }
 342
 343     if (!needWork) {
 344       append(s);
 345     } else {
 346       append("=?UTF-8?Q?"); // quoted printable
 347       foreach (char ch; s) {
 348         if (ch <= ' ') ch = '_';
 349         if (!isSpecial(ch) && ch != '=' && ch != '?') {
 350           append(ch);
 351         } else {
 352           append("=");
 353           append(hexd[(cast(ubyte)ch)>>4]);
 354           append(hexd[(cast(ubyte)ch)&0x0f]);
 355         }
 356       }
 357       append("?=");
 358     }
 359   }
 360
 361   void appendB64Encoded (const(void)[] buf, uint maxlinelen=76) {
 362     static immutable string b64alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
 363     ubyte[3] bts = void;
 364     uint btspos = 0;
 365     uint linelen = 0;
 366
 367     void putB64Char (immutable char ch) nothrow @trusted @nogc {
 368       if (maxlinelen && linelen >= maxlinelen) { append("\r\n"); linelen = 0; }
 369       append(ch);
 370       ++linelen;
 371     }
 372
 373     void encodeChunk () nothrow @trusted @nogc {
 374       if (btspos == 0) return;
 375       putB64Char(b64alphabet.ptr[(bts.ptr[0]&0xfc)>>2]);
 376       if (btspos == 1) {
 377         putB64Char(b64alphabet.ptr[(bts.ptr[0]&0x03)<<4]);
 378         /*static if (padding)*/ { putB64Char('='); putB64Char('='); }
 379       } else {
 380         // 2 or more
 381         putB64Char(b64alphabet.ptr[((bts.ptr[0]&0x03)<<4)|((bts.ptr[1]&0xf0)>>4)]);
 382         if (btspos == 2) {
 383           putB64Char(b64alphabet.ptr[(bts.ptr[1]&0x0f)<<2]);
 384           /*static if (padding)*/ putB64Char('=');
 385         } else {
 386           // 3 bytes
 387           putB64Char(b64alphabet.ptr[((bts.ptr[1]&0x0f)<<2)|((bts.ptr[2]&0xc0)>>6)]);
 388           putB64Char(b64alphabet.ptr[bts.ptr[2]&0x3f]);
 389         }
 390       }
 391       btspos = 0;
 392     }
 393
 394     foreach (immutable ubyte ib; (cast(const(ubyte)[])buf)[]) {
 395       bts.ptr[btspos++] = ib;
 396       if (btspos == 3) encodeChunk();
 397     }
 398     if (btspos != 0) encodeChunk();
 399
 400     if (maxlinelen) append("\r\n");
 401   }
 402
 403   void appendNum(T) (const T v) if (__traits(isIntegral, T)) {
 404     import core.stdc.stdio : snprintf;
 405     static if (T.sizeof <= 4) {
 406       char[32] buf = void;
 407       static if (__traits(isUnsigned, T)) {
 408         auto len = snprintf(buf.ptr, buf.length, "%u", cast(uint)v);
 409       } else {
 410         auto len = snprintf(buf.ptr, buf.length, "%d", cast(int)v);
 411       }
 412       append(buf[0..len]);
 413     } else {
 414       char[128] buf = void;
 415       static if (__traits(isUnsigned, T)) {
 416         auto len = snprintf(buf.ptr, buf.length, "%llu", cast(ulong)v);
 417       } else {
 418         auto len = snprintf(buf.ptr, buf.length, "%lld", cast(long)v);
 419       }
 420       append(buf[0..len]);
 421     }
 422   }
 423
 424   void removeASCIICtrls () {
 425     static bool isCtrl (const char ch) pure nothrow @safe @nogc {
 426       pragma(inline, true);
 427       return (ch < 32 && ch != '\t' && ch != '\n');
 428     }
 429
 430     bool needWork = false;
 431     foreach (immutable char ch; getData) if (isCtrl(ch)) { needWork = true; break; }
 432     if (!needWork) return;
 433
 434     makeUnique();
 435     char[] buf = cast(char[])datap.ptr[0..datap.used];
 436     foreach (ref char ch; buf) if (isCtrl(ch)) ch = ' ';
 437   }
 438
 439   // only for ASCII
 440   void lowerInPlace () {
 441     bool needWork = false;
 442     foreach (immutable char ch; getData) if (ch >= 'A' && ch <= 'Z') { needWork = true; break; }
 443     if (!needWork) return;
 444
 445     makeUnique();
 446     char[] buf = cast(char[])datap.ptr[0..datap.used];
 447     foreach (ref char ch; buf) if (ch >= 'A' && ch <= 'Z') ch += 32;
 448   }
 449 }
 450
 451
 452 // ////////////////////////////////////////////////////////////////////////// //
 453 // simple exponential running average
 454 struct RunningAverageExp {
 455 protected:
 456   double mFadeoff = 0.1; // 10%
 457   double mCurrValue = 0.0;
 458   ulong mStartTime = 0;
 459   uint mProcessed = 0;
 460   uint mTotal = 0;
 461   uint mProgTrh = 1024;
 462   ulong mTimerNextUpdate = 0;
 463
 464 public nothrow @trusted @nogc:
 465   this (const double aFadeoff) { pragma(inline, true);mFadeoff = aFadeoff; }
 466
 467   void reset () { pragma(inline, true);mCurrValue = 0.0; }
 468
 469   @property double fadeoff () const pure { pragma(inline, true);return mFadeoff; }
 470   @property void fadeoff (const double aFadeoff) { pragma(inline, true);mFadeoff = aFadeoff; }
 471
 472   void update (const double newValue) { pragma(inline, true);mCurrValue = mFadeoff*newValue+(1.0-mFadeoff)*mCurrValue; }
 473
 474   @property double value () const pure { pragma(inline, true);return mCurrValue; }
 475   @property void value (const double aValue) { pragma(inline, true);mCurrValue = aValue; }
 476
 477   @property uint uintValue () const pure { pragma(inline, true);return cast(uint)mCurrValue; }
 478
 479   void startTimer (uint total) {
 480     mStartTime = GetTickCount();
 481     mProcessed = 0;
 482     mTotal = total;
 483     mCurrValue = 0.0;
 484     mTimerNextUpdate = 0;
 485   }
 486
 487   @property uint progressThreshold () const pure { pragma(inline, true); return mProgTrh; }
 488   @property void progressThreshold (uint trh) { pragma(inline, true); mProgTrh = trh; }
 489
 490   @property uint timerTotal () const pure { pragma(inline, true); return mTotal; }
 491   @property void timerTotal (uint total) { pragma(inline, true); mTotal = total; }
 492
 493   // returns `true` if it is time to show progress
 494   bool updateProcessed (uint count, bool delta=true) {
 495     if (!count) return false;
 496     immutable uint prev = mProcessed;
 497     if (delta) mProcessed += count; else mProcessed = count;
 498     if (mProgTrh == 0 || prev == 0) return true;
 499     return (prev/mProgTrh != mProcessed/mProgTrh);
 500   }
 501
 502   // seconds
 503   void getTimeETA() (out uint time, out uint eta, out uint curr, out uint total, out uint percent, ulong ctime=ulong.max) {
 504     if (ctime == ulong.max) ctime = GetTickCount();
 505     immutable uint secs = cast(uint)(ctime-mStartTime);
 506     time = secs;
 507     if (secs && mTotal && mProcessed) {
 508       //secs/count*total
 509       eta = cast(uint)(cast(ulong)secs*mTotal/mProcessed);
 510       if (mCurrValue < eta) mCurrValue = eta;
 511       update(eta);
 512       if (mCurrValue < eta) mCurrValue = eta;
 513     }
 514     eta = cast(uint)mCurrValue;
 515
 516     curr = mProcessed;
 517     total = mTotal;
 518     percent = (mTotal ? cast(uint)(cast(ulong)mProcessed*100U/mTotal) : 0U);
 519   }
 520
 521   bool updateProcessedWithProgress (uint count, bool delta=true, bool forceWrite=false, bool writeNL=false) {
 522     if (!updateProcessed(count, delta)) {
 523       if (!forceWrite) return false;
 524     }
 525
 526     immutable ctime = GetTickCount();
 527     if (!forceWrite && mTimerNextUpdate > ctime) return false;
 528     mTimerNextUpdate = ctime+1;
 529
 530     uint secs, eta, curr, tot, prc;
 531     getTimeETA(out secs, out eta, out curr, out tot, out prc, ctime);
 532
 533     char[128] buf = void;
 534     uint h, m, s;
 535
 536     import core.stdc.stdio : snprintf;
 537     auto len = snprintf(buf.ptr, buf.length, "\r%s[%u/%u] %3u%%", (writeNL ? "".ptr : " ".ptr), curr, tot, prc);
 538     extractHMS(secs, out h, out m, out s);
 539     if (len < buf.length) len += snprintf(buf.ptr+len, buf.length-len, "  %02u:%02u:%02u", h, m, s);
 540     extractHMS(eta, out h, out m, out s);
 541     if (len < buf.length) len += snprintf(buf.ptr+len, buf.length-len, "  %02u:%02u:%02u", h, m, s);
 542     if (len < buf.length) len += snprintf(buf.ptr+len, buf.length-len, "\x1b[K%c", (writeNL ? '\n' : '\r'));
 543
 544     import core.sys.posix.unistd : write, STDOUT_FILENO;
 545     write(STDOUT_FILENO, buf.ptr, cast(usize)len);
 546
 547     return true;
 548   }
 549
 550   static extractHMS (uint secs, out uint h, out uint m, out uint s) {
 551     s = secs%60; secs /= 60;
 552     m = secs%60; secs /= 60;
 553     h = secs;
 554   }
 555
 556   static ulong GetTickCount() () {
 557     import core.time;
 558     immutable MonoTime ctt = MonoTime.currTime;
 559     return cast(ulong)ctt.ticks/cast(ulong)ctt.ticksPerSecond;
 560   }
 561 };