sqbase_experiment/zsq_convert.d

   1 // this converts old chiroptera mailbox CHMT database to new SQLite database
   2 // this only puts message data into "messages" table; you need to rebuild
   3 // other tables with the separate utility
   4 module zsq_convert is aliced;
   5
   6 import chiro_sqbase;
   7 import chiro_decode;
   8 import chiro_parse;
   9
  10
  11 import iv.sq3;
  12 import iv.strex;
  13 import iv.vfs;
  14 import iv.vfs.io;
  15
  16
  17 ////////////////////////////////////////////////////////////////////////////////
  18 void main (string[] args) {
  19   int docompress = 1;
  20
  21   if (args.length > 1) {
  22     foreach (string s; args[1..$]) {
  23       if (s.strEquCI("--nocompress")) docompress = 0;
  24     }
  25   }
  26
  27   writeln("reading text database...");
  28   char[] alltext;
  29   {
  30     auto fl = VFile("index.chmt");
  31     auto sz = fl.size;
  32     if (sz > 0x3fffffffU) throw new Exception("text database too big");
  33     if (sz > 0) {
  34       alltext = new char[cast(uint)sz];
  35       fl.rawReadExact(alltext[]);
  36     }
  37   }
  38
  39   writeln("creating db...");
  40   auto db = chiroRecreateDB();
  41
  42   auto stInsMsg = db.statement("INSERT INTO messages (uid,headers,body) VALUES(NULL,ChiroPack(ChiroNormHeaders(:hdrs),:dopack),ChiroPack(:body,:dopack));");
  43   //db.execute("begin transaction;");
  44
  45   usize allsize = alltext.length;
  46   uint count = 0;
  47
  48   while (alltext.length) {
  49     usize end = findMessageEnd(alltext);
  50     const(char)[] msg = alltext[0..end];
  51     usize hdrend = findHeadersEnd(msg);
  52     const(char)[] hdrs = msg[0..hdrend];
  53     hdrend = skipOneLine(msg, hdrend);
  54     const(char)[] body = msg[hdrend..$];
  55
  56     // nobody needs final blanks; no, really
  57     // it is forbidden to use binary encoding in emails anyway, so don't bother keeping them intact
  58     hdrs = hdrs.xstripright;
  59     body = body.xstripright;
  60
  61     stInsMsg
  62       .bindBlob(":hdrs", hdrs)
  63       .bindBlob(":body", body)
  64       .bind(":dopack", docompress)
  65       .doAll();
  66
  67     version(none) {
  68       auto msgid = findHeaderField(hdrs, "Message-ID");
  69       if (msgid.length) writeln("MSGID: <", msgid, ">");
  70     }
  71
  72     version(none) {
  73       import iv.encoding;
  74       auto subj = findHeaderField(hdrs, "Subject").decodeSubj.subjRemoveRe;
  75       auto from = findHeaderField(hdrs, "From").decodeSubj;
  76       writeln(" FROM: {", from.recodeToKOI8, "}");
  77       writeln(" SUBJ: {", subj.recodeToKOI8, "}");
  78     }
  79
  80     alltext = cutTopMessage(alltext);
  81
  82     ++count;
  83     if (count%64 == 1) {
  84       usize cpos = allsize-alltext.length;
  85       write(count, " ", cast(ulong)cpos*100u/cast(ulong)allsize, "%\r");
  86       //db.execute("commit transaction;");
  87       //db.execute("begin transaction;");
  88     }
  89   }
  90
  91   writeln(count, " messages found.");
  92   //writeln("commiting final transaction...");
  93   //db.execute("commit transaction;");
  94
  95   writeln("closing the db");
  96   db.close();
  97 }