filterengine.d

   1 /* E-Mail Client
   2  * coded by Ketmar // Invisible Vector <ketmar@ketmar.no-ip.org>
   3  * Understanding is not required. Only obedience.
   4  *
   5  * This program is free software: you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation, either version 3 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  17  */
  18 module filterengine is aliced;
  19 private:
  20
  21 import iv.cmdcon;
  22 import iv.encoding;
  23 import iv.strex;
  24
  25 import maildb;
  26
  27
  28 // ////////////////////////////////////////////////////////////////////////// //
  29 public struct FilterData {
  30   Article art; // in/out
  31   string destfolder; // in/out; set to null to delete
  32   Bogo bogo = Bogo.Error; // "error" means "not checked" here
  33   bool markRead;
  34   bool softDelete;
  35 }
  36
  37
  38 // ////////////////////////////////////////////////////////////////////////// //
  39 private abstract class FilterAction {
  40   bool perform (ref FilterData fda); // return `true` to stop
  41
  42   static FilterAction parse (ref string[] tokens) {
  43     if (tokens.length == 0) return null;
  44     //TODO: use CTFE to build list
  45     if (tokens[0] == "move") {
  46       if (tokens.length < 2) throw new Exception("invalid move rule");
  47       auto res = new FilterActionMove(tokens[1]);
  48       tokens = tokens[2..$];
  49       return res;
  50     }
  51     if (tokens[0] == "exec") {
  52       if (tokens.length < 2) throw new Exception("invalid exec rule");
  53       auto res = new FilterActionExec(tokens[1]);
  54       tokens = tokens[2..$];
  55       return res;
  56     }
  57     if (tokens[0] == "delete") { tokens = tokens[1..$]; return new FilterActionDelete(); }
  58     if (tokens[0] == "spam") { tokens = tokens[1..$]; return new FilterActionSpam(); }
  59     if (tokens[0] == "ham") { tokens = tokens[1..$]; return new FilterActionHam(); }
  60     if (tokens[0] == "read") { tokens = tokens[1..$]; return new FilterActionRead(); }
  61     return null;
  62   }
  63 }
  64
  65
  66 // args[1]: article (without dot, dot-unstuffed)
  67 // write string to stdout, with tokens:
  68 //   ham
  69 //   spam
  70 //   delete
  71 //   softdelete
  72 //   read
  73 //   move dest
  74 // start line with "-" to stop action
  75 private class FilterActionExec : FilterAction {
  76   string command;
  77
  78   this (string acommand) {
  79     if (acommand.length == 0) throw new Exception("can't exec nothing");
  80     command = acommand;
  81   }
  82
  83   override bool perform (ref FilterData fda) {
  84     import std.stdio : File;
  85     import std.process;
  86     try {
  87       // write article to file
  88       import std.uuid;
  89       UUID id = randomUUID();
  90       char[] buf;
  91       void deleteTempFile () {
  92         if (buf.length) try { import std.file : remove; remove(buf); } catch (Exception e) {}
  93       }
  94       scope(exit) {
  95         deleteTempFile();
  96         delete buf;
  97       }
  98       buf.reserve(2+16*2+42);
  99       buf ~= "/tmp/_temp_";
 100       foreach (immutable ubyte b; id.data[]) {
 101         buf ~= "0123456789abcdef"[b>>4];
 102         buf ~= "0123456789abcdef"[b&0x0f];
 103       }
 104       buf ~= ".eml";
 105       {
 106         auto fo = File(buf, "w");
 107         foreach (ConString s; fda.art.allLinesIterator) fo.writeln(s);
 108         fo.close();
 109       }
 110       conwriteln("EXEC filter '", command, "'...");
 111       auto pid = pipeProcess([command, buf], Redirect.all, null, Config.none, "/tmp");
 112       string action = pid.stdout.readln.xstrip;
 113       bool doStop = (action.length && action[0] == '-');
 114       if (doStop) action = action[1..$].xstrip;
 115       pid.pid.wait();
 116       conwriteln("EXEC filter '", command, "' action: ", action, " (", doStop, ")");
 117            if (action == "spam") fda.bogo = Bogo.Spam;
 118       else if (action == "ham") fda.bogo = Bogo.Ham;
 119       else if (action == "delete") fda.destfolder = null;
 120       else if (action == "softdelete") fda.softDelete = true;
 121       else if (action == "read") fda.markRead = true;
 122       else if (action == "noop") {}
 123       else if (action == "nothing") {}
 124       else if (action.startsWith("move")) {
 125         if (action.length < 6 || action[4] > ' ') throw new Exception("invalid action: '"~action~"'");
 126         action = action[5..$].xstrip;
 127         if (action.length > 1 && action[0] == '"' && action[$-1] == '"') action = action[1..$-1];
 128         if (action.length == 0) throw new Exception("can't move to empty folder");
 129         fda.destfolder = action;
 130       } else if (action.length) {
 131         throw new Exception("invalid action: '"~action~"'");
 132       }
 133       return doStop;
 134     } catch (Exception e) {
 135       conwriteln("EXEC filter error: ", e.msg);
 136     }
 137     return false;
 138   }
 139 }
 140
 141
 142 private class FilterActionMove : FilterAction {
 143   string folder;
 144
 145   this (string afolder) {
 146     if (afolder.length == 0) throw new Exception("can't move to empty folder");
 147     folder = afolder;
 148   }
 149
 150   override bool perform (ref FilterData fda) {
 151     conwriteln("MOVE article from '", fda.art.fromname.recodeToKOI8, "' <", fda.art.frommail.recodeToKOI8, "> (", fda.art.subj.recodeToKOI8, ") -> ", folder);
 152     fda.destfolder = folder;
 153     return false;
 154   }
 155 }
 156
 157
 158 // final action
 159 private class FilterActionDelete : FilterAction {
 160   override bool perform (ref FilterData fda) {
 161     conwriteln("DELETE article from '", fda.art.fromname.recodeToKOI8, "' <", fda.art.frommail.recodeToKOI8, "> (", fda.art.subj.recodeToKOI8, ")");
 162     fda.destfolder = null;
 163     return true;
 164   }
 165 }
 166
 167
 168 private class FilterActionSpam : FilterAction {
 169   override bool perform (ref FilterData fda) {
 170     conwriteln("SPAM article from '", fda.art.fromname.recodeToKOI8, "' <", fda.art.frommail.recodeToKOI8, "> (", fda.art.subj.recodeToKOI8, ")");
 171     fda.bogo = Bogo.Spam;
 172     return false;
 173   }
 174 }
 175
 176
 177 private class FilterActionHam : FilterAction {
 178   override bool perform (ref FilterData fda) {
 179     conwriteln("HAM article from '", fda.art.fromname.recodeToKOI8, "' <", fda.art.frommail.recodeToKOI8, "> (", fda.art.subj.recodeToKOI8, ")");
 180     fda.bogo = Bogo.Ham;
 181     return false;
 182   }
 183 }
 184
 185
 186 private class FilterActionRead : FilterAction {
 187   override bool perform (ref FilterData fda) {
 188     fda.markRead = true;
 189     return false;
 190   }
 191 }
 192
 193
 194 // ////////////////////////////////////////////////////////////////////////// //
 195 private abstract class FilterCompare {
 196   bool check (const(char)[] s);
 197
 198   static FilterCompare parse (ref string[] tokens) {
 199     if (tokens.length == 0) return null;
 200     //TODO: use CTFE to build list
 201     if (tokens[0] == "matchcase") {
 202       if (tokens.length < 2) throw new Exception("invalid matchcase rule");
 203       if (tokens[1].length == 0) throw new Exception("matchcase with nothing");
 204       auto res = new FilterCompareMatchCase(tokens[1]);
 205       tokens = tokens[2..$];
 206       return res;
 207     }
 208     return null;
 209   }
 210 }
 211
 212 private class FilterCompareMatchCase : FilterCompare {
 213   string pat;
 214
 215   this (string apat) { pat = apat; }
 216
 217   override bool check (const(char)[] s) { return (s.indexOf(pat) >= 0); }
 218 }
 219
 220
 221 // ////////////////////////////////////////////////////////////////////////// //
 222 private abstract class FilterChecker {
 223   static FilterChecker parse (ref string[] tokens) {
 224     if (tokens.length == 0) return null;
 225     //TODO: use CTFE to build list
 226     if (tokens[0] == "fromname") {
 227       if (tokens.length < 2) throw new Exception("invalid fromname rule");
 228       tokens = tokens[1..$];
 229       auto res = new FilterCheckerFromName();
 230       res.cmp = FilterCompare.parse(tokens);
 231       if (res.cmp is null) throw new Exception("invalid fromname rule");
 232       return res;
 233     }
 234     if (tokens[0] == "frommail") {
 235       if (tokens.length < 2) throw new Exception("invalid frommail rule");
 236       tokens = tokens[1..$];
 237       auto res = new FilterCheckerFromMail();
 238       res.cmp = FilterCompare.parse(tokens);
 239       if (res.cmp is null) throw new Exception("invalid frommail rule");
 240       return res;
 241     }
 242     if (tokens[0] == "header") {
 243       if (tokens.length < 2) throw new Exception("invalid header rule");
 244       auto res = new FilterCheckerHeaderField(tokens[1]);
 245       tokens = tokens[2..$];
 246       res.cmp = FilterCompare.parse(tokens);
 247       if (res.cmp is null) throw new Exception("invalid frommail rule");
 248       return res;
 249     }
 250     if (tokens[0] == "to") {
 251       if (tokens.length < 2) throw new Exception("invalid to rule");
 252       auto res = new FilterCheckerHeaderField("to");
 253       tokens = tokens[1..$];
 254       res.cmp = FilterCompare.parse(tokens);
 255       if (res.cmp is null) throw new Exception("invalid frommail rule");
 256       return res;
 257     }
 258     if (tokens[0] == "to_or_cc") {
 259       if (tokens.length < 2) throw new Exception("invalid to_or_cc rule");
 260       tokens = tokens[1..$];
 261       auto c0 = new FilterCheckerHeaderField("to");
 262       c0.cmp = FilterCompare.parse(tokens);
 263       if (c0.cmp is null) throw new Exception("invalid to_or_cc rule");
 264       auto c1 = new FilterCheckerHeaderField("cc");
 265       c1.cmp = c0.cmp;
 266       auto res = new FilterCheckerBinExpr!"or"(c0, c1);
 267       return res;
 268     }
 269     return null;
 270   }
 271
 272   static FilterChecker parseCheckers (ref string[] tokens) {
 273     FilterChecker res = parse(tokens);
 274     if (res is null || tokens.length == 0) return res;
 275     if (tokens[0] == "|" || tokens[0] == "&") {
 276       if (tokens.length < 2) throw new Exception("invalid logic op");
 277       string op = tokens[0];
 278       tokens = tokens[1..$];
 279       FilterChecker right = parseCheckers(tokens);
 280       if (right is null) throw new Exception("invalid logic op");
 281            if (op == "|") res = new FilterCheckerBinExpr!"or"(res, right);
 282       else if (op == "&") res = new FilterCheckerBinExpr!"and"(res, right);
 283       else throw new Exception("invalid logic op");
 284     }
 285     return res;
 286   }
 287
 288   bool check (ref FilterData fda);
 289 }
 290
 291
 292 // ////////////////////////////////////////////////////////////////////////// //
 293 private class FilterCheckerBinExpr(string op) : FilterChecker if (op == "or" || op == "and") {
 294   FilterChecker left, right;
 295
 296   this (FilterChecker l, FilterChecker r) { left = l; right = r; }
 297
 298   override bool check (ref FilterData fda) {
 299     bool res = (left !is null ? left.check(fda) : false);
 300     static if (op == "or") {
 301       if (res) return true;
 302       if (right is null) return false;
 303       return right.check(fda);
 304     } else static if (op == "and") {
 305       if (!res) return false;
 306       if (right is null) return false;
 307       return right.check(fda);
 308     } else {
 309       static assert(0, "wtf?!");
 310     }
 311   }
 312 }
 313
 314
 315 // ////////////////////////////////////////////////////////////////////////// //
 316 private abstract class FilterCheckerHeaderCompare : FilterChecker {
 317   FilterCompare cmp;
 318
 319   override bool check (ref FilterData fda) {
 320     if (cmp is null) return false;
 321     return cmp.check(getCompareString(fda));
 322   }
 323
 324   const(char)[] getCompareString (ref FilterData fda);
 325 }
 326
 327 private class FilterCheckerFromName : FilterCheckerHeaderCompare {
 328   override const(char)[] getCompareString (ref FilterData fda) { return fda.art.fromname; }
 329 }
 330
 331 private class FilterCheckerFromMail : FilterCheckerHeaderCompare {
 332   override const(char)[] getCompareString (ref FilterData fda) { return fda.art.frommail; }
 333 }
 334
 335 private class FilterCheckerHeaderField : FilterCheckerHeaderCompare {
 336   string fldname;
 337
 338   this (string afldname) {
 339     if (afldname.length == 0) throw new Exception("no field name");
 340     fldname = afldname;
 341   }
 342
 343   override const(char)[] getCompareString (ref FilterData fda) {
 344     auto hproc = fda.art.headersIterator;
 345     while (!hproc.empty) {
 346       if (hproc.fieldName.strEquCI(fldname)) return hproc.curline;
 347       hproc.popFront();
 348     }
 349     return null;
 350   }
 351 }
 352
 353
 354 // ////////////////////////////////////////////////////////////////////////// //
 355 public final class Filter {
 356 public:
 357   string name;
 358   FilterChecker checker;
 359   FilterAction[] actions;
 360   bool stop;
 361
 362 public:
 363   this (ConString cli) {
 364     string[] tokens;
 365     for (;;) {
 366       auto tk = ConCommand.getWord(cli);
 367       if (tk is null) break;
 368       tokens ~= tk.idup;
 369     }
 370     if (tokens.length < 1) throw new Exception("invalid filter: no name");
 371     if (tokens[0].length == 0) throw new Exception("invalid filter: empty name");
 372     if (tokens.length < 2) throw new Exception("invalid filter '"~tokens[0]~"': no rules");
 373     //conwriteln("=======================");
 374     name = tokens[0];
 375     tokens = tokens[1..$];
 376     //conwriteln("00tokens: ", tokens);
 377     try {
 378       checker = FilterChecker.parseCheckers(tokens);
 379       //conwriteln("01tokens: ", tokens);
 380       if (checker is null) throw new Exception("no checker");
 381       while (tokens.length) {
 382         auto action = FilterAction.parse(tokens);
 383         //conwriteln("02tokens: ", tokens);
 384         if (action is null) break;
 385         actions ~= action;
 386         if (tokens.length > 0 && tokens[0] == "stop") {
 387           stop = true;
 388           tokens = tokens[1..$];
 389         }
 390       }
 391       if (tokens.length) {
 392         //conwriteln("03tokens: ", tokens);
 393         throw new Exception("extra tokens");
 394       }
 395       if (actions.length == 0) throw new Exception("no checker");
 396     } catch (Exception e) {
 397       conwriteln("**** TOKENS LEFT: ", tokens);
 398       throw new Exception("filter '"~name~"' error: "~e.msg);
 399     }
 400   }
 401 }
 402
 403
 404 // ////////////////////////////////////////////////////////////////////////// //
 405 __gshared Filter[] prefilters, postfilters;
 406
 407
 408 shared static this () {
 409   conRegFunc!((ConFuncVA cmdl) {
 410     try {
 411       auto flt = new Filter(cmdl.cmdline);
 412       conwriteln("pre_filter: '", flt.name, "'");
 413       foreach (ref ff; prefilters) {
 414         if (ff.name == flt.name) { ff = flt; return; } // replace
 415       }
 416       prefilters ~= flt;
 417     } catch (Exception e) {
 418       conwriteln("FILTER ERROR: ", e.msg);
 419       conwriteln(cmdl.cmdline);
 420     }
 421   })("pre_filter", "add pre-spamcheck filter");
 422
 423   conRegFunc!((ConFuncVA cmdl) {
 424     try {
 425       auto flt = new Filter(cmdl.cmdline);
 426       conwriteln("filter: '", flt.name, "'");
 427       foreach (ref ff; postfilters) {
 428         if (ff.name == flt.name) { ff = flt; return; } // replace
 429       }
 430       postfilters ~= flt;
 431     } catch (Exception e) {
 432       conwriteln("FILTER ERROR: ", e.msg);
 433       conwriteln(cmdl.cmdline);
 434     }
 435   })("filter", "add post-spamcheck filter");
 436 }
 437
 438
 439 // ////////////////////////////////////////////////////////////////////////// //
 440 public void doFiltering(string stage) (ref FilterData fda) if (stage == "pre" || stage == "post") {
 441        static if (stage == "pre") Filter[] list = prefilters;
 442   else static if (stage == "post") Filter[] list = postfilters;
 443   else static assert(0, "wtf?!");
 444   foreach (Filter flt; list) {
 445     if (flt.checker.check(fda)) {
 446       conwriteln((flt.stop ? "FINAL " : ""), "FILTER '", flt.name, " HIT at article from '", fda.art.fromname.recodeToKOI8, "' <", fda.art.frommail.recodeToKOI8, "> (", fda.art.subj.recodeToKOI8, ")");
 447       bool stop = false;
 448       foreach (auto act; flt.actions) {
 449         if (act.perform(fda)) { stop = true; break; }
 450       }
 451       if (flt.stop) return;
 452       if (stop) {
 453         conwriteln("STOP FILTER '", flt.name, " HIT at article from '", fda.art.fromname.recodeToKOI8, "' <", fda.art.frommail.recodeToKOI8, "> (", fda.art.subj.recodeToKOI8, ")");
 454         return;
 455       }
 456     }
 457   }
 458 }