filterengine.d

   1 /* E-Mail Client
   2  * coded by Ketmar // Invisible Vector <ketmar@ketmar.no-ip.org>
   3  * Understanding is not required. Only obedience.
   4  *
   5  * This program is free software: you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation, version 3 of the License ONLY.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  16  */
  17 module filterengine /*is aliced*/;
  18 private:
  19
  20 import iv.alice;
  21 import iv.cmdcon;
  22 import iv.encoding;
  23 import iv.strex;
  24
  25 import maildb;
  26
  27
  28 // ////////////////////////////////////////////////////////////////////////// //
  29 public struct FilterData {
  30   Article art; // in/out
  31   string destfolder; // in/out; set to null to delete
  32   Bogo bogo = Bogo.Error; // "error" means "not checked" here
  33   bool markRead;
  34   bool softDelete;
  35 }
  36
  37
  38 // ////////////////////////////////////////////////////////////////////////// //
  39 private abstract class FilterAction {
  40   bool perform (ref FilterData fda); // return `true` to stop
  41
  42   static FilterAction parse (ref string[] tokens) {
  43     if (tokens.length == 0) return null;
  44     //TODO: use CTFE to build list
  45     if (tokens[0] == "move") {
  46       if (tokens.length < 2) throw new Exception("invalid move rule");
  47       auto res = new FilterActionMove(tokens[1]);
  48       tokens = tokens[2..$];
  49       return res;
  50     }
  51     if (tokens[0] == "exec") {
  52       if (tokens.length < 2) throw new Exception("invalid exec rule");
  53       auto res = new FilterActionExec(tokens[1]);
  54       tokens = tokens[2..$];
  55       return res;
  56     }
  57     if (tokens[0] == "delete") { tokens = tokens[1..$]; return new FilterActionDelete(); }
  58     if (tokens[0] == "spam") { tokens = tokens[1..$]; return new FilterActionSpam(); }
  59     if (tokens[0] == "ham") { tokens = tokens[1..$]; return new FilterActionHam(); }
  60     if (tokens[0] == "read") { tokens = tokens[1..$]; return new FilterActionRead(); }
  61     return null;
  62   }
  63 }
  64
  65
  66 // args[1]: article (without dot, dot-unstuffed)
  67 // write string to stdout, with tokens:
  68 //   ham
  69 //   spam
  70 //   delete
  71 //   softdelete
  72 //   read
  73 //   move dest
  74 // start line with "-" to stop action
  75 private class FilterActionExec : FilterAction {
  76   string command;
  77
  78   this (string acommand) {
  79     if (acommand.length == 0) throw new Exception("can't exec nothing");
  80     command = acommand;
  81   }
  82
  83   override bool perform (ref FilterData fda) {
  84     import std.stdio : File;
  85     import std.process;
  86     try {
  87       // write article to file
  88       import std.uuid;
  89       UUID id = randomUUID();
  90       char[] buf;
  91       void deleteTempFile () {
  92         if (buf.length) try { import std.file : remove; remove(buf); } catch (Exception e) {}
  93       }
  94       scope(exit) {
  95         deleteTempFile();
  96         delete buf;
  97       }
  98       buf.reserve(2+16*2+42);
  99       buf ~= "/tmp/_temp_";
 100       foreach (immutable ubyte b; id.data[]) {
 101         buf ~= "0123456789abcdef"[b>>4];
 102         buf ~= "0123456789abcdef"[b&0x0f];
 103       }
 104       buf ~= ".eml";
 105       {
 106         auto fo = File(buf, "w");
 107         foreach (ConString s; fda.art.allLinesIterator) fo.writeln(s);
 108         fo.close();
 109       }
 110       conwriteln("EXEC filter '", command, "'...");
 111       auto pid = pipeProcess([command, buf], Redirect.all, null, Config.none, "/tmp");
 112       string action = pid.stdout.readln.xstrip;
 113       bool doStop = (action.length && action[0] == '-');
 114       if (doStop) action = action[1..$].xstrip;
 115       pid.pid.wait();
 116       conwriteln("EXEC filter '", command, "' action: ", action, " (", doStop, ")");
 117            if (action == "spam") fda.bogo = Bogo.Spam;
 118       else if (action == "ham") fda.bogo = Bogo.Ham;
 119       else if (action == "delete") fda.destfolder = null;
 120       else if (action == "softdelete") fda.softDelete = true;
 121       else if (action == "read") fda.markRead = true;
 122       else if (action == "noop") {}
 123       else if (action == "nothing") {}
 124       else if (action.startsWith("move")) {
 125         if (action.length < 6 || action[4] > ' ') throw new Exception("invalid action: '"~action~"'");
 126         action = action[5..$].xstrip;
 127         if (action.length > 1 && action[0] == '"' && action[$-1] == '"') action = action[1..$-1];
 128         if (action.length == 0) throw new Exception("can't move to empty folder");
 129         fda.destfolder = action;
 130       } else if (action.length) {
 131         throw new Exception("invalid action: '"~action~"'");
 132       }
 133       return doStop;
 134     } catch (Exception e) {
 135       conwriteln("EXEC filter error: ", e.msg);
 136     }
 137     return false;
 138   }
 139 }
 140
 141
 142 private class FilterActionMove : FilterAction {
 143   string folder;
 144
 145   this (string afolder) {
 146     if (afolder.length == 0) throw new Exception("can't move to empty folder");
 147     folder = afolder;
 148   }
 149
 150   override bool perform (ref FilterData fda) {
 151     conwriteln("MOVE article from '", fda.art.fromname.recodeToKOI8, "' <", fda.art.frommail.recodeToKOI8, "> (", fda.art.subj.recodeToKOI8, ") -> ", folder);
 152     fda.destfolder = folder;
 153     return false;
 154   }
 155 }
 156
 157
 158 // final action
 159 private class FilterActionDelete : FilterAction {
 160   override bool perform (ref FilterData fda) {
 161     conwriteln("DELETE article from '", fda.art.fromname.recodeToKOI8, "' <", fda.art.frommail.recodeToKOI8, "> (", fda.art.subj.recodeToKOI8, ")");
 162     fda.destfolder = null;
 163     return true;
 164   }
 165 }
 166
 167
 168 private class FilterActionSpam : FilterAction {
 169   override bool perform (ref FilterData fda) {
 170     conwriteln("SPAM article from '", fda.art.fromname.recodeToKOI8, "' <", fda.art.frommail.recodeToKOI8, "> (", fda.art.subj.recodeToKOI8, ")");
 171     fda.bogo = Bogo.Spam;
 172     return false;
 173   }
 174 }
 175
 176
 177 private class FilterActionHam : FilterAction {
 178   override bool perform (ref FilterData fda) {
 179     conwriteln("HAM article from '", fda.art.fromname.recodeToKOI8, "' <", fda.art.frommail.recodeToKOI8, "> (", fda.art.subj.recodeToKOI8, ")");
 180     fda.bogo = Bogo.Ham;
 181     return false;
 182   }
 183 }
 184
 185
 186 private class FilterActionRead : FilterAction {
 187   override bool perform (ref FilterData fda) {
 188     fda.markRead = true;
 189     return false;
 190   }
 191 }
 192
 193
 194 // ////////////////////////////////////////////////////////////////////////// //
 195 private abstract class FilterCompare {
 196   bool check (const(char)[] s);
 197
 198   static FilterCompare parse (ref string[] tokens) {
 199     if (tokens.length == 0) return null;
 200     //TODO: use CTFE to build list
 201     if (tokens[0] == "matchcase") {
 202       if (tokens.length < 2) throw new Exception("invalid matchcase rule");
 203       if (tokens[1].length == 0) throw new Exception("matchcase with nothing");
 204       auto res = new FilterCompareMatchCase(tokens[1], true);
 205       tokens = tokens[2..$];
 206       return res;
 207     }
 208     if (tokens[0] == "match") {
 209       if (tokens.length < 2) throw new Exception("invalid match rule");
 210       if (tokens[1].length == 0) throw new Exception("match with nothing");
 211       auto res = new FilterCompareMatchCase(tokens[1], false);
 212       tokens = tokens[2..$];
 213       return res;
 214     }
 215     return null;
 216   }
 217 }
 218
 219 private class FilterCompareMatchCase : FilterCompare {
 220   string pat;
 221   bool caseSens;
 222
 223   this (string apat, bool acaseSens) { pat = apat; caseSens = acaseSens; }
 224
 225   override bool check (const(char)[] s) {
 226     if (pat.length == 0) return false;
 227     bool atStart = false, atEnd = false;
 228     if (pat[0] == '^') { atStart = true; pat = pat[1..$]; }
 229     if (pat.length && pat[$-1] == '$') { atEnd = true; pat = pat[0..$-1]; }
 230     if (pat.length == 0) {
 231       if (atStart || atEnd) return (s.length > 0);
 232       return false;
 233     }
 234     if (atStart && atEnd) return (caseSens ? (s == pat) : s.strEquCI(pat));
 235     if (atStart) return (caseSens ? s.startsWith(pat) : s.startsWithCI(pat));
 236     if (atEnd) return (caseSens ? s.endsWith(pat) : s.endsWithCI(pat));
 237     if (caseSens) return (s.indexOf(pat) >= 0);
 238     // case insensitive
 239     if (s.length < pat.length) return false;
 240     //if (s.length == pat.length) return s.strEquCI(pat);
 241     foreach (immutable idx; 0..s.length-pat.length+1) {
 242       if (s[idx..idx+pat.length].strEquCI(pat)) return true;
 243     }
 244     return false;
 245   }
 246 }
 247
 248
 249 // ////////////////////////////////////////////////////////////////////////// //
 250 private abstract class FilterChecker {
 251   static FilterChecker parse (ref string[] tokens) {
 252     if (tokens.length == 0) return null;
 253     //TODO: use CTFE to build list
 254     if (tokens[0] == "fromname") {
 255       if (tokens.length < 2) throw new Exception("invalid fromname rule");
 256       tokens = tokens[1..$];
 257       auto res = new FilterCheckerFromName();
 258       res.cmp = FilterCompare.parse(tokens);
 259       if (res.cmp is null) throw new Exception("invalid fromname rule");
 260       return res;
 261     }
 262     if (tokens[0] == "frommail") {
 263       if (tokens.length < 2) throw new Exception("invalid frommail rule");
 264       tokens = tokens[1..$];
 265       auto res = new FilterCheckerFromMail();
 266       res.cmp = FilterCompare.parse(tokens);
 267       if (res.cmp is null) throw new Exception("invalid frommail rule");
 268       return res;
 269     }
 270     if (tokens[0] == "header") {
 271       if (tokens.length < 2) throw new Exception("invalid header rule");
 272       auto res = new FilterCheckerHeaderField(tokens[1]);
 273       tokens = tokens[2..$];
 274       res.cmp = FilterCompare.parse(tokens);
 275       if (res.cmp is null) throw new Exception("invalid frommail rule");
 276       return res;
 277     }
 278     if (tokens[0] == "to") {
 279       if (tokens.length < 2) throw new Exception("invalid to rule");
 280       auto res = new FilterCheckerHeaderField("to");
 281       tokens = tokens[1..$];
 282       res.cmp = FilterCompare.parse(tokens);
 283       if (res.cmp is null) throw new Exception("invalid frommail rule");
 284       return res;
 285     }
 286     if (tokens[0] == "to_or_cc") {
 287       if (tokens.length < 2) throw new Exception("invalid to_or_cc rule");
 288       tokens = tokens[1..$];
 289       auto c0 = new FilterCheckerHeaderField("to");
 290       c0.cmp = FilterCompare.parse(tokens);
 291       if (c0.cmp is null) throw new Exception("invalid to_or_cc rule");
 292       auto c1 = new FilterCheckerHeaderField("cc");
 293       c1.cmp = c0.cmp;
 294       auto res = new FilterCheckerBinExpr!"or"(c0, c1);
 295       return res;
 296     }
 297     return null;
 298   }
 299
 300   static FilterChecker parseCheckers (ref string[] tokens) {
 301     FilterChecker res = parse(tokens);
 302     if (res is null || tokens.length == 0) return res;
 303     if (tokens[0] == "|" || tokens[0] == "&") {
 304       if (tokens.length < 2) throw new Exception("invalid logic op");
 305       string op = tokens[0];
 306       tokens = tokens[1..$];
 307       FilterChecker right = parseCheckers(tokens);
 308       if (right is null) throw new Exception("invalid logic op");
 309            if (op == "|") res = new FilterCheckerBinExpr!"or"(res, right);
 310       else if (op == "&") res = new FilterCheckerBinExpr!"and"(res, right);
 311       else throw new Exception("invalid logic op");
 312     }
 313     return res;
 314   }
 315
 316   bool check (ref FilterData fda);
 317 }
 318
 319
 320 // ////////////////////////////////////////////////////////////////////////// //
 321 private class FilterCheckerBinExpr(string op) : FilterChecker if (op == "or" || op == "and") {
 322   FilterChecker left, right;
 323
 324   this (FilterChecker l, FilterChecker r) { left = l; right = r; }
 325
 326   override bool check (ref FilterData fda) {
 327     bool res = (left !is null ? left.check(fda) : false);
 328     static if (op == "or") {
 329       if (res) return true;
 330       if (right is null) return false;
 331       return right.check(fda);
 332     } else static if (op == "and") {
 333       if (!res) return false;
 334       if (right is null) return false;
 335       return right.check(fda);
 336     } else {
 337       static assert(0, "wtf?!");
 338     }
 339   }
 340 }
 341
 342
 343 // ////////////////////////////////////////////////////////////////////////// //
 344 private abstract class FilterCheckerHeaderCompare : FilterChecker {
 345   FilterCompare cmp;
 346
 347   override bool check (ref FilterData fda) {
 348     if (cmp is null) return false;
 349     return cmp.check(getCompareString(fda));
 350   }
 351
 352   const(char)[] getCompareString (ref FilterData fda);
 353 }
 354
 355 private class FilterCheckerFromName : FilterCheckerHeaderCompare {
 356   override const(char)[] getCompareString (ref FilterData fda) { return fda.art.fromname; }
 357 }
 358
 359 private class FilterCheckerFromMail : FilterCheckerHeaderCompare {
 360   override const(char)[] getCompareString (ref FilterData fda) { return fda.art.frommail; }
 361 }
 362
 363 private class FilterCheckerHeaderField : FilterCheckerHeaderCompare {
 364   string fldname;
 365
 366   this (string afldname) {
 367     if (afldname.length == 0) throw new Exception("no field name");
 368     fldname = afldname;
 369   }
 370
 371   override const(char)[] getCompareString (ref FilterData fda) {
 372     auto hproc = fda.art.headersIterator;
 373     while (!hproc.empty) {
 374       if (hproc.fieldName.strEquCI(fldname)) return hproc.curline;
 375       hproc.popFront();
 376     }
 377     return null;
 378   }
 379 }
 380
 381
 382 // ////////////////////////////////////////////////////////////////////////// //
 383 public final class Filter {
 384 public:
 385   string name;
 386   FilterChecker checker;
 387   FilterAction[] actions;
 388   bool stop;
 389
 390 public:
 391   this (ConString cli) {
 392     string[] tokens;
 393     for (;;) {
 394       auto tk = ConCommand.getWord(cli);
 395       if (tk is null) break;
 396       tokens ~= tk.idup;
 397     }
 398     if (tokens.length < 1) throw new Exception("invalid filter: no name");
 399     if (tokens[0].length == 0) throw new Exception("invalid filter: empty name");
 400     if (tokens.length < 2) throw new Exception("invalid filter '"~tokens[0]~"': no rules");
 401     //conwriteln("=======================");
 402     name = tokens[0];
 403     tokens = tokens[1..$];
 404     //conwriteln("00tokens: ", tokens);
 405     try {
 406       checker = FilterChecker.parseCheckers(tokens);
 407       //conwriteln("01tokens: ", tokens);
 408       if (checker is null) throw new Exception("no checker");
 409       while (tokens.length) {
 410         auto action = FilterAction.parse(tokens);
 411         //conwriteln("02tokens: ", tokens);
 412         if (action is null) break;
 413         actions ~= action;
 414         if (tokens.length > 0 && tokens[0] == "stop") {
 415           stop = true;
 416           tokens = tokens[1..$];
 417         }
 418       }
 419       if (tokens.length) {
 420         //conwriteln("03tokens: ", tokens);
 421         throw new Exception("extra tokens");
 422       }
 423       if (actions.length == 0) throw new Exception("no checker");
 424     } catch (Exception e) {
 425       conwriteln("**** TOKENS LEFT: ", tokens);
 426       throw new Exception("filter '"~name~"' error: "~e.msg);
 427     }
 428   }
 429 }
 430
 431
 432 // ////////////////////////////////////////////////////////////////////////// //
 433 __gshared Filter[] prefilters, postfilters;
 434
 435
 436 shared static this () {
 437   conRegFunc!((ConFuncVA cmdl) {
 438     try {
 439       auto flt = new Filter(cmdl.cmdline);
 440       conwriteln("pre_filter: '", flt.name, "'");
 441       foreach (ref ff; prefilters) {
 442         if (ff.name == flt.name) { ff = flt; return; } // replace
 443       }
 444       prefilters ~= flt;
 445     } catch (Exception e) {
 446       conwriteln("FILTER ERROR: ", e.msg);
 447       conwriteln(cmdl.cmdline);
 448     }
 449   })("pre_filter", "add pre-spamcheck filter");
 450
 451   conRegFunc!((ConFuncVA cmdl) {
 452     try {
 453       auto flt = new Filter(cmdl.cmdline);
 454       conwriteln("filter: '", flt.name, "'");
 455       foreach (ref ff; postfilters) {
 456         if (ff.name == flt.name) { ff = flt; return; } // replace
 457       }
 458       postfilters ~= flt;
 459     } catch (Exception e) {
 460       conwriteln("FILTER ERROR: ", e.msg);
 461       conwriteln(cmdl.cmdline);
 462     }
 463   })("filter", "add post-spamcheck filter");
 464 }
 465
 466
 467 // ////////////////////////////////////////////////////////////////////////// //
 468 public void doFiltering(string stage) (ref FilterData fda) if (stage == "pre" || stage == "post") {
 469        static if (stage == "pre") Filter[] list = prefilters;
 470   else static if (stage == "post") Filter[] list = postfilters;
 471   else static assert(0, "wtf?!");
 472   foreach (Filter flt; list) {
 473     if (flt.checker.check(fda)) {
 474       conwriteln((flt.stop ? "FINAL " : ""), "FILTER '", flt.name, " HIT at article from '", fda.art.fromname.recodeToKOI8, "' <", fda.art.frommail.recodeToKOI8, "> (", fda.art.subj.recodeToKOI8, ")");
 475       bool stop = false;
 476       foreach (/*auto*/ act; flt.actions) {
 477         if (act.perform(fda)) { stop = true; break; }
 478       }
 479       if (flt.stop) return;
 480       if (stop) {
 481         conwriteln("STOP FILTER '", flt.name, " HIT at article from '", fda.art.fromname.recodeToKOI8, "' <", fda.art.frommail.recodeToKOI8, "> (", fda.art.subj.recodeToKOI8, ")");
 482         return;
 483       }
 484     }
 485   }
 486 }