message.cpp

   1 // What you get where:
   2
   3 // RFC5321.HELO/.EHLO domain
   4 // RFC5321.MailFrom   mailbox
   5 // RFC5322.From       mailbox-list
   6
   7 #include "message.hpp"
   8
   9 #include "Mailbox.hpp"
  10 #include "OpenARC.hpp"
  11 #include "OpenDKIM.hpp"
  12 #include "OpenDMARC.hpp"
  13 #include "esc.hpp"
  14 #include "fs.hpp"
  15 #include "iequal.hpp"
  16 #include "imemstream.hpp"
  17
  18 #include <cstring>
  19 #include <map>
  20
  21 #include <fmt/format.h>
  22 #include <fmt/ostream.h>
  23
  24 #include <boost/algorithm/string.hpp>
  25 #include <boost/iostreams/device/mapped_file.hpp>
  26
  27 #include <tao/pegtl.hpp>
  28 #include <tao/pegtl/contrib/abnf.hpp>
  29
  30 using std::begin;
  31 using std::end;
  32
  33 // DKIM key "selector"
  34 auto constexpr selector = "ghsmtp";
  35
  36 // RFC-5322 header names
  37 auto constexpr ARC_Authentication_Results = "ARC-Authentication-Results";
  38 auto constexpr ARC_Message_Signature      = "ARC-Message-Signature";
  39 auto constexpr ARC_Seal                   = "ARC-Seal";
  40
  41 auto constexpr Authentication_Results = "Authentication-Results";
  42 auto constexpr DKIM_Signature         = "DKIM-Signature";
  43 auto constexpr Delivered_To           = "Delivered-To";
  44 auto constexpr From                   = "From";
  45 auto constexpr Received_SPF           = "Received-SPF";
  46 auto constexpr Reply_To               = "Reply-To";
  47 auto constexpr Return_Path            = "Return-Path";
  48
  49 // MIME headers
  50 auto constexpr Content_Type = "Content-Type";
  51 auto constexpr MIME_Version = "MIME-Version";
  52
  53 // SPF Results
  54 auto constexpr Pass      = "Pass";
  55 auto constexpr Fail      = "Fail";
  56 auto constexpr SoftFail  = "SoftFail";
  57 auto constexpr Neutral   = "Neutral";
  58 auto constexpr None      = "None";
  59 auto constexpr TempError = "TempError";
  60 auto constexpr PermError = "PermError";
  61
  62 // SPF keys
  63 auto constexpr client_ip     = "client-ip";
  64 auto constexpr envelope_from = "envelope-from";
  65 auto constexpr problem       = "problem";
  66 auto constexpr receiver      = "receiver";
  67 auto constexpr identity      = "identity";
  68 auto constexpr mechanism     = "mechanism";
  69 // auto constexpr helo       = "helo"; // both key and value
  70
  71 // SPF identities
  72 auto constexpr helo     = "helo";
  73 auto constexpr mailfrom = "mailfrom";
  74
  75 using namespace tao::pegtl;
  76 using namespace tao::pegtl::abnf;
  77
  78 using namespace std::string_literals;
  79
  80 static std::string make_string(std::string_view v)
  81 {
  82   return std::string(v.begin(),
  83                      static_cast<size_t>(std::distance(v.begin(), v.end())));
  84 }
  85
  86 static std::string_view trim(std::string_view v)
  87 {
  88   auto constexpr WS = " \t";
  89   v.remove_prefix(std::min(v.find_first_not_of(WS), v.size()));
  90   v.remove_suffix(std::min(v.size() - v.find_last_not_of(WS) - 1, v.size()));
  91   return v;
  92 }
  93
  94 template <typename Input>
  95 static std::string_view make_view(Input const& in)
  96 {
  97   return std::string_view(in.begin(), std::distance(in.begin(), in.end()));
  98 }
  99
 100 namespace RFC5322 {
 101
 102 using dot   = one<'.'>;
 103 using colon = one<':'>;
 104
 105 // clang-format off
 106
 107 struct UTF8_tail        : range<'\x80', '\xBF'> {};
 108
 109 struct UTF8_1           : range<0x00, 0x7F> {};
 110
 111 struct UTF8_2           : seq<range<'\xC2', '\xDF'>, UTF8_tail> {};
 112
 113 struct UTF8_3           : sor<seq<one<'\xE0'>, range<'\xA0', '\xBF'>, UTF8_tail>,
 114                               seq<range<'\xE1', '\xEC'>, rep<2, UTF8_tail>>,
 115                               seq<one<'\xED'>, range<'\x80', '\x9F'>, UTF8_tail>,
 116                               seq<range<'\xEE', '\xEF'>, rep<2, UTF8_tail>>> {};
 117
 118 struct UTF8_4           : sor<seq<one<'\xF0'>, range<'\x90', '\xBF'>, rep<2, UTF8_tail>>,
 119                               seq<range<'\xF1', '\xF3'>, rep<3, UTF8_tail>>,
 120                               seq<one<'\xF4'>, range<'\x80', '\x8F'>, rep<2, UTF8_tail>>> {};
 121
 122 struct UTF8_non_ascii   : sor<UTF8_2, UTF8_3, UTF8_4> {};
 123
 124 struct VUCHAR           : sor<VCHAR, UTF8_non_ascii> {};
 125
 126 //.............................................................................
 127
 128 struct ftext            : ranges<33, 57, 59, 126> {};
 129
 130 struct field_name       : plus<ftext> {};
 131
 132 struct FWS              : seq<opt<seq<star<WSP>, eol>>, plus<WSP>> {};
 133
 134 // *([FWS] VCHAR) *WSP
 135 struct field_value      : seq<star<seq<opt<FWS>, VUCHAR>>, star<WSP>> {};
 136
 137 struct field            : seq<field_name, one<':'>, field_value, eol> {};
 138
 139 struct raw_field        : seq<field_name, one<':'>, field_value, eof> {};
 140
 141 struct fields           : star<field> {};
 142
 143 struct body             : until<eof> {};
 144
 145 struct message          : seq<fields, opt<seq<eol, body>>, eof> {};
 146
 147 //.............................................................................
 148
 149 // <https://tools.ietf.org/html/rfc2047>
 150
 151 //   especials = "(" / ")" / "<" / ">" / "@" / "," / ";" / ":" / "
 152 //               <"> / "/" / "[" / "]" / "?" / "." / "="
 153
 154 //   token = 1*<Any CHAR except SPACE, CTLs, and especials>
 155
 156 struct tchar47          : ranges<        // NUL..' '
 157                                  33, 33, // !
 158                               // 34, 34, // "
 159                                  35, 39, // #$%&'
 160                               // 40, 41, // ()
 161                                  42, 43, // *+
 162                               // 44, 44, // ,
 163                                  45, 45, // -
 164                               // 46, 47, // ./
 165                                  48, 57, // 0123456789
 166                               // 58, 64, // ;:<=>?@
 167                                  65, 90, // A..Z
 168                               // 91, 91, // [
 169                                  92, 92, // '\\'
 170                               // 93, 93, // ]
 171                                  94, 126 // ^_` a..z {|}~
 172                               // 127,127 // DEL
 173                                 > {};
 174
 175 struct token47           : plus<tchar47> {};
 176
 177 struct charset           : token47 {};
 178 struct encoding          : token47 {};
 179
 180 //   encoded-text = 1*<Any printable ASCII character other than "?"
 181 //                     or SPACE>
 182
 183 struct echar            : ranges<        // NUL..' '
 184                                  33, 62, // !..>
 185                               // 63, 63, // ?
 186                                  64, 126 // @A..Z[\]^_` a..z {|}~
 187                               // 127,127 // DEL
 188                                 > {};
 189
 190 struct encoded_text     : plus<echar> {};
 191
 192 //   encoded-word = "=?" charset "?" encoding "?" encoded-text "?="
 193
 194 // leading opt<FWS> is not in RFC 2047
 195
 196 struct encoded_word_book: seq<string<'=', '?'>,
 197                               charset, string<'?'>,
 198                               encoding, string<'?'>,
 199                               encoded_text,
 200                               string<'=', '?'>
 201                               > {};
 202
 203 struct encoded_word     : seq<opt<FWS>, encoded_word_book> {};
 204
 205 //.............................................................................
 206
 207 // Comments are recursive, hence the forward declaration:
 208 struct comment;
 209
 210 struct quoted_pair      : seq<one<'\\'>, sor<VUCHAR, WSP>> {};
 211
 212 // ctext is ASCII not '(' or ')' or '\\'
 213 struct ctext            : sor<ranges<33, 39, 42, 91, 93, 126>, UTF8_non_ascii> {};
 214
 215 struct ccontent         : sor<ctext, quoted_pair, comment, encoded_word> {};
 216
 217 // from <https://tools.ietf.org/html/rfc2047>
 218 // comment = "(" *(ctext / quoted-pair / comment / encoded-word) ")"
 219
 220 struct comment          : seq<one<'('>,
 221                               star<seq<opt<FWS>, ccontent>>,
 222                               opt<FWS>,
 223                               one<')'>
 224                               > {};
 225
 226 struct CFWS             : sor<seq<plus<seq<opt<FWS>, comment>, opt<FWS>>>,
 227                               FWS> {};
 228
 229 struct qtext            : sor<one<33>, ranges<35, 91, 93, 126>, UTF8_non_ascii> {};
 230
 231 struct qcontent         : sor<qtext, quoted_pair> {};
 232
 233 // Corrected in RFC-5322, errata ID: 3135 <https://www.rfc-editor.org/errata/eid3135>
 234 struct quoted_string    : seq<opt<CFWS>,
 235                               DQUOTE,
 236                               sor<seq<star<seq<opt<FWS>, qcontent>>, opt<FWS>>, FWS>,
 237                               DQUOTE,
 238                               opt<CFWS>
 239                               > {};
 240
 241 struct atext            : sor<ALPHA, DIGIT,
 242                               one<'!', '#',
 243                                   '$', '%',
 244                                   '&', '\'',
 245                                   '*', '+',
 246                                   '-', '/',
 247                                   '=', '?',
 248                                   '^', '_',
 249                                   '`', '{',
 250                                   '|', '}',
 251                                   '~'>,
 252                               UTF8_non_ascii> {};
 253
 254 struct atom             : seq<opt<CFWS>, plus<atext>, opt<CFWS>> {};
 255
 256 struct dot_atom_text    : list<plus<atext>, dot> {};
 257
 258 struct dot_atom         : seq<opt<CFWS>, dot_atom_text, opt<CFWS>> {};
 259
 260 struct word             : sor<atom, quoted_string> {};
 261
 262 struct phrase          : plus<sor<encoded_word, word>> {};
 263
 264 struct dec_octet        : sor<seq<string<'2','5'>, range<'0','5'>>,
 265                               seq<one<'2'>, range<'0','4'>, DIGIT>,
 266                               seq<range<'0', '1'>, rep<2, DIGIT>>,
 267                               rep_min_max<1, 2, DIGIT>> {};
 268
 269 struct ipv4_address     : seq<dec_octet, dot, dec_octet, dot, dec_octet, dot, dec_octet> {};
 270
 271 struct h16              : rep_min_max<1, 4, HEXDIG> {};
 272
 273 struct ls32             : sor<seq<h16, colon, h16>, ipv4_address> {};
 274
 275 struct dcolon           : two<':'> {};
 276
 277 struct ipv6_address     : sor<seq<                                          rep<6, h16, colon>, ls32>,
 278                               seq<                                  dcolon, rep<5, h16, colon>, ls32>,
 279                               seq<opt<h16                        >, dcolon, rep<4, h16, colon>, ls32>,
 280                               seq<opt<h16,     opt<   colon, h16>>, dcolon, rep<3, h16, colon>, ls32>,
 281                               seq<opt<h16, rep_opt<2, colon, h16>>, dcolon, rep<2, h16, colon>, ls32>,
 282                               seq<opt<h16, rep_opt<3, colon, h16>>, dcolon,        h16, colon,  ls32>,
 283                               seq<opt<h16, rep_opt<4, colon, h16>>, dcolon,                     ls32>,
 284                               seq<opt<h16, rep_opt<5, colon, h16>>, dcolon,                      h16>,
 285                               seq<opt<h16, rep_opt<6, colon, h16>>, dcolon                          >> {};
 286
 287 struct ip               : sor<ipv4_address, ipv6_address> {};
 288
 289 struct local_part       : sor<dot_atom, quoted_string> {};
 290
 291 struct dtext            : ranges<33, 90, 94, 126> {};
 292
 293 struct domain_literal   : seq<opt<CFWS>,
 294                               one<'['>,
 295                               star<seq<opt<FWS>, dtext>>,
 296                               opt<FWS>,
 297                               one<']'>,
 298                               opt<CFWS>> {};
 299
 300 struct domain           : sor<dot_atom, domain_literal> {};
 301
 302 // This addr_spec should be exactly the same as RFC5321 Mailbox, but it's not.
 303
 304 struct addr_spec        : seq<local_part, one<'@'>, domain> {};
 305
 306 struct addr_spec_only   : seq<addr_spec, eof> {};
 307
 308 struct result           : sor<TAO_PEGTL_ISTRING("Pass"),
 309                               TAO_PEGTL_ISTRING("Fail"),
 310                               TAO_PEGTL_ISTRING("SoftFail"),
 311                               TAO_PEGTL_ISTRING("Neutral"),
 312                               TAO_PEGTL_ISTRING("None"),
 313                               TAO_PEGTL_ISTRING("TempError"),
 314                               TAO_PEGTL_ISTRING("PermError")> {};
 315
 316 struct spf_key          : sor<TAO_PEGTL_ISTRING("client-ip"),
 317                               TAO_PEGTL_ISTRING("envelope-from"),
 318                               TAO_PEGTL_ISTRING("helo"),
 319                               TAO_PEGTL_ISTRING("problem"),
 320                               TAO_PEGTL_ISTRING("receiver"),
 321                               TAO_PEGTL_ISTRING("identity"),
 322                               TAO_PEGTL_ISTRING("mechanism")> {};
 323
 324 // This value syntax (allowing addr_spec) is not in accordance with RFC
 325 // 7208 (or 4408) but is what is effectivly used by libspf2 1.2.10 and
 326 // before.
 327
 328 struct spf_value        : sor<ip, addr_spec, dot_atom, quoted_string> {};
 329
 330 struct spf_kv_pair      : seq<spf_key, opt<CFWS>, one<'='>, spf_value> {};
 331
 332 struct spf_kv_list      : seq<spf_kv_pair,
 333                               star<seq<one<';'>, opt<CFWS>, spf_kv_pair>>,
 334                               opt<one<';'>>> {};
 335
 336 struct spf_header       : seq<opt<CFWS>,
 337                               result,
 338                               opt<seq<FWS, comment>>,
 339                               opt<seq<FWS, spf_kv_list>>> {};
 340
 341 struct spf_header_only  : seq<spf_header, eof> {};
 342
 343 //.............................................................................
 344
 345 struct display_name     : phrase {};
 346
 347 struct angle_addr       : seq<opt<CFWS>, one<'<'>, addr_spec, one<'>'>, opt<CFWS>> {};
 348
 349 struct name_addr        : seq<opt<display_name>, angle_addr> {};
 350
 351 struct mailbox          : sor<name_addr, addr_spec> {};
 352
 353 struct obs_mbox_list    : seq<star<seq<opt<CFWS>, one<','>>>,
 354                               mailbox,
 355                               star<one<','>, opt<sor<mailbox, CFWS>>>
 356                               > {};
 357
 358 struct mailbox_list     : sor<list<mailbox, one<','>>,
 359                               obs_mbox_list
 360                               > {};
 361
 362 // struct from          : seq<TAO_PEGTL_ISTRING("From:"),
 363 //                            mailbox_list
 364 //                            > {};
 365
 366 struct mailbox_list_only: seq<mailbox_list, eof> {};
 367
 368 //.............................................................................
 369
 370 // struct authres_header_field: seq<TAO_PEGTL_ISTRING("Authentication-Results:"),
 371 //                                  authres_payload> {};
 372
 373 //.............................................................................
 374
 375 // clang-format on
 376
 377 template <typename Rule>
 378 struct msg_action : nothing<Rule> {
 379 };
 380
 381 template <>
 382 struct msg_action<field_name> {
 383   template <typename Input>
 384   static void apply(Input const& in, ::message::parsed& msg)
 385   {
 386     msg.field_name = make_view(in);
 387   }
 388 };
 389
 390 template <>
 391 struct msg_action<field_value> {
 392   template <typename Input>
 393   static void apply(Input const& in, ::message::parsed& msg)
 394   {
 395     msg.field_value = make_view(in);
 396   }
 397 };
 398
 399 template <>
 400 struct msg_action<field> {
 401   template <typename Input>
 402   static void apply(Input const& in, ::message::parsed& msg)
 403   {
 404     msg.headers.emplace_back(
 405         ::message::header(msg.field_name, msg.field_value));
 406   }
 407 };
 408
 409 template <>
 410 struct msg_action<raw_field> {
 411   template <typename Input>
 412   static void apply(Input const& in, ::message::parsed& msg)
 413   {
 414     msg.headers.emplace_back(
 415         ::message::header(msg.field_name, msg.field_value));
 416   }
 417 };
 418
 419 template <>
 420 struct msg_action<body> {
 421   template <typename Input>
 422   static void apply(Input const& in, ::message::parsed& msg)
 423   {
 424     msg.body = make_view(in);
 425   }
 426 };
 427
 428 //.............................................................................
 429
 430 struct received_spf_parsed {
 431   bool parse(std::string_view input);
 432
 433   std::string_view result;
 434
 435   std::string_view key;
 436   std::string_view value;
 437
 438   std::vector<std::pair<std::string_view, std::string_view>> kv_list;
 439   std::map<std::string_view, std::string_view, ci_less>      kv_map;
 440 };
 441
 442 template <typename Rule>
 443 struct spf_action : nothing<Rule> {
 444 };
 445
 446 template <>
 447 struct spf_action<result> {
 448   template <typename Input>
 449   static void apply(const Input& in, received_spf_parsed& spf)
 450   {
 451     spf.result = make_view(in);
 452   }
 453 };
 454
 455 template <>
 456 struct spf_action<spf_key> {
 457   template <typename Input>
 458   static void apply(const Input& in, received_spf_parsed& spf)
 459   {
 460     spf.key = make_view(in);
 461   }
 462 };
 463
 464 template <>
 465 struct spf_action<spf_value> {
 466   template <typename Input>
 467   static void apply(const Input& in, received_spf_parsed& spf)
 468   {
 469     // RFC5322 syntax is full of optional WS, so we trim
 470     spf.value = trim(make_view(in));
 471   }
 472 };
 473
 474 template <>
 475 struct spf_action<spf_kv_pair> {
 476   template <typename Input>
 477   static void apply(const Input& in, received_spf_parsed& spf)
 478   {
 479     spf.kv_list.emplace_back(spf.key, spf.value);
 480     spf.key = spf.value = "";
 481   }
 482 };
 483
 484 template <>
 485 struct spf_action<spf_kv_list> {
 486   static void apply0(received_spf_parsed& spf)
 487   {
 488     for (auto kvp : spf.kv_list) {
 489       if (spf.kv_map.contains(kvp.first)) {
 490         LOG(WARNING) << "dup key: " << kvp.first << "=" << kvp.second;
 491         LOG(WARNING) << "    and: " << kvp.first << "="
 492                      << spf.kv_map[kvp.first];
 493       }
 494       spf.kv_map[kvp.first] = kvp.second;
 495     }
 496   }
 497 };
 498
 499 bool received_spf_parsed::parse(std::string_view input)
 500 {
 501   auto in{memory_input<>(input.data(), input.size(), "spf_header")};
 502   return tao::pegtl::parse<spf_header_only, spf_action>(in, *this);
 503 }
 504
 505 //.............................................................................
 506
 507 // Parse a grammar and extract each addr_spec
 508
 509 template <typename Rule>
 510 struct addr_specs_action : nothing<Rule> {
 511 };
 512
 513 template <>
 514 struct addr_specs_action<addr_spec> {
 515   template <typename Input>
 516   static void apply(Input const& in, std::vector<std::string>& addr_specs)
 517   {
 518     addr_specs.push_back(in.string());
 519   }
 520 };
 521
 522 } // namespace RFC5322
 523
 524 // Map SPF result string to DMARC policy code.
 525
 526 // FIXME: This mapping needs to be examined and confirmed, no time now.
 527
 528 static int result_to_pol(std::string_view result)
 529 {
 530   // clang-format off
 531   if (iequal(result, Pass))      return DMARC_POLICY_SPF_OUTCOME_PASS;
 532   if (iequal(result, Fail))      return DMARC_POLICY_SPF_OUTCOME_FAIL;
 533   if (iequal(result, SoftFail))  return DMARC_POLICY_SPF_OUTCOME_TMPFAIL;
 534   if (iequal(result, Neutral))   return DMARC_POLICY_SPF_OUTCOME_NONE;
 535   if (iequal(result, None))      return DMARC_POLICY_SPF_OUTCOME_NONE;
 536   if (iequal(result, TempError)) return DMARC_POLICY_SPF_OUTCOME_NONE;
 537   if (iequal(result, PermError)) return DMARC_POLICY_SPF_OUTCOME_NONE;
 538   LOG(WARNING) << "unknown SPF result: \"" << result << "\"";
 539   return DMARC_POLICY_SPF_OUTCOME_NONE;
 540   // clang-format on
 541 }
 542
 543 static void spf_result_to_dmarc(OpenDMARC::policy&            dmp,
 544                                 RFC5322::received_spf_parsed& spf)
 545 {
 546   if (spf.kv_map.contains(problem)) {
 547     LOG(WARNING) << "SPF problem: " << spf.kv_map[problem];
 548   }
 549
 550   auto const spf_pol = result_to_pol(spf.result);
 551
 552   std::string spf_dom;
 553
 554   int spf_origin;
 555
 556   if (spf.kv_map.contains(identity)) {
 557     if (iequal(spf.kv_map[identity], mailfrom)) {
 558       if (spf.kv_map.contains(envelope_from)) {
 559         if (Mailbox::validate(spf.kv_map[envelope_from])) {
 560           Mailbox mbx(spf.kv_map[envelope_from]);
 561           spf_dom    = mbx.domain().ascii();
 562           spf_origin = DMARC_POLICY_SPF_ORIGIN_MAILFROM;
 563
 564           auto const human_result = fmt::format(
 565               "{}, explicit origin mail from, mailbox {}", spf.result, mbx);
 566           LOG(INFO) << "SPF result " << human_result;
 567           dmp.store_spf(spf_dom.c_str(), spf_pol, spf_origin,
 568                         human_result.c_str());
 569           return;
 570         }
 571         else {
 572           LOG(WARNING) << "invalid mailbox in envelope-from: "
 573                        << spf.kv_map[envelope_from];
 574         }
 575       }
 576       else {
 577         LOG(WARNING)
 578             << "identity checked was mail from, but no envelope_from key";
 579       }
 580     }
 581     else if (iequal(spf.kv_map[identity], helo)) {
 582       if (spf.kv_map.contains(helo)) {
 583         if (Domain::validate(spf.kv_map[helo])) {
 584           Domain dom(spf.kv_map[helo]);
 585           spf_dom    = dom.ascii();
 586           spf_origin = DMARC_POLICY_SPF_ORIGIN_HELO;
 587
 588           auto const human_result = fmt::format(
 589               "{}, explicit origin hello, domain {}", spf.result, dom);
 590           LOG(INFO) << "SPF result " << human_result;
 591           dmp.store_spf(spf_dom.c_str(), spf_pol, spf_origin,
 592                         human_result.c_str());
 593           return;
 594         }
 595         else {
 596           LOG(WARNING) << "invalid domain in helo: " << spf.kv_map[helo];
 597         }
 598       }
 599       else {
 600         LOG(WARNING) << "identity checked was helo, but no helo key";
 601       }
 602     }
 603     else {
 604       LOG(WARNING) << "unknown identity " << spf.kv_map[identity];
 605     }
 606   }
 607   else {
 608     LOG(INFO) << "no explicit tag for which identity was checked";
 609   }
 610
 611   if (spf.kv_map.contains(envelope_from)) {
 612     auto const efrom = spf.kv_map[envelope_from];
 613
 614     if (efrom == "<>") {
 615       if (spf.kv_map.contains(helo)) {
 616         if (Domain::validate(spf.kv_map[helo])) {
 617           Domain dom(spf.kv_map[helo]);
 618           spf_dom    = dom.ascii();
 619           spf_origin = DMARC_POLICY_SPF_ORIGIN_HELO;
 620
 621           auto const human_result = fmt::format(
 622               "{}, RFC-5321.FROM is <>, implicit origin hello, domain {}",
 623               spf.result, dom);
 624           LOG(INFO) << "SPF result " << human_result;
 625           dmp.store_spf(spf_dom.c_str(), spf_pol, spf_origin,
 626                         human_result.c_str());
 627           return;
 628         }
 629         else {
 630           LOG(WARNING) << "RFC-5321.FROM is <> but helo is invalid domain: "
 631                        << spf.kv_map[helo];
 632         }
 633       }
 634       else {
 635         LOG(WARNING) << "envelope-from is <> but no helo key";
 636       }
 637     }
 638     else if (Mailbox::validate(efrom)) {
 639       // We're good to go
 640       Mailbox mbx(efrom);
 641       spf_dom    = mbx.domain().ascii();
 642       spf_origin = DMARC_POLICY_SPF_ORIGIN_MAILFROM;
 643
 644       auto const human_result = fmt::format(
 645           "{}, implicit origin mail from, mailbox {}", spf.result, mbx);
 646       LOG(INFO) << "SPF result " << human_result;
 647       dmp.store_spf(spf_dom.c_str(), spf_pol, spf_origin, human_result.c_str());
 648       return;
 649     }
 650     else {
 651       LOG(WARNING) << "envelope-from invalid mailbox: " << efrom;
 652     }
 653   }
 654   else if (spf.kv_map.contains(helo)) {
 655   }
 656   else {
 657     LOG(WARNING)
 658         << "no explicit \"identity\" key, and no envelope-from or helo key";
 659   }
 660 }
 661
 662 namespace message {
 663
 664 bool authentication(fs::path         config_path,
 665                     char const*      server,
 666                     message::parsed& msg)
 667 {
 668   LOG(INFO) << "add_authentication_results";
 669   CHECK(!msg.headers.empty());
 670
 671   // Run our message through OpenDKIM verify
 672
 673   OpenDKIM::verify dkv;
 674   for (auto header : msg.headers) {
 675     auto const hv = header.as_view();
 676     // LOG(INFO) << "header «" << esc(hv, esc_line_option::multi) << "»";
 677     dkv.header(hv);
 678   }
 679   dkv.eoh();
 680
 681   // LOG(INFO) << "body «" << msg.body << "»";
 682   dkv.body(msg.body);
 683
 684   dkv.eom();
 685
 686   OpenDMARC::policy dmp;
 687
 688   // Build up Authentication-Results header
 689   fmt::memory_buffer bfr;
 690
 691   // Grab 1st SPF record
 692   RFC5322::received_spf_parsed spf_parsed;
 693   if (auto hdr = std::find(begin(msg.headers), end(msg.headers), Received_SPF);
 694       hdr != end(msg.headers)) {
 695     if (spf_parsed.parse(hdr->value)) {
 696       fmt::format_to(bfr, ";\r\n       spf={}", spf_parsed.result);
 697
 698       // FIXME get comment in here
 699       // fmt::format_to(bfr, " ({}) ", );
 700
 701       if (spf_parsed.kv_map[envelope_from] != spf_parsed.kv_map[helo]) {
 702         fmt::format_to(bfr, " smtp.helo={}", spf_parsed.kv_map[helo]);
 703       }
 704       else {
 705         fmt::format_to(bfr, " smtp.mailfrom={}",
 706                        spf_parsed.kv_map[envelope_from]);
 707       }
 708
 709       if (spf_parsed.kv_map.contains(client_ip)) {
 710         std::string ip = make_string(spf_parsed.kv_map[client_ip]);
 711         dmp.connect(ip.c_str());
 712       }
 713
 714       spf_result_to_dmarc(dmp, spf_parsed);
 715     }
 716     else {
 717       LOG(WARNING) << "failed to parse " << hdr->value;
 718     }
 719   }
 720
 721   // Should be only one From:
 722   if (auto hdr = std::find(begin(msg.headers), end(msg.headers), From);
 723       hdr != end(msg.headers)) {
 724     auto const from_str = make_string(hdr->value);
 725
 726     memory_input<> from_in(from_str, "from");
 727     if (!parse<RFC5322::mailbox_list_only, RFC5322::addr_specs_action>(
 728             from_in, msg.from_addrs)) {
 729       LOG(WARNING) << "failed to parse From:" << from_str;
 730     }
 731
 732     for (auto hdr_next = std::next(hdr); hdr_next != end(msg.headers);
 733          hdr_next      = std::next(hdr_next)) {
 734       if (*hdr_next == From) {
 735         LOG(WARNING) << "additional RFC5322.From header found: "
 736                      << hdr_next->as_string();
 737       }
 738     }
 739   }
 740
 741   if (msg.from_addrs.empty()) {
 742     LOG(WARNING) << "No address in RFC5322.From header";
 743     return false;
 744   }
 745
 746   /*
 747     <https://tools.ietf.org/html/rfc7489#section-6.6>
 748     6.6.1.  Extract Author Domain
 749
 750     The case of a syntactically valid multi-valued RFC5322.From field
 751     presents a particular challenge.  The process in this case is to
 752     apply the DMARC check using each of those domains found in the
 753     RFC5322.From field as the Author Domain and apply the most strict
 754     policy selected among the checks that fail.
 755
 756   */
 757
 758   // FIXME
 759   if (msg.from_addrs.size() > 1) {
 760     LOG(WARNING) << "More than one address in RFC5322.From header";
 761   }
 762
 763   auto from_addr = msg.from_addrs[0];
 764
 765   boost::trim(from_addr);
 766
 767   if (!Mailbox::validate(from_addr)) {
 768     LOG(WARNING) << "Mailbox syntax valid for RFC-5322, not for RFC-5321: \""
 769                  << from_addr << "\"";
 770     // Maybe we can pick out a valid domain?
 771     return false;
 772   }
 773
 774   Mailbox from_mbx(from_addr);
 775   msg.dmarc_from        = from_mbx.as_string(Mailbox::domain_encoding::ascii);
 776   msg.dmarc_from_domain = from_mbx.domain().ascii();
 777
 778   LOG(INFO) << "dmarc_from_domain == " << msg.dmarc_from_domain;
 779   dmp.store_from_domain(msg.dmarc_from_domain.c_str());
 780
 781   // Check each DKIM sig, inform DMARC processor, put in AR
 782
 783   dkv.foreach_sig([&dmp, &bfr](char const* domain, bool passed,
 784                                char const* identity, char const* selector,
 785                                char const* b) {
 786     int const result = passed ? DMARC_POLICY_DKIM_OUTCOME_PASS
 787                               : DMARC_POLICY_DKIM_OUTCOME_FAIL;
 788     auto const human_result = (passed ? "pass" : "fail");
 789
 790     LOG(INFO) << "DKIM check for " << domain << " " << human_result;
 791
 792     dmp.store_dkim(domain, result, human_result);
 793
 794     auto bs = std::string_view(b, strlen(b)).substr(0, 8);
 795
 796     fmt::format_to(bfr, ";\r\n       dkim={}", human_result);
 797     fmt::format_to(bfr, " header.i={}", identity);
 798     fmt::format_to(bfr, " header.s={}", selector);
 799     fmt::format_to(bfr, " header.b=\"{}\"", bs);
 800   });
 801
 802   // Set DMARC status in AR
 803
 804   auto const dmarc_passed = dmp.query_dmarc(msg.dmarc_from_domain.c_str());
 805
 806   auto const dmarc_result = (dmarc_passed ? "pass" : "fail");
 807   LOG(INFO) << "DMARC " << dmarc_result;
 808
 809   fmt::format_to(bfr, ";\r\n       dmarc={} header.from={}", dmarc_result,
 810                  msg.dmarc_from_domain);
 811
 812   // ARC
 813
 814   OpenARC::verify arv;
 815   for (auto header : msg.headers) {
 816     arv.header(header.as_view());
 817   }
 818   arv.eoh();
 819   arv.body(msg.body);
 820   arv.eom();
 821
 822   LOG(INFO) << "ARC status  == " << arv.chain_status_str();
 823   LOG(INFO) << "ARC custody == " << arv.chain_custody_str();
 824
 825   auto const arc_status = arv.chain_status_str();
 826
 827   fmt::format_to(bfr, ";\r\n       arc={}", arc_status);
 828
 829   // New AR header on the top
 830
 831   auto const ar_results = [&bfr]() {
 832     // Ug, OpenARC adds an extra one, arc.c:3213
 833     auto s = fmt::to_string(bfr);
 834     if (s.length() && s[0] == ';')
 835       s.erase(0, 1);
 836     return s;
 837   }();
 838
 839   msg.ar_str =
 840       fmt::format("{}: {};{}", Authentication_Results, server, ar_results);
 841
 842   LOG(INFO) << "new AR header " << msg.ar_str;
 843   CHECK(msg.parse_hdr(msg.ar_str));
 844
 845   // Run our message through ARC::sign
 846
 847   OpenARC::sign ars;
 848
 849   if (iequal(arc_status, "none")) {
 850     ars.set_cv_none();
 851   }
 852   else if (iequal(arc_status, "fail")) {
 853     ars.set_cv_fail();
 854   }
 855   else if (iequal(arc_status, "pass")) {
 856     ars.set_cv_pass();
 857   }
 858   else {
 859     ars.set_cv_unkn();
 860   }
 861
 862   for (auto const& header : msg.headers) {
 863     ars.header(header.as_view());
 864   }
 865   ars.eoh();
 866   ars.body(msg.body);
 867   ars.eom();
 868
 869   auto const key_file = (config_path / selector).replace_extension("private");
 870   if (!fs::exists(key_file)) {
 871     LOG(WARNING) << "can't find key file " << key_file;
 872     return dmarc_passed;
 873   }
 874   boost::iostreams::mapped_file_source priv;
 875   priv.open(key_file);
 876
 877   if (ars.seal(server, selector, server, priv.data(), priv.size(),
 878                ar_results.c_str())) {
 879     msg.arc_hdrs = ars.whole_seal();
 880     for (auto const& hdr : msg.arc_hdrs) {
 881       CHECK(msg.parse_hdr(hdr));
 882     }
 883   }
 884   else {
 885     LOG(INFO) << "failed to generate seal";
 886   }
 887
 888   OpenARC::verify arv2;
 889   for (auto const& header : msg.headers) {
 890     arv2.header(header.as_view());
 891   }
 892   arv2.eoh();
 893   arv2.body(msg.body);
 894   arv2.eom();
 895
 896   LOG(INFO) << "check ARC status  == " << arv2.chain_status_str();
 897   LOG(INFO) << "check ARC custody == " << arv2.chain_custody_str();
 898
 899   return dmarc_passed;
 900 }
 901
 902 void print_spf_envelope_froms(char const* file, message::parsed& msg)
 903 {
 904   CHECK(!msg.headers.empty());
 905   for (auto const& hdr : msg.headers) {
 906     if (hdr == Received_SPF) {
 907       RFC5322::received_spf_parsed spf_parsed;
 908       if (spf_parsed.parse(hdr.value)) {
 909         std::cout << spf_parsed.kv_map[envelope_from] << '\n';
 910         break;
 911       }
 912       else {
 913         LOG(WARNING) << "failed to parse " << file << ":\n" << hdr.as_string();
 914       }
 915     }
 916   }
 917 }
 918
 919 void remove_delivery_headers(message::parsed& msg)
 920 {
 921   // Remove headers that are added by the "delivery agent"
 922   // aka (Session::added_headers_)
 923   msg.headers.erase(
 924       std::remove(msg.headers.begin(), msg.headers.end(), Return_Path),
 925       msg.headers.end());
 926
 927   // just in case, but right now this header should not exist.
 928   msg.headers.erase(
 929       std::remove(msg.headers.begin(), msg.headers.end(), Delivered_To),
 930       msg.headers.end());
 931 }
 932
 933 void dkim_check(fs::path config_path, char const* domain, message::parsed& msg)
 934 {
 935   LOG(INFO) << "dkim";
 936
 937   CHECK(!msg.body.empty());
 938
 939   OpenDKIM::verify dkv;
 940
 941   // Run our message through OpenDKIM verify
 942
 943   for (auto header : msg.headers) {
 944     auto const hv = header.as_view();
 945     dkv.header(hv);
 946   }
 947   dkv.eoh();
 948   dkv.body(msg.body);
 949   dkv.eom();
 950
 951   // Check each DKIM sig, inform DMARC processor, put in AR
 952
 953   dkv.foreach_sig([](char const* domain, bool passed, char const* identity,
 954                      char const* selector, char const* b) {
 955     auto const human_result = (passed ? "pass" : "fail");
 956
 957     auto bs = std::string_view(b, strlen(b)).substr(0, 8);
 958
 959     LOG(INFO) << "DKIM check bfor " << domain << " " << human_result;
 960     LOG(INFO) << " header.i=" << identity;
 961     LOG(INFO) << " header.s=" << selector;
 962     LOG(INFO) << " header.b=\"" << bs << "\"";
 963   });
 964 }
 965
 966 //.............................................................................
 967
 968 bool parsed::parse(std::string_view input)
 969 {
 970   auto in{memory_input<>(input.data(), input.size(), "message")};
 971   return tao::pegtl::parse<RFC5322::message, RFC5322::msg_action>(in, *this);
 972 }
 973
 974 bool parsed::parse_hdr(std::string_view input)
 975 {
 976   auto in{memory_input<>(input.data(), input.size(), "message")};
 977   if (tao::pegtl::parse<RFC5322::raw_field, RFC5322::msg_action>(in, *this)) {
 978     std::rotate(headers.rbegin(), headers.rbegin() + 1, headers.rend());
 979     return true;
 980   }
 981   return false;
 982 }
 983
 984 std::string parsed::as_string() const
 985 {
 986   fmt::memory_buffer bfr;
 987
 988   for (auto const& h : headers)
 989     fmt::format_to(bfr, "{}\r\n", h.as_string());
 990
 991   if (!body.empty())
 992     fmt::format_to(bfr, "\r\n{}", body);
 993
 994   return fmt::to_string(bfr);
 995 }
 996
 997 bool parsed::write(std::ostream& os) const
 998 {
 999   for (auto const& h : headers)
1000     os << h.as_string() << "\r\n";
1001
1002   if (!body.empty())
1003     os << "\r\n" << body;
1004
1005   return true;
1006 }
1007
1008 std::string header::as_string() const
1009 {
1010   return fmt::format("{}:{}", name, value);
1011 }
1012
1013 std::string_view parsed::get_header(std::string_view name) const
1014 {
1015   if (auto hdr = std::find(begin(headers), end(headers), name);
1016       hdr != end(headers)) {
1017     return trim(hdr->value);
1018   }
1019   return "";
1020 }
1021
1022 void rewrite(fs::path         config_path,
1023              Domain const&    sender,
1024              message::parsed& msg,
1025              std::string      mail_from,
1026              std::string      reply_to)
1027 {
1028   LOG(INFO) << "rewrite";
1029
1030   remove_delivery_headers(msg);
1031
1032   if (!mail_from.empty()) {
1033     msg.headers.erase(std::remove(msg.headers.begin(), msg.headers.end(), From),
1034                       msg.headers.end());
1035
1036     msg.from_str = mail_from;
1037     CHECK(msg.parse_hdr(msg.from_str));
1038   }
1039
1040   if (!reply_to.empty()) {
1041     msg.headers.erase(
1042         std::remove(msg.headers.begin(), msg.headers.end(), Reply_To),
1043         msg.headers.end());
1044
1045     msg.reply_to_str = reply_to;
1046     CHECK(msg.parse_hdr(msg.reply_to_str));
1047   }
1048
1049   // modify plain text body
1050
1051   /*
1052   if (iequal(msg.get_header(MIME_Version), "1.0") &&
1053       istarts_with(msg.get_header(Content_Type), "text/plain;")) {
1054     LOG(INFO) << "Adding footer to message body.";
1055     msg.body_str = msg.body;
1056     msg.body_str.append("\r\n\r\n\t-- Added Footer --\r\n");
1057     msg.body = msg.body_str;
1058   }
1059   else {
1060     LOG(INFO) << "Not adding footer to message body.";
1061     LOG(INFO) << "MIME-Version == " << msg.get_header(MIME_Version);
1062     LOG(INFO) << "Content-Type == " << msg.get_header(Content_Type);
1063   }
1064   // LOG(INFO) << "body == " << msg.body;
1065   */
1066
1067   auto const key_file = (config_path / selector).replace_extension("private");
1068   CHECK(fs::exists(key_file)) << "can't find key file " << key_file;
1069
1070   // DKIM sign
1071
1072   boost::iostreams::mapped_file_source priv;
1073   priv.open(key_file);
1074
1075   auto const key_str = std::string(priv.data(), priv.size());
1076
1077   // Run our message through DKIM::sign
1078   OpenDKIM::sign dks(key_str.c_str(), // textual data
1079                      selector, sender.ascii().c_str(),
1080                      OpenDKIM::sign::body_type::text);
1081   for (auto const& header : msg.headers) {
1082     dks.header(header.as_view());
1083   }
1084   dks.eoh();
1085   dks.body(msg.body);
1086   dks.eom();
1087
1088   fmt::memory_buffer bfr;
1089   msg.sig_str = fmt::format("DKIM-Signature: {}", dks.getsighdr());
1090   CHECK(msg.parse_hdr(msg.sig_str));
1091 }
1092
1093 } // namespace message