src/applications/metamta/engine/PhabricatorMailEmailEngine.php

   1 <?php
   2
   3 final class PhabricatorMailEmailEngine
   4   extends PhabricatorMailMessageEngine {
   5
   6   public function newMessage() {
   7     $mailer = $this->getMailer();
   8     $mail = $this->getMail();
   9
  10     $message = new PhabricatorMailEmailMessage();
  11
  12     $from_address = $this->newFromEmailAddress();
  13     $message->setFromAddress($from_address);
  14
  15     $reply_address = $this->newReplyToEmailAddress();
  16     if ($reply_address) {
  17       $message->setReplyToAddress($reply_address);
  18     }
  19
  20     $to_addresses = $this->newToEmailAddresses();
  21     $cc_addresses = $this->newCCEmailAddresses();
  22
  23     if (!$to_addresses && !$cc_addresses) {
  24       $mail->setMessage(
  25         pht(
  26           'Message has no valid recipients: all To/CC are disabled, '.
  27           'invalid, or configured not to receive this mail.'));
  28       return null;
  29     }
  30
  31     // If this email describes a mail processing error, we rate limit outbound
  32     // messages to each individual address. This prevents messes where
  33     // something is stuck in a loop or dumps a ton of messages on us suddenly.
  34     if ($mail->getIsErrorEmail()) {
  35       $all_recipients = array();
  36       foreach ($to_addresses as $to_address) {
  37         $all_recipients[] = $to_address->getAddress();
  38       }
  39       foreach ($cc_addresses as $cc_address) {
  40         $all_recipients[] = $cc_address->getAddress();
  41       }
  42       if ($this->shouldRateLimitMail($all_recipients)) {
  43         $mail->setMessage(
  44           pht(
  45             'This is an error email, but one or more recipients have '.
  46             'exceeded the error email rate limit. Declining to deliver '.
  47             'message.'));
  48         return null;
  49       }
  50     }
  51
  52     // Some mailers require a valid "To:" in order to deliver mail. If we
  53     // don't have any "To:", try to fill it in with a placeholder "To:".
  54     // If that also fails, move the "Cc:" line to "To:".
  55     if (!$to_addresses) {
  56       $void_address = $this->newVoidEmailAddress();
  57       $to_addresses = array($void_address);
  58     }
  59
  60     $to_addresses = $this->getUniqueEmailAddresses($to_addresses);
  61     $cc_addresses = $this->getUniqueEmailAddresses(
  62       $cc_addresses,
  63       $to_addresses);
  64
  65     $message->setToAddresses($to_addresses);
  66     $message->setCCAddresses($cc_addresses);
  67
  68     $attachments = $this->newEmailAttachments();
  69     $message->setAttachments($attachments);
  70
  71     $subject = $this->newEmailSubject();
  72     $message->setSubject($subject);
  73
  74     $headers = $this->newEmailHeaders();
  75     foreach ($this->newEmailThreadingHeaders($mailer) as $threading_header) {
  76       $headers[] = $threading_header;
  77     }
  78
  79     $stamps = $mail->getMailStamps();
  80     if ($stamps) {
  81       $headers[] = $this->newEmailHeader(
  82         'X-Phabricator-Stamps',
  83         implode(' ', $stamps));
  84     }
  85
  86     $must_encrypt = $mail->getMustEncrypt();
  87
  88     $raw_body = $mail->getBody();
  89     $body = $raw_body;
  90     if ($must_encrypt) {
  91       $parts = array();
  92
  93       $encrypt_uri = $mail->getMustEncryptURI();
  94       if (!strlen($encrypt_uri)) {
  95         $encrypt_phid = $mail->getRelatedPHID();
  96         if ($encrypt_phid) {
  97           $encrypt_uri = urisprintf(
  98             '/object/%s/',
  99             $encrypt_phid);
 100         }
 101       }
 102
 103       if (strlen($encrypt_uri)) {
 104         $parts[] = pht(
 105           'This secure message is notifying you of a change to this object:');
 106         $parts[] = PhabricatorEnv::getProductionURI($encrypt_uri);
 107       }
 108
 109       $parts[] = pht(
 110         'The content for this message can only be transmitted over a '.
 111         'secure channel. To view the message content, follow this '.
 112         'link:');
 113
 114       $parts[] = PhabricatorEnv::getProductionURI($mail->getURI());
 115
 116       $body = implode("\n\n", $parts);
 117     } else {
 118       $body = $raw_body;
 119     }
 120
 121     $body_limit = PhabricatorEnv::getEnvConfig('metamta.email-body-limit');
 122
 123     $body = phutil_string_cast($body);
 124     if (strlen($body) > $body_limit) {
 125       $body = id(new PhutilUTF8StringTruncator())
 126         ->setMaximumBytes($body_limit)
 127         ->truncateString($body);
 128       $body .= "\n";
 129       $body .= pht('(This email was truncated at %d bytes.)', $body_limit);
 130     }
 131     $message->setTextBody($body);
 132     $body_limit -= strlen($body);
 133
 134     // If we sent a different message body than we were asked to, record
 135     // what we actually sent to make debugging and diagnostics easier.
 136     if ($body !== $raw_body) {
 137       $mail->setDeliveredBody($body);
 138     }
 139
 140     if ($must_encrypt) {
 141       $send_html = false;
 142     } else {
 143       $send_html = $this->shouldSendHTML();
 144     }
 145
 146     if ($send_html) {
 147       $html_body = $mail->getHTMLBody();
 148       if (phutil_nonempty_string($html_body)) {
 149         // NOTE: We just drop the entire HTML body if it won't fit. Safely
 150         // truncating HTML is hard, and we already have the text body to fall
 151         // back to.
 152         if (strlen($html_body) <= $body_limit) {
 153           $message->setHTMLBody($html_body);
 154           $body_limit -= strlen($html_body);
 155         }
 156       }
 157     }
 158
 159     // Pass the headers to the mailer, then save the state so we can show
 160     // them in the web UI. If the mail must be encrypted, we remove headers
 161     // which are not on a strict whitelist to avoid disclosing information.
 162     $filtered_headers = $this->filterHeaders($headers, $must_encrypt);
 163     $message->setHeaders($filtered_headers);
 164
 165     $mail->setUnfilteredHeaders($headers);
 166     $mail->setDeliveredHeaders($headers);
 167
 168     if (PhabricatorEnv::getEnvConfig('phabricator.silent')) {
 169       $mail->setMessage(
 170         pht(
 171           'This software is running in silent mode. See `%s` '.
 172           'in the configuration to change this setting.',
 173           'phabricator.silent'));
 174
 175       return null;
 176     }
 177
 178     return $message;
 179   }
 180
 181 /* -(  Message Components  )------------------------------------------------- */
 182
 183   private function newFromEmailAddress() {
 184     $from_address = $this->newDefaultEmailAddress();
 185     $mail = $this->getMail();
 186
 187     // If the mail content must be encrypted, always disguise the sender.
 188     $must_encrypt = $mail->getMustEncrypt();
 189     if ($must_encrypt) {
 190       return $from_address;
 191     }
 192
 193     // If we have a raw "From" address, use that.
 194     $raw_from = $mail->getRawFrom();
 195     if ($raw_from) {
 196       list($from_email, $from_name) = $raw_from;
 197       return $this->newEmailAddress($from_email, $from_name);
 198     }
 199
 200     // Otherwise, use as much of the information for any sending entity as
 201     // we can.
 202     $from_phid = $mail->getFrom();
 203
 204     $actor = $this->getActor($from_phid);
 205     if ($actor) {
 206       $actor_email = $actor->getEmailAddress();
 207       $actor_name = $actor->getName();
 208     } else {
 209       $actor_email = null;
 210       $actor_name = null;
 211     }
 212
 213     $send_as_user = PhabricatorEnv::getEnvConfig('metamta.can-send-as-user');
 214     if ($send_as_user) {
 215       if ($actor_email !== null) {
 216         $from_address->setAddress($actor_email);
 217       }
 218     }
 219
 220     if ($actor_name !== null) {
 221       $from_address->setDisplayName($actor_name);
 222     }
 223
 224     return $from_address;
 225   }
 226
 227   private function newReplyToEmailAddress() {
 228     $mail = $this->getMail();
 229
 230     $reply_raw = $mail->getReplyTo();
 231     if (!phutil_nonempty_string($reply_raw)) {
 232       return null;
 233     }
 234
 235     $reply_address = new PhutilEmailAddress($reply_raw);
 236
 237     // If we have a sending object, change the display name.
 238     $from_phid = $mail->getFrom();
 239     $actor = $this->getActor($from_phid);
 240     if ($actor) {
 241       $reply_address->setDisplayName($actor->getName());
 242     }
 243
 244     // If we don't have a display name, fill in a default.
 245     if (!strlen($reply_address->getDisplayName())) {
 246       $reply_address->setDisplayName(PlatformSymbols::getPlatformServerName());
 247     }
 248
 249     return $reply_address;
 250   }
 251
 252   private function newToEmailAddresses() {
 253     $mail = $this->getMail();
 254
 255     $phids = $mail->getToPHIDs();
 256     $addresses = $this->newEmailAddressesFromActorPHIDs($phids);
 257
 258     foreach ($mail->getRawToAddresses() as $raw_address) {
 259       $addresses[] = new PhutilEmailAddress($raw_address);
 260     }
 261
 262     return $addresses;
 263   }
 264
 265   private function newCCEmailAddresses() {
 266     $mail = $this->getMail();
 267     $phids = $mail->getCcPHIDs();
 268     return $this->newEmailAddressesFromActorPHIDs($phids);
 269   }
 270
 271   private function newEmailAddressesFromActorPHIDs(array $phids) {
 272     $mail = $this->getMail();
 273     $phids = $mail->expandRecipients($phids);
 274
 275     $addresses = array();
 276     foreach ($phids as $phid) {
 277       $actor = $this->getActor($phid);
 278       if (!$actor) {
 279         continue;
 280       }
 281
 282       if (!$actor->isDeliverable()) {
 283         continue;
 284       }
 285
 286       $addresses[] = new PhutilEmailAddress($actor->getEmailAddress());
 287     }
 288
 289     return $addresses;
 290   }
 291
 292   private function newEmailSubject() {
 293     $mail = $this->getMail();
 294
 295     $is_threaded = (bool)$mail->getThreadID();
 296     $must_encrypt = $mail->getMustEncrypt();
 297
 298     $subject = array();
 299
 300     if ($is_threaded) {
 301       if ($this->shouldAddRePrefix()) {
 302         $subject[] = 'Re:';
 303       }
 304     }
 305
 306     $subject_prefix = $mail->getSubjectPrefix();
 307     $subject_prefix = phutil_string_cast($subject_prefix);
 308     $subject_prefix = trim($subject_prefix);
 309
 310     $subject[] = $subject_prefix;
 311
 312     // If mail content must be encrypted, we replace the subject with
 313     // a generic one.
 314     if ($must_encrypt) {
 315       $encrypt_subject = $mail->getMustEncryptSubject();
 316       if (!strlen($encrypt_subject)) {
 317         $encrypt_subject = pht('Object Updated');
 318       }
 319       $subject[] = $encrypt_subject;
 320     } else {
 321       $vary_prefix = $mail->getVarySubjectPrefix();
 322       if (phutil_nonempty_string($vary_prefix)) {
 323         if ($this->shouldVarySubject()) {
 324           $subject[] = $vary_prefix;
 325         }
 326       }
 327
 328       $subject[] = $mail->getSubject();
 329     }
 330
 331     foreach ($subject as $key => $part) {
 332       if (!phutil_nonempty_string($part)) {
 333         unset($subject[$key]);
 334       }
 335     }
 336
 337     $subject = implode(' ', $subject);
 338     return $subject;
 339   }
 340
 341   private function newEmailHeaders() {
 342     $mail = $this->getMail();
 343
 344     $headers = array();
 345
 346     $headers[] = $this->newEmailHeader(
 347       'X-Phabricator-Sent-This-Message',
 348       'Yes');
 349     $headers[] = $this->newEmailHeader(
 350       'X-Mail-Transport-Agent',
 351       'MetaMTA');
 352
 353     // Some clients respect this to suppress OOF and other auto-responses.
 354     $headers[] = $this->newEmailHeader(
 355       'X-Auto-Response-Suppress',
 356       'All');
 357
 358     $mailtags = $mail->getMailTags();
 359     if ($mailtags) {
 360       $tag_header = array();
 361       foreach ($mailtags as $mailtag) {
 362         $tag_header[] = '<'.$mailtag.'>';
 363       }
 364       $tag_header = implode(', ', $tag_header);
 365       $headers[] = $this->newEmailHeader(
 366         'X-Phabricator-Mail-Tags',
 367         $tag_header);
 368     }
 369
 370     $value = $mail->getHeaders();
 371     foreach ($value as $pair) {
 372       list($header_key, $header_value) = $pair;
 373
 374       // NOTE: If we have \n in a header, SES rejects the email.
 375       $header_value = str_replace("\n", ' ', $header_value);
 376       $headers[] = $this->newEmailHeader($header_key, $header_value);
 377     }
 378
 379     $is_bulk = $mail->getIsBulk();
 380     if ($is_bulk) {
 381       $headers[] = $this->newEmailHeader('Precedence', 'bulk');
 382     }
 383
 384     if ($mail->getMustEncrypt()) {
 385       $headers[] = $this->newEmailHeader('X-Phabricator-Must-Encrypt', 'Yes');
 386     }
 387
 388     $related_phid = $mail->getRelatedPHID();
 389     if ($related_phid) {
 390       $headers[] = $this->newEmailHeader('Thread-Topic', $related_phid);
 391     }
 392
 393     $headers[] = $this->newEmailHeader(
 394       'X-Phabricator-Mail-ID',
 395       $mail->getID());
 396
 397     $unique = Filesystem::readRandomCharacters(16);
 398     $headers[] = $this->newEmailHeader(
 399       'X-Phabricator-Send-Attempt',
 400       $unique);
 401
 402     return $headers;
 403   }
 404
 405   private function newEmailThreadingHeaders() {
 406     $mailer = $this->getMailer();
 407     $mail = $this->getMail();
 408
 409     $headers = array();
 410
 411     $thread_id = $mail->getThreadID();
 412     if (!phutil_nonempty_string($thread_id)) {
 413       return $headers;
 414     }
 415
 416     $is_first = $mail->getIsFirstMessage();
 417
 418     // NOTE: Gmail freaks out about In-Reply-To and References which aren't in
 419     // the form "<string@domain.tld>"; this is also required by RFC 2822,
 420     // although some clients are more liberal in what they accept.
 421     $domain = $this->newMailDomain();
 422     $thread_id = '<'.$thread_id.'@'.$domain.'>';
 423
 424     if ($is_first && $mailer->supportsMessageIDHeader()) {
 425       $headers[] = $this->newEmailHeader('Message-ID',  $thread_id);
 426     } else {
 427       $in_reply_to = $thread_id;
 428       $references = array($thread_id);
 429       $parent_id = $mail->getParentMessageID();
 430       if ($parent_id) {
 431         $in_reply_to = $parent_id;
 432         // By RFC 2822, the most immediate parent should appear last
 433         // in the "References" header, so this order is intentional.
 434         $references[] = $parent_id;
 435       }
 436       $references = implode(' ', $references);
 437       $headers[] = $this->newEmailHeader('In-Reply-To', $in_reply_to);
 438       $headers[] = $this->newEmailHeader('References',  $references);
 439     }
 440     $thread_index = $this->generateThreadIndex($thread_id, $is_first);
 441     $headers[] = $this->newEmailHeader('Thread-Index', $thread_index);
 442
 443     return $headers;
 444   }
 445
 446   private function newEmailAttachments() {
 447     $mail = $this->getMail();
 448
 449     // If the mail content must be encrypted, don't add attachments.
 450     $must_encrypt = $mail->getMustEncrypt();
 451     if ($must_encrypt) {
 452       return array();
 453     }
 454
 455     return $mail->getAttachments();
 456   }
 457
 458 /* -(  Preferences  )-------------------------------------------------------- */
 459
 460   private function shouldAddRePrefix() {
 461     $preferences = $this->getPreferences();
 462
 463     $value = $preferences->getSettingValue(
 464       PhabricatorEmailRePrefixSetting::SETTINGKEY);
 465
 466     return ($value == PhabricatorEmailRePrefixSetting::VALUE_RE_PREFIX);
 467   }
 468
 469   private function shouldVarySubject() {
 470     $preferences = $this->getPreferences();
 471
 472     $value = $preferences->getSettingValue(
 473       PhabricatorEmailVarySubjectsSetting::SETTINGKEY);
 474
 475     return ($value == PhabricatorEmailVarySubjectsSetting::VALUE_VARY_SUBJECTS);
 476   }
 477
 478   private function shouldSendHTML() {
 479     $preferences = $this->getPreferences();
 480
 481     $value = $preferences->getSettingValue(
 482       PhabricatorEmailFormatSetting::SETTINGKEY);
 483
 484     return ($value == PhabricatorEmailFormatSetting::VALUE_HTML_EMAIL);
 485   }
 486
 487
 488 /* -(  Utilities  )---------------------------------------------------------- */
 489
 490   private function newEmailHeader($name, $value) {
 491     return id(new PhabricatorMailHeader())
 492       ->setName($name)
 493       ->setValue($value);
 494   }
 495
 496   private function newEmailAddress($address, $name = null) {
 497     $object = id(new PhutilEmailAddress())
 498       ->setAddress($address);
 499
 500     if (strlen($name)) {
 501       $object->setDisplayName($name);
 502     }
 503
 504     return $object;
 505   }
 506
 507   public function newDefaultEmailAddress() {
 508     $raw_address = PhabricatorEnv::getEnvConfig('metamta.default-address');
 509
 510     if (!strlen($raw_address)) {
 511       $domain = $this->newMailDomain();
 512       $raw_address = "noreply@{$domain}";
 513     }
 514
 515     $address = new PhutilEmailAddress($raw_address);
 516
 517     if (!phutil_nonempty_string($address->getDisplayName())) {
 518       $address->setDisplayName(PlatformSymbols::getPlatformServerName());
 519     }
 520
 521     return $address;
 522   }
 523
 524   public function newVoidEmailAddress() {
 525     return $this->newDefaultEmailAddress();
 526   }
 527
 528   private function newMailDomain() {
 529     $domain = PhabricatorEnv::getEnvConfig('metamta.reply-handler-domain');
 530     if (strlen($domain)) {
 531       return $domain;
 532     }
 533
 534     $install_uri = PhabricatorEnv::getURI('/');
 535     $install_uri = new PhutilURI($install_uri);
 536
 537     return $install_uri->getDomain();
 538   }
 539
 540   private function filterHeaders(array $headers, $must_encrypt) {
 541     assert_instances_of($headers, 'PhabricatorMailHeader');
 542
 543     if (!$must_encrypt) {
 544       return $headers;
 545     }
 546
 547     $whitelist = array(
 548       'In-Reply-To',
 549       'Message-ID',
 550       'Precedence',
 551       'References',
 552       'Thread-Index',
 553       'Thread-Topic',
 554
 555       'X-Mail-Transport-Agent',
 556       'X-Auto-Response-Suppress',
 557
 558       'X-Phabricator-Sent-This-Message',
 559       'X-Phabricator-Must-Encrypt',
 560       'X-Phabricator-Mail-ID',
 561       'X-Phabricator-Send-Attempt',
 562     );
 563
 564     // NOTE: The major header we want to drop is "X-Phabricator-Mail-Tags".
 565     // This header contains a significant amount of meaningful information
 566     // about the object.
 567
 568     $whitelist_map = array();
 569     foreach ($whitelist as $term) {
 570       $whitelist_map[phutil_utf8_strtolower($term)] = true;
 571     }
 572
 573     foreach ($headers as $key => $header) {
 574       $name = $header->getName();
 575       $name = phutil_utf8_strtolower($name);
 576
 577       if (!isset($whitelist_map[$name])) {
 578         unset($headers[$key]);
 579       }
 580     }
 581
 582     return $headers;
 583   }
 584
 585   private function getUniqueEmailAddresses(
 586     array $addresses,
 587     array $exclude = array()) {
 588     assert_instances_of($addresses, 'PhutilEmailAddress');
 589     assert_instances_of($exclude, 'PhutilEmailAddress');
 590
 591     $seen = array();
 592
 593     foreach ($exclude as $address) {
 594       $seen[$address->getAddress()] = true;
 595     }
 596
 597     foreach ($addresses as $key => $address) {
 598       $raw_address = $address->getAddress();
 599
 600       if (isset($seen[$raw_address])) {
 601         unset($addresses[$key]);
 602         continue;
 603       }
 604
 605       $seen[$raw_address] = true;
 606     }
 607
 608     return array_values($addresses);
 609   }
 610
 611   private function generateThreadIndex($seed, $is_first_mail) {
 612     // When threading, Outlook ignores the 'References' and 'In-Reply-To'
 613     // headers that most clients use. Instead, it uses a custom 'Thread-Index'
 614     // header. The format of this header is something like this (from
 615     // camel-exchange-folder.c in Evolution Exchange):
 616
 617     /* A new post to a folder gets a 27-byte-long thread index. (The value
 618      * is apparently unique but meaningless.) Each reply to a post gets a
 619      * 32-byte-long thread index whose first 27 bytes are the same as the
 620      * parent's thread index. Each reply to any of those gets a
 621      * 37-byte-long thread index, etc. The Thread-Index header contains a
 622      * base64 representation of this value.
 623      */
 624
 625     // The specific implementation uses a 27-byte header for the first email
 626     // a recipient receives, and a random 5-byte suffix (32 bytes total)
 627     // thereafter. This means that all the replies are (incorrectly) siblings,
 628     // but it would be very difficult to keep track of the entire tree and this
 629     // gets us reasonable client behavior.
 630
 631     $base = substr(md5($seed), 0, 27);
 632     if (!$is_first_mail) {
 633       // Not totally sure, but it seems like outlook orders replies by
 634       // thread-index rather than timestamp, so to get these to show up in the
 635       // right order we use the time as the last 4 bytes.
 636       $base .= ' '.pack('N', time());
 637     }
 638
 639     return base64_encode($base);
 640   }
 641
 642   private function shouldRateLimitMail(array $all_recipients) {
 643     try {
 644       PhabricatorSystemActionEngine::willTakeAction(
 645         $all_recipients,
 646         new PhabricatorMetaMTAErrorMailAction(),
 647         1);
 648       return false;
 649     } catch (PhabricatorSystemActionRateLimitException $ex) {
 650       return true;
 651     }
 652   }
 653
 654 }