Filters/FilterMail.cs

   1
   2 //
   3 // FilterMail.cs
   4 //
   5 // Copyright (C) 2004-2005 Novell, Inc.
   6 //
   7 //
   8 //
   9 // Permission is hereby granted, free of charge, to any person obtaining a
  10 // copy of this software and associated documentation files (the "Software"),
  11 // to deal in the Software without restriction, including without limitation
  12 // the rights to use, copy, modify, merge, publish, distribute, sublicense,
  13 // and/or sell copies of the Software, and to permit persons to whom the
  14 // Software is furnished to do so, subject to the following conditions:
  15 //
  16 // The above copyright notice and this permission notice shall be included in
  17 // all copies or substantial portions of the Software.
  18 //
  19 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  20 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  21 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  22 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  23 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  24 // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  25 // DEALINGS IN THE SOFTWARE.
  26 //
  27
  28 using System;
  29 using System.Collections;
  30 using System.IO;
  31
  32 using GMime;
  33
  34 using Beagle;
  35 using Beagle.Daemon;
  36 using Beagle.Util;
  37
  38 namespace Beagle.Filters {
  39
  40         [PropertyKeywordMapping (Keyword="mailfrom",     PropertyName="fixme:from_name",    IsKeyword=false)]
  41         [PropertyKeywordMapping (Keyword="mailfromaddr", PropertyName="fixme:from_address", IsKeyword=false)]
  42         [PropertyKeywordMapping (Keyword="mailto",       PropertyName="fixme:to_name",      IsKeyword=false)]
  43         [PropertyKeywordMapping (Keyword="mailtoaddr",   PropertyName="fixme:to_address",   IsKeyword=false)]
  44         [PropertyKeywordMapping (Keyword="mailinglist",  PropertyName="fixme:mlist",        IsKeyword=true, Description="Mailing list id")]
  45         public class FilterMail : Beagle.Daemon.Filter, IDisposable {
  46
  47                 private static bool gmime_initialized = false;
  48
  49                 private GMime.Message message;
  50                 private PartHandler handler;
  51
  52                 public FilterMail ()
  53                 {
  54                         // 1: Make email addresses non-keyword, add sanitized version
  55                         //    for eaching for parts of an email address.
  56                         // 2: No need to separately add sanitized version of emails.
  57                         //    BeagleAnalyzer uses a tokenfilter taking care of this.
  58                         SetVersion (2);
  59
  60                         AddSupportedFlavor (FilterFlavor.NewFromMimeType ("message/rfc822"));
  61                 }
  62
  63                 protected override void DoOpen (FileInfo info)
  64                 {
  65                         if (!gmime_initialized) {
  66                                 try {
  67                                         GMime.Global.Init ();
  68                                         gmime_initialized = true;
  69                                 } catch {
  70                                         Error ();
  71                                         return;
  72                                 }
  73                         }
  74
  75                         int mail_fd = Mono.Unix.Native.Syscall.open (info.FullName, Mono.Unix.Native.OpenFlags.O_RDONLY);
  76
  77                         if (mail_fd == -1)
  78                                 throw new IOException (String.Format ("Unable to read {0} for parsing mail", info.FullName));
  79
  80                         GMime.StreamFs stream = new GMime.StreamFs (mail_fd);
  81                         GMime.Parser parser = new GMime.Parser (stream);
  82                         this.message = parser.ConstructMessage ();
  83                         stream.Dispose ();
  84                         parser.Dispose ();
  85
  86                         if (this.message == null)
  87                                 Error ();
  88                 }
  89
  90                 private bool HasAttachments (GMime.Object mime_part)
  91                 {
  92                         if (mime_part is GMime.MessagePart)
  93                                 return true;
  94
  95                         // Messages that are multipart/alternative shouldn't be considered as having
  96                         // attachments.  Unless of course they do.
  97                         if (mime_part is GMime.Multipart && mime_part.ContentType.Subtype.ToLower () != "alternative")
  98                                 return true;
  99
 100                         return false;
 101                 }
 102
 103                 protected override void DoPullProperties ()
 104                 {
 105                         string subject = GMime.Utils.HeaderDecodePhrase (this.message.Subject);
 106                         AddProperty (Property.New ("dc:title", subject));
 107
 108                         AddProperty (Property.NewDate ("fixme:date", message.Date.ToUniversalTime ()));
 109
 110                         GMime.InternetAddressList addrs;
 111                         addrs = this.message.GetRecipients (GMime.Message.RecipientType.To);
 112                         foreach (GMime.InternetAddress ia in addrs) {
 113                                 AddProperty (Property.NewUnsearched ("fixme:to", ia.ToString (false)));
 114                                 if (ia.AddressType != GMime.InternetAddressType.Group)
 115                                         AddProperty (Property.New ("fixme:to_address", ia.Addr));
 116
 117                                 AddProperty (Property.New ("fixme:to_name", ia.Name));
 118                         }
 119                         addrs.Dispose ();
 120
 121                         addrs = this.message.GetRecipients (GMime.Message.RecipientType.Cc);
 122                         foreach (GMime.InternetAddress ia in addrs) {
 123                                 AddProperty (Property.NewUnsearched ("fixme:cc", ia.ToString (false)));
 124                                 if (ia.AddressType != GMime.InternetAddressType.Group)
 125                                         AddProperty (Property.New ("fixme:cc_address", ia.Addr));
 126
 127                                 AddProperty (Property.New ("fixme:cc_name", ia.Name));
 128                         }
 129                         addrs.Dispose ();
 130
 131                         addrs = GMime.InternetAddressList.ParseString (GMime.Utils.HeaderDecodePhrase (this.message.Sender));
 132                         foreach (GMime.InternetAddress ia in addrs) {
 133                                 AddProperty (Property.NewUnsearched ("fixme:from", ia.ToString (false)));
 134                                 if (ia.AddressType != GMime.InternetAddressType.Group)
 135                                         AddProperty (Property.New ("fixme:from_address", ia.Addr));
 136
 137                                 AddProperty (Property.New ("fixme:from_name", ia.Name));
 138                         }
 139                         addrs.Dispose ();
 140
 141                         if (HasAttachments (this.message.MimePart))
 142                                 AddProperty (Property.NewFlag ("fixme:hasAttachments"));
 143
 144                         // Store the message ID and references are unsearched
 145                         // properties.  They will be used to generate
 146                         // conversations in the frontend.
 147                         string msgid = this.message.GetHeader ("Message-Id");
 148                         if (msgid != null)
 149                                 AddProperty (Property.NewUnsearched ("fixme:msgid", GMime.Utils.DecodeMessageId (msgid)));
 150
 151                         foreach (GMime.References refs in this.message.References)
 152                                 AddProperty (Property.NewUnsearched ("fixme:reference", refs.Msgid));
 153
 154                         string list_id = this.message.GetHeader ("List-Id");
 155                         if (list_id != null) {
 156                                 // FIXME: Might need some additional parsing.
 157                                 AddProperty (Property.NewKeyword ("fixme:mlist", GMime.Utils.HeaderDecodePhrase (list_id)));
 158                         }
 159
 160                         // KMail can store replies in the same folder
 161                         // Use issent flag to distinguish between incoming
 162                         // and outgoing message
 163                         string kmail_msg_sent = this.message.GetHeader ("X-KMail-Link-Type");
 164                         bool issent_is_set = false;
 165                         foreach (Property property in IndexableProperties) {
 166                                 if (property.Key == "fixme:isSent") {
 167                                         issent_is_set = true;
 168                                         break;
 169                                 }
 170                         }
 171                         if (!issent_is_set && kmail_msg_sent != null && kmail_msg_sent == "reply")
 172                                 AddProperty (Property.NewFlag ("fixme:isSent"));
 173                 }
 174
 175                 protected override void DoPullSetup ()
 176                 {
 177                         this.handler = new PartHandler (this);
 178                         using (GMime.Object mime_part = this.message.MimePart)
 179                                 this.handler.OnEachPart (mime_part);
 180
 181                         AddChildIndexables (this.handler.ChildIndexables);
 182                 }
 183
 184                 protected override void DoPull ()
 185                 {
 186                         if (handler.Reader == null) {
 187                                 Finished ();
 188                                 return;
 189                         }
 190
 191                         string l = handler.Reader.ReadLine ();
 192
 193                         if (l == null)
 194                                 Finished ();
 195                         else if (l.Length > 0) {
 196                                 AppendText (l);
 197                                 AppendStructuralBreak ();
 198                         }
 199                 }
 200
 201                 protected override void DoClose ()
 202                 {
 203                         Dispose ();
 204                 }
 205
 206                 public void Dispose ()
 207                 {
 208                         if (this.handler != null && this.handler.Reader != null)
 209                                 this.handler.Reader.Close ();
 210                         this.handler = null;
 211
 212                         if (this.message != null) {
 213                                 this.message.Dispose ();
 214                                 this.message = null;
 215                         }
 216                 }
 217
 218                 private class PartHandler {
 219                         private Beagle.Daemon.Filter filter;
 220                         private int count = 0; // parts handled so far
 221                         private int depth = 0; // part recursion depth
 222                         private ArrayList child_indexables = new ArrayList ();
 223                         private TextReader reader;
 224
 225                         // Blacklist a handful of common MIME types that are
 226                         // either pointless on their own or ones that we don't
 227                         // have filters for.
 228                         static private string[] blacklisted_mime_types = new string[] {
 229                                 "application/pgp-signature",
 230                                 "application/x-pkcs7-signature",
 231                                 "application/ms-tnef",
 232                                 "text/x-vcalendar",
 233                                 "text/x-vcard"
 234                         };
 235
 236                         public PartHandler (Beagle.Daemon.Filter filter)
 237                         {
 238                                 this.filter = filter;
 239                         }
 240
 241                         private bool IsMimeTypeHandled (string mime_type)
 242                         {
 243                                 foreach (FilterFlavor flavor in FilterFlavor.Flavors) {
 244                                         if (flavor.IsMatch (null, null, mime_type.ToLower ()))
 245                                                 return true;
 246                                 }
 247
 248                                 return false;
 249                         }
 250
 251                         public void OnEachPart (GMime.Object mime_part)
 252                         {
 253                                 GMime.Object part = null;
 254                                 bool part_needs_dispose = false;
 255
 256                                 //for (int i = 0; i < this.depth; i++)
 257                                 //  Console.Write ("  ");
 258                                 //Console.WriteLine ("Content-Type: {0}", mime_part.ContentType);
 259
 260                                 ++depth;
 261
 262                                 if (mime_part is GMime.MessagePart) {
 263                                         GMime.MessagePart msg_part = (GMime.MessagePart) mime_part;
 264
 265                                         using (GMime.Message message = msg_part.Message) {
 266                                                 using (GMime.Object subpart = message.MimePart)
 267                                                         this.OnEachPart (subpart);
 268                                         }
 269                                 } else if (mime_part is GMime.Multipart) {
 270                                         GMime.Multipart multipart = (GMime.Multipart) mime_part;
 271
 272                                         int num_parts = multipart.Number;
 273
 274                                         // If the mimetype is multipart/alternative, we only want to index
 275                                         // one part -- the richest one we can filter.
 276                                         if (mime_part.ContentType.Subtype.ToLower () == "alternative") {
 277                                                 // The richest formats are at the end, so work from there
 278                                                 // backward.
 279                                                 for (int i = num_parts - 1; i >= 0; i--) {
 280                                                         GMime.Object subpart = multipart.GetPart (i);
 281
 282                                                         if (IsMimeTypeHandled (subpart.ContentType.ToString ())) {
 283                                                                 part = subpart;
 284                                                                 part_needs_dispose = true;
 285                                                                 break;
 286                                                         } else {
 287                                                                 subpart.Dispose ();
 288                                                         }
 289                                                 }
 290                                         }
 291
 292                                         // If it's not alternative, or we don't know how to filter any of
 293                                         // the parts, treat them like a bunch of attachments.
 294                                         if (part == null) {
 295                                                 for (int i = 0; i < num_parts; i++) {
 296                                                         using (GMime.Object subpart = multipart.GetPart (i))
 297                                                                 this.OnEachPart (subpart);
 298                                                 }
 299                                         }
 300                                 } else if (mime_part is GMime.Part)
 301                                         part = mime_part;
 302                                 else
 303                                         throw new Exception (String.Format ("Unknown part type: {0}", part.GetType ()));
 304
 305                                 if (part != null) {
 306                                         System.IO.Stream stream = null;
 307
 308                                         using (GMime.DataWrapper content_obj = ((GMime.Part) part).ContentObject)
 309                                                 stream = content_obj.Stream;
 310
 311                                         // If this is the only part and it's plain text, we
 312                                         // want to just attach it to our filter instead of
 313                                         // creating a child indexable for it.
 314                                         bool no_child_needed = false;
 315
 316                                         string mime_type = part.ContentType.ToString ().ToLower ();
 317
 318                                         if (this.depth == 1 && this.count == 0) {
 319                                                 if (mime_type == "text/plain") {
 320                                                         no_child_needed = true;
 321
 322                                                         this.reader = new StreamReader (stream);
 323                                                 }
 324                                         }
 325
 326                                         if (!no_child_needed) {
 327                                                 // Check the mime type against the blacklist and don't index any
 328                                                 // parts that are contained within.  That way the user doesn't
 329                                                 // get flooded with pointless signatures and vcard and ical
 330                                                 // attachments along with (real) attachments.
 331
 332                                                 if (Array.IndexOf (blacklisted_mime_types, mime_type) == -1) {
 333                                                         string sub_uri = this.filter.Uri.ToString () + "#" + this.count;
 334                                                         Indexable child = new Indexable (new Uri (sub_uri));
 335
 336                                                         child.DisplayUri = new Uri (this.filter.DisplayUri.ToString () + "#" + this.count);
 337
 338                                                         child.HitType = "MailMessage";
 339                                                         child.MimeType = mime_type;
 340                                                         child.CacheContent = false;
 341
 342                                                         child.AddProperty (Property.NewKeyword ("fixme:attachment_title", ((GMime.Part)part).Filename));
 343
 344                                                         if (part.ContentType.Type.ToLower () == "text")
 345                                                                 child.SetTextReader (new StreamReader (stream));
 346                                                         else
 347                                                                 child.SetBinaryStream (stream);
 348
 349                                                         this.child_indexables.Add (child);
 350                                                 } else {
 351                                                         Log.Debug ("Skipping attachment {0}#{1} with blacklisted mime type {2}",
 352                                                                    this.filter.Uri, this.count, mime_type);
 353                                                 }
 354                                         }
 355
 356                                         this.count++;
 357                                 }
 358
 359                                 if (part_needs_dispose)
 360                                         part.Dispose ();
 361
 362                                 --depth;
 363                         }
 364
 365                         public ICollection ChildIndexables {
 366                                 get { return this.child_indexables; }
 367                         }
 368
 369                         public TextReader Reader {
 370                                 get { return this.reader; }
 371                         }
 372                 }
 373
 374
 375         }
 376
 377 }