beagled/EvolutionMailDriver/EvolutionMailIndexableGenerator.cs

   1
   2 //
   3 // EvolutionMailIndexableGenerator.cs
   4 //
   5 // Copyright (C) 2004 Novell, Inc.
   6 //
   7 //
   8 //
   9 // Permission is hereby granted, free of charge, to any person obtaining a
  10 // copy of this software and associated documentation files (the "Software"),
  11 // to deal in the Software without restriction, including without limitation
  12 // the rights to use, copy, modify, merge, publish, distribute, sublicense,
  13 // and/or sell copies of the Software, and to permit persons to whom the
  14 // Software is furnished to do so, subject to the following conditions:
  15 //
  16 // The above copyright notice and this permission notice shall be included in
  17 // all copies or substantial portions of the Software.
  18 //
  19 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  20 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  21 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  22 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  23 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  24 // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  25 // DEALINGS IN THE SOFTWARE.
  26 //
  27
  28 using System;
  29 using System.Collections;
  30 using System.IO;
  31 using System.Runtime.Serialization.Formatters.Binary;
  32 using System.Threading;
  33 using System.Xml;
  34
  35 using Beagle.Util;
  36 using Beagle.Daemon;
  37
  38 using Camel = Beagle.Util.Camel;
  39 using Mono.Posix;
  40
  41 namespace Beagle.Daemon.EvolutionMailDriver {
  42
  43         public abstract class EvolutionMailIndexableGenerator : IIndexableGenerator {
  44
  45                 private static bool gmime_initialized = false;
  46
  47                 private static ArrayList excludes = new ArrayList ();
  48
  49                 static EvolutionMailIndexableGenerator () {
  50                         foreach (ExcludeItem exclude in Conf.Indexing.Excludes)
  51                                 if (exclude.Type == ExcludeType.MailFolder)
  52                                         excludes.Add (exclude);
  53                 }
  54
  55                 protected EvolutionMailQueryable queryable;
  56
  57                 protected string account_name, folder_name;
  58                 protected int count, indexed_count;
  59
  60                 protected EvolutionMailIndexableGenerator (EvolutionMailQueryable queryable)
  61                 {
  62                         this.queryable = queryable;
  63                 }
  64
  65                 protected abstract string GetFolderName (FileSystemInfo info);
  66                 protected abstract bool Setup ();
  67                 protected abstract FileInfo CrawlFile { get; }
  68                 public abstract string GetTarget ();
  69                 public abstract bool HasNextIndexable ();
  70                 public abstract Indexable GetNextIndexable ();
  71                 public abstract void Checkpoint ();
  72
  73                 protected EvolutionMailQueryable Queryable {
  74                         get { return this.queryable; }
  75                 }
  76
  77                 protected static void InitializeGMime ()
  78                 {
  79                         if (!gmime_initialized) {
  80                                 GMime.Global.Init ();
  81                                 gmime_initialized = true;
  82                         }
  83                 }
  84
  85                 protected bool IsSpamFolder (string name)
  86                 {
  87                         if (name.ToLower () == "spam" || name.ToLower () == "junk")
  88                                 return true;
  89                         else
  90                                 return false;
  91                 }
  92
  93                 protected bool IgnoreFolder (string path)
  94                 {
  95                         // FIXME: Use System.IO.Path
  96                         foreach (ExcludeItem exclude in excludes) {
  97                                 if (exclude.IsMatch (path))
  98                                         return true;
  99                         }
 100                         return false;
 101                 }
 102
 103                 protected void CrawlFinished ()
 104                 {
 105                         // FIXME: This is a little sketchy
 106                         this.queryable.FileAttributesStore.AttachLastWriteTime (this.CrawlFile.FullName, DateTime.Now);
 107                 }
 108
 109                 public string StatusName {
 110                         get { return this.CrawlFile.FullName; }
 111                 }
 112
 113                 public override bool Equals (object o)
 114                 {
 115                         EvolutionMailIndexableGenerator generator = o as EvolutionMailIndexableGenerator;
 116
 117                         if (generator == null)
 118                                 return false;
 119
 120                         if (Object.ReferenceEquals (this, generator))
 121                                 return true;
 122
 123                         if (this.CrawlFile.FullName == generator.CrawlFile.FullName)
 124                                 return true;
 125                         else
 126                                 return false;
 127                 }
 128
 129                 public override int GetHashCode ()
 130                 {
 131                         return this.CrawlFile.FullName.GetHashCode ();
 132                 }
 133         }
 134
 135         public class EvolutionMailIndexableGeneratorMbox : EvolutionMailIndexableGenerator {
 136                 private FileInfo mbox_info;
 137                 private int mbox_fd = -1;
 138                 private GMime.StreamFs mbox_stream;
 139                 private GMime.Parser mbox_parser;
 140                 private long file_size;
 141
 142                 public EvolutionMailIndexableGeneratorMbox (EvolutionMailQueryable queryable, FileInfo mbox_info) : base (queryable)
 143                 {
 144                         this.mbox_info = mbox_info;
 145                 }
 146
 147                 protected override string GetFolderName (FileSystemInfo info)
 148                 {
 149                         FileInfo file_info = (FileInfo) info;
 150                         DirectoryInfo di;
 151                         string folder_name = "";
 152
 153                         di = file_info.Directory;
 154                         while (di != null) {
 155                                 // Evo uses ".sbd" as the extension on a folder
 156                                 if (di.Extension == ".sbd")
 157                                         folder_name = Path.Combine (Path.GetFileNameWithoutExtension (di.Name), folder_name);
 158                                 else
 159                                         break;
 160
 161                                 di = di.Parent;
 162                         }
 163
 164                         return Path.Combine (folder_name, file_info.Name);
 165                 }
 166
 167                 protected override bool Setup ()
 168                 {
 169                         this.account_name = "local@local";
 170                         this.folder_name = this.GetFolderName (this.mbox_info);
 171
 172                         if (this.IsSpamFolder (this.folder_name))
 173                                 return false;
 174
 175                         if (this.IgnoreFolder (this.mbox_info.FullName))
 176                                 return false;
 177
 178                         return true;
 179                 }
 180
 181                 private long MboxLastOffset {
 182                         get {
 183                                 string offset_str = this.queryable.ReadDataLine ("offset-" + this.folder_name.Replace ('/', '-'));
 184                                 long offset = Convert.ToInt64 (offset_str);
 185
 186                                 Logger.Log.Debug ("mbox {0} offset is {1}", this.mbox_info.Name, offset);
 187                                 return offset;
 188                         }
 189
 190                         set {
 191                                 this.queryable.WriteDataLine ("offset-" + this.folder_name.Replace ('/', '-'), value.ToString ());
 192                         }
 193                 }
 194
 195                 public override bool HasNextIndexable ()
 196                 {
 197                         if (this.account_name == null) {
 198                                 if (!Setup ())
 199                                         return false;
 200                         }
 201
 202                         if (this.mbox_fd < 0) {
 203                                 Logger.Log.Debug ("opening mbox {0}", this.mbox_info.Name);
 204
 205                                 try {
 206                                         InitializeGMime ();
 207                                 } catch (Exception e) {
 208                                         Logger.Log.Warn ("Caught exception trying to initalize gmime:");
 209                                         Logger.Log.Warn (e);
 210                                         return false;
 211                                 }
 212
 213                                 this.mbox_fd = Syscall.open (this.mbox_info.FullName, OpenFlags.O_RDONLY);
 214                                 this.mbox_stream = new GMime.StreamFs (this.mbox_fd);
 215                                 this.mbox_stream.Seek ((int) this.MboxLastOffset);
 216                                 this.mbox_parser = new GMime.Parser (this.mbox_stream);
 217                                 this.mbox_parser.ScanFrom = true;
 218
 219                                 FileInfo info = new FileInfo (this.mbox_info.FullName);
 220                                 this.file_size = info.Length;
 221                         }
 222
 223                         if (this.mbox_parser.Eos ()) {
 224                                 long offset = this.mbox_parser.FromOffset;
 225
 226                                 this.mbox_stream.Close ();
 227
 228                                 this.mbox_fd = -1;
 229                                 this.mbox_stream.Dispose ();
 230                                 this.mbox_stream = null;
 231                                 this.mbox_parser.Dispose ();
 232                                 this.mbox_parser = null;
 233
 234                                 EvolutionMailQueryable.log.Debug ("{0}: Finished indexing {1} messages",
 235                                                                   this.folder_name, this.indexed_count);
 236
 237                                 this.MboxLastOffset = offset;
 238                                 this.CrawlFinished ();
 239
 240                                 return false;
 241                         } else
 242                                 return true;
 243                 }
 244
 245                 public override Indexable GetNextIndexable ()
 246                 {
 247                         using (GMime.Message message = this.mbox_parser.ConstructMessage ()) {
 248                                 ++this.count;
 249
 250                                 // Work around what I think is a bug in GMime: If you
 251                                 // have a zero-byte file or seek to the end of a
 252                                 // file, parser.Eos () will return true until it
 253                                 // actually tries to read something off the wire.
 254                                 // Since parser.ConstructMessage() always returns a
 255                                 // message (which may also be a bug), we'll often get
 256                                 // one empty message which we need to deal with here.
 257                                 //
 258                                 // Check if its empty by seeing if the Headers
 259                                 // property is null or empty.
 260                                 if (message.Headers == null || message.Headers == "")
 261                                         return null;
 262
 263                                 string x_evolution = message.GetHeader ("X-Evolution");
 264                                 if (x_evolution == null || x_evolution == "") {
 265                                         EvolutionMailQueryable.log.Info ("{0}: Message at offset {1} has no X-Evolution header!",
 266                                                                          this.folder_name, this.mbox_parser.FromOffset);
 267                                         return null;
 268                                 }
 269
 270                                 int separator_idx = x_evolution.IndexOf ('-');
 271
 272                                 string uid_str = x_evolution.Substring (0, separator_idx);
 273                                 string uid = Convert.ToUInt32 (uid_str, 16).ToString (); // ugh.
 274                                 uint flags = Convert.ToUInt32 (x_evolution.Substring (separator_idx + 1), 16);
 275
 276                                 Indexable indexable = this.GMimeMessageToIndexable (uid, message, flags);
 277
 278                                 if (indexable == null)
 279                                         return null;
 280
 281                                 ++this.indexed_count;
 282
 283                                 // HACK: update your recipients
 284                                 EvolutionMailQueryable.AddAsYourRecipient (indexable);
 285
 286                                 return indexable;
 287                         }
 288                 }
 289
 290                 private static bool CheckFlags (uint flags, Camel.CamelFlags test)
 291                 {
 292                         return (flags & (uint) test) == (uint) test;
 293                 }
 294
 295                 private Indexable GMimeMessageToIndexable (string uid, GMime.Message message, uint flags)
 296                 {
 297                         // Don't index messages flagged as junk
 298                         if (CheckFlags (flags, Camel.CamelFlags.Junk))
 299                                 return null;
 300
 301                         System.Uri uri = EvolutionMailQueryable.EmailUri (this.account_name, this.folder_name, uid);
 302                         Indexable indexable = new Indexable (uri);
 303
 304                         indexable.Timestamp = message.Date;
 305                         indexable.Type = "MailMessage";
 306                         indexable.MimeType = "message/rfc822";
 307                         indexable.CacheContent = false;
 308
 309                         indexable.AddProperty (Property.NewKeyword ("fixme:client", "evolution"));
 310                         indexable.AddProperty (Property.NewKeyword ("fixme:account", "Local"));
 311                         indexable.AddProperty (Property.NewKeyword ("fixme:folder", this.folder_name));
 312
 313                         GMime.InternetAddressList addrs;
 314
 315                         addrs = message.GetRecipients (GMime.Message.RecipientType.To);
 316                         foreach (GMime.InternetAddress ia in addrs) {
 317                                 if (this.folder_name == "Sent")
 318                                         indexable.AddProperty (Property.NewKeyword ("fixme:sentTo", ia.Addr));
 319                         }
 320                         addrs.Dispose ();
 321
 322                         addrs = message.GetRecipients (GMime.Message.RecipientType.Cc);
 323                         foreach (GMime.InternetAddress ia in addrs) {
 324                                 if (this.folder_name == "Sent")
 325                                         indexable.AddProperty (Property.NewKeyword ("fixme:sentTo", ia.Addr));
 326                         }
 327                         addrs.Dispose ();
 328
 329                         addrs = GMime.InternetAddressList.ParseString (GMime.Utils.HeaderDecodePhrase (message.Sender));
 330                         foreach (GMime.InternetAddress ia in addrs) {
 331                                 if (this.folder_name != "Sent")
 332                                         indexable.AddProperty (Property.NewKeyword ("fixme:gotFrom", ia.Addr));
 333                         }
 334                         addrs.Dispose ();
 335
 336                         if (this.folder_name == "Sent")
 337                                 indexable.AddProperty (Property.NewFlag ("fixme:isSent"));
 338
 339                         if (this.folder_name == "Sent")
 340                                 indexable.AddProperty (Property.NewDate ("fixme:sentdate", message.Date));
 341                         else
 342                                 indexable.AddProperty (Property.NewDate ("fixme:received", message.Date));
 343
 344                         indexable.AddProperty (Property.NewKeyword ("fixme:flags", flags));
 345
 346                         if (CheckFlags (flags, Camel.CamelFlags.Answered))
 347                                 indexable.AddProperty (Property.NewFlag ("fixme:isAnswered"));
 348
 349                         if (CheckFlags (flags, Camel.CamelFlags.Deleted))
 350                                 indexable.AddProperty (Property.NewFlag ("fixme:isDeleted"));
 351
 352                         if (CheckFlags (flags, Camel.CamelFlags.Draft))
 353                                 indexable.AddProperty (Property.NewFlag ("fixme:isDraft"));
 354
 355                         if (CheckFlags (flags, Camel.CamelFlags.Flagged))
 356                                 indexable.AddProperty (Property.NewFlag ("fixme:isFlagged"));
 357
 358                         if (CheckFlags (flags, Camel.CamelFlags.Seen))
 359                                 indexable.AddProperty (Property.NewFlag ("fixme:isSeen"));
 360
 361                         if (CheckFlags (flags, Camel.CamelFlags.AnsweredAll))
 362                                 indexable.AddProperty (Property.NewFlag ("fixme:isAnsweredAll"));
 363
 364                         MemoryStream stream = new MemoryStream ();
 365                         message.WriteToStream (stream);
 366                         stream.Seek (0, SeekOrigin.Begin);
 367                         indexable.SetBinaryStream (stream);
 368
 369                         return indexable;
 370                 }
 371
 372                 public override void Checkpoint ()
 373                 {
 374                         long offset = -1;
 375
 376                         if (this.mbox_parser != null)
 377                                 this.MboxLastOffset = offset = this.mbox_parser.FromOffset;
 378
 379                         EvolutionMailQueryable.log.Debug ("{0}: indexed {1} messages ({2}/{3} bytes {4:###.0}%)",
 380                                                           this.folder_name, this.indexed_count,
 381                                                           offset, this.file_size,
 382                                                           100.0 * offset / this.file_size);
 383                 }
 384
 385                 public override string GetTarget ()
 386                 {
 387                         return "mbox-file:" + mbox_info.FullName;
 388                 }
 389
 390                 protected override FileInfo CrawlFile {
 391                         get { return this.mbox_info; }
 392                 }
 393         }
 394
 395         public class EvolutionMailIndexableGeneratorImap : EvolutionMailIndexableGenerator {
 396                 private enum ImapBackendType {
 397                         Imap,
 398                         Imap4
 399                 };
 400
 401                 private FileInfo summary_info;
 402                 private string imap_name;
 403                 private ImapBackendType backend_type;
 404                 private Camel.Summary summary;
 405                 private IEnumerator summary_enumerator;
 406                 private ICollection accounts;
 407                 private string folder_cache_name;
 408                 private Hashtable mapping;
 409                 private ArrayList deleted_list;
 410
 411                 public EvolutionMailIndexableGeneratorImap (EvolutionMailQueryable queryable, FileInfo summary_info) : base (queryable)
 412                 {
 413                         this.summary_info = summary_info;
 414                 }
 415
 416                 protected override string GetFolderName (FileSystemInfo info)
 417                 {
 418                         DirectoryInfo dir_info = (DirectoryInfo) info;
 419                         string folder_name = "";
 420
 421                         while (dir_info != null) {
 422                                 folder_name = Path.Combine (dir_info.Name, folder_name);
 423
 424                                 dir_info = dir_info.Parent;
 425
 426                                 if (dir_info.Name != "subfolders")
 427                                         break;
 428                                 else
 429                                         dir_info = dir_info.Parent;
 430                         }
 431
 432                         return folder_name;
 433                 }
 434
 435                 protected override bool Setup ()
 436                 {
 437                         string dir_name = summary_info.DirectoryName;
 438                         int imap_start_idx;
 439
 440                         int idx = dir_name.IndexOf (".evolution/mail/imap4/");
 441
 442                         if (idx >= 0) {
 443                                 this.backend_type = ImapBackendType.Imap4;
 444                                 imap_start_idx = idx + 22;
 445                         } else {
 446                                 this.backend_type = ImapBackendType.Imap;
 447                                 imap_start_idx = dir_name.IndexOf (".evolution/mail/imap/") + 21;
 448                         }
 449
 450                         string imap_start = dir_name.Substring (imap_start_idx);
 451                         this.imap_name = imap_start.Substring (0, imap_start.IndexOf ('/'));
 452
 453                         try {
 454                                 this.accounts = (ICollection) GConfThreadHelper.Get ("/apps/evolution/mail/accounts");
 455                         } catch (Exception ex) {
 456                                 EvolutionMailQueryable.log.Warn ("Caught exception in Setup(): " + ex.Message);
 457                                 EvolutionMailQueryable.log.Warn ("There are no configured evolution accounts, ignoring {0}", this.imap_name);
 458                                 return false;
 459                         }
 460
 461                         // This should only happen if we shut down while waiting for the GConf results to come back.
 462                         if (this.accounts == null)
 463                                 return false;
 464
 465                         foreach (string xml in this.accounts) {
 466                                 XmlDocument xmlDoc = new XmlDocument ();
 467
 468                                 xmlDoc.LoadXml (xml);
 469
 470                                 XmlNode account = xmlDoc.SelectSingleNode ("//account");
 471
 472                                 if (account == null)
 473                                         continue;
 474
 475                                 string uid = null;
 476
 477                                 foreach (XmlAttribute attr in account.Attributes) {
 478                                         if (attr.Name == "uid") {
 479                                                 uid = attr.InnerText;
 480                                                 break;
 481                                         }
 482                                 }
 483
 484                                 if (uid == null)
 485                                         continue;
 486
 487                                 XmlNode imap_url_node = xmlDoc.SelectSingleNode ("//source/url");
 488
 489                                 if (imap_url_node == null)
 490                                         continue;
 491
 492                                 string imap_url = imap_url_node.InnerText;
 493                                 // If there is a semicolon in the username part of the URL, it
 494                                 // indicates that there's an auth scheme there.  We don't care
 495                                 // about that, so remove it.
 496                                 int user_end = imap_url.IndexOf ('@');
 497                                 int semicolon = imap_url.IndexOf (';', 0, user_end + 1);
 498
 499                                 if (semicolon != -1)
 500                                         imap_url = imap_url.Substring (0, semicolon) + imap_url.Substring (user_end);
 501
 502                                 // Escape out additional @s in the name.  I hate the class libs so much.
 503                                 int lastIdx = this.imap_name.LastIndexOf ('@');
 504                                 if (this.imap_name.IndexOf ('@') != lastIdx) {
 505                                         string toEscape = this.imap_name.Substring (0, lastIdx);
 506                                         this.imap_name = toEscape.Replace ("@", "%40") + this.imap_name.Substring (lastIdx);
 507                                 }
 508
 509                                 string backend_url_prefix;
 510                                 if (this.backend_type == ImapBackendType.Imap)
 511                                         backend_url_prefix = "imap";
 512                                 else
 513                                         backend_url_prefix = "imap4";
 514
 515                                 if (imap_url.StartsWith (backend_url_prefix + "://" + this.imap_name + "/")) {
 516                                         this.account_name = uid;
 517                                         break;
 518                                 }
 519                         }
 520
 521                         if (account_name == null) {
 522                                 EvolutionMailQueryable.log.Info ("Unable to determine account name for {0}", this.imap_name);
 523                                 return false;
 524                         }
 525
 526                         // Need to check the directory on disk to see if it's a junk/spam folder,
 527                         // since the folder name will be "foo/spam" and not match the check below.
 528                         DirectoryInfo dir_info = new DirectoryInfo (dir_name);
 529                         if (this.IsSpamFolder (dir_info.Name))
 530                                 return false;
 531
 532                         // Check if the folder is listed in the configuration as to be excluded from indexing
 533                         if (this.IgnoreFolder (dir_info.FullName))
 534                                 return false;
 535
 536                         this.folder_name = GetFolderName (new DirectoryInfo (dir_name));
 537
 538                         return true;
 539                 }
 540
 541                 private string FolderCacheName {
 542                         get {
 543                                 if (this.folder_cache_name == null)
 544                                         this.folder_cache_name = "status-" + this.account_name + "-" + this.folder_name.Replace ('/', '-');
 545
 546                                 return this.folder_cache_name;
 547                         }
 548                 }
 549
 550                 private bool LoadCache ()
 551                 {
 552                         Stream cacheStream;
 553                         BinaryFormatter formatter;
 554
 555                         try {
 556                                 cacheStream = this.queryable.ReadDataStream (this.FolderCacheName);
 557                                 formatter = new BinaryFormatter ();
 558                                 this.mapping = formatter.Deserialize (cacheStream) as Hashtable;
 559                                 cacheStream.Close ();
 560                                 EvolutionMailQueryable.log.Debug ("Successfully loaded previous crawled data from disk: {0}", this.FolderCacheName);
 561
 562                                 return true;
 563                         } catch {
 564                                 this.mapping = new Hashtable ();
 565
 566                                 return false;
 567                         }
 568                 }
 569
 570                 private void SaveCache ()
 571                 {
 572                         Stream cacheStream;
 573                         BinaryFormatter formatter;
 574
 575                         cacheStream = this.queryable.WriteDataStream (this.FolderCacheName);
 576                         formatter = new BinaryFormatter ();
 577                         formatter.Serialize (cacheStream, mapping);
 578                         cacheStream.Close ();
 579                 }
 580
 581                 public override bool HasNextIndexable ()
 582                 {
 583                         if (this.account_name == null) {
 584                                 if (!Setup ())
 585                                         return false;
 586                         }
 587
 588                         if (this.mapping == null) {
 589                                 bool cache_loaded = this.LoadCache ();
 590
 591                                 this.deleted_list = new ArrayList (this.mapping.Keys);
 592                                 this.deleted_list.Sort ();
 593                                 Logger.Log.Debug ("Deleted list starting at {0} for {1}", this.deleted_list.Count, this.folder_name);
 594
 595                                 // Check to see if we even need to bother walking the summary
 596                                 if (cache_loaded && this.queryable.FileAttributesStore.IsUpToDate (this.CrawlFile.FullName)) {
 597                                         EvolutionMailQueryable.log.Debug ("{0}: summary has not been updated; crawl unncessary", this.folder_name);
 598                                         return false;
 599                                 }
 600                         }
 601
 602                         if (this.summary == null) {
 603                                 try {
 604                                         if (this.backend_type == ImapBackendType.Imap)
 605                                                 this.summary = Camel.Summary.LoadImapSummary (this.summary_info.FullName);
 606                                         else
 607                                                 this.summary = Camel.Summary.LoadImap4Summary (this.summary_info.FullName);
 608                                 } catch (Exception e) {
 609                                         EvolutionMailQueryable.log.Warn ("Unable to index {0}: {1}", this.folder_name,
 610                                                                          e.Message);
 611                                         return false;
 612                                 }
 613                         }
 614
 615                         if (this.summary_enumerator == null)
 616                                 this.summary_enumerator = this.summary.GetEnumerator ();
 617
 618                         if (this.summary_enumerator.MoveNext ())
 619                                 return true;
 620
 621                         Logger.Log.Debug ("Queuing up {0} removals for deleted messages in ", this.deleted_list.Count, this.folder_name);
 622                         foreach (string uid in this.deleted_list) {
 623                                 Uri uri = EvolutionMailQueryable.EmailUri (this.account_name, this.folder_name, uid);
 624
 625                                 // FIXME: This is kind of a hack, but it's the only way
 626                                 // with the IndexableGenerator to handle our removals.
 627                                 Scheduler.Task task = this.Queryable.NewRemoveTask (uri);
 628                                 task.Priority = Scheduler.Priority.Immediate;
 629                                 this.Queryable.ThisScheduler.Add (task);
 630                         }
 631
 632                         EvolutionMailQueryable.log.Debug ("{0}: Finished indexing {1} ({2}/{3} {4:###.0}%)",
 633                                                           this.folder_name, this.indexed_count, this.count,
 634                                                           this.summary.header.count,
 635                                                           100.0 * this.count / this.summary.header.count);
 636
 637                         this.SaveCache ();
 638                         this.CrawlFinished ();
 639
 640                         return false;
 641                 }
 642
 643                 // Kind of nasty, but we need the function.
 644                 [System.Runtime.InteropServices.DllImport("libglib-2.0.so.0")]
 645                 static extern int g_str_hash (string str);
 646
 647                 // Stolen from deep within e-d-s's camel-data-cache.c  Very evil.
 648                 private const int CAMEL_DATA_CACHE_MASK = ((1 << 6) - 1);
 649
 650                 public override Indexable GetNextIndexable ()
 651                 {
 652                         Indexable indexable = null;
 653
 654                         Camel.MessageInfo mi = (Camel.MessageInfo) this.summary_enumerator.Current;
 655
 656                         ++this.count;
 657
 658                         // Try to load the cached message data off disk
 659                         object flags = this.mapping[mi.uid];
 660
 661                         if (flags == null || (uint) flags != mi.flags) {
 662                                 string msg_file;
 663
 664                                 if (this.backend_type == ImapBackendType.Imap)
 665                                         msg_file = Path.Combine (summary_info.DirectoryName, mi.uid + ".");
 666                                 else {
 667                                         // This is taken from e-d-s's camel-data-cache.c.  No doubt
 668                                         // NotZed would scream bloody murder if he saw this here.
 669                                         int hash = (g_str_hash (mi.uid) >> 5) & CAMEL_DATA_CACHE_MASK;
 670                                         string cache_path = String.Format ("cache/{0:x}/{1}", hash, mi.uid);
 671                                         msg_file = Path.Combine (summary_info.DirectoryName, cache_path);
 672                                 }
 673
 674                                 indexable = this.CamelMessageToIndexable (mi, msg_file);
 675
 676                                 this.mapping[mi.uid] = mi.flags;
 677
 678                                 if (indexable != null) {
 679                                         ++this.indexed_count;
 680                                         this.deleted_list.Remove (mi.uid);
 681
 682                                         // HACK: update your recipients
 683                                         EvolutionMailQueryable.AddAsYourRecipient (indexable);
 684                                 }
 685                         } else if (flags != null)
 686                                 this.deleted_list.Remove (mi.uid);
 687
 688                         return indexable;
 689                 }
 690
 691                 private Uri CamelMessageUri (Camel.MessageInfo message_info)
 692                 {
 693                         return EvolutionMailQueryable.EmailUri (this.account_name, this.folder_name, message_info.uid);
 694                 }
 695
 696                 private Indexable CamelMessageToIndexable (Camel.MessageInfo messageInfo, string msg_file)
 697                 {
 698                         // Don't index messages flagged as junk
 699                         if (messageInfo.IsJunk)
 700                                 return null;
 701
 702                         // Many properties will be set by the filter when
 703                         // processing the cached data, if it's there.  So
 704                         // don't set a number of properties in that case.
 705                         bool have_content = File.Exists (msg_file);
 706
 707                         Uri uri = CamelMessageUri (messageInfo);
 708                         Indexable indexable = new Indexable (uri);
 709
 710                         indexable.Timestamp = messageInfo.Date;
 711                         indexable.MimeType = "message/rfc822";
 712                         indexable.Type = "MailMessage";
 713
 714                         indexable.AddProperty (Property.NewKeyword ("fixme:account",  this.imap_name));
 715                         indexable.AddProperty (Property.NewKeyword ("fixme:folder",   this.folder_name));
 716                         indexable.AddProperty (Property.NewKeyword ("fixme:client", "evolution"));
 717
 718                         if (!have_content)
 719                                 indexable.AddProperty (Property.New ("dc:title", messageInfo.subject));
 720
 721                         GMime.InternetAddressList addrs;
 722                         addrs = GMime.InternetAddressList.ParseString (messageInfo.to);
 723                         foreach (GMime.InternetAddress ia in addrs) {
 724                                 if (!have_content) {
 725                                         indexable.AddProperty (Property.NewKeyword ("fixme:to", ia.ToString (false)));
 726                                         indexable.AddProperty (Property.NewKeyword ("fixme:to_address", ia.Addr));
 727                                         indexable.AddProperty (Property.New ("fixme:to_name", ia.Name));
 728                                 }
 729
 730                                 if (this.folder_name == "Sent")
 731                                         indexable.AddProperty (Property.NewKeyword ("fixme:sentTo", ia.Addr));
 732                         }
 733                         addrs.Dispose ();
 734
 735                         addrs = GMime.InternetAddressList.ParseString (messageInfo.cc);
 736                         foreach (GMime.InternetAddress ia in addrs) {
 737                                 if (!have_content) {
 738                                         indexable.AddProperty (Property.NewKeyword ("fixme:cc", ia.ToString (false)));
 739                                         indexable.AddProperty (Property.NewKeyword ("fixme:cc_address", ia.Addr));
 740                                         indexable.AddProperty (Property.New ("fixme:cc_name", ia.Name));
 741                                 }
 742
 743                                 if (this.folder_name == "Sent")
 744                                         indexable.AddProperty (Property.NewKeyword ("fixme:sentTo", ia.Addr));
 745                         }
 746                         addrs.Dispose ();
 747
 748                         addrs = GMime.InternetAddressList.ParseString (messageInfo.from);
 749                         foreach (GMime.InternetAddress ia in addrs) {
 750                                 if (!have_content) {
 751                                         indexable.AddProperty (Property.NewKeyword ("fixme:from", ia.ToString (false)));
 752                                         indexable.AddProperty (Property.NewKeyword ("fixme:from_address", ia.Addr));
 753                                         indexable.AddProperty (Property.New ("fixme:from_name", ia.Name));
 754                                 }
 755
 756                                 if (this.folder_name != "Sent")
 757                                         indexable.AddProperty (Property.NewKeyword ("fixme:gotFrom", ia.Addr));
 758                         }
 759                         addrs.Dispose ();
 760
 761                         indexable.AddProperty (Property.NewKeyword ("fixme:mlist",    messageInfo.mlist));
 762                         indexable.AddProperty (Property.NewKeyword ("fixme:flags",    messageInfo.flags));
 763
 764                         if (messageInfo.received != DateTime.MinValue)
 765                                 indexable.AddProperty (Property.NewDate ("fixme:received", messageInfo.received));
 766
 767                         if (messageInfo.sent != DateTime.MinValue)
 768                                 indexable.AddProperty (Property.NewDate ("fixme:sentdate", messageInfo.sent));
 769
 770                         if (this.folder_name == "Sent")
 771                                 indexable.AddProperty (Property.NewFlag ("fixme:isSent"));
 772
 773                         if (messageInfo.IsAnswered)
 774                                 indexable.AddProperty (Property.NewFlag ("fixme:isAnswered"));
 775
 776                         if (messageInfo.IsDeleted)
 777                                 indexable.AddProperty (Property.NewFlag ("fixme:isDeleted"));
 778
 779                         if (messageInfo.IsDraft)
 780                                 indexable.AddProperty (Property.NewFlag ("fixme:isDraft"));
 781
 782                         if (messageInfo.IsFlagged)
 783                                 indexable.AddProperty (Property.NewFlag ("fixme:isFlagged"));
 784
 785                         if (messageInfo.IsSeen)
 786                                 indexable.AddProperty (Property.NewFlag ("fixme:isSeen"));
 787
 788                         if (messageInfo.HasAttachments && !have_content)
 789                                 indexable.AddProperty (Property.NewFlag ("fixme:hasAttachments"));
 790
 791                         if (messageInfo.IsAnsweredAll)
 792                                 indexable.AddProperty (Property.NewFlag ("fixme:isAnsweredAll"));
 793
 794                         if (have_content)
 795                                 indexable.ContentUri = UriFu.PathToFileUri (msg_file);
 796                         else
 797                                 indexable.NoContent = true;
 798
 799                         return indexable;
 800                 }
 801
 802                 public override void Checkpoint ()
 803                 {
 804                         EvolutionMailQueryable.log.Debug ("{0}: indexed {1} messages ({2}/{3} {4:###.0}%)",
 805                                                           this.folder_name, this.indexed_count, this.count,
 806                                                           this.summary.header.count,
 807                                                           100.0 * this.count / this.summary.header.count);
 808
 809                         this.SaveCache ();
 810                 }
 811
 812                 public override string GetTarget ()
 813                 {
 814                         return "summary-file:" + summary_info.FullName;
 815                 }
 816
 817                 protected override FileInfo CrawlFile {
 818                         get { return this.summary_info; }
 819                 }
 820         }
 821 }