beagled/EvolutionMailDriver/EvolutionMailIndexableGenerator.cs

   1
   2 //
   3 // EvolutionMailIndexableGenerator.cs
   4 //
   5 // Copyright (C) 2004 Novell, Inc.
   6 //
   7 //
   8 //
   9 // Permission is hereby granted, free of charge, to any person obtaining a
  10 // copy of this software and associated documentation files (the "Software"),
  11 // to deal in the Software without restriction, including without limitation
  12 // the rights to use, copy, modify, merge, publish, distribute, sublicense,
  13 // and/or sell copies of the Software, and to permit persons to whom the
  14 // Software is furnished to do so, subject to the following conditions:
  15 //
  16 // The above copyright notice and this permission notice shall be included in
  17 // all copies or substantial portions of the Software.
  18 //
  19 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  20 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  21 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  22 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  23 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  24 // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  25 // DEALINGS IN THE SOFTWARE.
  26 //
  27
  28 using System;
  29 using System.Collections;
  30 using System.IO;
  31 using System.Runtime.Serialization.Formatters.Binary;
  32 using System.Threading;
  33 using System.Xml;
  34
  35 using Beagle.Util;
  36 using Beagle.Daemon;
  37
  38 using Camel = Beagle.Util.Camel;
  39
  40 namespace Beagle.Daemon.EvolutionMailDriver {
  41
  42         public abstract class EvolutionMailIndexableGenerator : IIndexableGenerator {
  43
  44                 protected static bool Debug = false;
  45
  46                 private static bool gmime_initialized = false;
  47
  48                 private static ArrayList excludes = new ArrayList ();
  49
  50                 static EvolutionMailIndexableGenerator () {
  51                         foreach (ExcludeItem exclude in Conf.Indexing.Excludes)
  52                                 if (exclude.Type == ExcludeType.MailFolder)
  53                                         excludes.Add (exclude);
  54                 }
  55
  56                 protected EvolutionMailQueryable queryable;
  57
  58                 protected string account_name, folder_name;
  59                 protected int count, indexed_count;
  60
  61                 protected EvolutionMailIndexableGenerator (EvolutionMailQueryable queryable)
  62                 {
  63                         this.queryable = queryable;
  64                 }
  65
  66                 protected abstract string GetFolderName (FileSystemInfo info);
  67                 protected abstract bool Setup ();
  68                 protected abstract FileInfo CrawlFile { get; }
  69                 public abstract string GetTarget ();
  70                 public abstract bool HasNextIndexable ();
  71                 public abstract Indexable GetNextIndexable ();
  72                 public abstract void Checkpoint ();
  73
  74                 public void PostFlushHook ()
  75                 {
  76                         Checkpoint ();
  77                 }
  78
  79                 protected static void InitializeGMime ()
  80                 {
  81                         if (!gmime_initialized) {
  82                                 GMime.Global.Init ();
  83                                 gmime_initialized = true;
  84                         }
  85                 }
  86
  87                 protected bool IsSpamFolder (string name)
  88                 {
  89                         if (name.ToLower () == "spam" || name.ToLower () == "junk")
  90                                 return true;
  91                         else
  92                                 return false;
  93                 }
  94
  95                 protected bool IgnoreFolder (string path)
  96                 {
  97                         // FIXME: Use System.IO.Path
  98                         foreach (ExcludeItem exclude in excludes) {
  99                                 if (exclude.IsMatch (path))
 100                                         return true;
 101                         }
 102                         return false;
 103                 }
 104
 105                 protected void CrawlFinished ()
 106                 {
 107                         // FIXME: This is a little sketchy
 108                         this.queryable.FileAttributesStore.AttachLastWriteTime (this.CrawlFile.FullName, DateTime.UtcNow);
 109
 110                         this.queryable.SetGeneratorProgress (this, 100);
 111                 }
 112
 113                 public string StatusName {
 114                         get { return this.CrawlFile.FullName; }
 115                 }
 116
 117                 public override bool Equals (object o)
 118                 {
 119                         EvolutionMailIndexableGenerator generator = o as EvolutionMailIndexableGenerator;
 120
 121                         if (generator == null)
 122                                 return false;
 123
 124                         if (Object.ReferenceEquals (this, generator))
 125                                 return true;
 126
 127                         if (this.CrawlFile.FullName == generator.CrawlFile.FullName)
 128                                 return true;
 129                         else
 130                                 return false;
 131                 }
 132
 133                 public override int GetHashCode ()
 134                 {
 135                         return this.CrawlFile.FullName.GetHashCode ();
 136                 }
 137         }
 138
 139         public class EvolutionMailIndexableGeneratorMbox : EvolutionMailIndexableGenerator {
 140                 private FileInfo mbox_info;
 141                 private int mbox_fd = -1;
 142                 private GMime.StreamFs mbox_stream;
 143                 private GMime.Parser mbox_parser;
 144                 private long file_size;
 145
 146                 public EvolutionMailIndexableGeneratorMbox (EvolutionMailQueryable queryable, FileInfo mbox_info) : base (queryable)
 147                 {
 148                         this.mbox_info = mbox_info;
 149                 }
 150
 151                 protected override string GetFolderName (FileSystemInfo info)
 152                 {
 153                         FileInfo file_info = (FileInfo) info;
 154                         DirectoryInfo di;
 155                         string folder_name = "";
 156
 157                         di = file_info.Directory;
 158                         while (di != null) {
 159                                 // Evo uses ".sbd" as the extension on a folder
 160                                 if (di.Extension == ".sbd")
 161                                         folder_name = Path.Combine (Path.GetFileNameWithoutExtension (di.Name), folder_name);
 162                                 else
 163                                         break;
 164
 165                                 di = di.Parent;
 166                         }
 167
 168                         return Path.Combine (folder_name, file_info.Name);
 169                 }
 170
 171                 protected override bool Setup ()
 172                 {
 173                         this.account_name = "local@local";
 174                         this.folder_name = this.GetFolderName (this.mbox_info);
 175
 176                         if (this.IsSpamFolder (this.folder_name))
 177                                 return false;
 178
 179                         if (this.IgnoreFolder (this.mbox_info.FullName))
 180                                 return false;
 181
 182                         return true;
 183                 }
 184
 185                 private long MboxLastOffset {
 186                         get {
 187                                 string offset_str = this.queryable.ReadDataLine ("offset-" + this.folder_name.Replace ('/', '-'));
 188                                 long offset = Convert.ToInt64 (offset_str);
 189
 190                                 return offset;
 191                         }
 192
 193                         set {
 194                                 this.queryable.WriteDataLine ("offset-" + this.folder_name.Replace ('/', '-'), value.ToString ());
 195                         }
 196                 }
 197
 198                 public override bool HasNextIndexable ()
 199                 {
 200                         if (this.account_name == null) {
 201                                 if (!Setup ()) {
 202                                         this.queryable.RemoveGeneratorProgress (this);
 203                                         return false;
 204                                 }
 205                         }
 206
 207                         if (this.mbox_fd < 0) {
 208                                 Logger.Log.Debug ("Opening mbox {0}", this.mbox_info.Name);
 209
 210                                 try {
 211                                         InitializeGMime ();
 212                                 } catch (Exception e) {
 213                                         Logger.Log.Warn ("Caught exception trying to initalize gmime:");
 214                                         Logger.Log.Warn (e);
 215                                         return false;
 216                                 }
 217
 218                                 this.mbox_fd = Mono.Unix.Native.Syscall.open (this.mbox_info.FullName, Mono.Unix.Native.OpenFlags.O_RDONLY);
 219                                 this.mbox_stream = new GMime.StreamFs (this.mbox_fd);
 220                                 this.mbox_stream.Seek ((int) this.MboxLastOffset);
 221                                 this.mbox_parser = new GMime.Parser (this.mbox_stream);
 222                                 this.mbox_parser.ScanFrom = true;
 223
 224                                 FileInfo info = new FileInfo (this.mbox_info.FullName);
 225                                 this.file_size = info.Length;
 226                         }
 227
 228                         if (this.mbox_parser.Eos ()) {
 229                                 long offset = this.mbox_parser.FromOffset;
 230
 231                                 this.mbox_stream.Close ();
 232
 233                                 this.mbox_fd = -1;
 234                                 this.mbox_stream.Dispose ();
 235                                 this.mbox_stream = null;
 236                                 this.mbox_parser.Dispose ();
 237                                 this.mbox_parser = null;
 238
 239                                 Logger.Log.Debug ("{0}: Finished indexing {1} messages", this.folder_name, this.indexed_count);
 240
 241                                 if (offset >= 0)
 242                                         this.MboxLastOffset = offset;
 243                                 this.CrawlFinished ();
 244
 245                                 return false;
 246                         } else
 247                                 return true;
 248                 }
 249
 250                 public override Indexable GetNextIndexable ()
 251                 {
 252                         using (GMime.Message message = this.mbox_parser.ConstructMessage ()) {
 253                                 // Work around what I think is a bug in GMime: If you
 254                                 // have a zero-byte file or seek to the end of a
 255                                 // file, parser.Eos () will return true until it
 256                                 // actually tries to read something off the wire.
 257                                 // Since parser.ConstructMessage() always returns a
 258                                 // message (which may also be a bug), we'll often get
 259                                 // one empty message which we need to deal with here.
 260                                 //
 261                                 // Check if its empty by seeing if the Headers
 262                                 // property is null or empty.
 263                                 if (message == null || message.Headers == null || message.Headers == "")
 264                                         return null;
 265
 266                                 ++this.count;
 267
 268                                 string x_evolution = message.GetHeader ("X-Evolution");
 269                                 if (x_evolution == null || x_evolution == "") {
 270                                         Logger.Log.Info ("{0}: Message at offset {1} has no X-Evolution header!",
 271                                                          this.folder_name, this.mbox_parser.FromOffset);
 272                                         return null;
 273                                 }
 274
 275                                 // This extracts the UID and flags from the X-Evolution header.
 276                                 // It may also contain user-defined flags and tags, but we don't
 277                                 // support those right now.
 278                                 int separator_idx = x_evolution.IndexOf ('-');
 279
 280                                 string uid_str = x_evolution.Substring (0, separator_idx);
 281                                 string uid = Convert.ToUInt32 (uid_str, 16).ToString (); // ugh.
 282                                 uint flags = Convert.ToUInt32 (x_evolution.Substring (separator_idx + 1, 4), 16);
 283
 284                                 Indexable indexable = this.GMimeMessageToIndexable (uid, message, flags);
 285
 286                                 if (Debug) {
 287                                         Logger.Log.Debug ("Constructed message {0} with uid {1}, flags {2}.  Indexable {3} null",
 288                                                           this.count, uid, flags, indexable == null ? "" : "not");
 289                                 }
 290
 291                                 if (indexable == null)
 292                                         return null;
 293
 294                                 ++this.indexed_count;
 295
 296                                 return indexable;
 297                         }
 298                 }
 299
 300                 private static bool CheckFlags (uint flags, Camel.CamelFlags test)
 301                 {
 302                         return (flags & (uint) test) == (uint) test;
 303                 }
 304
 305                 private Indexable GMimeMessageToIndexable (string uid, GMime.Message message, uint flags)
 306                 {
 307                         // Don't index messages flagged as junk
 308                         if (CheckFlags (flags, Camel.CamelFlags.Junk))
 309                                 return null;
 310
 311                         System.Uri uri = EvolutionMailQueryable.EmailUri (this.account_name, this.folder_name, uid);
 312                         Indexable indexable = new Indexable (uri);
 313
 314                         indexable.Timestamp = message.Date.ToUniversalTime ();
 315                         indexable.HitType = "MailMessage";
 316                         indexable.MimeType = "message/rfc822";
 317                         indexable.CacheContent = false;
 318
 319                         indexable.AddProperty (Property.NewUnsearched ("fixme:client", "evolution"));
 320                         indexable.AddProperty (Property.NewUnsearched ("fixme:account", "Local"));
 321                         indexable.AddProperty (Property.NewUnsearched ("fixme:folder", this.folder_name));
 322
 323                         GMime.InternetAddressList addrs;
 324
 325                         addrs = message.GetRecipients (GMime.Message.RecipientType.To);
 326                         foreach (GMime.InternetAddress ia in addrs) {
 327                                 if (this.folder_name == "Sent" && ia.AddressType != GMime.InternetAddressType.Group)
 328                                         indexable.AddProperty (Property.NewUnsearched ("fixme:sentTo", ia.Addr));
 329                         }
 330                         addrs.Dispose ();
 331
 332                         addrs = message.GetRecipients (GMime.Message.RecipientType.Cc);
 333                         foreach (GMime.InternetAddress ia in addrs) {
 334                                 if (this.folder_name == "Sent"  && ia.AddressType != GMime.InternetAddressType.Group)
 335                                         indexable.AddProperty (Property.NewUnsearched ("fixme:sentTo", ia.Addr));
 336                         }
 337                         addrs.Dispose ();
 338
 339                         addrs = GMime.InternetAddressList.ParseString (GMime.Utils.HeaderDecodePhrase (message.Sender));
 340                         foreach (GMime.InternetAddress ia in addrs) {
 341                                 if (this.folder_name != "Sent"  && ia.AddressType != GMime.InternetAddressType.Group)
 342                                         indexable.AddProperty (Property.NewUnsearched ("fixme:gotFrom", ia.Addr));
 343                         }
 344                         addrs.Dispose ();
 345
 346                         if (this.folder_name == "Sent")
 347                                 indexable.AddProperty (Property.NewFlag ("fixme:isSent"));
 348
 349                         Property flag_prop = Property.NewUnsearched ("fixme:flags", flags);
 350                         flag_prop.IsMutable = true;
 351                         indexable.AddProperty (flag_prop);
 352
 353                         if (CheckFlags (flags, Camel.CamelFlags.Answered))
 354                                 indexable.AddProperty (Property.NewFlag ("fixme:isAnswered"));
 355
 356                         if (CheckFlags (flags, Camel.CamelFlags.Deleted))
 357                                 indexable.AddProperty (Property.NewFlag ("fixme:isDeleted"));
 358
 359                         if (CheckFlags (flags, Camel.CamelFlags.Draft))
 360                                 indexable.AddProperty (Property.NewFlag ("fixme:isDraft"));
 361
 362                         if (CheckFlags (flags, Camel.CamelFlags.Flagged))
 363                                 indexable.AddProperty (Property.NewFlag ("fixme:isFlagged"));
 364
 365                         if (CheckFlags (flags, Camel.CamelFlags.Seen))
 366                                 indexable.AddProperty (Property.NewFlag ("fixme:isSeen"));
 367
 368                         if (CheckFlags (flags, Camel.CamelFlags.AnsweredAll))
 369                                 indexable.AddProperty (Property.NewFlag ("fixme:isAnsweredAll"));
 370
 371                         indexable.SetBinaryStream (message.Stream);
 372
 373                         return indexable;
 374                 }
 375
 376                 public override void Checkpoint ()
 377                 {
 378                         long offset = -1;
 379
 380                         if (this.mbox_parser != null) {
 381                                 offset = this.mbox_parser.FromOffset;
 382
 383                                 if (offset >= 0)
 384                                         this.MboxLastOffset = offset;
 385                         }
 386
 387                         string progress = "";
 388                         if (this.file_size > 0 && offset > 0) {
 389                                 progress = String.Format (" ({0}/{1} bytes {2:###.0}%)",
 390                                                           offset, this.file_size, 100.0 * offset / this.file_size);
 391
 392                                 this.queryable.SetGeneratorProgress (this, (int) (100.0 * offset / this.file_size));
 393                         }
 394
 395                         Logger.Log.Debug ("{0}: indexed {1} messages{2}",
 396                                           this.folder_name, this.indexed_count, progress);
 397                 }
 398
 399                 public override string GetTarget ()
 400                 {
 401                         return "mbox-file:" + mbox_info.FullName;
 402                 }
 403
 404                 protected override FileInfo CrawlFile {
 405                         get { return this.mbox_info; }
 406                 }
 407         }
 408
 409         public class EvolutionMailIndexableGeneratorImap : EvolutionMailIndexableGenerator {
 410                 private enum ImapBackendType {
 411                         Imap,
 412                         Imap4
 413                 };
 414
 415                 private FileInfo summary_info;
 416                 private string imap_name;
 417                 private ImapBackendType backend_type;
 418                 private Camel.Summary summary;
 419                 private IEnumerator summary_enumerator;
 420                 private ICollection accounts;
 421                 private string folder_cache_name;
 422                 private Hashtable mapping;
 423                 private ArrayList deleted_list;
 424                 private bool delete_mode;
 425                 private int delete_count;
 426
 427                 public EvolutionMailIndexableGeneratorImap (EvolutionMailQueryable queryable, FileInfo summary_info) : base (queryable)
 428                 {
 429                         this.summary_info = summary_info;
 430                 }
 431
 432                 protected override string GetFolderName (FileSystemInfo info)
 433                 {
 434                         DirectoryInfo dir_info = (DirectoryInfo) info;
 435                         string folder_name = "";
 436
 437                         while (dir_info != null) {
 438                                 folder_name = Path.Combine (dir_info.Name, folder_name);
 439
 440                                 dir_info = dir_info.Parent;
 441
 442                                 if (dir_info.Name != "subfolders")
 443                                         break;
 444                                 else
 445                                         dir_info = dir_info.Parent;
 446                         }
 447
 448                         return folder_name;
 449                 }
 450
 451                 protected override bool Setup ()
 452                 {
 453                         string dir_name = summary_info.DirectoryName;
 454                         int imap_start_idx;
 455
 456                         int idx = dir_name.IndexOf (".evolution/mail/imap4/");
 457
 458                         if (idx >= 0) {
 459                                 this.backend_type = ImapBackendType.Imap4;
 460                                 imap_start_idx = idx + 22;
 461                         } else {
 462                                 this.backend_type = ImapBackendType.Imap;
 463                                 imap_start_idx = dir_name.IndexOf (".evolution/mail/imap/") + 21;
 464                         }
 465
 466                         string imap_start = dir_name.Substring (imap_start_idx);
 467                         this.imap_name = imap_start.Substring (0, imap_start.IndexOf ('/'));
 468
 469                         try {
 470                                 this.accounts = (ICollection) GConfThreadHelper.Get ("/apps/evolution/mail/accounts");
 471                         } catch (Exception ex) {
 472                                 Logger.Log.Warn ("Caught exception in Setup(): " + ex.Message);
 473                                 Logger.Log.Warn ("There are no configured evolution accounts, ignoring {0}", this.imap_name);
 474                                 return false;
 475                         }
 476
 477                         // This should only happen if we shut down while waiting for the GConf results to come back.
 478                         if (this.accounts == null)
 479                                 return false;
 480
 481                         foreach (string xml in this.accounts) {
 482                                 XmlDocument xmlDoc = new XmlDocument ();
 483
 484                                 xmlDoc.LoadXml (xml);
 485
 486                                 XmlNode account = xmlDoc.SelectSingleNode ("//account");
 487
 488                                 if (account == null)
 489                                         continue;
 490
 491                                 string uid = null;
 492
 493                                 foreach (XmlAttribute attr in account.Attributes) {
 494                                         if (attr.Name == "uid") {
 495                                                 uid = attr.InnerText;
 496                                                 break;
 497                                         }
 498                                 }
 499
 500                                 if (uid == null)
 501                                         continue;
 502
 503                                 XmlNode imap_url_node = xmlDoc.SelectSingleNode ("//source/url");
 504
 505                                 if (imap_url_node == null)
 506                                         continue;
 507
 508                                 string imap_url = imap_url_node.InnerText;
 509                                 // If there is a semicolon in the username part of the URL, it
 510                                 // indicates that there's an auth scheme there.  We don't care
 511                                 // about that, so remove it.
 512                                 int user_end = imap_url.IndexOf ('@');
 513                                 int semicolon = imap_url.IndexOf (';', 0, user_end + 1);
 514
 515                                 if (semicolon != -1)
 516                                         imap_url = imap_url.Substring (0, semicolon) + imap_url.Substring (user_end);
 517
 518                                 // Escape backslashes, which frequently appear when using IMAP against Exchange servers
 519                                 this.imap_name = this.imap_name.Replace ("\\", "%5c");
 520
 521                                 // Escape out additional @s in the name.  I hate the class libs so much.
 522                                 int lastIdx = this.imap_name.LastIndexOf ('@');
 523                                 if (this.imap_name.IndexOf ('@') != lastIdx) {
 524                                         string toEscape = this.imap_name.Substring (0, lastIdx);
 525                                         this.imap_name = toEscape.Replace ("@", "%40") + this.imap_name.Substring (lastIdx);
 526                                 }
 527
 528                                 string backend_url_prefix;
 529                                 if (this.backend_type == ImapBackendType.Imap)
 530                                         backend_url_prefix = "imap";
 531                                 else
 532                                         backend_url_prefix = "imap4";
 533
 534                                 if (imap_url.StartsWith (backend_url_prefix + "://" + this.imap_name + "/")) {
 535                                         this.account_name = uid;
 536                                         break;
 537                                 }
 538                         }
 539
 540                         if (this.account_name == null) {
 541                                 Logger.Log.Info ("Unable to determine account name for {0}", this.imap_name);
 542                                 return false;
 543                         }
 544
 545                         // Need to check the directory on disk to see if it's a junk/spam folder,
 546                         // since the folder name will be "foo/spam" and not match the check below.
 547                         DirectoryInfo dir_info = new DirectoryInfo (dir_name);
 548                         if (this.IsSpamFolder (dir_info.Name))
 549                                 return false;
 550
 551                         // Check if the folder is listed in the configuration as to be excluded from indexing
 552                         if (this.IgnoreFolder (dir_info.FullName))
 553                                 return false;
 554
 555                         this.folder_name = GetFolderName (new DirectoryInfo (dir_name));
 556
 557                         return true;
 558                 }
 559
 560                 private string FolderCacheName {
 561                         get {
 562                                 if (this.account_name == null || this.folder_name == null)
 563                                         return null;
 564
 565                                 if (this.folder_cache_name == null)
 566                                         this.folder_cache_name = "status-" + this.account_name + "-" + this.folder_name.Replace ('/', '-');
 567
 568                                 return this.folder_cache_name;
 569                         }
 570                 }
 571
 572                 private bool LoadCache ()
 573                 {
 574                         Stream cacheStream;
 575                         BinaryFormatter formatter;
 576
 577                         if (this.FolderCacheName == null) {
 578                                 this.mapping = new Hashtable ();
 579                                 return false;
 580                         }
 581
 582                         try {
 583                                 cacheStream = this.queryable.ReadDataStream (this.FolderCacheName);
 584                                 formatter = new BinaryFormatter ();
 585                                 this.mapping = formatter.Deserialize (cacheStream) as Hashtable;
 586                                 cacheStream.Close ();
 587                                 Logger.Log.Debug ("Successfully loaded previous crawled data from disk: {0}", this.FolderCacheName);
 588
 589                                 return true;
 590                         } catch {
 591                                 this.mapping = new Hashtable ();
 592
 593                                 return false;
 594                         }
 595                 }
 596
 597                 private void SaveCache ()
 598                 {
 599                         Stream cacheStream;
 600                         BinaryFormatter formatter;
 601
 602                         if (this.FolderCacheName == null)
 603                                 return;
 604
 605                         cacheStream = this.queryable.WriteDataStream (this.FolderCacheName);
 606                         formatter = new BinaryFormatter ();
 607                         formatter.Serialize (cacheStream, mapping);
 608                         cacheStream.Close ();
 609                 }
 610
 611                 public override bool HasNextIndexable ()
 612                 {
 613                         if (this.account_name == null) {
 614                                 if (!Setup ()) {
 615                                         this.queryable.RemoveGeneratorProgress (this);
 616                                         return false;
 617                                 }
 618                         }
 619
 620                         if (this.mapping == null) {
 621                                 bool cache_loaded = this.LoadCache ();
 622
 623                                 this.deleted_list = new ArrayList (this.mapping.Keys);
 624                                 this.deleted_list.Sort ();
 625                                 Logger.Log.Debug ("Deleted list starting at {0} for {1}", this.deleted_list.Count, this.folder_name);
 626
 627                                 // Check to see if we even need to bother walking the summary
 628                                 if (cache_loaded && this.queryable.FileAttributesStore.IsUpToDate (this.CrawlFile.FullName)) {
 629                                         Logger.Log.Debug ("{0}: summary has not been updated; crawl unncessary", this.folder_name);
 630                                         this.queryable.RemoveGeneratorProgress (this);
 631                                         return false;
 632                                 }
 633                         }
 634
 635                         if (this.summary == null) {
 636                                 try {
 637                                         if (this.backend_type == ImapBackendType.Imap)
 638                                                 this.summary = Camel.Summary.LoadImapSummary (this.summary_info.FullName);
 639                                         else
 640                                                 this.summary = Camel.Summary.LoadImap4Summary (this.summary_info.FullName);
 641                                 } catch (Exception e) {
 642                                         Logger.Log.Warn ("Unable to index {0}: {1}", this.folder_name,
 643                                                                          e.Message);
 644                                         this.queryable.RemoveGeneratorProgress (this);
 645                                         return false;
 646                                 }
 647                         }
 648
 649                         if (this.summary_enumerator == null)
 650                                 this.summary_enumerator = this.summary.GetEnumerator ();
 651
 652                         if (this.summary_enumerator.MoveNext ())
 653                                 return true;
 654
 655                         this.delete_mode = true;
 656
 657                         if (this.deleted_list.Count > 0)
 658                                 return true;
 659
 660                         string progress = "";
 661                         if (this.count > 0 && this.summary.header.count > 0) {
 662                                 progress = String.Format ("({0}/{1} {2:###.0}%)",
 663                                                           this.count,
 664                                                           this.summary.header.count,
 665                                                           100.0 * this.count / this.summary.header.count);
 666                         }
 667
 668                         Logger.Log.Debug ("{0}: Finished indexing {1} messages {2}, {3} messages deleted", this.folder_name, this.indexed_count, progress, this.delete_count);
 669
 670                         this.SaveCache ();
 671                         this.CrawlFinished ();
 672
 673                         return false;
 674                 }
 675
 676                 // Kind of nasty, but we need the function.
 677                 [System.Runtime.InteropServices.DllImport("libglib-2.0.so.0")]
 678                 static extern int g_str_hash (string str);
 679
 680                 // Stolen from deep within e-d-s's camel-data-cache.c  Very evil.
 681                 private const int CAMEL_DATA_CACHE_MASK = ((1 << 6) - 1);
 682
 683                 public override Indexable GetNextIndexable ()
 684                 {
 685                         Indexable indexable = null;
 686
 687                         // No more new messages to index, so start on the removals.
 688                         if (this.delete_mode) {
 689                                 string uid = (string) this.deleted_list [0];
 690                                 Uri uri = EvolutionMailQueryable.EmailUri (this.account_name, this.folder_name, uid);
 691
 692                                 indexable = new Indexable (IndexableType.Remove, uri);
 693
 694                                 this.deleted_list.RemoveAt (0);
 695                                 this.mapping.Remove (uid);
 696
 697                                 this.delete_count++;
 698
 699                                 return indexable;
 700                         }
 701
 702                         Camel.MessageInfo mi = (Camel.MessageInfo) this.summary_enumerator.Current;
 703
 704                         ++this.count;
 705
 706                         if (Debug) {
 707                                 Logger.Log.Debug ("Constructed message {0} with uid {1}, flags {2}.",
 708                                                   this.count, mi.uid, mi.flags);
 709                         }
 710
 711                         // Try to load the cached message data off disk
 712                         object flags = this.mapping[mi.uid];
 713
 714                         if (flags == null) {
 715                                 // New, previously unseen message
 716                                 string msg_file;
 717
 718                                 if (this.backend_type == ImapBackendType.Imap)
 719                                         msg_file = Path.Combine (summary_info.DirectoryName, mi.uid + ".");
 720                                 else {
 721                                         // This is taken from e-d-s's camel-data-cache.c.  No doubt
 722                                         // NotZed would scream bloody murder if he saw this here.
 723                                         int hash = (g_str_hash (mi.uid) >> 5) & CAMEL_DATA_CACHE_MASK;
 724                                         string cache_path = String.Format ("cache/{0:x}/{1}", hash, mi.uid);
 725                                         msg_file = Path.Combine (summary_info.DirectoryName, cache_path);
 726                                 }
 727
 728                                 indexable = this.CamelMessageToIndexable (mi, msg_file);
 729
 730                                 if (Debug)
 731                                         Logger.Log.Debug ("Unseen message, indexable {0} null", indexable == null ? "" : "not");
 732
 733                                 this.mapping[mi.uid] = mi.flags;
 734
 735                                 if (indexable != null)
 736                                         ++this.indexed_count;
 737                         } else if ((uint) flags != mi.flags) {
 738                                 // Previously seen message, but flags have changed.
 739                                 Uri uri = CamelMessageUri (mi);
 740                                 indexable = new Indexable (uri);
 741                                 indexable.Type = IndexableType.PropertyChange;
 742
 743                                 Property flag_prop = Property.NewUnsearched ("fixme:flags", mi.flags);
 744                                 flag_prop.IsMutable = true;
 745                                 indexable.AddProperty (flag_prop);
 746
 747                                 if (Debug)
 748                                         Logger.Log.Debug ("Previously seen message, flags changed: {0} -> {1}", flags, mi.flags);
 749
 750                                 ++this.indexed_count;
 751                         } else {
 752                                 if (Debug)
 753                                         Logger.Log.Debug ("Previously seen message, unchanged.");
 754                         }
 755
 756                         if (flags != null)
 757                                 this.deleted_list.Remove (mi.uid);
 758
 759                         return indexable;
 760                 }
 761
 762                 private Uri CamelMessageUri (Camel.MessageInfo message_info)
 763                 {
 764                         return EvolutionMailQueryable.EmailUri (this.account_name, this.folder_name, message_info.uid);
 765                 }
 766
 767                 private Indexable CamelMessageToIndexable (Camel.MessageInfo messageInfo, string msg_file)
 768                 {
 769                         // Don't index messages flagged as junk
 770                         if (messageInfo.IsJunk)
 771                                 return null;
 772
 773                         // Many properties will be set by the filter when
 774                         // processing the cached data, if it's there.  So
 775                         // don't set a number of properties in that case.
 776                         bool have_content = File.Exists (msg_file);
 777
 778                         Uri uri = CamelMessageUri (messageInfo);
 779                         Indexable indexable = new Indexable (uri);
 780
 781                         indexable.Timestamp = messageInfo.SentDate;
 782                         indexable.MimeType = "message/rfc822";
 783                         indexable.HitType = "MailMessage";
 784
 785                         indexable.AddProperty (Property.NewUnsearched ("fixme:account",  this.imap_name));
 786                         indexable.AddProperty (Property.NewUnsearched ("fixme:folder",   this.folder_name));
 787                         indexable.AddProperty (Property.NewUnsearched ("fixme:client", "evolution"));
 788
 789                         if (!have_content) {
 790                                 indexable.AddProperty (Property.New ("dc:title", GMime.Utils.HeaderDecodePhrase (messageInfo.subject)));
 791                                 indexable.AddProperty (Property.NewDate ("fixme:date", messageInfo.SentDate));
 792                         }
 793
 794                         GMime.InternetAddressList addrs;
 795                         addrs = GMime.InternetAddressList.ParseString (messageInfo.to);
 796                         foreach (GMime.InternetAddress ia in addrs) {
 797                                 if (!have_content) {
 798                                         indexable.AddProperty (Property.NewUnsearched ("fixme:to", ia.ToString (false)));
 799                                         if (ia.AddressType != GMime.InternetAddressType.Group) {
 800                                                 indexable.AddProperty (Property.New ("fixme:to_address", ia.Addr));
 801                                                 indexable.AddProperty (Property.New ("fixme:to_sanitized", StringFu.SanitizeEmail (ia.Addr)));
 802                                         }
 803                                         indexable.AddProperty (Property.New ("fixme:to_name", ia.Name));
 804                                 }
 805
 806                                 if (this.folder_name == "Sent" && ia.AddressType != GMime.InternetAddressType.Group)
 807                                         indexable.AddProperty (Property.NewUnsearched ("fixme:sentTo", ia.Addr));
 808                         }
 809                         addrs.Dispose ();
 810
 811                         addrs = GMime.InternetAddressList.ParseString (messageInfo.cc);
 812                         foreach (GMime.InternetAddress ia in addrs) {
 813                                 if (!have_content) {
 814                                         indexable.AddProperty (Property.NewUnsearched ("fixme:cc", ia.ToString (false)));
 815                                         if (ia.AddressType != GMime.InternetAddressType.Group) {
 816                                                 indexable.AddProperty (Property.New ("fixme:cc_address", ia.Addr));
 817                                                 indexable.AddProperty (Property.New ("fixme:cc_sanitized", StringFu.SanitizeEmail (ia.Addr)));
 818                                         }
 819                                         indexable.AddProperty (Property.New ("fixme:cc_name", ia.Name));
 820                                 }
 821
 822                                 if (this.folder_name == "Sent" && ia.AddressType != GMime.InternetAddressType.Group)
 823                                         indexable.AddProperty (Property.NewUnsearched ("fixme:sentTo", ia.Addr));
 824                         }
 825                         addrs.Dispose ();
 826
 827                         addrs = GMime.InternetAddressList.ParseString (messageInfo.from);
 828                         foreach (GMime.InternetAddress ia in addrs) {
 829                                 if (!have_content) {
 830                                         indexable.AddProperty (Property.NewUnsearched ("fixme:from", ia.ToString (false)));
 831                                         if (ia.AddressType != GMime.InternetAddressType.Group) {
 832                                                 indexable.AddProperty (Property.New ("fixme:from_address", ia.Addr));
 833                                                 indexable.AddProperty (Property.New ("fixme:from_sanitized", StringFu.SanitizeEmail (ia.Addr)));
 834                                         }
 835                                         indexable.AddProperty (Property.New ("fixme:from_name", ia.Name));
 836                                 }
 837
 838                                 if (this.folder_name != "Sent" && ia.AddressType != GMime.InternetAddressType.Group)
 839                                         indexable.AddProperty (Property.NewUnsearched ("fixme:gotFrom", ia.Addr));
 840                         }
 841                         addrs.Dispose ();
 842
 843                         indexable.AddProperty (Property.NewKeyword ("fixme:mlist", messageInfo.mlist));
 844
 845                         Property flag_prop = Property.NewUnsearched ("fixme:flags", messageInfo.flags);
 846                         flag_prop.IsMutable = true;
 847                         indexable.AddProperty (flag_prop);
 848
 849                         if (this.folder_name == "Sent")
 850                                 indexable.AddProperty (Property.NewFlag ("fixme:isSent"));
 851
 852                         if (messageInfo.IsAnswered)
 853                                 indexable.AddProperty (Property.NewFlag ("fixme:isAnswered"));
 854
 855                         if (messageInfo.IsDeleted)
 856                                 indexable.AddProperty (Property.NewFlag ("fixme:isDeleted"));
 857
 858                         if (messageInfo.IsDraft)
 859                                 indexable.AddProperty (Property.NewFlag ("fixme:isDraft"));
 860
 861                         if (messageInfo.IsFlagged)
 862                                 indexable.AddProperty (Property.NewFlag ("fixme:isFlagged"));
 863
 864                         if (messageInfo.IsSeen)
 865                                 indexable.AddProperty (Property.NewFlag ("fixme:isSeen"));
 866
 867                         if (messageInfo.HasAttachments && !have_content)
 868                                 indexable.AddProperty (Property.NewFlag ("fixme:hasAttachments"));
 869
 870                         if (messageInfo.IsAnsweredAll)
 871                                 indexable.AddProperty (Property.NewFlag ("fixme:isAnsweredAll"));
 872
 873                         if (have_content)
 874                                 indexable.ContentUri = UriFu.PathToFileUri (msg_file);
 875                         else
 876                                 indexable.NoContent = true;
 877
 878                         return indexable;
 879                 }
 880
 881                 public override void Checkpoint ()
 882                 {
 883                         if (this.summary != null) {
 884
 885                                 string progress = "";
 886                                 if (this.count > 0 && this.summary.header.count > 0) {
 887                                         progress = String.Format (" ({0}/{1} {2:###.0}%)",
 888                                                                   this.count,
 889                                                                   this.summary.header.count,
 890                                                                   100.0 * this.count / this.summary.header.count);
 891
 892                                         this.queryable.SetGeneratorProgress (this, (int) (100.0 * this.count / this.summary.header.count));
 893
 894                                 }
 895
 896                                 Logger.Log.Debug ("{0}: indexed {1} messages{2}",
 897                                                   this.folder_name, this.indexed_count, progress);
 898                         }
 899
 900                         this.SaveCache ();
 901                 }
 902
 903                 public override string GetTarget ()
 904                 {
 905                         return "summary-file:" + summary_info.FullName;
 906                 }
 907
 908                 protected override FileInfo CrawlFile {
 909                         get { return this.summary_info; }
 910                 }
 911         }
 912 }