beagled/EvolutionMailDriver/EvolutionMailIndexableGenerator.cs

   1
   2 //
   3 // EvolutionMailIndexableGenerator.cs
   4 //
   5 // Copyright (C) 2004 Novell, Inc.
   6 //
   7 //
   8 //
   9 // Permission is hereby granted, free of charge, to any person obtaining a
  10 // copy of this software and associated documentation files (the "Software"),
  11 // to deal in the Software without restriction, including without limitation
  12 // the rights to use, copy, modify, merge, publish, distribute, sublicense,
  13 // and/or sell copies of the Software, and to permit persons to whom the
  14 // Software is furnished to do so, subject to the following conditions:
  15 //
  16 // The above copyright notice and this permission notice shall be included in
  17 // all copies or substantial portions of the Software.
  18 //
  19 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  20 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  21 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  22 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  23 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  24 // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  25 // DEALINGS IN THE SOFTWARE.
  26 //
  27
  28 using System;
  29 using System.Collections;
  30 using System.IO;
  31 using System.Runtime.Serialization.Formatters.Binary;
  32 using System.Threading;
  33 using System.Xml;
  34
  35 using Beagle.Util;
  36 using Beagle.Daemon;
  37
  38 using Camel = Beagle.Util.Camel;
  39
  40 namespace Beagle.Daemon.EvolutionMailDriver {
  41
  42         public abstract class EvolutionMailIndexableGenerator : IIndexableGenerator {
  43
  44                 protected static bool Debug = false;
  45
  46                 private static bool gmime_initialized = false;
  47
  48                 private static ArrayList excludes = new ArrayList ();
  49
  50                 static EvolutionMailIndexableGenerator () {
  51                         foreach (ExcludeItem exclude in Conf.Indexing.Excludes)
  52                                 if (exclude.Type == ExcludeType.MailFolder)
  53                                         excludes.Add (exclude);
  54                 }
  55
  56                 protected EvolutionMailQueryable queryable;
  57
  58                 protected string account_name, folder_name;
  59                 protected int count, indexed_count;
  60
  61                 protected EvolutionMailIndexableGenerator (EvolutionMailQueryable queryable)
  62                 {
  63                         this.queryable = queryable;
  64                 }
  65
  66                 protected abstract string GetFolderName (FileSystemInfo info);
  67                 protected abstract bool Setup ();
  68                 protected abstract FileInfo CrawlFile { get; }
  69                 public abstract string GetTarget ();
  70                 public abstract bool HasNextIndexable ();
  71                 public abstract Indexable GetNextIndexable ();
  72                 public abstract void Checkpoint ();
  73
  74
  75                 protected double progress_percent;
  76                 public double ProgressPercent {
  77                         get { return progress_percent; }
  78                 }
  79
  80                 public void PostFlushHook ()
  81                 {
  82                         Checkpoint ();
  83                 }
  84
  85                 protected static void InitializeGMime ()
  86                 {
  87                         if (!gmime_initialized) {
  88                                 GMime.Global.Init ();
  89                                 gmime_initialized = true;
  90                         }
  91                 }
  92
  93                 protected bool IsSpamFolder (string name)
  94                 {
  95                         if (name.ToLower () == "spam" || name.ToLower () == "junk")
  96                                 return true;
  97                         else
  98                                 return false;
  99                 }
 100
 101                 protected bool IgnoreFolder (string path)
 102                 {
 103                         // FIXME: Use System.IO.Path
 104                         foreach (ExcludeItem exclude in excludes) {
 105                                 if (exclude.IsMatch (path))
 106                                         return true;
 107                         }
 108                         return false;
 109                 }
 110
 111                 protected virtual void CrawlFinished ()
 112                 {
 113                         this.queryable.RemoveGenerator (this);
 114                 }
 115
 116                 public string StatusName {
 117                         get { return this.CrawlFile.FullName; }
 118                 }
 119
 120                 public override bool Equals (object o)
 121                 {
 122                         EvolutionMailIndexableGenerator generator = o as EvolutionMailIndexableGenerator;
 123
 124                         if (generator == null)
 125                                 return false;
 126
 127                         if (Object.ReferenceEquals (this, generator))
 128                                 return true;
 129
 130                         if (this.CrawlFile.FullName == generator.CrawlFile.FullName)
 131                                 return true;
 132                         else
 133                                 return false;
 134                 }
 135
 136                 public override int GetHashCode ()
 137                 {
 138                         return this.CrawlFile.FullName.GetHashCode ();
 139                 }
 140         }
 141
 142         public class EvolutionMailIndexableGeneratorMbox : EvolutionMailIndexableGenerator {
 143                 private FileInfo mbox_info;
 144                 private int mbox_fd = -1;
 145                 private GMime.StreamFs mbox_stream;
 146                 private GMime.Parser mbox_parser;
 147                 private long file_size;
 148
 149                 public EvolutionMailIndexableGeneratorMbox (EvolutionMailQueryable queryable, FileInfo mbox_info) : base (queryable)
 150                 {
 151                         this.mbox_info = mbox_info;
 152                 }
 153
 154                 protected override string GetFolderName (FileSystemInfo info)
 155                 {
 156                         FileInfo file_info = (FileInfo) info;
 157                         DirectoryInfo di;
 158                         string folder_name = "";
 159
 160                         di = file_info.Directory;
 161                         while (di != null) {
 162                                 // Evo uses ".sbd" as the extension on a folder
 163                                 if (di.Extension == ".sbd")
 164                                         folder_name = Path.Combine (Path.GetFileNameWithoutExtension (di.Name), folder_name);
 165                                 else
 166                                         break;
 167
 168                                 di = di.Parent;
 169                         }
 170
 171                         return Path.Combine (folder_name, file_info.Name);
 172                 }
 173
 174                 protected override bool Setup ()
 175                 {
 176                         this.account_name = "local@local";
 177                         this.folder_name = this.GetFolderName (this.mbox_info);
 178
 179                         if (this.IsSpamFolder (this.folder_name))
 180                                 return false;
 181
 182                         if (this.IgnoreFolder (this.mbox_info.FullName))
 183                                 return false;
 184
 185                         return true;
 186                 }
 187
 188                 private long MboxLastOffset {
 189                         get {
 190                                 string offset_str = this.queryable.ReadDataLine ("offset-" + this.folder_name.Replace ('/', '-'));
 191                                 long offset = Convert.ToInt64 (offset_str);
 192
 193                                 return offset;
 194                         }
 195
 196                         set {
 197                                 this.queryable.WriteDataLine ("offset-" + this.folder_name.Replace ('/', '-'), value.ToString ());
 198                         }
 199                 }
 200
 201                 public override bool HasNextIndexable ()
 202                 {
 203                         if (this.account_name == null) {
 204                                 if (!Setup ()) {
 205                                         this.queryable.RemoveGenerator (this);
 206                                         return false;
 207                                 }
 208                         }
 209
 210                         if (this.mbox_fd < 0) {
 211                                 Logger.Log.Debug ("Opening mbox {0}", this.mbox_info.Name);
 212
 213                                 try {
 214                                         InitializeGMime ();
 215                                 } catch (Exception e) {
 216                                         Logger.Log.Warn (e, "Caught exception trying to initalize gmime:");
 217                                         return false;
 218                                 }
 219
 220                                 this.mbox_fd = Mono.Unix.Native.Syscall.open (this.mbox_info.FullName, Mono.Unix.Native.OpenFlags.O_RDONLY);
 221
 222                                 if (this.mbox_fd < 0) {
 223                                         Log.Error ("Unable to open {0}: {1}", this.mbox_info.FullName, Mono.Unix.Native.Stdlib.strerror (Mono.Unix.Native.Stdlib.GetLastError ()));
 224                                         return false;
 225                                 }
 226
 227                                 this.mbox_stream = new GMime.StreamFs (this.mbox_fd);
 228                                 this.mbox_stream.Seek ((int) this.MboxLastOffset);
 229                                 this.mbox_parser = new GMime.Parser (this.mbox_stream);
 230                                 this.mbox_parser.ScanFrom = true;
 231
 232                                 FileInfo info = new FileInfo (this.mbox_info.FullName);
 233                                 this.file_size = info.Length;
 234                         }
 235
 236                         if (this.mbox_parser.Eos ()) {
 237                                 long offset = this.mbox_parser.FromOffset;
 238
 239                                 this.mbox_stream.Close ();
 240
 241                                 this.mbox_fd = -1;
 242                                 this.mbox_stream.Dispose ();
 243                                 this.mbox_stream = null;
 244                                 this.mbox_parser.Dispose ();
 245                                 this.mbox_parser = null;
 246
 247                                 Logger.Log.Debug ("{0}: Finished indexing {1} messages", this.folder_name, this.indexed_count);
 248
 249                                 if (offset >= 0)
 250                                         this.MboxLastOffset = offset;
 251                                 this.CrawlFinished ();
 252
 253                                 return false;
 254                         } else
 255                                 return true;
 256                 }
 257
 258                 public override Indexable GetNextIndexable ()
 259                 {
 260                         using (GMime.Message message = this.mbox_parser.ConstructMessage ()) {
 261                                 // Work around what I think is a bug in GMime: If you
 262                                 // have a zero-byte file or seek to the end of a
 263                                 // file, parser.Eos () will return true until it
 264                                 // actually tries to read something off the wire.
 265                                 // Since parser.ConstructMessage() always returns a
 266                                 // message (which may also be a bug), we'll often get
 267                                 // one empty message which we need to deal with here.
 268                                 //
 269                                 // Check if its empty by seeing if the Headers
 270                                 // property is null or empty.
 271                                 if (message == null || message.Headers == null || message.Headers == "")
 272                                         return null;
 273
 274                                 ++this.count;
 275
 276                                 string x_evolution = message.GetHeader ("X-Evolution");
 277                                 if (x_evolution == null || x_evolution == "") {
 278                                         Logger.Log.Info ("{0}: Message at offset {1} has no X-Evolution header!",
 279                                                          this.folder_name, this.mbox_parser.FromOffset);
 280                                         return null;
 281                                 }
 282
 283                                 // This extracts the UID and flags from the X-Evolution header.
 284                                 // It may also contain user-defined flags and tags, but we don't
 285                                 // support those right now.
 286                                 int separator_idx = x_evolution.IndexOf ('-');
 287
 288                                 string uid_str = x_evolution.Substring (0, separator_idx);
 289                                 string uid = Convert.ToUInt32 (uid_str, 16).ToString (); // ugh.
 290                                 uint flags = Convert.ToUInt32 (x_evolution.Substring (separator_idx + 1, 4), 16);
 291
 292                                 Indexable indexable = this.GMimeMessageToIndexable (uid, message, flags);
 293
 294                                 if (Debug) {
 295                                         Logger.Log.Debug ("Constructed message {0} with uid {1}, flags {2}.  Indexable {3} null",
 296                                                           this.count, uid, flags, indexable == null ? "" : "not");
 297                                 }
 298
 299                                 if (indexable == null)
 300                                         return null;
 301
 302                                 ++this.indexed_count;
 303
 304                                 return indexable;
 305                         }
 306                 }
 307
 308                 private static bool CheckFlags (uint flags, Camel.CamelFlags test)
 309                 {
 310                         return (flags & (uint) test) == (uint) test;
 311                 }
 312
 313                 private Indexable GMimeMessageToIndexable (string uid, GMime.Message message, uint flags)
 314                 {
 315                         // Don't index messages flagged as junk
 316                         if (CheckFlags (flags, Camel.CamelFlags.Junk))
 317                                 return null;
 318
 319                         System.Uri uri = EvolutionMailQueryable.EmailUri (this.account_name, this.folder_name, uid);
 320                         Indexable indexable = new Indexable (uri);
 321
 322                         indexable.Timestamp = message.Date.ToUniversalTime ();
 323                         indexable.HitType = "MailMessage";
 324                         indexable.MimeType = "message/rfc822";
 325                         indexable.CacheContent = false;
 326
 327                         indexable.AddProperty (Property.NewUnsearched ("fixme:client", "evolution"));
 328                         indexable.AddProperty (Property.NewUnsearched ("fixme:account", "Local"));
 329                         indexable.AddProperty (Property.NewUnsearched ("fixme:folder", this.folder_name));
 330
 331                         GMime.InternetAddressList addrs;
 332
 333                         addrs = message.GetRecipients (GMime.Message.RecipientType.To);
 334                         foreach (GMime.InternetAddress ia in addrs) {
 335                                 if (this.folder_name == "Sent" && ia.AddressType != GMime.InternetAddressType.Group)
 336                                         indexable.AddProperty (Property.NewUnsearched ("fixme:sentTo", ia.Addr));
 337                         }
 338                         addrs.Dispose ();
 339
 340                         addrs = message.GetRecipients (GMime.Message.RecipientType.Cc);
 341                         foreach (GMime.InternetAddress ia in addrs) {
 342                                 if (this.folder_name == "Sent"  && ia.AddressType != GMime.InternetAddressType.Group)
 343                                         indexable.AddProperty (Property.NewUnsearched ("fixme:sentTo", ia.Addr));
 344                         }
 345                         addrs.Dispose ();
 346
 347                         addrs = GMime.InternetAddressList.ParseString (GMime.Utils.HeaderDecodePhrase (message.Sender));
 348                         foreach (GMime.InternetAddress ia in addrs) {
 349                                 if (this.folder_name != "Sent"  && ia.AddressType != GMime.InternetAddressType.Group)
 350                                         indexable.AddProperty (Property.NewUnsearched ("fixme:gotFrom", ia.Addr));
 351                         }
 352                         addrs.Dispose ();
 353
 354                         if (this.folder_name == "Sent")
 355                                 indexable.AddProperty (Property.NewFlag ("fixme:isSent"));
 356
 357                         Property flag_prop = Property.NewUnsearched ("fixme:flags", flags);
 358                         flag_prop.IsMutable = true;
 359                         indexable.AddProperty (flag_prop);
 360
 361                         if (CheckFlags (flags, Camel.CamelFlags.Answered))
 362                                 indexable.AddProperty (Property.NewFlag ("fixme:isAnswered"));
 363
 364                         if (CheckFlags (flags, Camel.CamelFlags.Deleted))
 365                                 indexable.AddProperty (Property.NewFlag ("fixme:isDeleted"));
 366
 367                         if (CheckFlags (flags, Camel.CamelFlags.Draft))
 368                                 indexable.AddProperty (Property.NewFlag ("fixme:isDraft"));
 369
 370                         if (CheckFlags (flags, Camel.CamelFlags.Flagged))
 371                                 indexable.AddProperty (Property.NewFlag ("fixme:isFlagged"));
 372
 373                         if (CheckFlags (flags, Camel.CamelFlags.Seen))
 374                                 indexable.AddProperty (Property.NewFlag ("fixme:isSeen"));
 375
 376                         if (CheckFlags (flags, Camel.CamelFlags.AnsweredAll))
 377                                 indexable.AddProperty (Property.NewFlag ("fixme:isAnsweredAll"));
 378
 379                         indexable.SetBinaryStream (message.Stream);
 380
 381                         return indexable;
 382                 }
 383
 384                 public override void Checkpoint ()
 385                 {
 386                         long offset = -1;
 387
 388                         if (this.mbox_parser != null) {
 389                                 offset = this.mbox_parser.FromOffset;
 390
 391                                 if (offset >= 0)
 392                                         this.MboxLastOffset = offset;
 393                         }
 394
 395                         string progress = "";
 396                         if (this.file_size > 0 && offset > 0) {
 397                                 progress = String.Format (" ({0}/{1} bytes {2:###.0}%)",
 398                                                           offset, this.file_size, 100.0 * offset / this.file_size);
 399
 400                                 this.progress_percent = 100.0 * offset / this.file_size;
 401                         }
 402
 403                         Logger.Log.Debug ("{0}: indexed {1} messages{2}",
 404                                           this.folder_name, this.indexed_count, progress);
 405                 }
 406
 407                 public override string GetTarget ()
 408                 {
 409                         return "mbox-file:" + mbox_info.FullName;
 410                 }
 411
 412                 protected override FileInfo CrawlFile {
 413                         get { return this.mbox_info; }
 414                 }
 415         }
 416
 417         public class EvolutionMailIndexableGeneratorImap : EvolutionMailIndexableGenerator {
 418                 private enum ImapBackendType {
 419                         Imap,
 420                         Imap4
 421                 };
 422
 423                 private FileInfo summary_info;
 424                 private string imap_name;
 425                 private ImapBackendType backend_type;
 426                 private Camel.Summary summary;
 427                 private IEnumerator summary_enumerator;
 428                 private EvolutionSummaryTracker tracker;
 429                 private DateTime start_crawl_time;
 430                 private ArrayList deleted_list;
 431                 private bool delete_mode;
 432                 private int delete_count;
 433
 434                 public EvolutionMailIndexableGeneratorImap (EvolutionMailQueryable queryable, FileInfo summary_info) : base (queryable)
 435                 {
 436                         this.summary_info = summary_info;
 437                 }
 438
 439                 protected override string GetFolderName (FileSystemInfo info)
 440                 {
 441                         DirectoryInfo dir_info = (DirectoryInfo) info;
 442                         string folder_name = "";
 443
 444                         while (dir_info != null) {
 445                                 folder_name = Path.Combine (dir_info.Name, folder_name);
 446
 447                                 dir_info = dir_info.Parent;
 448
 449                                 if (dir_info.Name != "subfolders")
 450                                         break;
 451                                 else
 452                                         dir_info = dir_info.Parent;
 453                         }
 454
 455                         return folder_name;
 456                 }
 457
 458                 protected override bool Setup ()
 459                 {
 460                         string dir_name = summary_info.DirectoryName;
 461                         int imap_start_idx;
 462
 463                         int idx = dir_name.IndexOf (".evolution/mail/imap4/");
 464
 465                         if (idx >= 0) {
 466                                 this.backend_type = ImapBackendType.Imap4;
 467                                 imap_start_idx = idx + 22;
 468                         } else {
 469                                 this.backend_type = ImapBackendType.Imap;
 470                                 imap_start_idx = dir_name.IndexOf (".evolution/mail/imap/") + 21;
 471                         }
 472
 473                         string imap_start = dir_name.Substring (imap_start_idx);
 474                         this.imap_name = imap_start.Substring (0, imap_start.IndexOf ('/'));
 475
 476                         ICollection accounts = null;
 477
 478                         try {
 479                                 accounts = (ICollection) GConfThreadHelper.Get ("/apps/evolution/mail/accounts");
 480                         } catch (Exception ex) {
 481                                 Logger.Log.Warn ("Caught exception in Setup(): " + ex.Message);
 482                                 Logger.Log.Warn ("There are no configured evolution accounts, ignoring {0}", this.imap_name);
 483                                 return false;
 484                         }
 485
 486                         // This should only happen if we shut down while waiting for the GConf results to come back.
 487                         if (accounts == null)
 488                                 return false;
 489
 490                         foreach (string xml in accounts) {
 491                                 XmlDocument xmlDoc = new XmlDocument ();
 492
 493                                 xmlDoc.LoadXml (xml);
 494
 495                                 XmlNode account = xmlDoc.SelectSingleNode ("//account");
 496
 497                                 if (account == null)
 498                                         continue;
 499
 500                                 string uid = null;
 501
 502                                 foreach (XmlAttribute attr in account.Attributes) {
 503                                         if (attr.Name == "uid") {
 504                                                 uid = attr.InnerText;
 505                                                 break;
 506                                         }
 507                                 }
 508
 509                                 if (uid == null)
 510                                         continue;
 511
 512                                 XmlNode imap_url_node = xmlDoc.SelectSingleNode ("//source/url");
 513
 514                                 if (imap_url_node == null)
 515                                         continue;
 516
 517                                 string imap_url = imap_url_node.InnerText;
 518                                 // If there is a semicolon in the username part of the URL, it
 519                                 // indicates that there's an auth scheme there.  We don't care
 520                                 // about that, so remove it.
 521                                 int user_end = imap_url.IndexOf ('@');
 522                                 int semicolon = imap_url.IndexOf (';', 0, user_end + 1);
 523
 524                                 if (semicolon != -1)
 525                                         imap_url = imap_url.Substring (0, semicolon) + imap_url.Substring (user_end);
 526
 527                                 // Escape backslashes, which frequently appear when using IMAP against Exchange servers
 528                                 this.imap_name = this.imap_name.Replace ("\\", "%5c");
 529
 530                                 // Escape out additional @s in the name.  I hate the class libs so much.
 531                                 int lastIdx = this.imap_name.LastIndexOf ('@');
 532                                 if (this.imap_name.IndexOf ('@') != lastIdx) {
 533                                         string toEscape = this.imap_name.Substring (0, lastIdx);
 534                                         this.imap_name = toEscape.Replace ("@", "%40") + this.imap_name.Substring (lastIdx);
 535                                 }
 536
 537                                 string backend_url_prefix;
 538                                 if (this.backend_type == ImapBackendType.Imap)
 539                                         backend_url_prefix = "imap";
 540                                 else
 541                                         backend_url_prefix = "imap4";
 542
 543                                 if (imap_url.StartsWith (backend_url_prefix + "://" + this.imap_name + "/")) {
 544                                         this.account_name = uid;
 545                                         break;
 546                                 }
 547                         }
 548
 549                         if (this.account_name == null) {
 550                                 Logger.Log.Info ("Unable to determine account name for {0}", this.imap_name);
 551                                 return false;
 552                         }
 553
 554                         // Need to check the directory on disk to see if it's a junk/spam folder,
 555                         // since the folder name will be "foo/spam" and not match the check below.
 556                         DirectoryInfo dir_info = new DirectoryInfo (dir_name);
 557                         if (this.IsSpamFolder (dir_info.Name))
 558                                 return false;
 559
 560                         // Check if the folder is listed in the configuration as to be excluded from indexing
 561                         if (this.IgnoreFolder (dir_info.FullName))
 562                                 return false;
 563
 564                         this.folder_name = GetFolderName (new DirectoryInfo (dir_name));
 565
 566                         return true;
 567                 }
 568
 569                 public override bool HasNextIndexable ()
 570                 {
 571                         if (this.account_name == null) {
 572                                 if (!Setup ()) {
 573                                         this.queryable.RemoveGenerator (this);
 574                                         return false;
 575                                 }
 576                         }
 577
 578                         if (this.tracker == null) {
 579                                 if (this.queryable.FileAttributesStore.IsUpToDate (this.CrawlFile.FullName)) {
 580                                         Logger.Log.Debug ("{0}: summary has not been updated; crawl unncessary", this.folder_name);
 581                                         this.queryable.RemoveGenerator (this);
 582                                         return false;
 583                                 }
 584
 585                                 this.tracker = new EvolutionSummaryTracker (this.queryable.IndexDirectory, this.account_name, this.folder_name);
 586                                 this.start_crawl_time = DateTime.UtcNow;
 587                         }
 588
 589                         if (this.summary == null) {
 590                                 try {
 591                                         if (this.backend_type == ImapBackendType.Imap)
 592                                                 this.summary = Camel.Summary.LoadImapSummary (this.summary_info.FullName);
 593                                         else
 594                                                 this.summary = Camel.Summary.LoadImap4Summary (this.summary_info.FullName);
 595                                 } catch (Exception e) {
 596                                         Logger.Log.Warn (e, "Unable to index {0}:", this.folder_name);
 597                                         this.queryable.RemoveGenerator (this);
 598                                         return false;
 599                                 }
 600                         }
 601
 602                         if (this.summary_enumerator == null)
 603                                 this.summary_enumerator = this.summary.GetEnumerator ();
 604
 605                         if (this.summary_enumerator.MoveNext ())
 606                                 return true;
 607
 608                         this.delete_mode = true;
 609                         this.deleted_list = this.tracker.GetOlderThan (this.start_crawl_time);
 610
 611                         if (this.deleted_list.Count > 0)
 612                                 return true;
 613
 614                         this.deleted_list = null;
 615
 616                         string progress = "";
 617                         if (this.count > 0 && this.summary.header.count > 0) {
 618                                 progress = String.Format ("({0}/{1} {2:###.0}%)",
 619                                                           this.count,
 620                                                           this.summary.header.count,
 621                                                           100.0 * this.count / this.summary.header.count);
 622                         }
 623
 624                         Logger.Log.Debug ("{0}: Finished indexing {1} messages {2}, {3} messages deleted", this.folder_name, this.indexed_count, progress, this.delete_count);
 625
 626                         this.tracker.Close ();
 627                         this.tracker = null;
 628                         this.CrawlFinished ();
 629
 630                         return false;
 631                 }
 632
 633                 // Kind of nasty, but we need the function.
 634                 [System.Runtime.InteropServices.DllImport("libglib-2.0.so.0")]
 635                 static extern int g_str_hash (string str);
 636
 637                 // Stolen from deep within e-d-s's camel-data-cache.c  Very evil.
 638                 private const int CAMEL_DATA_CACHE_MASK = ((1 << 6) - 1);
 639
 640                 public override Indexable GetNextIndexable ()
 641                 {
 642                         Indexable indexable = null;
 643
 644                         // No more new messages to index, so start on the removals.
 645                         if (this.delete_mode) {
 646                                 string uid = (string) this.deleted_list [0];
 647                                 Uri uri = EvolutionMailQueryable.EmailUri (this.account_name, this.folder_name, uid);
 648
 649                                 indexable = new Indexable (IndexableType.Remove, uri);
 650
 651                                 this.deleted_list.RemoveAt (0);
 652                                 this.tracker.Remove (uid);
 653
 654                                 this.delete_count++;
 655
 656                                 return indexable;
 657                         }
 658
 659                         Camel.MessageInfo mi = (Camel.MessageInfo) this.summary_enumerator.Current;
 660
 661                         ++this.count;
 662
 663                         if (Debug) {
 664                                 Logger.Log.Debug ("Constructed message {0} with uid {1}, flags {2}.",
 665                                                   this.count, mi.uid, mi.flags);
 666                         }
 667
 668                         uint flags;
 669                         bool found = this.tracker.Get (mi.uid, out flags);
 670
 671                         if (! found) {
 672                                 // New, previously unseen message
 673                                 string msg_file;
 674
 675                                 if (this.backend_type == ImapBackendType.Imap)
 676                                         msg_file = Path.Combine (summary_info.DirectoryName, mi.uid + ".");
 677                                 else {
 678                                         // This is taken from e-d-s's camel-data-cache.c.  No doubt
 679                                         // NotZed would scream bloody murder if he saw this here.
 680                                         int hash = (g_str_hash (mi.uid) >> 5) & CAMEL_DATA_CACHE_MASK;
 681                                         string cache_path = String.Format ("cache/{0:x}/{1}", hash, mi.uid);
 682                                         msg_file = Path.Combine (summary_info.DirectoryName, cache_path);
 683                                 }
 684
 685                                 indexable = this.CamelMessageToIndexable (mi, msg_file);
 686
 687                                 if (Debug)
 688                                         Logger.Log.Debug ("Unseen message, indexable {0} null", indexable == null ? "" : "not");
 689
 690                                 if (indexable != null)
 691                                         ++this.indexed_count;
 692                         } else if (found && flags != mi.flags) {
 693                                 // Previously seen message, but flags have changed.
 694                                 Uri uri = CamelMessageUri (mi);
 695                                 indexable = new Indexable (uri);
 696                                 indexable.Type = IndexableType.PropertyChange;
 697
 698                                 Property flag_prop = Property.NewUnsearched ("fixme:flags", mi.flags);
 699                                 flag_prop.IsMutable = true;
 700                                 indexable.AddProperty (flag_prop);
 701
 702                                 if (Debug)
 703                                         Logger.Log.Debug ("Previously seen message, flags changed: {0} -> {1}", flags, mi.flags);
 704
 705                                 ++this.indexed_count;
 706                         } else {
 707                                 if (Debug)
 708                                         Logger.Log.Debug ("Previously seen message, unchanged.");
 709                         }
 710
 711                         this.tracker.Update (mi.uid, mi.flags);
 712
 713                         return indexable;
 714                 }
 715
 716                 private Uri CamelMessageUri (Camel.MessageInfo message_info)
 717                 {
 718                         return EvolutionMailQueryable.EmailUri (this.account_name, this.folder_name, message_info.uid);
 719                 }
 720
 721                 private Indexable CamelMessageToIndexable (Camel.MessageInfo messageInfo, string msg_file)
 722                 {
 723                         // Don't index messages flagged as junk
 724                         if (messageInfo.IsJunk)
 725                                 return null;
 726
 727                         // Many properties will be set by the filter when
 728                         // processing the cached data, if it's there.  So
 729                         // don't set a number of properties in that case.
 730                         bool have_content = File.Exists (msg_file);
 731
 732                         Uri uri = CamelMessageUri (messageInfo);
 733                         Indexable indexable = new Indexable (uri);
 734
 735                         indexable.Timestamp = messageInfo.SentDate;
 736                         indexable.MimeType = "message/rfc822";
 737                         indexable.HitType = "MailMessage";
 738
 739                         indexable.AddProperty (Property.NewUnsearched ("fixme:account",  this.imap_name));
 740                         indexable.AddProperty (Property.NewUnsearched ("fixme:folder",   this.folder_name));
 741                         indexable.AddProperty (Property.NewUnsearched ("fixme:client", "evolution"));
 742
 743                         if (!have_content) {
 744                                 indexable.AddProperty (Property.New ("dc:title", GMime.Utils.HeaderDecodePhrase (messageInfo.subject)));
 745                                 indexable.AddProperty (Property.NewDate ("fixme:date", messageInfo.SentDate));
 746                         }
 747
 748                         GMime.InternetAddressList addrs;
 749                         addrs = GMime.InternetAddressList.ParseString (messageInfo.to);
 750                         foreach (GMime.InternetAddress ia in addrs) {
 751                                 if (!have_content) {
 752                                         indexable.AddProperty (Property.NewUnsearched ("fixme:to", ia.ToString (false)));
 753                                         if (ia.AddressType != GMime.InternetAddressType.Group)
 754                                                 indexable.AddProperty (Property.New ("fixme:to_address", ia.Addr));
 755
 756                                         indexable.AddProperty (Property.New ("fixme:to_name", ia.Name));
 757                                 }
 758
 759                                 if (this.folder_name == "Sent" && ia.AddressType != GMime.InternetAddressType.Group)
 760                                         indexable.AddProperty (Property.NewUnsearched ("fixme:sentTo", ia.Addr));
 761                         }
 762                         addrs.Dispose ();
 763
 764                         addrs = GMime.InternetAddressList.ParseString (messageInfo.cc);
 765                         foreach (GMime.InternetAddress ia in addrs) {
 766                                 if (!have_content) {
 767                                         indexable.AddProperty (Property.NewUnsearched ("fixme:cc", ia.ToString (false)));
 768                                         if (ia.AddressType != GMime.InternetAddressType.Group)
 769                                                 indexable.AddProperty (Property.New ("fixme:cc_address", ia.Addr));
 770
 771                                         indexable.AddProperty (Property.New ("fixme:cc_name", ia.Name));
 772                                 }
 773
 774                                 if (this.folder_name == "Sent" && ia.AddressType != GMime.InternetAddressType.Group)
 775                                         indexable.AddProperty (Property.NewUnsearched ("fixme:sentTo", ia.Addr));
 776                         }
 777                         addrs.Dispose ();
 778
 779                         addrs = GMime.InternetAddressList.ParseString (messageInfo.from);
 780                         foreach (GMime.InternetAddress ia in addrs) {
 781                                 if (!have_content) {
 782                                         indexable.AddProperty (Property.NewUnsearched ("fixme:from", ia.ToString (false)));
 783                                         if (ia.AddressType != GMime.InternetAddressType.Group)
 784                                                 indexable.AddProperty (Property.New ("fixme:from_address", ia.Addr));
 785
 786                                         indexable.AddProperty (Property.New ("fixme:from_name", ia.Name));
 787                                 }
 788
 789                                 if (this.folder_name != "Sent" && ia.AddressType != GMime.InternetAddressType.Group)
 790                                         indexable.AddProperty (Property.NewUnsearched ("fixme:gotFrom", ia.Addr));
 791                         }
 792                         addrs.Dispose ();
 793
 794                         indexable.AddProperty (Property.NewKeyword ("fixme:mlist", messageInfo.mlist));
 795
 796                         Property flag_prop = Property.NewUnsearched ("fixme:flags", messageInfo.flags);
 797                         flag_prop.IsMutable = true;
 798                         indexable.AddProperty (flag_prop);
 799
 800                         if (this.folder_name == "Sent")
 801                                 indexable.AddProperty (Property.NewFlag ("fixme:isSent"));
 802
 803                         if (messageInfo.IsAnswered)
 804                                 indexable.AddProperty (Property.NewFlag ("fixme:isAnswered"));
 805
 806                         if (messageInfo.IsDeleted)
 807                                 indexable.AddProperty (Property.NewFlag ("fixme:isDeleted"));
 808
 809                         if (messageInfo.IsDraft)
 810                                 indexable.AddProperty (Property.NewFlag ("fixme:isDraft"));
 811
 812                         if (messageInfo.IsFlagged)
 813                                 indexable.AddProperty (Property.NewFlag ("fixme:isFlagged"));
 814
 815                         if (messageInfo.IsSeen)
 816                                 indexable.AddProperty (Property.NewFlag ("fixme:isSeen"));
 817
 818                         if (messageInfo.HasAttachments && !have_content)
 819                                 indexable.AddProperty (Property.NewFlag ("fixme:hasAttachments"));
 820
 821                         if (messageInfo.IsAnsweredAll)
 822                                 indexable.AddProperty (Property.NewFlag ("fixme:isAnsweredAll"));
 823
 824                         if (have_content)
 825                                 indexable.ContentUri = UriFu.PathToFileUri (msg_file);
 826                         else
 827                                 indexable.NoContent = true;
 828
 829                         return indexable;
 830                 }
 831
 832                 public override void Checkpoint ()
 833                 {
 834                         if (this.summary != null) {
 835
 836                                 string progress = "";
 837                                 if (this.count > 0 && this.summary.header.count > 0) {
 838                                         progress = String.Format (" ({0}/{1} {2:###.0}%)",
 839                                                                   this.count,
 840                                                                   this.summary.header.count,
 841                                                                   100.0 * this.count / this.summary.header.count);
 842
 843                                         this.progress_percent = 100.0 * this.count / this.summary.header.count;
 844
 845                                 }
 846
 847                                 Logger.Log.Debug ("{0}: indexed {1} messages{2}",
 848                                                   this.folder_name, this.indexed_count, progress);
 849                         }
 850
 851                         if (this.tracker != null)
 852                                 this.tracker.Checkpoint ();
 853                 }
 854
 855                 protected override void CrawlFinished ()
 856                 {
 857                         this.queryable.FileAttributesStore.AttachLastWriteTime (this.CrawlFile.FullName, this.start_crawl_time);
 858                         base.CrawlFinished ();
 859                 }
 860
 861                 public override string GetTarget ()
 862                 {
 863                         return "summary-file:" + summary_info.FullName;
 864                 }
 865
 866                 protected override FileInfo CrawlFile {
 867                         get { return this.summary_info; }
 868                 }
 869         }
 870 }