beagled/EvolutionMailDriver/EvolutionMailDriver.cs

   1 //
   2 // EvolutionMailDriver.cs
   3 //
   4 // Copyright (C) 2004 Novell, Inc.
   5 //
   6 //
   7 //
   8 // Permission is hereby granted, free of charge, to any person obtaining a
   9 // copy of this software and associated documentation files (the "Software"),
  10 // to deal in the Software without restriction, including without limitation
  11 // the rights to use, copy, modify, merge, publish, distribute, sublicense,
  12 // and/or sell copies of the Software, and to permit persons to whom the
  13 // Software is furnished to do so, subject to the following conditions:
  14 //
  15 // The above copyright notice and this permission notice shall be included in
  16 // all copies or substantial portions of the Software.
  17 //
  18 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  21 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  22 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  23 // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  24 // DEALINGS IN THE SOFTWARE.
  25 //
  26
  27
  28 using System;
  29 using System.Collections;
  30 using System.IO;
  31 using System.Threading;
  32
  33 using Beagle.Util;
  34 using Camel = Beagle.Util.Camel;
  35
  36 using LNI = Lucene.Net.Index;
  37
  38 namespace Beagle.Daemon.EvolutionMailDriver {
  39
  40         [QueryableFlavor (Name="Mail", Domain=QueryDomain.Local, RequireInotify=false)]
  41         public class EvolutionMailQueryable : LuceneQueryable {
  42
  43                 public int polling_interval_in_seconds = 60;
  44
  45                 public static Logger log = Logger.Get ("mail");
  46                 private string local_path, imap_path, imap4_path;
  47
  48                 private MailCrawler crawler;
  49
  50                 // Index versions
  51                 // 1: Original version, stored all recipient addresses as a
  52                 //    RFC822 string
  53                 // 2: Stores recipients in separate properties,
  54                 //    filters/indexes all attachments
  55                 private const int INDEX_VERSION = 2;
  56
  57                 public EvolutionMailQueryable () : base ("MailIndex", INDEX_VERSION)
  58                 {
  59                         this.local_path = Path.Combine (PathFinder.HomeDir, ".evolution/mail/local");
  60                         this.imap_path = Path.Combine (PathFinder.HomeDir, ".evolution/mail/imap");
  61                         this.imap4_path = Path.Combine (PathFinder.HomeDir, ".evolution/mail/imap4");
  62                 }
  63
  64                 private void Crawl ()
  65                 {
  66                         crawler.Crawl ();
  67                         foreach (FileInfo file in crawler.Summaries)
  68                                 IndexSummary (file);
  69                         foreach (FileInfo file in crawler.Mboxes)
  70                                 IndexMbox (file);
  71                 }
  72
  73                 private void CrawlHook (Scheduler.Task task)
  74                 {
  75                         Crawl ();
  76                         task.Reschedule = true;
  77                         task.TriggerTime = DateTime.Now.AddSeconds (polling_interval_in_seconds);
  78                 }
  79
  80                 //////////////////////////////////////////////////////////////////////////////////////////////
  81
  82                 // Evil hack: manipulate the lucene index directly to extract
  83                 // statistics about who you sent mail to.
  84                 static private void AnalyzeYourRecipients (LuceneQueryingDriver driver)
  85                 {
  86                         Logger.Log.Debug ("FIXME: AnalyzeYourRecipients commented out!");
  87 #if false
  88                         LNI.IndexReader reader;
  89                         reader = LNI.IndexReader.Open (driver.Store);
  90
  91                         LNI.TermEnum term_enum;
  92                         term_enum = reader.Terms ();
  93
  94                         const string prop_field = "prop:k:fixme:sentTo";
  95
  96                         LNI.Term skip_to;
  97                         skip_to = new LNI.Term (prop_field, ""); // ACK!
  98
  99                         term_enum.SkipTo (skip_to);
 100
 101                         int N = 0;
 102                         while (term_enum.Next ()) {
 103                                 LNI.Term t = term_enum.Term ();
 104                                 if (t.Field () != prop_field) // Double-ACK!
 105                                         break;
 106
 107                                 AddAsYourRecipient (t.Text (), term_enum.DocFreq ());
 108                                 ++N;
 109                         }
 110                         term_enum.Close ();
 111
 112                         reader.Close ();
 113 #endif
 114                 }
 115
 116                 static private Hashtable your_recipient_weights = new Hashtable ();
 117                 static private int total_recipient_weight = 0;
 118
 119                 static private void AddAsYourRecipient (string address, int weight)
 120                 {
 121                         object obj = your_recipient_weights [address];
 122                         int n = obj != null ? (int) obj : 0;
 123                         your_recipient_weights [address] = n + weight;
 124                 }
 125
 126                 // FIXME: We update the weights when we add indexables, but not
 127                 // when we remove records... the daemon has to be restarted
 128                 // to pick up those changes.
 129                 static public void AddAsYourRecipient (Indexable indexable)
 130                 {
 131                         if (indexable == null)
 132                                 return;
 133                         foreach (Property prop in indexable.Properties)
 134                                 if (prop.Key == "fixme:sentTo")
 135                                         AddAsYourRecipient (prop.Value, 1);
 136                 }
 137
 138                 static private double AnalyzeYourRecipientsMultiplier (Hit hit)
 139                 {
 140                         // The first magic constant: exempt mail that is less than
 141                         // one month old.
 142                         if (hit.DaysSinceTimestamp < 30)
 143                                 return 1.0;
 144
 145                         int weight = 0;
 146                         foreach (Property prop in hit.Properties) {
 147
 148                                 // This optimization only counts for mails you
 149                                 // received: if you sent this mail, bail out immediately.
 150                                 if (prop.Key == "fixme:isSent")
 151                                         return 1.0;
 152
 153                                 if (prop.Key == "fixme:gotFrom") {
 154                                         object obj = your_recipient_weights [prop.Value];
 155                                         if (obj != null)
 156                                                 weight = (int) obj;
 157                                         break;
 158                                 }
 159                         }
 160
 161                         const double min_multiplier = 0.6;
 162                         const int weight_threshold = 10;
 163
 164                         if (weight <= 0)
 165                                 return min_multiplier;
 166
 167                         if (weight >= weight_threshold)
 168                                 return 1.0;
 169
 170                         // t == 0.0 if weight == 0
 171                         // t == 1.0 if weight == weight_threshold
 172                         double t = weight / (double) weight_threshold;
 173
 174                         double multiplier;
 175
 176                         multiplier = min_multiplier + (1 - min_multiplier) * t;
 177
 178                         return multiplier;
 179
 180                 }
 181
 182                 //////////////////////////////////////////////////////////////////////////////////////////////
 183
 184                 private void StartWorker ()
 185                 {
 186                         Logger.Log.Info ("Starting Evolution mail backend");
 187
 188                         Stopwatch stopwatch = new Stopwatch ();
 189                         stopwatch.Start ();
 190
 191                         // Check that we have data to index
 192                         if ((! Directory.Exists (this.local_path)) && (! Directory.Exists (this.imap_path))) {
 193                                 // No mails present, repoll every minute
 194                                 log.Warn ("Evolution mail store not found, watching for it.");
 195                                 GLib.Timeout.Add (60000, new GLib.TimeoutHandler (CheckForMailData));
 196                                 return;
 197                         }
 198
 199                         // This might be dangerous.
 200                         AnalyzeYourRecipients (Driver);
 201
 202                         // Get notification when an index or summary file changes
 203                         if (Inotify.Enabled) {
 204                                 Watch (this.local_path);
 205                                 Watch (this.imap_path);
 206                                 Watch (this.imap4_path);
 207                         }
 208
 209                         Logger.Log.Debug ("Starting mail crawl");
 210                         crawler = new MailCrawler (this.local_path, this.imap_path, this.imap4_path);
 211                         Crawl ();
 212                         Logger.Log.Debug ("Mail crawl finished");
 213
 214                         // If we don't have inotify, we have to poll the file system.  Ugh.
 215                         if (! Inotify.Enabled) {
 216                                 Scheduler.Task task = Scheduler.TaskFromHook (new Scheduler.TaskHook (CrawlHook));
 217                                 task.Tag = "Crawling ~/.evolution to find summary changes";
 218                                 ThisScheduler.Add (task);
 219                         }
 220
 221                         stopwatch.Stop ();
 222                         Logger.Log.Info ("Evolution mail driver worker thread done in {0}",
 223                                          stopwatch);
 224                 }
 225
 226                 public override void Start ()
 227                 {
 228                         Logger.Log.Info ("Starting Evolution mail backend");
 229                         base.Start ();
 230
 231                         ExceptionHandlingThread.Start (new ThreadStart (StartWorker));
 232                 }
 233
 234                 private void Watch (string path)
 235                 {
 236                         DirectoryInfo root = new DirectoryInfo (path);
 237                         if (! root.Exists)
 238                                 return;
 239
 240                         Queue queue = new Queue ();
 241                         queue.Enqueue (root);
 242
 243                         while (queue.Count > 0) {
 244                                 DirectoryInfo dir = queue.Dequeue () as DirectoryInfo;
 245
 246                                 if (! dir.Exists)
 247                                         continue;
 248
 249                                 Inotify.Subscribe (dir.FullName, OnInotifyEvent,
 250                                                         Inotify.EventType.Create
 251                                                         | Inotify.EventType.Delete
 252                                                         | Inotify.EventType.MovedTo);
 253
 254                                 foreach (DirectoryInfo subdir in dir.GetDirectories ())
 255                                         queue.Enqueue (subdir);
 256                         }
 257                 }
 258
 259                 private void OnInotifyEvent (Inotify.Watch watch,
 260                                                  string path,
 261                                              string subitem,
 262                                              string srcpath,
 263                                              Inotify.EventType type)
 264                 {
 265                         if (subitem == "")
 266                                 return;
 267
 268                         string fullPath = Path.Combine (path, subitem);
 269
 270                         if ((type & Inotify.EventType.Create) != 0 && (type & Inotify.EventType.IsDirectory) != 0) {
 271                                 Watch (fullPath);
 272                                 return;
 273                         }
 274
 275                         if ((type & Inotify.EventType.Delete) != 0 && (type & Inotify.EventType.IsDirectory) != 0) {
 276                                 watch.Unsubscribe ();
 277                                 return;
 278                         }
 279
 280                         if ((type & Inotify.EventType.MovedTo) != 0) {
 281                                 if (subitem == "summary") {
 282                                         // IMAP summary
 283                                         log.Info ("Reindexing updated IMAP summary: {0}", fullPath);
 284                                         this.IndexSummary (new FileInfo (fullPath));
 285                                 } else if (Path.GetExtension (fullPath) == ".ev-summary") {
 286                                         // mbox summary
 287                                         string mbox_file = Path.ChangeExtension (fullPath, null);
 288                                         log.Info ("Reindexing updated mbox: {0}", mbox_file);
 289                                         this.IndexMbox (new FileInfo (mbox_file));
 290                                 }
 291                         }
 292                 }
 293
 294                 private bool CheckForMailData ()
 295                 {
 296                         if ((! Directory.Exists (this.local_path)) && (! Directory.Exists (this.imap_path)))
 297                                 return true; // continue polling
 298
 299                         // Otherwise stop polling and start indexing
 300                         StartWorker();
 301                         return false;
 302                 }
 303
 304                 public string Name {
 305                         get { return "EvolutionMail"; }
 306                 }
 307
 308                 public void IndexSummary (FileInfo summaryInfo)
 309                 {
 310                         // If there's already a task running for this folder,
 311                         // don't interrupt it.
 312                         if (ThisScheduler.ContainsByTag (summaryInfo.FullName)) {
 313                                 Logger.Log.Debug ("Not adding task for already running task: {0}", summaryInfo.FullName);
 314                                 return;
 315                         }
 316
 317                         EvolutionMailIndexableGeneratorImap generator = new EvolutionMailIndexableGeneratorImap (this, summaryInfo);
 318                         Scheduler.Task task;
 319                         task = NewAddTask (generator, new Scheduler.Hook (generator.Checkpoint));
 320                         task.Tag = summaryInfo.FullName;
 321                         // IndexableGenerator tasks default to having priority Scheduler.Priority Generator
 322                         ThisScheduler.Add (task);
 323                 }
 324
 325                 public void IndexMbox (FileInfo mboxInfo)
 326                 {
 327                         // If there's already a task running for this mbox,
 328                         // don't interrupt it.
 329                         if (ThisScheduler.ContainsByTag (mboxInfo.FullName)) {
 330                                 Logger.Log.Debug ("Not adding task for already running task: {0}", mboxInfo.FullName);
 331                                 return;
 332                         }
 333
 334                         EvolutionMailIndexableGeneratorMbox generator = new EvolutionMailIndexableGeneratorMbox (this, mboxInfo);
 335                         Scheduler.Task task;
 336                         task = NewAddTask (generator, new Scheduler.Hook (generator.Checkpoint));
 337                         task.Tag = mboxInfo.FullName;
 338                         // IndexableGenerator tasks default to having priority Scheduler.Priority Generator
 339                         ThisScheduler.Add (task);
 340                 }
 341
 342                 public static Uri EmailUri (string accountName, string folderName, string uid)
 343                 {
 344                         return new Uri (String.Format ("email://{0}/{1};uid={2}",
 345                                                        accountName, folderName, uid));
 346                 }
 347
 348                 override protected double RelevancyMultiplier (Hit hit)
 349                 {
 350                         double t = 1.0;
 351
 352                         // FIXME: We should probably be more careful
 353                         // about how we combine these two numbers into one,
 354                         // since the cardinal value of the score is used for
 355                         // comparisons with hits of other types.  It isn't
 356                         // sufficient to just worry about the ordinal relationship
 357                         // between two scores.
 358                         t *= HalfLifeMultiplierFromProperty (hit, 0.25,
 359                                                              "fixme:received", "fixme:sentdate");
 360
 361                         t *= AnalyzeYourRecipientsMultiplier (hit);
 362
 363                         return t;
 364                 }
 365         }
 366
 367 }