beagled/ThunderbirdQueryable/ThunderbirdIndexer.cs

   1 //
   2 // ThunderbirdIndexer.cs: This class launches IndexableGenerators and makes sure instant-updates work
   3 //
   4 // Copyright (C) 2006 Pierre Östlund
   5 //
   6
   7 //
   8 // Permission is hereby granted, free of charge, to any person obtaining a copy
   9 // of this software and associated documentation files (the "Software"), to deal
  10 // in the Software without restriction, including without limitation the rights
  11 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  12 // copies of the Software, and to permit persons to whom the Software is
  13 // furnished to do so, subject to the following conditions:
  14 //
  15 // The above copyright notice and this permission notice shall be included in all
  16 // copies or substantial portions of the Software.
  17 //
  18 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  21 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  22 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  23 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  24 // SOFTWARE.
  25 //
  26
  27 using System;
  28 using System.IO;
  29 using System.Collections;
  30 using System.Reflection;
  31
  32 using Beagle.Util;
  33 using Beagle.Daemon;
  34 using TB = Beagle.Util.Thunderbird;
  35
  36 namespace Beagle.Daemon.ThunderbirdQueryable {
  37
  38         public class ThunderbirdIndexer {
  39                 private ThunderbirdQueryable queryable;
  40
  41                 private bool init_phase, first_lap;
  42                 private string[] root_paths;
  43                 private Hashtable supported_types;
  44                 private ArrayList account_list;
  45                 private ThunderbirdInotify inotify;
  46
  47                 public delegate void NotificationEventHandler (object o, NotificationEventArgs args);
  48
  49                 public ThunderbirdIndexer (ThunderbirdQueryable queryable, string[] root_paths)
  50                 {
  51                         this.queryable = queryable;
  52                         this.root_paths = root_paths;
  53                         this.supported_types = new Hashtable ();
  54                         this.init_phase = true;
  55                         this.first_lap = true;
  56                         this.account_list = new ArrayList ();
  57                         this.inotify = new ThunderbirdInotify ();
  58
  59                         LoadSupportedTypes ();
  60
  61                         foreach (string path in root_paths) {
  62                                 Inotify.Subscribe (path, OnInotifyEvent,
  63                                         Inotify.EventType.Delete |
  64                                         Inotify.EventType.MovedTo |
  65                                         Inotify.EventType.Modify |
  66                                         Inotify.EventType.Create);
  67                         }
  68
  69                         inotify.InotifyEvent += OnInotifyEvent;
  70                 }
  71
  72                 // Loads all supported types, checks if they have a correct constructor and is enabled
  73                 private void LoadSupportedTypes ()
  74                 {
  75                         Assembly assembly = Assembly.GetCallingAssembly ();
  76
  77                         foreach (Type type in ReflectionFu.ScanAssemblyForInterface (assembly, typeof (IIndexableGenerator))) {
  78
  79                                 foreach (ThunderbirdIndexableGeneratorAttribute attr in
  80                                         ReflectionFu.ScanTypeForAttribute (type, typeof (ThunderbirdIndexableGeneratorAttribute))) {
  81
  82                                         foreach (ConstructorInfo constructor in type.GetConstructors ()) {
  83
  84                                                 ParameterInfo[] parameters = constructor.GetParameters ();
  85                                                 if(parameters.Length != 3)
  86                                                         continue;
  87
  88                                                 if (parameters [0].ParameterType.Equals (typeof (ThunderbirdIndexer)) &&
  89                                                         parameters [1].ParameterType.Equals (typeof (TB.Account)) &&
  90                                                         parameters [2].ParameterType.Equals (typeof (string))) {
  91
  92                                                         // Make sure we should enable this type
  93                                                         if (attr.Enabled)
  94                                                                 supported_types [attr.Type]  = type;
  95
  96                                                 } else
  97                                                         Logger.Log.Debug ("{0} has an invalid constructor!", type.ToString ());
  98                                         }
  99                                 }
 100                         }
 101                 }
 102
 103                 public void Crawl ()
 104                 {
 105                         int launched = 0;
 106
 107                         foreach (string path in root_paths) {
 108                                 foreach (TB.Account account in Thunderbird.ReadAccounts (path)) {
 109                                         if (Shutdown.ShutdownRequested)
 110                                                 return;
 111
 112                                         if (supported_types [account.Type] == null)
 113                                                 continue;
 114
 115                                         IndexAccount (account);
 116                                         launched++;
 117                                 }
 118                         }
 119
 120                         init_phase = false;
 121                         Logger.Log.Info ("Indexing {0} ({1}) Thunderbird account(s) spread over {2} profile(s)",
 122                                 launched, account_list.Count, root_paths.Length);
 123
 124                         // Clean out old stuff in case no IndexableGenerator was launched
 125                         if (launched == 0)
 126                                 ChildComplete ();
 127                 }
 128
 129                 public void IndexAccount (TB.Account account)
 130                 {
 131                         TB.Account stored_account = GetParentAccount (account.Path);
 132
 133                         // We need to act upon changes made to accounts during Thunderbird runtime.
 134                         // The user might change from plain to SSL, which leads to a new port number
 135                         // that has to be taken in account for when indexing.
 136                         if (stored_account == null && Directory.Exists (account.Path) && supported_types [account.Type] != null) {
 137                                 account_list.Add (account);
 138                                 IndexDirectory (account.Path);
 139                                 //Logger.Log.Info ("Indexing {0} account {1}", account.Type.ToString (), account.Server);
 140
 141                         } else if (stored_account == null && File.Exists (account.Path) && supported_types [account.Type] != null) {
 142                                 account_list.Add (account);
 143                                 IndexFile (account.Path);
 144                                 //Logger.Log.Info ("Indexing {0} account {1}", account.Type.ToString (), account.Server);
 145
 146                         } else if (stored_account != null &&
 147                                 (stored_account.Server != account.Server ||
 148                                 stored_account.Port != account.Port ||
 149                                 stored_account.Type != account.Type ||
 150                                 stored_account.Delimiter != account.Delimiter)) {
 151
 152                                 account_list.Remove (stored_account);
 153                                 account_list.Add (account);
 154
 155                                 // Make sure all running indexables are aware of this since it can affect the way they index
 156                                 NotificationEventArgs args;
 157                                 args = new NotificationEventArgs (NotificationType.UpdateAccountInformation, stored_account);
 158                                 args.Data = (object) account;
 159                                 OnNotification (args);
 160
 161                                 Logger.Log.Info ("Updated {0} with new account details", account.Server);
 162                         }
 163                 }
 164
 165                 public void IndexFile (string file)
 166                 {
 167                         TB.Account account = GetParentAccount (file);
 168
 169                         if (account == null || supported_types [account.Type] == null || Thunderbird.GetFileSize (file) < 1)
 170                                 return;
 171
 172                         object[] param = new object[] {this, account, file};
 173                         ThunderbirdIndexableGenerator generator = Activator.CreateInstance (
 174                                 (Type) supported_types [account.Type], param) as ThunderbirdIndexableGenerator;
 175
 176                         AddIIndexableTask (generator, file);
 177                 }
 178
 179                 private void IndexDirectory (string directory)
 180                 {
 181                         Queue pending = new Queue ();
 182
 183                         pending.Enqueue (directory);
 184                         while (pending.Count > 0) {
 185                                 string dir = pending.Dequeue () as string;
 186
 187                                 foreach (string subdir in DirectoryWalker.GetDirectories (dir)) {
 188                                         if (Shutdown.ShutdownRequested)
 189                                                 return;
 190
 191                                         pending.Enqueue (subdir);
 192                                 }
 193
 194                                 if (Inotify.Enabled) {
 195                                                 inotify.Watch (dir,
 196                                                 Inotify.EventType.Modify |
 197                                                 Inotify.EventType.Create |
 198                                                 Inotify.EventType.Delete |
 199                                                 Inotify.EventType.MovedFrom |
 200                                                 Inotify.EventType.MovedTo);
 201                                 }
 202
 203                                 foreach (string file in DirectoryWalker.GetItems
 204                                         (dir, new DirectoryWalker.FileFilter (Thunderbird.IsMorkFile))) {
 205                                         if (Shutdown.ShutdownRequested)
 206                                                 return;
 207
 208                                         IndexFile (file);
 209                                 }
 210                         }
 211                 }
 212
 213                 public void RemoveAccount (TB.Account account)
 214                 {
 215                         TB.Account acc = GetParentAccount (account.Path);
 216
 217                         if (acc == null)
 218                                 return;
 219
 220                         ScheduleRemoval (Property.NewKeyword ("fixme:account", acc.Server), Scheduler.Priority.Delayed);
 221                         OnNotification (new NotificationEventArgs (NotificationType.StopIndexing, account));
 222                         account_list.Remove (acc);
 223                 }
 224
 225                 private void AddIIndexableTask (IIndexableGenerator generator, string tag)
 226                 {
 227                         if (queryable.ThisScheduler.ContainsByTag (tag)) {
 228                                 Logger.Log.Debug ("Not adding a Task for already running: {0}", tag);
 229                                 return;
 230                         }
 231
 232                         Scheduler.Task task = queryable.NewAddTask (generator);
 233                         task.Tag = tag;
 234                         queryable.ThisScheduler.Add (task);
 235                 }
 236
 237                 private void ScheduleRemoval (Property prop, Scheduler.Priority priority)
 238                 {
 239                         if (queryable.ThisScheduler.ContainsByTag (prop.ToString ())) {
 240                                 Logger.Log.Debug ("Not adding a Task for already running: {0}", prop.ToString ());
 241                                 return;
 242                         }
 243
 244                         Scheduler.Task task = queryable.NewRemoveByPropertyTask (prop);
 245                         task.Priority = priority;
 246                         task.SubPriority = 0;
 247                         queryable.ThisScheduler.Add (task);
 248                 }
 249
 250                 public void ScheduleRemoval (Uri[] uris, string tag, Scheduler.Priority priority)
 251                 {
 252                         if (queryable.ThisScheduler.ContainsByTag (tag)) {
 253                                 Logger.Log.Debug ("Not adding a Task for already running: {0}", tag);
 254                                 return;
 255                         }
 256
 257                         Scheduler.Task task = queryable.NewAddTask (new UriRemovalIndexableGenerator (uris));
 258                         task.Priority = priority;
 259                         task.SubPriority = 0;
 260                         queryable.ThisScheduler.Add (task);
 261                 }
 262
 263                 public void UpdateAccounts (string root_path)
 264                 {
 265                         ArrayList new_accounts;
 266
 267                         try {
 268                                 new_accounts = Thunderbird.ReadAccounts (root_path);
 269                         } catch (Exception e) {
 270                                 Logger.Log.Warn ("Failed when reading Thunderbird accounts: {0}, account may have been added or removed", e);
 271                                 return;
 272                         }
 273
 274                         // Add all accounts again to make sure things are updated the way they should
 275                         foreach (TB.Account account in new_accounts)
 276                                 IndexAccount (account);
 277
 278                         // Remove non-existing accounts
 279                         foreach (TB.Account existing in account_list) {
 280                                 bool found = false;
 281
 282                                 foreach (TB.Account new_account in new_accounts) {
 283                                         if (existing.Path == new_account.Path) {
 284                                                 found = true;
 285                                                 break;
 286                                         }
 287                                 }
 288
 289                                 if (!found)
 290                                                 RemoveAccount (existing);
 291                         }
 292                 }
 293
 294                 public TB.Account GetParentAccount (string directory)
 295                 {
 296                         foreach (TB.Account acc in account_list) {
 297                                 if (directory.StartsWith (acc.Path))
 298                                         return acc;
 299                         }
 300
 301                         return null;
 302                 }
 303
 304                 private void OnInotifyEvent (Inotify.Watch watch,
 305                                                 string path,
 306                                                 string subitem,
 307                                                 string srcpath,
 308                                                 Inotify.EventType type)
 309                 {
 310                         if (subitem == null)
 311                                 return;
 312
 313                         string full_path = Path.Combine (path, subitem);
 314
 315                         // If prefs.js is deleted... then we have nothing at all to index
 316                         if (((type & Inotify.EventType.MovedTo) != 0 && srcpath == Path.Combine (path, "prefs.js")) ||
 317                                 ((type & Inotify.EventType.Delete) != 0 && subitem == "prefs.js")) {
 318
 319                                 foreach (TB.Account account in account_list)
 320                                         RemoveAccount (account);
 321                                 return;
 322                         }
 323
 324                         // Update in case an account was removed or added
 325                         // Thunderbird saves prefs.js with a different name and then replacing the old one
 326                         // by "moving" it over the existing prefs.js. That's why MoveTo is used as inotfy type.
 327                         if ((((type & Inotify.EventType.Modify) != 0 || (type & Inotify.EventType.MovedTo) != 0 ||
 328                                 (type & Inotify.EventType.Create) != 0) && subitem == "prefs.js")) {
 329
 330                                 UpdateAccounts (path);
 331                                 return;
 332                         }
 333
 334                         // In case the address book file have been moved or deleted, we have to stop indexing it
 335                         if (((type & Inotify.EventType.MovedTo) != 0 && srcpath == Path.Combine (path, "abook.mab")) ||
 336                                 ((type & Inotify.EventType.Delete) != 0 && subitem == "abook.mab")) {
 337
 338                                 TB.Account account = GetParentAccount (full_path);
 339
 340                                 if (account != null)
 341                                         RemoveAccount (account);
 342
 343                                 return;
 344                         }
 345
 346                         // In case of a newly created addressbook, the current address book is modified or an old
 347                         // address book is moved to where the address book can be found: either start indexing
 348                         // or restart an already indexing IndeaxbleGenerator.
 349                         if ((((type & Inotify.EventType.Modify) != 0 || (type & Inotify.EventType.MovedTo) != 0 ||
 350                                 (type & Inotify.EventType.Create) != 0) && subitem == "abook.mab")) {
 351
 352                                 TB.Account account = GetParentAccount (full_path);
 353
 354                                 if (account == null && File.Exists (full_path)) {
 355                                         UpdateAccounts (path);
 356                                         return;
 357                                 } else if (account == null)
 358                                         return;
 359
 360                                 // Tell any running indexable about this or start a new one
 361                                 if (queryable.ThisScheduler.ContainsByTag (full_path))
 362                                         OnNotification (new NotificationEventArgs (NotificationType.RestartIndexing, account));
 363                                 else
 364                                         IndexFile (full_path);
 365
 366                                 return;
 367                         }
 368
 369                         // Re-index files when needed
 370                         if ((type & Inotify.EventType.Modify) != 0) {
 371                                 TB.Account account = GetParentAccount (full_path);
 372
 373                                 if (account == null || !Thunderbird.IsMorkFile (path, subitem))
 374                                         return;
 375
 376                                 // In case we have a running IndexableGenerator, tell it that we have a file that needs to
 377                                 // be re-indexed.
 378                                 if (queryable.ThisScheduler.ContainsByTag (full_path))
 379                                         OnNotification (new NotificationEventArgs (NotificationType.RestartIndexing, account));
 380                                 else
 381                                         IndexFile (full_path);
 382
 383                                 return;
 384                         }
 385
 386                         // Index newly created directories
 387                         if ((type & Inotify.EventType.Create) != 0 && (type & Inotify.EventType.IsDirectory) != 0) {
 388                                 if (GetParentAccount (full_path) != null && Inotify.Enabled)
 389                                         Inotify.Subscribe (full_path, OnInotifyEvent, Inotify.EventType.All);
 390
 391                                 return;
 392                         }
 393                 }
 394
 395                 public void ChildComplete ()
 396                 {
 397                         if (NotificationEvent != null || init_phase || !first_lap)
 398                                 return;
 399
 400                         if (Thunderbird.Debug)
 401                                 Logger.Log.Debug ("Removing old Thunderbird objects");
 402
 403                         Scheduler.Task task = queryable.NewRemoveTaskByDate (ThunderbirdQueryable.IndexingStart);
 404                         task.Priority = Scheduler.Priority.Idle;
 405                         task.Tag = "RemoveOldThunderbirdMails";
 406                         queryable.ThisScheduler.Add (task);
 407
 408                         // This makes sure that ChildComplete will only clean out all mails once in a lifetime
 409                         // (of the Thunderbird backend that is)
 410                         first_lap = false;
 411                 }
 412
 413                 protected virtual void OnNotification(NotificationEventArgs args)
 414                 {
 415                         if (NotificationEvent != null)
 416                                 NotificationEvent (this, args);
 417                 }
 418
 419                 public event NotificationEventHandler NotificationEvent;
 420
 421                 public LuceneAccess Lucene {
 422                         get { return queryable.Lucene; }
 423                 }
 424         }
 425
 426         /////////////////////////////////////////////////////////////////////////////////////
 427
 428         public enum NotificationType {
 429                 StopIndexing,
 430                 RestartIndexing,
 431                 UpdateAccountInformation
 432         }
 433
 434         /////////////////////////////////////////////////////////////////////////////////////
 435
 436         public class NotificationEventArgs  : EventArgs
 437         {
 438                 private NotificationType type;
 439                 private TB.Account account;
 440                 private object data;
 441
 442                 public NotificationEventArgs (NotificationType type, TB.Account account)
 443                 {
 444                         this.type = type;
 445                         this.account = account;
 446                 }
 447
 448                 public NotificationType Type {
 449                         get { return type; }
 450                 }
 451
 452                 public TB.Account Account {
 453                         get { return account; }
 454                 }
 455
 456                 public object Data {
 457                         get { return data; }
 458                         set { data =value; }
 459                 }
 460         }
 461
 462 }