?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Indexable is not marked _done_ until all the child indexables (including child of...
[beagle.git] / beagled / ThunderbirdQueryable / ThunderbirdIndexer.cs
blobb2a04331072909b8c509145ca64f204322c0bd72
1 //
2 // ThunderbirdIndexer.cs: This class launches IndexableGenerators and makes sure instant-updates work
3 //
4 // Copyright (C) 2006 Pierre Östlund
5 //
6
7 //
8 // Permission is hereby granted, free of charge, to any person obtaining a copy
9 // of this software and associated documentation files (the "Software"), to deal
10 // in the Software without restriction, including without limitation the rights
11 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
12 // copies of the Software, and to permit persons to whom the Software is
13 // furnished to do so, subject to the following conditions:
14 //
15 // The above copyright notice and this permission notice shall be included in all
16 // copies or substantial portions of the Software.
17 //
18 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
21 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
22 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
23 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
24 // SOFTWARE.
25 //
26
27 using System;
28 using System.IO;
29 using System.Collections;
30 using System.Reflection;
31
32 using Beagle.Util;
33 using Beagle.Daemon;
34 using Beagle.Daemon.ThunderbirdQueryable;
35 using TB = Beagle.Util.Thunderbird;
36
37 [assembly: ThunderbirdDefinedGenerators (
38 typeof (ContactIndexableGenerator),
39 typeof (MailIndexableGenerator),
40 typeof (MoveMailIndexableGenerator),
41 typeof (NntpIndexableGenerator),
42 typeof (RssIndexableGenerator)
43 )]
44
45 namespace Beagle.Daemon.ThunderbirdQueryable {
46
47 public class ThunderbirdIndexer {
48 private ThunderbirdQueryable queryable;
49
50 private bool init_phase, first_lap;
51 private string[] root_paths;
52 private Hashtable supported_types;
53 private ArrayList account_list;
54 private ThunderbirdInotify inotify;
55
56 public delegate void NotificationEventHandler (object o, NotificationEventArgs args);
57
58 public ThunderbirdIndexer (ThunderbirdQueryable queryable, string[] root_paths)
59 {
60 this.queryable = queryable;
61 this.root_paths = root_paths;
62 this.supported_types = new Hashtable ();
63 this.init_phase = true;
64 this.first_lap = true;
65 this.account_list = new ArrayList ();
66 this.inotify = new ThunderbirdInotify ();
67
68 LoadSupportedTypes ();
69
70 foreach (string path in root_paths) {
71 Inotify.Subscribe (path, OnInotifyEvent,
72 Inotify.EventType.Delete |
73 Inotify.EventType.MovedTo |
74 Inotify.EventType.Modify |
75 Inotify.EventType.Create);
76 }
77
78 inotify.InotifyEvent += OnInotifyEvent;
79 }
80
81 // Loads all supported types, checks if they have a correct constructor and is enabled
82 private void LoadSupportedTypes ()
83 {
84 Assembly assembly = Assembly.GetCallingAssembly ();
85
86 foreach (Type type in ReflectionFu.GetTypesFromAssemblyAttribute (assembly, typeof (ThunderbirdDefinedGeneratorsAttribute))) {
87
88 foreach (ThunderbirdIndexableGeneratorAttribute attr in
89 ReflectionFu.ScanTypeForAttribute (type, typeof (ThunderbirdIndexableGeneratorAttribute))) {
90
91 foreach (ConstructorInfo constructor in type.GetConstructors ()) {
92
93 ParameterInfo[] parameters = constructor.GetParameters ();
94 if(parameters.Length != 3)
95 continue;
96
97 if (parameters [0].ParameterType.Equals (typeof (ThunderbirdIndexer)) &&
98 parameters [1].ParameterType.Equals (typeof (TB.Account)) &&
99 parameters [2].ParameterType.Equals (typeof (string))) {
100
101 // Make sure we should enable this type
102 if (attr.Enabled)
103 supported_types [attr.Type] = type;
104
105 } else
106 Logger.Log.Debug ("{0} has an invalid constructor!", type.ToString ());
107 }
108 }
109 }
110 }
111
112 public void Crawl ()
113 {
114 int launched = 0;
115 TB.AccountReader reader = null;
116
117 foreach (string path in root_paths) {
118 try {
119 reader = new TB.AccountReader (path);
120 } catch (Exception e) {
121 Logger.Log.Warn (e, "Failed to load accounts: {0}", e.Message);
122 continue;
123 }
124
125 foreach (TB.Account account in reader) {
126 if (Shutdown.ShutdownRequested)
127 return;
128
129 if (supported_types [account.Type] == null)
130 continue;
131
132 IndexAccount (account);
133 launched++;
134 }
135 }
136
137 init_phase = false;
138 Logger.Log.Info ("Indexing {0} ({1}) Thunderbird account(s) spread over {2} profile(s)",
139 launched, account_list.Count, root_paths.Length);
140
141 // Clean out old stuff in case no IndexableGenerator was launched
142 if (launched == 0)
143 ChildComplete ();
144 }
145
146 public void IndexAccount (TB.Account account)
147 {
148 TB.Account stored_account = GetParentAccount (account.Path);
149
150 // We need to act upon changes made to accounts during Thunderbird runtime.
151 // The user might change from plain to SSL, which leads to a new port number
152 // that has to be taken in account for when indexing.
153 if (stored_account == null && Directory.Exists (account.Path) && supported_types [account.Type] != null) {
154 account_list.Add (account);
155 IndexDirectory (account.Path);
156 //Logger.Log.Info ("Indexing {0} account {1}", account.Type.ToString (), account.Server);
157
158 } else if (stored_account == null && File.Exists (account.Path) && supported_types [account.Type] != null) {
159 account_list.Add (account);
160 IndexFile (account.Path);
161 //Logger.Log.Info ("Indexing {0} account {1}", account.Type.ToString (), account.Server);
162
163 } else if (stored_account != null &&
164 (stored_account.Server != account.Server ||
165 stored_account.Port != account.Port ||
166 stored_account.Type != account.Type ||
167 stored_account.Delimiter != account.Delimiter)) {
168
169 account_list.Remove (stored_account);
170 account_list.Add (account);
171
172 // Make sure all running indexables are aware of this since it can affect the way they index
173 NotificationEventArgs args;
174 args = new NotificationEventArgs (NotificationType.UpdateAccountInformation, stored_account);
175 args.Data = (object) account;
176 OnNotification (args);
177
178 Logger.Log.Info ("Updated {0} with new account details", account.Server);
179 }
180 }
181
182 public void IndexFile (string file)
183 {
184 TB.Account account = GetParentAccount (file);
185
186 if (account == null || supported_types [account.Type] == null || Thunderbird.GetFileSize (file) < 1)
187 return;
188
189 object[] param = new object[] {this, account, file};
190 ThunderbirdIndexableGenerator generator = Activator.CreateInstance (
191 (Type) supported_types [account.Type], param) as ThunderbirdIndexableGenerator;
192
193 AddIIndexableTask (generator, file);
194 }
195
196 private void IndexDirectory (string directory)
197 {
198 Queue pending = new Queue ();
199
200 pending.Enqueue (directory);
201 while (pending.Count > 0) {
202 string dir = pending.Dequeue () as string;
203
204 foreach (string subdir in DirectoryWalker.GetDirectories (dir)) {
205 if (Shutdown.ShutdownRequested)
206 return;
207
208 pending.Enqueue (subdir);
209 }
210
211 if (Inotify.Enabled) {
212 inotify.Watch (dir,
213 Inotify.EventType.Modify |
214 Inotify.EventType.Create |
215 Inotify.EventType.Delete |
216 Inotify.EventType.MovedFrom |
217 Inotify.EventType.MovedTo);
218 }
219
220 foreach (string file in DirectoryWalker.GetItems
221 (dir, new DirectoryWalker.FileFilter (Thunderbird.IsMorkFile))) {
222 if (Shutdown.ShutdownRequested)
223 return;
224
225 IndexFile (file);
226 }
227 }
228 }
229
230 public void RemoveAccount (TB.Account account)
231 {
232 TB.Account acc = GetParentAccount (account.Path);
233
234 if (acc == null)
235 return;
236
237 ScheduleRemoval (Property.NewKeyword ("fixme:account", acc.Server), Scheduler.Priority.Delayed);
238 OnNotification (new NotificationEventArgs (NotificationType.StopIndexing, account));
239 account_list.Remove (acc);
240 }
241
242 private void AddIIndexableTask (IIndexableGenerator generator, string tag)
243 {
244 if (queryable.ThisScheduler.ContainsByTag (tag)) {
245 Logger.Log.Debug ("Not adding a Task for already running: {0}", tag);
246 return;
247 }
248
249 Scheduler.Task task = queryable.NewAddTask (generator);
250 task.Tag = tag;
251 queryable.ThisScheduler.Add (task);
252 }
253
254 private void ScheduleRemoval (Property prop, Scheduler.Priority priority)
255 {
256 if (queryable.ThisScheduler.ContainsByTag (prop.ToString ())) {
257 Logger.Log.Debug ("Not adding a Task for already running: {0}", prop.ToString ());
258 return;
259 }
260
261 Scheduler.Task task = queryable.NewRemoveByPropertyTask (prop);
262 task.Priority = priority;
263 task.SubPriority = 0;
264 queryable.ThisScheduler.Add (task);
265 }
266
267 public void ScheduleRemoval (Uri[] uris, string tag, Scheduler.Priority priority)
268 {
269 if (queryable.ThisScheduler.ContainsByTag (tag)) {
270 Logger.Log.Debug ("Not adding a Task for already running: {0}", tag);
271 return;
272 }
273
274 Scheduler.Task task = queryable.NewAddTask (new UriRemovalIndexableGenerator (uris));
275 task.Priority = priority;
276 task.SubPriority = 0;
277 queryable.ThisScheduler.Add (task);
278 }
279
280 public void UpdateAccounts (string root_path)
281 {
282 TB.AccountReader new_accounts = null;
283
284 try {
285 new_accounts = new TB.AccountReader (root_path);
286 } catch (Exception e) {
287 Logger.Log.Warn ("Failed when reading Thunderbird accounts: {0}, account may have been added or removed", e);
288 return;
289 }
290
291 // Add all accounts again to make sure things are updated the way they should
292 foreach (TB.Account account in new_accounts)
293 IndexAccount (account);
294
295 // Remove non-existing accounts
296 foreach (TB.Account existing in account_list) {
297 bool found = false;
298
299 foreach (TB.Account new_account in new_accounts) {
300 if (existing.Path == new_account.Path) {
301 found = true;
302 break;
303 }
304 }
305
306 if (!found)
307 RemoveAccount (existing);
308 }
309 }
310
311 public TB.Account GetParentAccount (string directory)
312 {
313 foreach (TB.Account acc in account_list) {
314 if (directory.StartsWith (acc.Path))
315 return acc;
316 }
317
318 return null;
319 }
320
321 private void OnInotifyEvent (Inotify.Watch watch,
322 string path,
323 string subitem,
324 string srcpath,
325 Inotify.EventType type)
326 {
327 if (subitem == null)
328 return;
329
330 string full_path = Path.Combine (path, subitem);
331
332 // If prefs.js is deleted... then we have nothing at all to index
333 if (((type & Inotify.EventType.MovedTo) != 0 && srcpath == Path.Combine (path, "prefs.js")) ||
334 ((type & Inotify.EventType.Delete) != 0 && subitem == "prefs.js")) {
335
336 foreach (TB.Account account in account_list)
337 RemoveAccount (account);
338 return;
339 }
340
341 // Update in case an account was removed or added
342 // Thunderbird saves prefs.js with a different name and then replacing the old one
343 // by "moving" it over the existing prefs.js. That's why MoveTo is used as inotfy type.
344 if ((((type & Inotify.EventType.Modify) != 0 || (type & Inotify.EventType.MovedTo) != 0 ||
345 (type & Inotify.EventType.Create) != 0) && subitem == "prefs.js")) {
346
347 UpdateAccounts (path);
348 return;
349 }
350
351 // In case the address book file have been moved or deleted, we have to stop indexing it
352 if (((type & Inotify.EventType.MovedTo) != 0 && srcpath == Path.Combine (path, "abook.mab")) ||
353 ((type & Inotify.EventType.Delete) != 0 && subitem == "abook.mab")) {
354
355 TB.Account account = GetParentAccount (full_path);
356
357 if (account != null)
358 RemoveAccount (account);
359
360 return;
361 }
362
363 // In case of a newly created addressbook, the current address book is modified or an old
364 // address book is moved to where the address book can be found: either start indexing
365 // or restart an already indexing IndeaxbleGenerator.
366 if ((((type & Inotify.EventType.Modify) != 0 || (type & Inotify.EventType.MovedTo) != 0 ||
367 (type & Inotify.EventType.Create) != 0) && subitem == "abook.mab")) {
368
369 TB.Account account = GetParentAccount (full_path);
370
371 if (account == null && File.Exists (full_path)) {
372 UpdateAccounts (path);
373 return;
374 } else if (account == null)
375 return;
376
377 // Tell any running indexable about this or start a new one
378 if (queryable.ThisScheduler.ContainsByTag (full_path))
379 OnNotification (new NotificationEventArgs (NotificationType.RestartIndexing, account));
380 else
381 IndexFile (full_path);
382
383 return;
384 }
385
386 // Re-index files when needed
387 if ((type & Inotify.EventType.Modify) != 0) {
388 TB.Account account = GetParentAccount (full_path);
389
390 if (account == null || !Thunderbird.IsMorkFile (path, subitem))
391 return;
392
393 // In case we have a running IndexableGenerator, tell it that we have a file that needs to
394 // be re-indexed.
395 if (queryable.ThisScheduler.ContainsByTag (full_path))
396 OnNotification (new NotificationEventArgs (NotificationType.RestartIndexing, account));
397 else
398 IndexFile (full_path);
399
400 return;
401 }
402
403 // Index newly created directories
404 if ((type & Inotify.EventType.Create) != 0 && (type & Inotify.EventType.IsDirectory) != 0) {
405 if (GetParentAccount (full_path) != null && Inotify.Enabled)
406 Inotify.Subscribe (full_path, OnInotifyEvent, Inotify.EventType.All);
407
408 return;
409 }
410 }
411
412 public void ChildComplete ()
413 {
414 if (NotificationEvent != null || init_phase || !first_lap)
415 return;
416
417 if (Thunderbird.Debug)
418 Logger.Log.Debug ("Removing old Thunderbird objects");
419
420 Scheduler.Task task = queryable.NewRemoveTaskByDate (ThunderbirdQueryable.IndexingStart);
421 task.Priority = Scheduler.Priority.Idle;
422 task.Tag = "RemoveOldThunderbirdMails";
423 queryable.ThisScheduler.Add (task);
424
425 // This makes sure that ChildComplete will only clean out all mails once in a lifetime
426 // (of the Thunderbird backend that is)
427 first_lap = false;
428 }
429
430 protected virtual void OnNotification(NotificationEventArgs args)
431 {
432 if (NotificationEvent != null)
433 NotificationEvent (this, args);
434 }
435
436 public event NotificationEventHandler NotificationEvent;
437
438 public LuceneAccess Lucene {
439 get { return queryable.Lucene; }
440 }
441 }
442
443 /////////////////////////////////////////////////////////////////////////////////////
444
445 [AttributeUsage (AttributeTargets.Assembly)]
446 public class ThunderbirdDefinedGeneratorsAttribute : TypeCacheAttribute {
447 public ThunderbirdDefinedGeneratorsAttribute (params Type[] types) : base (types) { }
448 }
449
450 /////////////////////////////////////////////////////////////////////////////////////
451
452 public enum NotificationType {
453 StopIndexing,
454 RestartIndexing,
455 UpdateAccountInformation
456 }
457
458 /////////////////////////////////////////////////////////////////////////////////////
459
460 public class NotificationEventArgs : EventArgs
461 {
462 private NotificationType type;
463 private TB.Account account;
464 private object data;
465
466 public NotificationEventArgs (NotificationType type, TB.Account account)
467 {
468 this.type = type;
469 this.account = account;
470 }
471
472 public NotificationType Type {
473 get { return type; }
474 }
475
476 public TB.Account Account {
477 get { return account; }
478 }
479
480 public object Data {
481 get { return data; }
482 set { data =value; }
483 }
484 }
485
486 }
A Beagle Git Repo