beagled/MozillaQueryable/MozillaQueryable.cs

   1 //
   2 // MozillaQueryable.cs
   3 //
   4 // Copyright (C) 2004 Novell, Inc.
   5 //
   6 // Authors:
   7 //   Fredrik Hedberg (fredrik.hedberg@avafan.com)
   8 //
   9
  10 //
  11 // Permission is hereby granted, free of charge, to any person obtaining a
  12 // copy of this software and associated documentation files (the "Software"),
  13 // to deal in the Software without restriction, including without limitation
  14 // the rights to use, copy, modify, merge, publish, distribute, sublicense,
  15 // and/or sell copies of the Software, and to permit persons to whom the
  16 // Software is furnished to do so, subject to the following conditions:
  17 //
  18 // The above copyright notice and this permission notice shall be included in
  19 // all copies or substantial portions of the Software.
  20 //
  21 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  22 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  23 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  24 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  25 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  26 // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  27 // DEALINGS IN THE SOFTWARE.
  28 //
  29
  30
  31 using System;
  32 using System.Collections;
  33 using System.IO;
  34 using System.Threading;
  35
  36 using Beagle.Util;
  37 using Beagle.Util.Mozilla;
  38
  39 namespace Beagle.Daemon.Mozilla {
  40
  41         internal class MozillaCrawler : Crawler {
  42
  43                 MozillaQueryable queryable;
  44
  45                 public MozillaCrawler (MozillaQueryable queryable, string fingerprint) : base (fingerprint)
  46                 {
  47                         this.queryable = queryable;
  48                 }
  49
  50                 protected override bool SkipByName (FileSystemInfo info)
  51                 {
  52                         if (info as DirectoryInfo != null)
  53                                 return false;
  54
  55                         if (File.Exists (info + ".msf"))
  56                                 return false;
  57
  58                         return true;
  59                 }
  60
  61                 protected override void CrawlFile (FileSystemInfo info)
  62                 {
  63                         FileInfo file = info as FileInfo;
  64
  65                         if (file != null)
  66                                 this.queryable.Index (file);
  67                 }
  68         }
  69
  70         [QueryableFlavor (Name="Mail", Domain=QueryDomain.Local)]
  71         public class MozillaMailQueryable : MozillaQueryable
  72         {
  73                 public MozillaMailQueryable () : base ("MozillaMailIndex")
  74                 {
  75                         AddAccountType ("none");
  76                         AddAccountType ("imap");
  77                 }
  78
  79                 protected override Indexable MessageToIndexable (Message message)
  80                 {
  81                         Uri uri = new Uri (String.Format ("email:///{0};id={1}", message.Path, message.Id));
  82
  83                         Indexable indexable = new Indexable (uri);
  84                         indexable.Type = "MailMessage";
  85
  86                         indexable.AddProperty (Property.New ("fixme:client", "mozilla"));
  87
  88                         indexable.AddProperty (Property.New ("dc:title", message.Subject));
  89
  90                         indexable.AddProperty (Property.New ("fixme:subject", message.Subject));
  91                         indexable.AddProperty (Property.New ("fixme:to", message.To));
  92                         indexable.AddProperty (Property.New ("fixme:from", message.From));
  93
  94                         indexable.AddProperty (Property.New ("fixme:offset", message.Offset));
  95
  96                         StringReader reader = new StringReader (message.Body);
  97                         indexable.SetTextReader (reader);
  98
  99                         return indexable;
 100                 }
 101         }
 102
 103         [QueryableFlavor (Name="Feed", Domain=QueryDomain.Local)]
 104         public class MozillaFeedQueryable : MozillaQueryable
 105         {
 106                 public MozillaFeedQueryable () : base ("MozillaFeedIndex")
 107                 {
 108                         AddAccountType ("rss");
 109                 }
 110
 111                 protected override Indexable MessageToIndexable (Message message)
 112                 {
 113                         Uri uri = new Uri (String.Format ("feed:///{0};id={1}", message.Path, message.Id));
 114
 115                         Indexable indexable = new Indexable (uri);
 116                         indexable.MimeType = "text/html";
 117                         indexable.Type = "FeedItem";
 118
 119                         indexable.AddProperty (Property.New ("fixme:client", "mozilla"));
 120
 121                         indexable.AddProperty(Property.New ("dc:title", message.Subject));
 122                         indexable.AddProperty(Property.New ("fixme:author", message.From));
 123                         //indexable.AddProperty(Property.NewDate ("fixme:published", item.PubDate));
 124                         indexable.AddProperty(Property.NewUnsearched ("fixme:itemuri", message.Headers ["Content-Base"]));
 125
 126                         indexable.AddProperty (Property.New ("fixme:offset", message.Offset));
 127
 128                         StringReader reader = new StringReader (message.Body);
 129                         indexable.SetTextReader (reader);
 130
 131                         return indexable;
 132                 }
 133         }
 134
 135         public abstract class MozillaQueryable : LuceneQueryable {
 136
 137                 public static Logger log = Logger.Get ("mozilla");
 138
 139                 private MozillaCrawler crawler;
 140
 141                 private ArrayList accountTypes = new ArrayList ();
 142
 143                 public MozillaQueryable (string indexname) : base (indexname)
 144                 {
 145                 }
 146
 147
 148                 protected void AddAccountType (string str)
 149                 {
 150                         accountTypes.Add (str);
 151                 }
 152
 153                 private void StartWorker ()
 154                 {
 155                         Stopwatch stopwatch = new Stopwatch ();
 156                         stopwatch.Start ();
 157
 158                         Shutdown.ShutdownEvent += OnShutdown;
 159
 160                         this.crawler = new MozillaCrawler (this, this.Driver.Fingerprint);
 161
 162                         foreach (Profile profile in Profile.ReadProfiles ()) {
 163                                 foreach (Account account in profile.Accounts) {
 164                                         if (!accountTypes.Contains (account.Type))
 165                                                 continue;
 166
 167                                         this.crawler.ScheduleCrawl (new DirectoryInfo (account.Path), -1);
 168
 169                                         FileSystemWatcher fsw = new FileSystemWatcher ();
 170                                         fsw.Path = account.Path;
 171                                         fsw.IncludeSubdirectories = true;
 172
 173                                         fsw.Changed += new FileSystemEventHandler (OnChanged);
 174                                         fsw.Created += new FileSystemEventHandler (OnChanged);
 175                                         fsw.Deleted += new FileSystemEventHandler (OnChanged);
 176
 177                                         fsw.EnableRaisingEvents = true;
 178                                 }
 179                         }
 180
 181                         Shutdown.ShutdownEvent += OnShutdown;
 182
 183                         this.crawler.StopWhenEmpty ();
 184
 185                         stopwatch.Stop ();
 186                         Logger.Log.Info ("MozillaQueryable worker thread done in {0}",
 187                                          stopwatch);
 188                 }
 189
 190                 private void OnChanged (object source, FileSystemEventArgs args)
 191                 {
 192                         switch (args.ChangeType) {
 193                         case WatcherChangeTypes.Changed:
 194                         case WatcherChangeTypes.Created:
 195                                 if (File.Exists (args.FullPath + ".msf"))
 196                                         Index (new FileInfo (args.FullPath));
 197                                 break;
 198                         case WatcherChangeTypes.Deleted:
 199                                 // FIXME: Do
 200                                 break;
 201                         }
 202                 }
 203
 204                 public override void Start ()
 205                 {
 206                         base.Start ();
 207
 208                         ExceptionHandlingThread.Start (new ThreadStart (StartWorker));
 209                 }
 210
 211                 private void OnShutdown ()
 212                 {
 213                         this.crawler.Stop ();
 214                 }
 215
 216                 public string Name {
 217                         get { return "EvolutionMail"; }
 218                 }
 219
 220                 public void Index (FileInfo file)
 221                 {
 222                         Scheduler.TaskGroup group = NewMarkingTaskGroup (file.FullName, file.LastWriteTime);
 223
 224                         MessageReader reader = new MessageReader (file.FullName);
 225
 226                         while (reader.HasMoreMessages) {
 227                                 Message message = reader.NextMessage;
 228                                 Indexable indexable = MessageToIndexable (message);
 229
 230                                 Scheduler.Task task = NewAddTask (indexable);
 231                                 task.Priority = Scheduler.Priority.Delayed;
 232                                 task.SubPriority = 0;
 233                                 task.AddTaskGroup (group);
 234                                 ThisScheduler.Add (task);
 235                         }
 236
 237                 }
 238
 239                 protected abstract Indexable MessageToIndexable (Message message);
 240         }
 241 }