beagled/GaimLogQueryable/GaimLogQueryable.cs

   1 //
   2 // GaimLogQueryable.cs
   3 //
   4 // Copyright (C) 2004 Novell, Inc.
   5 //
   6
   7 //
   8 // Permission is hereby granted, free of charge, to any person obtaining a
   9 // copy of this software and associated documentation files (the "Software"),
  10 // to deal in the Software without restriction, including without limitation
  11 // the rights to use, copy, modify, merge, publish, distribute, sublicense,
  12 // and/or sell copies of the Software, and to permit persons to whom the
  13 // Software is furnished to do so, subject to the following conditions:
  14 //
  15 // The above copyright notice and this permission notice shall be included in
  16 // all copies or substantial portions of the Software.
  17 //
  18 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  21 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  22 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  23 // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  24 // DEALINGS IN THE SOFTWARE.
  25 //
  26
  27 using System;
  28 using System.Collections;
  29 using System.IO;
  30 using System.Text;
  31 using System.Threading;
  32
  33 using Beagle.Daemon;
  34 using Beagle.Util;
  35
  36 namespace Beagle.Daemon.GaimLogQueryable {
  37
  38         // FIXME: This should be rather renamed to Gaim to be compliant with other backend names
  39         [QueryableFlavor (Name="GaimLog", Domain=QueryDomain.Local, RequireInotify=false)]
  40         public class GaimLogQueryable : LuceneFileQueryable, IIndexableGenerator {
  41
  42                 private static Logger log = Logger.Get ("GaimLogQueryable");
  43
  44                 private string config_dir, log_dir, icons_dir;
  45
  46                 private int polling_interval_in_seconds = 60;
  47
  48                 private GaimBuddyListReader list = new GaimBuddyListReader ();
  49
  50                 public GaimLogQueryable () : base ("GaimLogIndex")
  51                 {
  52                         config_dir = Path.Combine (PathFinder.HomeDir, ".gaim");
  53                         log_dir = Path.Combine (config_dir, "logs");
  54                         icons_dir = Path.Combine (config_dir, "icons");
  55                 }
  56
  57                 /////////////////////////////////////////////////
  58
  59                 private void StartWorker()
  60                 {
  61                         if (! Directory.Exists (log_dir)) {
  62                                 GLib.Timeout.Add (60000, new GLib.TimeoutHandler (CheckForExistence));
  63                                 return;
  64                         }
  65
  66                         log.Info ("Starting Gaim log backend");
  67
  68                         Stopwatch stopwatch = new Stopwatch ();
  69                         stopwatch.Start ();
  70
  71                         if (Inotify.Enabled) {
  72                                 Log.Info ("Setting up inotify watches on gaim log directories");
  73                                 Crawl (false);
  74                         }
  75
  76                         Scheduler.Task task;
  77                         task = NewAddTask (this);
  78                         task.Tag = "Crawling Gaim logs";
  79                         task.Source = this;
  80
  81                         if (!Inotify.Enabled) {
  82                                 Scheduler.TaskGroup group = Scheduler.NewTaskGroup ("Repeating gaim log crawler", null, AddCrawlTask);
  83                                 task.AddTaskGroup (group);
  84                         }
  85
  86                         ThisScheduler.Add (task);
  87
  88                         stopwatch.Stop ();
  89
  90                         log.Info ("Gaim log backend worker thread done in {0}", stopwatch);
  91                 }
  92
  93                 public override void Start ()
  94                 {
  95                         base.Start ();
  96
  97                         ExceptionHandlingThread.Start (new ThreadStart (StartWorker));
  98                 }
  99
 100                 /////////////////////////////////////////////////
 101
 102                 private void AddCrawlTask ()
 103                 {
 104                         Scheduler.Task task = Scheduler.TaskFromHook (new Scheduler.TaskHook (CrawlHook));
 105                         task.Tag = "Crawling ~/.gaim/logs to find new logfiles";
 106                         task.Source = this;
 107                         ThisScheduler.Add (task);
 108                 }
 109
 110                 private void CrawlHook (Scheduler.Task task)
 111                 {
 112                         Crawl (true);
 113                         task.Reschedule = true;
 114                         task.TriggerTime = DateTime.Now.AddSeconds (polling_interval_in_seconds);
 115                 }
 116
 117                 private void Crawl (bool index)
 118                 {
 119                         if (Inotify.Enabled)
 120                                 Inotify.Subscribe (log_dir, OnInotifyNewProtocol, Inotify.EventType.Create);
 121
 122                         // Walk through protocol subdirs
 123                         foreach (string proto_dir in DirectoryWalker.GetDirectories (log_dir))
 124                                 CrawlProtocolDirectory (proto_dir, index);
 125                 }
 126
 127                 private void CrawlProtocolDirectory (string proto_dir, bool index)
 128                 {
 129                         if (Inotify.Enabled)
 130                                 Inotify.Subscribe (proto_dir, OnInotifyNewAccount, Inotify.EventType.Create);
 131
 132                         // Walk through accounts
 133                         foreach (string account_dir in DirectoryWalker.GetDirectories (proto_dir))
 134                                 CrawlAccountDirectory (account_dir, index);
 135                 }
 136
 137                 private void CrawlAccountDirectory (string account_dir, bool index)
 138                 {
 139                         if (Inotify.Enabled)
 140                                 Inotify.Subscribe (account_dir, OnInotifyNewRemote, Inotify.EventType.Create);
 141
 142                         // Walk through remote user conversations
 143                         foreach (string remote_dir in DirectoryWalker.GetDirectories (account_dir)) {
 144                                 if (remote_dir.IndexOf (".system") < 0)
 145                                         CrawlRemoteDirectory (remote_dir, index);
 146                         }
 147                 }
 148
 149                 private void CrawlRemoteDirectory (string remote_dir, bool index)
 150                 {
 151                         if (Inotify.Enabled)
 152                                 Inotify.Subscribe (remote_dir, OnInotifyNewConversation, Inotify.EventType.CloseWrite | Inotify.EventType.Modify);
 153
 154                         if (index) {
 155                                 foreach (FileInfo file in DirectoryWalker.GetFileInfos (remote_dir))
 156                                         if (FileIsInteresting (file.Name))
 157                                                 IndexLog (file.FullName, Scheduler.Priority.Delayed);
 158                         }
 159                 }
 160
 161                 /////////////////////////////////////////////////
 162
 163                 public string StatusName {
 164                         get { return "GaimLogQueryable"; }
 165                 }
 166
 167                 private IEnumerator log_files = null;
 168
 169                 public void PostFlushHook () { }
 170
 171                 public bool HasNextIndexable ()
 172                 {
 173                         if (log_files == null)
 174                                 log_files = DirectoryWalker.GetFileInfosRecursive (log_dir).GetEnumerator ();
 175
 176                         return log_files.MoveNext ();
 177                 }
 178
 179                 public Indexable GetNextIndexable ()
 180                 {
 181                         FileInfo file = (FileInfo) log_files.Current;
 182
 183                         if (! file.Exists)
 184                                 return null;
 185
 186                         if (IsUpToDate (file.FullName))
 187                                 return null;
 188
 189                         Indexable indexable = ImLogToIndexable (file.FullName);
 190
 191                         return indexable;
 192                 }
 193
 194                 /////////////////////////////////////////////////
 195
 196                 private bool CheckForExistence ()
 197                 {
 198                         if (!Directory.Exists (log_dir))
 199                                 return true;
 200
 201                         this.Start ();
 202
 203                         return false;
 204                 }
 205
 206                 private bool FileIsInteresting (string filename)
 207                 {
 208                         if (filename.Length < 21)
 209                                 return false;
 210
 211                         string ext = Path.GetExtension (filename);
 212                         if (ext != ".txt" && ext != ".html")
 213                                 return false;
 214
 215                         // Pre-gaim 2.0.0 logs are in the format "2005-07-22.161521.txt".  Afterward a
 216                         // timezone field as added, ie. "2005-07-22.161521-0500EST.txt".
 217                         //
 218                         // This is a lot uglier than a regexp, but they are so damn expensive.
 219
 220                         return Char.IsDigit (filename [0]) && Char.IsDigit (filename [1])
 221                                 && Char.IsDigit (filename [2]) && Char.IsDigit (filename [3])
 222                                 && filename [4] == '-'
 223                                 && Char.IsDigit (filename [5]) && Char.IsDigit (filename [6])
 224                                 && filename [7] == '-'
 225                                 && Char.IsDigit (filename [8]) && Char.IsDigit (filename [9])
 226                                 && filename [10] == '.'
 227                                 && Char.IsDigit (filename [11]) && Char.IsDigit (filename [12])
 228                                 && Char.IsDigit (filename [13]) && Char.IsDigit (filename [14])
 229                                 && Char.IsDigit (filename [15]) && Char.IsDigit (filename [16])
 230                                 && (filename [17] == '+' || filename [17] == '-' || filename [17] == '.');
 231                 }
 232
 233                 /////////////////////////////////////////////////
 234
 235                 private void OnInotifyNewProtocol (Inotify.Watch watch,
 236                                                 string path, string subitem, string srcpath,
 237                                                 Inotify.EventType type)
 238                 {
 239                         if (subitem.Length == 0 || (type & Inotify.EventType.IsDirectory) == 0)
 240                                 return;
 241
 242                         CrawlProtocolDirectory (Path.Combine (path, subitem), true);
 243                 }
 244
 245                 private void OnInotifyNewAccount (Inotify.Watch watch,
 246                                                 string path, string subitem, string srcpath,
 247                                                 Inotify.EventType type)
 248                 {
 249                         if (subitem.Length == 0 || (type & Inotify.EventType.IsDirectory) == 0)
 250                                 return;
 251
 252                         CrawlAccountDirectory (Path.Combine (path, subitem), true);
 253                 }
 254
 255                 private void OnInotifyNewRemote (Inotify.Watch watch,
 256                                                 string path, string subitem, string srcpath,
 257                                                 Inotify.EventType type)
 258                 {
 259                         if (subitem.Length == 0 || (type & Inotify.EventType.IsDirectory) == 0)
 260                                 return;
 261
 262                         CrawlRemoteDirectory (Path.Combine (path, subitem), true);
 263                 }
 264
 265                 private void OnInotifyNewConversation (Inotify.Watch watch,
 266                                                 string path, string subitem, string srcpath,
 267                                                 Inotify.EventType type)
 268                 {
 269                         if (subitem.Length == 0 || (type & Inotify.EventType.IsDirectory) != 0)
 270                                 return;
 271
 272                         if (FileIsInteresting (subitem))
 273                                 IndexLog (Path.Combine (path, subitem), Scheduler.Priority.Immediate);
 274                 }
 275
 276                 /////////////////////////////////////////////////
 277
 278                 private static Indexable ImLogToIndexable (string filename)
 279                 {
 280                         Uri uri = UriFu.PathToFileUri (filename);
 281                         Indexable indexable = new Indexable (uri);
 282                         indexable.ContentUri = uri;
 283                         indexable.Timestamp = File.GetLastWriteTimeUtc (filename);
 284                         indexable.MimeType = GaimLog.MimeType;
 285                         indexable.HitType = "IMLog";
 286                         indexable.CacheContent = false;
 287
 288                         return indexable;
 289                 }
 290
 291                 private void IndexLog (string filename, Scheduler.Priority priority)
 292                 {
 293                         if (! File.Exists (filename))
 294                                 return;
 295
 296                         if (IsUpToDate (filename))
 297                                 return;
 298
 299                         Indexable indexable = ImLogToIndexable (filename);
 300                         Scheduler.Task task = NewAddTask (indexable);
 301                         task.Priority = priority;
 302                         task.SubPriority = 0;
 303                         ThisScheduler.Add (task);
 304                 }
 305
 306                 override protected double RelevancyMultiplier (Hit hit)
 307                 {
 308                         return HalfLifeMultiplierFromProperty (hit, 0.25, "fixme:endtime", "fixme:starttime");
 309                 }
 310
 311                 override protected bool HitFilter (Hit hit)
 312                 {
 313                         // If the protocol isn't set (because maybe we got an
 314                         // exception while we were indexing), this isn't a
 315                         // valid hit.
 316                         if (hit ["fixme:protocol"] == null) {
 317                                 Log.Warn ("Discarding IM log hit with missing protocol info: {0}", hit.Uri);
 318                                 return false;
 319                         }
 320
 321                         string speakingto = hit ["fixme:speakingto"];
 322
 323                         // We have no idea who we're speaking to.  Bad, but we
 324                         // still want to present it.
 325                         if (speakingto == null || speakingto == String.Empty)
 326                                 return true;
 327
 328                         ImBuddy buddy = list.Search (speakingto);
 329
 330                         // We might still want to see a chat even if someone's
 331                         // not on our buddy list.
 332                         if (buddy == null)
 333                                 return true;
 334
 335                         if (buddy.Alias != "")
 336                                 hit.AddProperty (Beagle.Property.NewKeyword ("fixme:speakingto_alias", buddy.Alias));
 337
 338                         if (buddy.BuddyIconLocation != "")
 339                                 hit.AddProperty (Beagle.Property.NewUnsearched ("fixme:speakingto_icon", Path.Combine (icons_dir, buddy.BuddyIconLocation)));
 340
 341                         return true;
 342                 }
 343
 344                 override public string GetSnippet (string [] query_terms, Hit hit)
 345                 {
 346                         TextReader reader;
 347                         reader = TextCache.UserCache.GetReader (hit.Uri);
 348                         if (reader == null)
 349                                 return null;
 350                         HtmlRemovingReader html_removing_reader = new HtmlRemovingReader (reader);
 351                         string snippet = SnippetFu.GetSnippet (query_terms, html_removing_reader);
 352                         html_removing_reader.Close ();
 353
 354                         return snippet;
 355                 }
 356
 357         }
 358 }
 359