beagled/GaimLogQueryable/GaimLogQueryable.cs

   1 //
   2 // GaimLogQueryable.cs
   3 //
   4 // Copyright (C) 2004 Novell, Inc.
   5 //
   6
   7 //
   8 // Permission is hereby granted, free of charge, to any person obtaining a
   9 // copy of this software and associated documentation files (the "Software"),
  10 // to deal in the Software without restriction, including without limitation
  11 // the rights to use, copy, modify, merge, publish, distribute, sublicense,
  12 // and/or sell copies of the Software, and to permit persons to whom the
  13 // Software is furnished to do so, subject to the following conditions:
  14 //
  15 // The above copyright notice and this permission notice shall be included in
  16 // all copies or substantial portions of the Software.
  17 //
  18 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  21 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  22 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  23 // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  24 // DEALINGS IN THE SOFTWARE.
  25 //
  26
  27 using System;
  28 using System.Collections;
  29 using System.IO;
  30 using System.Text;
  31 using System.Threading;
  32
  33 using Beagle.Daemon;
  34 using Beagle.Util;
  35
  36 namespace Beagle.Daemon.GaimLogQueryable {
  37
  38         // FIXME: This should be rather renamed to Gaim to be compliant with other backend names
  39         [QueryableFlavor (Name="GaimLog", Domain=QueryDomain.Local, RequireInotify=false)]
  40         public class GaimLogQueryable : LuceneFileQueryable, IIndexableGenerator {
  41
  42                 private string config_dir, log_dir, icons_dir;
  43
  44                 private int polling_interval_in_seconds = 60;
  45
  46                 private bool crawling = false;
  47
  48                 private GaimBuddyListReader list = new GaimBuddyListReader ();
  49
  50                 public GaimLogQueryable () : base ("GaimLogIndex")
  51                 {
  52                         config_dir = Path.Combine (PathFinder.HomeDir, ".gaim");
  53                         log_dir = Path.Combine (config_dir, "logs");
  54                         icons_dir = Path.Combine (config_dir, "icons");
  55                 }
  56
  57                 /////////////////////////////////////////////////
  58
  59                 private void StartWorker()
  60                 {
  61                         if (! Directory.Exists (log_dir)) {
  62                                 GLib.Timeout.Add (60000, new GLib.TimeoutHandler (CheckForExistence));
  63                                 return;
  64                         }
  65
  66                         Log.Info ("Starting Gaim log backend");
  67
  68                         Stopwatch stopwatch = new Stopwatch ();
  69                         stopwatch.Start ();
  70
  71                         if (Inotify.Enabled) {
  72                                 Log.Info ("Setting up inotify watches on gaim log directories");
  73                                 Crawl (false);
  74                         }
  75
  76                         Scheduler.Task task;
  77                         task = NewAddTask (this);
  78                         task.Tag = "Crawling Gaim logs";
  79                         task.Source = this;
  80
  81                         if (!Inotify.Enabled) {
  82                                 Scheduler.TaskGroup group = Scheduler.NewTaskGroup ("Repeating gaim log crawler", null, AddCrawlTask);
  83                                 task.AddTaskGroup (group);
  84                         }
  85
  86                         ThisScheduler.Add (task);
  87
  88                         stopwatch.Stop ();
  89
  90                         Log.Info ("Gaim log backend worker thread done in {0}", stopwatch);
  91                 }
  92
  93                 public override void Start ()
  94                 {
  95                         base.Start ();
  96
  97                         ExceptionHandlingThread.Start (new ThreadStart (StartWorker));
  98                 }
  99
 100                 protected override bool IsIndexing {
 101                         get { return crawling; }
 102                 }
 103
 104                 /////////////////////////////////////////////////
 105
 106                 private void AddCrawlTask ()
 107                 {
 108                         Scheduler.Task task = Scheduler.TaskFromHook (new Scheduler.TaskHook (CrawlHook));
 109                         task.Tag = "Crawling ~/.gaim/logs to find new logfiles";
 110                         task.Source = this;
 111                         ThisScheduler.Add (task);
 112                 }
 113
 114                 private void CrawlHook (Scheduler.Task task)
 115                 {
 116                         Crawl (true);
 117                         task.Reschedule = true;
 118                         task.TriggerTime = DateTime.Now.AddSeconds (polling_interval_in_seconds);
 119                 }
 120
 121                 private void Crawl (bool index)
 122                 {
 123                         this.crawling = true;
 124
 125                         if (Inotify.Enabled)
 126                                 Inotify.Subscribe (log_dir, OnInotifyNewProtocol, Inotify.EventType.Create);
 127
 128                         // Walk through protocol subdirs
 129                         foreach (string proto_dir in DirectoryWalker.GetDirectories (log_dir))
 130                                 CrawlProtocolDirectory (proto_dir, index);
 131                 }
 132
 133                 private void CrawlProtocolDirectory (string proto_dir, bool index)
 134                 {
 135                         if (Inotify.Enabled)
 136                                 Inotify.Subscribe (proto_dir, OnInotifyNewAccount, Inotify.EventType.Create);
 137
 138                         // Walk through accounts
 139                         foreach (string account_dir in DirectoryWalker.GetDirectories (proto_dir))
 140                                 CrawlAccountDirectory (account_dir, index);
 141                 }
 142
 143                 private void CrawlAccountDirectory (string account_dir, bool index)
 144                 {
 145                         if (Inotify.Enabled)
 146                                 Inotify.Subscribe (account_dir, OnInotifyNewRemote, Inotify.EventType.Create);
 147
 148                         // Walk through remote user conversations
 149                         foreach (string remote_dir in DirectoryWalker.GetDirectories (account_dir)) {
 150                                 if (remote_dir.IndexOf (".system") < 0)
 151                                         CrawlRemoteDirectory (remote_dir, index);
 152                         }
 153                 }
 154
 155                 private void CrawlRemoteDirectory (string remote_dir, bool index)
 156                 {
 157                         if (Inotify.Enabled)
 158                                 Inotify.Subscribe (remote_dir, OnInotifyNewConversation, Inotify.EventType.CloseWrite | Inotify.EventType.Modify);
 159
 160                         if (index) {
 161                                 foreach (FileInfo file in DirectoryWalker.GetFileInfos (remote_dir))
 162                                         if (FileIsInteresting (file.Name))
 163                                                 IndexLog (file.FullName, Scheduler.Priority.Delayed);
 164
 165                                 crawling = false;
 166                         }
 167                 }
 168
 169                 /////////////////////////////////////////////////
 170
 171                 public string StatusName {
 172                         get { return "GaimLogQueryable"; }
 173                 }
 174
 175                 private IEnumerator log_files = null;
 176
 177                 public void PostFlushHook () { }
 178
 179                 public bool HasNextIndexable ()
 180                 {
 181                         if (log_files == null)
 182                                 log_files = DirectoryWalker.GetFileInfosRecursive (log_dir).GetEnumerator ();
 183
 184                         if (log_files.MoveNext ())
 185                                 return true;
 186                         else {
 187                                 crawling = false;
 188                                 return false;
 189                         }
 190                 }
 191
 192                 public Indexable GetNextIndexable ()
 193                 {
 194                         FileInfo file = (FileInfo) log_files.Current;
 195
 196                         if (! file.Exists)
 197                                 return null;
 198
 199                         if (IsUpToDate (file.FullName))
 200                                 return null;
 201
 202                         Indexable indexable = ImLogToIndexable (file.FullName);
 203
 204                         return indexable;
 205                 }
 206
 207                 /////////////////////////////////////////////////
 208
 209                 private bool CheckForExistence ()
 210                 {
 211                         if (!Directory.Exists (log_dir))
 212                                 return true;
 213
 214                         this.Start ();
 215
 216                         return false;
 217                 }
 218
 219                 private bool FileIsInteresting (string filename)
 220                 {
 221                         if (filename.Length < 21)
 222                                 return false;
 223
 224                         string ext = Path.GetExtension (filename);
 225                         if (ext != ".txt" && ext != ".html")
 226                                 return false;
 227
 228                         // Pre-gaim 2.0.0 logs are in the format "2005-07-22.161521.txt".  Afterward a
 229                         // timezone field as added, ie. "2005-07-22.161521-0500EST.txt".
 230                         //
 231                         // This is a lot uglier than a regexp, but they are so damn expensive.
 232
 233                         return Char.IsDigit (filename [0]) && Char.IsDigit (filename [1])
 234                                 && Char.IsDigit (filename [2]) && Char.IsDigit (filename [3])
 235                                 && filename [4] == '-'
 236                                 && Char.IsDigit (filename [5]) && Char.IsDigit (filename [6])
 237                                 && filename [7] == '-'
 238                                 && Char.IsDigit (filename [8]) && Char.IsDigit (filename [9])
 239                                 && filename [10] == '.'
 240                                 && Char.IsDigit (filename [11]) && Char.IsDigit (filename [12])
 241                                 && Char.IsDigit (filename [13]) && Char.IsDigit (filename [14])
 242                                 && Char.IsDigit (filename [15]) && Char.IsDigit (filename [16])
 243                                 && (filename [17] == '+' || filename [17] == '-' || filename [17] == '.');
 244                 }
 245
 246                 /////////////////////////////////////////////////
 247
 248                 private void OnInotifyNewProtocol (Inotify.Watch watch,
 249                                                 string path, string subitem, string srcpath,
 250                                                 Inotify.EventType type)
 251                 {
 252                         if (subitem.Length == 0 || (type & Inotify.EventType.IsDirectory) == 0)
 253                                 return;
 254
 255                         CrawlProtocolDirectory (Path.Combine (path, subitem), true);
 256                 }
 257
 258                 private void OnInotifyNewAccount (Inotify.Watch watch,
 259                                                 string path, string subitem, string srcpath,
 260                                                 Inotify.EventType type)
 261                 {
 262                         if (subitem.Length == 0 || (type & Inotify.EventType.IsDirectory) == 0)
 263                                 return;
 264
 265                         CrawlAccountDirectory (Path.Combine (path, subitem), true);
 266                 }
 267
 268                 private void OnInotifyNewRemote (Inotify.Watch watch,
 269                                                 string path, string subitem, string srcpath,
 270                                                 Inotify.EventType type)
 271                 {
 272                         if (subitem.Length == 0 || (type & Inotify.EventType.IsDirectory) == 0)
 273                                 return;
 274
 275                         CrawlRemoteDirectory (Path.Combine (path, subitem), true);
 276                 }
 277
 278                 private void OnInotifyNewConversation (Inotify.Watch watch,
 279                                                 string path, string subitem, string srcpath,
 280                                                 Inotify.EventType type)
 281                 {
 282                         if (subitem.Length == 0 || (type & Inotify.EventType.IsDirectory) != 0)
 283                                 return;
 284
 285                         if (FileIsInteresting (subitem))
 286                                 IndexLog (Path.Combine (path, subitem), Scheduler.Priority.Immediate);
 287                 }
 288
 289                 /////////////////////////////////////////////////
 290
 291                 private static Indexable ImLogToIndexable (string filename)
 292                 {
 293                         Uri uri = UriFu.PathToFileUri (filename);
 294                         Indexable indexable = new Indexable (uri);
 295                         indexable.ContentUri = uri;
 296                         indexable.Timestamp = File.GetLastWriteTimeUtc (filename);
 297                         indexable.MimeType = GaimLog.MimeType;
 298                         indexable.HitType = "IMLog";
 299                         indexable.CacheContent = false;
 300
 301                         return indexable;
 302                 }
 303
 304                 private void IndexLog (string filename, Scheduler.Priority priority)
 305                 {
 306                         if (! File.Exists (filename))
 307                                 return;
 308
 309                         if (IsUpToDate (filename))
 310                                 return;
 311
 312                         Indexable indexable = ImLogToIndexable (filename);
 313                         Scheduler.Task task = NewAddTask (indexable);
 314                         task.Priority = priority;
 315                         task.SubPriority = 0;
 316                         ThisScheduler.Add (task);
 317                 }
 318
 319                 override protected double RelevancyMultiplier (Hit hit)
 320                 {
 321                         return HalfLifeMultiplierFromProperty (hit, 0.25, "fixme:endtime", "fixme:starttime");
 322                 }
 323
 324                 override protected bool HitFilter (Hit hit)
 325                 {
 326                         // If the protocol isn't set (because maybe we got an
 327                         // exception while we were indexing), this isn't a
 328                         // valid hit.
 329                         if (hit ["fixme:protocol"] == null) {
 330                                 Log.Warn ("Discarding IM log hit with missing protocol info: {0}", hit.Uri);
 331                                 return false;
 332                         }
 333
 334                         string speakingto = hit ["fixme:speakingto"];
 335
 336                         // We have no idea who we're speaking to.  Bad, but we
 337                         // still want to present it.
 338                         if (speakingto == null || speakingto == String.Empty)
 339                                 return true;
 340
 341                         ImBuddy buddy = list.Search (speakingto);
 342
 343                         // We might still want to see a chat even if someone's
 344                         // not on our buddy list.
 345                         if (buddy == null)
 346                                 return true;
 347
 348                         if (buddy.Alias != "")
 349                                 hit.AddProperty (Beagle.Property.NewKeyword ("fixme:speakingto_alias", buddy.Alias));
 350
 351                         if (buddy.BuddyIconLocation != "")
 352                                 hit.AddProperty (Beagle.Property.NewUnsearched ("fixme:speakingto_icon", Path.Combine (icons_dir, buddy.BuddyIconLocation)));
 353
 354                         return true;
 355                 }
 356
 357                 override public string GetSnippet (string [] query_terms, Hit hit)
 358                 {
 359                         TextReader reader;
 360                         reader = TextCache.UserCache.GetReader (hit.Uri);
 361                         if (reader == null)
 362                                 return null;
 363                         HtmlRemovingReader html_removing_reader = new HtmlRemovingReader (reader);
 364                         string snippet = SnippetFu.GetSnippet (query_terms, html_removing_reader);
 365                         html_removing_reader.Close ();
 366
 367                         return snippet;
 368                 }
 369
 370         }
 371 }
 372