beagled/IndexingServiceQueryable/IndexingServiceQueryable.cs

   1 //
   2 // IndexingServiceQueryable.cs
   3 //
   4 // Copyright (C) 2005 Novell, Inc.
   5 //
   6
   7 //
   8 // Permission is hereby granted, free of charge, to any person obtaining a
   9 // copy of this software and associated documentation files (the "Software"),
  10 // to deal in the Software without restriction, including without limitation
  11 // the rights to use, copy, modify, merge, publish, distribute, sublicense,
  12 // and/or sell copies of the Software, and to permit persons to whom the
  13 // Software is furnished to do so, subject to the following conditions:
  14 //
  15 // The above copyright notice and this permission notice shall be included in
  16 // all copies or substantial portions of the Software.
  17 //
  18 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  21 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  22 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  23 // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  24 // DEALINGS IN THE SOFTWARE.
  25 //
  26
  27 // The IndexingService has two modes of operation: one is through the standard
  28 // message-passing system and one where a slightly-structured file is dropped
  29 // into a known location on the filesystem.
  30 //
  31 // (1) Messaging: An IndexingServiceRequest message is sent containing URIs of
  32 // items to remove and Indexables to add.  This is more reliable, and is best
  33 // for clients which will also be utilizing Beagle for searching.
  34 //
  35 // (2) Files: The file to be indexed is dropped into the ~/.beagle/ToIndex
  36 // directory.  Another file with the same name prepended with a period is
  37 // also dropped into the directory.  In that file is the metadata for the
  38 // file being indexed.  The first line is the URI of the data being indexed.
  39 // The second line is the hit type.  The third line is the mime type.  Then
  40 // there are zero or more properties in the form "type:key=value", where
  41 // "type" is either 't' for text or 'k' for keyword.  This method is a lot
  42 // easier to use, but requires that Beagle have inotify support enabled to
  43 // work.
  44
  45 using System;
  46 using System.Collections;
  47 using System.IO;
  48 using System.Threading;
  49
  50 using Beagle.Daemon;
  51 using Beagle.Util;
  52
  53 namespace Beagle.Daemon.IndexingServiceQueryable {
  54
  55         [QueryableFlavor (Name="IndexingService", Domain=QueryDomain.Local, RequireInotify=false)]
  56         public class IndexingServiceQueryable : LuceneQueryable {
  57
  58                 public IndexingServiceQueryable () : base ("IndexingServiceIndex")
  59                 {
  60                         Server.RegisterRequestMessageHandler (typeof (IndexingServiceRequest), new Server.RequestMessageHandler (HandleMessage));
  61                 }
  62
  63                 public override void Start ()
  64                 {
  65                         base.Start ();
  66
  67                         ExceptionHandlingThread.Start (new ThreadStart (StartWorker));
  68                 }
  69
  70                 private void StartWorker ()
  71                 {
  72                         string index_path = Path.Combine (PathFinder.StorageDir, "ToIndex");
  73
  74                         if (!Directory.Exists (index_path))
  75                                 Directory.CreateDirectory (index_path);
  76
  77                         if (Inotify.Enabled)
  78                                 Inotify.Subscribe (index_path, OnInotifyEvent, Inotify.EventType.CloseWrite);
  79
  80                         Logger.Log.Info ("Setting up an initial crawl of the IndexingService directory");
  81
  82                         IndexableGenerator generator = new IndexableGenerator (GetIndexables (index_path));
  83                         Scheduler.Task task = NewAddTask (generator);
  84                         task.Tag = "IndexingService initial crawl";
  85                         ThisScheduler.Add (task);
  86                 }
  87
  88                 private IEnumerable GetIndexables (string path)
  89                 {
  90                         foreach (FileInfo file in DirectoryWalker.GetFileInfos (path)) {
  91                                 if (file.Name [0] == '.')
  92                                         continue;
  93
  94                                 if (File.Exists (Path.Combine (file.DirectoryName, "." + file.Name)))
  95                                         yield return FileToIndexable (file);
  96                         }
  97
  98                         yield break;
  99                 }
 100
 101                 private Indexable FileToIndexable (FileInfo data_file)
 102                 {
 103                         FileInfo meta_file = new FileInfo (Path.Combine (data_file.DirectoryName, "." + data_file.Name));
 104                         FileStream meta_stream;
 105
 106                         try {
 107                                 meta_stream = meta_file.Open (FileMode.Open, FileAccess.Read, FileShare.Read);
 108                         } catch (FileNotFoundException) {
 109                                 // The meta file disappeared before we could
 110                                 // open it.
 111                                 return null;
 112                         }
 113
 114                         StreamReader reader = new StreamReader (meta_stream);
 115
 116                         // First line of the file is a URI
 117                         string line = reader.ReadLine ();
 118                         Uri uri;
 119
 120                         try {
 121                                 uri = new Uri (line);
 122                         } catch (Exception e) {
 123                                 Logger.Log.Warn (e, "IndexingService: Unable to parse URI in {0}:", meta_file.FullName);
 124                                 meta_stream.Close ();
 125                                 return null;
 126                         }
 127
 128                         Indexable indexable = new Indexable (uri);
 129                         indexable.Timestamp = data_file.LastWriteTimeUtc;
 130                         indexable.ContentUri = UriFu.PathToFileUri (data_file.FullName);
 131                         indexable.DeleteContent = true;
 132
 133                         // Second line is the hit type
 134                         line = reader.ReadLine ();
 135                         if (line == null) {
 136                                 Logger.Log.Warn ("IndexingService: EOF reached trying to read hit type from {0}",
 137                                                  meta_file.FullName);
 138                                 meta_stream.Close ();
 139                                 return null;
 140                         } else if (line != String.Empty)
 141                                 indexable.HitType = line;
 142
 143                         // Third line is the mime type
 144                         line = reader.ReadLine ();
 145                         if (line == null) {
 146                                 Logger.Log.Warn ("IndexingService: EOF reached trying to read mime type from {0}",
 147                                                  meta_file.FullName);
 148                                 meta_stream.Close ();
 149                                 return null;
 150                         } else if (line != String.Empty)
 151                                 indexable.MimeType = line;
 152
 153                         // Following lines are properties in "t:key=value" format
 154                         do {
 155                                 line = reader.ReadLine ();
 156
 157                                 if (line != null && line != String.Empty) {
 158                                         bool keyword = false;
 159
 160                                         if (line[0] == 'k')
 161                                                 keyword = true;
 162                                         else if (line[0] != 't') {
 163                                                 Logger.Log.Warn ("IndexingService: Unknown property type: '{0}'", line[0]);
 164                                                 continue;
 165                                         }
 166
 167                                         int i = line.IndexOf ('=');
 168
 169                                         if (i == -1) {
 170                                                 Logger.Log.Warn ("IndexingService: Unknown property line: '{0}'", line);
 171                                                 continue;
 172                                         }
 173
 174                                         // FIXME: We should probably handle date types
 175                                         if (keyword) {
 176                                                 indexable.AddProperty (Property.NewUnsearched (line.Substring (2, i - 2),
 177                                                                                             line.Substring (i + 1)));
 178                                         } else {
 179                                                 indexable.AddProperty (Property.New (line.Substring (2, i - 2),
 180                                                                                      line.Substring (i + 1)));
 181                                         }
 182                                 }
 183                         } while (line != null);
 184
 185                         indexable.LocalState ["MetaFile"] = meta_file;
 186
 187                         // Ok, we're finished with the meta file.  It will be
 188                         // deleted in PostAddHook ().
 189                         meta_stream.Close ();
 190
 191                         return indexable;
 192                 }
 193
 194                 // Bleh, we need to keep around a list of pending items to be
 195                 // indexed so that we don't actually index it twice because
 196                 // the order of the creation of the data file and meta file
 197                 // isn't defined.
 198                 private ArrayList pending_files = new ArrayList ();
 199
 200                 private void OnInotifyEvent (Inotify.Watch watch,
 201                                              string path,
 202                                              string subitem,
 203                                              string srcpath,
 204                                              Inotify.EventType type)
 205                 {
 206                         if (subitem == "")
 207                                 return;
 208
 209                         if (subitem[0] == '.') {
 210                                 string data_file = Path.Combine (path, subitem.Substring (1));
 211
 212                                 lock (pending_files) {
 213                                         if (File.Exists (data_file) && ! pending_files.Contains (data_file)) {
 214                                                 pending_files.Add (data_file);
 215                                                 IndexFile (new FileInfo (data_file));
 216                                         }
 217                                 }
 218                         } else {
 219                                 string meta_file = Path.Combine (path, "." + subitem);
 220                                 string data_file = Path.Combine (path, subitem);
 221
 222                                 lock (pending_files) {
 223                                         if (File.Exists (meta_file) && ! pending_files.Contains (data_file)) {
 224                                                 pending_files.Add (data_file);
 225                                                 IndexFile (new FileInfo (data_file));
 226                                         }
 227                                 }
 228                         }
 229                 }
 230
 231                 private void IndexFile (FileInfo data_file)
 232                 {
 233                         Indexable indexable = FileToIndexable (data_file);
 234
 235                         if (indexable == null) // The file disappeared
 236                                 return;
 237
 238                         Scheduler.Task task = NewAddTask (indexable);
 239                         task.Priority = Scheduler.Priority.Immediate;
 240                         ThisScheduler.Add (task);
 241                 }
 242
 243                 protected override void PostAddHook (Indexable indexable, IndexerAddedReceipt receipt)
 244                 {
 245                         FileInfo meta_file = (FileInfo) indexable.LocalState ["MetaFile"];
 246                         meta_file.Delete ();
 247
 248                         lock (pending_files)
 249                                 pending_files.Remove (indexable.ContentUri.LocalPath);
 250                 }
 251
 252                 private class IndexableGenerator : IIndexableGenerator {
 253                         private IEnumerator to_add_enumerator;
 254                         private int count = -1, done_count = 0;
 255
 256                         public IndexableGenerator (IEnumerable to_add)
 257                         {
 258                                 this.to_add_enumerator = to_add.GetEnumerator ();
 259                         }
 260
 261                         public IndexableGenerator (ICollection to_add) : this ((IEnumerable) to_add)
 262                         {
 263                                 this.count = to_add.Count;
 264                         }
 265
 266                         public Indexable GetNextIndexable ()
 267                         {
 268                                 return to_add_enumerator.Current as Indexable;
 269                         }
 270
 271                         public bool HasNextIndexable ()
 272                         {
 273                                 ++done_count;
 274                                 return to_add_enumerator.MoveNext ();
 275                         }
 276
 277                         public string StatusName {
 278                                 get {
 279                                         if (count == -1)
 280                                                 return String.Format ("IndexingService: {0}", done_count);
 281                                         else
 282                                                 return String.Format ("IndexingService: {0} of {1}", done_count, count);
 283                                 }
 284                         }
 285
 286                         public void PostFlushHook ()
 287                         { }
 288                 }
 289
 290                 private ResponseMessage HandleMessage (RequestMessage msg)
 291                 {
 292                         IndexingServiceRequest isr = (IndexingServiceRequest) msg;
 293
 294                         foreach (Uri uri in isr.ToRemove) {
 295                                 Scheduler.Task task = NewRemoveTask (uri);
 296                                 ThisScheduler.Add (task);
 297                         }
 298
 299                         // FIXME: There should be a way for the request to control the
 300                         // scheduler priority of the task.
 301
 302                         if (isr.ToAdd.Count > 0) {
 303                                 IIndexableGenerator ind_gen = new IndexableGenerator (isr.ToAdd);
 304                                 Scheduler.Task task = NewAddTask (ind_gen);
 305                                 task.Priority = Scheduler.Priority.Immediate;
 306                                 ThisScheduler.Add (task);
 307                         }
 308
 309                         // FIXME: There should be an asynchronous response  (fired by a Scheduler.Hook)
 310                         // that fires when all of the items have been added to the index.
 311
 312                         // No response
 313                         return new EmptyResponse ();
 314                 }
 315
 316         }
 317
 318 }