beagled/BuildIndex.cs

   1 //
   2 // BuildIndex.cs
   3 //
   4 // Copyright (C) 2005 Novell, Inc.
   5 //
   6
   7 //
   8 // Permission is hereby granted, free of charge, to any person obtaining a copy
   9 // of this software and associated documentation files (the "Software"), to deal
  10 // in the Software without restriction, including without limitation the rights
  11 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  12 // copies of the Software, and to permit persons to whom the Software is
  13 // furnished to do so, subject to the following conditions:
  14 //
  15 // The above copyright notice and this permission notice shall be included in all
  16 // copies or substantial portions of the Software.
  17 //
  18 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  21 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  22 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  23 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  24 // SOFTWARE.
  25 //
  26
  27 using System;
  28 using System.IO;
  29 using System.Net;
  30 using System.Threading;
  31 using System.Collections;
  32
  33 using System.Xml;
  34 using System.Xml.Serialization;
  35
  36 using Beagle;
  37 using Beagle.Util;
  38 using FSQ = Beagle.Daemon.FileSystemQueryable.FileSystemQueryable;
  39
  40 namespace Beagle.Daemon
  41 {
  42         class BuildIndex
  43         {
  44                 static bool arg_recursive = false, arg_debug = false, arg_cache_text = false, arg_disable_filtering = false;
  45
  46                 static Hashtable remap_table = new Hashtable ();
  47
  48                 static string arg_output, arg_tag;
  49
  50                 /////////////////////////////////////////////////////////
  51
  52                 static FileAttributesStore_Sqlite backing_fa_store;
  53                 static FileAttributesStore fa_store;
  54
  55                 static LuceneIndexingDriver driver;
  56
  57                 static bool crawling = true, shutdown = false;
  58
  59                 static ArrayList allowed_patterns = new ArrayList ();
  60                 static ArrayList denied_patterns = new ArrayList ();
  61
  62                 static Queue pending_files = new Queue ();
  63                 static Queue pending_directories = new Queue ();
  64
  65                 const int BATCH_SIZE = 30;
  66
  67                 /////////////////////////////////////////////////////////
  68
  69                 static void Main (string [] args)
  70                 {
  71                         if (args.Length < 2)
  72                                 PrintUsage ();
  73
  74                         int i = 0;
  75                         while (i < args.Length) {
  76
  77                                 string arg = args [i];
  78                                 ++i;
  79                                 string next_arg = i < args.Length ? args [i] : null;
  80
  81                                 switch (arg) {
  82                                 case "-h":
  83                                 case "--help":
  84                                         PrintUsage ();
  85                                         break;
  86
  87                                 case "--tag":
  88                                         if (next_arg != null)
  89                                                 arg_tag = next_arg;
  90                                         ++i;
  91                                         break;
  92
  93                                 case "-r":
  94                                 case "--recursive":
  95                                         arg_recursive = true;
  96                                         break;
  97
  98                                 case "--enable-text-cache":
  99                                         arg_cache_text = true;
 100                                         break;
 101
 102                                 case "--remap":
 103                                         if (next_arg == null)
 104                                                 break;
 105
 106                                         int j = next_arg.IndexOf (":");
 107
 108                                         if (j == -1) {
 109                                                 Console.WriteLine ("Invalid remap argument: {0}", next_arg);
 110                                                 Environment.Exit (1);
 111                                         }
 112
 113                                         remap_table [next_arg.Substring (0, j)] = next_arg.Substring (j+1);
 114
 115                                         ++i;
 116                                         break;
 117
 118                                 case "--target":
 119                                         if (next_arg != null)
 120                                                 arg_output = Path.IsPathRooted (next_arg) ? next_arg : Path.GetFullPath (next_arg);
 121                                         ++i;
 122                                         break;
 123
 124                                 case "--disable-filtering":
 125                                         arg_disable_filtering = true;
 126                                         break;
 127
 128                                 case "--allow-pattern":
 129                                         if (next_arg == null)
 130                                                 break;
 131
 132                                         if (next_arg.IndexOf (',') != -1) {
 133                                                 foreach (string pattern in next_arg.Split (','))
 134                                                         allowed_patterns.Add (new ExcludeItem (ExcludeType.Pattern, pattern));
 135
 136                                         } else {
 137                                                 allowed_patterns.Add (new ExcludeItem (ExcludeType.Pattern, next_arg));
 138                                         }
 139
 140                                         ++i;
 141                                         break;
 142
 143                                 case "--deny-pattern":
 144                                         if (next_arg == null)
 145                                                 break;
 146
 147                                         if (next_arg.IndexOf (',') != -1) {
 148                                                 foreach (string pattern in next_arg.Split (','))
 149                                                         denied_patterns.Add (new ExcludeItem (ExcludeType.Pattern, pattern));
 150
 151                                         } else {
 152                                                 denied_patterns.Add (new ExcludeItem (ExcludeType.Pattern, next_arg));
 153                                         }
 154
 155                                         ++i;
 156                                         break;
 157
 158                                 default:
 159                                         string path = Path.IsPathRooted (arg) ? arg : Path.GetFullPath (arg);
 160
 161                                         if (Directory.Exists (path))
 162                                                         pending_directories.Enqueue (new DirectoryInfo (path));
 163                                         else if (File.Exists (path))
 164                                                 pending_files.Enqueue (new FileInfo (path));
 165                                         break;
 166                                 }
 167                         }
 168
 169                         /////////////////////////////////////////////////////////
 170
 171                         if (!Directory.Exists (Path.GetDirectoryName (arg_output))) {
 172                                 Console.WriteLine ("Index directory not available for construction: {0}", arg_output);
 173                                 Environment.Exit (1);
 174                         }
 175
 176                         driver = new LuceneIndexingDriver (arg_output);
 177                         driver.TextCache = (arg_cache_text) ? new TextCache (arg_output) : null;
 178
 179                         backing_fa_store = new FileAttributesStore_Sqlite (driver.TopDirectory, driver.Fingerprint);
 180                         fa_store = new FileAttributesStore (backing_fa_store);
 181
 182                         // Set up signal handlers
 183                         SetupSignalHandlers ();
 184
 185                         // Start the thread that does the crawling
 186                         ExceptionHandlingThread.Start (new ThreadStart (CrawlWorker));
 187
 188                         // Start the thread that does the actual indexing
 189                         ExceptionHandlingThread.Start (new ThreadStart (IndexWorker));
 190                 }
 191
 192                 /////////////////////////////////////////////////////////////////
 193
 194                 static void CrawlWorker ()
 195                 {
 196                         Logger.Log.Debug ("Starting CrawlWorker");
 197
 198
 199                         int count_dirs = 0;
 200
 201                         while (pending_directories.Count > 0) {
 202                                 DirectoryInfo dir = (DirectoryInfo) pending_directories.Dequeue ();
 203
 204                                 try {
 205                                         if (arg_recursive)
 206                                                 foreach (DirectoryInfo subdir in DirectoryWalker.GetDirectoryInfos (dir))
 207                                                         if (!Ignore (subdir))
 208                                                                 pending_directories.Enqueue (subdir);
 209
 210                                         foreach (FileInfo file in DirectoryWalker.GetFileInfos (dir))
 211                                                 if (!Ignore (file))
 212                                                         pending_files.Enqueue (file);
 213
 214                                 } catch (DirectoryNotFoundException e) {}
 215
 216                                 if (shutdown)
 217                                         break;
 218
 219                                 count_dirs++;
 220                         }
 221
 222                         Logger.Log.Debug ("Scanned {0} files in {1} directories", pending_files.Count, count_dirs);
 223                         Logger.Log.Debug ("CrawlWorker Done");
 224
 225                         crawling = false;
 226                 }
 227
 228                 /////////////////////////////////////////////////////////////////
 229
 230                 static IndexerReceipt [] FlushIndexer (IIndexer indexer)
 231                 {
 232                         IndexerReceipt [] receipts;
 233                         receipts = indexer.FlushAndBlock ();
 234
 235
 236                         foreach (IndexerReceipt raw_r in receipts) {
 237
 238                                 if (raw_r is IndexerAddedReceipt) {
 239                                         // Update the file attributes
 240                                         IndexerAddedReceipt r = (IndexerAddedReceipt) raw_r;
 241
 242                                         string path = r.Uri.LocalPath;
 243
 244                                         FileAttributes attr;
 245                                         attr = fa_store.ReadOrCreate (path);
 246
 247                                         attr.LastWriteTime = FileSystem.GetLastWriteTime (path);
 248                                         attr.FilterName = r.FilterName;
 249                                         attr.FilterVersion = r.FilterVersion;
 250
 251                                         fa_store.Write (attr);
 252
 253                                 } else if (raw_r is IndexerChildIndexablesReceipt) {
 254                                         // Add any child indexables back into our indexer
 255                                         IndexerChildIndexablesReceipt r = (IndexerChildIndexablesReceipt) raw_r;
 256                                         foreach (Indexable i in r.Children)
 257                                                 indexer.Add (i);
 258                                 }
 259                         }
 260
 261                         return receipts;
 262                 }
 263
 264                 static void IndexWorker ()
 265                 {
 266                         Logger.Log.Debug ("Starting IndexWorker");
 267
 268                         Indexable indexable;
 269                         int pending_adds = 0;
 270
 271                         while (!shutdown) {
 272                                 if (pending_files.Count > 0) {
 273                                         FileInfo file = (FileInfo) pending_files.Dequeue ();
 274                                         Uri uri = UriFu.PathToFileUri (file.FullName);
 275
 276                                         // Check that we really should be indexing the file
 277                                         if (!file.Exists || Ignore (file) || fa_store.IsUpToDate (file.FullName))
 278                                                 continue;
 279
 280                                         // Create the indexable and add the standard properties we
 281                                         // use in the FileSystemQueryable.
 282                                         indexable = new Indexable (uri);
 283                                         FSQ.AddStandardPropertiesToIndexable (indexable, file.Name, Guid.Empty, false);
 284
 285                                         // Disable filtering and only index file attributes
 286                                         if (arg_disable_filtering)
 287                                                 indexable.Filtering = IndexableFiltering.Never;
 288
 289                                         // Tag the item for easy identification (for say, removal)
 290                                         if (arg_tag != null)
 291                                                 indexable.AddProperty (Property.NewKeyword("Tag", arg_tag));
 292
 293                                         driver.Add (indexable);
 294                                         ++pending_adds;
 295
 296                                         if (pending_adds % BATCH_SIZE == 0) {
 297                                                 Logger.Log.Debug ("Flushing driver, {0} items in queue", pending_files.Count);
 298                                                 FlushIndexer (driver);
 299                                                 pending_adds = 0;
 300                                         }
 301                                 } else if (crawling) {
 302                                         //Logger.Log.Debug ("IndexWorker: La la la...");
 303                                         Thread.Sleep (50);
 304                                 } else {
 305                                         break;
 306                                 }
 307                         }
 308
 309                         // Call Flush one last time.
 310                         // This should be a totally safe no-op if there are no pending operations.
 311                         // FIXME: This is incorrect.  We will drop any children in the final flush.
 312                         FlushIndexer (driver);
 313
 314                         backing_fa_store.Flush ();
 315
 316                         Logger.Log.Debug ("IndexWorker Done");
 317                 }
 318
 319                 /////////////////////////////////////////////////////////////////
 320
 321                 // From BeagleDaemon.cs
 322
 323                 // The integer values of the Mono.Posix.Signal enumeration don't actually
 324                 // match the Linux signal numbers of Linux.  Oops!
 325                 // This is fixed in Mono.Unix, but for the moment we want to maintain
 326                 // compatibility with mono 1.0.x.
 327                 const int ACTUAL_LINUX_SIGINT  = 2;
 328                 const int ACTUAL_LINUX_SIGQUIT = 3;
 329                 const int ACTUAL_LINUX_SIGTERM = 15;
 330
 331                 static void SetupSignalHandlers ()
 332                 {
 333                         // Force OurSignalHandler to be JITed
 334                         OurSignalHandler (-1);
 335
 336                         // Set up our signal handler
 337                         Mono.Posix.Syscall.sighandler_t sig_handler;
 338                         sig_handler = new Mono.Posix.Syscall.sighandler_t (OurSignalHandler);
 339                         Mono.Posix.Syscall.signal (ACTUAL_LINUX_SIGINT, sig_handler);
 340                         Mono.Posix.Syscall.signal (ACTUAL_LINUX_SIGQUIT, sig_handler);
 341                         Mono.Posix.Syscall.signal (ACTUAL_LINUX_SIGTERM, sig_handler);
 342                 }
 343
 344                 static void OurSignalHandler (int signal)
 345                 {
 346                         // This allows us to call OurSignalHandler w/o doing anything.
 347                         // We want to call it once to ensure that it is pre-JITed.
 348                         if (signal < 0)
 349                                 return;
 350
 351                         Logger.Log.Debug ("Shutdown Requested");
 352                         shutdown = true;
 353                 }
 354
 355                 /////////////////////////////////////////////////////////////////
 356
 357                 static void PrintUsage ()
 358                 {
 359                         string usage =
 360                                 "beagle-build-index: Build an index.\n" +
 361                                 "Web page: http://www.gnome.org/projects/beagle\n" +
 362                                 "Copyright (C) 2005 Novell, Inc.\n\n";
 363
 364                         usage +=
 365                                 "Usage: beagle-build-index [OPTIONS] --target <index_path> <path> [path]\n\n" +
 366                                 "Options:\n" +
 367                                 "  --remap [path1:path2]\t\tRemap data paths to fit target. \n" +
 368                                 "  --tag [tag]\t\t\tTag index data for identification.\n" +
 369                                 "  --recursive\t\t\tCrawl source path recursivly.\n" +
 370                                 "  --enable-text-cache\t\tBuild text-cache of documents used for snippets.\n" +
 371                                 "  --disable-filtering\t\tDisable all filtering of files. Only index attributes.\n" +
 372                                 "  --allow-pattern [pattern]\tOnly allow files that match the pattern to be indexed.\n" +
 373                                 "  --deny-pattern [pattern]\tKeep any files that match the pattern from being indexed.\n" +
 374                                 "  --debug\t\t\tEcho verbose debugging information.\n";
 375
 376                         Console.WriteLine (usage);
 377                         Environment.Exit (0);
 378                 }
 379
 380                 /////////////////////////////////////////////////////////
 381
 382                 static Uri RemapUri (Uri uri)
 383                 {
 384                         // FIXME: This is ghetto
 385                         foreach (DictionaryEntry dict in remap_table) {
 386                                 if (uri.LocalPath.IndexOf ((string) dict.Key) == -1)
 387                                         continue;
 388                                 return new Uri (uri.LocalPath.Replace ((string) dict.Key, (string) dict.Value));
 389                         }
 390                         return uri;
 391                 }
 392
 393                 static bool Ignore (DirectoryInfo directory)
 394                 {
 395                         if (directory.Name.StartsWith ("."))
 396                                 return true;
 397
 398                         return false;
 399                 }
 400
 401                 static bool Ignore (FileInfo file)
 402                 {
 403                         if (file.Name.StartsWith ("."))
 404                                 return true;
 405
 406                         if (FileSystem.IsSymLink (file.FullName))
 407                                 return true;
 408
 409                         if (allowed_patterns.Count > 0) {
 410                                 foreach (ExcludeItem pattern in allowed_patterns)
 411                                         if (pattern.IsMatch (file.Name))
 412                                                 return false;
 413
 414                                 return true;
 415                         }
 416
 417                         foreach (ExcludeItem pattern in denied_patterns)
 418                                 if (pattern.IsMatch (file.Name))
 419                                         return true;
 420
 421                         // FIXME: Add more stuff here
 422
 423                         return false;
 424                 }
 425         }
 426 }