beagled/QueryDriver.cs

   1 //
   2 // QueryDriver.cs
   3 //
   4 // Copyright (C) 2004 Novell, Inc.
   5 //
   6
   7 //
   8 // Permission is hereby granted, free of charge, to any person obtaining a
   9 // copy of this software and associated documentation files (the "Software"),
  10 // to deal in the Software without restriction, including without limitation
  11 // the rights to use, copy, modify, merge, publish, distribute, sublicense,
  12 // and/or sell copies of the Software, and to permit persons to whom the
  13 // Software is furnished to do so, subject to the following conditions:
  14 //
  15 // The above copyright notice and this permission notice shall be included in
  16 // all copies or substantial portions of the Software.
  17 //
  18 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  21 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  22 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  23 // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  24 // DEALINGS IN THE SOFTWARE.
  25 //
  26
  27 using System;
  28 using System.IO;
  29 using System.Collections;
  30 using System.Reflection;
  31 using System.Text;
  32 using System.Threading;
  33 using Beagle.Util;
  34 namespace Beagle.Daemon {
  35
  36         public class QueryDriver {
  37
  38                 // Contains list of queryables explicitly asked by --allow-backend or --backend name
  39                 // --allow-backend/--backend name : dont read config information and only start backend 'name'
  40                 static ArrayList excl_allowed_queryables = new ArrayList ();
  41
  42                 // Contains list of denied queryables from config/arguments (every queryable is enabled by default)
  43                 // Unless overruled by --allow-backend/--backend name, deny backend only if names appears here.
  44                 static ArrayList denied_queryables = new ArrayList ();
  45
  46                 static bool to_read_conf = true; // read backends from conf if true
  47                 static bool done_reading_conf = false;
  48
  49                 static private void ReadBackendsFromConf ()
  50                 {
  51                         if (! to_read_conf || done_reading_conf)
  52                                 return;
  53
  54                         // set flag here to stop Allow() from calling ReadBackendsFromConf() again
  55                         done_reading_conf = true;
  56
  57                         // To allow static indexes, "static" should be in allowed_queryables
  58                         if (Conf.Daemon.AllowStaticBackend)
  59                                 Allow ("static");
  60
  61                         if (Conf.Daemon.DeniedBackends == null)
  62                                 return;
  63
  64                         foreach (string name in Conf.Daemon.DeniedBackends)
  65                                 denied_queryables.Add (name.ToLower ());
  66                 }
  67
  68                 static public void OnlyAllow (string name)
  69                 {
  70                         excl_allowed_queryables.Add (name.ToLower ());
  71                         to_read_conf = false;
  72                 }
  73
  74                 static public void Allow (string name)
  75                 {
  76                         if (! done_reading_conf && to_read_conf)
  77                                 ReadBackendsFromConf ();
  78
  79                         denied_queryables.Remove (name.ToLower ());
  80                 }
  81
  82                 static public void Deny (string name)
  83                 {
  84                         if (! done_reading_conf && to_read_conf)
  85                                 ReadBackendsFromConf ();
  86
  87                         name = name.ToLower ();
  88                         if (!denied_queryables.Contains (name))
  89                                 denied_queryables.Add (name);
  90                 }
  91
  92                 static private bool UseQueryable (string name)
  93                 {
  94                         name = name.ToLower ();
  95
  96                         if (excl_allowed_queryables.Contains (name))
  97                                 return true;
  98                         if (excl_allowed_queryables.Count != 0)
  99                                 return false;
 100
 101                         if (denied_queryables.Contains (name))
 102                                 return false;
 103
 104                         return true;
 105                 }
 106
 107                 //////////////////////////////////////////////////////////////////////////////////////
 108
 109                 // Paths to static queryables
 110
 111                 static ArrayList static_queryables = new ArrayList ();
 112
 113                 static public void AddStaticQueryable (string path) {
 114
 115                         if (! static_queryables.Contains (path))
 116                                 static_queryables.Add (path);
 117                 }
 118
 119                 //////////////////////////////////////////////////////////////////////////////////////
 120
 121                 // Delay before starting the indexing process
 122
 123                 static int indexing_delay = 60;  // Default to 60 seconds
 124
 125                 public static int IndexingDelay {
 126                         set { indexing_delay = value; }
 127                 }
 128
 129                 //////////////////////////////////////////////////////////////////////////////////////
 130
 131                 // Use introspection to find all classes that implement IQueryable, the construct
 132                 // associated Queryables objects.
 133
 134                 static ArrayList queryables = new ArrayList ();
 135                 static Hashtable iqueryable_to_queryable = new Hashtable ();
 136
 137                 static bool ThisApiSoVeryIsBroken (Type m, object criteria)
 138                 {
 139                         return m == (Type) criteria;
 140                 }
 141
 142                 static bool TypeImplementsInterface (Type t, Type iface)
 143                 {
 144                         Type[] impls = t.FindInterfaces (new TypeFilter (ThisApiSoVeryIsBroken),
 145                                                          iface);
 146                         return impls.Length > 0;
 147                 }
 148
 149                 // For every type in the assembly that
 150                 // (1) implements IQueryable
 151                 // (2) Has a QueryableFlavor attribute attached
 152                 // assemble a Queryable object and stick it into our list of queryables.
 153                 static void ScanAssembly (Assembly assembly)
 154                 {
 155                         int count = 0;
 156
 157                         foreach (Type type in ReflectionFu.ScanAssemblyForInterface (assembly, typeof (IQueryable))) {
 158                                 bool type_accepted = false;
 159                                 foreach (QueryableFlavor flavor in ReflectionFu.ScanTypeForAttribute (type, typeof (QueryableFlavor))) {
 160                                         if (! UseQueryable (flavor.Name))
 161                                                 continue;
 162
 163                                         if (flavor.RequireInotify && ! Inotify.Enabled) {
 164                                                 Logger.Log.Warn ("Can't start backend '{0}' without inotify", flavor.Name);
 165                                                 continue;
 166                                         }
 167
 168                                         if (flavor.RequireExtendedAttributes && ! ExtendedAttribute.Supported) {
 169                                                 Logger.Log.Warn ("Can't start backend '{0}' without extended attributes", flavor.Name);
 170                                                 continue;
 171                                         }
 172
 173                                         IQueryable iq = null;
 174                                         try {
 175                                                 iq = Activator.CreateInstance (type) as IQueryable;
 176                                         } catch (Exception e) {
 177                                                 Logger.Log.Error (e, "Caught exception while instantiating {0} backend", flavor.Name);
 178                                         }
 179
 180                                         if (iq != null) {
 181                                                 Queryable q = new Queryable (flavor, iq);
 182                                                 queryables.Add (q);
 183                                                 iqueryable_to_queryable [iq] = q;
 184                                                 ++count;
 185                                                 type_accepted = true;
 186                                                 break;
 187                                         }
 188                                 }
 189
 190                                 if (! type_accepted)
 191                                         continue;
 192
 193                                 object[] attributes = type.GetCustomAttributes (false);
 194                                 foreach (object attribute in attributes) {
 195                                         PropertyKeywordMapping mapping = attribute as PropertyKeywordMapping;
 196                                         if (mapping == null)
 197                                                 continue;
 198                                         //Logger.Log.Debug (mapping.Keyword + " => "
 199                                         //              + mapping.PropertyName +
 200                                         //              + " is-keyword=" + mapping.IsKeyword + " ("
 201                                         //              + mapping.Description + ") "
 202                                         //              + "(" + type.FullName + ")");
 203                                         PropertyKeywordFu.RegisterMapping (mapping);
 204                                 }
 205
 206                         }
 207                         Logger.Log.Debug ("Found {0} backends in {1}", count, assembly.Location);
 208                 }
 209
 210                 ////////////////////////////////////////////////////////
 211
 212                 public static void ReadKeywordMappings ()
 213                 {
 214                         Logger.Log.Debug ("Reading mapping from filters");
 215                         ArrayList assemblies = ReflectionFu.ScanEnvironmentForAssemblies ("BEAGLE_FILTER_PATH", PathFinder.FilterDir);
 216
 217                         foreach (Assembly assembly in assemblies) {
 218                                 foreach (Type type in assembly.GetTypes ())
 219                                         if (type.FullName.StartsWith ("Beagle.Filters")) {
 220
 221                                                 if (type.IsNestedPrivate ||
 222                                                     type.IsNestedPublic ||
 223                                                     type.IsNestedAssembly ||
 224                                                     type.IsNestedFamily)
 225                                                         continue;
 226
 227                                                 object[] attributes = type.GetCustomAttributes (false);
 228                                                 foreach (object attribute in attributes) {
 229
 230                                                         PropertyKeywordMapping mapping = attribute as PropertyKeywordMapping;
 231                                                         if (mapping == null)
 232                                                                 continue;
 233                                                         //Logger.Log.Debug (mapping.Keyword + " => "
 234                                                         //              + mapping.PropertyName
 235                                                         //              + " is-keyword=" + mapping.IsKeyword + " ("
 236                                                         //              + mapping.Description + ") "
 237                                                         //              + "(" + type.FullName + ")");
 238                                                         PropertyKeywordFu.RegisterMapping (mapping);
 239                                                 }
 240                                         }
 241                         }
 242                 }
 243
 244                 ////////////////////////////////////////////////////////
 245
 246                 // Scans PathFinder.SystemIndexesDir after available
 247                 // system-wide indexes.
 248                 static void LoadSystemIndexes ()
 249                 {
 250                         if (!Directory.Exists (PathFinder.SystemIndexesDir))
 251                                 return;
 252
 253                         Logger.Log.Info ("Loading system static indexes.");
 254
 255                         int count = 0;
 256
 257                         foreach (DirectoryInfo index_dir in new DirectoryInfo (PathFinder.SystemIndexesDir).GetDirectories ()) {
 258                                 if (! UseQueryable (index_dir.Name))
 259                                         continue;
 260
 261                                 if (LoadStaticQueryable (index_dir, QueryDomain.System))
 262                                         count++;
 263                         }
 264
 265                         Logger.Log.Info ("Found {0} system-wide indexes.", count);
 266                 }
 267
 268                 // Scans configuration for user-specified index paths
 269                 // to load StaticQueryables from.
 270                 static void LoadStaticQueryables ()
 271                 {
 272                         int count = 0;
 273
 274                         if (UseQueryable ("static")) {
 275                                 Logger.Log.Info ("Loading user-configured static indexes.");
 276                                 foreach (string path in Conf.Daemon.StaticQueryables)
 277                                         static_queryables.Add (path);
 278                         }
 279
 280                         foreach (string path in static_queryables) {
 281                                 DirectoryInfo index_dir = new DirectoryInfo (StringFu.SanitizePath (path));
 282
 283                                 if (!index_dir.Exists)
 284                                         continue;
 285
 286                                 // FIXME: QueryDomain might be other than local
 287                                 if (LoadStaticQueryable (index_dir, QueryDomain.Local))
 288                                         count++;
 289                         }
 290
 291                         Logger.Log.Info ("Found {0} user-configured static indexes..", count);
 292                 }
 293
 294                 // Instantiates and loads a StaticQueryable from an index directory
 295                 static private bool LoadStaticQueryable (DirectoryInfo index_dir, QueryDomain query_domain)
 296                 {
 297                         StaticQueryable static_queryable = null;
 298
 299                         if (!index_dir.Exists)
 300                                 return false;
 301
 302                         try {
 303                                 static_queryable = new StaticQueryable (index_dir.Name, index_dir.FullName, true);
 304                         } catch (InvalidOperationException) {
 305                                 Logger.Log.Warn ("Unable to create read-only index (likely due to index version mismatch): {0}", index_dir.FullName);
 306                                 return false;
 307                         } catch (Exception e) {
 308                                 Logger.Log.Error (e, "Caught exception while instantiating static queryable: {0}", index_dir.Name);
 309                                 return false;
 310                         }
 311
 312                         if (static_queryable != null) {
 313                                 QueryableFlavor flavor = new QueryableFlavor ();
 314                                 flavor.Name = index_dir.Name;
 315                                 flavor.Domain = query_domain;
 316
 317                                 Queryable queryable = new Queryable (flavor, static_queryable);
 318                                 queryables.Add (queryable);
 319
 320                                 iqueryable_to_queryable [static_queryable] = queryable;
 321
 322                                 return true;
 323                         }
 324
 325                         return false;
 326                 }
 327
 328                 ////////////////////////////////////////////////////////
 329
 330                 private static ArrayList assemblies = null;
 331
 332                 // Perform expensive initialization steps all at once.
 333                 // Should be done before SignalHandler comes into play.
 334                 static public void Init ()
 335                 {
 336                         ReadBackendsFromConf ();
 337                         assemblies = ReflectionFu.ScanEnvironmentForAssemblies ("BEAGLE_BACKEND_PATH", PathFinder.BackendDir);
 338                 }
 339
 340                 static public void Start ()
 341                 {
 342                         // Only add the executing assembly if we haven't already loaded it.
 343                         if (assemblies.IndexOf (Assembly.GetExecutingAssembly ()) == -1)
 344                                 assemblies.Add (Assembly.GetExecutingAssembly ());
 345
 346                         foreach (Assembly assembly in assemblies) {
 347                                 ScanAssembly (assembly);
 348
 349                                 // This allows backends to define their
 350                                 // own executors.
 351                                 Server.ScanAssemblyForExecutors (assembly);
 352                         }
 353
 354                         ReadKeywordMappings ();
 355
 356                         LoadSystemIndexes ();
 357                         LoadStaticQueryables ();
 358
 359                         if (indexing_delay <= 0 || Environment.GetEnvironmentVariable ("BEAGLE_EXERCISE_THE_DOG") != null)
 360                                 StartQueryables ();
 361                         else {
 362                                 Logger.Log.Debug ("Waiting {0} seconds before starting queryables", indexing_delay);
 363                                 GLib.Timeout.Add ((uint) indexing_delay * 1000, new GLib.TimeoutHandler (StartQueryables));
 364                         }
 365                 }
 366
 367                 static private bool StartQueryables ()
 368                 {
 369                         Logger.Log.Debug ("Starting queryables");
 370
 371                         foreach (Queryable q in queryables) {
 372                                 Logger.Log.Info ("Starting backend: '{0}'", q.Name);
 373                                 q.Start ();
 374                         }
 375
 376                         return false;
 377                 }
 378
 379                 static public string ListBackends ()
 380                 {
 381                         ArrayList assemblies = ReflectionFu.ScanEnvironmentForAssemblies ("BEAGLE_BACKEND_PATH", PathFinder.BackendDir);
 382
 383                         // Only add the executing assembly if we haven't already loaded it.
 384                         if (assemblies.IndexOf (Assembly.GetExecutingAssembly ()) == -1)
 385                                 assemblies.Add (Assembly.GetExecutingAssembly ());
 386
 387                         string ret = "User:\n";
 388
 389                         foreach (Assembly assembly in assemblies) {
 390                                 foreach (Type type in ReflectionFu.ScanAssemblyForInterface (assembly, typeof (IQueryable))) {
 391                                         foreach (QueryableFlavor flavor in ReflectionFu.ScanTypeForAttribute (type, typeof (QueryableFlavor)))
 392                                                 ret += String.Format (" - {0}\n", flavor.Name);
 393                                 }
 394                         }
 395
 396                         if (!Directory.Exists (PathFinder.SystemIndexesDir))
 397                                 return ret;
 398
 399                         ret += "System:\n";
 400                         foreach (DirectoryInfo index_dir in new DirectoryInfo (PathFinder.SystemIndexesDir).GetDirectories ()) {
 401                                 ret += String.Format (" - {0}\n", index_dir.Name);
 402                         }
 403
 404                         return ret;
 405                 }
 406
 407                 static public Queryable GetQueryable (string name)
 408                 {
 409                         foreach (Queryable q in queryables) {
 410                                 if (q.Name == name)
 411                                         return q;
 412                         }
 413
 414                         return null;
 415                 }
 416
 417                 static public Queryable GetQueryable (IQueryable iqueryable)
 418                 {
 419                         return (Queryable) iqueryable_to_queryable [iqueryable];
 420                 }
 421
 422                 ////////////////////////////////////////////////////////
 423
 424                 public delegate void ChangedHandler (Queryable            queryable,
 425                                                      IQueryableChangeData changeData);
 426
 427                 static public event ChangedHandler ChangedEvent;
 428
 429                 // A method to fire the ChangedEvent event.
 430                 static public void QueryableChanged (IQueryable           iqueryable,
 431                                                      IQueryableChangeData change_data)
 432                 {
 433                         if (ChangedEvent != null) {
 434                                 Queryable queryable = iqueryable_to_queryable [iqueryable] as Queryable;
 435                                 ChangedEvent (queryable, change_data);
 436                         }
 437                 }
 438
 439                 ////////////////////////////////////////////////////////
 440
 441                 private class QueryClosure : IQueryWorker {
 442
 443                         Queryable queryable;
 444                         Query query;
 445                         IQueryResult result;
 446                         IQueryableChangeData change_data;
 447
 448                         public QueryClosure (Queryable            queryable,
 449                                              Query                query,
 450                                              QueryResult          result,
 451                                              IQueryableChangeData change_data)
 452                         {
 453                                 this.queryable = queryable;
 454                                 this.query = query;
 455                                 this.result = result;
 456                                 this.change_data = change_data;
 457                         }
 458
 459                         public void DoWork ()
 460                         {
 461                                 queryable.DoQuery (query, result, change_data);
 462                         }
 463                 }
 464
 465                 static public void DoOneQuery (Queryable            queryable,
 466                                                Query                query,
 467                                                QueryResult          result,
 468                                                IQueryableChangeData change_data)
 469                 {
 470                         if (queryable.AcceptQuery (query)) {
 471                                 QueryClosure qc = new QueryClosure (queryable, query, result, change_data);
 472                                 result.AttachWorker (qc);
 473                         }
 474                 }
 475
 476                 static void AddSearchTermInfo (QueryPart          part,
 477                                                SearchTermResponse response, StringBuilder sb)
 478                 {
 479                         if (part.Logic == QueryPartLogic.Prohibited)
 480                                 return;
 481
 482                         if (part is QueryPart_Or) {
 483                                 ICollection sub_parts;
 484                                 sub_parts = ((QueryPart_Or) part).SubParts;
 485                                 foreach (QueryPart qp in sub_parts)
 486                                         AddSearchTermInfo (qp, response, sb);
 487                                 return;
 488                         }
 489
 490                         if (! (part is QueryPart_Text))
 491                                 return;
 492
 493                         QueryPart_Text tp;
 494                         tp = (QueryPart_Text) part;
 495
 496                         string [] split;
 497                         split = tp.Text.Split (' ');
 498
 499                         // First, remove stop words
 500                         for (int i = 0; i < split.Length; ++i)
 501                                 if (LuceneCommon.IsStopWord (split [i]))
 502                                         split [i] = null;
 503
 504                         // Assemble the phrase minus stop words
 505                         sb.Length = 0;
 506                         for (int i = 0; i < split.Length; ++i) {
 507                                 if (split [i] == null)
 508                                         continue;
 509                                 if (sb.Length > 0)
 510                                         sb.Append (' ');
 511                                 sb.Append (split [i]);
 512                         }
 513                         response.ExactText.Add (sb.ToString ());
 514
 515                         // Now assemble a stemmed version
 516                         sb.Length = 0; // clear the previous value
 517                         for (int i = 0; i < split.Length; ++i) {
 518                                 if (split [i] == null)
 519                                         continue;
 520                                 if (sb.Length > 0)
 521                                         sb.Append (' ');
 522                                 sb.Append (LuceneCommon.Stem (split [i]));
 523                         }
 524                         response.StemmedText.Add (sb.ToString ());
 525                 }
 526
 527                 ////////////////////////////////////////////////////////
 528
 529                 static private void DehumanizeQuery (Query query)
 530                 {
 531                         // We need to remap any QueryPart_Human parts into
 532                         // lower-level part types.  First, we find any
 533                         // QueryPart_Human parts and explode them into
 534                         // lower-level types.
 535                         ArrayList new_parts = null;
 536                         foreach (QueryPart abstract_part in query.Parts) {
 537                                 if (abstract_part is QueryPart_Human) {
 538                                         QueryPart_Human human = abstract_part as QueryPart_Human;
 539                                         if (new_parts == null)
 540                                                 new_parts = new ArrayList ();
 541                                         foreach (QueryPart sub_part in QueryStringParser.Parse (human.QueryString))
 542                                                 new_parts.Add (sub_part);
 543                                 }
 544                         }
 545
 546                         // If we found any QueryPart_Human parts, copy the
 547                         // non-Human parts over and then replace the parts in
 548                         // the query.
 549                         if (new_parts != null) {
 550                                 foreach (QueryPart abstract_part in query.Parts) {
 551                                         if (! (abstract_part is QueryPart_Human))
 552                                                 new_parts.Add (abstract_part);
 553                                 }
 554
 555                                 query.ClearParts ();
 556                                 foreach (QueryPart part in new_parts)
 557                                         query.AddPart (part);
 558                         }
 559
 560                 }
 561
 562                 static private SearchTermResponse AssembleSearchTermResponse (Query query)
 563                 {
 564                         StringBuilder sb = new StringBuilder ();
 565                         SearchTermResponse search_term_response;
 566                         search_term_response = new SearchTermResponse ();
 567                         foreach (QueryPart part in query.Parts)
 568                                 AddSearchTermInfo (part, search_term_response, sb);
 569                         return search_term_response;
 570                 }
 571
 572                 static private void QueryEachQueryable (Query       query,
 573                                                         QueryResult result)
 574                 {
 575                         // The extra pair of calls to WorkerStart/WorkerFinished ensures:
 576                         // (1) that the QueryResult will fire the StartedEvent
 577                         // and FinishedEvent, even if no queryable accepts the
 578                         // query.
 579                         // (2) that the FinishedEvent will only get called when all of the
 580                         // backends have had time to finish.
 581
 582                         object dummy_worker = new object ();
 583
 584                         if (! result.WorkerStart (dummy_worker))
 585                                 return;
 586
 587                         foreach (Queryable queryable in queryables)
 588                                 DoOneQuery (queryable, query, result, null);
 589
 590                         result.WorkerFinished (dummy_worker);
 591                 }
 592
 593                 static public void DoQueryLocal (Query       query,
 594                                                  QueryResult result)
 595                 {
 596                         DehumanizeQuery (query);
 597
 598                         SearchTermResponse search_term_response;
 599                         search_term_response = AssembleSearchTermResponse (query);
 600                         query.ProcessSearchTermResponse (search_term_response);
 601
 602                         QueryEachQueryable (query, result);
 603                 }
 604
 605                 static public void DoQuery (Query                                query,
 606                                             QueryResult                          result,
 607                                             RequestMessageExecutor.AsyncResponse send_response)
 608                 {
 609                         DehumanizeQuery (query);
 610
 611                         SearchTermResponse search_term_response;
 612                         search_term_response = AssembleSearchTermResponse (query);
 613                         send_response (search_term_response);
 614
 615                         QueryEachQueryable (query, result);
 616                 }
 617
 618                 ////////////////////////////////////////////////////////
 619
 620                 static public IEnumerable GetIndexInformation ()
 621                 {
 622                         foreach (Queryable q in queryables)
 623                                 yield return q.GetQueryableStatus ();
 624                 }
 625
 626                 ////////////////////////////////////////////////////////
 627
 628                 static public bool IsIndexing {
 629                         get {
 630                                 foreach (Queryable q in queryables) {
 631                                         QueryableStatus status = q.GetQueryableStatus ();
 632
 633                                         if (status.IsIndexing)
 634                                                 return true;
 635                                 }
 636
 637                                 return false;
 638                         }
 639                 }
 640         }
 641 }