beagled/QueryDriver.cs

   1 //
   2 // QueryDriver.cs
   3 //
   4 // Copyright (C) 2004 Novell, Inc.
   5 //
   6
   7 //
   8 // Permission is hereby granted, free of charge, to any person obtaining a
   9 // copy of this software and associated documentation files (the "Software"),
  10 // to deal in the Software without restriction, including without limitation
  11 // the rights to use, copy, modify, merge, publish, distribute, sublicense,
  12 // and/or sell copies of the Software, and to permit persons to whom the
  13 // Software is furnished to do so, subject to the following conditions:
  14 //
  15 // The above copyright notice and this permission notice shall be included in
  16 // all copies or substantial portions of the Software.
  17 //
  18 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  21 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  22 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  23 // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  24 // DEALINGS IN THE SOFTWARE.
  25 //
  26
  27 using System;
  28 using System.IO;
  29 using System.Collections;
  30 using System.Reflection;
  31 using System.Text;
  32 using System.Threading;
  33 using Beagle.Util;
  34 namespace Beagle.Daemon {
  35
  36         public class QueryDriver {
  37
  38                 // Contains list of queryables explicitly asked by --allow-backend or --backend name
  39                 // --allow-backend/--backend name : dont read config information and only start backend 'name'
  40                 static ArrayList excl_allowed_queryables = new ArrayList ();
  41
  42                 // Contains list of denied queryables from config/arguments (every queryable is enabled by default)
  43                 // Unless overruled by --allow-backend/--backend name, deny backend only if names appears here.
  44                 static ArrayList denied_queryables = new ArrayList ();
  45
  46                 static bool to_read_conf = true; // read backends from conf if true
  47                 static bool done_reading_conf = false;
  48
  49                 static private void ReadBackendsFromConf ()
  50                 {
  51                         if (! to_read_conf || done_reading_conf)
  52                                 return;
  53
  54                         // set flag here to stop Allow() from calling ReadBackendsFromConf() again
  55                         done_reading_conf = true;
  56
  57                         // To allow static indexes, "static" should be in allowed_queryables
  58                         if (Conf.Daemon.AllowStaticBackend)
  59                                 Allow ("static");
  60
  61                         if (Conf.Daemon.DeniedBackends == null)
  62                                 return;
  63
  64                         foreach (string name in Conf.Daemon.DeniedBackends)
  65                                 denied_queryables.Add (name.ToLower ());
  66                 }
  67
  68                 static public void OnlyAllow (string name)
  69                 {
  70                         excl_allowed_queryables.Add (name.ToLower ());
  71                         to_read_conf = false;
  72                 }
  73
  74                 static public void Allow (string name)
  75                 {
  76                         if (! done_reading_conf && to_read_conf)
  77                                 ReadBackendsFromConf ();
  78
  79                         denied_queryables.Remove (name.ToLower ());
  80                 }
  81
  82                 static public void Deny (string name)
  83                 {
  84                         if (! done_reading_conf && to_read_conf)
  85                                 ReadBackendsFromConf ();
  86
  87                         name = name.ToLower ();
  88                         if (!denied_queryables.Contains (name))
  89                                 denied_queryables.Add (name);
  90                 }
  91
  92                 static private bool UseQueryable (string name)
  93                 {
  94                         name = name.ToLower ();
  95
  96                         if (excl_allowed_queryables.Contains (name))
  97                                 return true;
  98                         if (excl_allowed_queryables.Count != 0)
  99                                 return false;
 100
 101                         if (denied_queryables.Contains (name))
 102                                 return false;
 103
 104                         return true;
 105                 }
 106
 107                 //////////////////////////////////////////////////////////////////////////////////////
 108
 109                 // Paths to static queryables
 110
 111                 static ArrayList static_queryables = new ArrayList ();
 112
 113                 static public void AddStaticQueryable (string path) {
 114
 115                         if (! static_queryables.Contains (path))
 116                                 static_queryables.Add (path);
 117                 }
 118
 119                 //////////////////////////////////////////////////////////////////////////////////////
 120
 121                 // Delay before starting the indexing process
 122
 123                 static int indexing_delay = 60;  // Default to 60 seconds
 124
 125                 public static int IndexingDelay {
 126                         set { indexing_delay = value; }
 127                 }
 128
 129                 //////////////////////////////////////////////////////////////////////////////////////
 130
 131                 // Use introspection to find all classes that implement IQueryable, the construct
 132                 // associated Queryables objects.
 133
 134                 static ArrayList queryables = new ArrayList ();
 135                 static Hashtable iqueryable_to_queryable = new Hashtable ();
 136
 137                 static bool ThisApiSoVeryIsBroken (Type m, object criteria)
 138                 {
 139                         return m == (Type) criteria;
 140                 }
 141
 142                 static bool TypeImplementsInterface (Type t, Type iface)
 143                 {
 144                         Type[] impls = t.FindInterfaces (new TypeFilter (ThisApiSoVeryIsBroken),
 145                                                          iface);
 146                         return impls.Length > 0;
 147                 }
 148
 149                 // For every type in the assembly that
 150                 // (1) implements IQueryable
 151                 // (2) Has a QueryableFlavor attribute attached
 152                 // assemble a Queryable object and stick it into our list of queryables.
 153                 static void ScanAssembly (Assembly assembly)
 154                 {
 155                         int count = 0;
 156
 157                         foreach (Type type in ReflectionFu.ScanAssemblyForInterface (assembly, typeof (IQueryable))) {
 158                                 bool type_accepted = false;
 159                                 foreach (QueryableFlavor flavor in ReflectionFu.ScanTypeForAttribute (type, typeof (QueryableFlavor))) {
 160                                         if (! UseQueryable (flavor.Name))
 161                                                 continue;
 162
 163                                         if (flavor.RequireInotify && ! Inotify.Enabled) {
 164                                                 Logger.Log.Warn ("Can't start backend '{0}' without inotify", flavor.Name);
 165                                                 continue;
 166                                         }
 167
 168                                         if (flavor.RequireExtendedAttributes && ! ExtendedAttribute.Supported) {
 169                                                 Logger.Log.Warn ("Can't start backend '{0}' without extended attributes", flavor.Name);
 170                                                 continue;
 171                                         }
 172
 173                                         IQueryable iq = null;
 174                                         try {
 175                                                 iq = Activator.CreateInstance (type) as IQueryable;
 176                                         } catch (Exception e) {
 177                                                 Logger.Log.Error ("Caught exception while instantiating {0} backend", flavor.Name);
 178                                                 Logger.Log.Error (e);
 179                                         }
 180
 181                                         if (iq != null) {
 182                                                 Queryable q = new Queryable (flavor, iq);
 183                                                 queryables.Add (q);
 184                                                 iqueryable_to_queryable [iq] = q;
 185                                                 ++count;
 186                                                 type_accepted = true;
 187                                                 break;
 188                                         }
 189                                 }
 190
 191                                 if (! type_accepted)
 192                                         continue;
 193
 194                                 object[] attributes = type.GetCustomAttributes (false);
 195                                 foreach (object attribute in attributes) {
 196                                         PropertyKeywordMapping mapping = attribute as PropertyKeywordMapping;
 197                                         if (mapping == null)
 198                                                 continue;
 199                                         //Logger.Log.Debug (mapping.Keyword + " => "
 200                                         //              + mapping.PropertyName +
 201                                         //              + " is-keyword=" + mapping.IsKeyword + " ("
 202                                         //              + mapping.Description + ") "
 203                                         //              + "(" + type.FullName + ")");
 204                                         PropertyKeywordFu.RegisterMapping (mapping);
 205                                 }
 206
 207                         }
 208                         Logger.Log.Debug ("Found {0} backends in {1}", count, assembly.Location);
 209                 }
 210
 211                 ////////////////////////////////////////////////////////
 212
 213                 public static void ReadKeywordMappings ()
 214                 {
 215                         Logger.Log.Debug ("Reading mapping from filters");
 216                         ArrayList assemblies = ReflectionFu.ScanEnvironmentForAssemblies ("BEAGLE_FILTER_PATH", PathFinder.FilterDir);
 217
 218                         foreach (Assembly assembly in assemblies) {
 219                                 foreach (Type type in assembly.GetTypes ())
 220                                         if (type.FullName.StartsWith ("Beagle.Filters")) {
 221
 222                                                 if (type.IsNestedPrivate ||
 223                                                     type.IsNestedPublic ||
 224                                                     type.IsNestedAssembly ||
 225                                                     type.IsNestedFamily)
 226                                                         continue;
 227
 228                                                 object[] attributes = type.GetCustomAttributes (false);
 229                                                 foreach (object attribute in attributes) {
 230
 231                                                         PropertyKeywordMapping mapping = attribute as PropertyKeywordMapping;
 232                                                         if (mapping == null)
 233                                                                 continue;
 234                                                         //Logger.Log.Debug (mapping.Keyword + " => "
 235                                                         //              + mapping.PropertyName
 236                                                         //              + " is-keyword=" + mapping.IsKeyword + " ("
 237                                                         //              + mapping.Description + ") "
 238                                                         //              + "(" + type.FullName + ")");
 239                                                         PropertyKeywordFu.RegisterMapping (mapping);
 240                                                 }
 241                                         }
 242                         }
 243                 }
 244
 245                 ////////////////////////////////////////////////////////
 246
 247                 // Scans PathFinder.SystemIndexesDir after available
 248                 // system-wide indexes.
 249                 static void LoadSystemIndexes ()
 250                 {
 251                         if (!Directory.Exists (PathFinder.SystemIndexesDir))
 252                                 return;
 253
 254                         Logger.Log.Info ("Loading system static indexes.");
 255
 256                         int count = 0;
 257
 258                         foreach (DirectoryInfo index_dir in new DirectoryInfo (PathFinder.SystemIndexesDir).GetDirectories ()) {
 259                                 if (! UseQueryable (index_dir.Name))
 260                                         continue;
 261
 262                                 if (LoadStaticQueryable (index_dir, QueryDomain.System))
 263                                         count++;
 264                         }
 265
 266                         Logger.Log.Info ("Found {0} system-wide indexes.", count);
 267                 }
 268
 269                 // Scans configuration for user-specified index paths
 270                 // to load StaticQueryables from.
 271                 static void LoadStaticQueryables ()
 272                 {
 273                         int count = 0;
 274
 275                         if (UseQueryable ("static")) {
 276                                 Logger.Log.Info ("Loading user-configured static indexes.");
 277                                 foreach (string path in Conf.Daemon.StaticQueryables)
 278                                         static_queryables.Add (path);
 279                         }
 280
 281                         foreach (string path in static_queryables) {
 282                                 DirectoryInfo index_dir = new DirectoryInfo (StringFu.SanitizePath (path));
 283
 284                                 if (!index_dir.Exists)
 285                                         continue;
 286
 287                                 // FIXME: QueryDomain might be other than local
 288                                 if (LoadStaticQueryable (index_dir, QueryDomain.Local))
 289                                         count++;
 290                         }
 291
 292                         Logger.Log.Info ("Found {0} user-configured static indexes..", count);
 293                 }
 294
 295                 // Instantiates and loads a StaticQueryable from an index directory
 296                 static private bool LoadStaticQueryable (DirectoryInfo index_dir, QueryDomain query_domain)
 297                 {
 298                         StaticQueryable static_queryable = null;
 299
 300                         if (!index_dir.Exists)
 301                                 return false;
 302
 303                         try {
 304                                 static_queryable = new StaticQueryable (index_dir.Name, index_dir.FullName, true);
 305                         } catch (InvalidOperationException) {
 306                                 Logger.Log.Warn ("Unable to create read-only index (likely due to index version mismatch): {0}", index_dir.FullName);
 307                                 return false;
 308                         } catch (Exception e) {
 309                                 Logger.Log.Error ("Caught exception while instantiating static queryable: {0}", index_dir.Name);
 310                                 Logger.Log.Error (e);
 311                                 return false;
 312                         }
 313
 314                         if (static_queryable != null) {
 315                                 QueryableFlavor flavor = new QueryableFlavor ();
 316                                 flavor.Name = index_dir.Name;
 317                                 flavor.Domain = query_domain;
 318
 319                                 Queryable queryable = new Queryable (flavor, static_queryable);
 320                                 queryables.Add (queryable);
 321
 322                                 iqueryable_to_queryable [static_queryable] = queryable;
 323
 324                                 return true;
 325                         }
 326
 327                         return false;
 328                 }
 329
 330                 ////////////////////////////////////////////////////////
 331
 332                 private static ArrayList assemblies = null;
 333
 334                 // Perform expensive initialization steps all at once.
 335                 // Should be done before SignalHandler comes into play.
 336                 static public void Init ()
 337                 {
 338                         ReadBackendsFromConf ();
 339                         assemblies = ReflectionFu.ScanEnvironmentForAssemblies ("BEAGLE_BACKEND_PATH", PathFinder.BackendDir);
 340                 }
 341
 342                 static public void Start ()
 343                 {
 344                         // Only add the executing assembly if we haven't already loaded it.
 345                         if (assemblies.IndexOf (Assembly.GetExecutingAssembly ()) == -1)
 346                                 assemblies.Add (Assembly.GetExecutingAssembly ());
 347
 348                         foreach (Assembly assembly in assemblies) {
 349                                 ScanAssembly (assembly);
 350
 351                                 // This allows backends to define their
 352                                 // own executors.
 353                                 Server.ScanAssemblyForExecutors (assembly);
 354                         }
 355
 356                         ReadKeywordMappings ();
 357
 358                         LoadSystemIndexes ();
 359                         LoadStaticQueryables ();
 360
 361                         if (indexing_delay <= 0 || Environment.GetEnvironmentVariable ("BEAGLE_EXERCISE_THE_DOG") != null)
 362                                 StartQueryables ();
 363                         else {
 364                                 Logger.Log.Debug ("Waiting {0} seconds before starting queryables", indexing_delay);
 365                                 GLib.Timeout.Add ((uint) indexing_delay * 1000, new GLib.TimeoutHandler (StartQueryables));
 366                         }
 367                 }
 368
 369                 static private bool StartQueryables ()
 370                 {
 371                         Logger.Log.Debug ("Starting queryables");
 372
 373                         foreach (Queryable q in queryables) {
 374                                 Logger.Log.Info ("Starting backend: '{0}'", q.Name);
 375                                 q.Start ();
 376                         }
 377
 378                         return false;
 379                 }
 380
 381                 static public string ListBackends ()
 382                 {
 383                         ArrayList assemblies = ReflectionFu.ScanEnvironmentForAssemblies ("BEAGLE_BACKEND_PATH", PathFinder.BackendDir);
 384
 385                         // Only add the executing assembly if we haven't already loaded it.
 386                         if (assemblies.IndexOf (Assembly.GetExecutingAssembly ()) == -1)
 387                                 assemblies.Add (Assembly.GetExecutingAssembly ());
 388
 389                         string ret = "User:\n";
 390
 391                         foreach (Assembly assembly in assemblies) {
 392                                 foreach (Type type in ReflectionFu.ScanAssemblyForInterface (assembly, typeof (IQueryable))) {
 393                                         foreach (QueryableFlavor flavor in ReflectionFu.ScanTypeForAttribute (type, typeof (QueryableFlavor)))
 394                                                 ret += String.Format (" - {0}\n", flavor.Name);
 395                                 }
 396                         }
 397
 398                         if (!Directory.Exists (PathFinder.SystemIndexesDir))
 399                                 return ret;
 400
 401                         ret += "System:\n";
 402                         foreach (DirectoryInfo index_dir in new DirectoryInfo (PathFinder.SystemIndexesDir).GetDirectories ()) {
 403                                 ret += String.Format (" - {0}\n", index_dir.Name);
 404                         }
 405
 406                         return ret;
 407                 }
 408
 409                 static public Queryable GetQueryable (string name)
 410                 {
 411                         foreach (Queryable q in queryables) {
 412                                 if (q.Name == name)
 413                                         return q;
 414                         }
 415
 416                         return null;
 417                 }
 418
 419                 static public Queryable GetQueryable (IQueryable iqueryable)
 420                 {
 421                         return (Queryable) iqueryable_to_queryable [iqueryable];
 422                 }
 423
 424                 ////////////////////////////////////////////////////////
 425
 426                 public delegate void ChangedHandler (Queryable            queryable,
 427                                                      IQueryableChangeData changeData);
 428
 429                 static public event ChangedHandler ChangedEvent;
 430
 431                 // A method to fire the ChangedEvent event.
 432                 static public void QueryableChanged (IQueryable           iqueryable,
 433                                                      IQueryableChangeData change_data)
 434                 {
 435                         if (ChangedEvent != null) {
 436                                 Queryable queryable = iqueryable_to_queryable [iqueryable] as Queryable;
 437                                 ChangedEvent (queryable, change_data);
 438                         }
 439                 }
 440
 441                 ////////////////////////////////////////////////////////
 442
 443                 private class QueryClosure : IQueryWorker {
 444
 445                         Queryable queryable;
 446                         Query query;
 447                         IQueryResult result;
 448                         IQueryableChangeData change_data;
 449
 450                         public QueryClosure (Queryable            queryable,
 451                                              Query                query,
 452                                              QueryResult          result,
 453                                              IQueryableChangeData change_data)
 454                         {
 455                                 this.queryable = queryable;
 456                                 this.query = query;
 457                                 this.result = result;
 458                                 this.change_data = change_data;
 459                         }
 460
 461                         public void DoWork ()
 462                         {
 463                                 queryable.DoQuery (query, result, change_data);
 464                         }
 465                 }
 466
 467                 static public void DoOneQuery (Queryable            queryable,
 468                                                Query                query,
 469                                                QueryResult          result,
 470                                                IQueryableChangeData change_data)
 471                 {
 472                         if (queryable.AcceptQuery (query)) {
 473                                 QueryClosure qc = new QueryClosure (queryable, query, result, change_data);
 474                                 result.AttachWorker (qc);
 475                         }
 476                 }
 477
 478                 static void AddSearchTermInfo (QueryPart          part,
 479                                                SearchTermResponse response, StringBuilder sb)
 480                 {
 481                         if (part.Logic == QueryPartLogic.Prohibited)
 482                                 return;
 483
 484                         if (part is QueryPart_Or) {
 485                                 ICollection sub_parts;
 486                                 sub_parts = ((QueryPart_Or) part).SubParts;
 487                                 foreach (QueryPart qp in sub_parts)
 488                                         AddSearchTermInfo (qp, response, sb);
 489                                 return;
 490                         }
 491
 492                         if (! (part is QueryPart_Text))
 493                                 return;
 494
 495                         QueryPart_Text tp;
 496                         tp = (QueryPart_Text) part;
 497
 498                         string [] split;
 499                         split = tp.Text.Split (' ');
 500
 501                         // First, remove stop words
 502                         for (int i = 0; i < split.Length; ++i)
 503                                 if (LuceneCommon.IsStopWord (split [i]))
 504                                         split [i] = null;
 505
 506                         // Assemble the phrase minus stop words
 507                         sb.Length = 0;
 508                         for (int i = 0; i < split.Length; ++i) {
 509                                 if (split [i] == null)
 510                                         continue;
 511                                 if (sb.Length > 0)
 512                                         sb.Append (' ');
 513                                 sb.Append (split [i]);
 514                         }
 515                         response.ExactText.Add (sb.ToString ());
 516
 517                         // Now assemble a stemmed version
 518                         sb.Length = 0; // clear the previous value
 519                         for (int i = 0; i < split.Length; ++i) {
 520                                 if (split [i] == null)
 521                                         continue;
 522                                 if (sb.Length > 0)
 523                                         sb.Append (' ');
 524                                 sb.Append (LuceneCommon.Stem (split [i]));
 525                         }
 526                         response.StemmedText.Add (sb.ToString ());
 527                 }
 528
 529                 ////////////////////////////////////////////////////////
 530
 531                 static private void DehumanizeQuery (Query query)
 532                 {
 533                         // We need to remap any QueryPart_Human parts into
 534                         // lower-level part types.  First, we find any
 535                         // QueryPart_Human parts and explode them into
 536                         // lower-level types.
 537                         ArrayList new_parts = null;
 538                         foreach (QueryPart abstract_part in query.Parts) {
 539                                 if (abstract_part is QueryPart_Human) {
 540                                         QueryPart_Human human = abstract_part as QueryPart_Human;
 541                                         if (new_parts == null)
 542                                                 new_parts = new ArrayList ();
 543                                         foreach (QueryPart sub_part in QueryStringParser.Parse (human.QueryString))
 544                                                 new_parts.Add (sub_part);
 545                                 }
 546                         }
 547
 548                         // If we found any QueryPart_Human parts, copy the
 549                         // non-Human parts over and then replace the parts in
 550                         // the query.
 551                         if (new_parts != null) {
 552                                 foreach (QueryPart abstract_part in query.Parts) {
 553                                         if (! (abstract_part is QueryPart_Human))
 554                                                 new_parts.Add (abstract_part);
 555                                 }
 556
 557                                 query.ClearParts ();
 558                                 foreach (QueryPart part in new_parts)
 559                                         query.AddPart (part);
 560                         }
 561
 562                 }
 563
 564                 static private SearchTermResponse AssembleSearchTermResponse (Query query)
 565                 {
 566                         StringBuilder sb = new StringBuilder ();
 567                         SearchTermResponse search_term_response;
 568                         search_term_response = new SearchTermResponse ();
 569                         foreach (QueryPart part in query.Parts)
 570                                 AddSearchTermInfo (part, search_term_response, sb);
 571                         return search_term_response;
 572                 }
 573
 574                 static private void QueryEachQueryable (Query       query,
 575                                                         QueryResult result)
 576                 {
 577                         // The extra pair of calls to WorkerStart/WorkerFinished ensures:
 578                         // (1) that the QueryResult will fire the StartedEvent
 579                         // and FinishedEvent, even if no queryable accepts the
 580                         // query.
 581                         // (2) that the FinishedEvent will only get called when all of the
 582                         // backends have had time to finish.
 583
 584                         object dummy_worker = new object ();
 585
 586                         if (! result.WorkerStart (dummy_worker))
 587                                 return;
 588
 589                         foreach (Queryable queryable in queryables)
 590                                 DoOneQuery (queryable, query, result, null);
 591
 592                         result.WorkerFinished (dummy_worker);
 593                 }
 594
 595                 static public void DoQueryLocal (Query       query,
 596                                                  QueryResult result)
 597                 {
 598                         DehumanizeQuery (query);
 599
 600                         SearchTermResponse search_term_response;
 601                         search_term_response = AssembleSearchTermResponse (query);
 602                         query.ProcessSearchTermResponse (search_term_response);
 603
 604                         QueryEachQueryable (query, result);
 605                 }
 606
 607                 static public void DoQuery (Query                                query,
 608                                             QueryResult                          result,
 609                                             RequestMessageExecutor.AsyncResponse send_response)
 610                 {
 611                         DehumanizeQuery (query);
 612
 613                         SearchTermResponse search_term_response;
 614                         search_term_response = AssembleSearchTermResponse (query);
 615                         send_response (search_term_response);
 616
 617                         QueryEachQueryable (query, result);
 618                 }
 619
 620                 ////////////////////////////////////////////////////////
 621
 622                 static public IEnumerable GetIndexInformation ()
 623                 {
 624                         foreach (Queryable q in queryables)
 625                                 yield return q.GetQueryableStatus ();
 626                 }
 627
 628                 ////////////////////////////////////////////////////////
 629
 630                 static public bool IsIndexing {
 631                         get {
 632                                 foreach (Queryable q in queryables) {
 633                                         QueryableStatus status = q.GetQueryableStatus ();
 634
 635                                         if (status.IsIndexing)
 636                                                 return true;
 637                                 }
 638
 639                                 return false;
 640                         }
 641                 }
 642         }
 643 }