beagled/QueryDriver.cs

   1 //
   2 // QueryDriver.cs
   3 //
   4 // Copyright (C) 2004 Novell, Inc.
   5 //
   6
   7 //
   8 // Permission is hereby granted, free of charge, to any person obtaining a
   9 // copy of this software and associated documentation files (the "Software"),
  10 // to deal in the Software without restriction, including without limitation
  11 // the rights to use, copy, modify, merge, publish, distribute, sublicense,
  12 // and/or sell copies of the Software, and to permit persons to whom the
  13 // Software is furnished to do so, subject to the following conditions:
  14 //
  15 // The above copyright notice and this permission notice shall be included in
  16 // all copies or substantial portions of the Software.
  17 //
  18 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  21 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  22 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  23 // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  24 // DEALINGS IN THE SOFTWARE.
  25 //
  26
  27 using System;
  28 using System.IO;
  29 using System.Collections;
  30 using System.Reflection;
  31 using System.Text;
  32 using System.Threading;
  33 using Beagle.Util;
  34 namespace Beagle.Daemon {
  35
  36         public class QueryDriver {
  37
  38                 // Contains list of queryables explicitly asked by --allow-backend or --backend name
  39                 // --allow-backend/--backend name : dont read config information and only start backend 'name'
  40                 static ArrayList excl_allowed_queryables = new ArrayList ();
  41
  42                 // Contains list of denied queryables from config/arguments (every queryable is enabled by default)
  43                 // Unless overruled by --allow-backend/--backend name, deny backend only if names appears here.
  44                 static ArrayList denied_queryables = new ArrayList ();
  45
  46                 static bool to_read_conf = true; // read backends from conf if true
  47                 static bool done_reading_conf = false;
  48
  49                 static private void ReadBackendsFromConf ()
  50                 {
  51                         if (! to_read_conf || done_reading_conf)
  52                                 return;
  53
  54                         // set flag here to stop Allow() from calling ReadBackendsFromConf() again
  55                         done_reading_conf = true;
  56
  57                         // To allow static indexes, "static" should be in allowed_queryables
  58                         if (Conf.Daemon.AllowStaticBackend)
  59                                 Allow ("static");
  60
  61                         if (Conf.Daemon.DeniedBackends == null)
  62                                 return;
  63
  64                         foreach (string name in Conf.Daemon.DeniedBackends)
  65                                 denied_queryables.Add (name.ToLower ());
  66                 }
  67
  68                 static public void OnlyAllow (string name)
  69                 {
  70                         excl_allowed_queryables.Add (name.ToLower ());
  71                         to_read_conf = false;
  72                 }
  73
  74                 static public void Allow (string name)
  75                 {
  76                         if (! done_reading_conf && to_read_conf)
  77                                 ReadBackendsFromConf ();
  78
  79                         denied_queryables.Remove (name.ToLower ());
  80                 }
  81
  82                 static public void Deny (string name)
  83                 {
  84                         if (! done_reading_conf && to_read_conf)
  85                                 ReadBackendsFromConf ();
  86
  87                         name = name.ToLower ();
  88                         if (!denied_queryables.Contains (name))
  89                                 denied_queryables.Add (name);
  90                 }
  91
  92                 static private bool UseQueryable (string name)
  93                 {
  94                         name = name.ToLower ();
  95
  96                         if (excl_allowed_queryables.Contains (name))
  97                                 return true;
  98                         if (excl_allowed_queryables.Count != 0)
  99                                 return false;
 100
 101                         if (denied_queryables.Contains (name))
 102                                 return false;
 103
 104                         return true;
 105                 }
 106
 107                 //////////////////////////////////////////////////////////////////////////////////////
 108
 109                 // Paths to static queryables
 110
 111                 static ArrayList static_queryables = new ArrayList ();
 112
 113                 static public void AddStaticQueryable (string path) {
 114
 115                         if (! static_queryables.Contains (path))
 116                                 static_queryables.Add (path);
 117                 }
 118
 119                 //////////////////////////////////////////////////////////////////////////////////////
 120
 121                 // Delay before starting the indexing process
 122
 123                 static int indexing_delay = 60;  // Default to 60 seconds
 124
 125                 public static int IndexingDelay {
 126                         set { indexing_delay = value; }
 127                 }
 128
 129                 //////////////////////////////////////////////////////////////////////////////////////
 130
 131                 // Use introspection to find all classes that implement IQueryable, the construct
 132                 // associated Queryables objects.
 133
 134                 static ArrayList queryables = new ArrayList ();
 135                 static Hashtable iqueryable_to_queryable = new Hashtable ();
 136
 137                 static bool ThisApiSoVeryIsBroken (Type m, object criteria)
 138                 {
 139                         return m == (Type) criteria;
 140                 }
 141
 142                 static bool TypeImplementsInterface (Type t, Type iface)
 143                 {
 144                         Type[] impls = t.FindInterfaces (new TypeFilter (ThisApiSoVeryIsBroken),
 145                                                          iface);
 146                         return impls.Length > 0;
 147                 }
 148
 149                 // For every type in the assembly that
 150                 // (1) implements IQueryable
 151                 // (2) Has a QueryableFlavor attribute attached
 152                 // assemble a Queryable object and stick it into our list of queryables.
 153                 static void ScanAssembly (Assembly assembly)
 154                 {
 155                         int count = 0;
 156
 157                         foreach (Type type in ReflectionFu.ScanAssemblyForInterface (assembly, typeof (IQueryable))) {
 158                                 bool type_accepted = false;
 159                                 foreach (QueryableFlavor flavor in ReflectionFu.ScanTypeForAttribute (type, typeof (QueryableFlavor))) {
 160                                         if (! UseQueryable (flavor.Name))
 161                                                 continue;
 162
 163                                         if (flavor.RequireInotify && ! Inotify.Enabled) {
 164                                                 Logger.Log.Warn ("Can't start backend '{0}' without inotify", flavor.Name);
 165                                                 continue;
 166                                         }
 167
 168                                         if (flavor.RequireExtendedAttributes && ! ExtendedAttribute.Supported) {
 169                                                 Logger.Log.Warn ("Can't start backend '{0}' without extended attributes", flavor.Name);
 170                                                 continue;
 171                                         }
 172
 173                                         IQueryable iq = null;
 174                                         try {
 175                                                 iq = Activator.CreateInstance (type) as IQueryable;
 176                                         } catch (Exception e) {
 177                                                 Logger.Log.Error ("Caught exception while instantiating {0} backend", flavor.Name);
 178                                                 Logger.Log.Error (e);
 179                                         }
 180
 181                                         if (iq != null) {
 182                                                 Queryable q = new Queryable (flavor, iq);
 183                                                 queryables.Add (q);
 184                                                 iqueryable_to_queryable [iq] = q;
 185                                                 ++count;
 186                                                 type_accepted = true;
 187                                                 break;
 188                                         }
 189                                 }
 190
 191                                 if (! type_accepted)
 192                                         continue;
 193
 194                                 object[] attributes = type.GetCustomAttributes (false);
 195                                 foreach (object attribute in attributes) {
 196                                         PropertyKeywordMapping mapping = attribute as PropertyKeywordMapping;
 197                                         if (mapping == null)
 198                                                 continue;
 199                                         //Logger.Log.Debug (mapping.Keyword + " => "
 200                                         //              + mapping.PropertyName +
 201                                         //              + " is-keyword=" + mapping.IsKeyword + " ("
 202                                         //              + mapping.Description + ") "
 203                                         //              + "(" + type.FullName + ")");
 204                                         PropertyKeywordFu.RegisterMapping (mapping);
 205                                 }
 206
 207                         }
 208                         Logger.Log.Debug ("Found {0} backends in {1}", count, assembly.Location);
 209                 }
 210
 211                 ////////////////////////////////////////////////////////
 212
 213                 public static void ReadKeywordMappings ()
 214                 {
 215                         Logger.Log.Debug ("Reading mapping from filters");
 216                         ArrayList assemblies = ReflectionFu.ScanEnvironmentForAssemblies ("BEAGLE_FILTER_PATH", PathFinder.FilterDir);
 217
 218                         foreach (Assembly assembly in assemblies) {
 219                                 foreach (Type type in assembly.GetTypes ())
 220                                         if (type.FullName.StartsWith ("Beagle.Filters")) {
 221
 222                                                 if (type.IsNestedPrivate ||
 223                                                     type.IsNestedPublic ||
 224                                                     type.IsNestedAssembly ||
 225                                                     type.IsNestedFamily)
 226                                                         continue;
 227
 228                                                 object[] attributes = type.GetCustomAttributes (false);
 229                                                 foreach (object attribute in attributes) {
 230
 231                                                         PropertyKeywordMapping mapping = attribute as PropertyKeywordMapping;
 232                                                         if (mapping == null)
 233                                                                 continue;
 234                                                         //Logger.Log.Debug (mapping.Keyword + " => "
 235                                                         //              + mapping.PropertyName
 236                                                         //              + " is-keyword=" + mapping.IsKeyword + " ("
 237                                                         //              + mapping.Description + ") "
 238                                                         //              + "(" + type.FullName + ")");
 239                                                         PropertyKeywordFu.RegisterMapping (mapping);
 240                                                 }
 241                                         }
 242                         }
 243                 }
 244
 245                 ////////////////////////////////////////////////////////
 246
 247                 // Scans PathFinder.SystemIndexesDir after available
 248                 // system-wide indexes.
 249                 static void LoadSystemIndexes ()
 250                 {
 251                         if (!Directory.Exists (PathFinder.SystemIndexesDir))
 252                                 return;
 253
 254                         Logger.Log.Info ("Loading system static indexes.");
 255
 256                         int count = 0;
 257
 258                         foreach (DirectoryInfo index_dir in new DirectoryInfo (PathFinder.SystemIndexesDir).GetDirectories ()) {
 259                                 if (! UseQueryable (index_dir.Name))
 260                                         continue;
 261
 262                                 if (LoadStaticQueryable (index_dir, QueryDomain.System))
 263                                         count++;
 264                         }
 265
 266                         Logger.Log.Info ("Found {0} system-wide indexes.", count);
 267                 }
 268
 269                 // Scans configuration for user-specified index paths
 270                 // to load StaticQueryables from.
 271                 static void LoadStaticQueryables ()
 272                 {
 273                         int count = 0;
 274
 275                         if (UseQueryable ("static")) {
 276                                 Logger.Log.Info ("Loading user-configured static indexes.");
 277                                 foreach (string path in Conf.Daemon.StaticQueryables)
 278                                         static_queryables.Add (path);
 279                         }
 280
 281                         foreach (string path in static_queryables) {
 282                                 DirectoryInfo index_dir = new DirectoryInfo (StringFu.SanitizePath (path));
 283
 284                                 if (!index_dir.Exists)
 285                                         continue;
 286
 287                                 // FIXME: QueryDomain might be other than local
 288                                 if (LoadStaticQueryable (index_dir, QueryDomain.Local))
 289                                         count++;
 290                         }
 291
 292                         Logger.Log.Info ("Found {0} user-configured static indexes..", count);
 293                 }
 294
 295                 // Instantiates and loads a StaticQueryable from an index directory
 296                 static private bool LoadStaticQueryable (DirectoryInfo index_dir, QueryDomain query_domain)
 297                 {
 298                         StaticQueryable static_queryable = null;
 299
 300                         if (!index_dir.Exists)
 301                                 return false;
 302
 303                         try {
 304                                 static_queryable = new StaticQueryable (index_dir.Name, index_dir.FullName, true);
 305                         } catch (InvalidOperationException) {
 306                                 Logger.Log.Warn ("Unable to create read-only index (likely due to index version mismatch): {0}", index_dir.FullName);
 307                                 return false;
 308                         } catch (Exception e) {
 309                                 Logger.Log.Error ("Caught exception while instantiating static queryable: {0}", index_dir.Name);
 310                                 Logger.Log.Error (e);
 311                                 return false;
 312                         }
 313
 314                         if (static_queryable != null) {
 315                                 QueryableFlavor flavor = new QueryableFlavor ();
 316                                 flavor.Name = index_dir.Name;
 317                                 flavor.Domain = query_domain;
 318
 319                                 Queryable queryable = new Queryable (flavor, static_queryable);
 320                                 queryables.Add (queryable);
 321
 322                                 iqueryable_to_queryable [static_queryable] = queryable;
 323
 324                                 return true;
 325                         }
 326
 327                         return false;
 328                 }
 329
 330                 ////////////////////////////////////////////////////////
 331
 332                 static public void Start ()
 333                 {
 334                         ReadBackendsFromConf ();
 335
 336                         ArrayList assemblies = ReflectionFu.ScanEnvironmentForAssemblies ("BEAGLE_BACKEND_PATH", PathFinder.BackendDir);
 337
 338                         // Only add the executing assembly if we haven't already loaded it.
 339                         if (assemblies.IndexOf (Assembly.GetExecutingAssembly ()) == -1)
 340                                 assemblies.Add (Assembly.GetExecutingAssembly ());
 341
 342                         foreach (Assembly assembly in assemblies) {
 343                                 ScanAssembly (assembly);
 344
 345                                 // This allows backends to define their
 346                                 // own executors.
 347                                 Server.ScanAssemblyForExecutors (assembly);
 348                         }
 349
 350                         ReadKeywordMappings ();
 351
 352                         LoadSystemIndexes ();
 353                         LoadStaticQueryables ();
 354
 355                         if (indexing_delay <= 0 || Environment.GetEnvironmentVariable ("BEAGLE_EXERCISE_THE_DOG") != null)
 356                                 StartQueryables ();
 357                         else {
 358                                 Logger.Log.Debug ("Waiting {0} seconds before starting queryables", indexing_delay);
 359                                 GLib.Timeout.Add ((uint) indexing_delay * 1000, new GLib.TimeoutHandler (StartQueryables));
 360                         }
 361                 }
 362
 363                 static private bool StartQueryables ()
 364                 {
 365                         Logger.Log.Debug ("Starting queryables");
 366
 367                         foreach (Queryable q in queryables) {
 368                                 Logger.Log.Info ("Starting backend: '{0}'", q.Name);
 369                                 q.Start ();
 370                         }
 371
 372                         return false;
 373                 }
 374
 375                 static public string ListBackends ()
 376                 {
 377                         ArrayList assemblies = ReflectionFu.ScanEnvironmentForAssemblies ("BEAGLE_BACKEND_PATH", PathFinder.BackendDir);
 378
 379                         // Only add the executing assembly if we haven't already loaded it.
 380                         if (assemblies.IndexOf (Assembly.GetExecutingAssembly ()) == -1)
 381                                 assemblies.Add (Assembly.GetExecutingAssembly ());
 382
 383                         string ret = "User:\n";
 384
 385                         foreach (Assembly assembly in assemblies) {
 386                                 foreach (Type type in ReflectionFu.ScanAssemblyForInterface (assembly, typeof (IQueryable))) {
 387                                         foreach (QueryableFlavor flavor in ReflectionFu.ScanTypeForAttribute (type, typeof (QueryableFlavor)))
 388                                                 ret += String.Format (" - {0}\n", flavor.Name);
 389                                 }
 390                         }
 391
 392                         if (!Directory.Exists (PathFinder.SystemIndexesDir))
 393                                 return ret;
 394
 395                         ret += "System:\n";
 396                         foreach (DirectoryInfo index_dir in new DirectoryInfo (PathFinder.SystemIndexesDir).GetDirectories ()) {
 397                                 ret += String.Format (" - {0}\n", index_dir.Name);
 398                         }
 399
 400                         return ret;
 401                 }
 402
 403                 static public Queryable GetQueryable (string name)
 404                 {
 405                         foreach (Queryable q in queryables) {
 406                                 if (q.Name == name)
 407                                         return q;
 408                         }
 409
 410                         return null;
 411                 }
 412
 413                 static public Queryable GetQueryable (IQueryable iqueryable)
 414                 {
 415                         return (Queryable) iqueryable_to_queryable [iqueryable];
 416                 }
 417
 418                 ////////////////////////////////////////////////////////
 419
 420                 public delegate void ChangedHandler (Queryable            queryable,
 421                                                      IQueryableChangeData changeData);
 422
 423                 static public event ChangedHandler ChangedEvent;
 424
 425                 // A method to fire the ChangedEvent event.
 426                 static public void QueryableChanged (IQueryable           iqueryable,
 427                                                      IQueryableChangeData change_data)
 428                 {
 429                         if (ChangedEvent != null) {
 430                                 Queryable queryable = iqueryable_to_queryable [iqueryable] as Queryable;
 431                                 ChangedEvent (queryable, change_data);
 432                         }
 433                 }
 434
 435                 ////////////////////////////////////////////////////////
 436
 437                 private class QueryClosure : IQueryWorker {
 438
 439                         Queryable queryable;
 440                         Query query;
 441                         IQueryResult result;
 442                         IQueryableChangeData change_data;
 443
 444                         public QueryClosure (Queryable            queryable,
 445                                              Query                query,
 446                                              QueryResult          result,
 447                                              IQueryableChangeData change_data)
 448                         {
 449                                 this.queryable = queryable;
 450                                 this.query = query;
 451                                 this.result = result;
 452                                 this.change_data = change_data;
 453                         }
 454
 455                         public void DoWork ()
 456                         {
 457                                 queryable.DoQuery (query, result, change_data);
 458                         }
 459                 }
 460
 461                 static public void DoOneQuery (Queryable            queryable,
 462                                                Query                query,
 463                                                QueryResult          result,
 464                                                IQueryableChangeData change_data)
 465                 {
 466                         if (queryable.AcceptQuery (query)) {
 467                                 QueryClosure qc = new QueryClosure (queryable, query, result, change_data);
 468                                 result.AttachWorker (qc);
 469                         }
 470                 }
 471
 472                 static void AddSearchTermInfo (QueryPart          part,
 473                                                SearchTermResponse response, StringBuilder sb)
 474                 {
 475                         if (part.Logic == QueryPartLogic.Prohibited)
 476                                 return;
 477
 478                         if (part is QueryPart_Or) {
 479                                 ICollection sub_parts;
 480                                 sub_parts = ((QueryPart_Or) part).SubParts;
 481                                 foreach (QueryPart qp in sub_parts)
 482                                         AddSearchTermInfo (qp, response, sb);
 483                                 return;
 484                         }
 485
 486                         if (! (part is QueryPart_Text))
 487                                 return;
 488
 489                         QueryPart_Text tp;
 490                         tp = (QueryPart_Text) part;
 491
 492                         string [] split;
 493                         split = tp.Text.Split (' ');
 494
 495                         // First, remove stop words
 496                         for (int i = 0; i < split.Length; ++i)
 497                                 if (LuceneCommon.IsStopWord (split [i]))
 498                                         split [i] = null;
 499
 500                         // Assemble the phrase minus stop words
 501                         sb.Length = 0;
 502                         for (int i = 0; i < split.Length; ++i) {
 503                                 if (split [i] == null)
 504                                         continue;
 505                                 if (sb.Length > 0)
 506                                         sb.Append (' ');
 507                                 sb.Append (split [i]);
 508                         }
 509                         response.ExactText.Add (sb.ToString ());
 510
 511                         // Now assemble a stemmed version
 512                         sb.Length = 0; // clear the previous value
 513                         for (int i = 0; i < split.Length; ++i) {
 514                                 if (split [i] == null)
 515                                         continue;
 516                                 if (sb.Length > 0)
 517                                         sb.Append (' ');
 518                                 sb.Append (LuceneCommon.Stem (split [i]));
 519                         }
 520                         response.StemmedText.Add (sb.ToString ());
 521                 }
 522
 523                 ////////////////////////////////////////////////////////
 524
 525                 static private void DehumanizeQuery (Query query)
 526                 {
 527                         // We need to remap any QueryPart_Human parts into
 528                         // lower-level part types.  First, we find any
 529                         // QueryPart_Human parts and explode them into
 530                         // lower-level types.
 531                         ArrayList new_parts = null;
 532                         foreach (QueryPart abstract_part in query.Parts) {
 533                                 if (abstract_part is QueryPart_Human) {
 534                                         QueryPart_Human human = abstract_part as QueryPart_Human;
 535                                         if (new_parts == null)
 536                                                 new_parts = new ArrayList ();
 537                                         foreach (QueryPart sub_part in QueryStringParser.Parse (human.QueryString))
 538                                                 new_parts.Add (sub_part);
 539                                 }
 540                         }
 541
 542                         // If we found any QueryPart_Human parts, copy the
 543                         // non-Human parts over and then replace the parts in
 544                         // the query.
 545                         if (new_parts != null) {
 546                                 foreach (QueryPart abstract_part in query.Parts) {
 547                                         if (! (abstract_part is QueryPart_Human))
 548                                                 new_parts.Add (abstract_part);
 549                                 }
 550
 551                                 query.ClearParts ();
 552                                 foreach (QueryPart part in new_parts)
 553                                         query.AddPart (part);
 554                         }
 555
 556                 }
 557
 558                 static private SearchTermResponse AssembleSearchTermResponse (Query query)
 559                 {
 560                         StringBuilder sb = new StringBuilder ();
 561                         SearchTermResponse search_term_response;
 562                         search_term_response = new SearchTermResponse ();
 563                         foreach (QueryPart part in query.Parts)
 564                                 AddSearchTermInfo (part, search_term_response, sb);
 565                         return search_term_response;
 566                 }
 567
 568                 static private void QueryEachQueryable (Query       query,
 569                                                         QueryResult result)
 570                 {
 571                         // The extra pair of calls to WorkerStart/WorkerFinished ensures:
 572                         // (1) that the QueryResult will fire the StartedEvent
 573                         // and FinishedEvent, even if no queryable accepts the
 574                         // query.
 575                         // (2) that the FinishedEvent will only get called when all of the
 576                         // backends have had time to finish.
 577
 578                         object dummy_worker = new object ();
 579
 580                         if (! result.WorkerStart (dummy_worker))
 581                                 return;
 582
 583                         foreach (Queryable queryable in queryables)
 584                                 DoOneQuery (queryable, query, result, null);
 585
 586                         result.WorkerFinished (dummy_worker);
 587                 }
 588
 589                 static public void DoQueryLocal (Query       query,
 590                                                  QueryResult result)
 591                 {
 592                         DehumanizeQuery (query);
 593
 594                         SearchTermResponse search_term_response;
 595                         search_term_response = AssembleSearchTermResponse (query);
 596                         query.ProcessSearchTermResponse (search_term_response);
 597
 598                         QueryEachQueryable (query, result);
 599                 }
 600
 601                 static public void DoQuery (Query                                query,
 602                                             QueryResult                          result,
 603                                             RequestMessageExecutor.AsyncResponse send_response)
 604                 {
 605                         DehumanizeQuery (query);
 606
 607                         SearchTermResponse search_term_response;
 608                         search_term_response = AssembleSearchTermResponse (query);
 609                         send_response (search_term_response);
 610
 611                         QueryEachQueryable (query, result);
 612                 }
 613
 614                 ////////////////////////////////////////////////////////
 615
 616                 static public IEnumerable GetIndexInformation ()
 617                 {
 618                         foreach (Queryable q in queryables)
 619                                 yield return q.GetQueryableStatus ();
 620                 }
 621
 622                 ////////////////////////////////////////////////////////
 623
 624                 static public bool IsIndexing {
 625                         get {
 626                                 foreach (Queryable q in queryables) {
 627                                         QueryableStatus status = q.GetQueryableStatus ();
 628
 629                                         if (status.IsIndexing)
 630                                                 return true;
 631                                 }
 632
 633                                 return false;
 634                         }
 635                 }
 636         }
 637 }