beagled/QueryDriver.cs

   1 //
   2 // QueryDriver.cs
   3 //
   4 // Copyright (C) 2004 Novell, Inc.
   5 //
   6
   7 //
   8 // Permission is hereby granted, free of charge, to any person obtaining a
   9 // copy of this software and associated documentation files (the "Software"),
  10 // to deal in the Software without restriction, including without limitation
  11 // the rights to use, copy, modify, merge, publish, distribute, sublicense,
  12 // and/or sell copies of the Software, and to permit persons to whom the
  13 // Software is furnished to do so, subject to the following conditions:
  14 //
  15 // The above copyright notice and this permission notice shall be included in
  16 // all copies or substantial portions of the Software.
  17 //
  18 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  21 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  22 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  23 // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  24 // DEALINGS IN THE SOFTWARE.
  25 //
  26
  27 using System;
  28 using System.IO;
  29 using System.Collections;
  30 using System.Reflection;
  31 using System.Text;
  32 using System.Threading;
  33 using Beagle.Util;
  34 namespace Beagle.Daemon {
  35
  36         public class QueryDriver {
  37
  38                 // Enable or Disable specific Queryables by name
  39
  40                 static ArrayList allowed_queryables = new ArrayList ();
  41                 static ArrayList denied_queryables = new ArrayList ();
  42
  43                 static public void Allow (string name)
  44                 {
  45                         allowed_queryables.Add (name.ToLower ());
  46                 }
  47
  48                 static public void Deny (string name)
  49                 {
  50                         denied_queryables.Add (name.ToLower ());
  51                 }
  52
  53                 static private bool UseQueryable (string name)
  54                 {
  55                         if (allowed_queryables.Count == 0
  56                             && denied_queryables.Count == 0)
  57                                 return true;
  58
  59                         name = name.ToLower ();
  60
  61                         if (allowed_queryables.Count > 0) {
  62                                 foreach (string allowed in allowed_queryables)
  63                                         if (name == allowed)
  64                                                 return true;
  65                                 return false;
  66                         }
  67
  68                         foreach (string denied in denied_queryables)
  69                                 if (name == denied)
  70                                         return false;
  71                         return true;
  72
  73                 }
  74
  75                 //////////////////////////////////////////////////////////////////////////////////////
  76
  77                 // Paths to static queryables
  78
  79                 static ArrayList static_queryables = new ArrayList ();
  80
  81                 static public void AddStaticQueryable (string path) {
  82
  83                         if (! static_queryables.Contains (path))
  84                                 static_queryables.Add (path);
  85                 }
  86
  87                 //////////////////////////////////////////////////////////////////////////////////////
  88
  89                 // Delay before starting the indexing process
  90
  91                 static int indexing_delay = 60;  // Default to 60 seconds
  92
  93                 public static int IndexingDelay {
  94                         set { indexing_delay = value; }
  95                 }
  96
  97                 //////////////////////////////////////////////////////////////////////////////////////
  98
  99                 // Use introspection to find all classes that implement IQueryable, the construct
 100                 // associated Queryables objects.
 101
 102                 static ArrayList queryables = new ArrayList ();
 103                 static Hashtable iqueryable_to_queryable = new Hashtable ();
 104
 105                 static bool ThisApiSoVeryIsBroken (Type m, object criteria)
 106                 {
 107                         return m == (Type) criteria;
 108                 }
 109
 110                 static bool TypeImplementsInterface (Type t, Type iface)
 111                 {
 112                         Type[] impls = t.FindInterfaces (new TypeFilter (ThisApiSoVeryIsBroken),
 113                                                          iface);
 114                         return impls.Length > 0;
 115                 }
 116
 117                 // For every type in the assembly that
 118                 // (1) implements IQueryable
 119                 // (2) Has a QueryableFlavor attribute attached
 120                 // assemble a Queryable object and stick it into our list of queryables.
 121                 static void ScanAssembly (Assembly assembly)
 122                 {
 123                         int count = 0;
 124
 125                         foreach (Type type in ReflectionFu.ScanAssemblyForInterface (assembly, typeof (IQueryable))) {
 126                                 foreach (QueryableFlavor flavor in ReflectionFu.ScanTypeForAttribute (type, typeof (QueryableFlavor))) {
 127                                         if (! UseQueryable (flavor.Name))
 128                                                 continue;
 129
 130                                         if (flavor.RequireInotify && ! Inotify.Enabled) {
 131                                                 Logger.Log.Warn ("Can't start backend '{0}' without inotify", flavor.Name);
 132                                                 continue;
 133                                         }
 134
 135                                         if (flavor.RequireExtendedAttributes && ! ExtendedAttribute.Supported) {
 136                                                 Logger.Log.Warn ("Can't start backend '{0}' without extended attributes", flavor.Name);
 137                                                 continue;
 138                                         }
 139
 140                                         IQueryable iq = null;
 141                                         try {
 142                                                 iq = Activator.CreateInstance (type) as IQueryable;
 143                                         } catch (Exception e) {
 144                                                 Logger.Log.Error ("Caught exception while instantiating {0} backend", flavor.Name);
 145                                                 Logger.Log.Error (e);
 146                                         }
 147
 148                                         if (iq != null) {
 149                                                 Queryable q = new Queryable (flavor, iq);
 150                                                 queryables.Add (q);
 151                                                 iqueryable_to_queryable [iq] = q;
 152                                                 ++count;
 153                                                 break;
 154                                         }
 155                                 }
 156                         }
 157                         Logger.Log.Debug ("Found {0} backends in {1}", count, assembly.Location);
 158                 }
 159
 160                 ////////////////////////////////////////////////////////
 161
 162                 // Scans PathFinder.SystemIndexesDir after available
 163                 // system-wide indexes.
 164                 static void LoadSystemIndexes ()
 165                 {
 166                         if (!Directory.Exists (PathFinder.SystemIndexesDir))
 167                                 return;
 168
 169                         int count = 0;
 170
 171                         foreach (DirectoryInfo index_dir in new DirectoryInfo (PathFinder.SystemIndexesDir).GetDirectories ()) {
 172                                 if (! UseQueryable (index_dir.Name))
 173                                         continue;
 174
 175                                 if (LoadStaticQueryable (index_dir, QueryDomain.System))
 176                                         count++;
 177                         }
 178
 179                         Logger.Log.Debug ("Found {0} system-wide indexes", count);
 180                 }
 181
 182                 // Scans configuration for user-specified index paths
 183                 // to load StaticQueryables from.
 184                 static void LoadStaticQueryables ()
 185                 {
 186                         int count = 0;
 187
 188                         if (UseQueryable ("static")) {
 189                                 foreach (string path in Conf.Daemon.StaticQueryables)
 190                                         static_queryables.Add (path);
 191                         }
 192
 193                         foreach (string path in static_queryables) {
 194                                 DirectoryInfo index_dir = new DirectoryInfo (StringFu.SanitizePath (path));
 195
 196                                 if (!index_dir.Exists)
 197                                         continue;
 198
 199                                 // FIXME: QueryDomain might be other than local
 200                                 if (LoadStaticQueryable (index_dir, QueryDomain.Local))
 201                                         count++;
 202                         }
 203
 204                         Logger.Log.Debug ("Found {0} user-configured static queryables", count);
 205                 }
 206
 207                 // Instantiates and loads a StaticQueryable from an index directory
 208                 static private bool LoadStaticQueryable (DirectoryInfo index_dir, QueryDomain query_domain)
 209                 {
 210                         StaticQueryable static_queryable = null;
 211
 212                         if (!index_dir.Exists)
 213                                 return false;
 214
 215                         try {
 216                                 static_queryable = new StaticQueryable (index_dir.Name, index_dir.FullName, true);
 217                         } catch (InvalidOperationException) {
 218                                 Logger.Log.Warn ("Unable to create read-only index (likely due to index version mismatch): {0}", index_dir.FullName);
 219                                 return false;
 220                         } catch (Exception e) {
 221                                 Logger.Log.Error ("Caught exception while instantiating static queryable: {0}", index_dir.Name);
 222                                 Logger.Log.Error (e);
 223                                 return false;
 224                         }
 225
 226                         if (static_queryable != null) {
 227                                 QueryableFlavor flavor = new QueryableFlavor ();
 228                                 flavor.Name = index_dir.Name;
 229                                 flavor.Domain = query_domain;
 230
 231                                 Queryable queryable = new Queryable (flavor, static_queryable);
 232                                 queryables.Add (queryable);
 233
 234                                 iqueryable_to_queryable [static_queryable] = queryable;
 235
 236                                 return true;
 237                         }
 238
 239                         return false;
 240                 }
 241
 242                 ////////////////////////////////////////////////////////
 243
 244                 static public void Start ()
 245                 {
 246                         ArrayList assemblies = ReflectionFu.ScanEnvironmentForAssemblies ("BEAGLE_BACKEND_PATH", PathFinder.BackendDir);
 247
 248                         // Only add the executing assembly if we haven't already loaded it.
 249                         if (assemblies.IndexOf (Assembly.GetExecutingAssembly ()) == -1)
 250                                 assemblies.Add (Assembly.GetExecutingAssembly ());
 251
 252                         foreach (Assembly assembly in assemblies) {
 253                                 ScanAssembly (assembly);
 254
 255                                 // This allows backends to define their
 256                                 // own executors.
 257                                 Server.ScanAssemblyForExecutors (assembly);
 258                         }
 259
 260                         LoadSystemIndexes ();
 261                         LoadStaticQueryables ();
 262
 263                         if (indexing_delay <= 0 || Environment.GetEnvironmentVariable ("BEAGLE_EXERCISE_THE_DOG") != null)
 264                                 StartQueryables ();
 265                         else {
 266                                 Logger.Log.Debug ("Waiting {0} seconds before starting queryables", indexing_delay);
 267                                 GLib.Timeout.Add ((uint) indexing_delay * 1000, new GLib.TimeoutHandler (StartQueryables));
 268                         }
 269                 }
 270
 271                 static private bool StartQueryables ()
 272                 {
 273                         Logger.Log.Debug ("Starting queryables");
 274
 275                         foreach (Queryable q in queryables)
 276                                 q.Start ();
 277
 278                         return false;
 279                 }
 280
 281                 static public string ListBackends ()
 282                 {
 283                         ArrayList assemblies = ReflectionFu.ScanEnvironmentForAssemblies ("BEAGLE_BACKEND_PATH", PathFinder.BackendDir);
 284
 285                         // Only add the executing assembly if we haven't already loaded it.
 286                         if (assemblies.IndexOf (Assembly.GetExecutingAssembly ()) == -1)
 287                                 assemblies.Add (Assembly.GetExecutingAssembly ());
 288
 289                         string ret = "User:\n";
 290
 291                         foreach (Assembly assembly in assemblies) {
 292                                 foreach (Type type in ReflectionFu.ScanAssemblyForInterface (assembly, typeof (IQueryable))) {
 293                                         foreach (QueryableFlavor flavor in ReflectionFu.ScanTypeForAttribute (type, typeof (QueryableFlavor)))
 294                                                 ret += String.Format (" - {0}\n", flavor.Name);
 295                                 }
 296                         }
 297
 298                         if (!Directory.Exists (PathFinder.SystemIndexesDir))
 299                                 return ret;
 300
 301                         ret += "System:\n";
 302                         foreach (DirectoryInfo index_dir in new DirectoryInfo (PathFinder.SystemIndexesDir).GetDirectories ()) {
 303                                 ret += String.Format (" - {0}\n", index_dir.Name);
 304                         }
 305
 306                         return ret;
 307                 }
 308
 309                 static public Queryable GetQueryable (string name)
 310                 {
 311                         foreach (Queryable q in queryables) {
 312                                 if (q.Name == name)
 313                                         return q;
 314                         }
 315
 316                         return null;
 317                 }
 318
 319                 static public Queryable GetQueryable (IQueryable iqueryable)
 320                 {
 321                         return (Queryable) iqueryable_to_queryable [iqueryable];
 322                 }
 323
 324                 ////////////////////////////////////////////////////////
 325
 326                 public delegate void ChangedHandler (Queryable            queryable,
 327                                                      IQueryableChangeData changeData);
 328
 329                 static public event ChangedHandler ChangedEvent;
 330
 331                 // A method to fire the ChangedEvent event.
 332                 static public void QueryableChanged (IQueryable           iqueryable,
 333                                                      IQueryableChangeData change_data)
 334                 {
 335                         if (ChangedEvent != null) {
 336                                 Queryable queryable = iqueryable_to_queryable [iqueryable] as Queryable;
 337                                 ChangedEvent (queryable, change_data);
 338                         }
 339                 }
 340
 341                 ////////////////////////////////////////////////////////
 342
 343                 private class QueryClosure : IQueryWorker {
 344
 345                         Queryable queryable;
 346                         Query query;
 347                         IQueryResult result;
 348                         IQueryableChangeData change_data;
 349
 350                         public QueryClosure (Queryable            queryable,
 351                                              Query                query,
 352                                              QueryResult          result,
 353                                              IQueryableChangeData change_data)
 354                         {
 355                                 this.queryable = queryable;
 356                                 this.query = query;
 357                                 this.result = result;
 358                                 this.change_data = change_data;
 359                         }
 360
 361                         public void DoWork ()
 362                         {
 363                                 queryable.DoQuery (query, result, change_data);
 364                         }
 365                 }
 366
 367                 static public void DoOneQuery (Queryable            queryable,
 368                                                Query                query,
 369                                                QueryResult          result,
 370                                                IQueryableChangeData change_data)
 371                 {
 372                         if (queryable.AcceptQuery (query)) {
 373                                 QueryClosure qc = new QueryClosure (queryable, query, result, change_data);
 374                                 result.AttachWorker (qc);
 375                         }
 376                 }
 377
 378                 static void AddSearchTermInfo (QueryPart          part,
 379                                                SearchTermResponse response, StringBuilder sb)
 380                 {
 381                         if (part.Logic == QueryPartLogic.Prohibited)
 382                                 return;
 383
 384                         if (part is QueryPart_Or) {
 385                                 ICollection sub_parts;
 386                                 sub_parts = ((QueryPart_Or) part).SubParts;
 387                                 foreach (QueryPart qp in sub_parts)
 388                                         AddSearchTermInfo (qp, response, sb);
 389                                 return;
 390                         }
 391
 392                         if (! (part is QueryPart_Text))
 393                                 return;
 394
 395                         QueryPart_Text tp;
 396                         tp = (QueryPart_Text) part;
 397
 398                         string [] split;
 399                         split = tp.Text.Split (' ');
 400
 401                         // First, remove stop words
 402                         for (int i = 0; i < split.Length; ++i)
 403                                 if (LuceneCommon.IsStopWord (split [i]))
 404                                         split [i] = null;
 405
 406                         // Assemble the phrase minus stop words
 407                         sb.Length = 0;
 408                         for (int i = 0; i < split.Length; ++i) {
 409                                 if (split [i] == null)
 410                                         continue;
 411                                 if (sb.Length > 0)
 412                                         sb.Append (' ');
 413                                 sb.Append (split [i]);
 414                         }
 415                         response.ExactText.Add (sb.ToString ());
 416
 417                         // Now assemble a stemmed version
 418                         sb.Length = 0; // clear the previous value
 419                         for (int i = 0; i < split.Length; ++i) {
 420                                 if (split [i] == null)
 421                                         continue;
 422                                 if (sb.Length > 0)
 423                                         sb.Append (' ');
 424                                 sb.Append (LuceneCommon.Stem (split [i]));
 425                         }
 426                         response.StemmedText.Add (sb.ToString ());
 427                 }
 428
 429                 ////////////////////////////////////////////////////////
 430
 431                 static private void DehumanizeQuery (Query query)
 432                 {
 433                         // We need to remap any QueryPart_Human parts into
 434                         // lower-level part types.  First, we find any
 435                         // QueryPart_Human parts and explode them into
 436                         // lower-level types.
 437                         ArrayList new_parts = null;
 438                         foreach (QueryPart abstract_part in query.Parts) {
 439                                 if (abstract_part is QueryPart_Human) {
 440                                         QueryPart_Human human = abstract_part as QueryPart_Human;
 441                                         if (new_parts == null)
 442                                                 new_parts = new ArrayList ();
 443                                         foreach (QueryPart sub_part in QueryStringParser.Parse (human.QueryString))
 444                                                 new_parts.Add (sub_part);
 445                                 }
 446                         }
 447
 448                         // If we found any QueryPart_Human parts, copy the
 449                         // non-Human parts over and then replace the parts in
 450                         // the query.
 451                         if (new_parts != null) {
 452                                 foreach (QueryPart abstract_part in query.Parts) {
 453                                         if (! (abstract_part is QueryPart_Human))
 454                                                 new_parts.Add (abstract_part);
 455                                 }
 456
 457                                 query.ClearParts ();
 458                                 foreach (QueryPart part in new_parts)
 459                                         query.AddPart (part);
 460                         }
 461
 462                 }
 463
 464                 static private SearchTermResponse AssembleSearchTermResponse (Query query)
 465                 {
 466                         StringBuilder sb = new StringBuilder ();
 467                         SearchTermResponse search_term_response;
 468                         search_term_response = new SearchTermResponse ();
 469                         foreach (QueryPart part in query.Parts)
 470                                 AddSearchTermInfo (part, search_term_response, sb);
 471                         return search_term_response;
 472                 }
 473
 474                 static private void QueryEachQueryable (Query       query,
 475                                                         QueryResult result)
 476                 {
 477                         // The extra pair of calls to WorkerStart/WorkerFinished ensures:
 478                         // (1) that the QueryResult will fire the StartedEvent
 479                         // and FinishedEvent, even if no queryable accepts the
 480                         // query.
 481                         // (2) that the FinishedEvent will only get called when all of the
 482                         // backends have had time to finish.
 483
 484                         object dummy_worker = new object ();
 485
 486                         if (! result.WorkerStart (dummy_worker))
 487                                 return;
 488
 489                         foreach (Queryable queryable in queryables)
 490                                 DoOneQuery (queryable, query, result, null);
 491
 492                         result.WorkerFinished (dummy_worker);
 493                 }
 494
 495                 static public void DoQueryLocal (Query       query,
 496                                                  QueryResult result)
 497                 {
 498                         DehumanizeQuery (query);
 499
 500                         SearchTermResponse search_term_response;
 501                         search_term_response = AssembleSearchTermResponse (query);
 502                         query.ProcessSearchTermResponse (search_term_response);
 503
 504                         QueryEachQueryable (query, result);
 505                 }
 506
 507                 static public void DoQuery (Query                                query,
 508                                             QueryResult                          result,
 509                                             RequestMessageExecutor.AsyncResponse send_response)
 510                 {
 511                         DehumanizeQuery (query);
 512
 513                         SearchTermResponse search_term_response;
 514                         search_term_response = AssembleSearchTermResponse (query);
 515                         send_response (search_term_response);
 516
 517                         QueryEachQueryable (query, result);
 518                 }
 519
 520                 ////////////////////////////////////////////////////////
 521
 522                 static public string GetIndexInformation ()
 523                 {
 524                         StringBuilder builder = new StringBuilder ('\n');
 525
 526                         foreach (Queryable q in queryables) {
 527                                 QueryableStatus status = q.GetQueryableStatus ();
 528
 529                                 builder.Append ("Name: ").Append (status.Name).Append ('\n');
 530                                 builder.Append ("Count: ").Append (status.ItemCount).Append ('\n');
 531                                 builder.Append ("Indexing: ").Append (status.IsIndexing).Append ('\n');
 532                                 builder.Append ('\n');
 533                         }
 534
 535                         return builder.ToString ();
 536                 }
 537
 538                 ////////////////////////////////////////////////////////
 539
 540                 static public bool IsIndexing {
 541                         get {
 542                                 foreach (Queryable q in queryables) {
 543                                         QueryableStatus status = q.GetQueryableStatus ();
 544
 545                                         if (status.IsIndexing)
 546                                                 return true;
 547                                 }
 548
 549                                 return false;
 550                         }
 551                 }
 552         }
 553 }