beagled/QueryDriver.cs

   1 //
   2 // QueryDriver.cs
   3 //
   4 // Copyright (C) 2004 Novell, Inc.
   5 //
   6
   7 //
   8 // Permission is hereby granted, free of charge, to any person obtaining a
   9 // copy of this software and associated documentation files (the "Software"),
  10 // to deal in the Software without restriction, including without limitation
  11 // the rights to use, copy, modify, merge, publish, distribute, sublicense,
  12 // and/or sell copies of the Software, and to permit persons to whom the
  13 // Software is furnished to do so, subject to the following conditions:
  14 //
  15 // The above copyright notice and this permission notice shall be included in
  16 // all copies or substantial portions of the Software.
  17 //
  18 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  21 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  22 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  23 // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  24 // DEALINGS IN THE SOFTWARE.
  25 //
  26
  27 using System;
  28 using System.IO;
  29 using System.Collections;
  30 using System.Reflection;
  31 using System.Text;
  32 using System.Threading;
  33 using Beagle.Util;
  34 namespace Beagle.Daemon {
  35
  36         public class QueryDriver {
  37
  38                 // Enable or Disable specific Queryables by name
  39
  40                 static ArrayList allowed_queryables = new ArrayList ();
  41                 static ArrayList denied_queryables = new ArrayList ();
  42
  43                 static public void Allow (string name)
  44                 {
  45                         allowed_queryables.Add (name.ToLower ());
  46                 }
  47
  48                 static public void Deny (string name)
  49                 {
  50                         denied_queryables.Add (name.ToLower ());
  51                 }
  52
  53                 static private bool UseQueryable (string name)
  54                 {
  55                         if (allowed_queryables.Count == 0
  56                             && denied_queryables.Count == 0)
  57                                 return true;
  58
  59                         name = name.ToLower ();
  60
  61                         if (allowed_queryables.Count > 0) {
  62                                 foreach (string allowed in allowed_queryables)
  63                                         if (name == allowed)
  64                                                 return true;
  65                                 return false;
  66                         }
  67
  68                         foreach (string denied in denied_queryables)
  69                                 if (name == denied)
  70                                         return false;
  71                         return true;
  72
  73                 }
  74
  75                 //////////////////////////////////////////////////////////////////////////////////////
  76
  77                 // Paths to static queryables
  78
  79                 static ArrayList static_queryables = new ArrayList ();
  80
  81                 static public void AddStaticQueryable (string path) {
  82
  83                         if (! static_queryables.Contains (path))
  84                                 static_queryables.Add (path);
  85                 }
  86
  87                 //////////////////////////////////////////////////////////////////////////////////////
  88
  89                 // Use introspection to find all classes that implement IQueryable, the construct
  90                 // associated Queryables objects.
  91
  92                 static ArrayList queryables = new ArrayList ();
  93                 static Hashtable iqueryable_to_queryable = new Hashtable ();
  94
  95                 static bool ThisApiSoVeryIsBroken (Type m, object criteria)
  96                 {
  97                         return m == (Type) criteria;
  98                 }
  99
 100                 static bool TypeImplementsInterface (Type t, Type iface)
 101                 {
 102                         Type[] impls = t.FindInterfaces (new TypeFilter (ThisApiSoVeryIsBroken),
 103                                                          iface);
 104                         return impls.Length > 0;
 105                 }
 106
 107                 // For every type in the assembly that
 108                 // (1) implements IQueryable
 109                 // (2) Has a QueryableFlavor attribute attached
 110                 // assemble a Queryable object and stick it into our list of queryables.
 111                 static void ScanAssembly (Assembly assembly)
 112                 {
 113                         int count = 0;
 114
 115                         foreach (Type type in GetQueryableTypes (assembly)) {
 116                                 foreach (QueryableFlavor flavor in GetQueryableFlavors (type)) {
 117                                         if (! UseQueryable (flavor.Name))
 118                                                 continue;
 119
 120                                         if (flavor.RequireInotify && ! Inotify.Enabled) {
 121                                                 Logger.Log.Warn ("Can't start backend '{0}' without inotify", flavor.Name);
 122                                                 continue;
 123                                         }
 124
 125                                         if (flavor.RequireExtendedAttributes && ! ExtendedAttribute.Supported) {
 126                                                 Logger.Log.Warn ("Can't start backend '{0}' without extended attributes", flavor.Name);
 127                                                 continue;
 128                                         }
 129
 130                                         IQueryable iq = null;
 131                                         try {
 132                                                 iq = Activator.CreateInstance (type) as IQueryable;
 133                                         } catch (Exception e) {
 134                                                 Logger.Log.Error ("Caught exception while instantiating {0} backend", flavor.Name);
 135                                                 Logger.Log.Error (e);
 136                                         }
 137
 138                                         if (iq != null) {
 139                                                 Queryable q = new Queryable (flavor, iq);
 140                                                 queryables.Add (q);
 141                                                 iqueryable_to_queryable [iq] = q;
 142                                                 ++count;
 143                                                 break;
 144                                         }
 145                                 }
 146                         }
 147                         Logger.Log.Debug ("Found {0} types in {1}", count, assembly.FullName);
 148                 }
 149
 150                 ////////////////////////////////////////////////////////
 151
 152                 // Scans PathFinder.SystemIndexesDir after available
 153                 // system-wide indexes.
 154                 static void LoadSystemIndexes ()
 155                 {
 156                         if (!Directory.Exists (PathFinder.SystemIndexesDir))
 157                                 return;
 158
 159                         int count = 0;
 160
 161                         foreach (DirectoryInfo index_dir in new DirectoryInfo (PathFinder.SystemIndexesDir).GetDirectories ()) {
 162                                 if (! UseQueryable (index_dir.Name))
 163                                         continue;
 164
 165                                 if (LoadStaticQueryable (index_dir, QueryDomain.System))
 166                                         count++;
 167                         }
 168
 169                         Logger.Log.Debug ("Found {0} system-wide indexes", count);
 170                 }
 171
 172                 // Scans configuration for user-specified index paths
 173                 // to load StaticQueryables from.
 174                 static void LoadStaticQueryables ()
 175                 {
 176                         int count = 0;
 177
 178                         foreach (string path in Conf.Daemon.StaticQueryables)
 179                                 static_queryables.Add (path);
 180
 181                         foreach (string path in static_queryables) {
 182                                 DirectoryInfo index_dir = new DirectoryInfo (path);
 183
 184                                 if (!index_dir.Exists)
 185                                         continue;
 186
 187                                 // FIXME: QueryDomain might be other than local
 188                                 if (LoadStaticQueryable (index_dir, QueryDomain.Local))
 189                                         count++;
 190                         }
 191
 192                         Logger.Log.Debug ("Found {0} user-configured static queryables", count);
 193                 }
 194
 195                 // Instantiates and loads a StaticQueryable from an index directory
 196                 static private bool LoadStaticQueryable (DirectoryInfo index_dir, QueryDomain query_domain)
 197                 {
 198                         StaticQueryable static_queryable = null;
 199
 200                         if (!index_dir.Exists)
 201                                 return false;
 202
 203                         try {
 204                                 static_queryable = new StaticQueryable (index_dir.Name, index_dir.FullName, true);
 205                         } catch (Exception e) {
 206                                 Logger.Log.Error ("Caught exception while instantiating static queryable: {0}", index_dir.Name);
 207                                 Logger.Log.Error (e);
 208                                 return false;
 209                         }
 210
 211                         if (static_queryable != null) {
 212                                 QueryableFlavor flavor = new QueryableFlavor ();
 213                                 flavor.Name = index_dir.Name;
 214                                 flavor.Domain = query_domain;
 215
 216                                 Queryable queryable = new Queryable (flavor, static_queryable);
 217                                 queryables.Add (queryable);
 218
 219                                 iqueryable_to_queryable [static_queryable] = queryable;
 220
 221                                 return true;
 222                         }
 223
 224                         return false;
 225                 }
 226
 227                 ////////////////////////////////////////////////////////
 228
 229                 static private Type[] GetQueryableTypes (Assembly assembly)
 230                 {
 231                         Type[] assembly_types = assembly.GetTypes ();
 232                         ArrayList types = new ArrayList (assembly_types.Length);
 233
 234                         foreach (Type type in assembly_types)
 235                                 if (TypeImplementsInterface (type, typeof (IQueryable)))
 236                                         types.Add (type);
 237
 238                         return (Type[]) types.ToArray (typeof (Type));
 239                 }
 240
 241                 static private QueryableFlavor[] GetQueryableFlavors (Type type)
 242                 {
 243                         object[] attributes = Attribute.GetCustomAttributes (type);
 244                         ArrayList flavors = new ArrayList (attributes.Length);
 245
 246                         foreach (object obj in attributes) {
 247                                         QueryableFlavor flavor = obj as QueryableFlavor;
 248                                         if (flavor != null)
 249                                                 flavors.Add (flavor);
 250                         }
 251
 252                         return (QueryableFlavor[]) flavors.ToArray (typeof (QueryableFlavor));
 253                 }
 254
 255                 static private Assembly[] GetAssemblies ()
 256                 {
 257                         Assembly[] assemblies;
 258                         int i = 0;
 259                         DirectoryInfo backends = new DirectoryInfo (PathFinder.BackendDir);
 260
 261                         if (backends.Exists) {
 262                                 FileInfo[] assembly_files = backends.GetFiles ("*.dll");
 263                                 assemblies = new Assembly [assembly_files.Length + 1];
 264
 265                                 foreach (FileInfo assembly in assembly_files)
 266                                         assemblies[i++] = Assembly.LoadFile (assembly.ToString ());
 267
 268                         } else {
 269                                 assemblies = new Assembly [1];
 270                         }
 271
 272                         assemblies[i] = Assembly.GetExecutingAssembly ();
 273
 274                         return assemblies;
 275                 }
 276
 277                 static public void Start ()
 278                 {
 279                         foreach (Assembly assembly in GetAssemblies ()) {
 280                                 ScanAssembly (assembly);
 281
 282                                 // This allows backends to define their
 283                                 // own executors.
 284                                 Server.ScanAssemblyForExecutors (assembly);
 285                         }
 286
 287                         // FIXME: This is not nice at all
 288                         if (UseQueryable ("system"))
 289                                 LoadSystemIndexes ();
 290                         if (UseQueryable ("static"))
 291                                 LoadStaticQueryables ();
 292
 293                         foreach (Queryable q in queryables)
 294                                 q.Start ();
 295                 }
 296
 297                 static public string ListBackends ()
 298                 {
 299                         string ret = "User:\n";
 300                         foreach (Assembly assembly in GetAssemblies ())
 301                                 foreach (Type type in GetQueryableTypes (assembly))
 302                                         foreach (QueryableFlavor flavor in GetQueryableFlavors (type))
 303                                 ret += String.Format (" - {0}\n", flavor.Name);
 304
 305                         if (!Directory.Exists (PathFinder.SystemIndexesDir))
 306                                 return ret;
 307
 308                         ret += "System:\n";
 309                         foreach (DirectoryInfo index_dir in new DirectoryInfo (PathFinder.SystemIndexesDir).GetDirectories ()) {
 310                                 ret += String.Format (" - {0}\n", index_dir.Name);
 311                         }
 312
 313                         return ret;
 314                 }
 315
 316                 static public Queryable GetQueryable (string name)
 317                 {
 318                         foreach (Queryable q in queryables) {
 319                                 if (q.Name == name)
 320                                         return q;
 321                         }
 322
 323                         return null;
 324                 }
 325
 326                 ////////////////////////////////////////////////////////
 327
 328                 public delegate void ChangedHandler (Queryable            queryable,
 329                                                      IQueryableChangeData changeData);
 330
 331                 static public event ChangedHandler ChangedEvent;
 332
 333                 // A method to fire the ChangedEvent event.
 334                 static public void QueryableChanged (IQueryable           iqueryable,
 335                                                      IQueryableChangeData change_data)
 336                 {
 337                         if (ChangedEvent != null) {
 338                                 Queryable queryable = iqueryable_to_queryable [iqueryable] as Queryable;
 339                                 ChangedEvent (queryable, change_data);
 340                         }
 341                 }
 342
 343                 ////////////////////////////////////////////////////////
 344
 345                 private class MarkAndForwardHits : IQueryResult {
 346
 347                         IQueryResult result;
 348                         string name;
 349
 350                         public MarkAndForwardHits (IQueryResult result, string name)
 351                         {
 352                                 this.result = result;
 353                                 this.name = name;
 354                         }
 355
 356                         public void Add (ICollection some_hits)
 357                         {
 358                                 foreach (Hit hit in some_hits)
 359                                         if (hit != null)
 360                                                 hit.SourceObjectName = name;
 361                                 result.Add (some_hits);
 362                         }
 363
 364                         public void Subtract (ICollection some_uris)
 365                         {
 366                                 result.Subtract (some_uris);
 367                         }
 368                 }
 369
 370                 private class QueryClosure : IQueryWorker {
 371
 372                         Queryable queryable;
 373                         Query query;
 374                         IQueryResult result;
 375                         IQueryableChangeData change_data;
 376
 377                         public QueryClosure (Queryable            queryable,
 378                                              Query                query,
 379                                              QueryResult          result,
 380                                              IQueryableChangeData change_data)
 381                         {
 382                                 this.queryable = queryable;
 383                                 this.query = query;
 384                                 this.result = new MarkAndForwardHits (result, queryable.Name);
 385                                 this.change_data = change_data;
 386                         }
 387
 388                         public void DoWork ()
 389                         {
 390                                 queryable.DoQuery (query, result, change_data);
 391                         }
 392                 }
 393
 394                 static public void DoOneQuery (Queryable            queryable,
 395                                                Query                query,
 396                                                QueryResult          result,
 397                                                IQueryableChangeData change_data)
 398                 {
 399                         if (queryable.AcceptQuery (query)) {
 400                                 QueryClosure qc = new QueryClosure (queryable, query, result, change_data);
 401                                 result.AttachWorker (qc);
 402                         }
 403                 }
 404
 405                 static void AddSearchTermInfo (QueryPart          part,
 406                                                SearchTermResponse response)
 407                 {
 408                         if (part.Logic == QueryPartLogic.Prohibited)
 409                                 return;
 410
 411                         if (part is QueryPart_Or) {
 412                                 ICollection sub_parts;
 413                                 sub_parts = ((QueryPart_Or) part).SubParts;
 414                                 foreach (QueryPart qp in sub_parts)
 415                                         AddSearchTermInfo (qp, response);
 416                                 return;
 417                         }
 418
 419                         if (! (part is QueryPart_Text))
 420                                 return;
 421
 422                         QueryPart_Text tp;
 423                         tp = (QueryPart_Text) part;
 424
 425                         string [] split;
 426                         split = tp.Text.Split (' ');
 427
 428                         // First, remove stop words
 429                         for (int i = 0; i < split.Length; ++i)
 430                                 if (LuceneCommon.IsStopWord (split [i]))
 431                                         split [i] = null;
 432
 433                         // Assemble the phrase minus stop words
 434                         StringBuilder sb;
 435                         sb = new StringBuilder ();
 436                         for (int i = 0; i < split.Length; ++i) {
 437                                 if (split [i] == null)
 438                                         continue;
 439                                 if (sb.Length > 0)
 440                                         sb.Append (' ');
 441                                 sb.Append (split [i]);
 442                         }
 443                         response.ExactText.Add (sb.ToString ());
 444
 445                         // Now assemble a stemmed version
 446                         sb.Length = 0; // clear the previous value
 447                         for (int i = 0; i < split.Length; ++i) {
 448                                 if (split [i] == null)
 449                                         continue;
 450                                 if (sb.Length > 0)
 451                                         sb.Append (' ');
 452                                 sb.Append (LuceneCommon.Stem (split [i]));
 453                         }
 454                         response.StemmedText.Add (sb.ToString ());
 455                 }
 456
 457                 ////////////////////////////////////////////////////////
 458
 459                 static private void DehumanizeQuery (Query query)
 460                 {
 461                         // We need to remap any QueryPart_Human parts into
 462                         // lower-level part types.  First, we find any
 463                         // QueryPart_Human parts and explode them into
 464                         // lower-level types.
 465                         ArrayList new_parts = null;
 466                         foreach (QueryPart abstract_part in query.Parts) {
 467                                 if (abstract_part is QueryPart_Human) {
 468                                         QueryPart_Human human = abstract_part as QueryPart_Human;
 469                                         if (new_parts == null)
 470                                                 new_parts = new ArrayList ();
 471                                         foreach (QueryPart sub_part in QueryStringParser.Parse (human.QueryString))
 472                                                 new_parts.Add (sub_part);
 473                                 }
 474                         }
 475
 476                         // If we found any QueryPart_Human parts, copy the
 477                         // non-Human parts over and then replace the parts in
 478                         // the query.
 479                         if (new_parts != null) {
 480                                 foreach (QueryPart abstract_part in query.Parts) {
 481                                         if (! (abstract_part is QueryPart_Human))
 482                                                 new_parts.Add (abstract_part);
 483                                 }
 484
 485                                 query.ClearParts ();
 486                                 foreach (QueryPart part in new_parts)
 487                                         query.AddPart (part);
 488                         }
 489
 490                 }
 491
 492                 static private SearchTermResponse AssembleSearchTermResponse (Query query)
 493                 {
 494                         SearchTermResponse search_term_response;
 495                         search_term_response = new SearchTermResponse ();
 496                         foreach (QueryPart part in query.Parts)
 497                                 AddSearchTermInfo (part, search_term_response);
 498                         return search_term_response;
 499                 }
 500
 501                 static private void QueryEachQueryable (Query       query,
 502                                                         QueryResult result)
 503                 {
 504                         // The extra pair of calls to WorkerStart/WorkerFinished ensures:
 505                         // (1) that the QueryResult will fire the StartedEvent
 506                         // and FinishedEvent, even if no queryable accepts the
 507                         // query.
 508                         // (2) that the FinishedEvent will only get called when all of the
 509                         // backends have had time to finish.
 510
 511                         object dummy_worker = new object ();
 512
 513                         if (! result.WorkerStart (dummy_worker))
 514                                 return;
 515
 516                         foreach (Queryable queryable in queryables)
 517                                 DoOneQuery (queryable, query, result, null);
 518
 519                         result.WorkerFinished (dummy_worker);
 520                 }
 521
 522                 static public void DoQueryLocal (Query       query,
 523                                                  QueryResult result)
 524                 {
 525                         DehumanizeQuery (query);
 526
 527                         SearchTermResponse search_term_response;
 528                         search_term_response = AssembleSearchTermResponse (query);
 529                         query.ProcessSearchTermResponse (search_term_response);
 530
 531                         QueryEachQueryable (query, result);
 532                 }
 533
 534                 static public void DoQuery (Query                                query,
 535                                             QueryResult                          result,
 536                                             RequestMessageExecutor.AsyncResponse send_response)
 537                 {
 538                         DehumanizeQuery (query);
 539
 540                         SearchTermResponse search_term_response;
 541                         search_term_response = AssembleSearchTermResponse (query);
 542                         send_response (search_term_response);
 543
 544                         QueryEachQueryable (query, result);
 545                 }
 546
 547                 ////////////////////////////////////////////////////////
 548
 549                 static public string GetIndexInformation ()
 550                 {
 551                         StringBuilder builder = new StringBuilder ("\n");
 552
 553                         foreach (Queryable q in queryables) {
 554                                 builder.AppendFormat ("Name: {0}\n", q.Name);
 555                                 builder.AppendFormat ("Count: {0}\n", q.GetItemCount ());
 556                                 builder.Append ("\n");
 557                         }
 558
 559                         return builder.ToString ();
 560                 }
 561         }
 562 }