?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merging from head
[beagle.git] / beagled / QueryDriver.cs
blobedfd9a07c633bcf509af25cda31039e21420215b
1 //
2 // QueryDriver.cs
3 //
4 // Copyright (C) 2004 Novell, Inc.
5 //
6
7 //
8 // Permission is hereby granted, free of charge, to any person obtaining a
9 // copy of this software and associated documentation files (the "Software"),
10 // to deal in the Software without restriction, including without limitation
11 // the rights to use, copy, modify, merge, publish, distribute, sublicense,
12 // and/or sell copies of the Software, and to permit persons to whom the
13 // Software is furnished to do so, subject to the following conditions:
14 //
15 // The above copyright notice and this permission notice shall be included in
16 // all copies or substantial portions of the Software.
17 //
18 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
21 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
22 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
23 // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
24 // DEALINGS IN THE SOFTWARE.
25 //
26
27 using System;
28 using System.IO;
29 using System.Collections;
30 using System.Reflection;
31 using System.Text;
32 using System.Threading;
33 using Beagle.Util;
34 namespace Beagle.Daemon {
35
36 public class QueryDriver {
37
38 // Contains list of queryables explicitly asked by --allow-backend or --backend name
39 // --allow-backend/--backend name : dont read config information and only start backend 'name'
40 static ArrayList excl_allowed_queryables = new ArrayList ();
41
42 // Contains list of denied queryables from config/arguments (every queryable is enabled by default)
43 // Unless overruled by --allow-backend/--backend name, deny backend only if names appears here.
44 static ArrayList denied_queryables = new ArrayList ();
45
46 static bool to_read_conf = true; // read backends from conf if true
47 static bool done_reading_conf = false;
48
49 static private void ReadBackendsFromConf ()
50 {
51 if (! to_read_conf || done_reading_conf)
52 return;
53
54 // set flag here to stop Allow() from calling ReadBackendsFromConf() again
55 done_reading_conf = true;
56
57 // To allow static indexes, "static" should be in allowed_queryables
58 if (Conf.Daemon.AllowStaticBackend)
59 Allow ("static");
60
61 if (Conf.Daemon.DeniedBackends == null)
62 return;
63
64 foreach (string name in Conf.Daemon.DeniedBackends)
65 denied_queryables.Add (name.ToLower ());
66 }
67
68 static public void OnlyAllow (string name)
69 {
70 excl_allowed_queryables.Add (name.ToLower ());
71 to_read_conf = false;
72 }
73
74 static public void Allow (string name)
75 {
76 if (! done_reading_conf && to_read_conf)
77 ReadBackendsFromConf ();
78
79 denied_queryables.Remove (name.ToLower ());
80 }
81
82 static public void Deny (string name)
83 {
84 if (! done_reading_conf && to_read_conf)
85 ReadBackendsFromConf ();
86
87 name = name.ToLower ();
88 if (!denied_queryables.Contains (name))
89 denied_queryables.Add (name);
90 }
91
92 static private bool UseQueryable (string name)
93 {
94 name = name.ToLower ();
95
96 if (excl_allowed_queryables.Contains (name))
97 return true;
98 if (excl_allowed_queryables.Count != 0)
99 return false;
100
101 if (denied_queryables.Contains (name))
102 return false;
103
104 return true;
105 }
106
107 //////////////////////////////////////////////////////////////////////////////////////
108
109 // Paths to static queryables
110
111 static ArrayList static_queryables = new ArrayList ();
112
113 static public void AddStaticQueryable (string path) {
114
115 if (! static_queryables.Contains (path))
116 static_queryables.Add (path);
117 }
118
119 //////////////////////////////////////////////////////////////////////////////////////
120
121 // Delay before starting the indexing process
122
123 static int indexing_delay = 60; // Default to 60 seconds
124
125 public static int IndexingDelay {
126 set { indexing_delay = value; }
127 }
128
129 //////////////////////////////////////////////////////////////////////////////////////
130
131 // Use introspection to find all classes that implement IQueryable, the construct
132 // associated Queryables objects.
133
134 static ArrayList queryables = new ArrayList ();
135 static Hashtable iqueryable_to_queryable = new Hashtable ();
136
137 static bool ThisApiSoVeryIsBroken (Type m, object criteria)
138 {
139 return m == (Type) criteria;
140 }
141
142 static bool TypeImplementsInterface (Type t, Type iface)
143 {
144 Type[] impls = t.FindInterfaces (new TypeFilter (ThisApiSoVeryIsBroken),
145 iface);
146 return impls.Length > 0;
147 }
148
149 // Find the types in the assembly that
150 // (1) register themselves in AssemblyInfo.cs:IQueryableTypes and
151 // (2) has a QueryableFlavor attribute attached
152 // assemble a Queryable object and stick it into our list of queryables.
153 static void ScanAssemblyForQueryables (Assembly assembly)
154 {
155 int count = 0;
156
157 foreach (Type type in ReflectionFu.GetTypesFromAssemblyAttribute (assembly, typeof (IQueryableTypesAttribute))) {
158 bool type_accepted = false;
159 foreach (QueryableFlavor flavor in ReflectionFu.ScanTypeForAttribute (type, typeof (QueryableFlavor))) {
160 if (! UseQueryable (flavor.Name))
161 continue;
162
163 if (flavor.RequireInotify && ! Inotify.Enabled) {
164 Logger.Log.Warn ("Can't start backend '{0}' without inotify", flavor.Name);
165 continue;
166 }
167
168 if (flavor.RequireExtendedAttributes && ! ExtendedAttribute.Supported) {
169 Logger.Log.Warn ("Can't start backend '{0}' without extended attributes", flavor.Name);
170 continue;
171 }
172
173 IQueryable iq = null;
174 try {
175 iq = Activator.CreateInstance (type) as IQueryable;
176 } catch (Exception e) {
177 Logger.Log.Error (e, "Caught exception while instantiating {0} backend", flavor.Name);
178 }
179
180 if (iq != null) {
181 Queryable q = new Queryable (flavor, iq);
182 queryables.Add (q);
183 iqueryable_to_queryable [iq] = q;
184 ++count;
185 type_accepted = true;
186 break;
187 }
188 }
189
190 if (! type_accepted)
191 continue;
192
193 object[] attributes = type.GetCustomAttributes (false);
194 foreach (object attribute in attributes) {
195 PropertyKeywordMapping mapping = attribute as PropertyKeywordMapping;
196 if (mapping == null)
197 continue;
198 //Logger.Log.Debug (mapping.Keyword + " => "
199 // + mapping.PropertyName +
200 // + " is-keyword=" + mapping.IsKeyword + " ("
201 // + mapping.Description + ") "
202 // + "(" + type.FullName + ")");
203 PropertyKeywordFu.RegisterMapping (mapping);
204 }
205
206 }
207 Logger.Log.Debug ("Found {0} backends in {1}", count, assembly.Location);
208 }
209
210 ////////////////////////////////////////////////////////
211
212 public static void ReadKeywordMappings ()
213 {
214 Logger.Log.Debug ("Reading mapping from filters");
215 ArrayList assemblies = ReflectionFu.ScanEnvironmentForAssemblies ("BEAGLE_FILTER_PATH", PathFinder.FilterDir);
216
217 foreach (Assembly assembly in assemblies) {
218 foreach (Type type in ReflectionFu.GetTypesFromAssemblyAttribute (assembly, typeof (FilterTypesAttribute))) {
219 object[] attributes = type.GetCustomAttributes (false);
220 foreach (object attribute in attributes) {
221
222 PropertyKeywordMapping mapping = attribute as PropertyKeywordMapping;
223 if (mapping == null)
224 continue;
225 //Logger.Log.Debug (mapping.Keyword + " => "
226 // + mapping.PropertyName
227 // + " is-keyword=" + mapping.IsKeyword + " ("
228 // + mapping.Description + ") "
229 // + "(" + type.FullName + ")");
230 PropertyKeywordFu.RegisterMapping (mapping);
231 }
232 }
233 }
234 }
235
236 ////////////////////////////////////////////////////////
237
238 // Scans PathFinder.SystemIndexesDir after available
239 // system-wide indexes.
240 static void LoadSystemIndexes ()
241 {
242 if (!Directory.Exists (PathFinder.SystemIndexesDir))
243 return;
244
245 Logger.Log.Info ("Loading system static indexes.");
246
247 int count = 0;
248
249 foreach (DirectoryInfo index_dir in new DirectoryInfo (PathFinder.SystemIndexesDir).GetDirectories ()) {
250 if (! UseQueryable (index_dir.Name))
251 continue;
252
253 if (LoadStaticQueryable (index_dir, QueryDomain.System))
254 count++;
255 }
256
257 Logger.Log.Info ("Found {0} system-wide indexes.", count);
258 }
259
260 // Scans configuration for user-specified index paths
261 // to load StaticQueryables from.
262 static void LoadStaticQueryables ()
263 {
264 int count = 0;
265
266 if (UseQueryable ("static")) {
267 Logger.Log.Info ("Loading user-configured static indexes.");
268 foreach (string path in Conf.Daemon.StaticQueryables)
269 static_queryables.Add (path);
270 }
271
272 foreach (string path in static_queryables) {
273 DirectoryInfo index_dir = new DirectoryInfo (StringFu.SanitizePath (path));
274
275 if (!index_dir.Exists)
276 continue;
277
278 // FIXME: QueryDomain might be other than local
279 if (LoadStaticQueryable (index_dir, QueryDomain.Local))
280 count++;
281 }
282
283 Logger.Log.Info ("Found {0} user-configured static indexes..", count);
284 }
285
286 // Instantiates and loads a StaticQueryable from an index directory
287 static private bool LoadStaticQueryable (DirectoryInfo index_dir, QueryDomain query_domain)
288 {
289 StaticQueryable static_queryable = null;
290
291 if (!index_dir.Exists)
292 return false;
293
294 try {
295 static_queryable = new StaticQueryable (index_dir.Name, index_dir.FullName, true);
296 } catch (InvalidOperationException) {
297 Logger.Log.Warn ("Unable to create read-only index (likely due to index version mismatch): {0}", index_dir.FullName);
298 return false;
299 } catch (Exception e) {
300 Logger.Log.Error (e, "Caught exception while instantiating static queryable: {0}", index_dir.Name);
301 return false;
302 }
303
304 if (static_queryable != null) {
305 QueryableFlavor flavor = new QueryableFlavor ();
306 flavor.Name = index_dir.Name;
307 flavor.Domain = query_domain;
308
309 Queryable queryable = new Queryable (flavor, static_queryable);
310 queryables.Add (queryable);
311
312 iqueryable_to_queryable [static_queryable] = queryable;
313
314 return true;
315 }
316
317 return false;
318 }
319
320 ////////////////////////////////////////////////////////
321
322 private static ArrayList assemblies = null;
323
324 // Perform expensive initialization steps all at once.
325 // Should be done before SignalHandler comes into play.
326 static public void Init ()
327 {
328 ReadBackendsFromConf ();
329 assemblies = ReflectionFu.ScanEnvironmentForAssemblies ("BEAGLE_BACKEND_PATH", PathFinder.BackendDir);
330 }
331
332 private static bool queryables_started = false;
333
334 static public void Start ()
335 {
336 // Only add the executing assembly if we haven't already loaded it.
337 if (assemblies.IndexOf (Assembly.GetExecutingAssembly ()) == -1)
338 assemblies.Add (Assembly.GetExecutingAssembly ());
339
340 foreach (Assembly assembly in assemblies) {
341 ScanAssemblyForQueryables (assembly);
342
343 // This allows backends to define their
344 // own executors.
345 Server.ScanAssemblyForExecutors (assembly);
346 }
347
348 assemblies = null;
349
350 ReadKeywordMappings ();
351
352 LoadSystemIndexes ();
353 LoadStaticQueryables ();
354
355 if (indexing_delay <= 0 || Environment.GetEnvironmentVariable ("BEAGLE_EXERCISE_THE_DOG") != null)
356 StartQueryables ();
357 else {
358 Logger.Log.Debug ("Waiting {0} seconds before starting queryables", indexing_delay);
359 GLib.Timeout.Add ((uint) indexing_delay * 1000, new GLib.TimeoutHandler (StartQueryables));
360 }
361 }
362
363 static private bool StartQueryables ()
364 {
365 Logger.Log.Debug ("Starting queryables");
366
367 foreach (Queryable q in queryables) {
368 Logger.Log.Info ("Starting backend: '{0}'", q.Name);
369 q.Start ();
370 }
371
372 queryables_started = true;
373
374 return false;
375 }
376
377 static public string ListBackends ()
378 {
379 ArrayList assemblies = ReflectionFu.ScanEnvironmentForAssemblies ("BEAGLE_BACKEND_PATH", PathFinder.BackendDir);
380
381 // Only add the executing assembly if we haven't already loaded it.
382 if (assemblies.IndexOf (Assembly.GetExecutingAssembly ()) == -1)
383 assemblies.Add (Assembly.GetExecutingAssembly ());
384
385 string ret = "User:\n";
386
387 foreach (Assembly assembly in assemblies) {
388 foreach (Type type in ReflectionFu.GetTypesFromAssemblyAttribute (assembly, typeof (IQueryableTypesAttribute))) {
389 foreach (QueryableFlavor flavor in ReflectionFu.ScanTypeForAttribute (type, typeof (QueryableFlavor)))
390 ret += String.Format (" - {0}\n", flavor.Name);
391 }
392 }
393
394 if (!Directory.Exists (PathFinder.SystemIndexesDir))
395 return ret;
396
397 ret += "System:\n";
398 foreach (DirectoryInfo index_dir in new DirectoryInfo (PathFinder.SystemIndexesDir).GetDirectories ()) {
399 ret += String.Format (" - {0}\n", index_dir.Name);
400 }
401
402 return ret;
403 }
404
405 static public Queryable GetQueryable (string name)
406 {
407 foreach (Queryable q in queryables) {
408 if (q.Name == name)
409 return q;
410 }
411
412 return null;
413 }
414
415 static public Queryable GetQueryable (IQueryable iqueryable)
416 {
417 return (Queryable) iqueryable_to_queryable [iqueryable];
418 }
419
420 ////////////////////////////////////////////////////////
421
422 public delegate void ChangedHandler (Queryable queryable,
423 IQueryableChangeData changeData);
424
425 static public event ChangedHandler ChangedEvent;
426
427 // A method to fire the ChangedEvent event.
428 static public void QueryableChanged (IQueryable iqueryable,
429 IQueryableChangeData change_data)
430 {
431 if (ChangedEvent != null) {
432 Queryable queryable = iqueryable_to_queryable [iqueryable] as Queryable;
433 ChangedEvent (queryable, change_data);
434 }
435 }
436
437 ////////////////////////////////////////////////////////
438
439 private class QueryClosure : IQueryWorker {
440
441 Queryable queryable;
442 Query query;
443 IQueryResult result;
444 IQueryableChangeData change_data;
445
446 public QueryClosure (Queryable queryable,
447 Query query,
448 QueryResult result,
449 IQueryableChangeData change_data)
450 {
451 this.queryable = queryable;
452 this.query = query;
453 this.result = result;
454 this.change_data = change_data;
455 }
456
457 public void DoWork ()
458 {
459 queryable.DoQuery (query, result, change_data);
460 }
461 }
462
463 static public void DoOneQuery (Queryable queryable,
464 Query query,
465 QueryResult result,
466 IQueryableChangeData change_data)
467 {
468 if (queryable.AcceptQuery (query)) {
469 QueryClosure qc = new QueryClosure (queryable, query, result, change_data);
470 result.AttachWorker (qc);
471 }
472 }
473
474 static void AddSearchTermInfo (QueryPart part,
475 SearchTermResponse response, StringBuilder sb)
476 {
477 if (part.Logic == QueryPartLogic.Prohibited)
478 return;
479
480 if (part is QueryPart_Or) {
481 ICollection sub_parts;
482 sub_parts = ((QueryPart_Or) part).SubParts;
483 foreach (QueryPart qp in sub_parts)
484 AddSearchTermInfo (qp, response, sb);
485 return;
486 }
487
488 if (! (part is QueryPart_Text))
489 return;
490
491 QueryPart_Text tp;
492 tp = (QueryPart_Text) part;
493
494 string [] split;
495 split = tp.Text.Split (' ');
496
497 // First, remove stop words
498 for (int i = 0; i < split.Length; ++i)
499 if (LuceneCommon.IsStopWord (split [i]))
500 split [i] = null;
501
502 // Assemble the phrase minus stop words
503 sb.Length = 0;
504 for (int i = 0; i < split.Length; ++i) {
505 if (split [i] == null)
506 continue;
507 if (sb.Length > 0)
508 sb.Append (' ');
509 sb.Append (split [i]);
510 }
511 response.ExactText.Add (sb.ToString ());
512
513 // Now assemble a stemmed version
514 sb.Length = 0; // clear the previous value
515 for (int i = 0; i < split.Length; ++i) {
516 if (split [i] == null)
517 continue;
518 if (sb.Length > 0)
519 sb.Append (' ');
520 sb.Append (LuceneCommon.Stem (split [i]));
521 }
522 response.StemmedText.Add (sb.ToString ());
523 }
524
525 ////////////////////////////////////////////////////////
526
527 static private void DehumanizeQuery (Query query)
528 {
529 // We need to remap any QueryPart_Human parts into
530 // lower-level part types. First, we find any
531 // QueryPart_Human parts and explode them into
532 // lower-level types.
533 ArrayList new_parts = null;
534 foreach (QueryPart abstract_part in query.Parts) {
535 if (abstract_part is QueryPart_Human) {
536 QueryPart_Human human = abstract_part as QueryPart_Human;
537 if (new_parts == null)
538 new_parts = new ArrayList ();
539 foreach (QueryPart sub_part in QueryStringParser.Parse (human.QueryString))
540 new_parts.Add (sub_part);
541 }
542 }
543
544 // If we found any QueryPart_Human parts, copy the
545 // non-Human parts over and then replace the parts in
546 // the query.
547 if (new_parts != null) {
548 foreach (QueryPart abstract_part in query.Parts) {
549 if (! (abstract_part is QueryPart_Human))
550 new_parts.Add (abstract_part);
551 }
552
553 query.ClearParts ();
554 foreach (QueryPart part in new_parts)
555 query.AddPart (part);
556 }
557
558 }
559
560 static private SearchTermResponse AssembleSearchTermResponse (Query query)
561 {
562 StringBuilder sb = new StringBuilder ();
563 SearchTermResponse search_term_response;
564 search_term_response = new SearchTermResponse ();
565 foreach (QueryPart part in query.Parts)
566 AddSearchTermInfo (part, search_term_response, sb);
567 return search_term_response;
568 }
569
570 static private void QueryEachQueryable (Query query,
571 QueryResult result)
572 {
573 // The extra pair of calls to WorkerStart/WorkerFinished ensures:
574 // (1) that the QueryResult will fire the StartedEvent
575 // and FinishedEvent, even if no queryable accepts the
576 // query.
577 // (2) that the FinishedEvent will only get called when all of the
578 // backends have had time to finish.
579
580 object dummy_worker = new object ();
581
582 if (! result.WorkerStart (dummy_worker))
583 return;
584
585 foreach (Queryable queryable in queryables)
586 DoOneQuery (queryable, query, result, null);
587
588 result.WorkerFinished (dummy_worker);
589 }
590
591 static public void DoQueryLocal (Query query,
592 QueryResult result)
593 {
594 DehumanizeQuery (query);
595
596 SearchTermResponse search_term_response;
597 search_term_response = AssembleSearchTermResponse (query);
598 query.ProcessSearchTermResponse (search_term_response);
599
600 QueryEachQueryable (query, result);
601 }
602
603 static public void DoQuery (Query query,
604 QueryResult result,
605 RequestMessageExecutor.AsyncResponse send_response)
606 {
607 DehumanizeQuery (query);
608
609 SearchTermResponse search_term_response;
610 search_term_response = AssembleSearchTermResponse (query);
611 send_response (search_term_response);
612
613 QueryEachQueryable (query, result);
614 }
615
616 ////////////////////////////////////////////////////////
617
618 static public IEnumerable GetIndexInformation ()
619 {
620 foreach (Queryable q in queryables)
621 yield return q.GetQueryableStatus ();
622 }
623
624 ////////////////////////////////////////////////////////
625
626 static public bool IsIndexing {
627 get {
628 // If the backends haven't been started yet,
629 // there is at least the initial setup. Just
630 // assume all the backends are indexing.
631 if (! queryables_started)
632 return true;
633
634 foreach (Queryable q in queryables) {
635 QueryableStatus status = q.GetQueryableStatus ();
636
637 if (status == null)
638 return false;
639
640 if (status.IsIndexing)
641 return true;
642 }
643
644 return false;
645 }
646 }
647 }
648 }
A Beagle Git Repo