?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Remove debug statements from KCal backends.
[beagle.git] / beagled / QueryDriver.cs
blobdaab67b29af9b07d30130adfa36a44382dfb2ad5
1 //
2 // QueryDriver.cs
3 //
4 // Copyright (C) 2004 Novell, Inc.
5 //
6
7 //
8 // Permission is hereby granted, free of charge, to any person obtaining a
9 // copy of this software and associated documentation files (the "Software"),
10 // to deal in the Software without restriction, including without limitation
11 // the rights to use, copy, modify, merge, publish, distribute, sublicense,
12 // and/or sell copies of the Software, and to permit persons to whom the
13 // Software is furnished to do so, subject to the following conditions:
14 //
15 // The above copyright notice and this permission notice shall be included in
16 // all copies or substantial portions of the Software.
17 //
18 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
21 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
22 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
23 // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
24 // DEALINGS IN THE SOFTWARE.
25 //
26
27 using System;
28 using System.IO;
29 using System.Collections;
30 using System.Reflection;
31 using System.Text;
32 using System.Threading;
33 using Beagle.Util;
34 namespace Beagle.Daemon {
35
36 public class QueryDriver {
37
38 // Contains list of queryables explicitly asked by --allow-backend or --backend name
39 // --allow-backend/--backend name : dont read config information and only start backend 'name'
40 static ArrayList excl_allowed_queryables = new ArrayList ();
41
42 // Contains list of denied queryables from config/arguments (every queryable is enabled by default)
43 // Unless overruled by --allow-backend/--backend name, deny backend only if names appears here.
44 static ArrayList denied_queryables = new ArrayList ();
45
46 static bool to_read_conf = true; // read backends from conf if true
47 static bool done_reading_conf = false;
48
49 static private void ReadBackendsFromConf ()
50 {
51 if (! to_read_conf || done_reading_conf)
52 return;
53
54 // set flag here to stop Allow() from calling ReadBackendsFromConf() again
55 done_reading_conf = true;
56
57 // To allow static indexes, "static" should be in allowed_queryables
58 if (Conf.Daemon.AllowStaticBackend)
59 Allow ("static");
60
61 if (Conf.Daemon.DeniedBackends == null)
62 return;
63
64 foreach (string name in Conf.Daemon.DeniedBackends)
65 denied_queryables.Add (name.ToLower ());
66 }
67
68 static public void OnlyAllow (string name)
69 {
70 excl_allowed_queryables.Add (name.ToLower ());
71 to_read_conf = false;
72 }
73
74 static public void Allow (string name)
75 {
76 if (! done_reading_conf && to_read_conf)
77 ReadBackendsFromConf ();
78
79 denied_queryables.Remove (name.ToLower ());
80 }
81
82 static public void Deny (string name)
83 {
84 if (! done_reading_conf && to_read_conf)
85 ReadBackendsFromConf ();
86
87 name = name.ToLower ();
88 if (!denied_queryables.Contains (name))
89 denied_queryables.Add (name);
90 }
91
92 static private bool UseQueryable (string name)
93 {
94 name = name.ToLower ();
95
96 if (excl_allowed_queryables.Contains (name))
97 return true;
98 if (excl_allowed_queryables.Count != 0)
99 return false;
100
101 if (denied_queryables.Contains (name))
102 return false;
103
104 return true;
105 }
106
107 //////////////////////////////////////////////////////////////////////////////////////
108
109 // Paths to static queryables
110
111 static ArrayList static_queryables = new ArrayList ();
112
113 static public void AddStaticQueryable (string path) {
114
115 if (! static_queryables.Contains (path))
116 static_queryables.Add (path);
117 }
118
119 //////////////////////////////////////////////////////////////////////////////////////
120
121 // Delay before starting the indexing process
122
123 static int indexing_delay = 60; // Default to 60 seconds
124
125 public static int IndexingDelay {
126 set { indexing_delay = value; }
127 }
128
129 //////////////////////////////////////////////////////////////////////////////////////
130
131 // Use introspection to find all classes that implement IQueryable, the construct
132 // associated Queryables objects.
133
134 static ArrayList queryables = new ArrayList ();
135 static Hashtable iqueryable_to_queryable = new Hashtable ();
136
137 static bool ThisApiSoVeryIsBroken (Type m, object criteria)
138 {
139 return m == (Type) criteria;
140 }
141
142 static bool TypeImplementsInterface (Type t, Type iface)
143 {
144 Type[] impls = t.FindInterfaces (new TypeFilter (ThisApiSoVeryIsBroken),
145 iface);
146 return impls.Length > 0;
147 }
148
149 // Find the types in the assembly that
150 // (1) register themselves in AssemblyInfo.cs:IQueryableTypes and
151 // (2) has a QueryableFlavor attribute attached
152 // assemble a Queryable object and stick it into our list of queryables.
153 static void ScanAssemblyForQueryables (Assembly assembly)
154 {
155 int count = 0;
156
157 foreach (Type type in ReflectionFu.GetTypesFromAssemblyAttribute (assembly, typeof (IQueryableTypesAttribute))) {
158 bool type_accepted = false;
159 foreach (QueryableFlavor flavor in ReflectionFu.ScanTypeForAttribute (type, typeof (QueryableFlavor))) {
160 if (! UseQueryable (flavor.Name))
161 continue;
162
163 if (flavor.RequireInotify && ! Inotify.Enabled) {
164 Logger.Log.Warn ("Can't start backend '{0}' without inotify", flavor.Name);
165 continue;
166 }
167
168 if (flavor.RequireExtendedAttributes && ! ExtendedAttribute.Supported) {
169 Logger.Log.Warn ("Can't start backend '{0}' without extended attributes", flavor.Name);
170 continue;
171 }
172
173 IQueryable iq = null;
174 try {
175 iq = Activator.CreateInstance (type) as IQueryable;
176 } catch (Exception e) {
177 Logger.Log.Error (e, "Caught exception while instantiating {0} backend", flavor.Name);
178 }
179
180 if (iq != null) {
181 Queryable q = new Queryable (flavor, iq);
182 queryables.Add (q);
183 iqueryable_to_queryable [iq] = q;
184 ++count;
185 type_accepted = true;
186 break;
187 }
188 }
189
190 if (! type_accepted)
191 continue;
192
193 object[] attributes = type.GetCustomAttributes (false);
194 foreach (object attribute in attributes) {
195 PropertyKeywordMapping mapping = attribute as PropertyKeywordMapping;
196 if (mapping == null)
197 continue;
198 //Logger.Log.Debug (mapping.Keyword + " => "
199 // + mapping.PropertyName +
200 // + " is-keyword=" + mapping.IsKeyword + " ("
201 // + mapping.Description + ") "
202 // + "(" + type.FullName + ")");
203 PropertyKeywordFu.RegisterMapping (mapping);
204 }
205
206 }
207 Logger.Log.Debug ("Found {0} backends in {1}", count, assembly.Location);
208 }
209
210 ////////////////////////////////////////////////////////
211
212 public static void ReadKeywordMappings ()
213 {
214 Logger.Log.Debug ("Reading mapping from filters");
215 ArrayList assemblies = ReflectionFu.ScanEnvironmentForAssemblies ("BEAGLE_FILTER_PATH", PathFinder.FilterDir);
216
217 foreach (Assembly assembly in assemblies) {
218 foreach (Type type in ReflectionFu.GetTypesFromAssemblyAttribute (assembly, typeof (FilterTypesAttribute))) {
219 object[] attributes = type.GetCustomAttributes (false);
220 foreach (object attribute in attributes) {
221
222 PropertyKeywordMapping mapping = attribute as PropertyKeywordMapping;
223 if (mapping == null)
224 continue;
225 //Logger.Log.Debug (mapping.Keyword + " => "
226 // + mapping.PropertyName
227 // + " is-keyword=" + mapping.IsKeyword + " ("
228 // + mapping.Description + ") "
229 // + "(" + type.FullName + ")");
230 PropertyKeywordFu.RegisterMapping (mapping);
231 }
232 }
233 }
234 }
235
236 ////////////////////////////////////////////////////////
237
238 // Scans PathFinder.SystemIndexesDir after available
239 // system-wide indexes.
240 static void LoadSystemIndexes ()
241 {
242 if (!Directory.Exists (PathFinder.SystemIndexesDir))
243 return;
244
245 Logger.Log.Info ("Loading system static indexes.");
246
247 int count = 0;
248
249 foreach (DirectoryInfo index_dir in new DirectoryInfo (PathFinder.SystemIndexesDir).GetDirectories ()) {
250 if (! UseQueryable (index_dir.Name))
251 continue;
252
253 if (LoadStaticQueryable (index_dir, QueryDomain.System))
254 count++;
255 }
256
257 Logger.Log.Info ("Found {0} system-wide indexes.", count);
258 }
259
260 // Scans configuration for user-specified index paths
261 // to load StaticQueryables from.
262 static void LoadStaticQueryables ()
263 {
264 int count = 0;
265
266 if (UseQueryable ("static")) {
267 Logger.Log.Info ("Loading user-configured static indexes.");
268 foreach (string path in Conf.Daemon.StaticQueryables)
269 static_queryables.Add (path);
270 }
271
272 foreach (string path in static_queryables) {
273 DirectoryInfo index_dir = new DirectoryInfo (StringFu.SanitizePath (path));
274
275 if (!index_dir.Exists)
276 continue;
277
278 // FIXME: QueryDomain might be other than local
279 if (LoadStaticQueryable (index_dir, QueryDomain.Local))
280 count++;
281 }
282
283 Logger.Log.Info ("Found {0} user-configured static indexes..", count);
284 }
285
286 // Instantiates and loads a StaticQueryable from an index directory
287 static private bool LoadStaticQueryable (DirectoryInfo index_dir, QueryDomain query_domain)
288 {
289 StaticQueryable static_queryable = null;
290
291 if (!index_dir.Exists)
292 return false;
293
294 try {
295 static_queryable = new StaticQueryable (index_dir.Name, index_dir.FullName, true);
296 } catch (InvalidOperationException) {
297 Logger.Log.Warn ("Unable to create read-only index (likely due to index version mismatch): {0}", index_dir.FullName);
298 return false;
299 } catch (Exception e) {
300 Logger.Log.Error (e, "Caught exception while instantiating static queryable: {0}", index_dir.Name);
301 return false;
302 }
303
304 if (static_queryable != null) {
305 QueryableFlavor flavor = new QueryableFlavor ();
306 flavor.Name = index_dir.Name;
307 flavor.Domain = query_domain;
308
309 Queryable queryable = new Queryable (flavor, static_queryable);
310 queryables.Add (queryable);
311
312 iqueryable_to_queryable [static_queryable] = queryable;
313
314 return true;
315 }
316
317 return false;
318 }
319
320 ////////////////////////////////////////////////////////
321
322 private static ArrayList assemblies = null;
323
324 // Perform expensive initialization steps all at once.
325 // Should be done before SignalHandler comes into play.
326 static public void Init ()
327 {
328 ReadBackendsFromConf ();
329 assemblies = ReflectionFu.ScanEnvironmentForAssemblies ("BEAGLE_BACKEND_PATH", PathFinder.BackendDir);
330 }
331
332 static public void Start ()
333 {
334 // Only add the executing assembly if we haven't already loaded it.
335 if (assemblies.IndexOf (Assembly.GetExecutingAssembly ()) == -1)
336 assemblies.Add (Assembly.GetExecutingAssembly ());
337
338 foreach (Assembly assembly in assemblies) {
339 ScanAssemblyForQueryables (assembly);
340
341 // This allows backends to define their
342 // own executors.
343 Server.ScanAssemblyForExecutors (assembly);
344 }
345
346 assemblies = null;
347
348 ReadKeywordMappings ();
349
350 LoadSystemIndexes ();
351 LoadStaticQueryables ();
352
353 if (indexing_delay <= 0 || Environment.GetEnvironmentVariable ("BEAGLE_EXERCISE_THE_DOG") != null)
354 StartQueryables ();
355 else {
356 Logger.Log.Debug ("Waiting {0} seconds before starting queryables", indexing_delay);
357 GLib.Timeout.Add ((uint) indexing_delay * 1000, new GLib.TimeoutHandler (StartQueryables));
358 }
359 }
360
361 static private bool StartQueryables ()
362 {
363 Logger.Log.Debug ("Starting queryables");
364
365 foreach (Queryable q in queryables) {
366 Logger.Log.Info ("Starting backend: '{0}'", q.Name);
367 q.Start ();
368 }
369
370 return false;
371 }
372
373 static public string ListBackends ()
374 {
375 ArrayList assemblies = ReflectionFu.ScanEnvironmentForAssemblies ("BEAGLE_BACKEND_PATH", PathFinder.BackendDir);
376
377 // Only add the executing assembly if we haven't already loaded it.
378 if (assemblies.IndexOf (Assembly.GetExecutingAssembly ()) == -1)
379 assemblies.Add (Assembly.GetExecutingAssembly ());
380
381 string ret = "User:\n";
382
383 foreach (Assembly assembly in assemblies) {
384 foreach (Type type in ReflectionFu.GetTypesFromAssemblyAttribute (assembly, typeof (IQueryableTypesAttribute))) {
385 foreach (QueryableFlavor flavor in ReflectionFu.ScanTypeForAttribute (type, typeof (QueryableFlavor)))
386 ret += String.Format (" - {0}\n", flavor.Name);
387 }
388 }
389
390 if (!Directory.Exists (PathFinder.SystemIndexesDir))
391 return ret;
392
393 ret += "System:\n";
394 foreach (DirectoryInfo index_dir in new DirectoryInfo (PathFinder.SystemIndexesDir).GetDirectories ()) {
395 ret += String.Format (" - {0}\n", index_dir.Name);
396 }
397
398 return ret;
399 }
400
401 static public Queryable GetQueryable (string name)
402 {
403 foreach (Queryable q in queryables) {
404 if (q.Name == name)
405 return q;
406 }
407
408 return null;
409 }
410
411 static public Queryable GetQueryable (IQueryable iqueryable)
412 {
413 return (Queryable) iqueryable_to_queryable [iqueryable];
414 }
415
416 ////////////////////////////////////////////////////////
417
418 public delegate void ChangedHandler (Queryable queryable,
419 IQueryableChangeData changeData);
420
421 static public event ChangedHandler ChangedEvent;
422
423 // A method to fire the ChangedEvent event.
424 static public void QueryableChanged (IQueryable iqueryable,
425 IQueryableChangeData change_data)
426 {
427 if (ChangedEvent != null) {
428 Queryable queryable = iqueryable_to_queryable [iqueryable] as Queryable;
429 ChangedEvent (queryable, change_data);
430 }
431 }
432
433 ////////////////////////////////////////////////////////
434
435 private class QueryClosure : IQueryWorker {
436
437 Queryable queryable;
438 Query query;
439 IQueryResult result;
440 IQueryableChangeData change_data;
441
442 public QueryClosure (Queryable queryable,
443 Query query,
444 QueryResult result,
445 IQueryableChangeData change_data)
446 {
447 this.queryable = queryable;
448 this.query = query;
449 this.result = result;
450 this.change_data = change_data;
451 }
452
453 public void DoWork ()
454 {
455 queryable.DoQuery (query, result, change_data);
456 }
457 }
458
459 static public void DoOneQuery (Queryable queryable,
460 Query query,
461 QueryResult result,
462 IQueryableChangeData change_data)
463 {
464 if (queryable.AcceptQuery (query)) {
465 QueryClosure qc = new QueryClosure (queryable, query, result, change_data);
466 result.AttachWorker (qc);
467 }
468 }
469
470 static void AddSearchTermInfo (QueryPart part,
471 SearchTermResponse response, StringBuilder sb)
472 {
473 if (part.Logic == QueryPartLogic.Prohibited)
474 return;
475
476 if (part is QueryPart_Or) {
477 ICollection sub_parts;
478 sub_parts = ((QueryPart_Or) part).SubParts;
479 foreach (QueryPart qp in sub_parts)
480 AddSearchTermInfo (qp, response, sb);
481 return;
482 }
483
484 if (! (part is QueryPart_Text))
485 return;
486
487 QueryPart_Text tp;
488 tp = (QueryPart_Text) part;
489
490 string [] split;
491 split = tp.Text.Split (' ');
492
493 // First, remove stop words
494 for (int i = 0; i < split.Length; ++i)
495 if (LuceneCommon.IsStopWord (split [i]))
496 split [i] = null;
497
498 // Assemble the phrase minus stop words
499 sb.Length = 0;
500 for (int i = 0; i < split.Length; ++i) {
501 if (split [i] == null)
502 continue;
503 if (sb.Length > 0)
504 sb.Append (' ');
505 sb.Append (split [i]);
506 }
507 response.ExactText.Add (sb.ToString ());
508
509 // Now assemble a stemmed version
510 sb.Length = 0; // clear the previous value
511 for (int i = 0; i < split.Length; ++i) {
512 if (split [i] == null)
513 continue;
514 if (sb.Length > 0)
515 sb.Append (' ');
516 sb.Append (LuceneCommon.Stem (split [i]));
517 }
518 response.StemmedText.Add (sb.ToString ());
519 }
520
521 ////////////////////////////////////////////////////////
522
523 static private void DehumanizeQuery (Query query)
524 {
525 // We need to remap any QueryPart_Human parts into
526 // lower-level part types. First, we find any
527 // QueryPart_Human parts and explode them into
528 // lower-level types.
529 ArrayList new_parts = null;
530 foreach (QueryPart abstract_part in query.Parts) {
531 if (abstract_part is QueryPart_Human) {
532 QueryPart_Human human = abstract_part as QueryPart_Human;
533 if (new_parts == null)
534 new_parts = new ArrayList ();
535 foreach (QueryPart sub_part in QueryStringParser.Parse (human.QueryString))
536 new_parts.Add (sub_part);
537 }
538 }
539
540 // If we found any QueryPart_Human parts, copy the
541 // non-Human parts over and then replace the parts in
542 // the query.
543 if (new_parts != null) {
544 foreach (QueryPart abstract_part in query.Parts) {
545 if (! (abstract_part is QueryPart_Human))
546 new_parts.Add (abstract_part);
547 }
548
549 query.ClearParts ();
550 foreach (QueryPart part in new_parts)
551 query.AddPart (part);
552 }
553
554 }
555
556 static private SearchTermResponse AssembleSearchTermResponse (Query query)
557 {
558 StringBuilder sb = new StringBuilder ();
559 SearchTermResponse search_term_response;
560 search_term_response = new SearchTermResponse ();
561 foreach (QueryPart part in query.Parts)
562 AddSearchTermInfo (part, search_term_response, sb);
563 return search_term_response;
564 }
565
566 static private void QueryEachQueryable (Query query,
567 QueryResult result)
568 {
569 // The extra pair of calls to WorkerStart/WorkerFinished ensures:
570 // (1) that the QueryResult will fire the StartedEvent
571 // and FinishedEvent, even if no queryable accepts the
572 // query.
573 // (2) that the FinishedEvent will only get called when all of the
574 // backends have had time to finish.
575
576 object dummy_worker = new object ();
577
578 if (! result.WorkerStart (dummy_worker))
579 return;
580
581 foreach (Queryable queryable in queryables)
582 DoOneQuery (queryable, query, result, null);
583
584 result.WorkerFinished (dummy_worker);
585 }
586
587 static public void DoQueryLocal (Query query,
588 QueryResult result)
589 {
590 DehumanizeQuery (query);
591
592 SearchTermResponse search_term_response;
593 search_term_response = AssembleSearchTermResponse (query);
594 query.ProcessSearchTermResponse (search_term_response);
595
596 QueryEachQueryable (query, result);
597 }
598
599 static public void DoQuery (Query query,
600 QueryResult result,
601 RequestMessageExecutor.AsyncResponse send_response)
602 {
603 DehumanizeQuery (query);
604
605 SearchTermResponse search_term_response;
606 search_term_response = AssembleSearchTermResponse (query);
607 send_response (search_term_response);
608
609 QueryEachQueryable (query, result);
610 }
611
612 ////////////////////////////////////////////////////////
613
614 static public IEnumerable GetIndexInformation ()
615 {
616 foreach (Queryable q in queryables)
617 yield return q.GetQueryableStatus ();
618 }
619
620 ////////////////////////////////////////////////////////
621
622 static public bool IsIndexing {
623 get {
624 foreach (Queryable q in queryables) {
625 QueryableStatus status = q.GetQueryableStatus ();
626
627 if (status == null)
628 return false;
629
630 if (status.IsIndexing)
631 return true;
632 }
633
634 return false;
635 }
636 }
637 }
638 }
A Beagle Git Repo