?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Add --enable-deletion option to buildindex. If used, buildindex will remove deleted...
[beagle.git] / beagled / BeagleDaemon.cs
blobfb7fcd08bbac78471ac4f5a57bbb24a85c7eae6d
1 //
2 // BeagleDaemon.cs
3 //
4 // Copyright (C) 2004-2006 Novell, Inc.
5 //
6
7 //
8 // Permission is hereby granted, free of charge, to any person obtaining a
9 // copy of this software and associated documentation files (the "Software"),
10 // to deal in the Software without restriction, including without limitation
11 // the rights to use, copy, modify, merge, publish, distribute, sublicense,
12 // and/or sell copies of the Software, and to permit persons to whom the
13 // Software is furnished to do so, subject to the following conditions:
14 //
15 // The above copyright notice and this permission notice shall be included in
16 // all copies or substantial portions of the Software.
17 //
18 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
21 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
22 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
23 // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
24 // DEALINGS IN THE SOFTWARE.
25 //
26
27 using System;
28 using System.Collections;
29 using System.IO;
30 using System.Reflection;
31 using System.Runtime.InteropServices;
32 using System.Threading;
33
34 using Gtk;
35
36 using Beagle.Util;
37
38 #if ENABLE_WEBSERVICES
39 using Beagle.WebService;
40 #endif
41
42 namespace Beagle.Daemon {
43 class BeagleDaemon {
44
45 public static Thread MainLoopThread = null;
46
47 private static Server server = null;
48
49 private static bool arg_replace = false;
50 private static bool arg_disable_scheduler = false;
51 private static bool arg_indexing_test_mode = false;
52
53 public static bool StartServer ()
54 {
55 Logger.Log.Debug ("Starting messaging server");
56 try {
57 server = new Server ("socket");
58 server.Start ();
59 } catch (InvalidOperationException) {
60 return false;
61 }
62
63 return true;
64 }
65
66 public static void ReplaceExisting ()
67 {
68 Logger.Log.Info ("Attempting to replace another beagled.");
69
70 do {
71 ShutdownRequest request = new ShutdownRequest ();
72 Logger.Log.Info ("Sending Shutdown");
73 request.Send ();
74 // Give it a second to shut down the messaging server
75 Thread.Sleep (1000);
76 } while (! StartServer ());
77 }
78
79 private static void LogMemoryUsage ()
80 {
81 while (! Shutdown.ShutdownRequested) {
82 int vm_size = SystemInformation.VmSize;
83 int vm_rss = SystemInformation.VmRss;
84
85 Logger.Log.Debug ("Memory usage: VmSize={0:.0} MB, VmRSS={1:.0} MB, GC.GetTotalMemory={2}",
86 vm_size/1024.0, vm_rss/1024.0, GC.GetTotalMemory (false));
87
88 if (vm_size > 300 * 1024) {
89 Logger.Log.Debug ("VmSize too large --- shutting down");
90 Shutdown.BeginShutdown ();
91 }
92
93 Thread.Sleep (5000);
94 }
95 }
96
97 private static void PrintUsage ()
98 {
99 string usage =
100 "beagled: The daemon to the Beagle search system.\n" +
101 "Web page: http://beagle-project.org\n" +
102 "Copyright (C) 2004-2006 Novell, Inc.\n\n";
103
104 usage +=
105 "Usage: beagled [OPTIONS]\n\n" +
106 "Options:\n" +
107 " --foreground, --fg\tRun the daemon in the foreground.\n" +
108 " --background, --bg\tRun the daemon in the background.\n" +
109 " --replace\t\tReplace a running daemon with a new instance.\n" +
110 " --debug\t\tWrite out debugging information.\n" +
111 " --debug-memory\tWrite out debugging information about memory use.\n" +
112 " --indexing-test-mode\tRun in foreground, and exit when fully indexed.\n" +
113 " --indexing-delay\tTime to wait before indexing. (Default 60 seconds)\n" +
114 " --backend\t\tConfigure which backends to use. Specifically:\n" +
115 " --backend <name>\tOnly start backend 'name'\n" +
116 " --backend +<name>\tAdditionally start backend 'name'\n" +
117 " --backend -<name>\tDisable backend 'name'\n" +
118 " --allow-backend\t(DEPRECATED) Start only the specific backend.\n" +
119 " --deny-backend\t(DEPRECATED) Deny a specific backend.\n" +
120 " --list-backends\tList all the available backends.\n" +
121 " --add-static-backend\tAdd a static backend by path.\n" +
122 " --disable-scheduler\tDisable the use of the scheduler.\n" +
123 " --help\t\tPrint this usage message.\n";
124
125 #if ENABLE_WEBSERVICES
126 usage += "\n" +
127 " --web-global\t\tAllow global access to the Web & WebService interfaces.\n" +
128 " --web-port\t\tPort to use for the internal web server.\n" +
129 " --web-root\t\tRoot directory to use for the internal web server.\n" +
130 " --web-disable\t\tDisable Web & WebServices functionality.\n";
131 #endif
132
133 Console.WriteLine (usage);
134 }
135
136 public static bool StartupProcess ()
137 {
138 // Profile our initialization
139 Stopwatch stopwatch = new Stopwatch ();
140 stopwatch.Start ();
141
142 SetupSignalHandlers ();
143
144 // Fire up our server
145 if (! StartServer ()) {
146 if (arg_replace)
147 {
148 #if ENABLE_WEBSERVICES
149 WebServiceBackEnd.Stop();
150 #endif
151 ReplaceExisting ();
152 }
153 else {
154 Logger.Log.Error ("Could not set up the listener for beagle requests. "
155 + "There is probably another beagled instance running. "
156 + "Use --replace to replace the running service");
157 Environment.Exit (1);
158 }
159 }
160
161 // Set up out-of-process indexing
162 if (Environment.GetEnvironmentVariable ("BEAGLE_ENABLE_IN_PROCESS_INDEXING") == null)
163 LuceneQueryable.IndexerHook = new LuceneQueryable.IndexerCreator (RemoteIndexer.NewRemoteIndexer);
164
165 // Initialize syncronization to keep the indexes local if PathFinder.HomeDir
166 // is on a non-block device, or if BEAGLE_SYNCHRONIZE_LOCALLY is set
167 if ((! SystemInformation.IsPathOnBlockDevice (PathFinder.HomeDir) && Conf.Daemon.IndexSynchronization) ||
168 Environment.GetEnvironmentVariable ("BEAGLE_SYNCHRONIZE_LOCALLY") != null)
169 IndexSynchronization.Initialize ();
170
171 // Start the query driver.
172 Logger.Log.Debug ("Starting QueryDriver");
173 QueryDriver.Start ();
174
175 // Start the Global Scheduler thread
176 if (! arg_disable_scheduler) {
177 Logger.Log.Debug ("Starting Scheduler thread");
178 Scheduler.Global.Start ();
179 }
180
181 // Start our Inotify threads
182 Inotify.Start ();
183
184 // Test if the FileAdvise stuff is working: This will print a
185 // warning if not. The actual advice calls will fail silently.
186 FileAdvise.TestAdvise ();
187
188 #if ENABLE_WEBSERVICES
189 //Beagle Web, WebService access initialization code:
190 WebServiceBackEnd.Start();
191 #endif
192 Shutdown.ShutdownEvent += OnShutdown;
193
194 Conf.WatchForUpdates ();
195
196 stopwatch.Stop ();
197
198 Logger.Log.Debug ("Daemon initialization finished after {0}", stopwatch);
199
200 if (arg_indexing_test_mode) {
201 Thread.Sleep (1000); // Ugly paranoia: wait a second for the backends to settle.
202 Logger.Log.Debug ("Running in indexing test mode");
203 Scheduler.Global.EmptyQueueEvent += OnEmptySchedulerQueue;
204 Scheduler.Global.Add (null); // pulse the scheduler
205 }
206 return false;
207 }
208
209 static void OnEmptySchedulerQueue ()
210 {
211 Logger.Log.Debug ("Scheduler queue is empty: terminating immediately");
212 Shutdown.BeginShutdown ();
213 Environment.Exit (0); // Ugly work-around: We need to call Exit here to avoid deadlocking.
214 }
215
216 public static void Main (string[] args)
217 {
218 try {
219 DoMain (args);
220 } catch (Exception ex) {
221 Logger.Log.Error ("Unhandled exception thrown. Exiting immediately.");
222 Logger.Log.Error (ex);
223 Environment.Exit (1);
224 }
225 }
226
227 public static void DoMain (string[] args)
228 {
229 SystemInformation.SetProcessName ("beagled");
230
231 // Process the command-line arguments
232 bool arg_debug = false;
233 bool arg_debug_memory = false;
234 bool arg_fg = false;
235
236 int i = 0;
237 while (i < args.Length) {
238
239 string arg = args [i];
240 ++i;
241 string next_arg = i < args.Length ? args [i] : null;
242
243 switch (arg) {
244 case "-h":
245 case "--help":
246 PrintUsage ();
247 Environment.Exit (0);
248 break;
249
250 case "--heap-buddy":
251 // Silently ignore the --heap-buddy argument: it gets handled
252 // in the wrapper script.
253 break;
254
255 case "--list-backends":
256 Console.WriteLine ("Current available backends:");
257 Console.Write (QueryDriver.ListBackends ());
258 Environment.Exit (0);
259 break;
260
261 case "--fg":
262 case "--foreground":
263 arg_fg = true;
264 break;
265
266 case "--bg":
267 case "--background":
268 arg_fg = false;
269 break;
270
271 case "--replace":
272 arg_replace = true;
273 break;
274
275 case "--debug":
276 arg_debug = true;
277 break;
278
279 case "--debug-memory":
280 arg_debug = true;
281 arg_debug_memory = true;
282 break;
283
284 case "--indexing-test-mode":
285 arg_indexing_test_mode = true;
286 arg_fg = true;
287 break;
288
289 case "--backend":
290 if (next_arg == null) {
291 Console.WriteLine ("--backend requires a backend name");
292 Environment.Exit (1);
293 break;
294 }
295
296 if (next_arg.StartsWith ("--")) {
297 Console.WriteLine ("--backend requires a backend name. Invalid name '{0}'", next_arg);
298 Environment.Exit (1);
299 break;
300 }
301
302 if (next_arg [0] != '+' && next_arg [0] != '-')
303 QueryDriver.OnlyAllow (next_arg);
304 else {
305 if (next_arg [0] == '+')
306 QueryDriver.Allow (next_arg.Substring (1));
307 else
308 QueryDriver.Deny (next_arg.Substring (1));
309 }
310
311 ++i; // we used next_arg
312 break;
313
314 case "--allow-backend":
315 // --allow-backend is deprecated, use --backends 'name' instead
316 // it will disable reading the list of enabled/disabled backends
317 // from conf and start the backend given
318 if (next_arg != null)
319 QueryDriver.OnlyAllow (next_arg);
320 ++i; // we used next_arg
321 break;
322
323 case "--deny-backend":
324 // deprecated: use --backends -'name' instead
325 if (next_arg != null)
326 QueryDriver.Deny (next_arg);
327 ++i; // we used next_arg
328 break;
329
330 case "--add-static-backend":
331 if (next_arg != null)
332 QueryDriver.AddStaticQueryable (next_arg);
333 ++i;
334 break;
335
336 case "--disable-scheduler":
337 arg_disable_scheduler = true;
338 break;
339
340 case "--indexing-delay":
341 if (next_arg != null) {
342 try {
343 QueryDriver.IndexingDelay = Int32.Parse (next_arg);
344 } catch {
345 Console.WriteLine ("'{0}' is not a valid number of seconds", next_arg);
346 Environment.Exit (1);
347 }
348 }
349
350 ++i;
351 break;
352
353 case "--autostarted":
354 if (! Conf.Searching.Autostart) {
355 Console.WriteLine ("Autostarting is disabled, not starting");
356 Environment.Exit (0);
357 }
358 break;
359 #if ENABLE_WEBSERVICES
360 case "--web-global":
361 WebServiceBackEnd.web_global = true;
362 WebServiceBackEnd.web_start = true;
363 break;
364
365 case "--web-port":
366 WebServiceBackEnd.web_port = next_arg;
367 ++i;
368 WebServiceBackEnd.web_start = true;
369 break;
370
371 case "--web-root":
372 WebServiceBackEnd.web_rootDir = next_arg;
373 ++i;
374 WebServiceBackEnd.web_start = true;
375 break;
376
377 case "--web-disable":
378 WebServiceBackEnd.web_start = false;
379 break;
380 #endif
381 default:
382 Console.WriteLine ("Unknown argument '{0}'", arg);
383 Environment.Exit (1);
384 break;
385
386 }
387 }
388
389 if (arg_indexing_test_mode) {
390 LuceneQueryable.OptimizeRightAway = true;
391 }
392
393
394 // Bail out if we are trying to run as root
395 if (Environment.UserName == "root" && Environment.GetEnvironmentVariable ("SUDO_USER") != null) {
396 Console.WriteLine ("You appear to be running beagle using sudo. This can cause problems with");
397 Console.WriteLine ("permissions in your .beagle and .wapi directories if you later try to run");
398 Console.WriteLine ("as an unprivileged user. If you need to run beagle as root, please use");
399 Console.WriteLine ("'su -c' instead.");
400 Environment.Exit (-1);
401 }
402
403 if (Environment.UserName == "root" && ! Conf.Daemon.AllowRoot) {
404 Console.WriteLine ("You can not run beagle as root. Beagle is designed to run from your own");
405 Console.WriteLine ("user account. If you want to create multiuser or system-wide indexes, use");
406 Console.WriteLine ("the beagle-build-index tool.");
407 Console.WriteLine ();
408 Console.WriteLine ("You can override this setting using the beagle-config or beagle-settings tools.");
409 Environment.Exit (-1);
410 }
411
412 try {
413 string tmp = PathFinder.HomeDir;
414 } catch (Exception e) {
415 Console.WriteLine ("Unable to start the daemon: {0}", e.Message);
416 Environment.Exit (-1);
417 }
418
419 MainLoopThread = Thread.CurrentThread;
420
421 Log.Initialize (PathFinder.LogDir,
422 "Beagle",
423 // FIXME: We always turn on full debugging output! We are still
424 // debugging this code, after all...
425 //arg_debug ? LogLevel.Debug : LogLevel.Warn,
426 LogLevel.Debug,
427 arg_fg);
428
429 Logger.Log.Info ("Starting Beagle Daemon (version {0})", ExternalStringsHack.Version);
430
431 Logger.Log.Info ("Running on {0}", SystemInformation.MonoRuntimeVersion);
432
433 Logger.Log.Debug ("Command Line: {0}",
434 Environment.CommandLine != null ? Environment.CommandLine : "(null)");
435
436 if (! ExtendedAttribute.Supported) {
437 Logger.Log.Warn ("Extended attributes are not supported on this filesystem. " +
438 "Performance will suffer as a result.");
439 }
440
441 // Start our memory-logging thread
442 if (arg_debug_memory)
443 ExceptionHandlingThread.Start (new ThreadStart (LogMemoryUsage));
444
445 // Do BEAGLE_EXERCISE_THE_DOG_HARDER-related processing.
446 ExerciseTheDogHarder ();
447
448 if (Application.InitCheck ("beagled", ref args))
449 Logger.Log.Debug ("Established a connection to the X server");
450 else
451 Logger.Log.Debug ("Unable to establish a connection to the X server");
452
453 XSetIOErrorHandler (BeagleXIOErrorHandler);
454
455 // Defer all actual startup until the main loop is
456 // running. That way shutdowns during the startup
457 // process work correctly.
458 GLib.Idle.Add (new GLib.IdleHandler (StartupProcess));
459
460 // Start our event loop.
461 Logger.Log.Debug ("Starting main loop");
462
463 Application.Run ();
464
465 // If we placed our sockets in a temp directory, try to clean it up
466 // Note: this may fail because the helper is still running
467 if (PathFinder.GetRemoteStorageDir (false) != PathFinder.StorageDir) {
468 try {
469 Directory.Delete (PathFinder.GetRemoteStorageDir (false));
470 } catch (IOException) { }
471 }
472
473 Logger.Log.Debug ("Leaving BeagleDaemon.Main");
474
475 if (arg_debug) {
476 Thread.Sleep (500);
477 ExceptionHandlingThread.SpewLiveThreads ();
478 }
479 }
480
481 /////////////////////////////////////////////////////////////////////////////
482
483 private delegate int XIOErrorHandler (IntPtr display);
484
485 [DllImport ("libX11.so.6")]
486 extern static private int XSetIOErrorHandler (XIOErrorHandler handler);
487
488 private static int BeagleXIOErrorHandler (IntPtr display)
489 {
490 Logger.Log.Debug ("Lost our connection to the X server! Trying to shut down gracefully");
491
492 if (! Shutdown.ShutdownRequested)
493 Shutdown.BeginShutdown ();
494
495 Logger.Log.Debug ("Xlib is forcing us to exit!");
496
497 ExceptionHandlingThread.SpewLiveThreads ();
498
499 // Returning will cause xlib to exit immediately.
500 return 0;
501 }
502
503 /////////////////////////////////////////////////////////////////////////////
504
505 static void SetupSignalHandlers ()
506 {
507 // Force OurSignalHandler to be JITed
508 OurSignalHandler (-1);
509
510 // Set up our signal handler
511 Mono.Unix.Native.Stdlib.signal (Mono.Unix.Native.Signum.SIGINT, OurSignalHandler);
512 Mono.Unix.Native.Stdlib.signal (Mono.Unix.Native.Signum.SIGTERM, OurSignalHandler);
513
514 // Ignore SIGPIPE
515 Mono.Unix.Native.Stdlib.signal (Mono.Unix.Native.Signum.SIGPIPE, Mono.Unix.Native.Stdlib.SIG_IGN);
516 }
517
518 // Our handler triggers an orderly shutdown when it receives a signal.
519 // However, this can be annoying if the process gets wedged during
520 // shutdown. To deal with that case, we make a note of the time when
521 // the first signal comes in, and we allow signals to unconditionally
522 // kill the process after 5 seconds have passed.
523 static DateTime signal_time = DateTime.MinValue;
524 static void OurSignalHandler (int signal)
525 {
526 // This allows us to call OurSignalHandler w/o doing anything.
527 // We want to call it once to ensure that it is pre-JITed.
528 if (signal < 0)
529 return;
530
531 Logger.Log.Debug ("Handling signal {0} ({1})", signal, (Mono.Unix.Native.Signum) signal);
532
533 bool first_signal = false;
534 if (signal_time == DateTime.MinValue) {
535 signal_time = DateTime.Now;
536 first_signal = true;
537 }
538
539 if (Shutdown.ShutdownRequested) {
540
541 if (first_signal) {
542 Logger.Log.Debug ("Shutdown already in progress.");
543 } else {
544 double t = (DateTime.Now - signal_time).TotalSeconds;
545 const double min_t = 5;
546
547 if (t < min_t) {
548 Logger.Log.Debug ("Signals can force an immediate shutdown in {0:0.00}s", min_t-t);
549 } else {
550 Logger.Log.Debug ("Forcing immediate shutdown.");
551 Environment.Exit (0);
552 }
553 }
554
555 } else {
556 Logger.Log.Debug ("Initiating shutdown in response to signal.");
557 Shutdown.BeginShutdown ();
558 }
559 }
560
561 /////////////////////////////////////////////////////////////////////////////
562
563 private static void OnShutdown ()
564 {
565 #if ENABLE_WEBSERVICES
566 WebServiceBackEnd.Stop();
567 #endif
568 // Stop our Inotify threads
569 Inotify.Stop ();
570
571 // Shut down the global scheduler
572 Scheduler.Global.Stop ();
573
574 // Stop the messaging server
575 server.Stop ();
576 }
577
578 /////////////////////////////////////////////////////////////////////////////
579
580
581 private static ArrayList exercise_files = new ArrayList ();
582
583 private static void ExerciseTheDogHarder ()
584 {
585 string path;
586 path = Environment.GetEnvironmentVariable ("BEAGLE_EXERCISE_THE_DOG_HARDER");
587 if (path == null)
588 return;
589
590 DirectoryInfo dir = new DirectoryInfo (path);
591 foreach (FileInfo file in dir.GetFiles ())
592 exercise_files.Add (file);
593 if (exercise_files.Count == 0)
594 return;
595
596 int N = 5;
597 if (N > exercise_files.Count)
598 N = exercise_files.Count;
599
600 for (int i = 0; i < N; ++i)
601 ExceptionHandlingThread.Start (new ThreadStart (ExerciseTheDogHarderWorker));
602 }
603
604 private static void ExerciseTheDogHarderWorker ()
605 {
606 Random rng = new Random ();
607
608 while (! Shutdown.ShutdownRequested) {
609
610 FileInfo file = null;
611 int i;
612
613
614 lock (exercise_files) {
615 do {
616 i = rng.Next (exercise_files.Count);
617 file = exercise_files [i] as FileInfo;
618 } while (file == null);
619 exercise_files [i] = null;
620 }
621
622 string target;
623 target = Path.Combine (PathFinder.HomeDir, "_HARDER_" + file.Name);
624
625 Logger.Log.Debug ("ETDH: Copying {0}", file.Name);
626 file.CopyTo (target, true);
627
628 lock (exercise_files)
629 exercise_files [i] = file;
630
631 Thread.Sleep (500 + rng.Next (500));
632 }
633 }
634 }
635 }
A Beagle Git Repo