?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
* tools/Info.cs: Add --list-backends, --list-static-indexes to
[beagle.git] / beagled / BeagleDaemon.cs
blob44138a9826ff6852bc3ec8767083d0311536ade1
1 //
2 // BeagleDaemon.cs
3 //
4 // Copyright (C) 2004-2006 Novell, Inc.
5 //
6
7 //
8 // Permission is hereby granted, free of charge, to any person obtaining a
9 // copy of this software and associated documentation files (the "Software"),
10 // to deal in the Software without restriction, including without limitation
11 // the rights to use, copy, modify, merge, publish, distribute, sublicense,
12 // and/or sell copies of the Software, and to permit persons to whom the
13 // Software is furnished to do so, subject to the following conditions:
14 //
15 // The above copyright notice and this permission notice shall be included in
16 // all copies or substantial portions of the Software.
17 //
18 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
21 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
22 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
23 // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
24 // DEALINGS IN THE SOFTWARE.
25 //
26
27 using System;
28 using System.Collections;
29 using System.IO;
30 using System.Reflection;
31 using System.Runtime.InteropServices;
32 using System.Threading;
33
34 using Gtk;
35
36 using Beagle.Util;
37
38 #if ENABLE_WEBSERVICES
39 using Beagle.WebService;
40 #endif
41
42 namespace Beagle.Daemon {
43 class BeagleDaemon {
44
45 public static Thread MainLoopThread = null;
46
47 private static Server server = null;
48
49 private static bool arg_replace = false;
50 private static bool arg_disable_scheduler = false;
51 private static bool arg_indexing_test_mode = false;
52
53 public static bool StartServer ()
54 {
55 Logger.Log.Debug ("Starting messaging server");
56 try {
57 server = new Server ("socket");
58 server.Start ();
59 } catch (InvalidOperationException) {
60 return false;
61 }
62
63 return true;
64 }
65
66 public static void ReplaceExisting ()
67 {
68 Logger.Log.Info ("Attempting to replace another beagled.");
69
70 do {
71 ShutdownRequest request = new ShutdownRequest ();
72 Logger.Log.Info ("Sending Shutdown");
73 request.Send ();
74 // Give it a second to shut down the messaging server
75 Thread.Sleep (1000);
76 } while (! StartServer ());
77 }
78
79 private static void LogMemoryUsage ()
80 {
81 while (! Shutdown.ShutdownRequested) {
82 int vm_size = SystemInformation.VmSize;
83 int vm_rss = SystemInformation.VmRss;
84
85 Logger.Log.Debug ("Memory usage: VmSize={0:.0} MB, VmRSS={1:.0} MB, GC.GetTotalMemory={2}",
86 vm_size/1024.0, vm_rss/1024.0, GC.GetTotalMemory (false));
87
88 if (vm_size > 300 * 1024) {
89 Logger.Log.Debug ("VmSize too large --- shutting down");
90 Shutdown.BeginShutdown ();
91 }
92
93 Thread.Sleep (5000);
94 }
95 }
96
97 private static void PrintUsage ()
98 {
99 string usage =
100 "beagled: The daemon to the Beagle search system.\n" +
101 "Web page: http://beagle-project.org\n" +
102 "Copyright (C) 2004-2006 Novell, Inc.\n\n";
103
104 usage +=
105 "Usage: beagled [OPTIONS]\n\n" +
106 "Options:\n" +
107 " --foreground, --fg\tRun the daemon in the foreground.\n" +
108 " --background, --bg\tRun the daemon in the background.\n" +
109 " --replace\t\tReplace a running daemon with a new instance.\n" +
110 " --debug\t\tWrite out debugging information.\n" +
111 " --debug-memory\tWrite out debugging information about memory use.\n" +
112 " --indexing-test-mode\tRun in foreground, and exit when fully indexed.\n" +
113 " --indexing-delay\tTime to wait before indexing. (Default 60 seconds)\n" +
114 " --backend [+-]name\t--backend name: only start backend 'name'\n" +
115 " \t--backend +name: also start backend 'name' besides those mentioned in config\n" +
116 " \t--backend -name: dont start backend 'name' if it is enabled in config\n" +
117 " --allow-backend\t(DEPRECATED) Start only the specific backend.\n" +
118 " --deny-backend\t(DEPRECATED) Deny a specific backend.\n" +
119 " --list-backends\tList all the available backends.\n" +
120 " --add-static-backend\tAdd a static backend by path.\n" +
121 " --disable-scheduler\tDisable the use of the scheduler.\n" +
122 " --help\t\tPrint this usage message.\n";
123
124 #if ENABLE_WEBSERVICES
125 usage += "\n" +
126 " --web-global\t\tAllow global access to the Web & WebService interfaces.\n" +
127 " --web-port\t\tPort to use for the internal web server.\n" +
128 " --web-root\t\tRoot directory to use for the internal web server.\n" +
129 " --web-disable\t\tDisable Web & WebServices functionality.\n";
130 #endif
131
132 Console.WriteLine (usage);
133 }
134
135 public static bool StartupProcess ()
136 {
137 // Profile our initialization
138 Stopwatch stopwatch = new Stopwatch ();
139 stopwatch.Start ();
140
141 SetupSignalHandlers ();
142
143 // Fire up our server
144 if (! StartServer ()) {
145 if (arg_replace)
146 {
147 #if ENABLE_WEBSERVICES
148 WebServiceBackEnd.Stop();
149 #endif
150 ReplaceExisting ();
151 }
152 else {
153 Logger.Log.Error ("Could not set up the listener for beagle requests. "
154 + "There is probably another beagled instance running. "
155 + "Use --replace to replace the running service");
156 Environment.Exit (1);
157 }
158 }
159
160 // Set up out-of-process indexing
161 if (Environment.GetEnvironmentVariable ("BEAGLE_ENABLE_IN_PROCESS_INDEXING") == null)
162 LuceneQueryable.IndexerHook = new LuceneQueryable.IndexerCreator (RemoteIndexer.NewRemoteIndexer);
163
164 // Initialize syncronization to keep the indexes local if PathFinder.HomeDir
165 // is on a non-block device, or if BEAGLE_SYNCHRONIZE_LOCALLY is set
166 if ((! SystemInformation.IsPathOnBlockDevice (PathFinder.HomeDir) && Conf.Daemon.IndexSynchronization) ||
167 Environment.GetEnvironmentVariable ("BEAGLE_SYNCHRONIZE_LOCALLY") != null)
168 IndexSynchronization.Initialize ();
169
170 // Start the query driver.
171 Logger.Log.Debug ("Starting QueryDriver");
172 QueryDriver.Start ();
173
174 // Start the Global Scheduler thread
175 if (! arg_disable_scheduler) {
176 Logger.Log.Debug ("Starting Scheduler thread");
177 Scheduler.Global.Start ();
178 }
179
180 // Start our Inotify threads
181 Inotify.Start ();
182
183 // Test if the FileAdvise stuff is working: This will print a
184 // warning if not. The actual advice calls will fail silently.
185 FileAdvise.TestAdvise ();
186
187 #if ENABLE_WEBSERVICES
188 //Beagle Web, WebService access initialization code:
189 WebServiceBackEnd.Start();
190 #endif
191 Shutdown.ShutdownEvent += OnShutdown;
192
193 Conf.WatchForUpdates ();
194
195 stopwatch.Stop ();
196
197 Logger.Log.Debug ("Daemon initialization finished after {0}", stopwatch);
198
199 if (arg_indexing_test_mode) {
200 Thread.Sleep (1000); // Ugly paranoia: wait a second for the backends to settle.
201 Logger.Log.Debug ("Running in indexing test mode");
202 Scheduler.Global.EmptyQueueEvent += OnEmptySchedulerQueue;
203 Scheduler.Global.Add (null); // pulse the scheduler
204 }
205 return false;
206 }
207
208 static void OnEmptySchedulerQueue ()
209 {
210 Logger.Log.Debug ("Scheduler queue is empty: terminating immediately");
211 Shutdown.BeginShutdown ();
212 Environment.Exit (0); // Ugly work-around: We need to call Exit here to avoid deadlocking.
213 }
214
215 public static void Main (string[] args)
216 {
217 try {
218 DoMain (args);
219 } catch (Exception ex) {
220 Logger.Log.Error ("Unhandled exception thrown. Exiting immediately.");
221 Logger.Log.Error (ex);
222 Environment.Exit (1);
223 }
224 }
225
226 public static void DoMain (string[] args)
227 {
228 SystemInformation.SetProcessName ("beagled");
229
230 // Process the command-line arguments
231 bool arg_debug = false;
232 bool arg_debug_memory = false;
233 bool arg_fg = false;
234
235 int i = 0;
236 while (i < args.Length) {
237
238 string arg = args [i];
239 ++i;
240 string next_arg = i < args.Length ? args [i] : null;
241
242 switch (arg) {
243 case "-h":
244 case "--help":
245 PrintUsage ();
246 Environment.Exit (0);
247 break;
248
249 case "--heap-buddy":
250 // Silently ignore the --heap-buddy argument: it gets handled
251 // in the wrapper script.
252 break;
253
254 case "--list-backends":
255 Console.WriteLine ("Current available backends:");
256 Console.Write (QueryDriver.ListBackends ());
257 Environment.Exit (0);
258 break;
259
260 case "--fg":
261 case "--foreground":
262 arg_fg = true;
263 break;
264
265 case "--bg":
266 case "--background":
267 arg_fg = false;
268 break;
269
270 case "--replace":
271 arg_replace = true;
272 break;
273
274 case "--debug":
275 arg_debug = true;
276 break;
277
278 case "--debug-memory":
279 arg_debug = true;
280 arg_debug_memory = true;
281 break;
282
283 case "--indexing-test-mode":
284 arg_indexing_test_mode = true;
285 arg_fg = true;
286 break;
287
288 case "--backend":
289 if (next_arg == null) {
290 Console.WriteLine ("--backend requires a backend name");
291 Environment.Exit (1);
292 break;
293 }
294
295 if (next_arg.StartsWith ("--")) {
296 Console.WriteLine ("--backend requires a backend name. Invalid name '{0}'", next_arg);
297 Environment.Exit (1);
298 break;
299 }
300
301 if (next_arg [0] != '+' && next_arg [0] != '-')
302 QueryDriver.OnlyAllow (next_arg);
303 else {
304 if (next_arg [0] == '+')
305 QueryDriver.Allow (next_arg.Substring (1));
306 else
307 QueryDriver.Deny (next_arg.Substring (1));
308 }
309
310 ++i; // we used next_arg
311 break;
312
313 case "--allow-backend":
314 // --allow-backend is deprecated, use --backends 'name' instead
315 // it will disable reading the list of enabled/disabled backends
316 // from conf and start the backend given
317 if (next_arg != null)
318 QueryDriver.OnlyAllow (next_arg);
319 ++i; // we used next_arg
320 break;
321
322 case "--deny-backend":
323 // deprecated: use --backends -'name' instead
324 if (next_arg != null)
325 QueryDriver.Deny (next_arg);
326 ++i; // we used next_arg
327 break;
328
329 case "--add-static-backend":
330 if (next_arg != null)
331 QueryDriver.AddStaticQueryable (next_arg);
332 ++i;
333 break;
334
335 case "--disable-scheduler":
336 arg_disable_scheduler = true;
337 break;
338
339 case "--indexing-delay":
340 if (next_arg != null) {
341 try {
342 QueryDriver.IndexingDelay = Int32.Parse (next_arg);
343 } catch {
344 Console.WriteLine ("'{0}' is not a valid number of seconds", next_arg);
345 Environment.Exit (1);
346 }
347 }
348
349 ++i;
350 break;
351
352 case "--autostarted":
353 if (! Conf.Searching.Autostart) {
354 Console.WriteLine ("Autostarting is disabled, not starting");
355 Environment.Exit (0);
356 }
357 break;
358 #if ENABLE_WEBSERVICES
359 case "--web-global":
360 WebServiceBackEnd.web_global = true;
361 WebServiceBackEnd.web_start = true;
362 break;
363
364 case "--web-port":
365 WebServiceBackEnd.web_port = next_arg;
366 ++i;
367 WebServiceBackEnd.web_start = true;
368 break;
369
370 case "--web-root":
371 WebServiceBackEnd.web_rootDir = next_arg;
372 ++i;
373 WebServiceBackEnd.web_start = true;
374 break;
375
376 case "--web-disable":
377 WebServiceBackEnd.web_start = false;
378 break;
379 #endif
380 default:
381 Console.WriteLine ("Unknown argument '{0}'", arg);
382 Environment.Exit (1);
383 break;
384
385 }
386 }
387
388 if (arg_indexing_test_mode) {
389 LuceneQueryable.OptimizeRightAway = true;
390 }
391
392
393 // Bail out if we are trying to run as root
394 if (Environment.UserName == "root" && Environment.GetEnvironmentVariable ("SUDO_USER") != null) {
395 Console.WriteLine ("You appear to be running beagle using sudo. This can cause problems with");
396 Console.WriteLine ("permissions in your .beagle and .wapi directories if you later try to run");
397 Console.WriteLine ("as an unprivileged user. If you need to run beagle as root, please use");
398 Console.WriteLine ("'su -c' instead.");
399 Environment.Exit (-1);
400 }
401
402 if (Environment.UserName == "root" && ! Conf.Daemon.AllowRoot) {
403 Console.WriteLine ("You can not run beagle as root. Beagle is designed to run from your own");
404 Console.WriteLine ("user account. If you want to create multiuser or system-wide indexes, use");
405 Console.WriteLine ("the beagle-build-index tool.");
406 Console.WriteLine ();
407 Console.WriteLine ("You can override this setting using the beagle-config or beagle-settings tools.");
408 Environment.Exit (-1);
409 }
410
411 try {
412 string tmp = PathFinder.HomeDir;
413 } catch (Exception e) {
414 Console.WriteLine ("Unable to start the daemon: {0}", e.Message);
415 Environment.Exit (-1);
416 }
417
418 MainLoopThread = Thread.CurrentThread;
419
420 Log.Initialize (PathFinder.LogDir,
421 "Beagle",
422 // FIXME: We always turn on full debugging output! We are still
423 // debugging this code, after all...
424 //arg_debug ? LogLevel.Debug : LogLevel.Warn,
425 LogLevel.Debug,
426 arg_fg);
427
428 Logger.Log.Info ("Starting Beagle Daemon (version {0})", ExternalStringsHack.Version);
429
430 Logger.Log.Info ("Running on {0}", SystemInformation.MonoRuntimeVersion);
431
432 Logger.Log.Debug ("Command Line: {0}",
433 Environment.CommandLine != null ? Environment.CommandLine : "(null)");
434
435 if (! ExtendedAttribute.Supported) {
436 Logger.Log.Warn ("Extended attributes are not supported on this filesystem. " +
437 "Performance will suffer as a result.");
438 }
439
440 // Start our memory-logging thread
441 if (arg_debug_memory)
442 ExceptionHandlingThread.Start (new ThreadStart (LogMemoryUsage));
443
444 // Do BEAGLE_EXERCISE_THE_DOG_HARDER-related processing.
445 ExerciseTheDogHarder ();
446
447 if (Application.InitCheck ("beagled", ref args))
448 Logger.Log.Debug ("Established a connection to the X server");
449 else
450 Logger.Log.Debug ("Unable to establish a connection to the X server");
451
452 XSetIOErrorHandler (BeagleXIOErrorHandler);
453
454 // Defer all actual startup until the main loop is
455 // running. That way shutdowns during the startup
456 // process work correctly.
457 GLib.Idle.Add (new GLib.IdleHandler (StartupProcess));
458
459 // Start our event loop.
460 Logger.Log.Debug ("Starting main loop");
461
462 Application.Run ();
463
464 // If we placed our sockets in a temp directory, try to clean it up
465 // Note: this may fail because the helper is still running
466 if (PathFinder.GetRemoteStorageDir (false) != PathFinder.StorageDir) {
467 try {
468 Directory.Delete (PathFinder.GetRemoteStorageDir (false));
469 } catch (IOException) { }
470 }
471
472 Logger.Log.Debug ("Leaving BeagleDaemon.Main");
473
474 if (arg_debug) {
475 Thread.Sleep (500);
476 ExceptionHandlingThread.SpewLiveThreads ();
477 }
478 }
479
480 /////////////////////////////////////////////////////////////////////////////
481
482 private delegate int XIOErrorHandler (IntPtr display);
483
484 [DllImport ("libX11.so.6")]
485 extern static private int XSetIOErrorHandler (XIOErrorHandler handler);
486
487 private static int BeagleXIOErrorHandler (IntPtr display)
488 {
489 Logger.Log.Debug ("Lost our connection to the X server! Trying to shut down gracefully");
490
491 if (! Shutdown.ShutdownRequested)
492 Shutdown.BeginShutdown ();
493
494 Logger.Log.Debug ("Xlib is forcing us to exit!");
495
496 ExceptionHandlingThread.SpewLiveThreads ();
497
498 // Returning will cause xlib to exit immediately.
499 return 0;
500 }
501
502 /////////////////////////////////////////////////////////////////////////////
503
504 static void SetupSignalHandlers ()
505 {
506 // Force OurSignalHandler to be JITed
507 OurSignalHandler (-1);
508
509 // Set up our signal handler
510 Mono.Unix.Native.Stdlib.signal (Mono.Unix.Native.Signum.SIGINT, OurSignalHandler);
511 Mono.Unix.Native.Stdlib.signal (Mono.Unix.Native.Signum.SIGTERM, OurSignalHandler);
512 if (Environment.GetEnvironmentVariable("BEAGLE_THERE_BE_NO_QUITTIN") == null)
513 Mono.Unix.Native.Stdlib.signal (Mono.Unix.Native.Signum.SIGQUIT, OurSignalHandler);
514
515 // Ignore SIGPIPE
516 Mono.Unix.Native.Stdlib.signal (Mono.Unix.Native.Signum.SIGPIPE, Mono.Unix.Native.Stdlib.SIG_IGN);
517 }
518
519 // Our handler triggers an orderly shutdown when it receives a signal.
520 // However, this can be annoying if the process gets wedged during
521 // shutdown. To deal with that case, we make a note of the time when
522 // the first signal comes in, and we allow signals to unconditionally
523 // kill the process after 5 seconds have passed.
524 static DateTime signal_time = DateTime.MinValue;
525 static void OurSignalHandler (int signal)
526 {
527 // This allows us to call OurSignalHandler w/o doing anything.
528 // We want to call it once to ensure that it is pre-JITed.
529 if (signal < 0)
530 return;
531
532 Logger.Log.Debug ("Handling signal {0} ({1})", signal, (Mono.Unix.Native.Signum) signal);
533
534 bool first_signal = false;
535 if (signal_time == DateTime.MinValue) {
536 signal_time = DateTime.Now;
537 first_signal = true;
538 }
539
540 if (Shutdown.ShutdownRequested) {
541
542 if (first_signal) {
543 Logger.Log.Debug ("Shutdown already in progress.");
544 } else {
545 double t = (DateTime.Now - signal_time).TotalSeconds;
546 const double min_t = 5;
547
548 if (t < min_t) {
549 Logger.Log.Debug ("Signals can force an immediate shutdown in {0:0.00}s", min_t-t);
550 } else {
551 Logger.Log.Debug ("Forcing immediate shutdown.");
552 Environment.Exit (0);
553 }
554 }
555
556 } else {
557 Logger.Log.Debug ("Initiating shutdown in response to signal.");
558 Shutdown.BeginShutdown ();
559 }
560 }
561
562 /////////////////////////////////////////////////////////////////////////////
563
564 private static void OnShutdown ()
565 {
566 #if ENABLE_WEBSERVICES
567 WebServiceBackEnd.Stop();
568 #endif
569 // Stop our Inotify threads
570 Inotify.Stop ();
571
572 // Shut down the global scheduler
573 Scheduler.Global.Stop ();
574
575 // Stop the messaging server
576 server.Stop ();
577 }
578
579 /////////////////////////////////////////////////////////////////////////////
580
581
582 private static ArrayList exercise_files = new ArrayList ();
583
584 private static void ExerciseTheDogHarder ()
585 {
586 string path;
587 path = Environment.GetEnvironmentVariable ("BEAGLE_EXERCISE_THE_DOG_HARDER");
588 if (path == null)
589 return;
590
591 DirectoryInfo dir = new DirectoryInfo (path);
592 foreach (FileInfo file in dir.GetFiles ())
593 exercise_files.Add (file);
594 if (exercise_files.Count == 0)
595 return;
596
597 int N = 5;
598 if (N > exercise_files.Count)
599 N = exercise_files.Count;
600
601 for (int i = 0; i < N; ++i)
602 ExceptionHandlingThread.Start (new ThreadStart (ExerciseTheDogHarderWorker));
603 }
604
605 private static void ExerciseTheDogHarderWorker ()
606 {
607 Random rng = new Random ();
608
609 while (! Shutdown.ShutdownRequested) {
610
611 FileInfo file = null;
612 int i;
613
614
615 lock (exercise_files) {
616 do {
617 i = rng.Next (exercise_files.Count);
618 file = exercise_files [i] as FileInfo;
619 } while (file == null);
620 exercise_files [i] = null;
621 }
622
623 string target;
624 target = Path.Combine (PathFinder.HomeDir, "_HARDER_" + file.Name);
625
626 Logger.Log.Debug ("ETDH: Copying {0}", file.Name);
627 file.CopyTo (target, true);
628
629 lock (exercise_files)
630 exercise_files [i] = file;
631
632 Thread.Sleep (500 + rng.Next (500));
633 }
634 }
635 }
636 }
A Beagle Git Repo