mogstored

   1 #!/usr/bin/perl
   2
   3 eval 'exec /usr/bin/perl -w -S $0 ${1+"$@"}'
   4     if 0; # not running under some shell
   5
   6 #
   7 # MogileFS storage node daemon
   8 #  (perlbal front-end)
   9 #
  10 # (c) 2004, Brad Fitzpatrick, <brad@danga.com>
  11 # (c) 2006-2007, Six Apart, Ltd.
  12
  13 use strict;
  14 use lib 'lib';
  15 use Mogstored::HTTPServer;
  16
  17 # based on where we found Mogstored::HTTPServer (a pure-perl module),
  18 # add the mogdeps/ subdirectory of that base to our @INC search
  19 # path, where all the misc Mogile dependencies are installed.
  20 BEGIN {
  21     my $libpath;
  22     if (! $ENV{MOGILE_NO_BUILTIN_DEPS} &&
  23         ($libpath = $INC{"Mogstored/HTTPServer.pm"}) &&
  24         $libpath =~ s!Mogstored/HTTPServer.pm$!!)
  25     {
  26         my $dep_dir = "${libpath}mogdeps";
  27         push @INC, $dep_dir;
  28         unless (($ENV{PERL5LIB} || "") =~ /$dep_dir/) {
  29             $ENV{PERL5LIB} = join(":",
  30                                   split(/:/, $ENV{PERL5LIB} || ""),
  31                                   $dep_dir);
  32         }
  33     }
  34 }
  35
  36 use IO::Socket::INET;
  37 use POSIX qw(WNOHANG);
  38 use Perlbal 1.73;
  39 use FindBin qw($Bin $RealScript);
  40 use Gearman::Server 1.08;
  41 use Gearman::Client::Async 0.93;
  42
  43 use Mogstored::HTTPServer::Perlbal;
  44 use Mogstored::HTTPServer::Lighttpd;
  45 use Mogstored::HTTPServer::None;
  46 use Mogstored::HTTPServer::Apache;
  47 use Mogstored::SideChannelListener;
  48 use Mogstored::SideChannelClient;
  49
  50 my $selfexe = "$Bin/$RealScript";
  51
  52 # State:
  53 my %on_death;             # pid -> subref (to run when pid dies)
  54 my %devnum_to_device;     # mogile device number (eg. 'dev1' would be '1') -> os device path (eg. '/dev/rd0')
  55 my %osdevnum_to_device;   # os device number (fetched via stat(file)[0]) -> os device path (ec. '/dev/rd0')
  56 my %iostat_listeners;     # fd => SideChannel client: clients interested in iostat data.
  57 my $iostat_available = 1; # bool: iostat working.  assume working to start.
  58 my ($iostat_pipe_r, $iostat_pipe_w);  # pipes for talking to iostat process
  59
  60 # Config:
  61 my $opt_daemonize;
  62 my $opt_config;
  63 my $opt_iostat = 1;  # default to on now
  64 my $max_conns = 10000;
  65 my $http_listen = "0.0.0.0:7500";
  66 my $mgmt_listen = "0.0.0.0:7501";
  67 my $docroot     = "/var/mogdata";
  68 my $default_config = "/etc/mogilefs/mogstored.conf";
  69 my $server      = $ENV{MOGSTORED_SERVER_TYPE} || "perlbal";
  70 my $serverbin   = "";
  71 my $pidfile     = undef;
  72
  73 # Rename binary in process list to make init scripts saner
  74 $0 = "mogstored";
  75
  76 my %config_opts = (
  77                    'iostat'       => \$opt_iostat,
  78                    'daemonize|d'  => \$opt_daemonize,
  79                    'config=s'     => \$opt_config,
  80                    'httplisten=s' => \$http_listen,
  81                    'mgmtlisten=s' => \$mgmt_listen,
  82                    'docroot=s'    => \$docroot,
  83                    'maxconns=i'   => \$max_conns,
  84                    'server=s'     => \$server,
  85                    'serverbin=s'  => \$serverbin,
  86                    'pidfile=s'    => \$pidfile,
  87                    );
  88 usage() unless Getopt::Long::GetOptions(%config_opts);
  89
  90 die "Unknown server type.  Valid options: --server={perlbal,lighttpd,apache,none}"
  91     unless $server =~ /^perlbal|lighttpd|apache|none$/;
  92
  93 $opt_config = $default_config if ! $opt_config && -e $default_config;
  94 load_config_file($opt_config => \%config_opts) if $opt_config;
  95
  96 # initialize basic required Perlbal machinery, for any HTTP server
  97 my $perlbal_init = qq{
  98    CREATE SERVICE mogstored
  99      SET role = web_server
 100      SET docroot = $docroot
 101
 102     # don't listen... this is just a stub service.
 103     CREATE SERVICE mgmt
 104        SET role = management
 105      ENABLE mgmt
 106 };
 107 $perlbal_init .= "\nSERVER pidfile = $pidfile" if defined($pidfile);
 108 Perlbal::run_manage_commands($perlbal_init , sub { print STDERR "$_[0]\n"; });
 109
 110 # start HTTP server
 111 my $httpsrv_class = "Mogstored::HTTPServer::" . ucfirst($server);
 112 my $httpsrv       = $httpsrv_class->new(
 113                                         listen   => $http_listen,
 114                                         docroot  => $docroot,
 115                                         maxconns => $max_conns,
 116                                         bin      => $serverbin,
 117                                         );
 118 $httpsrv->start;
 119
 120 if ($opt_daemonize) {
 121     $httpsrv->pre_daemonize;
 122     Perlbal::daemonize();
 123 } else {
 124     print "Running.\n";
 125 }
 126
 127 $httpsrv->post_daemonize;
 128
 129 # kill our children processes on exit:
 130 my $parent_pid = $$;
 131
 132 $SIG{TERM} = $SIG{INT} = sub {
 133     return unless $$ == $parent_pid; # don't let this be inherited
 134     kill 'TERM', grep { $_ } keys %on_death;
 135     POSIX::_exit(0);
 136 };
 137
 138 setup_iostat_pipes();
 139 start_disk_usage_process();
 140 start_iostat_process() if $opt_iostat;
 141 harvest_dead_children();  # every 2 seconds, it reschedules itself
 142 setup_sidechannel_listener();
 143 start_fidsizes_worker();
 144
 145 # now start the main loop
 146 Perlbal::run();
 147
 148 ############################################################################
 149
 150 sub usage {
 151     my $note = shift;
 152     $note = $note ? "NOTE: $note\n\n" : "";
 153
 154     die "${note}Usage: mogstored [OPTS]
 155
 156 OPTS:
 157  --daemonize  -d        Daemonize
 158  --config=<file>        Set config file (default is /etc/mogilefs/mogstored.conf)
 159  --httplisten=<ip:port> IP/Port HTTP server listens on
 160  --mgmtlisten=<ip:port> IP/Port management/sidechannel listens on
 161  --docroot=<path>       Docroot above device mount points.  Defaults to /var/mogdata
 162 ";
 163
 164 }
 165
 166 # accessor for SideChannelClient:
 167 sub Mogstored::iostat_available {
 168     return $iostat_available;
 169 }
 170
 171 sub load_config_file {
 172     my ($conffile, $opts) = @_;
 173
 174     # parse the mogstored config file, which is just lines of comments and
 175     # "key = value" lines, where keys are just the same as command line
 176     # options.
 177     die "Config file $opt_config doesn't exist.\n" unless -e $conffile;
 178     open my $fh, $conffile or die "Couldn't open config file for reading: $!";
 179     while (<$fh>) {
 180         s/\#.*//;
 181         next unless /\S/;
 182         if (/SERVER max_connect/i || /CREATE SERVICE/i) {
 183             usage("Your $opt_config file is the old syntax.  The new format is simply lines of <key> = <value> where keys are the same as mogstored's command line options.");
 184
 185         }
 186         die "Unknown config syntax: $_\n" unless /^\s*(\w+)\s*=\s*(.+?)\s*$/;
 187         my ($key, $val) = ($1, $2);
 188         my $dest;
 189         foreach my $ck (keys %$opts) {
 190             next unless $ck =~ /^$key\b/;
 191             $dest = $opts->{$ck};
 192         }
 193         die "Unknown config setting: $key\n" unless $dest;
 194         $$dest = $val;
 195     }
 196 }
 197
 198 sub harvest_dead_children {
 199     my $dead = waitpid(-1, WNOHANG);
 200     if ($dead > 0) {
 201         my $code = delete $on_death{$dead};
 202         $code->() if $code;
 203     }
 204     Danga::Socket->AddTimer(2, \&harvest_dead_children);
 205 }
 206
 207 sub start_fidsizes_worker {
 208
 209     # Note: in this case, this load is *before* the fork (which happens
 210     # in Gearman::Server's start_worker), so be careful nothing
 211     # heavy/gross is added to the FIDSizes worker.
 212     require Mogstored::ChildProcess::FIDSizes;
 213     my $class = "Mogstored::ChildProcess::FIDSizes";
 214     $class->pre_exec_init;
 215
 216     my $pid = Mogstored->gearman_server->start_worker(sub {
 217         $class->exec;
 218     });
 219
 220     # old Gearman::Servers didn't return pid integers
 221     if ($pid =~ /^\d+$/) {
 222         Mogstored->on_pid_death($pid, \&start_fidsizes_worker);
 223     }
 224 }
 225
 226 sub Mogstored::on_pid_death {
 227     my ($class, $pid, $code) = @_;
 228     $on_death{$pid} = $code;
 229 }
 230
 231 # returns $pid of child, if parent, else runs child.
 232 sub start_disk_usage_process {
 233     my $child = fork;
 234     unless (defined $child) {
 235         Perlbal::log('crit', "Fork error creating disk usage tracking process");
 236         return undef;
 237     }
 238
 239     # if we're the parent.
 240     if ($child) {
 241         $on_death{$child} = sub {
 242             start_disk_usage_process();  # start a new one
 243         };
 244         return $child;
 245     }
 246
 247     require Mogstored::ChildProcess::DiskUsage;
 248     my $class = "Mogstored::ChildProcess::DiskUsage";
 249     $class->pre_exec_init;
 250     $class->exec;
 251 }
 252
 253 sub Mogstored::iostat_subscribe {
 254     my ($class, $sock) = @_;
 255     $iostat_listeners{fileno($sock->sock)} = $sock;
 256 }
 257
 258 sub Mogstored::iostat_unsubscribe {
 259     my ($class, $sock) = @_;
 260     my $fdno = fileno($sock->sock);
 261     return unless defined $fdno;
 262     delete $iostat_listeners{$fdno};
 263 }
 264
 265 # to be honest, I have no clue why this exists.  I just had to move it
 266 # around for multi-server refactoring, and I felt better not
 267 # understanding it but preserving than killing it.  in particular, why
 268 # is this "graceful"?  (gets called from SideChannelClient's
 269 # die_gracefully)
 270 sub Mogstored::on_sidechannel_die_gracefully {
 271     if ($$ == $parent_pid) {
 272         kill 'TERM', grep { $_ } keys %on_death;
 273     }
 274 }
 275
 276 sub setup_sidechannel_listener {
 277     Mogstored::SideChannelListener->new($mgmt_listen);
 278 }
 279
 280 my $iostat_read_buf = "";
 281 sub setup_iostat_pipes {
 282     pipe ($iostat_pipe_r, $iostat_pipe_w);
 283     IO::Handle::blocking($iostat_pipe_r, 0);
 284     IO::Handle::blocking($iostat_pipe_w, 0);
 285
 286     Danga::Socket->AddOtherFds(fileno($iostat_pipe_r), sub {
 287         read_from_iostat_child();
 288     });
 289 }
 290
 291 sub start_iostat_process {
 292     my $pid = fork;
 293     unless (defined $pid) {
 294         warn "Fork for iostat failed: $!";
 295         return;
 296     }
 297
 298     if ($pid) {
 299         # Parent
 300         $on_death{$pid} = sub {
 301             start_iostat_process();
 302         };
 303         return;
 304     }
 305
 306     require Mogstored::ChildProcess::IOStat;
 307     my $class = "Mogstored::ChildProcess::IOStat";
 308     $class->pre_exec_init;
 309     $class->exec;
 310 }
 311
 312 sub Mogstored::get_iostat_writer_pipe { $iostat_pipe_w }
 313
 314 # (runs in parent event-loop process)
 315 sub read_from_iostat_child {
 316     my $data;
 317     my $rv = sysread($iostat_pipe_r, $data, 10240);
 318     return unless $rv && $rv > 0;
 319
 320     $iostat_read_buf .= $data;
 321
 322     # only write complete lines to sockets (in case for some reason we get
 323     # a partial read and child process dies...)
 324     while ($iostat_read_buf =~ s/(.+)\r?\n//) {
 325         my $line = $1;
 326         foreach my $out_sock (values %iostat_listeners) {
 327             # where $line will be like "dev53\t53.23" or a "." to signal end of a group of devices.
 328             $out_sock->write("$line\n");
 329         }
 330     }
 331 }
 332
 333 my $gearman_server;
 334 sub Mogstored::gearman_server {
 335     return $gearman_server ||= Gearman::Server->new;
 336 }
 337
 338 my $gearman_client;
 339 sub Mogstored::gearman_client {
 340     return $gearman_client ||=
 341         Gearman::Client::Async->new(job_servers => [ Mogstored->gearman_server ]);
 342 }
 343
 344 # Local Variables:
 345 # mode: perl
 346 # c-basic-indent: 4
 347 # indent-tabs-mode: nil
 348 # End:
 349
 350 __END__
 351
 352 =head1 NAME
 353
 354 mogstored -- MogileFS storage daemon
 355
 356 =head1 USAGE
 357
 358 This is the MogileFS storage daemon, which is just an HTTP server that
 359 supports PUT, DELETE, etc.  It's actually a wrapper around L<Perlbal>,
 360 doing all the proper Perlbal config for you.
 361
 362 In addition, it monitors disk usage, I/O activity, etc, which are
 363 checked from the L<MogileFS tracker|mogilefsd>.
 364
 365 =head1 AUTHORS
 366
 367 Brad Fitzpatrick E<lt>brad@danga.comE<gt>
 368
 369 Mark Smith E<lt>junior@danga.comE<gt>
 370
 371 Jonathan Steinert E<lt>jsteinert@sixapart.comE<gt>
 372
 373 =head1 ENVIRONMENT
 374
 375 =over 4
 376
 377 =item PERLBAL_XS_HEADERS
 378
 379 If defined and 0, Perlbal::XS::HTTPHeaders will not be used, if
 380 present.  Otherwise, it will be enabled by default, if installed and
 381 loadable.
 382
 383 =back
 384
 385 =head1 COPYRIGHT
 386
 387  Copyright 2004, Danga Interactive
 388  Copyright 2005-2006, Six Apart Ltd.
 389
 390 =head1 LICENSE
 391
 392 Same terms as Perl itself.  Artistic/GPLv2, at your choosing.
 393
 394 =head1 SEE ALSO
 395
 396 L<MogileFS::Overview> -- high level overview of MogileFS
 397
 398 L<mogilefsd> -- MogileFS daemon
 399
 400 L<http://danga.com/mogilefs/>