mogadm

   1 #!/usr/bin/perl
   2 # vim:ts=4 sw=4 ft=perl et:
   3
   4 use strict;
   5 use warnings;
   6 use Getopt::Long;
   7 use LWP::Simple;  # FIXME: use of this makes 'mog check' hang too long when multiple things down
   8 use Socket;
   9
  10 my @topcmds = qw(check stats host device domain class slave fsck rebalance settings);
  11 my $usage = {
  12     check => {
  13         des => "Check the state of the MogileFS world.",
  14     },
  15     stats => {
  16         des => "Show MogileFS system statistics.  (DEPRECATED: use mogstats instead)",
  17     },
  18     settings => {
  19         des => "Change/list server settings.",
  20         subcmd => {
  21             list => {
  22                 des => "List all server settings",
  23             },
  24             set => {
  25                 args => "<key> <value>",
  26                 des => "Set server setting 'key' to 'value'.",
  27             },
  28         },
  29     },
  30     host => {
  31         des => "Add/modify hosts.",
  32         subcmd => {
  33             list => {
  34                 des => "List all hosts.",
  35             },
  36             add => {
  37                 des => "Add a host to MogileFS.",
  38                 args => "<hostname> [opts]",
  39                 opts => {
  40                     "<hostname>"  => "Hostname of machine",
  41                     "--status=s"  => "One of {alive,down,readonly}.  Default 'down'.",
  42                     "--ip=s"      => "IP address of machine.",
  43                     "--port=i"    => "HTTP port of mogstored",
  44                     "--getport=i" => "Alternate HTTP port serving readonly traffic",
  45                     "--altip=s"   => "Alternate IP that is machine is reachable from",
  46                     "--altmask=s" => "Netmask which, when matches client, uses alt IP",
  47                 },
  48             },
  49             modify => {
  50                 des => "Modify a host's properties.",
  51                 args => "<hostname> [opts]",
  52                 opts => {
  53                     "<hostname>" => "Host name.",
  54                     "--status=s" => "One of {alive,down,readonly}.",
  55                     "--ip=s"     => "IP address of machine.",
  56                     "--port=i"   => "HTTP port of mogstored",
  57                     "--getport=i" => "Alternate HTTP port serving readonly traffic",
  58                     "--altip=s"   => "Alternate IP that is machine is reachable from",
  59                     "--altmask=s" => "Netmask which, when matches client, uses alt IP",
  60                 },
  61             },
  62             mark => {
  63                 des => "Change the status of a host.  (equivalent to 'modify --status')",
  64                 args => "<hostname> <status>",
  65                 opts => {
  66                     "<hostname>" => "Host name to bring up or down.",
  67                     "<status>"   => "One of {alive,down,readonly}.",
  68                 }
  69             },
  70             delete => {
  71                 des => "Delete a host.",
  72                 args => "<hostname>",
  73                 opts => {
  74                     "<hostname>" => "Host name to delete.",
  75                 },
  76             },
  77         },
  78     },
  79     device => {
  80         des => "Add/modify devices.",
  81         subcmd => {
  82             list => {
  83                 des => "List all devices, for each host.",
  84                 args => "[opts]",
  85                 opts => {
  86                     "--all"  => "Include dead devices in list.",
  87                     "--hostname=s" => "Limit results to a given host.",
  88                 },
  89             },
  90             summary => {
  91                 des => "List the summary of devices, for each host.",
  92                 args => "[opts]",
  93                 opts => {
  94                     "--status=s"  => "Devices of status A. Defaults to 'alive,readonly,drain'",
  95                     "--hostname=s" => "Limit results to a given host.",
  96                 },
  97             },
  98             add => {
  99                 des  => "Add a device to a host.",
 100                 args => "<hostname> <devid> [opts]",
 101                 opts => {
 102                     "<hostname>" => "Hostname to add a device",
 103                     "<devid>"    => "Numeric devid.  Never reuse these.",
 104                     "--status=s" => "One of 'alive' or 'down'.  Defaults to 'alive'.",
 105                 },
 106             },
 107             mark => {
 108                 des  => "Mark a device as {alive,dead,down,drain,readonly}",
 109                 args => "<hostname> <devid> <status>",
 110                 opts => {
 111                     "<hostname>" => "Hostname of device",
 112                     "<devid>"    => "Numeric devid to modify.",
 113                     "<status>"   => "One of {alive,dead,down,drain,readonly}",
 114                 },
 115             },
 116             modify => {
 117                 des => "Modify a device's properties.",
 118                 args => "<hostname> <devid> [opts]",
 119                 opts => {
 120                     "<hostname>" => "Hostname of device",
 121                     "<devid>"    => "Numeric devid to modify.",
 122                     "--status=s" => "One of {alive,dead,down,drain,readonly}",
 123                     "--weight=i" => "Positive numeric weight for device",
 124                 },
 125             },
 126             next => {
 127                 des => "Show the next available devid.",
 128             },
 129         },
 130     },
 131     domain => {
 132         des => "Add/modify domains (namespaces)",
 133         subcmd => {
 134             list => {
 135                 des => "List all hosts.",
 136             },
 137             add => {
 138                 des => "Add a domain (namespace)",
 139                 args => "<domain>",
 140                 opts => {
 141                     "<domain>" => "Domain (namespace) to add.",
 142                 },
 143             },
 144             delete => {
 145                 des => "Delete a domain.",
 146                 args => "<domain>",
 147                 opts => {
 148                     "<domain>" => "Domain (namespace) to add.",
 149                 },
 150             },
 151         },
 152     },
 153     class => {
 154         des => "Add/modify file classes.",
 155         subcmd => {
 156             list => {
 157                 des => "List all classes, for each domain.",
 158             },
 159             add => {
 160                 des => "Add a file class to a domain.",
 161                 args => "<domain> <class> [opts]",
 162                 opts => {
 163                     "<domain>" => "Domain to add class to.",
 164                     "<class>"  => "Name of class to add.",
 165                     "--mindevcount=i" => "Minimum number of replicas.",
 166                     "--replpolicy=s" => "Replication policy string.",
 167                     "--hashtype=s" => "Hash algorithm string ('MD5', 'NONE').",
 168                 },
 169             },
 170             modify => {
 171                 des => "Modify properties of a file class.",
 172                 args => "<domain> <class> [opts]",
 173                 opts => {
 174                     "<domain>" => "Domain to add class to.",
 175                     "<class>"  => "Name of class to add.",
 176                     "--mindevcount=i" => "Minimum number of replicas.",
 177                     "--replpolicy=s" => "Replication policy string.",
 178                     "--hashtype=s" => "Hash algorithm string ('MD5', 'NONE').",
 179                 },
 180             },
 181             delete => {
 182                 des => "Delete a file class from a domain.",
 183                 args => "<domain> <class>",
 184                 opts => {
 185                     "<domain>" => "Domain of class to delete.",
 186                     "<class>"  => "Class to delete.",
 187                 },
 188
 189             },
 190         },
 191     },
 192     slave => {
 193         des => 'Manipulate slave database information in a running mogilefsd.',
 194         subcmd => {
 195             list => {
 196                 des => 'List current store slave nodes.',
 197             },
 198             add => {
 199                 des => 'Add a slave node for store usage',
 200                 args => '<slave_key> [opts]',
 201                 opts => {
 202                     '--dsn=s' => "DBI DSN specifying what database to connect to.",
 203                     '--username=s' => "DBI username for connecting.",
 204                     '--password=s' => "DBI password for connecting.",
 205                 },
 206             },
 207             modify => {
 208                 des => 'Modify a slave node for store usage',
 209                 args => '<slave_key> [opts]',
 210                 opts => {
 211                     '--dsn=s' => "DBI DSN specifying what database to connect to.",
 212                     '--username=s' => "DBI username for connecting.",
 213                     '--password=s' => "DBI password for connecting.",
 214                 },
 215             },
 216             delete => {
 217                 des => 'Delete a slave node for store usage',
 218                 args => '<slave_key>',
 219             },
 220         },
 221     },
 222     rebalance => {
 223         des => "Control file rebalancing operations.",
 224         subcmd => {
 225             start => {
 226                 des => 'Start a rebalance job',
 227             },
 228             stop => {
 229                 des => 'Stop a rebalance job',
 230             },
 231             status => {
 232                 des => 'Show status of current rebalance job',
 233             },
 234             settings => {
 235                 des => 'Display rebalance settings',
 236             },
 237             test => {
 238                 des => 'Show what devices the current policy would match',
 239             },
 240             reset => {
 241                 des => 'Reset an existing policy',
 242             },
 243             policy => {
 244                 des => 'Add or adjust the current policy',
 245                 args => '[opts]',
 246                 opts => {
 247                     '--options=s' => "Policy string (see docs/wiki for details)",
 248                 },
 249             },
 250         },
 251     },
 252     fsck => {
 253         des => "Control a background filesystem check operation.",
 254         subcmd => {
 255             start => {
 256                 des => 'Start (or resume) background fsck',
 257             },
 258             stop => {
 259                 des => 'Stop (pause) background fsck',
 260             },
 261             status => {
 262                 des => 'Show fsck status',
 263             },
 264             reset => {
 265                 des => 'Reset fsck position back to the beginning',
 266                 args => '[opts]',
 267                 opts => {
 268                     '--policy-only' => "Check repl policy (assumed locations); don't stat storage nodes",
 269                     '--startpos=i'  => "FID to start at.",
 270                 }
 271             },
 272             clearlog => {
 273                 des => 'Clear the fsck log',
 274             },
 275             printlog => {
 276                 des => 'Display the fsck log',
 277             },
 278             taillog => {
 279                 des => 'Tail the fsck log',
 280             },
 281
 282         },
 283     },
 284 };
 285
 286 # load up our config files
 287 my %opts;
 288
 289 Getopt::Long::Configure("require_order", "pass_through");
 290 GetOptions(
 291         "trackers=s" => \$opts{trackers},
 292         "config=s"   => \$opts{config},
 293         "lib=s"      => \$opts{lib},
 294         "help"       => \$opts{help},
 295         "verbose"    => \$opts{verbose},
 296     ) or abortWithUsage();
 297 Getopt::Long::Configure("require_order", "no_pass_through");
 298
 299 my @configs = ($opts{config}, "$ENV{HOME}/.mogilefs.conf", "/etc/mogilefs/mogilefs.conf");
 300 foreach my $fn (reverse @configs) {
 301     next unless $fn && -e $fn;
 302     open FILE, "<$fn"
 303         or die "unable to open $fn: $!\n";
 304     while (<FILE>) {
 305         s/\#.*//;
 306         next unless m!^\s*(\w+)\s*=\s*(.+?)\s*$!;
 307         $opts{$1} = $2 unless ( defined $opts{$1} );
 308     }
 309     close FILE;
 310 }
 311
 312 # bail for help
 313 abortWithUsage() if $opts{help};
 314
 315 # make sure we have at least a topcmd
 316 my $topcmd = shift(@ARGV);
 317 abortWithUsage() unless $topcmd && $usage->{$topcmd};
 318
 319 # break up the trackers and ensure we got some
 320 if ($opts{trackers}) {
 321     $opts{trackers} = [ split(/\s*,\s*/, $opts{trackers}) ];
 322 }
 323 fail_text('no_trackers')
 324     unless ($opts{trackers} && @{$opts{trackers}}) || detect_local_tracker();
 325
 326 # okay, load up the libraries that we need
 327 if ($opts{lib}) {
 328     eval "use lib '$opts{lib}';";
 329 }
 330 eval "use MogileFS::Admin; use MogileFS::Client; 1;" or fail_text('cant_find_module');
 331
 332 # dispatch if it's special
 333 if ($topcmd eq 'check') {
 334     die "Unknown options/arguments to 'check' command.\n" if @ARGV;
 335     cmd_check();
 336 } elsif ($topcmd eq 'stats') {
 337     die "Unknown options/arguments to 'stats' command.\n" if @ARGV;
 338     cmd_stats();
 339 }
 340
 341 # get the verb
 342 my $verb = shift(@ARGV) or
 343     abort_with_topcmd_help($topcmd);
 344 my $cmdinfo = $usage->{$topcmd}{subcmd}{$verb};
 345 abort_with_topcmd_help($topcmd) unless $cmdinfo;
 346
 347 my $badargs = sub {
 348     my $msg = shift;
 349     abort_with_topcmd_help($topcmd, $verb, $msg);
 350 };
 351
 352 # get the non-option (non --foo) arguments:
 353 my %cmdargs;
 354 if (my $args = $cmdinfo->{args}) {
 355     my @args = split(/ /, $args);
 356     foreach my $arg (@args) {
 357         # positional (but named) parameter
 358         if ($arg =~ /^<(.+)>$/) {
 359             my $argname = $1;
 360             my $val = shift @ARGV;
 361             # map e.g. "dev5" to 5
 362             if ($argname eq "devid" && $val && $val =~ /^dev(\d+)$/) {
 363                 $val = $1;
 364             }
 365             $badargs->("Missing argument '$argname'") unless defined $val;
 366             $badargs->("Unexpected option.  Expected argument '$argname'") if $val =~ /^-/;
 367             $cmdargs{$argname} = $val;
 368         } elsif ($arg eq "[opts]") {
 369             # handled later.
 370         } else {
 371             die "INTERNAL ERROR.";
 372         }
 373     }
 374     $badargs->("Unexpected extra argument.") if @ARGV && $ARGV[0] !~ /^-/;
 375 } else {
 376     $badargs->("Unexpected arguments when expecting none.") if @ARGV;
 377 }
 378
 379 # parse the options
 380 if (my $opts = $cmdinfo->{opts}) {
 381     my %getopts;
 382     foreach (keys %$opts) {
 383         my $k = $_;
 384         next if $k =~ /^</;  # don't care about these.
 385         die "BOGUS KEY: '$k'" unless $k =~ /^--([\w\-]+)(=.+)?$/;
 386         my ($oname, $type) = ($1, $2 || "");
 387         $getopts{"$oname$type"} = \$cmdargs{$1};
 388     }
 389     GetOptions(%getopts)
 390         or abort_with_topcmd_help($topcmd, $verb);
 391 }
 392
 393 # see what we should do
 394 my $cmdsub = do {
 395     no strict 'refs';
 396     *{"cmd_${topcmd}_${verb}"} or abortWithUsage();
 397 };
 398
 399 # now call our lovely lovely sub
 400 $cmdsub->(\%cmdargs);
 401 exit 0;
 402
 403 sub detect_local_tracker {
 404     require IO::Socket::INET;
 405     my $loctrack = "127.0.0.1:7001";
 406     my $sock = IO::Socket::INET->new(PeerAddr => $loctrack, Timeout => 1);
 407     return 0 unless $sock;
 408     $opts{trackers} = [$loctrack];
 409     return 1;
 410 }
 411
 412 ###########################################################################
 413 ## command routines
 414 ###########################################################################
 415
 416 sub cmd_check {
 417     # step 1: we want to check each tracker for responsiveness
 418     my $now = time();
 419     my ($hosts, $devices);
 420     $| = 1;
 421     print "Checking trackers...\n";
 422     foreach my $t (@{$opts{trackers}}) {
 423         print "  $t ... ";
 424         my $mogadm = mogadm($t);
 425         if ($mogadm) {
 426             my $lhosts = hosts($mogadm);
 427             my $ldevs = devices($mogadm);
 428             if ($lhosts && $ldevs) {
 429                 print "OK\n";
 430                 $hosts = $lhosts;
 431                 $devices = $ldevs;
 432             } else {
 433                 print "REQUEST FAILURE (is the tracker up?)\n";
 434             }
 435         } else {
 436             print "INITIAL FAILURE (bad configuration?)\n";
 437         }
 438     }
 439
 440     # we should have hosts if we get here
 441     fail_text('no_hosts') unless $hosts;
 442     print "\n";
 443
 444     # step 2: now hit each of the hosts for responsiveness
 445     print "Checking hosts...\n";
 446     my @urls;
 447     foreach my $hostid (sort { $a <=> $b } keys %$hosts) {
 448         printf "  [%2d] %s ... ", $hostid, $hosts->{$hostid}->{hostname};
 449         if ($hosts->{$hostid}->{status} eq 'alive') {
 450             my $url = 'http://' . $hosts->{$hostid}->{hostip} . ':' . $hosts->{$hostid}->{http_port} . '/';
 451             my $file = get($url);
 452             if (defined $file) {
 453                 print "OK\n";
 454                 push @urls, [ $hostid, $url ];
 455             } else {
 456                 print "REQUEST FAILURE FETCHING: $url\n";
 457             }
 458         } else {
 459             print "skipping; status = $hosts->{$hostid}->{status}\n";
 460         }
 461     }
 462
 463     # everything should be chill
 464     fail_text('no_devices') unless @urls;
 465     print "\n";
 466
 467     # step 3: check devices for each host
 468     print "Checking devices...\n";
 469     printf "  host device      %10s %10s %10s %7s  %7s   %4s\n", 'size(G)', 'used(G)', 'free(G)', 'use% ', 'ob state', 'I/O%';
 470     printf "  ---- ------------ ---------- ---------- ---------- ------ ---------- -----\n";
 471     my %total;
 472     # Initialize to zero so that the total outputs doesn't need to check for undefined.
 473     map { $total{$_} = 0; } qw(total used avail);
 474
 475     foreach my $hosturl (@urls) {
 476         my ($hostid, $url) = @$hosturl;
 477         my $devs = $devices->{$hostid};
 478         DEV: foreach my $devid (sort { $a <=> $b } keys %$devs) {
 479             my $dev = $devs->{$devid};
 480             my $status = $dev->{status} || "??";
 481             next if $status eq "dead";
 482
 483             printf "  [%2d] %-7s", $hostid, "dev$devid";
 484
 485             my $usage = get($url . "/dev$devid/usage");
 486             if (! defined $usage) {
 487                 print "REQUEST FAILURE FETCHING: $url" . "dev$devid/usage\n";
 488                 next;
 489             }
 490             if (length($usage) < 1) {
 491                 print "USAGE FILE BROKEN OR EMPTY: $url" . "dev$devid/usage\n";
 492                 next;
 493             }
 494             my %data = ( map { split(/:\s+/, $_) } split(/\r?\n/, $usage) );
 495             foreach (qw(time used total avail)) {
 496                 $data{$_} = 0 if (!$data{$_} ||
 497                                    $data{$_} !~ /\A\d+(\.\d+)?\Z/);
 498             }
 499             foreach (qw(available device disk)) {
 500                 if (! exists $data{$_} || !$data{$_}) {
 501                     print "MISSING FIELD ($_) FROM USAGE FILE: $url" . "dev$devid/usage\n";
 502                     next DEV;
 503                 }
 504             }
 505             $data{age} = $now - $data{time};
 506             $data{used} /= 1024**2;
 507             $data{total} /= 1024**2;
 508             $data{available} /= 1024**2;
 509             $data{avail} = $data{available};
 510             my $pct = 100 - $data{available}/$data{total}*100;
 511             $total{used} += $data{used};
 512             $total{avail} += $data{avail};
 513             $total{total} += $data{total};
 514
 515             my $util = "N/A";
 516             if (defined($dev->{utilization}) && $dev->{utilization} =~ /\A\d+(\.\d+)?\Z/) {
 517                 $util = sprintf("%.1f", $dev->{utilization});
 518             }
 519
 520             printf("     %10.3f %10.3f %10.3f %6.2f%%  %-7s %5s\n",
 521                    (map { $data{$_} } qw(total used avail)),
 522                    $pct, ($dev->{observed_state} || "?"),
 523                    $util);
 524         }
 525     }
 526     my $pct = 0;
 527     # Avoid division by zero
 528     $pct = 100 - $total{avail}/$total{total}*100 if($total{total} > 0);
 529
 530     printf "  ---- ------------ ---------- ---------- ---------- ------\n";
 531     printf "             total:%10.3f %10.3f %10.3f %6.2f%%\n", (map { $total{$_} } qw(total used avail)), $pct;
 532
 533     # if we get here, all's well
 534     ok();
 535 }
 536
 537 sub cmd_stats {
 538     fail("mogadm stats is deprecated by new 'mogstats' utility");
 539 }
 540
 541 sub cmd_host_list {
 542     my $hosts = hosts();
 543     fail_text('no_hosts') unless $hosts;
 544
 545     foreach my $hostid (sort keys %$hosts) {
 546         my $host = $hosts->{$hostid};
 547         print "$host->{hostname} [$hostid]: $host->{status}\n";
 548         my @data = (
 549             'IP', "$host->{hostip}:$host->{http_port}",
 550             'Alt IP', $host->{altip},
 551             'Alt Mask', $host->{altmask},
 552             'GET Port', $host->{http_get_port},
 553         );
 554         while (my ($k, $v) = splice(@data, 0, 2)) {
 555             next unless $v;
 556             printf "  %-10s\%s\n", "$k:", $v;
 557         }
 558         print "\n";
 559     }
 560     ok();
 561 }
 562
 563 sub cmd_host_add {
 564     my $args = shift;
 565
 566     my $hosts = hosts_byname();
 567     fail_text('no_hosts') unless $hosts;
 568
 569     my $name = delete $args->{hostname};
 570     cmd_help_die("No hostname") unless $name;
 571     fail('Host already exists.') if $hosts->{$name};
 572
 573     # make sure we have an ip
 574     unless ($args->{ip}) {
 575         my $addr = gethostbyname($name);
 576         fail_text('host_add_no_ip') unless $addr;
 577         $args->{ip} = inet_ntoa($addr);
 578     }
 579
 580     # defaults
 581     $args->{port}   ||= 7500;
 582     $args->{status} ||= 'down';
 583
 584     # FIXME: verify the status can't be 'alive' if we can't get to ip:port
 585     # OR BETTER: also make default status the reachability of that ip:port
 586
 587     # now create the host
 588     my $mogadm = mogadm();
 589     $mogadm->create_host($name, $args);
 590     if ($mogadm->err) {
 591         fail("Failure creating host: " . $mogadm->errstr);
 592     }
 593
 594     ok('Host has been created.');
 595 }
 596
 597 sub cmd_host_modify {
 598     my $args = shift;
 599     my $name = delete $args->{hostname};
 600
 601     # FIXME: verify the status can't be 'alive' if we can't get to ip:port
 602
 603     # now modify the host
 604     my $mogadm = mogadm();
 605     $mogadm->update_host($name, $args);
 606     if ($mogadm->err) {
 607         fail("Failure modifying host: " . $mogadm->errstr);
 608     }
 609
 610     ok('Host has been modified.');
 611 }
 612
 613 sub cmd_host_delete {
 614     my $args = shift;
 615     my $name = delete $args->{hostname};
 616
 617     # now modify the host
 618     my $mogadm = mogadm();
 619     $mogadm->delete_host($name);
 620     if ($mogadm->err) {
 621         fail("Failure deleting host: " . $mogadm->errstr);
 622     }
 623
 624     ok('Host has been deleted.');
 625 }
 626
 627 sub cmd_host_mark {
 628     my $args = shift;
 629
 630     my $mogadm = mogadm();
 631     $mogadm->update_host($args->{hostname}, { status => $args->{status} });
 632     if ($mogadm->err) {
 633         fail("Failure updating host status: " . $mogadm->errstr);
 634     }
 635
 636     ok('Host status updated.');
 637 }
 638
 639 sub cmd_domain_list {
 640     # actually lists domains and classes
 641     my $domains = domains() or
 642         fail_text('no_domains');
 643     # now iterate
 644     printf " %-20s %-20s %-12s  %-12s %-7s\n", "domain", "class", "mindevcount", "replpolicy", "hashtype";
 645     printf "%-20s %-20s %-12s %-12s %-7s\n", '-' x 20, '-' x 20, '-' x 13, '-' x 12, '-' x 7;
 646     foreach my $domain (sort keys %$domains) {
 647         foreach my $class (sort keys %{$domains->{$domain}}) {
 648             my $dom = $domains->{$domain}->{$class};
 649             printf " %-20s %-20s      %-8d %-13s %-7s\n", $domain, $class,
 650                 $dom->{mindevcount} || 0, $dom->{replpolicy} || '',
 651                 $dom->{hashtype} || '';
 652         }
 653         print "\n";
 654     }
 655
 656     ok();
 657 }
 658
 659 sub cmd_domain_add {
 660     my $args = shift;
 661
 662     my $domain = delete $args->{domain};
 663
 664     # create
 665     my $mogadm = mogadm();
 666     $mogadm->create_domain($domain);
 667     if ($mogadm->err) {
 668         fail('Error creating domain: ' . $mogadm->errstr);
 669     }
 670
 671     ok('Domain created.');
 672 }
 673
 674 sub cmd_domain_delete {
 675     my $args = shift;
 676
 677     my $domain = $args->{domain};
 678
 679     # destroy
 680     my $mogadm = mogadm();
 681     $mogadm->delete_domain($domain);
 682     if ($mogadm->err) {
 683         fail('Error deleting domain: ' . $mogadm->errstr);
 684     }
 685
 686     ok('Domain deleted.');
 687 }
 688
 689 sub cmd_class_list {
 690     # same, pass it through
 691     cmd_domain_list();
 692 }
 693
 694 sub cmd_class_add {
 695     my $args = shift;
 696
 697     my $domain = delete $args->{domain};
 698     my $class  = delete $args->{class};
 699
 700     cmd_help_die() unless $domain && $class;
 701
 702     $args->{mindevcount} ||= 2;
 703     $args->{replpolicy}  ||= '';
 704
 705     my $mogadm = mogadm();
 706     $mogadm->create_class($domain, $class, $args);
 707     if ($mogadm->err) {
 708         fail('Error creating class: ' . $mogadm->errstr);
 709     }
 710
 711     ok('Class created.');
 712 }
 713
 714 sub cmd_class_modify {
 715     my $args = shift;
 716
 717     my $domain = delete $args->{domain};
 718     my $class  = delete $args->{class};
 719
 720     cmd_help_die() unless $domain && $class;
 721
 722     $args->{mindevcount} ||= 2;
 723     $args->{replpolicy}  ||= '';
 724
 725     my $mogadm = mogadm();
 726     $mogadm->update_class($domain, $class, $args);
 727     if ($mogadm->err) {
 728         fail('Error updating class: ' . $mogadm->errstr);
 729     }
 730
 731     ok('Class updated.');
 732 }
 733
 734 sub cmd_class_delete {
 735     my $args = shift;
 736
 737     my $domain = $args->{domain};
 738     my $class  = $args->{class};
 739
 740     cmd_help_die() unless $domain && $class;
 741
 742     my $mogadm = mogadm();
 743     $mogadm->delete_class($domain, $class);
 744     if ($mogadm->err) {
 745         fail('Error deleting class: ' . $mogadm->errstr);
 746     }
 747
 748     ok('Class deleted.');
 749 }
 750
 751 sub cmd_device_add {
 752     my $args = shift;
 753
 754     my $hosts = hosts() or
 755         fail_text('no_hosts');
 756
 757     my $host  = $args->{hostname};
 758     my $devid = $args->{devid};
 759     my $state = $args->{status} || "alive";
 760
 761     cmd_help_die("devid should be numeric") unless $devid =~ /^\d+$/;
 762
 763     # FIXME: server should be fixed to verify via HTTP that the devid directory exists
 764
 765     my $mogadm = mogadm();
 766     $mogadm->create_device(hostname => $host, devid => $devid, state => $state);
 767
 768     if ($mogadm->err) {
 769         fail('Error adding device: ' . $mogadm->errstr);
 770     }
 771
 772     ok('Device added.');
 773 }
 774
 775 sub warn_on_drain {
 776     my $args = shift;
 777     if ($args->{status} && $args->{status} eq 'drain') {
 778         print "***NOTE***: As of server version 2.40 'drain' has changed. See docs/wiki\n";
 779     }
 780 }
 781
 782 sub cmd_device_mark {
 783     my $args = shift;
 784
 785     warn_on_drain($args);
 786     my $mogadm = mogadm();
 787     $mogadm->change_device_state($args->{hostname},
 788                                  $args->{devid},
 789                                  $args->{status});
 790     if ($mogadm->err) {
 791         fail('Error updating device: ' . $mogadm->errstr);
 792     }
 793
 794     ok('Device updated.');
 795 }
 796
 797 sub cmd_device_modify {
 798     my $args = shift;
 799     my $hostname = delete $args->{hostname};
 800     my $devid = delete $args->{devid};
 801
 802     warn_on_drain($args);
 803     my $mogadm = mogadm();
 804     $mogadm->update_device($hostname, $devid, $args);
 805
 806     if ($mogadm->err) {
 807         fail('Error updating device: ' . $mogadm->errstr);
 808     }
 809
 810     ok('Device updated.');
 811 }
 812
 813 sub cmd_device_list {
 814     my $args = shift;
 815
 816     my $hosts = hosts();
 817     fail_text('no_hosts') unless $hosts;
 818
 819     my $devs = devices();
 820     fail_text('no_devices') unless $devs;
 821     my $hostname = $args->{hostname};
 822
 823     foreach my $hostid (sort keys %$hosts) {
 824         my $host = $hosts->{$hostid};
 825         if (defined $hostname && $hostname ne $host->{hostname}) {
 826             next;
 827         }
 828         print "$host->{hostname} [$hostid]: $host->{status}\n";
 829
 830         printf "%7s  %-7s %10s %10s %10s %10s\n", '', '', 'used(G)', 'free(G)', 'total(G)', 'weight(%)';
 831         foreach my $devid (sort keys %{$devs->{$hostid} || {}}) {
 832             my $dev = $devs->{$hostid}->{$devid};
 833             next if $dev->{status} eq "dead" && ! $args->{all};
 834
 835             my $total = $dev->{mb_total} / 1024;
 836             my $used = $dev->{mb_used} / 1024;
 837             my $free = $total - $used;
 838             printf "%7s: %7s %10.3f %10.3f %10.3f %10u\n", "dev$devid", $dev->{status}, $used, $free, $total, $dev->{weight};
 839         }
 840
 841         print "\n";
 842     }
 843
 844     ok();
 845 }
 846
 847 sub cmd_device_summary {
 848     my $args = shift;
 849     my $hostname = $args->{hostname};
 850     my %show_state;
 851     $show_state{$_} = 1 foreach split(/,/, ($args->{status} || "alive,readonly,drain"));
 852
 853     my $hosts = hosts();
 854     fail_text('no_hosts') unless $hosts;
 855
 856     my $devs = devices();
 857     fail_text('no_devices') unless $devs;
 858
 859     printf "%-15s %6s %7s  %8s %8s %8s %8s\n", 'Hostname', 'HostID', 'Status', 'used(G)', 'free(G)', 'total(G)', '%Used';
 860     foreach my $hostid (sort keys %$hosts) {
 861         my $host = $hosts->{$hostid};
 862         if (defined $hostname && $hostname ne $host->{hostname}) {
 863             next;
 864         }
 865         my ($total,$used) = (0, 0);
 866
 867         foreach my $devid (sort keys %{$devs->{$hostid} || {}}) {
 868             my $dev = $devs->{$hostid}->{$devid};
 869             next unless $show_state{$dev->{status}};
 870
 871             my $devtotal = $dev->{mb_total} / 1024;
 872             my $devused  = $dev->{mb_used} / 1024;
 873
 874             $total += $devtotal;
 875             $used  += $devused;
 876         }
 877         my $free = $total - $used;
 878         printf "%-15s [%4d]: %6s", $host->{hostname}, $hostid, $host->{status};
 879         printf "  %8.3f %8.3f %8.3f ", $used, $free, $total;
 880         printf "%8.2f", 100*$used/$total if $total;
 881         print "\n";
 882     }
 883
 884     ok();
 885
 886 }
 887
 888 sub cmd_device_next {
 889     my $devs = mogadm()->get_devices;
 890     fail_text('no_devices') unless @$devs;
 891     my $maxid = 0;
 892     foreach my $dev (@$devs) {
 893         my $devid = $dev->{devid};
 894         if ($devid > $maxid) {
 895             $maxid = $devid;
 896         }
 897     }
 898
 899     print($maxid + 1, "\n");
 900     ok();
 901 }
 902
 903 sub cmd_slave_list {
 904     my $mogadm = mogadm();
 905
 906     my $slaves = $mogadm->slave_list();
 907
 908     foreach my $key (sort keys %$slaves) {
 909         my $slavedata = $slaves->{$key};
 910         my ($dsn, $username, $password) = @$slavedata;
 911         print "$key --dsn=$dsn --username=$username --password=$password\n";
 912     }
 913
 914     ok();
 915 }
 916
 917 sub cmd_slave_add {
 918     my $mogadm = mogadm();
 919     my $args = shift;
 920
 921     my $rc = $mogadm->slave_add($args->{slave_key}, $args->{dsn}, $args->{username}, $args->{password});
 922
 923     if ($rc) {
 924         ok("Slave added");
 925     } else {
 926         fail("Slave failed to be added");
 927     }
 928 }
 929
 930 sub cmd_slave_modify {
 931     my $mogadm = mogadm();
 932     my $args = shift;
 933
 934     my $key = delete $args->{slave_key} or cmd_help_die("Key argument is required");
 935
 936     my $rc = $mogadm->slave_modify($key, %$args);
 937
 938     if ($rc) {
 939         ok("Slave modify success");
 940     } else {
 941         fail("Slave modify failure: " . $mogadm->errstr);
 942     }
 943 }
 944
 945 sub cmd_slave_delete {
 946     my $mogadm = mogadm();
 947
 948     my $args = shift;
 949
 950     my $rc = $mogadm->slave_delete($args->{slave_key});
 951
 952     if ($rc) {
 953         ok("Slave deleted");
 954     } else {
 955         fail("Slave delete failed");
 956     }
 957 }
 958
 959 sub cmd_rebalance_start {
 960     my $mogadm = mogadm();
 961     my $res = $mogadm->rebalance_start || fail($mogadm->errstr);
 962     ok("rebalance started");
 963 }
 964
 965 sub cmd_rebalance_stop {
 966     my $mogadm = mogadm();
 967     my $res = $mogadm->rebalance_stop || fail($mogadm->errstr);
 968     ok("rebalance stopped");
 969 }
 970
 971 sub cmd_rebalance_reset {
 972     my $mogadm = mogadm();
 973     my $res = $mogadm->rebalance_reset || fail($mogadm->errstr);
 974     ok("rebalance reset");
 975 }
 976
 977 # TODO: Make output prettier? Put hostname next to device name, print device
 978 # info?
 979 sub cmd_rebalance_test {
 980     my $mogadm = mogadm();
 981     my $res = $mogadm->rebalance_test || fail($mogadm->errstr);
 982     print "Tested rebalance policy...\n";
 983     my $s = $mogadm->server_settings;
 984     print "Policy: ", $s->{rebal_policy}, "\n\n";
 985     print "Source devices:\n";
 986     for my $dev (sort split /,/, $res->{sdevs}) {
 987         print " - ", $dev, "\n";
 988     }
 989     print "Destination devices:\n";
 990     for my $dev (sort split /,/, $res->{ddevs}) {
 991         print " - ", $dev, "\n";
 992     }
 993 }
 994
 995 sub cmd_rebalance_status {
 996     my $mogadm = mogadm();
 997
 998     my $ss  = $mogadm->server_settings or fail ($mogadm->errstr);
 999     my $res = $mogadm->rebalance_status or fail ($mogadm->errstr);
1000     if ($ss->{rebal_host}) {
1001         print "Rebalance is running\n";
1002     } else {
1003         print "Rebalance is stopped\n";
1004     }
1005     print "Rebalance status:\n";
1006     for my $o (sort split /\s+/, $res->{state}) {
1007         my ($k, $v) = split /=/, $o;
1008         printf("%25s = %-s\n", $k, $v);
1009     }
1010 }
1011
1012 sub cmd_rebalance_policy {
1013     my $mogadm = mogadm();
1014     my $args   = shift;
1015
1016     my $res = $mogadm->rebalance_set_policy($args->{options})
1017         or fail($mogadm->errstr);
1018
1019     ok("changed policy setting");
1020 }
1021
1022 sub cmd_rebalance_settings {
1023     my $mogadm = mogadm();
1024
1025     my $ss = $mogadm->server_settings
1026         or fail("can't get settings");
1027     foreach my $k (sort keys %$ss) {
1028         next unless ($k =~ '^rebal_');
1029         next if ($k eq 'rebal_state');
1030         printf("%25s = %-s\n", $k, $ss->{$k});
1031     }
1032 }
1033
1034 sub cmd_fsck_start {
1035     my $mogadm = mogadm();
1036     my $res = $mogadm->fsck_start || fail($mogadm->errstr);
1037     ok("fsck started");
1038 }
1039
1040 sub cmd_fsck_stop {
1041     my $mogadm = mogadm();
1042     my $res = $mogadm->fsck_stop || fail($mogadm->errstr);
1043     ok("fsck stopped");
1044 }
1045
1046 sub cmd_fsck_reset {
1047     my $mogadm = mogadm();
1048     my $args = shift;
1049     my $res = $mogadm->fsck_reset(
1050                                   policy_only => $args->{"policy-only"},
1051                                   startpos => $args->{"startpos"},
1052                                   )
1053         or fail($mogadm->errstr);
1054     ok("fsck stopped");
1055 }
1056
1057 sub cmd_fsck_clearlog {
1058     my $mogadm = mogadm();
1059     my $res = $mogadm->fsck_clearlog || fail($mogadm->errstr);
1060     ok("fsck log cleared");
1061 }
1062
1063 sub _log_dump {
1064     my %opts   = @_;
1065     my $max    = $opts{start};
1066     my $mogadm = mogadm();
1067
1068     my $fmt = "%-20s %5s %13s %10s\n";
1069     printf($fmt, "unixtime", "event", "fid", "devid");
1070     while (1) {
1071         my @rows = $mogadm->fsck_log_rows(after_logid => $max);
1072         unless (@rows) {
1073             $opts{on_stall}->();
1074             next;
1075         }
1076         foreach my $row (@rows) {
1077             printf($fmt,
1078                    $row->{utime},
1079                    $row->{evcode},
1080                    $row->{fid},
1081                    $row->{devid} || "-");
1082             $max = $row->{logid};
1083         }
1084     }
1085 }
1086
1087 sub cmd_fsck_printlog {
1088     _log_dump(start     => 0,
1089               on_stall => sub { exit 0; });
1090 }
1091
1092 sub cmd_fsck_taillog {
1093     my $mogadm = mogadm();
1094     my $status = $mogadm->fsck_status
1095         or fail("can't get fsck status");
1096     _log_dump(start     => $status->{max_logid} - 20,
1097               on_stall => sub { sleep 5; });
1098 }
1099
1100 sub cmd_fsck_status {
1101     my $mogadm = mogadm();
1102     my $status = $mogadm->fsck_status
1103         or fail("can't get fsck status");
1104
1105     my %known = map { $_ => 1 } qw(
1106                                    current_time
1107                                    max_logid
1108                                    );
1109     my $st = sub {
1110         my $k = shift;
1111         $known{$k} = 1;
1112         return $status->{$k};
1113     };
1114
1115     my $line = sub {
1116         printf("%11s: %-s\n", @_);
1117     };
1118     print "\n";
1119     my $host = $st->('host');
1120     $line->("Running", $st->('running') ? "Yes (on $host)" : "No");
1121
1122     my $ratio = $st->('end_fid') ? ($st->('max_fid_checked') / $st->('end_fid')) : 0;
1123     my $perc  = sprintf("%0.02f%%", 100 * $ratio);
1124
1125     $line->("Status",
1126             $st->('max_fid_checked') . " / " . $st->('end_fid')
1127             . " ($perc)");
1128     my $elap = $st->('start_time') ?
1129         (($st->('stop_time') || $st->('current_time')) - $st->('start_time')) :
1130         0;
1131     my $as_time = sub {
1132         my $s = shift;
1133         return int($s) . "s" if $s < 60;
1134         return int($s/60) . "m";
1135     };
1136     my $per_sec = $elap ? ($st->('max_fid_checked') / $elap) : 0;
1137     $line->("Time",  sprintf("%s (%d fids/s; %s remain)",
1138                              $as_time->($elap),
1139                              sprintf("%0.1f", $per_sec),
1140                              $as_time->($per_sec ?
1141                                         (($st->('end_fid') - $st->('max_fid_checked'))
1142                                          / $per_sec) :
1143                                         0)));
1144
1145     $line->("Check Type", ($st->('policy_only') ?
1146                            "Repl policy only (skip file checks)" :
1147                            "Normal (check policy + files)"));
1148
1149     if (my @unk = grep { !$known{$_} } sort keys %$status) {
1150         print "\n";
1151         foreach (@unk) {
1152             $line->("[$_]", $status->{$_});
1153         }
1154     }
1155     print "\n";
1156 }
1157
1158 sub cmd_settings_list {
1159     my $mogadm = mogadm();
1160     unless ($mogadm->can("server_settings")) {
1161         fail("settings commands require MogileFS::Client >= 1.07");
1162     }
1163     my $ss = $mogadm->server_settings
1164         or fail("can't get settings");
1165     foreach my $k (sort keys %$ss) {
1166         # Don't list noisy "setting"
1167         next if ($k =~ '^rebal');
1168         printf("%25s = %-s\n", $k, $ss->{$k});
1169     }
1170 }
1171
1172 sub cmd_settings_set {
1173     my $mogadm = mogadm();
1174     unless ($mogadm->can("set_server_setting")) {
1175         fail("settings commands require MogileFS::Client >= 1.07");
1176     }
1177     my $args = shift;
1178
1179     $mogadm->set_server_setting($args->{key}, $args->{value})
1180         or fail($mogadm->errstr);
1181     ok();
1182 }
1183
1184 ###########################################################################
1185 ## helper routines
1186 ###########################################################################
1187
1188 sub abortWithUsage {
1189     my $ret = "Usage:  (enter any command prefix, leaving off options, for further help)\n\n";
1190     foreach my $cmd (@topcmds) {
1191         my $sbc = $usage->{$cmd}->{subcmd};
1192         if ($sbc) {
1193             $ret .= "  mogadm $cmd ...\n";
1194         } else {
1195             $ret .= sprintf("  mogadm %-25s %-s\n",
1196                             "$cmd",
1197                             $usage->{$cmd}{des} || "");
1198             next;
1199         }
1200         foreach my $v (sort keys %$sbc) {
1201             my $scv = $usage->{$cmd}{subcmd}{$v};
1202             $ret .= "       ";
1203             my $dotdot = $scv->{args} ? "..." : "";
1204             $ret .= sprintf("  %-25s %-s\n",
1205                             "$cmd $v $dotdot",
1206                             $scv->{des} || "");
1207
1208         }
1209     }
1210     print $ret, "\n";
1211     exit(1);
1212 }
1213
1214 sub abort_with_topcmd_help {
1215     my ($cmd, $verb, $msg) = @_;
1216     if ($msg) {
1217         print "\nERROR: $msg\n\n";
1218     }
1219     my $cmdsfx = $verb ? "-$verb" : "";
1220     my $ret = "Help for '$cmd$cmdsfx' command:\n";
1221     unless ($verb) {
1222         $ret .= " (enter any command prefix, leaving off options, for further help)\n";
1223     }
1224     $ret .= "\n";
1225     foreach my $subcmdv (sort keys %{$usage->{$cmd}{subcmd}}) {
1226         next if $verb && $verb ne $subcmdv;
1227         my $scv = $usage->{$cmd}{subcmd}{$subcmdv};
1228         $ret .= sprintf("  %-50s %-s\n",
1229                         "mogadm $cmd $subcmdv " . ($scv->{args} || ""),
1230                         $scv->{des});
1231     }
1232     print $ret, "\n";
1233     if ($verb) {
1234         my $scv = $usage->{$cmd}{subcmd}{$verb};
1235         foreach my $opt (sort {
1236             (substr($b, 0, 1) cmp substr($a, 0, 1)) ||
1237                 $a cmp $b
1238         } keys %{$scv->{opts} || {}})
1239         {
1240             printf("      %-20s %s\n", $opt, $scv->{opts}->{$opt});
1241         }
1242         print "\n";
1243     }
1244     exit 1;
1245 }
1246
1247 sub cmd_help_die {
1248     my ($msg) = @_;
1249     abort_with_topcmd_help($topcmd, $verb, $msg);
1250 }
1251
1252
1253 sub text {
1254     return {
1255
1256         ######################################################################
1257         cant_find_module => <<END,
1258 Error loading modules: $@
1259
1260 Please ensure that you have the MogileFS::Admin and MogileFS::Client
1261 modules and all necessary dependencies installed in a location in your
1262 search path.  Or, add a search path to mogadm:
1263
1264     mogadm --lib=/path/to/lib
1265
1266 Or add it to the configuration file:
1267
1268     lib = /path/to/lib
1269 END
1270
1271         ######################################################################
1272         no_mogadm => <<END,
1273 Unable to access MogileFS tracker and/or instantiate a MogileFS::Admin object.
1274 END
1275
1276         ######################################################################
1277         no_domains => "Unable to retrieve domains from tracker(s).\n",
1278
1279         ######################################################################
1280         no_devices => "No devices found on tracker(s).\n",
1281
1282         ######################################################################
1283         host_add_no_ip => <<END,
1284 Hostname does not resolve to an IP, and you didn\'t specify one on the options
1285 list.  Please either verify the host resolves, or try again:
1286
1287     mogadm host add <hostname> --ip=<ipaddr> [...]
1288 END
1289
1290         ######################################################################
1291         no_hosts => <<END,
1292 Unable to retrieve host information from tracker(s).
1293 END
1294
1295         ######################################################################
1296         no_trackers => <<END,
1297 In order to use the mogadm toolkit, you need to provide the information about
1298 where your trackers are.
1299
1300 In your configuration file:
1301
1302     trackers = 10.10.0.33:7001, 10.10.0.34:7001
1303
1304 Or on the command line
1305
1306     mogadm --trackers=10.10.0.33:7001,10.10.0.34:7001
1307 END
1308
1309     }->{$_[0]} || "UNDEFINED [$_[0]]";
1310 }
1311
1312 sub fail_text {
1313     print STDERR text($_[0]) . "\n";
1314     exit 1;
1315 }
1316
1317 sub fail {
1318     print STDERR $_[0] . "\n";
1319     exit 1;
1320 }
1321
1322 sub ok {
1323     if ($opts{verbose}) {
1324         print STDOUT $_[0] . "\n" if $_[0];
1325     }
1326     exit 0;
1327 }
1328
1329 sub mogadm {
1330     my $host = shift();
1331     if ($host) {
1332         $host = [ $host ] unless ref $host;
1333     } else {
1334         $host = $opts{trackers};
1335     }
1336     # 10 seconds is the max time used for any of the admin locks (fsck status)
1337     # plus we leave a bit of time for work.
1338     my $timeout = 15;
1339     $timeout = $opts{timeout} if $opts{timeout} && $opts{timeout} =~ /^[0-9]+$/;
1340 #    $MogileFS::DEBUG = 2;
1341     my $mogadm = MogileFS::Admin->new( hosts => $host, timeout => $timeout );
1342     fail_text('no_mogadm') unless $mogadm;
1343     return $mogadm;
1344 }
1345
1346 sub stats {
1347     my $mogadm = shift() || mogadm();
1348     my $res;
1349     eval {
1350         $res = $mogadm->get_stats();
1351     };
1352     return undef if $@;
1353     return $res;
1354 }
1355
1356 sub hosts_byname {
1357     my $mogadm = shift() || mogadm();
1358     fail_text('no_mogadm') unless $mogadm;
1359
1360     my $res;
1361     eval {
1362         $res = _array_to_hashref($mogadm->get_hosts(), 'hostname');
1363     };
1364     return undef if $@;
1365     return $res;
1366 }
1367
1368 sub hosts {
1369     my $mogadm = shift() || mogadm();
1370     fail_text('no_mogadm') unless $mogadm;
1371
1372     my $res;
1373     eval {
1374         $res = _array_to_hashref($mogadm->get_hosts(), 'hostid');
1375     };
1376     return undef if $@;
1377     return $res;
1378 }
1379
1380 sub devices {
1381     my $mogadm = shift() || mogadm();
1382     fail_text('no_mogadm') unless $mogadm;
1383
1384     my $res;
1385     eval {
1386         $res = _array_to_hashref($mogadm->get_devices(), [ 'hostid', 'devid' ]);
1387     };
1388     return undef if $@;
1389     return $res;
1390 }
1391
1392 sub domains {
1393     my $mogadm = shift() || mogadm();
1394     fail_text('no_mogadm') unless $mogadm;
1395
1396     my $res;
1397     eval {
1398         $res = $mogadm->get_domains();
1399     };
1400     return undef if $@;
1401     return $res;
1402 }
1403
1404 sub _array_to_hashref {
1405     my ($array, $key) = @_;
1406     die "bad caller to _array_to_hashref\n"
1407         unless $array && $key;
1408     $key = [ $key ] unless ref $key eq 'ARRAY';
1409     my $kmax = scalar(@$key) - 1;
1410
1411     # and a dose of handwavium...
1412     my %res;
1413     foreach my $row (@$array) {
1414         my $ref = \%res;
1415         for (my $i = 0; $i <= $kmax; $i++) {
1416             if ($i == $kmax) {
1417                 # we're on the last key so just assign into $ref
1418                 $ref->{$row->{$key->[$i]}} = $row;
1419             } else {
1420                 # not on the last, so keep descending
1421                 $ref->{$row->{$key->[$i]}} ||= {};
1422                 $ref = $ref->{$row->{$key->[$i]}};
1423             }
1424         }
1425     }
1426
1427     # return result.. duh
1428     return \%res;
1429 }
1430
1431 __END__
1432
1433 =head1 NAME
1434
1435 mogadm - MogileFS admin tool
1436
1437 =head1 SYNOPSIS
1438
1439  $ mogadm [config options] <argument(s)> [argument options]
1440
1441  $ mogadm
1442    ....
1443    (prints contextual help, if missing command/arguments)
1444    ...
1445
1446 =head1 OPTIONS
1447
1448 =over 8
1449
1450 =item B<--lib=/path/to/lib>
1451
1452 Set this option to a path to include this directory in the module
1453 search path.
1454
1455 =item B<--trackers=10.0.0.117:7001,10.0.0.118:7001,...>
1456
1457 Use these MogileFS trackers for status information.
1458
1459 =back
1460
1461 =head1 ARGUMENTS
1462
1463 =over 8
1464
1465 =item B<check>
1466
1467 Check to ensure that all of the MogileFS system components are functioning
1468 and that we can contact everybody.  The quickest way of ensuring that the
1469 entire MogileFS system is functional from the current machine's point of view.
1470
1471 =item B<host add E<lt>hostE<gt> [host options]>
1472
1473 =item B<host modify E<lt>hostE<gt> [host options]>
1474
1475 =item B<host mark E<lt>hostE<gt> E<lt>statusE<gt>>
1476
1477 =item B<host delete E<lt>hostE<gt>>
1478
1479 =item B<host list>
1480
1481 Functions for manipulating hosts.  For add and modify, host options is in
1482 the format of normal command line options and can include anything in the
1483 L</"HOST OPTIONS"> section.
1484
1485 =item B<device add E<lt>hostE<gt> E<lt>device idE<gt>>
1486
1487 =item B<device mark E<lt>hostE<gt> E<lt>device idE<gt> E<lt>statusE<gt>>
1488
1489 =item B<device modify E<lt>hostE<gt> E<lt>deviceE<gt> [device options]>
1490
1491 =item B<device delete E<lt>hostE<gt> E<lt>deviceE<gt>>
1492
1493 =item B<device list>
1494
1495 =item B<device next>
1496
1497 Functions for manipulating devices. For add and modify, device options are in
1498 the format of normal command line options and can include anything in the
1499 L</"DEVICE OPTIONS"> section.
1500
1501 =item B<domain add E<lt>domainE<gt>>
1502
1503 =item B<domain delete E<lt>domainE<gt>>
1504
1505 =item B<domain list>
1506
1507 Simple commands for managing MogileFS domains.  Note that you cannot delete
1508 a domain unless it has no classes and is devoid of files.
1509
1510 =item B<class add E<lt>domainE<gt> E<lt>classE<gt> [class options]>
1511
1512 =item B<class modify E<lt>domainE<gt> E<lt>classE<gt> [class options]>
1513
1514 =item B<class delete E<lt>domainE<gt> E<lt>classE<gt>>
1515
1516 =item B<class list>
1517
1518 Commands for working with classes.  Please see the L</"CLASS OPTIONS"> section
1519 for the options to use with add/modify.  Also, delete requires that the class
1520 have no files in it before it will allow the deletion.
1521
1522 =item B<slave ...>
1523
1524 Add/remove slaves replicating from MogileFS master database.
1525
1526 TODO: detail this
1527
1528 Run B<mogadm slave> by itself for contextual help.
1529
1530 =item B<fsck printlog>
1531
1532 =item B<fsck taillog>
1533
1534 =item B<fsck clearlog>
1535
1536 Display or clear the log of fsck events.
1537
1538 =item B<fsck reset [fsck options]>
1539
1540 Reset fsck position back to the beginning.  Please see the L</"FSCK OPTIONS">
1541 section for options to use with fsck.
1542
1543 =item B<fsck start>
1544
1545 Start (or resume) background fsck from the last checked fid. If you want to
1546 check every fid, you must call B<fsck reset> before calling start.
1547
1548 =item B<fsck status>
1549
1550 Show the status of the presently active (or last if none active) fsck. This
1551 includes what FIDs are being checked, time statistics, check type as well as a
1552 summary of problems encountered so far.
1553
1554 =item B<fsck stop>
1555
1556 Stop (pause) background fsck
1557
1558 =item B<settings list>
1559
1560 Display all present MogileFS settings.
1561
1562 =item B<settings set E<lt>keyE<gt> E<lt>valueE<gt>>
1563
1564 Set the server setting for 'key' to 'value'.
1565
1566 The current settings are E<lt>enable_rebalanceE<gt> (set to 1 to start
1567 rebalance mode to move files to under-used devices) and
1568 E<lt>memcache_serversE<gt> (enable memcached caching in the tracker).
1569
1570 =back
1571
1572 =head1 HOST OPTIONS
1573
1574 =over 8
1575
1576 =item B<--ip=E<lt>ip of hostE<gt>>
1577
1578 =item B<--port=E<lt>port of mogstored on hostE<gt>>
1579
1580 Contact information for the host.  This is the minimum set of information needed
1581 to setup a host.
1582
1583 =item B<--getport=E<lt>alternate retrieval part on hostE<gt>>
1584
1585 If provided, causes the tracker to use this port for retrieving files.  Uploads are
1586 still processed at the standard port.
1587
1588 =item B<--altip=E<lt>alternate IPE<gt>>
1589
1590 =item B<--altmask=E<lt>mask to activate alternate IPE<gt>>
1591
1592 If a client request comes in from an IP that matches the alternate mask, then the
1593 host IP is treated as the alternate IP instead of the standard IP.  This can be
1594 used, for example, if you have two networks and you need to return one IP to
1595 reach the node on one network, but a second IP to reach it on the alternate
1596 network.
1597
1598 =item B<--status=E<lt>host statusE<gt>>
1599
1600 Valid host statuses are one of: alive, down, readonly.
1601
1602 =back
1603
1604 =head1 DEVICE OPTIONS
1605
1606 =over 8
1607
1608 =item B<--status=E<lt>device statusE<gt>>
1609
1610 Valid device statuses are one of: alive, dead, down, drain, readonly.
1611
1612 =item B<--weight=E<lt>device weight<gt>>
1613
1614 The weight used in calculation of preferred paths. It must be a positive
1615 integer.
1616
1617 =back
1618
1619 =head1 CLASS OPTIONS
1620
1621 =over 8
1622
1623 =item B<--mindevcount=E<lt>valueE<gt>>
1624
1625 Number of devices the files in this class should be replicated across.  Can be
1626 set to anything >= 1.
1627
1628 =item B<--replpolicy=E<lt>valueE<gt>>
1629
1630 Stringified replication policy. ie "MultipleHosts(3)" is equivalent to a
1631 --mindevcount=3. See documentation or plugins on alternative policies.
1632
1633 =item B<--hashtype=E<lt>valueE<gt>>
1634
1635 Name of the hash algorithm used for checksumming.  "MD5" or "NONE" for no
1636 checksumming.
1637
1638 =back
1639
1640 =head1 FSCK OPTIONS
1641
1642 =over 8
1643
1644 =item B<--policy-only>
1645
1646 Check replication policy (assumed locations) only; don't stat storage nodes for
1647 actual file presence.
1648
1649 =back
1650
1651 =head1 EXAMPLES
1652
1653 Host manipulation:
1654
1655     $ mogadm host list
1656     $ mogadm host add foo.local
1657     $ mogadm host add foo.local --status=down --ip=10.0.0.34 --port=7900
1658     $ mogadm host mark foo.local down
1659     $ mogadm host modify foo.local --port=7500
1660     $ mogadm host delete foo.local
1661
1662 Device manipulation:
1663
1664     $ mogadm device list
1665     $ mogadm device summary
1666     $ mogadm device summary --status=dead,down
1667     $ mogadm device next
1668     $ mogadm device add foo.local 16
1669     $ mogadm device add foo.local 17 --status=alive
1670     $ mogadm device mark foo.local 17 down
1671     $ mogadm device modify foo.local 17 --status=alive --weight=10
1672     $ mogadm device delete foo.local 17
1673
1674 Domain manipulation:
1675
1676     $ mogadm domain list
1677     $ mogadm domain add first.domain
1678     $ mogadm domain delete first.domain
1679
1680 Class manipulation
1681
1682     $ mogadm class list
1683     $ mogadm class add first.domain my.class
1684     $ mogadm class add first.domain my.class --mindevcount=3
1685     $ mogadm class add first.domain my.class --replpolicy="MultipleHosts(3)"
1686     $ mogadm class modify first.domain my.class --mindevcount=2
1687     $ mogadm class modify first.domain my.class --replpolicy="MultipleHosts(3)"
1688     $ mogadm class delete first.domain my.class
1689
1690 Check the status of your entire MogileFS system:
1691
1692     $ mogadm check
1693
1694 Check every file in the entire MogileFS system:
1695
1696     $ mogadm fsck reset
1697     $ mogadm fsck start
1698     $ mogadm fsck status
1699     $ mogadm fsck printlog
1700
1701 See all the things mogadm can do:
1702
1703     $ mogadm
1704
1705 Get help on a sub-command:
1706
1707     $ mogadm device
1708
1709
1710 =head1 CONFIGURATION
1711
1712 It is recommended that you create a configuration file such as C</etc/mogilefs/mogilefs.conf> (or at C<~/.mogilefs.conf>) to
1713 be used for configuration information.  Basically all you need is something like:
1714
1715     trackers = 10.0.0.23:7001, 10.0.0.15:7001
1716
1717     # if MogileFS::Admin files aren't installed in standard places:
1718     lib = /home/mogilefs/cgi-bin
1719
1720 Note that these can also be specified on the command line, as per above.
1721
1722 =head1 AUTHOR
1723
1724 Brad Fitzpatrick E<lt>L<brad@danga.com>E<gt>
1725
1726 Mark Smith E<lt>L<junior@danga.com>E<gt>
1727
1728 Leon Brocard E<lt>L<acme@astray.com>E<gt>, open sourced permissions from Foxtons Ltd.
1729
1730 Robin H. Johnson E<lt>robbat2@orbis-terrarum.netE<gt>
1731
1732 =head1 BUGS
1733
1734 Please report any on the MogileFS mailing list: L<http://groups.google.com/group/mogile/>.
1735
1736 =head1 LICENSE
1737
1738 Licensed for use and redistribution under the same terms as Perl itself.
1739
1740 =cut