lib/MogileFS/Device.pm

   1 package MogileFS::Device;
   2 use strict;
   3 use warnings;
   4 use Carp qw/croak/;
   5 use MogileFS::Server;
   6 use MogileFS::Util qw(throw);
   7 use MogileFS::Util qw(okay_args device_state error);
   8
   9 =head1
  10
  11 MogileFS::Device - device class
  12
  13 =cut
  14
  15 BEGIN {
  16     my $testing = $ENV{TESTING} ? 1 : 0;
  17     eval "sub TESTING () { $testing }";
  18 }
  19
  20 my @observed_fields = qw/observed_state utilization reject_bad_md5/;
  21 my @fields = (qw/hostid status weight mb_total mb_used mb_asof devid/,
  22     @observed_fields);
  23
  24 sub new_from_args {
  25     my ($class, $args, $host_factory) = @_;
  26     my $self = bless {
  27         host_factory => $host_factory,
  28         %{$args},
  29     }, $class;
  30
  31     # FIXME: No guarantee (as of now?) that hosts get loaded before devs.
  32     #$self->host || die "No host for $self->{devid} (host $self->{hostid})";
  33
  34     croak "invalid device observed state '$self->{observed_state}', valid: writeable, readable, unreachable"
  35         if $self->{observed_state} && $self->{observed_state} !~ /^(?:writeable|readable|unreachable)$/;
  36
  37     return $self;
  38 }
  39
  40 # Instance methods
  41
  42 sub id     { return $_[0]{devid} }
  43 sub devid  { return $_[0]{devid} }
  44 sub name   { return $_[0]{devid} }
  45 sub status { return $_[0]{status} }
  46 sub weight { return $_[0]{weight} }
  47 sub hostid { return $_[0]{hostid} }
  48
  49 sub host {
  50     my $self = shift;
  51     return $self->{host_factory}->get_by_id($self->{hostid});
  52 }
  53
  54 # returns 0 if not known, else [0,1]
  55 sub percent_free {
  56     my $self = shift;
  57     return 0 unless $self->{mb_total} && defined $self->{mb_used};
  58     return 1 - ($self->{mb_used} / $self->{mb_total});
  59 }
  60
  61 # returns undef if not known, else [0,1]
  62 sub percent_full {
  63     my $self = shift;
  64     return undef unless $self->{mb_total} && defined $self->{mb_used};
  65     return $self->{mb_used} / $self->{mb_total};
  66 }
  67
  68 # FIXME: $self->mb_free?
  69 sub fields {
  70     my $self = shift;
  71     my @tofetch = @_ ? @_ : @fields;
  72     my $ret = { (map { $_ => $self->{$_} } @tofetch),
  73         'mb_free' => $self->mb_free };
  74     return $ret;
  75 }
  76
  77 sub observed_fields {
  78     return $_[0]->fields(@observed_fields);
  79 }
  80
  81 sub observed_utilization {
  82     my $self = shift;
  83
  84     if (TESTING) {
  85         my $weight_varname = 'T_FAKE_IO_DEV' . $self->id;
  86         return $ENV{$weight_varname} if defined $ENV{$weight_varname};
  87     }
  88
  89     return $self->{utilization};
  90 }
  91
  92 sub host_ok {
  93     my $host = $_[0]->host;
  94     return ($host && $host->observed_reachable);
  95 }
  96
  97 sub observed_writeable {
  98     my $self = shift;
  99     return 0 unless $self->host_ok;
 100     return $self->{observed_state} && $self->{observed_state} eq 'writeable';
 101 }
 102
 103 sub observed_readable {
 104     my $self = shift;
 105     return 0 unless $self->host_ok;
 106     return $self->{observed_state} && $self->{observed_state} eq 'readable';
 107 }
 108
 109 sub observed_unreachable {
 110     my $self = shift;
 111     # host is unreachability implies device unreachability
 112     return 1 unless $self->host_ok;
 113     return $self->{observed_state} && $self->{observed_state} eq 'unreachable';
 114 }
 115
 116 # FIXME: This pattern is weird. Store the object on new?
 117 sub dstate {
 118     my $ds = device_state($_[0]->status);
 119     return $ds if $ds;
 120     error("dev$_[0]->{devid} has bogus status '$_[0]->{status}', pretending 'down'");
 121     return device_state("down");
 122 }
 123
 124 sub can_delete_from {
 125     return $_[0]->dstate->can_delete_from;
 126 }
 127
 128 # this method is for Monitor, other workers should use should_read_from
 129 sub can_read_from {
 130     return $_[0]->host->alive && $_[0]->dstate->can_read_from;
 131 }
 132
 133 # this is the only method a worker should call for checking for readability
 134 sub should_read_from {
 135     return $_[0]->can_read_from && ($_[0]->observed_readable || $_[0]->observed_writeable);
 136 }
 137
 138 # FIXME: Is there a (unrelated to this code) bug where new files aren't tested
 139 # against the free space limit before being stored or replicated somewhere?
 140 sub should_get_new_files {
 141     my $self   = shift;
 142     my $dstate = $self->dstate;
 143
 144     return 0 unless $dstate->should_get_new_files;
 145     return 0 unless $self->observed_writeable;
 146     return 0 unless $self->host->alive;
 147     # have enough disk space? (default: 100MB)
 148     my $min_free = MogileFS->config("min_free_space");
 149     return 0 if $self->{mb_total} &&
 150         $self->mb_free < $min_free;
 151
 152     return 1;
 153 }
 154
 155 sub mb_free {
 156     my $self = shift;
 157     return $self->{mb_total} - $self->{mb_used}
 158         if $self->{mb_total} && defined $self->{mb_used};
 159 }
 160
 161 sub mb_used {
 162     return $_[0]->{mb_used};
 163 }
 164
 165 # currently the same policy, but leaving it open for differences later.
 166 sub should_get_replicated_files {
 167     return $_[0]->should_get_new_files;
 168 }
 169
 170 sub not_on_hosts {
 171     my ($self, @hosts) = @_;
 172     my @hostids   = map { ref($_) ? $_->id : $_ } @hosts;
 173     my $my_hostid = $self->id;
 174     return (grep { $my_hostid == $_ } @hostids) ? 0 : 1;
 175 }
 176
 177 # "cached" by nature of the monitor worker testing this.
 178 sub doesnt_know_mkcol {
 179     return $_[0]->{no_mkcol};
 180 }
 181
 182 # Gross class-based singleton cache.
 183 my %dir_made;  # /dev<n>/path -> $time
 184 my $dir_made_lastclean = 0;
 185 # returns 1 on success, 0 on failure
 186 sub create_directory {
 187     my ($self, $uri) = @_;
 188     return 1 if $self->doesnt_know_mkcol || MogileFS::Config->server_setting_cached('skip_mkcol');
 189
 190     # rfc2518 says we "should" use a trailing slash. Some servers
 191     # (nginx) appears to require it.
 192     $uri .= '/' unless $uri =~ m/\/$/;
 193
 194     return 1 if $dir_made{$uri};
 195
 196     my $hostid = $self->hostid;
 197     my $host   = $self->host;
 198     my $hostip = $host->ip        or return 0;
 199     my $port   = $host->http_port or return 0;
 200     my $peer = "$hostip:$port";
 201
 202     my $sock = IO::Socket::INET->new(PeerAddr => $peer, Timeout => 1)
 203         or return 0;
 204
 205     print $sock "MKCOL $uri HTTP/1.0\r\n".
 206         "Content-Length: 0\r\n\r\n";
 207
 208     my $ans = <$sock>;
 209
 210     # if they don't support this method, remember that
 211     if ($ans && $ans =~ m!HTTP/1\.[01] (400|501)!) {
 212         $self->{no_mkcol} = 1;
 213         # TODO: move this into method in *monitor* worker
 214         return 1;
 215     }
 216
 217     return 0 unless $ans && $ans =~ m!^HTTP/1.[01] 2\d\d!;
 218
 219     my $now = time();
 220     $dir_made{$uri} = $now;
 221
 222     # cleanup %dir_made occasionally.
 223     my $clean_interval = 300;  # every 5 minutes.
 224     if ($dir_made_lastclean < $now - $clean_interval) {
 225         $dir_made_lastclean = $now;
 226         foreach my $k (keys %dir_made) {
 227             delete $dir_made{$k} if $dir_made{$k} < $now - 3600;
 228         }
 229     }
 230     return 1;
 231 }
 232
 233 sub fid_list {
 234     my ($self, %opts) = @_;
 235     my $limit = delete $opts{limit};
 236     croak("No limit specified") unless $limit && $limit =~ /^\d+$/;
 237     croak("Unknown options to fid_list") if %opts;
 238
 239     my $sto = Mgd::get_store();
 240     my $fidids = $sto->get_fidids_by_device($self->devid, $limit);
 241     return map {
 242         MogileFS::FID->new($_)
 243     } @{$fidids || []};
 244 }
 245
 246 sub fid_chunks {
 247     my ($self, %opts) = @_;
 248
 249     my $sto = Mgd::get_store();
 250     # storage function does validation.
 251     my $fidids = $sto->get_fidid_chunks_by_device(devid => $self->devid, %opts);
 252     return map {
 253         MogileFS::FID->new($_)
 254     } @{$fidids || []};
 255 }
 256
 257 sub forget_about {
 258     my ($self, $fid) = @_;
 259     Mgd::get_store()->remove_fidid_from_devid($fid->id, $self->id);
 260     return 1;
 261 }
 262
 263 sub usage_url {
 264     my $self = shift;
 265     my $host     = $self->host;
 266     my $get_port = $host->http_get_port;
 267     my $hostip   = $host->ip;
 268     return "http://$hostip:$get_port/dev$self->{devid}/usage";
 269 }
 270
 271 sub can_change_to_state {
 272     my ($self, $newstate) = @_;
 273     # don't allow dead -> alive transitions.  (yes, still possible
 274     # to go dead -> readonly -> alive to bypass this, but this is
 275     # all more of a user-education thing than an absolute policy)
 276     return 0 if $self->dstate->is_perm_dead && $newstate eq 'alive';
 277     return 1;
 278 }
 279
 280 sub vivify_directories {
 281     my ($self, $path) = @_;
 282
 283     # $path is something like:
 284     #    http://10.0.0.26:7500/dev2/0/000/148/0000148056.fid
 285
 286     # three directories we'll want to make:
 287     #    http://10.0.0.26:7500/dev2/0
 288     #    http://10.0.0.26:7500/dev2/0/000
 289     #    http://10.0.0.26:7500/dev2/0/000/148
 290
 291     croak "non-HTTP mode no longer supported" unless $path =~ /^http/;
 292     return 0 unless $path =~ m!/dev(\d+)/(\d+)/(\d\d\d)/(\d\d\d)/\d+\.fid$!;
 293     my ($devid, $p1, $p2, $p3) = ($1, $2, $3, $4);
 294
 295     die "devid mismatch" unless $self->id == $devid;
 296
 297     $self->create_directory("/dev$devid/$p1");
 298     $self->create_directory("/dev$devid/$p1/$p2");
 299     $self->create_directory("/dev$devid/$p1/$p2/$p3");
 300 }
 301
 302 # FIXME: Remove this once vestigial code is removed.
 303 sub set_observed_utilization {
 304     return 1;
 305 }
 306
 307 # Compatibility interface since this old routine is unfortunately called
 308 # internally within plugins. This data should be passed into any hooks which
 309 # may need it?
 310 # Currently an issue with MogileFS::Network + ZoneLocal
 311 # Remove this in 2012.
 312 sub devices {
 313     return Mgd::device_factory()->get_all;
 314 }
 315
 316 sub reject_bad_md5 {
 317     return $_[0]->{reject_bad_md5};
 318 }
 319
 320 1;