lib/MogileFS/Device.pm

   1 package MogileFS::Device;
   2 use strict;
   3 use warnings;
   4 use Carp qw/croak/;
   5 use MogileFS::Util qw(throw);
   6 use MogileFS::Util qw(okay_args device_state error);
   7
   8 =head1
   9
  10 MogileFS::Device - device class
  11
  12 =cut
  13
  14 BEGIN {
  15     my $testing = $ENV{TESTING} ? 1 : 0;
  16     eval "sub TESTING () { $testing }";
  17 }
  18
  19 my @observed_fields = qw/observed_state utilization/;
  20 my @fields = (qw/hostid status weight mb_total mb_used mb_asof devid/,
  21     @observed_fields);
  22
  23 sub new_from_args {
  24     my ($class, $args, $host_factory) = @_;
  25     my $self = bless {
  26         host_factory => $host_factory,
  27         %{$args},
  28     }, $class;
  29
  30     # FIXME: No guarantee (as of now?) that hosts get loaded before devs.
  31     #$self->host || die "No host for $self->{devid} (host $self->{hostid})";
  32
  33     croak "invalid device observed state '$self->{observed_state}', valid: writeable, readable, unreachable"
  34         if $self->{observed_state} && $self->{observed_state} !~ /^(?:writeable|readable|unreachable)$/;
  35
  36     return $self;
  37 }
  38
  39 # Instance methods
  40
  41 sub id     { return $_[0]{devid} }
  42 sub devid  { return $_[0]{devid} }
  43 sub name   { return $_[0]{devid} }
  44 sub status { return $_[0]{status} }
  45 sub weight { return $_[0]{weight} }
  46 sub hostid { return $_[0]{hostid} }
  47
  48 sub host {
  49     my $self = shift;
  50     return $self->{host_factory}->get_by_id($self->{hostid});
  51 }
  52
  53 # returns 0 if not known, else [0,1]
  54 sub percent_free {
  55     my $self = shift;
  56     return 0 unless $self->{mb_total} && defined $self->{mb_used};
  57     return 1 - ($self->{mb_used} / $self->{mb_total});
  58 }
  59
  60 # returns undef if not known, else [0,1]
  61 sub percent_full {
  62     my $self = shift;
  63     return undef unless $self->{mb_total} && defined $self->{mb_used};
  64     return $self->{mb_used} / $self->{mb_total};
  65 }
  66
  67 # FIXME: $self->mb_free?
  68 sub fields {
  69     my $self = shift;
  70     my @tofetch = @_ ? @_ : @fields;
  71     my $ret = { (map { $_ => $self->{$_} } @tofetch),
  72         'mb_free' => $self->mb_free };
  73     return $ret;
  74 }
  75
  76 sub observed_fields {
  77     return $_[0]->fields(@observed_fields);
  78 }
  79
  80 sub observed_utilization {
  81     my $self = shift;
  82
  83     if (TESTING) {
  84         my $weight_varname = 'T_FAKE_IO_DEV' . $self->id;
  85         return $ENV{$weight_varname} if defined $ENV{$weight_varname};
  86     }
  87
  88     return $self->{utilization};
  89 }
  90
  91 sub observed_writeable {
  92     my $self = shift;
  93     return 0 unless $self->{observed_state} && $self->{observed_state} eq 'writeable';
  94     my $host = $self->host or return 0;
  95     return 0 unless $host->observed_reachable;
  96     return 1;
  97 }
  98
  99 sub observed_readable {
 100     my $self = shift;
 101     return $self->{observed_state} && $self->{observed_state} eq 'readable';
 102 }
 103
 104 sub observed_unreachable {
 105     my $self = shift;
 106     return $self->{observed_state} && $self->{observed_state} eq 'unreachable';
 107 }
 108
 109 # FIXME: This pattern is weird. Store the object on new?
 110 sub dstate {
 111     my $ds = device_state($_[0]->status);
 112     return $ds if $ds;
 113     error("dev$_[0]->{devid} has bogus status '$_[0]->{status}', pretending 'down'");
 114     return device_state("down");
 115 }
 116
 117 sub can_delete_from {
 118     return $_[0]->dstate->can_delete_from;
 119 }
 120
 121 sub can_read_from {
 122     return $_[0]->dstate->can_read_from;
 123 }
 124
 125 # FIXME: Is there a (unrelated to this code) bug where new files aren't tested
 126 # against the free space limit before being stored or replicated somewhere?
 127 sub should_get_new_files {
 128     my $self   = shift;
 129     my $dstate = $self->dstate;
 130
 131     return 0 unless $dstate->should_get_new_files;
 132     return 0 unless $self->observed_writeable;
 133     return 0 unless $self->host->should_get_new_files;
 134     # have enough disk space? (default: 100MB)
 135     my $min_free = MogileFS->config("min_free_space");
 136     return 0 if $self->{mb_total} &&
 137         $self->mb_free < $min_free;
 138
 139     return 1;
 140 }
 141
 142 sub mb_free {
 143     my $self = shift;
 144     return $self->{mb_total} - $self->{mb_used}
 145         if $self->{mb_total} && $self->{mb_used};
 146 }
 147
 148 sub mb_used {
 149     return $_[0]->{mb_used};
 150 }
 151
 152 # currently the same policy, but leaving it open for differences later.
 153 sub should_get_replicated_files {
 154     return $_[0]->should_get_new_files;
 155 }
 156
 157 sub not_on_hosts {
 158     my ($self, @hosts) = @_;
 159     my @hostids   = map { ref($_) ? $_->id : $_ } @hosts;
 160     my $my_hostid = $self->id;
 161     return (grep { $my_hostid == $_ } @hostids) ? 0 : 1;
 162 }
 163
 164 # "cached" by nature of the monitor worker testing this.
 165 sub doesnt_know_mkcol {
 166     return $_[0]->{no_mkcol};
 167 }
 168
 169 # Gross class-based singleton cache.
 170 my %dir_made;  # /dev<n>/path -> $time
 171 my $dir_made_lastclean = 0;
 172 # returns 1 on success, 0 on failure
 173 sub create_directory {
 174     my ($self, $uri) = @_;
 175     return 1 if $self->doesnt_know_mkcol;
 176
 177     # rfc2518 says we "should" use a trailing slash. Some servers
 178     # (nginx) appears to require it.
 179     $uri .= '/' unless $uri =~ m/\/$/;
 180
 181     return 1 if $dir_made{$uri};
 182
 183     my $hostid = $self->hostid;
 184     my $host   = $self->host;
 185     my $hostip = $host->ip        or return 0;
 186     my $port   = $host->http_port or return 0;
 187     my $peer = "$hostip:$port";
 188
 189     my $sock = IO::Socket::INET->new(PeerAddr => $peer, Timeout => 1)
 190         or return 0;
 191
 192     print $sock "MKCOL $uri HTTP/1.0\r\n".
 193         "Content-Length: 0\r\n\r\n";
 194
 195     my $ans = <$sock>;
 196
 197     # if they don't support this method, remember that
 198     if ($ans && $ans =~ m!HTTP/1\.[01] (400|501)!) {
 199         $self->{no_mkcol} = 1;
 200         # TODO: move this into method in *monitor* worker
 201         return 1;
 202     }
 203
 204     return 0 unless $ans && $ans =~ m!^HTTP/1.[01] 2\d\d!;
 205
 206     my $now = time();
 207     $dir_made{$uri} = $now;
 208
 209     # cleanup %dir_made occasionally.
 210     my $clean_interval = 300;  # every 5 minutes.
 211     if ($dir_made_lastclean < $now - $clean_interval) {
 212         $dir_made_lastclean = $now;
 213         foreach my $k (keys %dir_made) {
 214             delete $dir_made{$k} if $dir_made{$k} < $now - 3600;
 215         }
 216     }
 217     return 1;
 218 }
 219
 220 sub fid_list {
 221     my ($self, %opts) = @_;
 222     my $limit = delete $opts{limit};
 223     croak("No limit specified") unless $limit && $limit =~ /^\d+$/;
 224     croak("Unknown options to fid_list") if %opts;
 225
 226     my $sto = Mgd::get_store();
 227     my $fidids = $sto->get_fidids_by_device($self->devid, $limit);
 228     return map {
 229         MogileFS::FID->new($_)
 230     } @{$fidids || []};
 231 }
 232
 233 sub fid_chunks {
 234     my ($self, %opts) = @_;
 235
 236     my $sto = Mgd::get_store();
 237     # storage function does validation.
 238     my $fidids = $sto->get_fidid_chunks_by_device(devid => $self->devid, %opts);
 239     return map {
 240         MogileFS::FID->new($_)
 241     } @{$fidids || []};
 242 }
 243
 244 sub forget_about {
 245     my ($self, $fid) = @_;
 246     Mgd::get_store()->remove_fidid_from_devid($fid->id, $self->id);
 247     return 1;
 248 }
 249
 250 sub usage_url {
 251     my $self = shift;
 252     my $host     = $self->host;
 253     my $get_port = $host->http_get_port;
 254     my $hostip   = $host->ip;
 255     return "http://$hostip:$get_port/dev$self->{devid}/usage";
 256 }
 257
 258 sub can_change_to_state {
 259     my ($self, $newstate) = @_;
 260     # don't allow dead -> alive transitions.  (yes, still possible
 261     # to go dead -> readonly -> alive to bypass this, but this is
 262     # all more of a user-education thing than an absolute policy)
 263     return 0 if $self->dstate->is_perm_dead && $newstate eq 'alive';
 264     return 1;
 265 }
 266
 267 sub vivify_directories {
 268     my ($self, $path) = @_;
 269
 270     # $path is something like:
 271     #    http://10.0.0.26:7500/dev2/0/000/148/0000148056.fid
 272
 273     # three directories we'll want to make:
 274     #    http://10.0.0.26:7500/dev2/0
 275     #    http://10.0.0.26:7500/dev2/0/000
 276     #    http://10.0.0.26:7500/dev2/0/000/148
 277
 278     croak "non-HTTP mode no longer supported" unless $path =~ /^http/;
 279     return 0 unless $path =~ m!/dev(\d+)/(\d+)/(\d\d\d)/(\d\d\d)/\d+\.fid$!;
 280     my ($devid, $p1, $p2, $p3) = ($1, $2, $3, $4);
 281
 282     die "devid mismatch" unless $self->id == $devid;
 283
 284     $self->create_directory("/dev$devid/$p1");
 285     $self->create_directory("/dev$devid/$p1/$p2");
 286     $self->create_directory("/dev$devid/$p1/$p2/$p3");
 287 }
 288
 289 # FIXME: Remove this once vestigial code is removed.
 290 sub set_observed_utilization {
 291     return 1;
 292 }
 293
 294 # Compatibility interface since this old routine is unfortunately called
 295 # internally within plugins. This data should be passed into any hooks which
 296 # may need it?
 297 # Currently an issue with MogileFS::Network + ZoneLocal
 298 # Remove this in 2012.
 299 sub devices {
 300     return Mgd::device_factory()->get_all;
 301 }
 302
 303 1;