2 package MogileFS
::Client
;
6 MogileFS::Client - Client library for the MogileFS distributed file system.
12 # create client object w/ server-configured namespace
14 $mogc = MogileFS::Client->new(domain => "foo.com::my_namespace",
15 hosts => ['10.0.0.2:7001', '10.0.0.3:7001']);
18 # mogile is a flat namespace. no paths.
19 $key = "image_of_userid:$userid";
20 # must be configured on server
21 $class = "user_images";
22 $fh = $mogc->new_file($key, $class);
27 die "Error writing file: " . $mogc->errcode . ": " . $mogc->errstr;
30 # Find the URLs that the file was replicated to.
31 # May change over time.
32 @urls = $mogc->get_paths($key);
39 This module is a client library for the MogileFS distributed file system. The class method 'new' creates a client object against a
40 particular mogilefs tracker and domain. This object may then be used to store and retrieve content easily from MogileFS.
49 'domain', # scalar: the MogileFS domain (namespace).
50 'backend', # MogileFS::Backend object
51 'readonly', # bool: if set, client won't permit write actions/etc. just reads.
52 'hooks', # hash: hookname -> coderef
55 use MogileFS
::Backend
;
56 use MogileFS
::NewHTTPFile
;
57 use MogileFS
::ClientHTTPFile
;
59 our $VERSION = '1.16';
67 $client = MogileFS::Client->new( %OPTIONS );
69 Creates a new MogileFS::Client object.
71 Returns MogileFS::Client object on success, or dies on failure.
79 Arrayref of 'host:port' strings to connect to as backend trackers in this client.
83 String representing the mogile domain which this MogileFS client is associated with. (All create/delete/fetch operations
84 will be performed against this mogile domain). See the mogadm shell command and its 'domain' category of operations for
85 information on manipulating the list of possible domains on a MogileFS system.
92 my MogileFS
::Client
$self = shift;
93 $self = fields
::new
($self) unless ref $self;
95 return $self->_init(@_);
100 $mogc->reload( %OPTIONS )
102 Re-init the object, like you'd just reconstructed it with 'new', but change it in-place instead. Useful if you have a system which reloads a config file, and you want to update a singleton $mogc handle's config value.
107 my MogileFS
::Client
$self = shift;
108 return undef unless $self;
110 return $self->_init(@_);
114 my MogileFS
::Client
$self = shift;
118 # FIXME: add actual validation
120 # by default, set readonly off
121 $self->{readonly
} = $args{readonly
} ?
1 : 0;
123 # get domain (required)
124 $self->{domain
} = $args{domain
} or
125 _fail
("constructor requires parameter 'domain'");
127 # create a new backend object if there's not one already,
128 # otherwise call a reload on the existing one
129 if ($self->{backend
}) {
130 $self->{backend
}->reload( hosts
=> $args{hosts
} );
132 $self->{backend
} = MogileFS
::Backend
->new( hosts
=> $args{hosts
},
133 timeout
=> $args{timeout
},
136 _fail
("cannot instantiate MogileFS::Backend") unless $self->{backend
};
139 _debug
("MogileFS object: [$self]", $self);
146 Returns a scalar of form "ip:port", representing the last mogilefsd
147 'tracker' server which was talked to.
152 my MogileFS
::Client
$self = shift;
153 return $self->{backend
}->last_tracker;
158 Returns string representation of the last error that occurred. It
159 includes the error code (same as method 'errcode') and a space before
160 the optional English error message.
162 This isn't necessarily guaranteed to reset after a successful
163 operation. Only call it after another operation returns an error.
169 my MogileFS
::Client
$self = shift;
170 return $self->{backend
}->errstr;
175 Returns an error code. Not a number, but a string identifier
176 (e.g. "no_domain") which is stable for use in error handling logic.
178 This isn't necessarily guaranteed to reset after a successful
179 operation. Only call it after another operation returns an error.
184 my MogileFS
::Client
$self = shift;
185 return $self->{backend
}->errcode;
188 =head2 force_disconnect
190 Forces the client to disconnect from the tracker, causing it to reconnect
191 when the next request is made. It will reconnect to a different tracker if
192 possible. A paranoid application may wish to do to this before retrying a
193 failed command, on the off chance that another tracker may be working better.
197 sub force_disconnect
{
198 my MogileFS
::Client
$self = shift;
199 return $self->{backend
}->force_disconnect();
204 $is_readonly = $mogc->readonly
207 Getter/setter to mark this client object as read-only. Purely a local
208 operation/restriction, doesn't do a network operation to the mogilefsd
214 my MogileFS
::Client
$self = shift;
215 return $self->{readonly
} = $_[0] ?
1 : 0 if @_;
216 return $self->{readonly
};
221 $mogc->new_file($key)
222 $mogc->new_file($key, $class)
223 $mogc->new_file($key, $class, $content_length)
224 $mogc->new_file($key, $class, $content_length , $opts_hashref)
226 Start creating a new filehandle with the given key, and option given
229 Returns a filehandle you should then print to, and later close to
230 complete the operation. B<NOTE:> check the return value from close!
231 If your close didn't succeed, the file didn't get saved!
233 $opts_hashref can contain keys:
239 Explicitly specify the fid number to use, rather than it being automatically allocated.
241 =item create_open_args
243 Hashref of extra key/value pairs to send to mogilefsd in create_open phase.
245 =item create_close_args
247 Hashref of extra key/value pairs to send to mogilefsd in create_close phase.
251 Use MogileFS::ClientHTTPFile which will not load the entire file into memory
252 like the default MogileFS::NewHTTPFile but requires that the storage node
253 HTTP servers support the Content-Range header in PUT requests and is a little
260 # returns MogileFS::NewHTTPFile object, or undef if no device
261 # available for writing
262 # ARGS: ( key, class, bytes?, opts? )
263 # where bytes is optional and the length of the file and opts is also optional
264 # and is a hashref of options. supported options: fid = unique file id to use
265 # instead of just picking one in the database.
267 my MogileFS
::Client
$self = shift;
268 return undef if $self->{readonly
};
270 my ($key, $class, $bytes, $opts) = @_;
274 # Extra args to be passed along with the create_open and create_close commands.
275 # Any internally generated args of the same name will overwrite supplied ones in
277 my $create_open_args = $opts->{create_open_args
} || {};
278 my $create_close_args = $opts->{create_close_args
} || {};
280 $self->run_hook('new_file_start', $self, $key, $class, $opts);
282 my $res = $self->{backend
}->do_request
285 domain
=> $self->{domain
},
288 fid
=> $opts->{fid
} || 0, # fid should be specified, or pass 0 meaning to auto-generate one
292 my $dests = []; # [ [devid,path], [devid,path], ... ]
294 # determine old vs. new format to populate destinations
295 unless (exists $res->{dev_count
}) {
296 push @
$dests, [ $res->{devid
}, $res->{path
} ];
298 for my $i (1..$res->{dev_count
}) {
299 push @
$dests, [ $res->{"devid_$i"}, $res->{"path_$i"} ];
303 my $main_dest = shift @
$dests;
304 my ($main_devid, $main_path) = ($main_dest->[0], $main_dest->[1]);
306 # create a MogileFS::NewHTTPFile object, based off of IO::File
307 unless ($main_path =~ m!^http://!) {
308 Carp
::croak
("This version of MogileFS::Client no longer supports non-http storage URLs.\n");
311 $self->run_hook('new_file_end', $self, $key, $class, $opts);
313 return IO
::WrapTie
::wraptie
( ( $opts->{largefile
}
314 ?
'MogileFS::ClientHTTPFile'
315 : 'MogileFS::NewHTTPFile' ),
319 devid
=> $main_devid,
320 backup_dests
=> $dests,
323 content_length
=> $bytes+0,
324 create_close_args
=> $create_close_args,
331 $mogc->edit_file($key, $opts_hashref)
333 Edit the file with the the given key.
336 B<NOTE:> edit_file is currently EXPERIMENTAL and not recommended for
337 production use. MogileFS is primarily designed for storing files
338 for later retrieval, rather than editing. Use of this function may lead to
339 poor performance and, until it has been proven mature, should be
340 considered to also potentially cause data loss.
342 B<NOTE:> use of this function requires support for the DAV 'MOVE'
343 verb and partial PUT (i.e. Content-Range in PUT) on the back-end
344 storage servers (e.g. apache with mod_dav).
346 Returns a seekable filehandle you can read/write to. Calling this
347 function may invalidate some or all URLs you currently have for this
348 key, so you should call ->get_paths again afterwards if you need
351 On close of the filehandle, the new file contents will replace the
352 previous contents (and again invalidate any existing URLs).
354 By default, the file contents are preserved on open, but you may
355 specify the overwrite option to zero the file first. The seek position
356 is at the beginning of the file, but you may seek to the end to append.
358 $opts_hashref can contain keys:
364 The edit will overwrite the file, equivalent to opening with '>'.
372 my MogileFS
::Client
$self = shift;
373 return undef if $self->{readonly
};
375 my($key, $opts) = @_;
377 my $res = $self->{backend
}->do_request
379 domain
=> $self->{domain
},
383 my $moveReq = HTTP
::Request
->new('MOVE', $res->{oldpath
});
384 $moveReq->header(Destination
=> $res->{newpath
});
385 my $ua = LWP
::UserAgent
->new;
386 my $resp = $ua->request($moveReq);
387 unless ($resp->is_success) {
388 warn "Failed to MOVE $res->{oldpath} to $res->{newpath}";
392 return IO
::WrapTie
::wraptie
('MogileFS::ClientHTTPFile',
395 path
=> $res->{newpath
},
396 devid
=> $res->{devid
},
397 class => $res->{class},
399 overwrite
=> $opts->{overwrite
},
405 $mogc->read_file($key)
407 Read the file with the the given key.
409 Returns a seekable filehandle you can read() from. Note that you cannot
410 read line by line using <$fh> notation.
412 Takes the same options as get_paths (which is called internally to get
413 the URIs to read from).
418 my MogileFS
::Client
$self = shift;
420 my @paths = $self->get_paths(@_);
422 my $path = shift @paths;
426 my @backup_dests = map { [ undef, $_ ] } @paths;
428 return IO
::WrapTie
::wraptie
('MogileFS::ClientHTTPFile',
430 backup_dests
=> \
@backup_dests,
437 $mogc->store_file($key, $class, $fh_or_filename[, $opts_hashref])
439 Wrapper around new_file, print, and close.
441 Given a key, class, and a filehandle or filename, stores the file
442 contents in MogileFS. Returns the number of bytes stored on success,
445 $opts_hashref can contain keys for new_file, and also the following:
451 Number of bytes to read and write and write at once out of the larger file.
452 Defaults to 8192 bytes. Increasing this can increase performance at the cost
453 of more memory used while uploading the file.
454 Note that this mostly helps when using largefile => 1
461 my MogileFS
::Client
$self = shift;
462 return undef if $self->{readonly
};
464 my($key, $class, $file, $opts) = @_;
465 $self->run_hook('store_file_start', $self, $key, $class, $opts);
467 my $chunk_size = $opts->{chunk_size
} || 8192;
468 my $fh = $self->new_file($key, $class, undef, $opts) or return;
473 open $fh_from, $file or return;
476 while (my $len = read $fh_from, my($chunk), $chunk_size) {
481 $self->run_hook('store_file_end', $self, $key, $class, $opts);
483 close $fh_from unless ref $file;
484 $fh->close or return;
490 $mogc->store_content($key, $class, $content[, $opts]);
492 Wrapper around new_file, print, and close. Given a key, class, and
493 file contents (scalar or scalarref), stores the file contents in
494 MogileFS. Returns the number of bytes stored on success, undef on
500 my MogileFS
::Client
$self = shift;
501 return undef if $self->{readonly
};
503 my($key, $class, $content, $opts) = @_;
505 $self->run_hook('store_content_start', $self, $key, $class, $opts);
507 my $fh = $self->new_file($key, $class, undef, $opts) or return;
508 $content = ref($content) eq 'SCALAR' ?
$$content : $content;
509 $fh->print($content);
511 $self->run_hook('store_content_end', $self, $key, $class, $opts);
513 $fh->close or return;
519 @paths = $mogc->get_paths($key)
520 @paths = $mogc->get_paths($key, $no_verify_bool); # old way
521 @paths = $mogc->get_paths($key, { noverify => $bool }); # new way
523 Given a key, returns an array of all the locations (HTTP URLs) that
524 the file has been replicated to.
530 If the "no verify" option is set, the mogilefsd tracker doesn't verify
531 that the first item returned in the list is up/alive. Skipping that
532 check is faster, so use "noverify" if your application can do it
533 faster/smarter. For instance, when giving L<Perlbal> a list of URLs
534 to reproxy to, Perlbal can intelligently find one that's alive, so use
535 noverify and get out of mod_perl or whatever as soon as possible.
539 If the zone option is set to 'alt', the mogilefsd tracker will use the
540 alternative IP for each host if available, while constructing the paths.
544 If the pathcount option is set to a positive integer greater than 2, the
545 mogilefsd tracker will attempt to return that many different paths (if
546 available) to the same file. If not present or out of range, this value
554 # get_paths(key, noverify)
556 # get_paths(key, { noverify => 0/1, zone => "alt", pathcount => 2..N });
557 # but with both, second parameter is optional
559 # returns list of URLs that key can be found at, or the empty
560 # list on either error or no paths
562 my MogileFS
::Client
$self = shift;
563 my ($key, $opts) = @_;
565 # handle parameters, if any
566 my ($noverify, $zone);
568 $opts = { noverify
=> $opts };
572 $noverify = 1 if $opts->{noverify
};
573 $zone = $opts->{zone
};
575 if (my $pathcount = delete $opts->{pathcount
}) {
576 $extra_args{pathcount
} = $pathcount;
579 $self->run_hook('get_paths_start', $self, $key, $opts);
581 my $res = $self->{backend
}->do_request
583 domain
=> $self->{domain
},
585 noverify
=> $noverify ?
1 : 0,
590 my @paths = map { $res->{"path$_"} } (1..$res->{paths
});
592 $self->run_hook('get_paths_end', $self, $key, $opts);
599 $dataref = $mogc->get_file_data($key)
601 Wrapper around get_paths & LWP, which returns scalarref of file
602 contents in a scalarref.
604 Don't use for large data, as it all comes back to you in one string.
608 # given a key, returns a scalar reference pointing at a string containing
609 # the contents of the file. takes one parameter; a scalar key to get the
612 # given a key, load some paths and get data
613 my MogileFS
::Client
$self = $_[0];
614 my ($key, $timeout) = ($_[1], $_[2]);
616 my @paths = $self->get_paths($key, 1);
617 return undef unless @paths;
620 foreach my $path (@paths) {
621 next unless defined $path;
622 if ($path =~ m!^http://!) {
624 my $ua = new LWP
::UserAgent
;
625 $ua->timeout($timeout || 10);
627 my $res = $ua->get($path);
628 if ($res->is_success) {
629 my $contents = $res->content;
634 # open the file from disk and just grab it all
635 open FILE
, "<$path" or next;
637 { local $/ = undef; $contents = <FILE
>; }
639 return \
$contents if $contents;
649 Delete a key from MogileFS.
653 # this method returns undef only on a fatal error such as inability to actually
654 # delete a resource and inability to contact the server. attempting to delete
655 # something that doesn't exist counts as success, as it doesn't exist.
657 my MogileFS
::Client
$self = shift;
658 return undef if $self->{readonly
};
662 my $rv = $self->{backend
}->do_request
664 domain
=> $self->{domain
},
668 # if it's unknown_key, not an error
669 return undef unless defined $rv ||
670 $self->{backend
}->{lasterr
} eq 'unknown_key';
677 $mogc->rename($oldkey, $newkey);
679 Rename file (key) in MogileFS from oldkey to newkey. Returns true on
680 success, failure otherwise.
684 # this method renames a file. it returns an undef on error (only a fatal error
685 # is considered as undef; "file didn't exist" isn't an error).
687 my MogileFS
::Client
$self = shift;
688 return undef if $self->{readonly
};
690 my ($fkey, $tkey) = @_;
692 my $rv = $self->{backend
}->do_request
694 domain
=> $self->{domain
},
699 # if it's unknown_key, not an error
700 return undef unless defined $rv ||
701 $self->{backend
}->{lasterr
} eq 'unknown_key';
708 my $info_gob = $mogc->file_debug(fid => $fid);
710 my $info_gob = $mogc->file_debug(key => $key);
712 Thoroughly search for any database notes about a particular fid. Searchable by
713 raw fidid, or by domain and key. B<Use sparingly>. Command hits the master
714 database numerous times, and if you're using it in production something is
717 To be used with troubleshooting broken/odd files and errors from mogilefsd.
722 my MogileFS
::Client
$self = shift;
724 $opts{domain
} = $self->{domain
} unless exists $opts{domain
};
726 my $res = $self->{backend
}->do_request
735 my $fid = $mogc->file_info($key, { devices => 0 });
737 Used to return metadata about a file. Returns the domain, class, expected
738 length, devcount, etc. Optionally device ids (not paths) can be returned as
741 Should be used for informational purposes, and not usually for dynamically
747 my MogileFS
::Client
$self = shift;
748 my ($key, $opts) = @_;
751 $extra{devices
} = delete $opts->{devices
};
752 die "Unknown arguments: " . join(', ', keys %$opts) if keys %$opts;
754 my $res = $self->{backend
}->do_request
756 domain
=> $self->{domain
},
765 $keys = $mogc->list_keys($prefix, $after[, $limit]);
766 ($after, $keys) = $mogc->list_keys($prefix, $after[, $limit]);
768 Used to get a list of keys matching a certain prefix.
770 $prefix specifies what you want to get a list of. $after is the item
771 specified as a return value from this function last time you called
772 it. $limit is optional and defaults to 1000 keys returned.
774 In list context, returns ($after, $keys). In scalar context, returns
775 arrayref of keys. The value $after is to be used as $after when you
776 call this function again.
778 When there are no more keys in the list, you will get back undef or
784 my MogileFS
::Client
$self = shift;
785 my ($prefix, $after, $limit) = @_;
787 my $res = $self->{backend
}->do_request
789 domain
=> $self->{domain
},
795 # construct our list of keys and the new after value
796 my $resafter = $res->{next_after
};
798 for (my $i = 1; $i <= $res->{key_count
}+0; $i++) {
799 push @
$reslist, $res->{"key_$i"};
801 return wantarray ?
($resafter, $reslist) : $reslist;
806 $mogc->foreach_key( %OPTIONS, sub { my $key = shift; ... } );
807 $mogc->foreach_key( prefix => "foo:", sub { my $key = shift; ... } );
810 Functional interface/wrapper around list_keys.
812 Given some %OPTIONS (currently only one, "prefix"), calls your callback
813 for each key matching the provided prefix.
818 my MogileFS
::Client
$self = shift;
820 Carp
::croak
("Last parameter not a subref") unless ref $callback eq "CODE";
822 my $prefix = delete $opts{prefix
};
823 Carp
::croak
("Unknown option(s): " . join(", ", keys %opts)) if %opts;
829 while ($count == $max) {
830 my $res = $self->{backend
}->do_request
832 domain
=> $self->{domain
},
837 $count = $res->{key_count
}+0;
838 for (my $i = 1; $i <= $count; $i++) {
839 $callback->($res->{"key_$i"});
841 $last = $res->{"key_$count"};
846 # just makes some sleeping happen. first and only argument is number of
847 # seconds to instruct backend thread to sleep for.
849 my MogileFS
::Client
$self = shift;
850 my $duration = shift;
852 $self->{backend
}->do_request("sleep", { duration
=> $duration + 0 })
860 $mogc->update_class($key, $newclass);
862 Update the replication class of a pre-existing file, causing
863 the file to become more or less replicated.
868 my MogileFS
::Client
$self = shift;
869 my ($key, $class) = @_;
870 my $res = $self->{backend
}->do_request
872 domain
=> $self->{domain
},
881 $mogc->set_pref_ip({ "10.0.0.2" => "10.2.0.2" });
883 Weird option for old, weird network architecture. Sets a mapping
884 table of preferred alternate IPs, if reachable. For instance, if
885 trying to connect to 10.0.0.2 in the above example, the module would
886 instead try to connect to 10.2.0.2 quickly first, then then fall back
887 to 10.0.0.2 if 10.2.0.2 wasn't reachable.
891 # expects as argument a hashref of "standard-ip" => "preferred-ip"
893 my MogileFS
::Client
$self = shift;
894 $self->{backend
}->set_pref_ip(shift)
898 =head1 PLUGIN METHODS
904 # used to support plugins that have modified the server, this builds things into
905 # an argument list and passes them back to the server
906 # TODO: there is no readonly protection here? does it matter? should we check
907 # with the server to see what methods they support? (and if they should be disallowed
908 # when the client is in readonly mode?)
910 # remove everything up to the last colon, so we only have the method name left
911 my $method = $AUTOLOAD;
914 return if $method eq 'DESTROY';
919 # create a method to pass this on back
921 my MogileFS
::Client
$self = shift;
922 # pre-assemble the arguments into a hashref
925 $args->{"arg" . ++$ct} = shift() while @_;
926 $args->{"argcount"} = $ct;
928 # now put in standard args
929 $args->{"domain"} = $self->{domain
};
931 # now call and return whatever we get back from the backend
932 return $self->{backend
}->do_request("plugin_$method", $args);
948 my MogileFS
::Client
$self = shift;
949 my $hookname = shift || return;
952 $self->{hooks
}->{$hookname} = shift;
954 delete $self->{hooks
}->{$hookname};
959 my MogileFS
::Client
$self = shift;
960 my $hookname = shift || return;
962 my $hook = $self->{hooks
}->{$hookname};
965 eval { $hook->(@_) };
967 warn "MogileFS::Client hook '$hookname' threw error: $@\n" if $@
;
970 =head2 add_backend_hook
976 sub add_backend_hook
{
977 my MogileFS
::Client
$self = shift;
978 my $backend = $self->{backend
};
980 $backend->add_hook(@_);
984 ################################################################################
985 # MogileFS class methods
989 croak
"MogileFS: $_[0]";
993 return 1 unless $MogileFS::DEBUG
;
999 eval "use Data::Dumper;";
1000 print STDERR
"$msg\n" . Dumper
($ref) . "\n";
1010 L<http://www.danga.com/mogilefs/>
1014 This module is Copyright 2003-2004 Brad Fitzpatrick,
1015 and copyright 2005-2007 Six Apart, Ltd.
1017 All rights reserved.
1019 You may distribute under the terms of either the GNU General Public
1020 License or the Artistic License, as specified in the Perl README file.
1024 This is free software. IT COMES WITHOUT WARRANTY OF ANY KIND.
1028 Brad Fitzpatrick <brad@danga.com>
1030 Brad Whitaker <whitaker@danga.com>
1032 Mark Smith <marksmith@danga.com>