Fixed spacing in db/debug/diff/externalstore/objectcache folder
[mediawiki.git] / includes / filebackend / FileBackendStore.php
blobea379bc9113ca83953935c06e5ddae7372cbedfb
1 <?php
2 /**
3 * Base class for all backends using particular storage medium.
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, write to the Free Software Foundation, Inc.,
17 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 * http://www.gnu.org/copyleft/gpl.html
20 * @file
21 * @ingroup FileBackend
22 * @author Aaron Schulz
25 /**
26 * @brief Base class for all backends using particular storage medium.
28 * This class defines the methods as abstract that subclasses must implement.
29 * Outside callers should *not* use functions with "Internal" in the name.
31 * The FileBackend operations are implemented using basic functions
32 * such as storeInternal(), copyInternal(), deleteInternal() and the like.
33 * This class is also responsible for path resolution and sanitization.
35 * @ingroup FileBackend
36 * @since 1.19
38 abstract class FileBackendStore extends FileBackend {
39 /** @var BagOStuff */
40 protected $memCache;
41 /** @var ProcessCacheLRU */
42 protected $cheapCache; // Map of paths to small (RAM/disk) cache items
43 /** @var ProcessCacheLRU */
44 protected $expensiveCache; // Map of paths to large (RAM/disk) cache items
46 /** @var Array Map of container names to sharding settings */
47 protected $shardViaHashLevels = array(); // (container name => config array)
49 protected $maxFileSize = 4294967296; // integer bytes (4GiB)
51 const CACHE_TTL = 10; // integer; TTL in seconds for process cache entries
53 /**
54 * @see FileBackend::__construct()
56 * @param $config Array
58 public function __construct( array $config ) {
59 parent::__construct( $config );
60 $this->memCache = new EmptyBagOStuff(); // disabled by default
61 $this->cheapCache = new ProcessCacheLRU( 300 );
62 $this->expensiveCache = new ProcessCacheLRU( 5 );
65 /**
66 * Get the maximum allowable file size given backend
67 * medium restrictions and basic performance constraints.
68 * Do not call this function from places outside FileBackend and FileOp.
70 * @return integer Bytes
72 final public function maxFileSizeInternal() {
73 return $this->maxFileSize;
76 /**
77 * Check if a file can be created or changed at a given storage path.
78 * FS backends should check if the parent directory exists, files can be
79 * written under it, and that any file already there is writable.
80 * Backends using key/value stores should check if the container exists.
82 * @param $storagePath string
83 * @return bool
85 abstract public function isPathUsableInternal( $storagePath );
87 /**
88 * Create a file in the backend with the given contents.
89 * This will overwrite any file that exists at the destination.
90 * Do not call this function from places outside FileBackend and FileOp.
92 * $params include:
93 * - content : the raw file contents
94 * - dst : destination storage path
95 * - headers : HTTP header name/value map
96 * - async : Status will be returned immediately if supported.
97 * If the status is OK, then its value field will be
98 * set to a FileBackendStoreOpHandle object.
99 * - dstExists : Whether a file exists at the destination (optimization).
100 * Callers can use "false" if no existing file is being changed.
102 * @param array $params
103 * @return Status
105 final public function createInternal( array $params ) {
106 wfProfileIn( __METHOD__ );
107 wfProfileIn( __METHOD__ . '-' . $this->name );
108 if ( strlen( $params['content'] ) > $this->maxFileSizeInternal() ) {
109 $status = Status::newFatal( 'backend-fail-maxsize',
110 $params['dst'], $this->maxFileSizeInternal() );
111 } else {
112 $status = $this->doCreateInternal( $params );
113 $this->clearCache( array( $params['dst'] ) );
114 if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) {
115 $this->deleteFileCache( $params['dst'] ); // persistent cache
118 wfProfileOut( __METHOD__ . '-' . $this->name );
119 wfProfileOut( __METHOD__ );
120 return $status;
124 * @see FileBackendStore::createInternal()
125 * @return Status
127 abstract protected function doCreateInternal( array $params );
130 * Store a file into the backend from a file on disk.
131 * This will overwrite any file that exists at the destination.
132 * Do not call this function from places outside FileBackend and FileOp.
134 * $params include:
135 * - src : source path on disk
136 * - dst : destination storage path
137 * - headers : HTTP header name/value map
138 * - async : Status will be returned immediately if supported.
139 * If the status is OK, then its value field will be
140 * set to a FileBackendStoreOpHandle object.
141 * - dstExists : Whether a file exists at the destination (optimization).
142 * Callers can use "false" if no existing file is being changed.
144 * @param array $params
145 * @return Status
147 final public function storeInternal( array $params ) {
148 wfProfileIn( __METHOD__ );
149 wfProfileIn( __METHOD__ . '-' . $this->name );
150 if ( filesize( $params['src'] ) > $this->maxFileSizeInternal() ) {
151 $status = Status::newFatal( 'backend-fail-maxsize',
152 $params['dst'], $this->maxFileSizeInternal() );
153 } else {
154 $status = $this->doStoreInternal( $params );
155 $this->clearCache( array( $params['dst'] ) );
156 if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) {
157 $this->deleteFileCache( $params['dst'] ); // persistent cache
160 wfProfileOut( __METHOD__ . '-' . $this->name );
161 wfProfileOut( __METHOD__ );
162 return $status;
166 * @see FileBackendStore::storeInternal()
167 * @return Status
169 abstract protected function doStoreInternal( array $params );
172 * Copy a file from one storage path to another in the backend.
173 * This will overwrite any file that exists at the destination.
174 * Do not call this function from places outside FileBackend and FileOp.
176 * $params include:
177 * - src : source storage path
178 * - dst : destination storage path
179 * - ignoreMissingSource : do nothing if the source file does not exist
180 * - headers : HTTP header name/value map
181 * - async : Status will be returned immediately if supported.
182 * If the status is OK, then its value field will be
183 * set to a FileBackendStoreOpHandle object.
184 * - dstExists : Whether a file exists at the destination (optimization).
185 * Callers can use "false" if no existing file is being changed.
187 * @param array $params
188 * @return Status
190 final public function copyInternal( array $params ) {
191 wfProfileIn( __METHOD__ );
192 wfProfileIn( __METHOD__ . '-' . $this->name );
193 $status = $this->doCopyInternal( $params );
194 $this->clearCache( array( $params['dst'] ) );
195 if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) {
196 $this->deleteFileCache( $params['dst'] ); // persistent cache
198 wfProfileOut( __METHOD__ . '-' . $this->name );
199 wfProfileOut( __METHOD__ );
200 return $status;
204 * @see FileBackendStore::copyInternal()
205 * @return Status
207 abstract protected function doCopyInternal( array $params );
210 * Delete a file at the storage path.
211 * Do not call this function from places outside FileBackend and FileOp.
213 * $params include:
214 * - src : source storage path
215 * - ignoreMissingSource : do nothing if the source file does not exist
216 * - async : Status will be returned immediately if supported.
217 * If the status is OK, then its value field will be
218 * set to a FileBackendStoreOpHandle object.
220 * @param array $params
221 * @return Status
223 final public function deleteInternal( array $params ) {
224 wfProfileIn( __METHOD__ );
225 wfProfileIn( __METHOD__ . '-' . $this->name );
226 $status = $this->doDeleteInternal( $params );
227 $this->clearCache( array( $params['src'] ) );
228 $this->deleteFileCache( $params['src'] ); // persistent cache
229 wfProfileOut( __METHOD__ . '-' . $this->name );
230 wfProfileOut( __METHOD__ );
231 return $status;
235 * @see FileBackendStore::deleteInternal()
236 * @return Status
238 abstract protected function doDeleteInternal( array $params );
241 * Move a file from one storage path to another in the backend.
242 * This will overwrite any file that exists at the destination.
243 * Do not call this function from places outside FileBackend and FileOp.
245 * $params include:
246 * - src : source storage path
247 * - dst : destination storage path
248 * - ignoreMissingSource : do nothing if the source file does not exist
249 * - headers : HTTP header name/value map
250 * - async : Status will be returned immediately if supported.
251 * If the status is OK, then its value field will be
252 * set to a FileBackendStoreOpHandle object.
253 * - dstExists : Whether a file exists at the destination (optimization).
254 * Callers can use "false" if no existing file is being changed.
256 * @param array $params
257 * @return Status
259 final public function moveInternal( array $params ) {
260 wfProfileIn( __METHOD__ );
261 wfProfileIn( __METHOD__ . '-' . $this->name );
262 $status = $this->doMoveInternal( $params );
263 $this->clearCache( array( $params['src'], $params['dst'] ) );
264 $this->deleteFileCache( $params['src'] ); // persistent cache
265 if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) {
266 $this->deleteFileCache( $params['dst'] ); // persistent cache
268 wfProfileOut( __METHOD__ . '-' . $this->name );
269 wfProfileOut( __METHOD__ );
270 return $status;
274 * @see FileBackendStore::moveInternal()
275 * @return Status
277 protected function doMoveInternal( array $params ) {
278 unset( $params['async'] ); // two steps, won't work here :)
279 $nsrc = FileBackend::normalizeStoragePath( $params['src'] );
280 $ndst = FileBackend::normalizeStoragePath( $params['dst'] );
281 // Copy source to dest
282 $status = $this->copyInternal( $params );
283 if ( $nsrc !== $ndst && $status->isOK() ) {
284 // Delete source (only fails due to races or network problems)
285 $status->merge( $this->deleteInternal( array( 'src' => $params['src'] ) ) );
286 $status->setResult( true, $status->value ); // ignore delete() errors
288 return $status;
292 * Alter metadata for a file at the storage path.
293 * Do not call this function from places outside FileBackend and FileOp.
295 * $params include:
296 * - src : source storage path
297 * - headers : HTTP header name/value map
298 * - async : Status will be returned immediately if supported.
299 * If the status is OK, then its value field will be
300 * set to a FileBackendStoreOpHandle object.
302 * @param array $params
303 * @return Status
305 final public function describeInternal( array $params ) {
306 wfProfileIn( __METHOD__ );
307 wfProfileIn( __METHOD__ . '-' . $this->name );
308 if ( count( $params['headers'] ) ) {
309 $status = $this->doDescribeInternal( $params );
310 $this->clearCache( array( $params['src'] ) );
311 $this->deleteFileCache( $params['src'] ); // persistent cache
312 } else {
313 $status = Status::newGood(); // nothing to do
315 wfProfileOut( __METHOD__ . '-' . $this->name );
316 wfProfileOut( __METHOD__ );
317 return $status;
321 * @see FileBackendStore::describeInternal()
322 * @return Status
324 protected function doDescribeInternal( array $params ) {
325 return Status::newGood();
329 * No-op file operation that does nothing.
330 * Do not call this function from places outside FileBackend and FileOp.
332 * @param array $params
333 * @return Status
335 final public function nullInternal( array $params ) {
336 return Status::newGood();
340 * @see FileBackend::concatenate()
341 * @return Status
343 final public function concatenate( array $params ) {
344 wfProfileIn( __METHOD__ );
345 wfProfileIn( __METHOD__ . '-' . $this->name );
346 $status = Status::newGood();
348 // Try to lock the source files for the scope of this function
349 $scopeLockS = $this->getScopedFileLocks( $params['srcs'], LockManager::LOCK_UW, $status );
350 if ( $status->isOK() ) {
351 // Actually do the file concatenation...
352 $start_time = microtime( true );
353 $status->merge( $this->doConcatenate( $params ) );
354 $sec = microtime( true ) - $start_time;
355 if ( !$status->isOK() ) {
356 wfDebugLog( 'FileOperation', get_class( $this ) . " failed to concatenate " .
357 count( $params['srcs'] ) . " file(s) [$sec sec]" );
361 wfProfileOut( __METHOD__ . '-' . $this->name );
362 wfProfileOut( __METHOD__ );
363 return $status;
367 * @see FileBackendStore::concatenate()
368 * @return Status
370 protected function doConcatenate( array $params ) {
371 $status = Status::newGood();
372 $tmpPath = $params['dst']; // convenience
373 unset( $params['latest'] ); // sanity
375 // Check that the specified temp file is valid...
376 wfSuppressWarnings();
377 $ok = ( is_file( $tmpPath ) && filesize( $tmpPath ) == 0 );
378 wfRestoreWarnings();
379 if ( !$ok ) { // not present or not empty
380 $status->fatal( 'backend-fail-opentemp', $tmpPath );
381 return $status;
384 // Get local FS versions of the chunks needed for the concatenation...
385 $fsFiles = $this->getLocalReferenceMulti( $params );
386 foreach ( $fsFiles as $path => &$fsFile ) {
387 if ( !$fsFile ) { // chunk failed to download?
388 $fsFile = $this->getLocalReference( array( 'src' => $path ) );
389 if ( !$fsFile ) { // retry failed?
390 $status->fatal( 'backend-fail-read', $path );
391 return $status;
395 unset( $fsFile ); // unset reference so we can reuse $fsFile
397 // Get a handle for the destination temp file
398 $tmpHandle = fopen( $tmpPath, 'ab' );
399 if ( $tmpHandle === false ) {
400 $status->fatal( 'backend-fail-opentemp', $tmpPath );
401 return $status;
404 // Build up the temp file using the source chunks (in order)...
405 foreach ( $fsFiles as $virtualSource => $fsFile ) {
406 // Get a handle to the local FS version
407 $sourceHandle = fopen( $fsFile->getPath(), 'rb' );
408 if ( $sourceHandle === false ) {
409 fclose( $tmpHandle );
410 $status->fatal( 'backend-fail-read', $virtualSource );
411 return $status;
413 // Append chunk to file (pass chunk size to avoid magic quotes)
414 if ( !stream_copy_to_stream( $sourceHandle, $tmpHandle ) ) {
415 fclose( $sourceHandle );
416 fclose( $tmpHandle );
417 $status->fatal( 'backend-fail-writetemp', $tmpPath );
418 return $status;
420 fclose( $sourceHandle );
422 if ( !fclose( $tmpHandle ) ) {
423 $status->fatal( 'backend-fail-closetemp', $tmpPath );
424 return $status;
427 clearstatcache(); // temp file changed
429 return $status;
433 * @see FileBackend::doPrepare()
434 * @return Status
436 final protected function doPrepare( array $params ) {
437 wfProfileIn( __METHOD__ );
438 wfProfileIn( __METHOD__ . '-' . $this->name );
440 $status = Status::newGood();
441 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
442 if ( $dir === null ) {
443 $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
444 wfProfileOut( __METHOD__ . '-' . $this->name );
445 wfProfileOut( __METHOD__ );
446 return $status; // invalid storage path
449 if ( $shard !== null ) { // confined to a single container/shard
450 $status->merge( $this->doPrepareInternal( $fullCont, $dir, $params ) );
451 } else { // directory is on several shards
452 wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
453 list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
454 foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
455 $status->merge( $this->doPrepareInternal( "{$fullCont}{$suffix}", $dir, $params ) );
459 wfProfileOut( __METHOD__ . '-' . $this->name );
460 wfProfileOut( __METHOD__ );
461 return $status;
465 * @see FileBackendStore::doPrepare()
466 * @return Status
468 protected function doPrepareInternal( $container, $dir, array $params ) {
469 return Status::newGood();
473 * @see FileBackend::doSecure()
474 * @return Status
476 final protected function doSecure( array $params ) {
477 wfProfileIn( __METHOD__ );
478 wfProfileIn( __METHOD__ . '-' . $this->name );
479 $status = Status::newGood();
481 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
482 if ( $dir === null ) {
483 $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
484 wfProfileOut( __METHOD__ . '-' . $this->name );
485 wfProfileOut( __METHOD__ );
486 return $status; // invalid storage path
489 if ( $shard !== null ) { // confined to a single container/shard
490 $status->merge( $this->doSecureInternal( $fullCont, $dir, $params ) );
491 } else { // directory is on several shards
492 wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
493 list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
494 foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
495 $status->merge( $this->doSecureInternal( "{$fullCont}{$suffix}", $dir, $params ) );
499 wfProfileOut( __METHOD__ . '-' . $this->name );
500 wfProfileOut( __METHOD__ );
501 return $status;
505 * @see FileBackendStore::doSecure()
506 * @return Status
508 protected function doSecureInternal( $container, $dir, array $params ) {
509 return Status::newGood();
513 * @see FileBackend::doPublish()
514 * @return Status
516 final protected function doPublish( array $params ) {
517 wfProfileIn( __METHOD__ );
518 wfProfileIn( __METHOD__ . '-' . $this->name );
519 $status = Status::newGood();
521 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
522 if ( $dir === null ) {
523 $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
524 wfProfileOut( __METHOD__ . '-' . $this->name );
525 wfProfileOut( __METHOD__ );
526 return $status; // invalid storage path
529 if ( $shard !== null ) { // confined to a single container/shard
530 $status->merge( $this->doPublishInternal( $fullCont, $dir, $params ) );
531 } else { // directory is on several shards
532 wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
533 list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
534 foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
535 $status->merge( $this->doPublishInternal( "{$fullCont}{$suffix}", $dir, $params ) );
539 wfProfileOut( __METHOD__ . '-' . $this->name );
540 wfProfileOut( __METHOD__ );
541 return $status;
545 * @see FileBackendStore::doPublish()
546 * @return Status
548 protected function doPublishInternal( $container, $dir, array $params ) {
549 return Status::newGood();
553 * @see FileBackend::doClean()
554 * @return Status
556 final protected function doClean( array $params ) {
557 wfProfileIn( __METHOD__ );
558 wfProfileIn( __METHOD__ . '-' . $this->name );
559 $status = Status::newGood();
561 // Recursive: first delete all empty subdirs recursively
562 if ( !empty( $params['recursive'] ) && !$this->directoriesAreVirtual() ) {
563 $subDirsRel = $this->getTopDirectoryList( array( 'dir' => $params['dir'] ) );
564 if ( $subDirsRel !== null ) { // no errors
565 foreach ( $subDirsRel as $subDirRel ) {
566 $subDir = $params['dir'] . "/{$subDirRel}"; // full path
567 $status->merge( $this->doClean( array( 'dir' => $subDir ) + $params ) );
569 unset( $subDirsRel ); // free directory for rmdir() on Windows (for FS backends)
573 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
574 if ( $dir === null ) {
575 $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
576 wfProfileOut( __METHOD__ . '-' . $this->name );
577 wfProfileOut( __METHOD__ );
578 return $status; // invalid storage path
581 // Attempt to lock this directory...
582 $filesLockEx = array( $params['dir'] );
583 $scopedLockE = $this->getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status );
584 if ( !$status->isOK() ) {
585 wfProfileOut( __METHOD__ . '-' . $this->name );
586 wfProfileOut( __METHOD__ );
587 return $status; // abort
590 if ( $shard !== null ) { // confined to a single container/shard
591 $status->merge( $this->doCleanInternal( $fullCont, $dir, $params ) );
592 $this->deleteContainerCache( $fullCont ); // purge cache
593 } else { // directory is on several shards
594 wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
595 list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
596 foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
597 $status->merge( $this->doCleanInternal( "{$fullCont}{$suffix}", $dir, $params ) );
598 $this->deleteContainerCache( "{$fullCont}{$suffix}" ); // purge cache
602 wfProfileOut( __METHOD__ . '-' . $this->name );
603 wfProfileOut( __METHOD__ );
604 return $status;
608 * @see FileBackendStore::doClean()
609 * @return Status
611 protected function doCleanInternal( $container, $dir, array $params ) {
612 return Status::newGood();
616 * @see FileBackend::fileExists()
617 * @return bool|null
619 final public function fileExists( array $params ) {
620 wfProfileIn( __METHOD__ );
621 wfProfileIn( __METHOD__ . '-' . $this->name );
622 $stat = $this->getFileStat( $params );
623 wfProfileOut( __METHOD__ . '-' . $this->name );
624 wfProfileOut( __METHOD__ );
625 return ( $stat === null ) ? null : (bool)$stat; // null => failure
629 * @see FileBackend::getFileTimestamp()
630 * @return bool
632 final public function getFileTimestamp( array $params ) {
633 wfProfileIn( __METHOD__ );
634 wfProfileIn( __METHOD__ . '-' . $this->name );
635 $stat = $this->getFileStat( $params );
636 wfProfileOut( __METHOD__ . '-' . $this->name );
637 wfProfileOut( __METHOD__ );
638 return $stat ? $stat['mtime'] : false;
642 * @see FileBackend::getFileSize()
643 * @return bool
645 final public function getFileSize( array $params ) {
646 wfProfileIn( __METHOD__ );
647 wfProfileIn( __METHOD__ . '-' . $this->name );
648 $stat = $this->getFileStat( $params );
649 wfProfileOut( __METHOD__ . '-' . $this->name );
650 wfProfileOut( __METHOD__ );
651 return $stat ? $stat['size'] : false;
655 * @see FileBackend::getFileStat()
656 * @return bool
658 final public function getFileStat( array $params ) {
659 $path = self::normalizeStoragePath( $params['src'] );
660 if ( $path === null ) {
661 return false; // invalid storage path
663 wfProfileIn( __METHOD__ );
664 wfProfileIn( __METHOD__ . '-' . $this->name );
665 $latest = !empty( $params['latest'] ); // use latest data?
666 if ( !$this->cheapCache->has( $path, 'stat', self::CACHE_TTL ) ) {
667 $this->primeFileCache( array( $path ) ); // check persistent cache
669 if ( $this->cheapCache->has( $path, 'stat', self::CACHE_TTL ) ) {
670 $stat = $this->cheapCache->get( $path, 'stat' );
671 // If we want the latest data, check that this cached
672 // value was in fact fetched with the latest available data.
673 if ( is_array( $stat ) ) {
674 if ( !$latest || $stat['latest'] ) {
675 wfProfileOut( __METHOD__ . '-' . $this->name );
676 wfProfileOut( __METHOD__ );
677 return $stat;
679 } elseif ( in_array( $stat, array( 'NOT_EXIST', 'NOT_EXIST_LATEST' ) ) ) {
680 if ( !$latest || $stat === 'NOT_EXIST_LATEST' ) {
681 wfProfileOut( __METHOD__ . '-' . $this->name );
682 wfProfileOut( __METHOD__ );
683 return false;
687 wfProfileIn( __METHOD__ . '-miss' );
688 wfProfileIn( __METHOD__ . '-miss-' . $this->name );
689 $stat = $this->doGetFileStat( $params );
690 wfProfileOut( __METHOD__ . '-miss-' . $this->name );
691 wfProfileOut( __METHOD__ . '-miss' );
692 if ( is_array( $stat ) ) { // file exists
693 $stat['latest'] = $latest;
694 $this->cheapCache->set( $path, 'stat', $stat );
695 $this->setFileCache( $path, $stat ); // update persistent cache
696 if ( isset( $stat['sha1'] ) ) { // some backends store SHA-1 as metadata
697 $this->cheapCache->set( $path, 'sha1',
698 array( 'hash' => $stat['sha1'], 'latest' => $latest ) );
700 } elseif ( $stat === false ) { // file does not exist
701 $this->cheapCache->set( $path, 'stat', $latest ? 'NOT_EXIST_LATEST' : 'NOT_EXIST' );
702 wfDebug( __METHOD__ . ": File $path does not exist.\n" );
703 } else { // an error occurred
704 wfDebug( __METHOD__ . ": Could not stat file $path.\n" );
706 wfProfileOut( __METHOD__ . '-' . $this->name );
707 wfProfileOut( __METHOD__ );
708 return $stat;
712 * @see FileBackendStore::getFileStat()
714 abstract protected function doGetFileStat( array $params );
717 * @see FileBackend::getFileContentsMulti()
718 * @return Array
720 public function getFileContentsMulti( array $params ) {
721 wfProfileIn( __METHOD__ );
722 wfProfileIn( __METHOD__ . '-' . $this->name );
724 $params = $this->setConcurrencyFlags( $params );
725 $contents = $this->doGetFileContentsMulti( $params );
727 wfProfileOut( __METHOD__ . '-' . $this->name );
728 wfProfileOut( __METHOD__ );
729 return $contents;
733 * @see FileBackendStore::getFileContentsMulti()
734 * @return Array
736 protected function doGetFileContentsMulti( array $params ) {
737 $contents = array();
738 foreach ( $this->doGetLocalReferenceMulti( $params ) as $path => $fsFile ) {
739 wfSuppressWarnings();
740 $contents[$path] = $fsFile ? file_get_contents( $fsFile->getPath() ) : false;
741 wfRestoreWarnings();
743 return $contents;
747 * @see FileBackend::getFileSha1Base36()
748 * @return bool|string
750 final public function getFileSha1Base36( array $params ) {
751 $path = self::normalizeStoragePath( $params['src'] );
752 if ( $path === null ) {
753 return false; // invalid storage path
755 wfProfileIn( __METHOD__ );
756 wfProfileIn( __METHOD__ . '-' . $this->name );
757 $latest = !empty( $params['latest'] ); // use latest data?
758 if ( $this->cheapCache->has( $path, 'sha1', self::CACHE_TTL ) ) {
759 $stat = $this->cheapCache->get( $path, 'sha1' );
760 // If we want the latest data, check that this cached
761 // value was in fact fetched with the latest available data.
762 if ( !$latest || $stat['latest'] ) {
763 wfProfileOut( __METHOD__ . '-' . $this->name );
764 wfProfileOut( __METHOD__ );
765 return $stat['hash'];
768 wfProfileIn( __METHOD__ . '-miss' );
769 wfProfileIn( __METHOD__ . '-miss-' . $this->name );
770 $hash = $this->doGetFileSha1Base36( $params );
771 wfProfileOut( __METHOD__ . '-miss-' . $this->name );
772 wfProfileOut( __METHOD__ . '-miss' );
773 $this->cheapCache->set( $path, 'sha1', array( 'hash' => $hash, 'latest' => $latest ) );
774 wfProfileOut( __METHOD__ . '-' . $this->name );
775 wfProfileOut( __METHOD__ );
776 return $hash;
780 * @see FileBackendStore::getFileSha1Base36()
781 * @return bool|string
783 protected function doGetFileSha1Base36( array $params ) {
784 $fsFile = $this->getLocalReference( $params );
785 if ( !$fsFile ) {
786 return false;
787 } else {
788 return $fsFile->getSha1Base36();
793 * @see FileBackend::getFileProps()
794 * @return Array
796 final public function getFileProps( array $params ) {
797 wfProfileIn( __METHOD__ );
798 wfProfileIn( __METHOD__ . '-' . $this->name );
799 $fsFile = $this->getLocalReference( $params );
800 $props = $fsFile ? $fsFile->getProps() : FSFile::placeholderProps();
801 wfProfileOut( __METHOD__ . '-' . $this->name );
802 wfProfileOut( __METHOD__ );
803 return $props;
807 * @see FileBackend::getLocalReferenceMulti()
808 * @return Array
810 final public function getLocalReferenceMulti( array $params ) {
811 wfProfileIn( __METHOD__ );
812 wfProfileIn( __METHOD__ . '-' . $this->name );
814 $params = $this->setConcurrencyFlags( $params );
816 $fsFiles = array(); // (path => FSFile)
817 $latest = !empty( $params['latest'] ); // use latest data?
818 // Reuse any files already in process cache...
819 foreach ( $params['srcs'] as $src ) {
820 $path = self::normalizeStoragePath( $src );
821 if ( $path === null ) {
822 $fsFiles[$src] = null; // invalid storage path
823 } elseif ( $this->expensiveCache->has( $path, 'localRef' ) ) {
824 $val = $this->expensiveCache->get( $path, 'localRef' );
825 // If we want the latest data, check that this cached
826 // value was in fact fetched with the latest available data.
827 if ( !$latest || $val['latest'] ) {
828 $fsFiles[$src] = $val['object'];
832 // Fetch local references of any remaning files...
833 $params['srcs'] = array_diff( $params['srcs'], array_keys( $fsFiles ) );
834 foreach ( $this->doGetLocalReferenceMulti( $params ) as $path => $fsFile ) {
835 $fsFiles[$path] = $fsFile;
836 if ( $fsFile ) { // update the process cache...
837 $this->expensiveCache->set( $path, 'localRef',
838 array( 'object' => $fsFile, 'latest' => $latest ) );
842 wfProfileOut( __METHOD__ . '-' . $this->name );
843 wfProfileOut( __METHOD__ );
844 return $fsFiles;
848 * @see FileBackendStore::getLocalReferenceMulti()
849 * @return Array
851 protected function doGetLocalReferenceMulti( array $params ) {
852 return $this->doGetLocalCopyMulti( $params );
856 * @see FileBackend::getLocalCopyMulti()
857 * @return Array
859 final public function getLocalCopyMulti( array $params ) {
860 wfProfileIn( __METHOD__ );
861 wfProfileIn( __METHOD__ . '-' . $this->name );
863 $params = $this->setConcurrencyFlags( $params );
864 $tmpFiles = $this->doGetLocalCopyMulti( $params );
866 wfProfileOut( __METHOD__ . '-' . $this->name );
867 wfProfileOut( __METHOD__ );
868 return $tmpFiles;
872 * @see FileBackendStore::getLocalCopyMulti()
873 * @return Array
875 abstract protected function doGetLocalCopyMulti( array $params );
878 * @see FileBackend::getFileHttpUrl()
879 * @return string|null
881 public function getFileHttpUrl( array $params ) {
882 return null; // not supported
886 * @see FileBackend::streamFile()
887 * @return Status
889 final public function streamFile( array $params ) {
890 wfProfileIn( __METHOD__ );
891 wfProfileIn( __METHOD__ . '-' . $this->name );
892 $status = Status::newGood();
894 $info = $this->getFileStat( $params );
895 if ( !$info ) { // let StreamFile handle the 404
896 $status->fatal( 'backend-fail-notexists', $params['src'] );
899 // Set output buffer and HTTP headers for stream
900 $extraHeaders = isset( $params['headers'] ) ? $params['headers'] : array();
901 $res = StreamFile::prepareForStream( $params['src'], $info, $extraHeaders );
902 if ( $res == StreamFile::NOT_MODIFIED ) {
903 // do nothing; client cache is up to date
904 } elseif ( $res == StreamFile::READY_STREAM ) {
905 wfProfileIn( __METHOD__ . '-send' );
906 wfProfileIn( __METHOD__ . '-send-' . $this->name );
907 $status = $this->doStreamFile( $params );
908 wfProfileOut( __METHOD__ . '-send-' . $this->name );
909 wfProfileOut( __METHOD__ . '-send' );
910 if ( !$status->isOK() ) {
911 // Per bug 41113, nasty things can happen if bad cache entries get
912 // stuck in cache. It's also possible that this error can come up
913 // with simple race conditions. Clear out the stat cache to be safe.
914 $this->clearCache( array( $params['src'] ) );
915 $this->deleteFileCache( $params['src'] );
916 trigger_error( "Bad stat cache or race condition for file {$params['src']}." );
918 } else {
919 $status->fatal( 'backend-fail-stream', $params['src'] );
922 wfProfileOut( __METHOD__ . '-' . $this->name );
923 wfProfileOut( __METHOD__ );
924 return $status;
928 * @see FileBackendStore::streamFile()
929 * @return Status
931 protected function doStreamFile( array $params ) {
932 $status = Status::newGood();
934 $fsFile = $this->getLocalReference( $params );
935 if ( !$fsFile ) {
936 $status->fatal( 'backend-fail-stream', $params['src'] );
937 } elseif ( !readfile( $fsFile->getPath() ) ) {
938 $status->fatal( 'backend-fail-stream', $params['src'] );
941 return $status;
945 * @see FileBackend::directoryExists()
946 * @return bool|null
948 final public function directoryExists( array $params ) {
949 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
950 if ( $dir === null ) {
951 return false; // invalid storage path
953 if ( $shard !== null ) { // confined to a single container/shard
954 return $this->doDirectoryExists( $fullCont, $dir, $params );
955 } else { // directory is on several shards
956 wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
957 list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
958 $res = false; // response
959 foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
960 $exists = $this->doDirectoryExists( "{$fullCont}{$suffix}", $dir, $params );
961 if ( $exists ) {
962 $res = true;
963 break; // found one!
964 } elseif ( $exists === null ) { // error?
965 $res = null; // if we don't find anything, it is indeterminate
968 return $res;
973 * @see FileBackendStore::directoryExists()
975 * @param string $container Resolved container name
976 * @param string $dir Resolved path relative to container
977 * @param array $params
978 * @return bool|null
980 abstract protected function doDirectoryExists( $container, $dir, array $params );
983 * @see FileBackend::getDirectoryList()
984 * @return Traversable|Array|null Returns null on failure
986 final public function getDirectoryList( array $params ) {
987 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
988 if ( $dir === null ) { // invalid storage path
989 return null;
991 if ( $shard !== null ) {
992 // File listing is confined to a single container/shard
993 return $this->getDirectoryListInternal( $fullCont, $dir, $params );
994 } else {
995 wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
996 // File listing spans multiple containers/shards
997 list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
998 return new FileBackendStoreShardDirIterator( $this,
999 $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params );
1004 * Do not call this function from places outside FileBackend
1006 * @see FileBackendStore::getDirectoryList()
1008 * @param string $container Resolved container name
1009 * @param string $dir Resolved path relative to container
1010 * @param array $params
1011 * @return Traversable|Array|null Returns null on failure
1013 abstract public function getDirectoryListInternal( $container, $dir, array $params );
1016 * @see FileBackend::getFileList()
1017 * @return Traversable|Array|null Returns null on failure
1019 final public function getFileList( array $params ) {
1020 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
1021 if ( $dir === null ) { // invalid storage path
1022 return null;
1024 if ( $shard !== null ) {
1025 // File listing is confined to a single container/shard
1026 return $this->getFileListInternal( $fullCont, $dir, $params );
1027 } else {
1028 wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
1029 // File listing spans multiple containers/shards
1030 list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
1031 return new FileBackendStoreShardFileIterator( $this,
1032 $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params );
1037 * Do not call this function from places outside FileBackend
1039 * @see FileBackendStore::getFileList()
1041 * @param string $container Resolved container name
1042 * @param string $dir Resolved path relative to container
1043 * @param array $params
1044 * @return Traversable|Array|null Returns null on failure
1046 abstract public function getFileListInternal( $container, $dir, array $params );
1049 * Return a list of FileOp objects from a list of operations.
1050 * Do not call this function from places outside FileBackend.
1052 * The result must have the same number of items as the input.
1053 * An exception is thrown if an unsupported operation is requested.
1055 * @param array $ops Same format as doOperations()
1056 * @return Array List of FileOp objects
1057 * @throws MWException
1059 final public function getOperationsInternal( array $ops ) {
1060 $supportedOps = array(
1061 'store' => 'StoreFileOp',
1062 'copy' => 'CopyFileOp',
1063 'move' => 'MoveFileOp',
1064 'delete' => 'DeleteFileOp',
1065 'create' => 'CreateFileOp',
1066 'describe' => 'DescribeFileOp',
1067 'null' => 'NullFileOp'
1070 $performOps = array(); // array of FileOp objects
1071 // Build up ordered array of FileOps...
1072 foreach ( $ops as $operation ) {
1073 $opName = $operation['op'];
1074 if ( isset( $supportedOps[$opName] ) ) {
1075 $class = $supportedOps[$opName];
1076 // Get params for this operation
1077 $params = $operation;
1078 // Append the FileOp class
1079 $performOps[] = new $class( $this, $params );
1080 } else {
1081 throw new MWException( "Operation '$opName' is not supported." );
1085 return $performOps;
1089 * Get a list of storage paths to lock for a list of operations
1090 * Returns an array with 'sh' (shared) and 'ex' (exclusive) keys,
1091 * each corresponding to a list of storage paths to be locked.
1092 * All returned paths are normalized.
1094 * @param array $performOps List of FileOp objects
1095 * @return Array ('sh' => list of paths, 'ex' => list of paths)
1097 final public function getPathsToLockForOpsInternal( array $performOps ) {
1098 // Build up a list of files to lock...
1099 $paths = array( 'sh' => array(), 'ex' => array() );
1100 foreach ( $performOps as $fileOp ) {
1101 $paths['sh'] = array_merge( $paths['sh'], $fileOp->storagePathsRead() );
1102 $paths['ex'] = array_merge( $paths['ex'], $fileOp->storagePathsChanged() );
1104 // Optimization: if doing an EX lock anyway, don't also set an SH one
1105 $paths['sh'] = array_diff( $paths['sh'], $paths['ex'] );
1106 // Get a shared lock on the parent directory of each path changed
1107 $paths['sh'] = array_merge( $paths['sh'], array_map( 'dirname', $paths['ex'] ) );
1109 return $paths;
1113 * @see FileBackend::getScopedLocksForOps()
1114 * @return Array
1116 public function getScopedLocksForOps( array $ops, Status $status ) {
1117 $paths = $this->getPathsToLockForOpsInternal( $this->getOperationsInternal( $ops ) );
1118 return array(
1119 $this->getScopedFileLocks( $paths['sh'], LockManager::LOCK_UW, $status ),
1120 $this->getScopedFileLocks( $paths['ex'], LockManager::LOCK_EX, $status )
1125 * @see FileBackend::doOperationsInternal()
1126 * @return Status
1128 final protected function doOperationsInternal( array $ops, array $opts ) {
1129 wfProfileIn( __METHOD__ );
1130 wfProfileIn( __METHOD__ . '-' . $this->name );
1131 $status = Status::newGood();
1133 // Fix up custom header name/value pairs...
1134 $ops = array_map( array( $this, 'stripInvalidHeadersFromOp' ), $ops );
1136 // Build up a list of FileOps...
1137 $performOps = $this->getOperationsInternal( $ops );
1139 // Acquire any locks as needed...
1140 if ( empty( $opts['nonLocking'] ) ) {
1141 // Build up a list of files to lock...
1142 $paths = $this->getPathsToLockForOpsInternal( $performOps );
1143 // Try to lock those files for the scope of this function...
1144 $scopeLockS = $this->getScopedFileLocks( $paths['sh'], LockManager::LOCK_UW, $status );
1145 $scopeLockE = $this->getScopedFileLocks( $paths['ex'], LockManager::LOCK_EX, $status );
1146 if ( !$status->isOK() ) {
1147 wfProfileOut( __METHOD__ . '-' . $this->name );
1148 wfProfileOut( __METHOD__ );
1149 return $status; // abort
1153 // Clear any file cache entries (after locks acquired)
1154 if ( empty( $opts['preserveCache'] ) ) {
1155 $this->clearCache();
1158 // Load from the persistent file and container caches
1159 $this->primeFileCache( $performOps );
1160 $this->primeContainerCache( $performOps );
1162 // Actually attempt the operation batch...
1163 $opts = $this->setConcurrencyFlags( $opts );
1164 $subStatus = FileOpBatch::attempt( $performOps, $opts, $this->fileJournal );
1166 // Merge errors into status fields
1167 $status->merge( $subStatus );
1168 $status->success = $subStatus->success; // not done in merge()
1170 wfProfileOut( __METHOD__ . '-' . $this->name );
1171 wfProfileOut( __METHOD__ );
1172 return $status;
1176 * @see FileBackend::doQuickOperationsInternal()
1177 * @return Status
1178 * @throws MWException
1180 final protected function doQuickOperationsInternal( array $ops ) {
1181 wfProfileIn( __METHOD__ );
1182 wfProfileIn( __METHOD__ . '-' . $this->name );
1183 $status = Status::newGood();
1185 // Fix up custom header name/value pairs...
1186 $ops = array_map( array( $this, 'stripInvalidHeadersFromOp' ), $ops );
1188 // Clear any file cache entries
1189 $this->clearCache();
1191 $supportedOps = array( 'create', 'store', 'copy', 'move', 'delete', 'null' );
1192 $async = ( $this->parallelize === 'implicit' );
1193 $maxConcurrency = $this->concurrency; // throttle
1195 $statuses = array(); // array of (index => Status)
1196 $fileOpHandles = array(); // list of (index => handle) arrays
1197 $curFileOpHandles = array(); // current handle batch
1198 // Perform the sync-only ops and build up op handles for the async ops...
1199 foreach ( $ops as $index => $params ) {
1200 if ( !in_array( $params['op'], $supportedOps ) ) {
1201 wfProfileOut( __METHOD__ . '-' . $this->name );
1202 wfProfileOut( __METHOD__ );
1203 throw new MWException( "Operation '{$params['op']}' is not supported." );
1205 $method = $params['op'] . 'Internal'; // e.g. "storeInternal"
1206 $subStatus = $this->$method( array( 'async' => $async ) + $params );
1207 if ( $subStatus->value instanceof FileBackendStoreOpHandle ) { // async
1208 if ( count( $curFileOpHandles ) >= $maxConcurrency ) {
1209 $fileOpHandles[] = $curFileOpHandles; // push this batch
1210 $curFileOpHandles = array();
1212 $curFileOpHandles[$index] = $subStatus->value; // keep index
1213 } else { // error or completed
1214 $statuses[$index] = $subStatus; // keep index
1217 if ( count( $curFileOpHandles ) ) {
1218 $fileOpHandles[] = $curFileOpHandles; // last batch
1220 // Do all the async ops that can be done concurrently...
1221 foreach ( $fileOpHandles as $fileHandleBatch ) {
1222 $statuses = $statuses + $this->executeOpHandlesInternal( $fileHandleBatch );
1224 // Marshall and merge all the responses...
1225 foreach ( $statuses as $index => $subStatus ) {
1226 $status->merge( $subStatus );
1227 if ( $subStatus->isOK() ) {
1228 $status->success[$index] = true;
1229 ++$status->successCount;
1230 } else {
1231 $status->success[$index] = false;
1232 ++$status->failCount;
1236 wfProfileOut( __METHOD__ . '-' . $this->name );
1237 wfProfileOut( __METHOD__ );
1238 return $status;
1242 * Execute a list of FileBackendStoreOpHandle handles in parallel.
1243 * The resulting Status object fields will correspond
1244 * to the order in which the handles where given.
1246 * @param array $handles List of FileBackendStoreOpHandle objects
1247 * @return Array Map of Status objects
1248 * @throws MWException
1250 final public function executeOpHandlesInternal( array $fileOpHandles ) {
1251 wfProfileIn( __METHOD__ );
1252 wfProfileIn( __METHOD__ . '-' . $this->name );
1253 foreach ( $fileOpHandles as $fileOpHandle ) {
1254 if ( !( $fileOpHandle instanceof FileBackendStoreOpHandle ) ) {
1255 wfProfileOut( __METHOD__ . '-' . $this->name );
1256 wfProfileOut( __METHOD__ );
1257 throw new MWException( "Given a non-FileBackendStoreOpHandle object." );
1258 } elseif ( $fileOpHandle->backend->getName() !== $this->getName() ) {
1259 wfProfileOut( __METHOD__ . '-' . $this->name );
1260 wfProfileOut( __METHOD__ );
1261 throw new MWException( "Given a FileBackendStoreOpHandle for the wrong backend." );
1264 $res = $this->doExecuteOpHandlesInternal( $fileOpHandles );
1265 foreach ( $fileOpHandles as $fileOpHandle ) {
1266 $fileOpHandle->closeResources();
1268 wfProfileOut( __METHOD__ . '-' . $this->name );
1269 wfProfileOut( __METHOD__ );
1270 return $res;
1274 * @see FileBackendStore::executeOpHandlesInternal()
1275 * @param array $fileOpHandles
1276 * @throws MWException
1277 * @return Array List of corresponding Status objects
1279 protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
1280 foreach ( $fileOpHandles as $fileOpHandle ) { // OK if empty
1281 throw new MWException( "This backend supports no asynchronous operations." );
1283 return array();
1287 * Strip long HTTP headers from a file operation.
1288 * Most headers are just numbers, but some are allowed to be long.
1289 * This function is useful for cleaning up headers and avoiding backend
1290 * specific errors, especially in the middle of batch file operations.
1292 * @param array $op Same format as doOperation()
1293 * @return Array
1295 protected function stripInvalidHeadersFromOp( array $op ) {
1296 static $longs = array( 'Content-Disposition' );
1297 if ( isset( $op['headers'] ) ) { // op sets HTTP headers
1298 foreach ( $op['headers'] as $name => $value ) {
1299 $maxHVLen = in_array( $name, $longs ) ? INF : 255;
1300 if ( strlen( $name ) > 255 || strlen( $value ) > $maxHVLen ) {
1301 trigger_error( "Header '$name: $value' is too long." );
1302 unset( $op['headers'][$name] );
1303 } elseif ( !strlen( $value ) ) {
1304 $op['headers'][$name] = ''; // null/false => ""
1308 return $op;
1312 * @see FileBackend::preloadCache()
1314 final public function preloadCache( array $paths ) {
1315 $fullConts = array(); // full container names
1316 foreach ( $paths as $path ) {
1317 list( $fullCont, , ) = $this->resolveStoragePath( $path );
1318 $fullConts[] = $fullCont;
1320 // Load from the persistent file and container caches
1321 $this->primeContainerCache( $fullConts );
1322 $this->primeFileCache( $paths );
1326 * @see FileBackend::clearCache()
1328 final public function clearCache( array $paths = null ) {
1329 if ( is_array( $paths ) ) {
1330 $paths = array_map( 'FileBackend::normalizeStoragePath', $paths );
1331 $paths = array_filter( $paths, 'strlen' ); // remove nulls
1333 if ( $paths === null ) {
1334 $this->cheapCache->clear();
1335 $this->expensiveCache->clear();
1336 } else {
1337 foreach ( $paths as $path ) {
1338 $this->cheapCache->clear( $path );
1339 $this->expensiveCache->clear( $path );
1342 $this->doClearCache( $paths );
1346 * Clears any additional stat caches for storage paths
1348 * @see FileBackend::clearCache()
1350 * @param array $paths Storage paths (optional)
1351 * @return void
1353 protected function doClearCache( array $paths = null ) {}
1356 * Is this a key/value store where directories are just virtual?
1357 * Virtual directories exists in so much as files exists that are
1358 * prefixed with the directory path followed by a forward slash.
1360 * @return bool
1362 abstract protected function directoriesAreVirtual();
1365 * Check if a container name is valid.
1366 * This checks for for length and illegal characters.
1368 * @param $container string
1369 * @return bool
1371 final protected static function isValidContainerName( $container ) {
1372 // This accounts for Swift and S3 restrictions while leaving room
1373 // for things like '.xxx' (hex shard chars) or '.seg' (segments).
1374 // This disallows directory separators or traversal characters.
1375 // Note that matching strings URL encode to the same string;
1376 // in Swift, the length restriction is *after* URL encoding.
1377 return preg_match( '/^[a-z0-9][a-z0-9-_]{0,199}$/i', $container );
1381 * Splits a storage path into an internal container name,
1382 * an internal relative file name, and a container shard suffix.
1383 * Any shard suffix is already appended to the internal container name.
1384 * This also checks that the storage path is valid and within this backend.
1386 * If the container is sharded but a suffix could not be determined,
1387 * this means that the path can only refer to a directory and can only
1388 * be scanned by looking in all the container shards.
1390 * @param $storagePath string
1391 * @return Array (container, path, container suffix) or (null, null, null) if invalid
1393 final protected function resolveStoragePath( $storagePath ) {
1394 list( $backend, $container, $relPath ) = self::splitStoragePath( $storagePath );
1395 if ( $backend === $this->name ) { // must be for this backend
1396 $relPath = self::normalizeContainerPath( $relPath );
1397 if ( $relPath !== null ) {
1398 // Get shard for the normalized path if this container is sharded
1399 $cShard = $this->getContainerShard( $container, $relPath );
1400 // Validate and sanitize the relative path (backend-specific)
1401 $relPath = $this->resolveContainerPath( $container, $relPath );
1402 if ( $relPath !== null ) {
1403 // Prepend any wiki ID prefix to the container name
1404 $container = $this->fullContainerName( $container );
1405 if ( self::isValidContainerName( $container ) ) {
1406 // Validate and sanitize the container name (backend-specific)
1407 $container = $this->resolveContainerName( "{$container}{$cShard}" );
1408 if ( $container !== null ) {
1409 return array( $container, $relPath, $cShard );
1415 return array( null, null, null );
1419 * Like resolveStoragePath() except null values are returned if
1420 * the container is sharded and the shard could not be determined
1421 * or if the path ends with '/'. The later case is illegal for FS
1422 * backends and can confuse listings for object store backends.
1424 * This function is used when resolving paths that must be valid
1425 * locations for files. Directory and listing functions should
1426 * generally just use resolveStoragePath() instead.
1428 * @see FileBackendStore::resolveStoragePath()
1430 * @param $storagePath string
1431 * @return Array (container, path) or (null, null) if invalid
1433 final protected function resolveStoragePathReal( $storagePath ) {
1434 list( $container, $relPath, $cShard ) = $this->resolveStoragePath( $storagePath );
1435 if ( $cShard !== null && substr( $relPath, -1 ) !== '/' ) {
1436 return array( $container, $relPath );
1438 return array( null, null );
1442 * Get the container name shard suffix for a given path.
1443 * Any empty suffix means the container is not sharded.
1445 * @param string $container Container name
1446 * @param string $relPath Storage path relative to the container
1447 * @return string|null Returns null if shard could not be determined
1449 final protected function getContainerShard( $container, $relPath ) {
1450 list( $levels, $base, $repeat ) = $this->getContainerHashLevels( $container );
1451 if ( $levels == 1 || $levels == 2 ) {
1452 // Hash characters are either base 16 or 36
1453 $char = ( $base == 36 ) ? '[0-9a-z]' : '[0-9a-f]';
1454 // Get a regex that represents the shard portion of paths.
1455 // The concatenation of the captures gives us the shard.
1456 if ( $levels === 1 ) { // 16 or 36 shards per container
1457 $hashDirRegex = '(' . $char . ')';
1458 } else { // 256 or 1296 shards per container
1459 if ( $repeat ) { // verbose hash dir format (e.g. "a/ab/abc")
1460 $hashDirRegex = $char . '/(' . $char . '{2})';
1461 } else { // short hash dir format (e.g. "a/b/c")
1462 $hashDirRegex = '(' . $char . ')/(' . $char . ')';
1465 // Allow certain directories to be above the hash dirs so as
1466 // to work with FileRepo (e.g. "archive/a/ab" or "temp/a/ab").
1467 // They must be 2+ chars to avoid any hash directory ambiguity.
1468 $m = array();
1469 if ( preg_match( "!^(?:[^/]{2,}/)*$hashDirRegex(?:/|$)!", $relPath, $m ) ) {
1470 return '.' . implode( '', array_slice( $m, 1 ) );
1472 return null; // failed to match
1474 return ''; // no sharding
1478 * Check if a storage path maps to a single shard.
1479 * Container dirs like "a", where the container shards on "x/xy",
1480 * can reside on several shards. Such paths are tricky to handle.
1482 * @param string $storagePath Storage path
1483 * @return bool
1485 final public function isSingleShardPathInternal( $storagePath ) {
1486 list( , , $shard ) = $this->resolveStoragePath( $storagePath );
1487 return ( $shard !== null );
1491 * Get the sharding config for a container.
1492 * If greater than 0, then all file storage paths within
1493 * the container are required to be hashed accordingly.
1495 * @param $container string
1496 * @return Array (integer levels, integer base, repeat flag) or (0, 0, false)
1498 final protected function getContainerHashLevels( $container ) {
1499 if ( isset( $this->shardViaHashLevels[$container] ) ) {
1500 $config = $this->shardViaHashLevels[$container];
1501 $hashLevels = (int)$config['levels'];
1502 if ( $hashLevels == 1 || $hashLevels == 2 ) {
1503 $hashBase = (int)$config['base'];
1504 if ( $hashBase == 16 || $hashBase == 36 ) {
1505 return array( $hashLevels, $hashBase, $config['repeat'] );
1509 return array( 0, 0, false ); // no sharding
1513 * Get a list of full container shard suffixes for a container
1515 * @param $container string
1516 * @return Array
1518 final protected function getContainerSuffixes( $container ) {
1519 $shards = array();
1520 list( $digits, $base ) = $this->getContainerHashLevels( $container );
1521 if ( $digits > 0 ) {
1522 $numShards = pow( $base, $digits );
1523 for ( $index = 0; $index < $numShards; $index++ ) {
1524 $shards[] = '.' . wfBaseConvert( $index, 10, $base, $digits );
1527 return $shards;
1531 * Get the full container name, including the wiki ID prefix
1533 * @param $container string
1534 * @return string
1536 final protected function fullContainerName( $container ) {
1537 if ( $this->wikiId != '' ) {
1538 return "{$this->wikiId}-$container";
1539 } else {
1540 return $container;
1545 * Resolve a container name, checking if it's allowed by the backend.
1546 * This is intended for internal use, such as encoding illegal chars.
1547 * Subclasses can override this to be more restrictive.
1549 * @param $container string
1550 * @return string|null
1552 protected function resolveContainerName( $container ) {
1553 return $container;
1557 * Resolve a relative storage path, checking if it's allowed by the backend.
1558 * This is intended for internal use, such as encoding illegal chars or perhaps
1559 * getting absolute paths (e.g. FS based backends). Note that the relative path
1560 * may be the empty string (e.g. the path is simply to the container).
1562 * @param string $container Container name
1563 * @param string $relStoragePath Storage path relative to the container
1564 * @return string|null Path or null if not valid
1566 protected function resolveContainerPath( $container, $relStoragePath ) {
1567 return $relStoragePath;
1571 * Get the cache key for a container
1573 * @param string $container Resolved container name
1574 * @return string
1576 private function containerCacheKey( $container ) {
1577 return wfMemcKey( 'backend', $this->getName(), 'container', $container );
1581 * Set the cached info for a container
1583 * @param string $container Resolved container name
1584 * @param array $val Information to cache
1585 * @return void
1587 final protected function setContainerCache( $container, array $val ) {
1588 $this->memCache->add( $this->containerCacheKey( $container ), $val, 14 * 86400 );
1592 * Delete the cached info for a container.
1593 * The cache key is salted for a while to prevent race conditions.
1595 * @param string $container Resolved container name
1596 * @return void
1598 final protected function deleteContainerCache( $container ) {
1599 if ( !$this->memCache->set( $this->containerCacheKey( $container ), 'PURGED', 300 ) ) {
1600 trigger_error( "Unable to delete stat cache for container $container." );
1605 * Do a batch lookup from cache for container stats for all containers
1606 * used in a list of container names, storage paths, or FileOp objects.
1607 * This loads the persistent cache values into the process cache.
1609 * @param $items Array
1610 * @return void
1612 final protected function primeContainerCache( array $items ) {
1613 wfProfileIn( __METHOD__ );
1614 wfProfileIn( __METHOD__ . '-' . $this->name );
1616 $paths = array(); // list of storage paths
1617 $contNames = array(); // (cache key => resolved container name)
1618 // Get all the paths/containers from the items...
1619 foreach ( $items as $item ) {
1620 if ( $item instanceof FileOp ) {
1621 $paths = array_merge( $paths, $item->storagePathsRead() );
1622 $paths = array_merge( $paths, $item->storagePathsChanged() );
1623 } elseif ( self::isStoragePath( $item ) ) {
1624 $paths[] = $item;
1625 } elseif ( is_string( $item ) ) { // full container name
1626 $contNames[$this->containerCacheKey( $item )] = $item;
1629 // Get all the corresponding cache keys for paths...
1630 foreach ( $paths as $path ) {
1631 list( $fullCont, , ) = $this->resolveStoragePath( $path );
1632 if ( $fullCont !== null ) { // valid path for this backend
1633 $contNames[$this->containerCacheKey( $fullCont )] = $fullCont;
1637 $contInfo = array(); // (resolved container name => cache value)
1638 // Get all cache entries for these container cache keys...
1639 $values = $this->memCache->getMulti( array_keys( $contNames ) );
1640 foreach ( $values as $cacheKey => $val ) {
1641 $contInfo[$contNames[$cacheKey]] = $val;
1644 // Populate the container process cache for the backend...
1645 $this->doPrimeContainerCache( array_filter( $contInfo, 'is_array' ) );
1647 wfProfileOut( __METHOD__ . '-' . $this->name );
1648 wfProfileOut( __METHOD__ );
1652 * Fill the backend-specific process cache given an array of
1653 * resolved container names and their corresponding cached info.
1654 * Only containers that actually exist should appear in the map.
1656 * @param array $containerInfo Map of resolved container names to cached info
1657 * @return void
1659 protected function doPrimeContainerCache( array $containerInfo ) {}
1662 * Get the cache key for a file path
1664 * @param string $path Normalized storage path
1665 * @return string
1667 private function fileCacheKey( $path ) {
1668 return wfMemcKey( 'backend', $this->getName(), 'file', sha1( $path ) );
1672 * Set the cached stat info for a file path.
1673 * Negatives (404s) are not cached. By not caching negatives, we can skip cache
1674 * salting for the case when a file is created at a path were there was none before.
1676 * @param string $path Storage path
1677 * @param array $val Stat information to cache
1678 * @return void
1680 final protected function setFileCache( $path, array $val ) {
1681 $path = FileBackend::normalizeStoragePath( $path );
1682 if ( $path === null ) {
1683 return; // invalid storage path
1685 $age = time() - wfTimestamp( TS_UNIX, $val['mtime'] );
1686 $ttl = min( 7 * 86400, max( 300, floor( .1 * $age ) ) );
1687 $this->memCache->add( $this->fileCacheKey( $path ), $val, $ttl );
1691 * Delete the cached stat info for a file path.
1692 * The cache key is salted for a while to prevent race conditions.
1693 * Since negatives (404s) are not cached, this does not need to be called when
1694 * a file is created at a path were there was none before.
1696 * @param string $path Storage path
1697 * @return void
1699 final protected function deleteFileCache( $path ) {
1700 $path = FileBackend::normalizeStoragePath( $path );
1701 if ( $path === null ) {
1702 return; // invalid storage path
1704 if ( !$this->memCache->set( $this->fileCacheKey( $path ), 'PURGED', 300 ) ) {
1705 trigger_error( "Unable to delete stat cache for file $path." );
1710 * Do a batch lookup from cache for file stats for all paths
1711 * used in a list of storage paths or FileOp objects.
1712 * This loads the persistent cache values into the process cache.
1714 * @param array $items List of storage paths or FileOps
1715 * @return void
1717 final protected function primeFileCache( array $items ) {
1718 wfProfileIn( __METHOD__ );
1719 wfProfileIn( __METHOD__ . '-' . $this->name );
1721 $paths = array(); // list of storage paths
1722 $pathNames = array(); // (cache key => storage path)
1723 // Get all the paths/containers from the items...
1724 foreach ( $items as $item ) {
1725 if ( $item instanceof FileOp ) {
1726 $paths = array_merge( $paths, $item->storagePathsRead() );
1727 $paths = array_merge( $paths, $item->storagePathsChanged() );
1728 } elseif ( self::isStoragePath( $item ) ) {
1729 $paths[] = FileBackend::normalizeStoragePath( $item );
1732 // Get rid of any paths that failed normalization...
1733 $paths = array_filter( $paths, 'strlen' ); // remove nulls
1734 // Get all the corresponding cache keys for paths...
1735 foreach ( $paths as $path ) {
1736 list( , $rel, ) = $this->resolveStoragePath( $path );
1737 if ( $rel !== null ) { // valid path for this backend
1738 $pathNames[$this->fileCacheKey( $path )] = $path;
1741 // Get all cache entries for these container cache keys...
1742 $values = $this->memCache->getMulti( array_keys( $pathNames ) );
1743 foreach ( $values as $cacheKey => $val ) {
1744 if ( is_array( $val ) ) {
1745 $path = $pathNames[$cacheKey];
1746 $this->cheapCache->set( $path, 'stat', $val );
1747 if ( isset( $val['sha1'] ) ) { // some backends store SHA-1 as metadata
1748 $this->cheapCache->set( $path, 'sha1',
1749 array( 'hash' => $val['sha1'], 'latest' => $val['latest'] ) );
1754 wfProfileOut( __METHOD__ . '-' . $this->name );
1755 wfProfileOut( __METHOD__ );
1759 * Set the 'concurrency' option from a list of operation options
1761 * @param array $opts Map of operation options
1762 * @return Array
1764 final protected function setConcurrencyFlags( array $opts ) {
1765 $opts['concurrency'] = 1; // off
1766 if ( $this->parallelize === 'implicit' ) {
1767 if ( !isset( $opts['parallelize'] ) || $opts['parallelize'] ) {
1768 $opts['concurrency'] = $this->concurrency;
1770 } elseif ( $this->parallelize === 'explicit' ) {
1771 if ( !empty( $opts['parallelize'] ) ) {
1772 $opts['concurrency'] = $this->concurrency;
1775 return $opts;
1780 * FileBackendStore helper class for performing asynchronous file operations.
1782 * For example, calling FileBackendStore::createInternal() with the "async"
1783 * param flag may result in a Status that contains this object as a value.
1784 * This class is largely backend-specific and is mostly just "magic" to be
1785 * passed to FileBackendStore::executeOpHandlesInternal().
1787 abstract class FileBackendStoreOpHandle {
1788 /** @var Array */
1789 public $params = array(); // params to caller functions
1790 /** @var FileBackendStore */
1791 public $backend;
1792 /** @var Array */
1793 public $resourcesToClose = array();
1795 public $call; // string; name that identifies the function called
1798 * Close all open file handles
1800 * @return void
1802 public function closeResources() {
1803 array_map( 'fclose', $this->resourcesToClose );
1808 * FileBackendStore helper function to handle listings that span container shards.
1809 * Do not use this class from places outside of FileBackendStore.
1811 * @ingroup FileBackend
1813 abstract class FileBackendStoreShardListIterator extends FilterIterator {
1814 /** @var FileBackendStore */
1815 protected $backend;
1816 /** @var Array */
1817 protected $params;
1819 protected $container; // string; full container name
1820 protected $directory; // string; resolved relative path
1822 /** @var Array */
1823 protected $multiShardPaths = array(); // (rel path => 1)
1826 * @param $backend FileBackendStore
1827 * @param string $container Full storage container name
1828 * @param string $dir Storage directory relative to container
1829 * @param array $suffixes List of container shard suffixes
1830 * @param array $params
1832 public function __construct(
1833 FileBackendStore $backend, $container, $dir, array $suffixes, array $params
1835 $this->backend = $backend;
1836 $this->container = $container;
1837 $this->directory = $dir;
1838 $this->params = $params;
1840 $iter = new AppendIterator();
1841 foreach ( $suffixes as $suffix ) {
1842 $iter->append( $this->listFromShard( $this->container . $suffix ) );
1845 parent::__construct( $iter );
1848 public function accept() {
1849 $rel = $this->getInnerIterator()->current(); // path relative to given directory
1850 $path = $this->params['dir'] . "/{$rel}"; // full storage path
1851 if ( $this->backend->isSingleShardPathInternal( $path ) ) {
1852 return true; // path is only on one shard; no issue with duplicates
1853 } elseif ( isset( $this->multiShardPaths[$rel] ) ) {
1854 // Don't keep listing paths that are on multiple shards
1855 return false;
1856 } else {
1857 $this->multiShardPaths[$rel] = 1;
1858 return true;
1862 public function rewind() {
1863 parent::rewind();
1864 $this->multiShardPaths = array();
1868 * Get the list for a given container shard
1870 * @param string $container Resolved container name
1871 * @return Iterator
1873 abstract protected function listFromShard( $container );
1877 * Iterator for listing directories
1879 class FileBackendStoreShardDirIterator extends FileBackendStoreShardListIterator {
1880 protected function listFromShard( $container ) {
1881 $list = $this->backend->getDirectoryListInternal(
1882 $container, $this->directory, $this->params );
1883 if ( $list === null ) {
1884 return new ArrayIterator( array() );
1885 } else {
1886 return is_array( $list ) ? new ArrayIterator( $list ) : $list;
1892 * Iterator for listing regular files
1894 class FileBackendStoreShardFileIterator extends FileBackendStoreShardListIterator {
1895 protected function listFromShard( $container ) {
1896 $list = $this->backend->getFileListInternal(
1897 $container, $this->directory, $this->params );
1898 if ( $list === null ) {
1899 return new ArrayIterator( array() );
1900 } else {
1901 return is_array( $list ) ? new ArrayIterator( $list ) : $list;