3 * Helper class for representing operations with transaction support.
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, write to the Free Software Foundation, Inc.,
17 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 * http://www.gnu.org/copyleft/gpl.html
21 * @ingroup FileBackend
22 * @author Aaron Schulz
26 * FileBackend helper class for representing operations.
27 * Do not use this class from places outside FileBackend.
29 * Methods called from FileOpBatch::attempt() should avoid throwing
30 * exceptions at all costs. FileOp objects should be lightweight in order
31 * to support large arrays in memory and serialization.
33 * @ingroup FileBackend
36 abstract class FileOp
{
38 protected $params = [];
40 /** @var FileBackendStore */
44 protected $state = self
::STATE_NEW
;
47 protected $failed = false;
50 protected $async = false;
55 /** @var bool Operation is not a no-op */
56 protected $doOperation = true;
59 protected $sourceSha1;
62 protected $overwriteSameCase;
65 protected $destExists;
67 /* Object life-cycle */
69 const STATE_CHECKED
= 2;
70 const STATE_ATTEMPTED
= 3;
73 * Build a new batch file operation transaction
75 * @param FileBackendStore $backend
76 * @param array $params
77 * @throws FileBackendError
79 final public function __construct( FileBackendStore
$backend, array $params ) {
80 $this->backend
= $backend;
81 list( $required, $optional, $paths ) = $this->allowedParams();
82 foreach ( $required as $name ) {
83 if ( isset( $params[$name] ) ) {
84 $this->params
[$name] = $params[$name];
86 throw new FileBackendError( "File operation missing parameter '$name'." );
89 foreach ( $optional as $name ) {
90 if ( isset( $params[$name] ) ) {
91 $this->params
[$name] = $params[$name];
94 foreach ( $paths as $name ) {
95 if ( isset( $this->params
[$name] ) ) {
96 // Normalize paths so the paths to the same file have the same string
97 $this->params
[$name] = self
::normalizeIfValidStoragePath( $this->params
[$name] );
103 * Normalize a string if it is a valid storage path
105 * @param string $path
108 protected static function normalizeIfValidStoragePath( $path ) {
109 if ( FileBackend
::isStoragePath( $path ) ) {
110 $res = FileBackend
::normalizeStoragePath( $path );
112 return ( $res !== null ) ?
$res : $path;
119 * Set the batch UUID this operation belongs to
121 * @param string $batchId
123 final public function setBatchId( $batchId ) {
124 $this->batchId
= $batchId;
128 * Get the value of the parameter with the given name
130 * @param string $name
131 * @return mixed Returns null if the parameter is not set
133 final public function getParam( $name ) {
134 return isset( $this->params
[$name] ) ?
$this->params
[$name] : null;
138 * Check if this operation failed precheck() or attempt()
142 final public function failed() {
143 return $this->failed
;
147 * Get a new empty predicates array for precheck()
151 final public static function newPredicates() {
152 return [ 'exists' => [], 'sha1' => [] ];
156 * Get a new empty dependency tracking array for paths read/written to
160 final public static function newDependencies() {
161 return [ 'read' => [], 'write' => [] ];
165 * Update a dependency tracking array to account for this operation
167 * @param array $deps Prior path reads/writes; format of FileOp::newPredicates()
170 final public function applyDependencies( array $deps ) {
171 $deps['read'] +
= array_fill_keys( $this->storagePathsRead(), 1 );
172 $deps['write'] +
= array_fill_keys( $this->storagePathsChanged(), 1 );
178 * Check if this operation changes files listed in $paths
180 * @param array $deps Prior path reads/writes; format of FileOp::newPredicates()
183 final public function dependsOn( array $deps ) {
184 foreach ( $this->storagePathsChanged() as $path ) {
185 if ( isset( $deps['read'][$path] ) ||
isset( $deps['write'][$path] ) ) {
186 return true; // "output" or "anti" dependency
189 foreach ( $this->storagePathsRead() as $path ) {
190 if ( isset( $deps['write'][$path] ) ) {
191 return true; // "flow" dependency
199 * Get the file journal entries for this file operation
201 * @param array $oPredicates Pre-op info about files (format of FileOp::newPredicates)
202 * @param array $nPredicates Post-op info about files (format of FileOp::newPredicates)
205 final public function getJournalEntries( array $oPredicates, array $nPredicates ) {
206 if ( !$this->doOperation
) {
207 return []; // this is a no-op
212 $pathsUsed = array_merge( $this->storagePathsRead(), $this->storagePathsChanged() );
213 foreach ( array_unique( $pathsUsed ) as $path ) {
214 $nullEntries[] = [ // assertion for recovery
217 'newSha1' => $this->fileSha1( $path, $oPredicates )
220 foreach ( $this->storagePathsChanged() as $path ) {
221 if ( $nPredicates['sha1'][$path] === false ) { // deleted
227 } else { // created/updated
229 'op' => $this->fileExists( $path, $oPredicates ) ?
'update' : 'create',
231 'newSha1' => $nPredicates['sha1'][$path]
236 return array_merge( $nullEntries, $updateEntries, $deleteEntries );
240 * Check preconditions of the operation without writing anything.
241 * This must update $predicates for each path that the op can change
242 * except when a failing status object is returned.
244 * @param array $predicates
247 final public function precheck( array &$predicates ) {
248 if ( $this->state
!== self
::STATE_NEW
) {
249 return Status
::newFatal( 'fileop-fail-state', self
::STATE_NEW
, $this->state
);
251 $this->state
= self
::STATE_CHECKED
;
252 $status = $this->doPrecheck( $predicates );
253 if ( !$status->isOK() ) {
254 $this->failed
= true;
261 * @param array $predicates
264 protected function doPrecheck( array &$predicates ) {
265 return Status
::newGood();
269 * Attempt the operation
273 final public function attempt() {
274 if ( $this->state
!== self
::STATE_CHECKED
) {
275 return Status
::newFatal( 'fileop-fail-state', self
::STATE_CHECKED
, $this->state
);
276 } elseif ( $this->failed
) { // failed precheck
277 return Status
::newFatal( 'fileop-fail-attempt-precheck' );
279 $this->state
= self
::STATE_ATTEMPTED
;
280 if ( $this->doOperation
) {
281 $status = $this->doAttempt();
282 if ( !$status->isOK() ) {
283 $this->failed
= true;
284 $this->logFailure( 'attempt' );
287 $status = Status
::newGood();
296 protected function doAttempt() {
297 return Status
::newGood();
301 * Attempt the operation in the background
305 final public function attemptAsync() {
307 $result = $this->attempt();
308 $this->async
= false;
314 * Get the file operation parameters
316 * @return array (required params list, optional params list, list of params that are paths)
318 protected function allowedParams() {
319 return [ [], [], [] ];
323 * Adjust params to FileBackendStore internal file calls
325 * @param array $params
326 * @return array (required params list, optional params list)
328 protected function setFlags( array $params ) {
329 return [ 'async' => $this->async
] +
$params;
333 * Get a list of storage paths read from for this operation
337 public function storagePathsRead() {
342 * Get a list of storage paths written to for this operation
346 public function storagePathsChanged() {
351 * Check for errors with regards to the destination file already existing.
352 * Also set the destExists, overwriteSameCase and sourceSha1 member variables.
353 * A bad status will be returned if there is no chance it can be overwritten.
355 * @param array $predicates
358 protected function precheckDestExistence( array $predicates ) {
359 $status = Status
::newGood();
360 // Get hash of source file/string and the destination file
361 $this->sourceSha1
= $this->getSourceSha1Base36(); // FS file or data string
362 if ( $this->sourceSha1
=== null ) { // file in storage?
363 $this->sourceSha1
= $this->fileSha1( $this->params
['src'], $predicates );
365 $this->overwriteSameCase
= false;
366 $this->destExists
= $this->fileExists( $this->params
['dst'], $predicates );
367 if ( $this->destExists
) {
368 if ( $this->getParam( 'overwrite' ) ) {
369 return $status; // OK
370 } elseif ( $this->getParam( 'overwriteSame' ) ) {
371 $dhash = $this->fileSha1( $this->params
['dst'], $predicates );
372 // Check if hashes are valid and match each other...
373 if ( !strlen( $this->sourceSha1
) ||
!strlen( $dhash ) ) {
374 $status->fatal( 'backend-fail-hashes' );
375 } elseif ( $this->sourceSha1
!== $dhash ) {
376 // Give an error if the files are not identical
377 $status->fatal( 'backend-fail-notsame', $this->params
['dst'] );
379 $this->overwriteSameCase
= true; // OK
382 return $status; // do nothing; either OK or bad status
384 $status->fatal( 'backend-fail-alreadyexists', $this->params
['dst'] );
394 * precheckDestExistence() helper function to get the source file SHA-1.
395 * Subclasses should overwride this if the source is not in storage.
397 * @return string|bool Returns false on failure
399 protected function getSourceSha1Base36() {
404 * Check if a file will exist in storage when this operation is attempted
406 * @param string $source Storage path
407 * @param array $predicates
410 final protected function fileExists( $source, array $predicates ) {
411 if ( isset( $predicates['exists'][$source] ) ) {
412 return $predicates['exists'][$source]; // previous op assures this
414 $params = [ 'src' => $source, 'latest' => true ];
416 return $this->backend
->fileExists( $params );
421 * Get the SHA-1 of a file in storage when this operation is attempted
423 * @param string $source Storage path
424 * @param array $predicates
425 * @return string|bool False on failure
427 final protected function fileSha1( $source, array $predicates ) {
428 if ( isset( $predicates['sha1'][$source] ) ) {
429 return $predicates['sha1'][$source]; // previous op assures this
430 } elseif ( isset( $predicates['exists'][$source] ) && !$predicates['exists'][$source] ) {
431 return false; // previous op assures this
433 $params = [ 'src' => $source, 'latest' => true ];
435 return $this->backend
->getFileSha1Base36( $params );
440 * Get the backend this operation is for
442 * @return FileBackendStore
444 public function getBackend() {
445 return $this->backend
;
449 * Log a file operation failure and preserve any temp files
451 * @param string $action
453 final public function logFailure( $action ) {
454 $params = $this->params
;
455 $params['failedAction'] = $action;
457 wfDebugLog( 'FileOperation', get_class( $this ) .
458 " failed (batch #{$this->batchId}): " . FormatJson
::encode( $params ) );
459 } catch ( Exception
$e ) {
460 // bad config? debug log error?
466 * Create a file in the backend with the given content.
467 * Parameters for this operation are outlined in FileBackend::doOperations().
469 class CreateFileOp
extends FileOp
{
470 protected function allowedParams() {
472 [ 'content', 'dst' ],
473 [ 'overwrite', 'overwriteSame', 'headers' ],
478 protected function doPrecheck( array &$predicates ) {
479 $status = Status
::newGood();
480 // Check if the source data is too big
481 if ( strlen( $this->getParam( 'content' ) ) > $this->backend
->maxFileSizeInternal() ) {
482 $status->fatal( 'backend-fail-maxsize',
483 $this->params
['dst'], $this->backend
->maxFileSizeInternal() );
484 $status->fatal( 'backend-fail-create', $this->params
['dst'] );
487 // Check if a file can be placed/changed at the destination
488 } elseif ( !$this->backend
->isPathUsableInternal( $this->params
['dst'] ) ) {
489 $status->fatal( 'backend-fail-usable', $this->params
['dst'] );
490 $status->fatal( 'backend-fail-create', $this->params
['dst'] );
494 // Check if destination file exists
495 $status->merge( $this->precheckDestExistence( $predicates ) );
496 $this->params
['dstExists'] = $this->destExists
; // see FileBackendStore::setFileCache()
497 if ( $status->isOK() ) {
498 // Update file existence predicates
499 $predicates['exists'][$this->params
['dst']] = true;
500 $predicates['sha1'][$this->params
['dst']] = $this->sourceSha1
;
503 return $status; // safe to call attempt()
506 protected function doAttempt() {
507 if ( !$this->overwriteSameCase
) {
508 // Create the file at the destination
509 return $this->backend
->createInternal( $this->setFlags( $this->params
) );
512 return Status
::newGood();
515 protected function getSourceSha1Base36() {
516 return Wikimedia\base_convert
( sha1( $this->params
['content'] ), 16, 36, 31 );
519 public function storagePathsChanged() {
520 return [ $this->params
['dst'] ];
525 * Store a file into the backend from a file on the file system.
526 * Parameters for this operation are outlined in FileBackend::doOperations().
528 class StoreFileOp
extends FileOp
{
529 protected function allowedParams() {
532 [ 'overwrite', 'overwriteSame', 'headers' ],
537 protected function doPrecheck( array &$predicates ) {
538 $status = Status
::newGood();
539 // Check if the source file exists on the file system
540 if ( !is_file( $this->params
['src'] ) ) {
541 $status->fatal( 'backend-fail-notexists', $this->params
['src'] );
544 // Check if the source file is too big
545 } elseif ( filesize( $this->params
['src'] ) > $this->backend
->maxFileSizeInternal() ) {
546 $status->fatal( 'backend-fail-maxsize',
547 $this->params
['dst'], $this->backend
->maxFileSizeInternal() );
548 $status->fatal( 'backend-fail-store', $this->params
['src'], $this->params
['dst'] );
551 // Check if a file can be placed/changed at the destination
552 } elseif ( !$this->backend
->isPathUsableInternal( $this->params
['dst'] ) ) {
553 $status->fatal( 'backend-fail-usable', $this->params
['dst'] );
554 $status->fatal( 'backend-fail-store', $this->params
['src'], $this->params
['dst'] );
558 // Check if destination file exists
559 $status->merge( $this->precheckDestExistence( $predicates ) );
560 $this->params
['dstExists'] = $this->destExists
; // see FileBackendStore::setFileCache()
561 if ( $status->isOK() ) {
562 // Update file existence predicates
563 $predicates['exists'][$this->params
['dst']] = true;
564 $predicates['sha1'][$this->params
['dst']] = $this->sourceSha1
;
567 return $status; // safe to call attempt()
570 protected function doAttempt() {
571 if ( !$this->overwriteSameCase
) {
572 // Store the file at the destination
573 return $this->backend
->storeInternal( $this->setFlags( $this->params
) );
576 return Status
::newGood();
579 protected function getSourceSha1Base36() {
580 MediaWiki\
suppressWarnings();
581 $hash = sha1_file( $this->params
['src'] );
582 MediaWiki\restoreWarnings
();
583 if ( $hash !== false ) {
584 $hash = Wikimedia\base_convert
( $hash, 16, 36, 31 );
590 public function storagePathsChanged() {
591 return [ $this->params
['dst'] ];
596 * Copy a file from one storage path to another in the backend.
597 * Parameters for this operation are outlined in FileBackend::doOperations().
599 class CopyFileOp
extends FileOp
{
600 protected function allowedParams() {
603 [ 'overwrite', 'overwriteSame', 'ignoreMissingSource', 'headers' ],
608 protected function doPrecheck( array &$predicates ) {
609 $status = Status
::newGood();
610 // Check if the source file exists
611 if ( !$this->fileExists( $this->params
['src'], $predicates ) ) {
612 if ( $this->getParam( 'ignoreMissingSource' ) ) {
613 $this->doOperation
= false; // no-op
614 // Update file existence predicates (cache 404s)
615 $predicates['exists'][$this->params
['src']] = false;
616 $predicates['sha1'][$this->params
['src']] = false;
618 return $status; // nothing to do
620 $status->fatal( 'backend-fail-notexists', $this->params
['src'] );
624 // Check if a file can be placed/changed at the destination
625 } elseif ( !$this->backend
->isPathUsableInternal( $this->params
['dst'] ) ) {
626 $status->fatal( 'backend-fail-usable', $this->params
['dst'] );
627 $status->fatal( 'backend-fail-copy', $this->params
['src'], $this->params
['dst'] );
631 // Check if destination file exists
632 $status->merge( $this->precheckDestExistence( $predicates ) );
633 $this->params
['dstExists'] = $this->destExists
; // see FileBackendStore::setFileCache()
634 if ( $status->isOK() ) {
635 // Update file existence predicates
636 $predicates['exists'][$this->params
['dst']] = true;
637 $predicates['sha1'][$this->params
['dst']] = $this->sourceSha1
;
640 return $status; // safe to call attempt()
643 protected function doAttempt() {
644 if ( $this->overwriteSameCase
) {
645 $status = Status
::newGood(); // nothing to do
646 } elseif ( $this->params
['src'] === $this->params
['dst'] ) {
647 // Just update the destination file headers
648 $headers = $this->getParam( 'headers' ) ?
: [];
649 $status = $this->backend
->describeInternal( $this->setFlags( [
650 'src' => $this->params
['dst'], 'headers' => $headers
653 // Copy the file to the destination
654 $status = $this->backend
->copyInternal( $this->setFlags( $this->params
) );
660 public function storagePathsRead() {
661 return [ $this->params
['src'] ];
664 public function storagePathsChanged() {
665 return [ $this->params
['dst'] ];
670 * Move a file from one storage path to another in the backend.
671 * Parameters for this operation are outlined in FileBackend::doOperations().
673 class MoveFileOp
extends FileOp
{
674 protected function allowedParams() {
677 [ 'overwrite', 'overwriteSame', 'ignoreMissingSource', 'headers' ],
682 protected function doPrecheck( array &$predicates ) {
683 $status = Status
::newGood();
684 // Check if the source file exists
685 if ( !$this->fileExists( $this->params
['src'], $predicates ) ) {
686 if ( $this->getParam( 'ignoreMissingSource' ) ) {
687 $this->doOperation
= false; // no-op
688 // Update file existence predicates (cache 404s)
689 $predicates['exists'][$this->params
['src']] = false;
690 $predicates['sha1'][$this->params
['src']] = false;
692 return $status; // nothing to do
694 $status->fatal( 'backend-fail-notexists', $this->params
['src'] );
698 // Check if a file can be placed/changed at the destination
699 } elseif ( !$this->backend
->isPathUsableInternal( $this->params
['dst'] ) ) {
700 $status->fatal( 'backend-fail-usable', $this->params
['dst'] );
701 $status->fatal( 'backend-fail-move', $this->params
['src'], $this->params
['dst'] );
705 // Check if destination file exists
706 $status->merge( $this->precheckDestExistence( $predicates ) );
707 $this->params
['dstExists'] = $this->destExists
; // see FileBackendStore::setFileCache()
708 if ( $status->isOK() ) {
709 // Update file existence predicates
710 $predicates['exists'][$this->params
['src']] = false;
711 $predicates['sha1'][$this->params
['src']] = false;
712 $predicates['exists'][$this->params
['dst']] = true;
713 $predicates['sha1'][$this->params
['dst']] = $this->sourceSha1
;
716 return $status; // safe to call attempt()
719 protected function doAttempt() {
720 if ( $this->overwriteSameCase
) {
721 if ( $this->params
['src'] === $this->params
['dst'] ) {
722 // Do nothing to the destination (which is also the source)
723 $status = Status
::newGood();
725 // Just delete the source as the destination file needs no changes
726 $status = $this->backend
->deleteInternal( $this->setFlags(
727 [ 'src' => $this->params
['src'] ]
730 } elseif ( $this->params
['src'] === $this->params
['dst'] ) {
731 // Just update the destination file headers
732 $headers = $this->getParam( 'headers' ) ?
: [];
733 $status = $this->backend
->describeInternal( $this->setFlags(
734 [ 'src' => $this->params
['dst'], 'headers' => $headers ]
737 // Move the file to the destination
738 $status = $this->backend
->moveInternal( $this->setFlags( $this->params
) );
744 public function storagePathsRead() {
745 return [ $this->params
['src'] ];
748 public function storagePathsChanged() {
749 return [ $this->params
['src'], $this->params
['dst'] ];
754 * Delete a file at the given storage path from the backend.
755 * Parameters for this operation are outlined in FileBackend::doOperations().
757 class DeleteFileOp
extends FileOp
{
758 protected function allowedParams() {
759 return [ [ 'src' ], [ 'ignoreMissingSource' ], [ 'src' ] ];
762 protected function doPrecheck( array &$predicates ) {
763 $status = Status
::newGood();
764 // Check if the source file exists
765 if ( !$this->fileExists( $this->params
['src'], $predicates ) ) {
766 if ( $this->getParam( 'ignoreMissingSource' ) ) {
767 $this->doOperation
= false; // no-op
768 // Update file existence predicates (cache 404s)
769 $predicates['exists'][$this->params
['src']] = false;
770 $predicates['sha1'][$this->params
['src']] = false;
772 return $status; // nothing to do
774 $status->fatal( 'backend-fail-notexists', $this->params
['src'] );
778 // Check if a file can be placed/changed at the source
779 } elseif ( !$this->backend
->isPathUsableInternal( $this->params
['src'] ) ) {
780 $status->fatal( 'backend-fail-usable', $this->params
['src'] );
781 $status->fatal( 'backend-fail-delete', $this->params
['src'] );
785 // Update file existence predicates
786 $predicates['exists'][$this->params
['src']] = false;
787 $predicates['sha1'][$this->params
['src']] = false;
789 return $status; // safe to call attempt()
792 protected function doAttempt() {
793 // Delete the source file
794 return $this->backend
->deleteInternal( $this->setFlags( $this->params
) );
797 public function storagePathsChanged() {
798 return [ $this->params
['src'] ];
803 * Change metadata for a file at the given storage path in the backend.
804 * Parameters for this operation are outlined in FileBackend::doOperations().
806 class DescribeFileOp
extends FileOp
{
807 protected function allowedParams() {
808 return [ [ 'src' ], [ 'headers' ], [ 'src' ] ];
811 protected function doPrecheck( array &$predicates ) {
812 $status = Status
::newGood();
813 // Check if the source file exists
814 if ( !$this->fileExists( $this->params
['src'], $predicates ) ) {
815 $status->fatal( 'backend-fail-notexists', $this->params
['src'] );
818 // Check if a file can be placed/changed at the source
819 } elseif ( !$this->backend
->isPathUsableInternal( $this->params
['src'] ) ) {
820 $status->fatal( 'backend-fail-usable', $this->params
['src'] );
821 $status->fatal( 'backend-fail-describe', $this->params
['src'] );
825 // Update file existence predicates
826 $predicates['exists'][$this->params
['src']] =
827 $this->fileExists( $this->params
['src'], $predicates );
828 $predicates['sha1'][$this->params
['src']] =
829 $this->fileSha1( $this->params
['src'], $predicates );
831 return $status; // safe to call attempt()
834 protected function doAttempt() {
835 // Update the source file's metadata
836 return $this->backend
->describeInternal( $this->setFlags( $this->params
) );
839 public function storagePathsChanged() {
840 return [ $this->params
['src'] ];
845 * Placeholder operation that has no params and does nothing
847 class NullFileOp
extends FileOp
{