includes/libs/filebackend/FileOpBatch.php

   1 <?php
   2 /**
   3  * Helper class for representing batch file operations.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  * @ingroup FileBackend
  22  */
  23
  24 namespace Wikimedia\FileBackend;
  25
  26 use StatusValue;
  27 use Wikimedia\FileBackend\FileOpHandle\FileBackendStoreOpHandle;
  28 use Wikimedia\FileBackend\FileOps\FileOp;
  29 use Wikimedia\FileBackend\FileOps\FileStatePredicates;
  30
  31 /**
  32  * Helper class for representing batch file operations.
  33  * Do not use this class from places outside FileBackend.
  34  *
  35  * Methods should avoid throwing exceptions at all costs.
  36  *
  37  * @ingroup FileBackend
  38  * @since 1.20
  39  */
  40 class FileOpBatch {
  41         /* Timeout related parameters */
  42         private const MAX_BATCH_SIZE = 1000; // integer
  43
  44         /**
  45          * Attempt to perform a series of file operations.
  46          * Callers are responsible for handling file locking.
  47          *
  48          * $opts is an array of options, including:
  49          *   - force        : Errors that would normally cause a rollback do not.
  50          *                    The remaining operations are still attempted if any fail.
  51          *   - concurrency  : Try to do this many operations in parallel when possible.
  52          *
  53          * The resulting StatusValue will be "OK" unless:
  54          *   - a) unexpected operation errors occurred (network partitions, disk full...)
  55          *   - b) predicted operation errors occurred and 'force' was not set
  56          *
  57          * @param FileOp[] $performOps List of FileOp operations
  58          * @param array $opts Batch operation options
  59          * @return StatusValue
  60          */
  61         public static function attempt( array $performOps, array $opts ) {
  62                 $status = StatusValue::newGood();
  63
  64                 $n = count( $performOps );
  65                 if ( $n > self::MAX_BATCH_SIZE ) {
  66                         $status->fatal( 'backend-fail-batchsize', $n, self::MAX_BATCH_SIZE );
  67
  68                         return $status;
  69                 }
  70
  71                 $ignoreErrors = !empty( $opts['force'] );
  72                 $maxConcurrency = $opts['concurrency'] ?? 1;
  73
  74                 $predicates = new FileStatePredicates(); // account for previous ops in prechecks
  75                 $curBatch = []; // concurrent FileOp sub-batch accumulation
  76                 $curBatchDeps = FileOp::newDependencies(); // paths used in FileOp sub-batch
  77                 $pPerformOps = []; // ordered list of concurrent FileOp sub-batches
  78                 $lastBackend = null; // last op backend name
  79                 // Do pre-checks for each operation; abort on failure...
  80                 foreach ( $performOps as $index => $fileOp ) {
  81                         $backendName = $fileOp->getBackend()->getName();
  82                         // Decide if this op can be done concurrently within this sub-batch
  83                         // or if a new concurrent sub-batch must be started after this one...
  84                         if ( $fileOp->dependsOn( $curBatchDeps )
  85                                 || count( $curBatch ) >= $maxConcurrency
  86                                 || ( $backendName !== $lastBackend && count( $curBatch ) )
  87                         ) {
  88                                 $pPerformOps[] = $curBatch; // push this batch
  89                                 $curBatch = []; // start a new sub-batch
  90                                 $curBatchDeps = FileOp::newDependencies();
  91                         }
  92                         $lastBackend = $backendName;
  93                         $curBatch[$index] = $fileOp; // keep index
  94                         // Update list of affected paths in this batch
  95                         $curBatchDeps = $fileOp->applyDependencies( $curBatchDeps );
  96                         // Simulate performing the operation...
  97                         $subStatus = $fileOp->precheck( $predicates ); // updates $predicates
  98                         $status->merge( $subStatus );
  99                         if ( !$subStatus->isOK() ) {
 100                                 // operation failed?
 101                                 $status->success[$index] = false;
 102                                 ++$status->failCount;
 103                                 if ( !$ignoreErrors ) {
 104                                         return $status; // abort
 105                                 }
 106                         }
 107                 }
 108                 // Push the last sub-batch
 109                 if ( count( $curBatch ) ) {
 110                         $pPerformOps[] = $curBatch;
 111                 }
 112
 113                 if ( $ignoreErrors ) { // treat precheck() fatals as mere warnings
 114                         $status->setResult( true, $status->value );
 115                 }
 116
 117                 // Attempt each operation (in parallel if allowed and possible)...
 118                 self::runParallelBatches( $pPerformOps, $status );
 119
 120                 return $status;
 121         }
 122
 123         /**
 124          * Attempt a list of file operations sub-batches in series.
 125          *
 126          * The operations *in* each sub-batch will be done in parallel.
 127          * The caller is responsible for making sure the operations
 128          * within any given sub-batch do not depend on each other.
 129          * This will abort remaining ops on failure.
 130          *
 131          * @param FileOp[][] $pPerformOps Batches of file ops (batches use original indexes)
 132          * @param StatusValue $status
 133          */
 134         protected static function runParallelBatches( array $pPerformOps, StatusValue $status ) {
 135                 $aborted = false; // set to true on unexpected errors
 136                 foreach ( $pPerformOps as $performOpsBatch ) {
 137                         if ( $aborted ) { // check batch op abort flag...
 138                                 // We can't continue (even with $ignoreErrors) as $predicates is wrong.
 139                                 // Log the remaining ops as failed for recovery...
 140                                 foreach ( $performOpsBatch as $i => $fileOp ) {
 141                                         $status->success[$i] = false;
 142                                         ++$status->failCount;
 143                                         $fileOp->logFailure( 'attempt_aborted' );
 144                                 }
 145                                 continue;
 146                         }
 147                         /** @var StatusValue[] $statuses */
 148                         $statuses = [];
 149                         $opHandles = [];
 150                         // Get the backend; all sub-batch ops belong to a single backend
 151                         /** @var FileBackendStore $backend */
 152                         $backend = reset( $performOpsBatch )->getBackend();
 153                         // Get the operation handles or actually do it if there is just one.
 154                         // If attemptAsync() returns a StatusValue, it was either due to an error
 155                         // or the backend does not support async ops and did it synchronously.
 156                         foreach ( $performOpsBatch as $i => $fileOp ) {
 157                                 if ( !isset( $status->success[$i] ) ) { // didn't already fail in precheck()
 158                                         // Parallel ops may be disabled in config due to missing dependencies,
 159                                         // (e.g. needing popen()). When they are, $performOpsBatch has size 1.
 160                                         $subStatus = ( count( $performOpsBatch ) > 1 )
 161                                                 ? $fileOp->attemptAsync()
 162                                                 : $fileOp->attempt();
 163                                         if ( $subStatus->value instanceof FileBackendStoreOpHandle ) {
 164                                                 $opHandles[$i] = $subStatus->value; // deferred
 165                                         } else {
 166                                                 $statuses[$i] = $subStatus; // done already
 167                                         }
 168                                 }
 169                         }
 170                         // Try to do all the operations concurrently...
 171                         $statuses += $backend->executeOpHandlesInternal( $opHandles );
 172                         // Marshall and merge all the responses (blocking)...
 173                         foreach ( $performOpsBatch as $i => $fileOp ) {
 174                                 if ( !isset( $status->success[$i] ) ) { // didn't already fail in precheck()
 175                                         $subStatus = $statuses[$i];
 176                                         $status->merge( $subStatus );
 177                                         if ( $subStatus->isOK() ) {
 178                                                 $status->success[$i] = true;
 179                                                 ++$status->successCount;
 180                                         } else {
 181                                                 $status->success[$i] = false;
 182                                                 ++$status->failCount;
 183                                                 $aborted = true; // set abort flag; we can't continue
 184                                         }
 185                                 }
 186                         }
 187                 }
 188         }
 189 }
 190
 191 /** @deprecated class alias since 1.43 */
 192 class_alias( FileOpBatch::class, 'FileOpBatch' );