3 * Proxy backend that mirrors writes to several internal backends.
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, write to the Free Software Foundation, Inc.,
17 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 * http://www.gnu.org/copyleft/gpl.html
21 * @ingroup FileBackend
22 * @author Aaron Schulz
26 * @brief Proxy backend that mirrors writes to several internal backends.
28 * This class defines a multi-write backend. Multiple backends can be
29 * registered to this proxy backend and it will act as a single backend.
30 * Use this when all access to those backends is through this proxy backend.
31 * At least one of the backends must be declared the "master" backend.
33 * Only use this class when transitioning from one storage system to another.
35 * Read operations are only done on the 'master' backend for consistency.
36 * Write operations are performed on all backends, in the order defined.
37 * If an operation fails on one backend it will be rolled back from the others.
39 * @ingroup FileBackend
42 class FileBackendMultiWrite
extends FileBackend
{
43 /** @var Array Prioritized list of FileBackendStore objects */
44 protected $backends = array(); // array of (backend index => backends)
45 protected $masterIndex = -1; // integer; index of master backend
46 protected $syncChecks = 0; // integer bitfield
48 /* Possible internal backend consistency checks */
53 * Construct a proxy backend that consists of several internal backends.
54 * Additional $config params include:
55 * 'backends' : Array of backend config and multi-backend settings.
56 * Each value is the config used in the constructor of a
57 * FileBackendStore class, but with these additional settings:
58 * 'class' : The name of the backend class
59 * 'isMultiMaster' : This must be set for one backend.
60 * 'template: : If given a backend name, this will use
61 * the config of that backend as a template.
62 * Values specified here take precedence.
63 * 'syncChecks' : Integer bitfield of internal backend sync checks to perform.
64 * Possible bits include self::CHECK_SIZE and self::CHECK_TIME.
65 * The checks are done before allowing any file operations.
66 * @param $config Array
68 public function __construct( array $config ) {
69 parent
::__construct( $config );
71 // Construct backends here rather than via registration
72 // to keep these backends hidden from outside the proxy.
73 foreach ( $config['backends'] as $index => $config ) {
74 if ( isset( $config['template'] ) ) {
75 // Config is just a modified version of a registered backend's.
76 // This should only be used when that config is used only be this backend.
77 $config = $config + FileBackendGroup
::singleton()->config( $config['template'] );
79 $name = $config['name'];
80 if ( isset( $namesUsed[$name] ) ) { // don't break FileOp predicates
81 throw new MWException( "Two or more backends defined with the name $name." );
83 $namesUsed[$name] = 1;
84 if ( !isset( $config['class'] ) ) {
85 throw new MWException( 'No class given for a backend config.' );
87 $class = $config['class'];
88 $this->backends
[$index] = new $class( $config );
89 if ( !empty( $config['isMultiMaster'] ) ) {
90 if ( $this->masterIndex
>= 0 ) {
91 throw new MWException( 'More than one master backend defined.' );
93 $this->masterIndex
= $index;
96 if ( $this->masterIndex
< 0 ) { // need backends and must have a master
97 throw new MWException( 'No master backend defined.' );
99 $this->syncChecks
= isset( $config['syncChecks'] )
100 ?
$config['syncChecks']
105 * @see FileBackend::doOperationsInternal()
108 final protected function doOperationsInternal( array $ops, array $opts ) {
109 $status = Status
::newGood();
111 $performOps = array(); // list of FileOp objects
112 $paths = array(); // storage paths read from or written to
113 // Build up a list of FileOps. The list will have all the ops
114 // for one backend, then all the ops for the next, and so on.
115 // These batches of ops are all part of a continuous array.
116 // Also build up a list of files read/changed...
117 foreach ( $this->backends
as $index => $backend ) {
118 $backendOps = $this->substOpBatchPaths( $ops, $backend );
119 // Add on the operation batch for this backend
120 $performOps = array_merge( $performOps,
121 $backend->getOperationsInternal( $backendOps ) );
122 if ( $index == 0 ) { // first batch
123 // Get the files used for these operations. Each backend has a batch of
124 // the same operations, so we only need to get them from the first batch.
125 $paths = $backend->getPathsToLockForOpsInternal( $performOps );
126 // Get the paths under the proxy backend's name
127 $paths['sh'] = $this->unsubstPaths( $paths['sh'] );
128 $paths['ex'] = $this->unsubstPaths( $paths['ex'] );
132 // Try to lock those files for the scope of this function...
133 if ( empty( $opts['nonLocking'] ) ) {
134 // Try to lock those files for the scope of this function...
135 $scopeLockS = $this->getScopedFileLocks( $paths['sh'], LockManager
::LOCK_UW
, $status );
136 $scopeLockE = $this->getScopedFileLocks( $paths['ex'], LockManager
::LOCK_EX
, $status );
137 if ( !$status->isOK() ) {
138 return $status; // abort
142 // Clear any cache entries (after locks acquired)
145 // Do a consistency check to see if the backends agree
146 if ( count( $this->backends
) > 1 ) {
147 $status->merge( $this->consistencyCheck( array_merge( $paths['sh'], $paths['ex'] ) ) );
148 if ( !$status->isOK() ) {
149 return $status; // abort
153 // Actually attempt the operation batch...
154 $subStatus = FileOpBatch
::attempt( $performOps, $opts, $this->fileJournal
);
159 // Make 'success', 'successCount', and 'failCount' fields reflect
160 // the overall operation, rather than all the batches for each backend.
161 // Do this by only using success values from the master backend's batch.
162 $batchStart = $this->masterIndex
* count( $ops );
163 $batchEnd = $batchStart +
count( $ops ) - 1;
164 for ( $i = $batchStart; $i <= $batchEnd; $i++
) {
165 if ( !isset( $subStatus->success
[$i] ) ) {
166 break; // failed out before trying this op
167 } elseif ( $subStatus->success
[$i] ) {
172 $success[] = $subStatus->success
[$i];
174 $subStatus->success
= $success;
175 $subStatus->successCount
= $successCount;
176 $subStatus->failCount
= $failCount;
178 // Merge errors into status fields
179 $status->merge( $subStatus );
180 $status->success
= $subStatus->success
; // not done in merge()
186 * Check that a set of files are consistent across all internal backends
188 * @param $paths Array
191 public function consistencyCheck( array $paths ) {
192 $status = Status
::newGood();
193 if ( $this->syncChecks
== 0 ) {
194 return $status; // skip checks
197 $mBackend = $this->backends
[$this->masterIndex
];
198 foreach ( array_unique( $paths ) as $path ) {
199 $params = array( 'src' => $path, 'latest' => true );
200 // Stat the file on the 'master' backend
201 $mStat = $mBackend->getFileStat( $this->substOpPaths( $params, $mBackend ) );
202 // Check of all clone backends agree with the master...
203 foreach ( $this->backends
as $index => $cBackend ) {
204 if ( $index === $this->masterIndex
) {
207 $cStat = $cBackend->getFileStat( $this->substOpPaths( $params, $cBackend ) );
208 if ( $mStat ) { // file is in master
209 if ( !$cStat ) { // file should exist
210 $status->fatal( 'backend-fail-synced', $path );
213 if ( $this->syncChecks
& self
::CHECK_SIZE
) {
214 if ( $cStat['size'] != $mStat['size'] ) { // wrong size
215 $status->fatal( 'backend-fail-synced', $path );
219 if ( $this->syncChecks
& self
::CHECK_TIME
) {
220 $mTs = wfTimestamp( TS_UNIX
, $mStat['mtime'] );
221 $cTs = wfTimestamp( TS_UNIX
, $cStat['mtime'] );
222 if ( abs( $mTs - $cTs ) > 30 ) { // outdated file somewhere
223 $status->fatal( 'backend-fail-synced', $path );
227 } else { // file is not in master
228 if ( $cStat ) { // file should not exist
229 $status->fatal( 'backend-fail-synced', $path );
239 * Substitute the backend name in storage path parameters
240 * for a set of operations with that of a given internal backend.
242 * @param $ops Array List of file operation arrays
243 * @param $backend FileBackendStore
246 protected function substOpBatchPaths( array $ops, FileBackendStore
$backend ) {
247 $newOps = array(); // operations
248 foreach ( $ops as $op ) {
249 $newOp = $op; // operation
250 foreach ( array( 'src', 'srcs', 'dst', 'dir' ) as $par ) {
251 if ( isset( $newOp[$par] ) ) { // string or array
252 $newOp[$par] = $this->substPaths( $newOp[$par], $backend );
261 * Same as substOpBatchPaths() but for a single operation
263 * @param $ops array File operation array
264 * @param $backend FileBackendStore
267 protected function substOpPaths( array $ops, FileBackendStore
$backend ) {
268 $newOps = $this->substOpBatchPaths( array( $ops ), $backend );
273 * Substitute the backend of storage paths with an internal backend's name
275 * @param $paths Array|string List of paths or single string path
276 * @param $backend FileBackendStore
277 * @return Array|string
279 protected function substPaths( $paths, FileBackendStore
$backend ) {
281 '!^mwstore://' . preg_quote( $this->name
) . '/!',
282 StringUtils
::escapeRegexReplacement( "mwstore://{$backend->getName()}/" ),
283 $paths // string or array
288 * Substitute the backend of internal storage paths with the proxy backend's name
290 * @param $paths Array|string List of paths or single string path
291 * @return Array|string
293 protected function unsubstPaths( $paths ) {
295 '!^mwstore://([^/]+)!',
296 StringUtils
::escapeRegexReplacement( "mwstore://{$this->name}" ),
297 $paths // string or array
302 * @see FileBackend::doQuickOperationsInternal()
305 public function doQuickOperationsInternal( array $ops ) {
306 // Do the operations on the master backend; setting Status fields...
307 $realOps = $this->substOpBatchPaths( $ops, $this->backends
[$this->masterIndex
] );
308 $status = $this->backends
[$this->masterIndex
]->doQuickOperations( $realOps );
309 // Propagate the operations to the clone backends...
310 foreach ( $this->backends
as $index => $backend ) {
311 if ( $index !== $this->masterIndex
) { // not done already
312 $realOps = $this->substOpBatchPaths( $ops, $backend );
313 $status->merge( $backend->doQuickOperations( $realOps ) );
320 * @see FileBackend::doPrepare()
323 protected function doPrepare( array $params ) {
324 $status = Status
::newGood();
325 foreach ( $this->backends
as $backend ) {
326 $realParams = $this->substOpPaths( $params, $backend );
327 $status->merge( $backend->doPrepare( $realParams ) );
333 * @see FileBackend::doSecure()
334 * @param $params array
337 protected function doSecure( array $params ) {
338 $status = Status
::newGood();
339 foreach ( $this->backends
as $backend ) {
340 $realParams = $this->substOpPaths( $params, $backend );
341 $status->merge( $backend->doSecure( $realParams ) );
347 * @see FileBackend::doClean()
348 * @param $params array
351 protected function doClean( array $params ) {
352 $status = Status
::newGood();
353 foreach ( $this->backends
as $backend ) {
354 $realParams = $this->substOpPaths( $params, $backend );
355 $status->merge( $backend->doClean( $realParams ) );
361 * @see FileBackend::concatenate()
362 * @param $params array
365 public function concatenate( array $params ) {
366 // We are writing to an FS file, so we don't need to do this per-backend
367 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
368 return $this->backends
[$this->masterIndex
]->concatenate( $realParams );
372 * @see FileBackend::fileExists()
373 * @param $params array
375 public function fileExists( array $params ) {
376 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
377 return $this->backends
[$this->masterIndex
]->fileExists( $realParams );
381 * @see FileBackend::getFileTimestamp()
382 * @param $params array
383 * @return bool|string
385 public function getFileTimestamp( array $params ) {
386 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
387 return $this->backends
[$this->masterIndex
]->getFileTimestamp( $realParams );
391 * @see FileBackend::getFileSize()
392 * @param $params array
395 public function getFileSize( array $params ) {
396 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
397 return $this->backends
[$this->masterIndex
]->getFileSize( $realParams );
401 * @see FileBackend::getFileStat()
402 * @param $params array
403 * @return Array|bool|null
405 public function getFileStat( array $params ) {
406 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
407 return $this->backends
[$this->masterIndex
]->getFileStat( $realParams );
411 * @see FileBackend::getFileContents()
412 * @param $params array
413 * @return bool|string
415 public function getFileContents( array $params ) {
416 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
417 return $this->backends
[$this->masterIndex
]->getFileContents( $realParams );
421 * @see FileBackend::getFileSha1Base36()
422 * @param $params array
423 * @return bool|string
425 public function getFileSha1Base36( array $params ) {
426 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
427 return $this->backends
[$this->masterIndex
]->getFileSha1Base36( $realParams );
431 * @see FileBackend::getFileProps()
432 * @param $params array
435 public function getFileProps( array $params ) {
436 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
437 return $this->backends
[$this->masterIndex
]->getFileProps( $realParams );
441 * @see FileBackend::streamFile()
442 * @param $params array
445 public function streamFile( array $params ) {
446 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
447 return $this->backends
[$this->masterIndex
]->streamFile( $realParams );
451 * @see FileBackend::getLocalReference()
452 * @param $params array
453 * @return FSFile|null
455 public function getLocalReference( array $params ) {
456 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
457 return $this->backends
[$this->masterIndex
]->getLocalReference( $realParams );
461 * @see FileBackend::getLocalCopy()
462 * @param $params array
463 * @return null|TempFSFile
465 public function getLocalCopy( array $params ) {
466 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
467 return $this->backends
[$this->masterIndex
]->getLocalCopy( $realParams );
471 * @see FileBackend::directoryExists()
472 * @param $params array
475 public function directoryExists( array $params ) {
476 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
477 return $this->backends
[$this->masterIndex
]->directoryExists( $realParams );
481 * @see FileBackend::getSubdirectoryList()
482 * @param $params array
483 * @return Array|null|Traversable
485 public function getDirectoryList( array $params ) {
486 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
487 return $this->backends
[$this->masterIndex
]->getDirectoryList( $realParams );
491 * @see FileBackend::getFileList()
492 * @param $params array
493 * @return Array|null|\Traversable
495 public function getFileList( array $params ) {
496 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
497 return $this->backends
[$this->masterIndex
]->getFileList( $realParams );
501 * @see FileBackend::clearCache()
503 public function clearCache( array $paths = null ) {
504 foreach ( $this->backends
as $backend ) {
505 $realPaths = is_array( $paths ) ?
$this->substPaths( $paths, $backend ) : null;
506 $backend->clearCache( $realPaths );
511 * @see FileBackend::getScopedLocksForOps()
513 public function getScopedLocksForOps( array $ops, Status
$status ) {
514 $fileOps = $this->backends
[$this->masterIndex
]->getOperationsInternal( $ops );
515 // Get the paths to lock from the master backend
516 $paths = $this->backends
[$this->masterIndex
]->getPathsToLockForOpsInternal( $fileOps );
517 // Get the paths under the proxy backend's name
518 $paths['sh'] = $this->unsubstPaths( $paths['sh'] );
519 $paths['ex'] = $this->unsubstPaths( $paths['ex'] );
521 $this->getScopedFileLocks( $paths['sh'], LockManager
::LOCK_UW
, $status ),
522 $this->getScopedFileLocks( $paths['ex'], LockManager
::LOCK_EX
, $status )