includes/filerepo/FileBackendDBRepoWrapper.php

   1 <?php
   2 /**
   3  * This program is free software; you can redistribute it and/or modify
   4  * it under the terms of the GNU General Public License as published by
   5  * the Free Software Foundation; either version 2 of the License, or
   6  * (at your option) any later version.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  11  * GNU General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along
  14  * with this program; if not, write to the Free Software Foundation, Inc.,
  15  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  16  * http://www.gnu.org/copyleft/gpl.html
  17  *
  18  * @file
  19  */
  20
  21 use MediaWiki\Output\StreamFile;
  22 use Shellbox\Command\BoxedCommand;
  23 use Wikimedia\FileBackend\FileBackend;
  24 use Wikimedia\Rdbms\IDatabase;
  25
  26 /**
  27  * Proxy backend that manages file layout rewriting for FileRepo.
  28  *
  29  * LocalRepo may be configured to store files under their title names or by SHA-1.
  30  * This acts as a shim in the latter case, providing backwards compatability for
  31  * most callers. All "public"/"deleted" zone files actually go in an "original"
  32  * container and are never changed.
  33  *
  34  * This requires something like thumb_handler.php and img_auth.php for client viewing of files.
  35  *
  36  * @ingroup FileRepo
  37  * @ingroup FileBackend
  38  * @since 1.25
  39  */
  40 class FileBackendDBRepoWrapper extends FileBackend {
  41         /** @var FileBackend */
  42         protected $backend;
  43         /** @var string */
  44         protected $repoName;
  45         /** @var Closure */
  46         protected $dbHandleFunc;
  47         /** @var MapCacheLRU */
  48         protected $resolvedPathCache;
  49         /** @var IDatabase[] */
  50         protected $dbs;
  51
  52         public function __construct( array $config ) {
  53                 /** @var FileBackend $backend */
  54                 $backend = $config['backend'];
  55                 $config['name'] = $backend->getName();
  56                 $config['domainId'] = $backend->getDomainId();
  57                 parent::__construct( $config );
  58                 $this->backend = $config['backend'];
  59                 $this->repoName = $config['repoName'];
  60                 $this->dbHandleFunc = $config['dbHandleFactory'];
  61                 $this->resolvedPathCache = new MapCacheLRU( 100 );
  62         }
  63
  64         /**
  65          * Get the underlying FileBackend that is being wrapped
  66          *
  67          * @return FileBackend
  68          */
  69         public function getInternalBackend() {
  70                 return $this->backend;
  71         }
  72
  73         /**
  74          * Translate a legacy "title" path to its "sha1" counterpart
  75          *
  76          * E.g. mwstore://local-backend/local-public/a/ab/<name>.jpg
  77          * => mwstore://local-backend/local-original/x/y/z/<sha1>.jpg
  78          *
  79          * @param string $path
  80          * @param bool $latest
  81          * @return string
  82          */
  83         public function getBackendPath( $path, $latest = true ) {
  84                 $paths = $this->getBackendPaths( [ $path ], $latest );
  85                 return current( $paths );
  86         }
  87
  88         /**
  89          * Translate legacy "title" paths to their "sha1" counterparts
  90          *
  91          * E.g. mwstore://local-backend/local-public/a/ab/<name>.jpg
  92          * => mwstore://local-backend/local-original/x/y/z/<sha1>.jpg
  93          *
  94          * @param string[] $paths
  95          * @param bool $latest
  96          * @return string[] Translated paths in same order
  97          */
  98         public function getBackendPaths( array $paths, $latest = true ) {
  99                 $db = $this->getDB( $latest ? DB_PRIMARY : DB_REPLICA );
 100
 101                 // @TODO: batching
 102                 $resolved = [];
 103                 foreach ( $paths as $i => $path ) {
 104                         if ( !$latest && $this->resolvedPathCache->hasField( $path, 'target', 10 ) ) {
 105                                 $resolved[$i] = $this->resolvedPathCache->getField( $path, 'target' );
 106                                 continue;
 107                         }
 108
 109                         [ , $container ] = FileBackend::splitStoragePath( $path );
 110
 111                         if ( $container === "{$this->repoName}-public" ) {
 112                                 $name = basename( $path );
 113                                 if ( str_contains( $path, '!' ) ) {
 114                                         $sha1 = $db->newSelectQueryBuilder()
 115                                                 ->select( 'oi_sha1' )
 116                                                 ->from( 'oldimage' )
 117                                                 ->where( [ 'oi_archive_name' => $name ] )
 118                                                 ->caller( __METHOD__ )->fetchField();
 119                                 } else {
 120                                         $sha1 = $db->newSelectQueryBuilder()
 121                                                 ->select( 'img_sha1' )
 122                                                 ->from( 'image' )
 123                                                 ->where( [ 'img_name' => $name ] )
 124                                                 ->caller( __METHOD__ )->fetchField();
 125                                 }
 126                                 if ( $sha1 === null || !strlen( $sha1 ) ) {
 127                                         $resolved[$i] = $path; // give up
 128                                         continue;
 129                                 }
 130                                 $resolved[$i] = $this->getPathForSHA1( $sha1 );
 131                                 $this->resolvedPathCache->setField( $path, 'target', $resolved[$i] );
 132                         } elseif ( $container === "{$this->repoName}-deleted" ) {
 133                                 $name = basename( $path ); // <hash>.<ext>
 134                                 $sha1 = substr( $name, 0, strpos( $name, '.' ) ); // ignore extension
 135                                 $resolved[$i] = $this->getPathForSHA1( $sha1 );
 136                                 $this->resolvedPathCache->setField( $path, 'target', $resolved[$i] );
 137                         } else {
 138                                 $resolved[$i] = $path;
 139                         }
 140                 }
 141
 142                 $res = [];
 143                 foreach ( $paths as $i => $path ) {
 144                         $res[$i] = $resolved[$i];
 145                 }
 146
 147                 return $res;
 148         }
 149
 150         protected function doOperationsInternal( array $ops, array $opts ) {
 151                 return $this->backend->doOperationsInternal( $this->mungeOpPaths( $ops ), $opts );
 152         }
 153
 154         protected function doQuickOperationsInternal( array $ops, array $opts ) {
 155                 return $this->backend->doQuickOperationsInternal( $this->mungeOpPaths( $ops ), $opts );
 156         }
 157
 158         protected function doPrepare( array $params ) {
 159                 return $this->backend->doPrepare( $params );
 160         }
 161
 162         protected function doSecure( array $params ) {
 163                 return $this->backend->doSecure( $params );
 164         }
 165
 166         protected function doPublish( array $params ) {
 167                 return $this->backend->doPublish( $params );
 168         }
 169
 170         protected function doClean( array $params ) {
 171                 return $this->backend->doClean( $params );
 172         }
 173
 174         public function concatenate( array $params ) {
 175                 return $this->translateSrcParams( __FUNCTION__, $params );
 176         }
 177
 178         public function fileExists( array $params ) {
 179                 return $this->translateSrcParams( __FUNCTION__, $params );
 180         }
 181
 182         public function getFileTimestamp( array $params ) {
 183                 return $this->translateSrcParams( __FUNCTION__, $params );
 184         }
 185
 186         public function getFileSize( array $params ) {
 187                 return $this->translateSrcParams( __FUNCTION__, $params );
 188         }
 189
 190         public function getFileStat( array $params ) {
 191                 return $this->translateSrcParams( __FUNCTION__, $params );
 192         }
 193
 194         public function getFileXAttributes( array $params ) {
 195                 return $this->translateSrcParams( __FUNCTION__, $params );
 196         }
 197
 198         public function getFileSha1Base36( array $params ) {
 199                 return $this->translateSrcParams( __FUNCTION__, $params );
 200         }
 201
 202         public function getFileProps( array $params ) {
 203                 return $this->translateSrcParams( __FUNCTION__, $params );
 204         }
 205
 206         public function streamFile( array $params ) {
 207                 // The stream methods use the file extension to determine the
 208                 // Content-Type (as MediaWiki should already validate it on upload).
 209                 // The translated SHA1 path has no extension, so this needs to use
 210                 // the untranslated path extension.
 211                 $type = StreamFile::contentTypeFromPath( $params['src'] );
 212                 if ( $type && $type != 'unknown/unknown' ) {
 213                         $params['headers'][] = "Content-type: $type";
 214                 }
 215                 return $this->translateSrcParams( __FUNCTION__, $params );
 216         }
 217
 218         public function getFileContentsMulti( array $params ) {
 219                 return $this->translateArrayResults( __FUNCTION__, $params );
 220         }
 221
 222         public function getLocalReferenceMulti( array $params ) {
 223                 return $this->translateArrayResults( __FUNCTION__, $params );
 224         }
 225
 226         public function getLocalCopyMulti( array $params ) {
 227                 return $this->translateArrayResults( __FUNCTION__, $params );
 228         }
 229
 230         public function getFileHttpUrl( array $params ) {
 231                 return $this->translateSrcParams( __FUNCTION__, $params );
 232         }
 233
 234         public function addShellboxInputFile( BoxedCommand $command, string $boxedName,
 235                 array $params
 236         ) {
 237                 $params['src'] = $this->getBackendPath( $params['src'], !empty( $params['latest'] ) );
 238                 return $this->backend->addShellboxInputFile( $command, $boxedName, $params );
 239         }
 240
 241         public function directoryExists( array $params ) {
 242                 return $this->backend->directoryExists( $params );
 243         }
 244
 245         public function getDirectoryList( array $params ) {
 246                 return $this->backend->getDirectoryList( $params );
 247         }
 248
 249         public function getFileList( array $params ) {
 250                 return $this->backend->getFileList( $params );
 251         }
 252
 253         public function getFeatures() {
 254                 return $this->backend->getFeatures();
 255         }
 256
 257         public function clearCache( ?array $paths = null ) {
 258                 $this->backend->clearCache( null ); // clear all
 259         }
 260
 261         public function preloadCache( array $paths ) {
 262                 $paths = $this->getBackendPaths( $paths );
 263                 $this->backend->preloadCache( $paths );
 264         }
 265
 266         public function preloadFileStat( array $params ) {
 267                 return $this->translateSrcParams( __FUNCTION__, $params );
 268         }
 269
 270         public function getScopedLocksForOps( array $ops, StatusValue $status ) {
 271                 return $this->backend->getScopedLocksForOps( $ops, $status );
 272         }
 273
 274         /**
 275          * Get the ultimate original storage path for a file
 276          *
 277          * Use this when putting a new file into the system
 278          *
 279          * @param string $sha1 File SHA-1 base36
 280          * @return string
 281          */
 282         public function getPathForSHA1( $sha1 ) {
 283                 if ( strlen( $sha1 ) < 3 ) {
 284                         throw new InvalidArgumentException( "Invalid file SHA-1." );
 285                 }
 286                 return $this->backend->getContainerStoragePath( "{$this->repoName}-original" ) .
 287                         "/{$sha1[0]}/{$sha1[1]}/{$sha1[2]}/{$sha1}";
 288         }
 289
 290         /**
 291          * Get a connection to the repo file registry DB
 292          *
 293          * @param int $index
 294          * @return IDatabase
 295          */
 296         protected function getDB( $index ) {
 297                 if ( !isset( $this->dbs[$index] ) ) {
 298                         $func = $this->dbHandleFunc;
 299                         $this->dbs[$index] = $func( $index );
 300                 }
 301                 return $this->dbs[$index];
 302         }
 303
 304         /**
 305          * Translates paths found in the "src" or "srcs" keys of a params array
 306          *
 307          * @param string $function
 308          * @param array $params
 309          * @return mixed
 310          */
 311         protected function translateSrcParams( $function, array $params ) {
 312                 $latest = !empty( $params['latest'] );
 313
 314                 if ( isset( $params['src'] ) ) {
 315                         $params['src'] = $this->getBackendPath( $params['src'], $latest );
 316                 }
 317
 318                 if ( isset( $params['srcs'] ) ) {
 319                         $params['srcs'] = $this->getBackendPaths( $params['srcs'], $latest );
 320                 }
 321
 322                 return $this->backend->$function( $params );
 323         }
 324
 325         /**
 326          * Translates paths when the backend function returns results keyed by paths
 327          *
 328          * @param string $function
 329          * @param array $params
 330          * @return array
 331          */
 332         protected function translateArrayResults( $function, array $params ) {
 333                 $origPaths = $params['srcs'];
 334                 $params['srcs'] = $this->getBackendPaths( $params['srcs'], !empty( $params['latest'] ) );
 335                 $pathMap = array_combine( $params['srcs'], $origPaths );
 336
 337                 $results = $this->backend->$function( $params );
 338
 339                 $contents = [];
 340                 foreach ( $results as $path => $result ) {
 341                         $contents[$pathMap[$path]] = $result;
 342                 }
 343
 344                 return $contents;
 345         }
 346
 347         /**
 348          * Translate legacy "title" source paths to their "sha1" counterparts
 349          *
 350          * This leaves destination paths alone since we don't want those to mutate
 351          *
 352          * @param array[] $ops
 353          * @return array[]
 354          */
 355         protected function mungeOpPaths( array $ops ) {
 356                 // Ops that use 'src' and do not mutate core file data there
 357                 static $srcRefOps = [ 'store', 'copy', 'describe' ];
 358                 foreach ( $ops as &$op ) {
 359                         if ( isset( $op['src'] ) && in_array( $op['op'], $srcRefOps ) ) {
 360                                 $op['src'] = $this->getBackendPath( $op['src'], true );
 361                         }
 362                         if ( isset( $op['srcs'] ) ) {
 363                                 $op['srcs'] = $this->getBackendPaths( $op['srcs'], true );
 364                         }
 365                 }
 366                 return $ops;
 367         }
 368 }