maintenance/purgeChangedFiles.php

   1 <?php
   2 /**
   3  * Scan the logging table and purge affected files within a timeframe.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  * @ingroup Maintenance
  22  */
  23
  24 use MediaWiki\Maintenance\Maintenance;
  25 use MediaWiki\Title\Title;
  26
  27 // @codeCoverageIgnoreStart
  28 require_once __DIR__ . '/Maintenance.php';
  29 // @codeCoverageIgnoreEnd
  30
  31 /**
  32  * Maintenance script that scans the deletion log and purges affected files
  33  * within a timeframe.
  34  *
  35  * @ingroup Maintenance
  36  */
  37 class PurgeChangedFiles extends Maintenance {
  38         /**
  39          * Mapping from type option to log type and actions.
  40          * @var array
  41          */
  42         private static $typeMappings = [
  43                 'created' => [
  44                         'upload' => [ 'upload' ],
  45                         'import' => [ 'upload', 'interwiki' ],
  46                 ],
  47                 'deleted' => [
  48                         'delete' => [ 'delete', 'revision' ],
  49                         'suppress' => [ 'delete', 'revision' ],
  50                 ],
  51                 'modified' => [
  52                         'upload' => [ 'overwrite', 'revert' ],
  53                         'move' => [ 'move', 'move_redir' ],
  54                 ],
  55         ];
  56
  57         /**
  58          * @var string
  59          */
  60         private $startTimestamp;
  61
  62         /**
  63          * @var string
  64          */
  65         private $endTimestamp;
  66
  67         public function __construct() {
  68                 parent::__construct();
  69                 $this->addDescription( 'Scan the logging table and purge files and thumbnails.' );
  70                 $this->addOption( 'starttime', 'Starting timestamp', true, true );
  71                 $this->addOption( 'endtime', 'Ending timestamp', true, true );
  72                 $this->addOption( 'type', 'Comma-separated list of types of changes to send purges for (' .
  73                         implode( ',', array_keys( self::$typeMappings ) ) . ',all)', false, true );
  74                 $this->addOption( 'htcp-dest', 'HTCP announcement destination (IP:port)', false, true );
  75                 $this->addOption( 'dry-run', 'Do not send purge requests' );
  76                 $this->addOption( 'sleep-per-batch', 'Milliseconds to sleep between batches', false, true );
  77                 $this->addOption( 'verbose', 'Show more output', false, false, 'v' );
  78                 $this->setBatchSize( 100 );
  79         }
  80
  81         public function execute() {
  82                 global $wgHTCPRouting;
  83
  84                 if ( $this->hasOption( 'htcp-dest' ) ) {
  85                         $parts = explode( ':', $this->getOption( 'htcp-dest' ), 2 );
  86                         if ( count( $parts ) < 2 ) {
  87                                 // Add default htcp port
  88                                 $parts[] = '4827';
  89                         }
  90
  91                         // Route all HTCP messages to provided host:port
  92                         $wgHTCPRouting = [
  93                                 '' => [ 'host' => $parts[0], 'port' => $parts[1] ],
  94                         ];
  95                         $this->verbose( "HTCP broadcasts to {$parts[0]}:{$parts[1]}\n" );
  96                 }
  97
  98                 // Find out which actions we should be concerned with
  99                 $typeOpt = $this->getOption( 'type', 'all' );
 100                 if ( $typeOpt === 'all' ) {
 101                         // Convert 'all' to all registered types
 102                         $typeOpt = implode( ',', array_keys( self::$typeMappings ) );
 103                 }
 104                 $typeList = explode( ',', $typeOpt );
 105                 foreach ( $typeList as $type ) {
 106                         if ( !isset( self::$typeMappings[$type] ) ) {
 107                                 $this->error( "\nERROR: Unknown type: {$type}\n" );
 108                                 $this->maybeHelp( true );
 109                         }
 110                 }
 111
 112                 // Validate the timestamps
 113                 $dbr = $this->getReplicaDB();
 114                 $this->startTimestamp = $dbr->timestamp( $this->getOption( 'starttime' ) );
 115                 $this->endTimestamp = $dbr->timestamp( $this->getOption( 'endtime' ) );
 116
 117                 if ( $this->startTimestamp > $this->endTimestamp ) {
 118                         $this->error( "\nERROR: starttime after endtime\n" );
 119                         $this->maybeHelp( true );
 120                 }
 121
 122                 // Turn on verbose when dry-run is enabled
 123                 if ( $this->hasOption( 'dry-run' ) ) {
 124                         $this->setOption( 'verbose', 1 );
 125                 }
 126
 127                 $this->verbose( 'Purging files that were: ' . implode( ', ', $typeList ) . "\n" );
 128                 foreach ( $typeList as $type ) {
 129                         $this->verbose( "Checking for {$type} files...\n" );
 130                         $this->purgeFromLogType( $type );
 131                         if ( !$this->hasOption( 'dry-run' ) ) {
 132                                 $this->verbose( "...{$type} files purged.\n\n" );
 133                         }
 134                 }
 135         }
 136
 137         /**
 138          * Purge cache and thumbnails for changes of the given type.
 139          *
 140          * @param string $type Type of change to find
 141          */
 142         protected function purgeFromLogType( $type ) {
 143                 $repo = $this->getServiceContainer()->getRepoGroup()->getLocalRepo();
 144                 $dbr = $this->getReplicaDB();
 145
 146                 foreach ( self::$typeMappings[$type] as $logType => $logActions ) {
 147                         $this->verbose( "Scanning for {$logType}/" . implode( ',', $logActions ) . "\n" );
 148
 149                         $res = $dbr->newSelectQueryBuilder()
 150                                 ->select( [ 'log_title', 'log_timestamp', 'log_params' ] )
 151                                 ->from( 'logging' )
 152                                 ->where( [
 153                                         'log_namespace' => NS_FILE,
 154                                         'log_type' => $logType,
 155                                         'log_action' => $logActions,
 156                                         $dbr->expr( 'log_timestamp', '>=', $this->startTimestamp ),
 157                                         $dbr->expr( 'log_timestamp', '<=', $this->endTimestamp ),
 158                                 ] )
 159                                 ->caller( __METHOD__ )->fetchResultSet();
 160
 161                         $bSize = 0;
 162                         foreach ( $res as $row ) {
 163                                 $file = $repo->newFile( Title::makeTitle( NS_FILE, $row->log_title ) );
 164
 165                                 if ( $this->hasOption( 'dry-run' ) ) {
 166                                         $this->verbose( "{$type}[{$row->log_timestamp}]: {$row->log_title}\n" );
 167                                         continue;
 168                                 }
 169
 170                                 // Purge current version and its thumbnails
 171                                 $file->purgeCache();
 172                                 // Purge the old versions and their thumbnails
 173                                 foreach ( $file->getHistory() as $oldFile ) {
 174                                         $oldFile->purgeCache();
 175                                 }
 176
 177                                 if ( $logType === 'delete' ) {
 178                                         // If there is an orphaned storage file... delete it
 179                                         if ( !$file->exists() && $repo->fileExists( $file->getPath() ) ) {
 180                                                 $dpath = $this->getDeletedPath( $repo, $file );
 181                                                 if ( $repo->fileExists( $dpath ) ) {
 182                                                         // Check to avoid data loss
 183                                                         $repo->getBackend()->delete( [ 'src' => $file->getPath() ] );
 184                                                         $this->verbose( "Deleted orphan file: {$file->getPath()}.\n" );
 185                                                 } else {
 186                                                         $this->error( "File was not deleted: {$file->getPath()}.\n" );
 187                                                 }
 188                                         }
 189
 190                                         // Purge items from fileachive table (rows are likely here)
 191                                         $this->purgeFromArchiveTable( $repo, $file );
 192                                 } elseif ( $logType === 'move' ) {
 193                                         // Purge the target file as well
 194
 195                                         $params = unserialize( $row->log_params );
 196                                         if ( isset( $params['4::target'] ) ) {
 197                                                 $target = $params['4::target'];
 198                                                 $targetFile = $repo->newFile( Title::makeTitle( NS_FILE, $target ) );
 199                                                 $targetFile->purgeCache();
 200                                                 $this->verbose( "Purged file {$target}; move target @{$row->log_timestamp}.\n" );
 201                                         }
 202                                 }
 203
 204                                 $this->verbose( "Purged file {$row->log_title}; {$type} @{$row->log_timestamp}.\n" );
 205
 206                                 if ( $this->hasOption( 'sleep-per-batch' ) && ++$bSize > $this->getBatchSize() ) {
 207                                         $bSize = 0;
 208                                         // sleep-per-batch is milliseconds, usleep wants micro seconds.
 209                                         usleep( 1000 * (int)$this->getOption( 'sleep-per-batch' ) );
 210                                 }
 211                         }
 212                 }
 213         }
 214
 215         protected function purgeFromArchiveTable( LocalRepo $repo, LocalFile $file ) {
 216                 $dbr = $repo->getReplicaDB();
 217                 $res = $dbr->newSelectQueryBuilder()
 218                         ->select( [ 'fa_archive_name' ] )
 219                         ->from( 'filearchive' )
 220                         ->where( [ 'fa_name' => $file->getName() ] )
 221                         ->caller( __METHOD__ )->fetchResultSet();
 222
 223                 foreach ( $res as $row ) {
 224                         if ( $row->fa_archive_name === null ) {
 225                                 // Was not an old version (current version names checked already)
 226                                 continue;
 227                         }
 228                         $ofile = $repo->newFromArchiveName( $file->getTitle(), $row->fa_archive_name );
 229                         // If there is an orphaned storage file still there...delete it
 230                         if ( !$file->exists() && $repo->fileExists( $ofile->getPath() ) ) {
 231                                 $dpath = $this->getDeletedPath( $repo, $ofile );
 232                                 if ( $repo->fileExists( $dpath ) ) {
 233                                         // Check to avoid data loss
 234                                         $repo->getBackend()->delete( [ 'src' => $ofile->getPath() ] );
 235                                         $this->output( "Deleted orphan file: {$ofile->getPath()}.\n" );
 236                                 } else {
 237                                         $this->error( "File was not deleted: {$ofile->getPath()}.\n" );
 238                                 }
 239                         }
 240                         $file->purgeOldThumbnails( $row->fa_archive_name );
 241                 }
 242         }
 243
 244         protected function getDeletedPath( LocalRepo $repo, LocalFile $file ) {
 245                 $hash = $repo->getFileSha1( $file->getPath() );
 246                 $key = "{$hash}.{$file->getExtension()}";
 247
 248                 return $repo->getDeletedHashPath( $key ) . $key;
 249         }
 250
 251         /**
 252          * Send an output message iff the 'verbose' option has been provided.
 253          *
 254          * @param string $msg Message to output
 255          */
 256         protected function verbose( $msg ) {
 257                 if ( $this->hasOption( 'verbose' ) ) {
 258                         $this->output( $msg );
 259                 }
 260         }
 261 }
 262
 263 // @codeCoverageIgnoreStart
 264 $maintClass = PurgeChangedFiles::class;
 265 require_once RUN_MAINTENANCE_IF_MAIN;
 266 // @codeCoverageIgnoreEnd