maintenance/findOrphanedFiles.php

   1 <?php
   2 /**
   3  * This program is free software; you can redistribute it and/or modify
   4  * it under the terms of the GNU General Public License as published by
   5  * the Free Software Foundation; either version 2 of the License, or
   6  * (at your option) any later version.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  11  * GNU General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along
  14  * with this program; if not, write to the Free Software Foundation, Inc.,
  15  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  16  * http://www.gnu.org/copyleft/gpl.html
  17  *
  18  * @file
  19  * @author Aaron Schulz
  20  */
  21
  22 require_once __DIR__ . '/Maintenance.php';
  23
  24 class FindOrphanedFiles extends Maintenance {
  25         function __construct() {
  26                 parent::__construct();
  27
  28                 $this->addDescription( "Find unregistered files in the 'public' repo zone." );
  29                 $this->addOption( 'subdir',
  30                         'Only scan files in this subdirectory (e.g. "a/a0")', false, true );
  31                 $this->addOption( 'verbose', "Mention file paths checked" );
  32                 $this->setBatchSize( 500 );
  33         }
  34
  35         function execute() {
  36                 $subdir = $this->getOption( 'subdir', '' );
  37                 $verbose = $this->hasOption( 'verbose' );
  38
  39                 $repo = RepoGroup::singleton()->getLocalRepo();
  40                 if ( $repo->hasSha1Storage() ) {
  41                         $this->error( "Local repo uses SHA-1 file storage names; aborting.", 1 );
  42                 }
  43
  44                 $directory = $repo->getZonePath( 'public' );
  45                 if ( $subdir != '' ) {
  46                         $directory .= "/$subdir/";
  47                 }
  48
  49                 if ( $verbose ) {
  50                         $this->output( "Scanning files under $directory:\n" );
  51                 }
  52
  53                 $list = $repo->getBackend()->getFileList( [ 'dir' => $directory ] );
  54                 if ( $list === null ) {
  55                         $this->error( "Could not get file listing.", 1 );
  56                 }
  57
  58                 $pathBatch = [];
  59                 foreach ( $list as $path ) {
  60                         if ( preg_match( '#^(thumb|deleted)/#', $path ) ) {
  61                                 continue; // handle ugly nested containers on stock installs
  62                         }
  63
  64                         $pathBatch[] = $path;
  65                         if ( count( $pathBatch ) >= $this->mBatchSize ) {
  66                                 $this->checkFiles( $repo, $pathBatch, $verbose );
  67                                 $pathBatch = [];
  68                         }
  69                 }
  70                 $this->checkFiles( $repo, $pathBatch, $verbose );
  71         }
  72
  73         protected function checkFiles( LocalRepo $repo, array $paths, $verbose ) {
  74                 if ( !count( $paths ) ) {
  75                         return;
  76                 }
  77
  78                 $dbr = $repo->getReplicaDB();
  79
  80                 $curNames = [];
  81                 $oldNames = [];
  82                 $imgIN = [];
  83                 $oiWheres = [];
  84                 foreach ( $paths as $path ) {
  85                         $name = basename( $path );
  86                         if ( preg_match( '#^archive/#', $path ) ) {
  87                                 if ( $verbose ) {
  88                                         $this->output( "Checking old file $name\n" );
  89                                 }
  90
  91                                 $oldNames[] = $name;
  92                                 list( , $base ) = explode( '!', $name, 2 ); // <TS_MW>!<img_name>
  93                                 $oiWheres[] = $dbr->makeList(
  94                                         [ 'oi_name' => $base, 'oi_archive_name' => $name ],
  95                                         LIST_AND
  96                                 );
  97                         } else {
  98                                 if ( $verbose ) {
  99                                         $this->output( "Checking current file $name\n" );
 100                                 }
 101
 102                                 $curNames[] = $name;
 103                                 $imgIN[] = $name;
 104                         }
 105                 }
 106
 107                 $res = $dbr->query(
 108                         $dbr->unionQueries(
 109                                 [
 110                                         $dbr->selectSQLText(
 111                                                 'image',
 112                                                 [ 'name' => 'img_name', 'old' => 0 ],
 113                                                 $imgIN ? [ 'img_name' => $imgIN ] : '1=0'
 114                                         ),
 115                                         $dbr->selectSQLText(
 116                                                 'oldimage',
 117                                                 [ 'name' => 'oi_archive_name', 'old' => 1 ],
 118                                                 $oiWheres ? $dbr->makeList( $oiWheres, LIST_OR ) : '1=0'
 119                                         )
 120                                 ],
 121                                 true // UNION ALL (performance)
 122                         ),
 123                         __METHOD__
 124                 );
 125
 126                 $curNamesFound = [];
 127                 $oldNamesFound = [];
 128                 foreach ( $res as $row ) {
 129                         if ( $row->old ) {
 130                                 $oldNamesFound[] = $row->name;
 131                         } else {
 132                                 $curNamesFound[] = $row->name;
 133                         }
 134                 }
 135
 136                 foreach ( array_diff( $curNames, $curNamesFound ) as $name ) {
 137                         $file = $repo->newFile( $name );
 138                         // Print name and public URL to ease recovery
 139                         if ( $file ) {
 140                                 $this->output( $name . "\n" . $file->getCanonicalUrl() . "\n\n" );
 141                         } else {
 142                                 $this->error( "Cannot get URL for bad file title '$name'" );
 143                         }
 144                 }
 145
 146                 foreach ( array_diff( $oldNames, $oldNamesFound ) as $name ) {
 147                         list( , $base ) = explode( '!', $name, 2 ); // <TS_MW>!<img_name>
 148                         $file = $repo->newFromArchiveName( Title::makeTitle( NS_FILE, $base ), $name );
 149                         // Print name and public URL to ease recovery
 150                         $this->output( $name . "\n" . $file->getCanonicalUrl() . "\n\n" );
 151                 }
 152         }
 153 }
 154
 155 $maintClass = 'FindOrphanedFiles';
 156 require_once RUN_MAINTENANCE_IF_MAIN;