maintenance/cleanupImages.php

   1 <?php
   2 /**
   3  * Clean up broken, unparseable upload filenames.
   4  *
   5  * Copyright © 2005-2006 Brooke Vibber <bvibber@wikimedia.org>
   6  * https://www.mediawiki.org/
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License as published by
  10  * the Free Software Foundation; either version 2 of the License, or
  11  * (at your option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License along
  19  * with this program; if not, write to the Free Software Foundation, Inc.,
  20  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  21  * http://www.gnu.org/copyleft/gpl.html
  22  *
  23  * @file
  24  * @author Brooke Vibber <bvibber@wikimedia.org>
  25  * @ingroup Maintenance
  26  */
  27
  28 use MediaWiki\Parser\Sanitizer;
  29 use MediaWiki\Title\Title;
  30
  31 // @codeCoverageIgnoreStart
  32 require_once __DIR__ . '/TableCleanup.php';
  33 // @codeCoverageIgnoreEnd
  34
  35 /**
  36  * Maintenance script to clean up broken, unparseable upload filenames.
  37  *
  38  * @ingroup Maintenance
  39  */
  40 class CleanupImages extends TableCleanup {
  41         /** @inheritDoc */
  42         protected $defaultParams = [
  43                 'table' => 'image',
  44                 'conds' => [],
  45                 'index' => 'img_name',
  46                 'callback' => 'processRow',
  47         ];
  48
  49         /** @var LocalRepo|null */
  50         private $repo;
  51
  52         public function __construct() {
  53                 parent::__construct();
  54                 $this->addDescription( 'Script to clean up broken, unparseable upload filenames' );
  55         }
  56
  57         protected function processRow( $row ) {
  58                 $source = $row->img_name;
  59                 if ( $source == '' ) {
  60                         // Ye olde empty rows. Just kill them.
  61                         $this->killRow( $source );
  62
  63                         $this->progress( 1 );
  64                         return;
  65                 }
  66
  67                 $cleaned = $source;
  68
  69                 // About half of old bad image names have percent-codes
  70                 $cleaned = rawurldecode( $cleaned );
  71
  72                 // We also have some HTML entities there
  73                 $cleaned = Sanitizer::decodeCharReferences( $cleaned );
  74
  75                 $contLang = $this->getServiceContainer()->getContentLanguage();
  76
  77                 // Some are old latin-1
  78                 $cleaned = $contLang->checkTitleEncoding( $cleaned );
  79
  80                 // Many of remainder look like non-normalized unicode
  81                 $cleaned = $contLang->normalize( $cleaned );
  82
  83                 $title = Title::makeTitleSafe( NS_FILE, $cleaned );
  84
  85                 if ( $title === null ) {
  86                         $this->output( "page $source ($cleaned) is illegal.\n" );
  87                         $safe = $this->buildSafeTitle( $cleaned );
  88                         if ( $safe === false ) {
  89                                 $this->progress( 0 );
  90                                 return;
  91                         }
  92                         $this->pokeFile( $source, $safe );
  93
  94                         $this->progress( 1 );
  95                         return;
  96                 }
  97
  98                 if ( $title->getDBkey() !== $source ) {
  99                         $munged = $title->getDBkey();
 100                         $this->output( "page $source ($munged) doesn't match self.\n" );
 101                         $this->pokeFile( $source, $munged );
 102
 103                         $this->progress( 1 );
 104                         return;
 105                 }
 106
 107                 $this->progress( 0 );
 108         }
 109
 110         /**
 111          * @param string $name
 112          */
 113         private function killRow( $name ) {
 114                 if ( $this->dryrun ) {
 115                         $this->output( "DRY RUN: would delete bogus row '$name'\n" );
 116                 } else {
 117                         $this->output( "deleting bogus row '$name'\n" );
 118                         $db = $this->getPrimaryDB();
 119                         $db->newDeleteQueryBuilder()
 120                                 ->deleteFrom( 'image' )
 121                                 ->where( [ 'img_name' => $name ] )
 122                                 ->caller( __METHOD__ )
 123                                 ->execute();
 124                 }
 125         }
 126
 127         /**
 128          * @param string $name
 129          * @return string
 130          */
 131         private function filePath( $name ) {
 132                 if ( $this->repo === null ) {
 133                         $this->repo = $this->getServiceContainer()->getRepoGroup()->getLocalRepo();
 134                 }
 135
 136                 return $this->repo->getRootDirectory() . '/' . $this->repo->getHashPath( $name ) . $name;
 137         }
 138
 139         private function imageExists( $name, $db ) {
 140                 return (bool)$db->newSelectQueryBuilder()
 141                         ->select( '1' )
 142                         ->from( 'image' )
 143                         ->where( [ 'img_name' => $name ] )
 144                         ->caller( __METHOD__ )
 145                         ->fetchField();
 146         }
 147
 148         private function pageExists( $name, $db ) {
 149                 return (bool)$db->newSelectQueryBuilder()
 150                         ->select( '1' )
 151                         ->from( 'page' )
 152                         ->where( [
 153                                 'page_namespace' => NS_FILE,
 154                                 'page_title' => $name,
 155                         ] )
 156                         ->caller( __METHOD__ )
 157                         ->fetchField();
 158         }
 159
 160         private function pokeFile( $orig, $new ) {
 161                 $path = $this->filePath( $orig );
 162                 if ( !file_exists( $path ) ) {
 163                         $this->output( "missing file: $path\n" );
 164                         $this->killRow( $orig );
 165
 166                         return;
 167                 }
 168
 169                 $db = $this->getPrimaryDB();
 170
 171                 /*
 172                  * To prevent key collisions in the update() statements below,
 173                  * if the target title exists in the image table, or if both the
 174                  * original and target titles exist in the page table, append
 175                  * increasing version numbers until the target title exists in
 176                  * neither.  (See also T18916.)
 177                  */
 178                 $version = 0;
 179                 $final = $new;
 180                 $conflict = ( $this->imageExists( $final, $db ) ||
 181                         ( $this->pageExists( $orig, $db ) && $this->pageExists( $final, $db ) ) );
 182
 183                 while ( $conflict ) {
 184                         $this->output( "Rename conflicts with '$final'...\n" );
 185                         $version++;
 186                         $final = $this->appendTitle( $new, "_$version" );
 187                         $conflict = ( $this->imageExists( $final, $db ) || $this->pageExists( $final, $db ) );
 188                 }
 189
 190                 $finalPath = $this->filePath( $final );
 191
 192                 if ( $this->dryrun ) {
 193                         $this->output( "DRY RUN: would rename $path to $finalPath\n" );
 194                 } else {
 195                         $this->output( "renaming $path to $finalPath\n" );
 196                         // @todo FIXME: Should this use File::move()?
 197                         $this->beginTransaction( $db, __METHOD__ );
 198                         $db->newUpdateQueryBuilder()
 199                                 ->update( 'image' )
 200                                 ->set( [ 'img_name' => $final ] )
 201                                 ->where( [ 'img_name' => $orig ] )
 202                                 ->caller( __METHOD__ )
 203                                 ->execute();
 204                         $db->newUpdateQueryBuilder()
 205                                 ->update( 'oldimage' )
 206                                 ->set( [ 'oi_name' => $final ] )
 207                                 ->where( [ 'oi_name' => $orig ] )
 208                                 ->caller( __METHOD__ )
 209                                 ->execute();
 210                         $db->newUpdateQueryBuilder()
 211                                 ->update( 'page' )
 212                                 ->set( [ 'page_title' => $final ] )
 213                                 ->where( [ 'page_title' => $orig, 'page_namespace' => NS_FILE ] )
 214                                 ->caller( __METHOD__ )
 215                                 ->execute();
 216                         $dir = dirname( $finalPath );
 217                         if ( !file_exists( $dir ) ) {
 218                                 if ( !wfMkdirParents( $dir, null, __METHOD__ ) ) {
 219                                         $this->output( "RENAME FAILED, COULD NOT CREATE $dir" );
 220                                         $this->rollbackTransaction( $db, __METHOD__ );
 221
 222                                         return;
 223                                 }
 224                         }
 225                         if ( rename( $path, $finalPath ) ) {
 226                                 $this->commitTransaction( $db, __METHOD__ );
 227                         } else {
 228                                 $this->error( "RENAME FAILED" );
 229                                 $this->rollbackTransaction( $db, __METHOD__ );
 230                         }
 231                 }
 232         }
 233
 234         private function appendTitle( $name, $suffix ) {
 235                 return preg_replace( '/^(.*)(\..*?)$/',
 236                         "\\1$suffix\\2", $name );
 237         }
 238
 239         private function buildSafeTitle( $name ) {
 240                 $x = preg_replace_callback(
 241                         '/([^' . Title::legalChars() . ']|~)/',
 242                         [ $this, 'hexChar' ],
 243                         $name );
 244
 245                 $test = Title::makeTitleSafe( NS_FILE, $x );
 246                 if ( $test === null || $test->getDBkey() !== $x ) {
 247                         $this->error( "Unable to generate safe title from '$name', got '$x'" );
 248
 249                         return false;
 250                 }
 251
 252                 return $x;
 253         }
 254 }
 255
 256 // @codeCoverageIgnoreStart
 257 $maintClass = CleanupImages::class;
 258 require_once RUN_MAINTENANCE_IF_MAIN;
 259 // @codeCoverageIgnoreEnd