maintenance/rebuildImages.php

   1 <?php
   2 /*
   3  * Script to update image metadata records
   4  *
   5  * Usage: php rebuildImages.php [--missing] [--dry-run]
   6  * Options:
   7  *   --missing  Crawl the uploads dir for images without records, and
   8  *              add them only.
   9  *
  10  * Copyright (C) 2005 Brion Vibber <brion@pobox.com>
  11  * http://www.mediawiki.org/
  12  *
  13  * This program is free software; you can redistribute it and/or modify
  14  * it under the terms of the GNU General Public License as published by
  15  * the Free Software Foundation; either version 2 of the License, or
  16  * (at your option) any later version.
  17  *
  18  * This program is distributed in the hope that it will be useful,
  19  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  20  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  21  * GNU General Public License for more details.
  22  *
  23  * You should have received a copy of the GNU General Public License along
  24  * with this program; if not, write to the Free Software Foundation, Inc.,
  25  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  26  * http://www.gnu.org/copyleft/gpl.html
  27  *
  28  * @author Brion Vibber <brion at pobox.com>
  29  * @package MediaWiki
  30  * @subpackage maintenance
  31  */
  32
  33 $options = array( 'missing', 'dry-run' );
  34
  35 require_once( 'commandLine.inc' );
  36 require_once( 'FiveUpgrade.inc' );
  37
  38 class ImageBuilder extends FiveUpgrade {
  39         function ImageBuilder( $dryrun = false ) {
  40                 parent::FiveUpgrade();
  41
  42                 $this->maxLag = 10; # if slaves are lagged more than 10 secs, wait
  43                 $this->dryrun = $dryrun;
  44         }
  45
  46         function build() {
  47                 $this->buildImage();
  48                 $this->buildOldImage();
  49         }
  50
  51         function init( $count, $table ) {
  52                 $this->processed = 0;
  53                 $this->updated = 0;
  54                 $this->count = $count;
  55                 $this->startTime = wfTime();
  56                 $this->table = $table;
  57         }
  58
  59         function progress( $updated ) {
  60                 $this->updated += $updated;
  61                 $this->processed++;
  62                 if( $this->processed % 100 != 0 ) {
  63                         return;
  64                 }
  65                 $portion = $this->processed / $this->count;
  66                 $updateRate = $this->updated / $this->processed;
  67
  68                 $now = wfTime();
  69                 $delta = $now - $this->startTime;
  70                 $estimatedTotalTime = $delta / $portion;
  71                 $eta = $this->startTime + $estimatedTotalTime;
  72
  73                 printf( "%s: %6.2f%% done on %s; ETA %s [%d/%d] %.2f/sec <%.2f%% updated>\n",
  74                         wfTimestamp( TS_DB, intval( $now ) ),
  75                         $portion * 100.0,
  76                         $this->table,
  77                         wfTimestamp( TS_DB, intval( $eta ) ),
  78                         $completed,
  79                         $this->count,
  80                         $rate,
  81                         $updateRate * 100.0 );
  82                 flush();
  83         }
  84
  85         function buildTable( $table, $key, $callback ) {
  86                 $fname = 'ImageBuilder::buildTable';
  87
  88                 $count = $this->dbw->selectField( $table, 'count(*)', '', $fname );
  89                 $this->init( $count, $table );
  90                 $this->log( "Processing $table..." );
  91
  92                 $tableName = $this->dbr->tableName( $table );
  93                 $sql = "SELECT * FROM $tableName";
  94                 $result = $this->dbr->query( $sql, $fname );
  95
  96                 while( $row = $this->dbr->fetchObject( $result ) ) {
  97                         $update = call_user_func( $callback, $row );
  98                         if( is_array( $update ) ) {
  99                                 if( !$this->dryrun ) {
 100                                         $this->dbw->update( $table,
 101                                                 $update,
 102                                                 array( $key => $row->$key ),
 103                                                 $fname );
 104                                 }
 105                                 $this->progress( 1 );
 106                         } else {
 107                                 $this->progress( 0 );
 108                         }
 109                 }
 110                 $this->log( "Finished $table... $this->updated of $this->processed rows updated" );
 111                 $this->dbr->freeResult( $result );
 112         }
 113
 114         function buildImage() {
 115                 $callback = array( &$this, 'imageCallback' );
 116                 $this->buildTable( 'image', 'img_name', $callback );
 117         }
 118
 119         function imageCallback( $row ) {
 120                 if( $row->img_width ) {
 121                         // Already processed
 122                         return null;
 123                 }
 124
 125                 // Fill in the new image info fields
 126                 $info = $this->imageInfo( $row->img_name );
 127
 128                 global $wgMemc, $wgDBname;
 129                 $key = $wgDBname . ":Image:" . md5( $row->img_name );
 130                 $wgMemc->delete( $key );
 131
 132                 return array(
 133                         'img_width'      => $info['width'],
 134                         'img_height'     => $info['height'],
 135                         'img_bits'       => $info['bits'],
 136                         'img_media_type' => $info['media'],
 137                         'img_major_mime' => $info['major'],
 138                         'img_minor_mime' => $info['minor'] );
 139         }
 140
 141
 142         function buildOldImage() {
 143                 $this->buildTable( 'oldimage', 'oi_archive_name',
 144                         array( &$this, 'oldimageCallback' ) );
 145         }
 146
 147         function oldimageCallback( $row ) {
 148                 if( $row->oi_width ) {
 149                         return null;
 150                 }
 151
 152                 // Fill in the new image info fields
 153                 $info = $this->imageInfo( $row->oi_archive_name, 'wfImageArchiveDir', $row->oi_name );
 154                 return array(
 155                         'oi_width'  => $info['width' ],
 156                         'oi_height' => $info['height'],
 157                         'oi_bits'   => $info['bits'  ] );
 158         }
 159
 160         function crawlMissing() {
 161                 global $wgUploadDirectory, $wgHashedUploadDirectory;
 162                 if( $wgHashedUploadDirectory ) {
 163                         for( $i = 0; $i < 16; $i++ ) {
 164                                 for( $j = 0; $j < 16; $j++ ) {
 165                                         $dir = sprintf( '%s%s%01x%s%02x',
 166                                                 $wgUploadDirectory,
 167                                                 DIRECTORY_SEPARATOR,
 168                                                 $i,
 169                                                 DIRECTORY_SEPARATOR,
 170                                                 $i * 16 + $j );
 171                                         $this->crawlDirectory( $dir );
 172                                 }
 173                         }
 174                 } else {
 175                         $this->crawlDirectory( $wgUploadDirectory );
 176                 }
 177         }
 178
 179         function crawlDirectory( $dir ) {
 180                 if( !file_exists( $dir ) ) {
 181                         return $this->log( "no directory, skipping $dir" );
 182                 }
 183                 if( !is_dir( $dir ) ) {
 184                         return $this->log( "not a directory?! skipping $dir" );
 185                 }
 186                 if( !is_readable( $dir ) ) {
 187                         return $this->log( "dir not readable, skipping $dir" );
 188                 }
 189                 $source = opendir( $dir );
 190                 if( $source === false ) {
 191                         return $this->log( "couldn't open dir, skipping $dir" );
 192                 }
 193
 194                 $this->log( "crawling $dir" );
 195                 while( false !== ( $filename = readdir( $source ) ) ) {
 196                         $fullpath = $dir . DIRECTORY_SEPARATOR . $filename;
 197                         if( is_dir( $fullpath ) ) {
 198                                 continue;
 199                         }
 200                         if( is_link( $fullpath ) ) {
 201                                 $this->log( "skipping symlink at $fullpath" );
 202                                 continue;
 203                         }
 204                         $this->checkMissingImage( $filename, $fullpath );
 205                 }
 206                 closedir( $source );
 207         }
 208
 209         function checkMissingImage( $filename, $fullpath ) {
 210                 $fname = 'ImageBuilder::checkMissingImage';
 211                 $row = $this->dbw->selectRow( 'image',
 212                         array( 'img_name' ),
 213                         array( 'img_name' => $filename ),
 214                         $fname );
 215
 216                 if( $row ) {
 217                         // already known, move on
 218                         return;
 219                 } else {
 220                         $this->addMissingImage( $filename, $fullpath );
 221                 }
 222         }
 223
 224         function addMissingImage( $filename, $fullpath ) {
 225                 $fname = 'ImageBuilder::addMissingImage';
 226
 227                 $size = filesize( $fullpath );
 228                 $info = $this->imageInfo( $filename );
 229                 $timestamp = $this->dbw->timestamp( filemtime( $fullpath ) );
 230
 231                 global $wgContLang;
 232                 $altname = $wgContLang->checkTitleEncoding( $filename );
 233                 if( $altname != $filename ) {
 234                         if( $this->dryrun ) {
 235                                 $filename = $altname;
 236                                 $this->log( "Estimating transcoding... $altname" );
 237                         } else {
 238                                 $filename = $this->renameFile( $filename );
 239                         }
 240                 }
 241
 242                 if( $filename == '' ) {
 243                         $this->log( "Empty filename for $fullpath" );
 244                         return;
 245                 }
 246
 247                 $fields = array(
 248                         'img_name'       => $filename,
 249                         'img_size'       => $size,
 250                         'img_width'      => $info['width'],
 251                         'img_height'     => $info['height'],
 252                         'img_metadata'   => '', // filled in on-demand
 253                         'img_bits'       => $info['bits'],
 254                         'img_media_type' => $info['media'],
 255                         'img_major_mime' => $info['major'],
 256                         'img_minor_mime' => $info['minor'],
 257                         'img_description' => '(recovered file, missing upload log entry)',
 258                         'img_user'        => 0,
 259                         'img_user_text'   => 'Conversion script',
 260                         'img_timestamp'   => $timestamp );
 261                 if( !$this->dryrun ) {
 262                         $this->dbw->insert( 'image', $fields, $fname );
 263                 }
 264                 $this->log( $fullpath );
 265         }
 266 }
 267
 268 $builder = new ImageBuilder( isset( $options['dry-run'] ) );
 269 if( isset( $options['missing'] ) ) {
 270         $builder->crawlMissing();
 271 } else {
 272         $builder->build();
 273 }
 274
 275 ?>