maintenance/cleanupTable.inc

   1 <?php
   2 /**
   3  * Generic table cleanup class. Already subclasses maintenance
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  * @ingroup Maintenance
  22  */
  23
  24 require_once( dirname( __FILE__ ) . '/Maintenance.php' );
  25
  26 class TableCleanup extends Maintenance {
  27         protected $defaultParams = array(
  28                 'table' => 'page',
  29                 'conds' => array(),
  30                 'index' => 'page_id',
  31                 'callback' => 'processRow',
  32         );
  33
  34         protected $dryrun = false;
  35         protected $maxLag = 10; # if slaves are lagged more than 10 secs, wait
  36         public $batchSize = 100;
  37         public $reportInterval = 100;
  38
  39         public function __construct() {
  40                 parent::__construct();
  41                 $this->addOption( 'dry-run', 'Perform a dry run' );
  42         }
  43
  44         public function execute() {
  45                 global $wgUser;
  46                 $wgUser = User::newFromName( 'Conversion script' );
  47                 $this->dryrun = $this->hasOption( 'dry-run' );
  48                 if ( $this->dryrun ) {
  49                         $this->output( "Checking for bad titles...\n" );
  50                 } else {
  51                         $this->output( "Checking and fixing bad titles...\n" );
  52                 }
  53                 $this->runTable( $this->defaultParams );
  54         }
  55
  56         protected function init( $count, $table ) {
  57                 $this->processed = 0;
  58                 $this->updated = 0;
  59                 $this->count = $count;
  60                 $this->startTime = wfTime();
  61                 $this->table = $table;
  62         }
  63
  64         protected function progress( $updated ) {
  65                 $this->updated += $updated;
  66                 $this->processed++;
  67                 if ( $this->processed % $this->reportInterval != 0 ) {
  68                         return;
  69                 }
  70                 $portion = $this->processed / $this->count;
  71                 $updateRate = $this->updated / $this->processed;
  72
  73                 $now = wfTime();
  74                 $delta = $now - $this->startTime;
  75                 $estimatedTotalTime = $delta / $portion;
  76                 $eta = $this->startTime + $estimatedTotalTime;
  77
  78                 $this->output(
  79                         sprintf( "%s %s: %6.2f%% done on %s; ETA %s [%d/%d] %.2f/sec <%.2f%% updated>\n",
  80                                 wfWikiID(),
  81                                 wfTimestamp( TS_DB, intval( $now ) ),
  82                                 $portion * 100.0,
  83                                 $this->table,
  84                                 wfTimestamp( TS_DB, intval( $eta ) ),
  85                                 $this->processed,
  86                                 $this->count,
  87                                 $this->processed / $delta,
  88                                 $updateRate * 100.0
  89                         )
  90                 );
  91                 flush();
  92         }
  93
  94         public function runTable( $params ) {
  95                 $dbr = wfGetDB( DB_SLAVE );
  96
  97                 if ( array_diff( array_keys( $params ),
  98                         array( 'table', 'conds', 'index', 'callback' ) ) )
  99                 {
 100                         throw new MWException( __METHOD__ . ': Missing parameter ' . implode( ', ', $params ) );
 101                 }
 102
 103                 $table = $params['table'];
 104                 // count(*) would melt the DB for huge tables, we can estimate here
 105                 $count = $dbr->estimateRowCount( $table, '*', '', __METHOD__ );
 106                 $this->init( $count, $table );
 107                 $this->output( "Processing $table...\n" );
 108
 109
 110                 $index = (array)$params['index'];
 111                 $indexConds = array();
 112                 $options = array(
 113                         'ORDER BY' => implode( ',', $index ),
 114                         'LIMIT' => $this->batchSize
 115                 );
 116                 $callback = array( $this, $params['callback'] );
 117
 118                 while ( true ) {
 119                         $conds = array_merge( $params['conds'], $indexConds );
 120                         $res = $dbr->select( $table, '*', $conds, __METHOD__, $options );
 121                         if ( !$res->numRows() ) {
 122                                 // Done
 123                                 break;
 124                         }
 125
 126                         foreach ( $res as $row ) {
 127                                 call_user_func( $callback, $row );
 128                         }
 129
 130                         if ( $res->numRows() < $this->batchSize ) {
 131                                 // Done
 132                                 break;
 133                         }
 134
 135                         // Update the conditions to select the next batch.
 136                         // Construct a condition string by starting with the least significant part
 137                         // of the index, and adding more significant parts progressively to the left
 138                         // of the string.
 139                         $nextCond = '';
 140                         foreach ( array_reverse( $index ) as $field ) {
 141                                 $encValue = $dbr->addQuotes( $row->$field );
 142                                 if ( $nextCond === '' ) {
 143                                         $nextCond = "$field > $encValue";
 144                                 } else {
 145                                         $nextCond = "$field > $encValue OR ($field = $encValue AND ($nextCond))";
 146                                 }
 147                         }
 148                         $indexConds = array( $nextCond );
 149                 }
 150
 151                 $this->output( "Finished $table... $this->updated of $this->processed rows updated\n" );
 152         }
 153
 154         protected function hexChar( $matches ) {
 155                 return sprintf( "\\x%02x", ord( $matches[1] ) );
 156         }
 157 }
 158
 159 class TableCleanupTest extends TableCleanup {
 160         function processRow( $row ) {
 161                 $this->progress( mt_rand( 0, 1 ) );
 162         }
 163 }
 164