maintenance/TableCleanup.php

   1 <?php
   2 /**
   3  * Generic class to cleanup a database table.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  * @ingroup Maintenance
  22  */
  23
  24 require_once __DIR__ . '/Maintenance.php';
  25
  26 /**
  27  * Generic class to cleanup a database table. Already subclasses Maintenance.
  28  *
  29  * @ingroup Maintenance
  30  */
  31 class TableCleanup extends Maintenance {
  32         protected $defaultParams = [
  33                 'table' => 'page',
  34                 'conds' => [],
  35                 'index' => 'page_id',
  36                 'callback' => 'processRow',
  37         ];
  38
  39         protected $dryrun = false;
  40         protected $reportInterval = 100;
  41
  42         protected $processed, $updated, $count, $startTime, $table;
  43
  44         public function __construct() {
  45                 parent::__construct();
  46                 $this->addOption( 'dry-run', 'Perform a dry run' );
  47                 $this->addOption( 'reporting-interval', 'How often to print status line' );
  48                 $this->setBatchSize( 100 );
  49         }
  50
  51         public function execute() {
  52                 global $wgUser;
  53
  54                 $this->reportInterval = $this->getOption( 'reporting-interval', $this->reportInterval );
  55
  56                 $this->dryrun = $this->hasOption( 'dry-run' );
  57
  58                 if ( $this->dryrun ) {
  59                         $user = User::newFromName( 'Conversion script' );
  60                         $this->output( "Checking for bad titles...\n" );
  61                 } else {
  62                         $user = User::newSystemUser( 'Conversion script', [ 'steal' => true ] );
  63                         $this->output( "Checking and fixing bad titles...\n" );
  64                 }
  65                 // Support deprecated use of the global
  66                 $wgUser = $user;
  67
  68                 $this->runTable( $this->defaultParams );
  69         }
  70
  71         protected function init( $count, $table ) {
  72                 $this->processed = 0;
  73                 $this->updated = 0;
  74                 $this->count = $count;
  75                 $this->startTime = microtime( true );
  76                 $this->table = $table;
  77         }
  78
  79         /**
  80          * @param int $updated
  81          */
  82         protected function progress( $updated ) {
  83                 $this->updated += $updated;
  84                 $this->processed++;
  85                 if ( $this->processed % $this->reportInterval != 0 ) {
  86                         return;
  87                 }
  88                 $portion = $this->processed / $this->count;
  89                 $updateRate = $this->updated / $this->processed;
  90
  91                 $now = microtime( true );
  92                 $delta = $now - $this->startTime;
  93                 $estimatedTotalTime = $delta / $portion;
  94                 $eta = $this->startTime + $estimatedTotalTime;
  95
  96                 $this->output(
  97                         sprintf( "%s %s: %6.2f%% done on %s; ETA %s [%d/%d] %.2f/sec <%.2f%% updated>\n",
  98                                 WikiMap::getCurrentWikiDbDomain()->getId(),
  99                                 wfTimestamp( TS_DB, intval( $now ) ),
 100                                 $portion * 100.0,
 101                                 $this->table,
 102                                 wfTimestamp( TS_DB, intval( $eta ) ),
 103                                 $this->processed,
 104                                 $this->count,
 105                                 $this->processed / $delta,
 106                                 $updateRate * 100.0
 107                         )
 108                 );
 109                 flush();
 110         }
 111
 112         /**
 113          * @param array $params
 114          * @throws MWException
 115          */
 116         public function runTable( $params ) {
 117                 $dbr = $this->getDB( DB_REPLICA );
 118
 119                 if ( array_diff( array_keys( $params ),
 120                         [ 'table', 'conds', 'index', 'callback' ] )
 121                 ) {
 122                         throw new MWException( __METHOD__ . ': Missing parameter ' . implode( ', ', $params ) );
 123                 }
 124
 125                 $table = $params['table'];
 126                 // count(*) would melt the DB for huge tables, we can estimate here
 127                 $count = $dbr->estimateRowCount( $table, '*', '', __METHOD__ );
 128                 $this->init( $count, $table );
 129                 $this->output( "Processing $table...\n" );
 130
 131                 $index = (array)$params['index'];
 132                 $indexConds = [];
 133                 $options = [
 134                         'ORDER BY' => implode( ',', $index ),
 135                         'LIMIT' => $this->getBatchSize()
 136                 ];
 137                 $callback = [ $this, $params['callback'] ];
 138
 139                 while ( true ) {
 140                         $conds = array_merge( $params['conds'], $indexConds );
 141                         $res = $dbr->select( $table, '*', $conds, __METHOD__, $options );
 142                         if ( !$res->numRows() ) {
 143                                 // Done
 144                                 break;
 145                         }
 146
 147                         foreach ( $res as $row ) {
 148                                 call_user_func( $callback, $row );
 149                         }
 150
 151                         if ( $res->numRows() < $this->getBatchSize() ) {
 152                                 // Done
 153                                 break;
 154                         }
 155
 156                         // Update the conditions to select the next batch.
 157                         // Construct a condition string by starting with the least significant part
 158                         // of the index, and adding more significant parts progressively to the left
 159                         // of the string.
 160                         $nextCond = '';
 161                         foreach ( array_reverse( $index ) as $field ) {
 162                                 $encValue = $dbr->addQuotes( $row->$field );
 163                                 if ( $nextCond === '' ) {
 164                                         $nextCond = "$field > $encValue";
 165                                 } else {
 166                                         $nextCond = "$field > $encValue OR ($field = $encValue AND ($nextCond))";
 167                                 }
 168                         }
 169                         $indexConds = [ $nextCond ];
 170                 }
 171
 172                 $this->output( "Finished $table... $this->updated of $this->processed rows updated\n" );
 173         }
 174
 175         /**
 176          * @param string[] $matches
 177          * @return string
 178          */
 179         protected function hexChar( $matches ) {
 180                 return sprintf( "\\x%02x", ord( $matches[1] ) );
 181         }
 182 }