maintenance/runBatchedQuery.php

   1 <?php
   2 /**
   3  * Run a database query in batches and wait for replica DBs. This is used on large
   4  * wikis to prevent replication lag from going through the roof when executing
   5  * large write queries.
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License as published by
   9  * the Free Software Foundation; either version 2 of the License, or
  10  * (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License along
  18  * with this program; if not, write to the Free Software Foundation, Inc.,
  19  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  20  * http://www.gnu.org/copyleft/gpl.html
  21  *
  22  * @file
  23  * @ingroup Maintenance
  24  */
  25
  26 require_once __DIR__ . '/Maintenance.php';
  27
  28 use Wikimedia\Rdbms\IDatabase;
  29
  30 /**
  31  * Maintenance script to run a database query in batches and wait for replica DBs.
  32  *
  33  * @ingroup Maintenance
  34  */
  35 class BatchedQueryRunner extends Maintenance {
  36         public function __construct() {
  37                 parent::__construct();
  38                 $this->addDescription(
  39                         "Run an update query on all rows of a table. " .
  40                         "Waits for replicas at appropriate intervals." );
  41                 $this->addOption( 'table', 'The table name', true, true );
  42                 $this->addOption( 'set', 'The SET clause', true, true );
  43                 $this->addOption( 'where', 'The WHERE clause', false, true );
  44                 $this->addOption( 'key', 'A column name, the values of which are unique', true, true );
  45                 $this->addOption( 'batch-size', 'The batch size (default 1000)', false, true );
  46                 $this->addOption( 'db', 'The database name, or omit to use the current wiki.', false, true );
  47         }
  48
  49         public function execute() {
  50                 $table = $this->getOption( 'table' );
  51                 $key = $this->getOption( 'key' );
  52                 $set = $this->getOption( 'set' );
  53                 $where = $this->getOption( 'where', null );
  54                 $where = $where === null ? [] : [ $where ];
  55                 $batchSize = $this->getOption( 'batch-size', 1000 );
  56
  57                 $dbName = $this->getOption( 'db', null );
  58                 if ( $dbName === null ) {
  59                         $dbw = $this->getDB( DB_MASTER );
  60                 } else {
  61                         $lbf = MediaWiki\MediaWikiServices::getInstance()->getDBLoadBalancerFactory();
  62                         $lb = $lbf->getMainLB( $dbName );
  63                         $dbw = $lb->getConnection( DB_MASTER, [], $dbName );
  64                 }
  65
  66                 $selectConds = $where;
  67                 $prevEnd = false;
  68
  69                 $n = 1;
  70                 do {
  71                         $this->output( "Batch $n: " );
  72                         $n++;
  73
  74                         // Note that the update conditions do not rely on atomicity of the
  75                         // SELECT query in order to guarantee that all rows are updated. The
  76                         // results of the SELECT are merely a partitioning hint. Simultaneous
  77                         // updates merely result in the wrong number of rows being updated
  78                         // in a batch.
  79
  80                         $res = $dbw->select( $table, $key, $selectConds, __METHOD__,
  81                                 [ 'ORDER BY' => $key, 'LIMIT' => $batchSize ] );
  82                         if ( $res->numRows() ) {
  83                                 $res->seek( $res->numRows() - 1 );
  84                                 $row = $res->fetchObject();
  85                                 $end = $dbw->addQuotes( $row->$key );
  86                                 $selectConds = array_merge( $where, [ "$key > $end" ] );
  87                                 $updateConds = array_merge( $where, [ "$key <= $end" ] );
  88                         } else {
  89                                 $updateConds = $where;
  90                         }
  91                         if ( $prevEnd !== false ) {
  92                                 $updateConds = array_merge( [ "$key > $prevEnd" ], $updateConds );
  93                         }
  94
  95                         $query = "UPDATE " . $dbw->tableName( $table ) .
  96                                 " SET " . $set .
  97                                 " WHERE " . $dbw->makeList( $updateConds, IDatabase::LIST_AND );
  98
  99                         $dbw->query( $query, __METHOD__ );
 100
 101                         $prevEnd = $end;
 102
 103                         $affected = $dbw->affectedRows();
 104                         $this->output( "$affected rows affected\n" );
 105                         wfWaitForSlaves();
 106                 } while ( $res->numRows() );
 107         }
 108
 109         public function getDbType() {
 110                 return Maintenance::DB_ADMIN;
 111         }
 112 }
 113
 114 $maintClass = "BatchedQueryRunner";
 115 require_once RUN_MAINTENANCE_IF_MAIN;