maintenance/runBatchedQuery.php

   1 <?php
   2 /**
   3  * Run a database query in batches and wait for replica DBs. This is used on large
   4  * wikis to prevent replication lag from going through the roof when executing
   5  * large write queries.
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License as published by
   9  * the Free Software Foundation; either version 2 of the License, or
  10  * (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License along
  18  * with this program; if not, write to the Free Software Foundation, Inc.,
  19  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  20  * http://www.gnu.org/copyleft/gpl.html
  21  *
  22  * @file
  23  * @ingroup Maintenance
  24  */
  25
  26 require_once __DIR__ . '/Maintenance.php';
  27
  28 /**
  29  * Maintenance script to run a database query in batches and wait for replica DBs.
  30  *
  31  * @ingroup Maintenance
  32  */
  33 class BatchedQueryRunner extends Maintenance {
  34         public function __construct() {
  35                 parent::__construct();
  36                 $this->addDescription(
  37                         "Run an update query on all rows of a table. " .
  38                         "Waits for replicas at appropriate intervals." );
  39                 $this->addOption( 'table', 'The table name', true, true );
  40                 $this->addOption( 'set', 'The SET clause', true, true );
  41                 $this->addOption( 'where', 'The WHERE clause', false, true );
  42                 $this->addOption( 'key', 'A column name, the values of which are unique', true, true );
  43                 $this->addOption( 'batch-size', 'The batch size (default 1000)', false, true );
  44                 $this->addOption( 'db', 'The database name, or omit to use the current wiki.', false, true );
  45         }
  46
  47         public function execute() {
  48                 $table = $this->getOption( 'table' );
  49                 $key = $this->getOption( 'key' );
  50                 $set = $this->getOption( 'set' );
  51                 $where = $this->getOption( 'where', null );
  52                 $where = $where === null ? [] : [ $where ];
  53                 $batchSize = $this->getOption( 'batch-size', 1000 );
  54
  55                 $dbName = $this->getOption( 'db', null );
  56                 if ( $dbName === null ) {
  57                         $dbw = $this->getDB( DB_MASTER );
  58                 } else {
  59                         $lbf = MediaWiki\MediaWikiServices::getInstance()->getDBLoadBalancerFactory();
  60                         $lb = $lbf->getMainLB( $dbName );
  61                         $dbw = $lb->getConnection( DB_MASTER, [], $dbName );
  62                 }
  63
  64                 $selectConds = $where;
  65                 $prevEnd = false;
  66
  67                 $n = 1;
  68                 do {
  69                         $this->output( "Batch $n: " );
  70                         $n++;
  71
  72                         // Note that the update conditions do not rely on atomicity of the
  73                         // SELECT query in order to guarantee that all rows are updated. The
  74                         // results of the SELECT are merely a partitioning hint. Simultaneous
  75                         // updates merely result in the wrong number of rows being updated
  76                         // in a batch.
  77
  78                         $res = $dbw->select( $table, $key, $selectConds, __METHOD__,
  79                                 [ 'ORDER BY' => $key, 'LIMIT' => $batchSize ] );
  80                         if ( $res->numRows() ) {
  81                                 $res->seek( $res->numRows() - 1 );
  82                                 $row = $res->fetchObject();
  83                                 $end = $dbw->addQuotes( $row->$key );
  84                                 $selectConds = array_merge( $where, [ "$key > $end" ] );
  85                                 $updateConds = array_merge( $where, [ "$key <= $end" ] );
  86                         } else {
  87                                 $updateConds = $where;
  88                         }
  89                         if ( $prevEnd !== false ) {
  90                                 $updateConds = array_merge( [ "$key > $prevEnd" ], $updateConds );
  91                         }
  92
  93                         $query = "UPDATE " . $dbw->tableName( $table ) .
  94                                 " SET " . $set .
  95                                 " WHERE " . $dbw->makeList( $updateConds, IDatabase::LIST_AND );
  96
  97                         $dbw->query( $query, __METHOD__ );
  98
  99                         $prevEnd = $end;
 100
 101                         $affected = $dbw->affectedRows();
 102                         $this->output( "$affected rows affected\n" );
 103                         wfWaitForSlaves();
 104                 } while ( $res->numRows() );
 105         }
 106
 107         public function getDbType() {
 108                 return Maintenance::DB_ADMIN;
 109         }
 110 }
 111
 112 $maintClass = "BatchedQueryRunner";
 113 require_once RUN_MAINTENANCE_IF_MAIN;