maintenance/runBatchedQuery.php

   1 <?php
   2 /**
   3  * Run a database query in batches and wait for replica DBs. This is used on large
   4  * wikis to prevent replication lag from going through the roof when executing
   5  * large write queries.
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License as published by
   9  * the Free Software Foundation; either version 2 of the License, or
  10  * (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License along
  18  * with this program; if not, write to the Free Software Foundation, Inc.,
  19  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  20  * http://www.gnu.org/copyleft/gpl.html
  21  *
  22  * @file
  23  * @ingroup Maintenance
  24  */
  25
  26 // @codeCoverageIgnoreStart
  27 require_once __DIR__ . '/Maintenance.php';
  28 // @codeCoverageIgnoreEnd
  29
  30 use MediaWiki\Maintenance\Maintenance;
  31 use Wikimedia\Rdbms\Platform\ISQLPlatform;
  32
  33 /**
  34  * Maintenance script to run a database query in batches and wait for replica DBs.
  35  *
  36  * @ingroup Maintenance
  37  */
  38 class RunBatchedQuery extends Maintenance {
  39         public function __construct() {
  40                 parent::__construct();
  41                 $this->addDescription(
  42                         "Run an update query on all rows of a table. " .
  43                         "Waits for replicas at appropriate intervals." );
  44                 $this->addOption( 'table', 'The table name', true, true );
  45                 $this->addOption( 'set', 'The SET clause', true, true );
  46                 $this->addOption( 'where', 'The WHERE clause', false, true );
  47                 $this->addOption( 'key', 'A column name, the values of which are unique', true, true );
  48                 $this->addOption( 'batch-size', 'The batch size (default 1000)', false, true );
  49                 $this->addOption( 'db', 'The database name, or omit to use the current wiki.', false, true );
  50         }
  51
  52         public function execute() {
  53                 $table = $this->getOption( 'table' );
  54                 $key = $this->getOption( 'key' );
  55                 $set = $this->getOption( 'set' );
  56                 $where = $this->getOption( 'where', null );
  57                 $where = $where === null ? [] : [ $where ];
  58                 $batchSize = $this->getOption( 'batch-size', 1000 );
  59
  60                 $dbName = $this->getOption( 'db', null );
  61                 if ( $dbName === null ) {
  62                         $dbw = $this->getPrimaryDB();
  63                 } else {
  64                         $dbw = $this->getServiceContainer()->getConnectionProvider()->getPrimaryDatabase( $dbName );
  65                 }
  66
  67                 $selectConds = $where;
  68                 $prevEnd = false;
  69
  70                 $n = 1;
  71                 do {
  72                         $this->output( "Batch $n: " );
  73                         $n++;
  74
  75                         // Note that the update conditions do not rely on the atomicity of the
  76                         // SELECT query in order to guarantee that all rows are updated. The
  77                         // results of the SELECT are merely a partitioning hint. Simultaneous
  78                         // updates merely result in the wrong number of rows being updated
  79                         // in a batch.
  80
  81                         $res = $dbw->newSelectQueryBuilder()
  82                                 ->select( $key )
  83                                 ->from( $table )
  84                                 ->where( $selectConds )
  85                                 ->orderBy( $key )
  86                                 ->limit( $batchSize )
  87                                 ->caller( __METHOD__ )
  88                                 ->fetchResultSet();
  89
  90                         if ( $res->numRows() ) {
  91                                 $res->seek( $res->numRows() - 1 );
  92                                 $row = $res->fetchObject();
  93                                 $end = $row->$key;
  94                                 $selectConds = array_merge( $where, [ $dbw->expr( $key, '>', $end ) ] );
  95                                 $updateConds = array_merge( $where, [ $dbw->expr( $key, '<=', $end ) ] );
  96                         } else {
  97                                 $updateConds = $where;
  98                                 $end = false;
  99                         }
 100                         if ( $prevEnd !== false ) {
 101                                 $updateConds = array_merge( [ $dbw->expr( $key, '>', $prevEnd ) ], $updateConds );
 102                         }
 103
 104                         $query = "UPDATE " . $dbw->tableName( $table ) .
 105                                 " SET " . $set .
 106                                 " WHERE " . $dbw->makeList( $updateConds, ISQLPlatform::LIST_AND );
 107
 108                         $dbw->query( $query, __METHOD__ );
 109
 110                         $prevEnd = $end;
 111
 112                         $affected = $dbw->affectedRows();
 113                         $this->output( "$affected rows affected\n" );
 114                         $this->waitForReplication();
 115                 } while ( $res->numRows() );
 116         }
 117
 118         public function getDbType() {
 119                 return Maintenance::DB_ADMIN;
 120         }
 121 }
 122
 123 // @codeCoverageIgnoreStart
 124 $maintClass = RunBatchedQuery::class;
 125 require_once RUN_MAINTENANCE_IF_MAIN;
 126 // @codeCoverageIgnoreEnd