maintenance/recountCategories.php

   1 <?php
   2 /**
   3  * Refreshes category counts.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  * @ingroup Maintenance
  22  */
  23
  24 require_once __DIR__ . '/Maintenance.php';
  25
  26 use MediaWiki\MediaWikiServices;
  27
  28 /**
  29  * Maintenance script that refreshes category membership counts in the category
  30  * table.
  31  *
  32  * (The populateCategory.php script will also recalculate counts, but
  33  * recountCategories only updates rows that need to be updated, making it more
  34  * efficient.)
  35  *
  36  * @ingroup Maintenance
  37  */
  38 class RecountCategories extends Maintenance {
  39         /** @var int */
  40         private $minimumId;
  41
  42         public function __construct() {
  43                 parent::__construct();
  44                 $this->addDescription( <<<'TEXT'
  45 This script refreshes the category membership counts stored in the category
  46 table. As time passes, these counts often drift from the actual number of
  47 category members. The script identifies rows where the value in the category
  48 table does not match the number of categorylinks rows for that category, and
  49 updates the category table accordingly.
  50
  51 To fully refresh the data in the category table, you need to run this script
  52 for all three modes. Alternatively, just one mode can be run if required.
  53 TEXT
  54                 );
  55                 $this->addOption(
  56                         'mode',
  57                         '(REQUIRED) Which category count column to recompute: "pages", "subcats", "files" or "all".',
  58                         true,
  59                         true
  60                 );
  61                 $this->addOption(
  62                         'begin',
  63                         'Only recount categories with cat_id greater than the given value',
  64                         false,
  65                         true
  66                 );
  67                 $this->addOption(
  68                         'throttle',
  69                         'Wait this many milliseconds after each batch. Default: 0',
  70                         false,
  71                         true
  72                 );
  73
  74                 $this->addOption(
  75                         'skip-cleanup',
  76                         'Skip running cleanupEmptyCategories if the "page" mode is selected',
  77                         false,
  78                         false
  79                 );
  80
  81                 $this->setBatchSize( 500 );
  82         }
  83
  84         public function execute() {
  85                 $originalMode = $this->getOption( 'mode' );
  86                 if ( !in_array( $originalMode, [ 'pages', 'subcats', 'files', 'all' ] ) ) {
  87                         $this->fatalError( 'Please specify a valid mode: one of "pages", "subcats", "files" or "all".' );
  88                 }
  89
  90                 if ( $originalMode === 'all' ) {
  91                         $modes = [ 'pages', 'subcats', 'files' ];
  92                 } else {
  93                         $modes = [ $originalMode ];
  94                 }
  95
  96                 foreach ( $modes as $mode ) {
  97                         $this->output( "Starting to recount {$mode} counts.\n" );
  98                         $this->minimumId = intval( $this->getOption( 'begin', 0 ) );
  99
 100                         // do the work, batch by batch
 101                         $affectedRows = 0;
 102                         while ( ( $result = $this->doWork( $mode ) ) !== false ) {
 103                                 $affectedRows += $result;
 104                                 usleep( $this->getOption( 'throttle', 0 ) * 1000 );
 105                         }
 106
 107                         $this->output( "Updated the {$mode} counts of $affectedRows categories.\n" );
 108                 }
 109
 110                 // Finished
 111                 $this->output( "Done!\n" );
 112                 if ( $originalMode !== 'all' ) {
 113                         $this->output( "Now run the script using the other --mode options if you haven't already.\n" );
 114                 }
 115
 116                 if ( in_array( 'pages', $modes ) ) {
 117                         if ( $this->hasOption( 'skip-cleanup' ) ) {
 118                                 $this->output(
 119                                         "Also run 'php cleanupEmptyCategories.php --mode remove' to remove empty,\n" .
 120                                         "nonexistent categories from the category table.\n\n" );
 121                         } else {
 122                                 $this->output( "Running cleanupEmptyCategories.php\n" );
 123                                 $cleanup = $this->runChild( CleanupEmptyCategories::class );
 124                                 '@phan-var CleanupEmptyCategories $cleanup';
 125                                 // Pass no options into the child because of a parameter collision between "mode", which
 126                                 // both scripts use but set to different values. We'll just use the defaults.
 127                                 $cleanup->loadParamsAndArgs( $this->mSelf, [], [] );
 128                                 // Force execution because we want to run it regardless of whether it's been run before.
 129                                 $cleanup->setForce( true );
 130                                 $cleanup->execute();
 131                         }
 132                 }
 133         }
 134
 135         protected function doWork( $mode ) {
 136                 $this->output( "Finding up to {$this->getBatchSize()} drifted rows " .
 137                         "greater than cat_id {$this->minimumId}...\n" );
 138
 139                 $countingConds = [ 'cl_to = cat_title' ];
 140                 if ( $mode === 'subcats' ) {
 141                         $countingConds['cl_type'] = 'subcat';
 142                 } elseif ( $mode === 'files' ) {
 143                         $countingConds['cl_type'] = 'file';
 144                 }
 145
 146                 $dbr = $this->getDB( DB_REPLICA, 'vslow' );
 147                 $countingSubquery = $dbr->selectSQLText( 'categorylinks',
 148                         'COUNT(*)',
 149                         $countingConds,
 150                         __METHOD__ );
 151
 152                 // First, let's find out which categories have drifted and need to be updated.
 153                 // The query counts the categorylinks for each category on the replica DB,
 154                 // but this data can't be used for updating the master, so we don't include it
 155                 // in the results.
 156                 $idsToUpdate = $dbr->selectFieldValues( 'category',
 157                         'cat_id',
 158                         [
 159                                 'cat_id > ' . (int)$this->minimumId,
 160                                 "cat_{$mode} != ($countingSubquery)"
 161                         ],
 162                         __METHOD__,
 163                         [ 'LIMIT' => $this->getBatchSize() ]
 164                 );
 165                 if ( !$idsToUpdate ) {
 166                         return false;
 167                 }
 168                 $this->output( "Updating cat_{$mode} field on " .
 169                         count( $idsToUpdate ) . " rows...\n" );
 170
 171                 // In the next batch, start where this query left off. The rows selected
 172                 // in this iteration shouldn't be selected again after being updated, but
 173                 // we still keep track of where we are up to, as extra protection against
 174                 // infinite loops.
 175                 $this->minimumId = end( $idsToUpdate );
 176
 177                 // Now, on master, find the correct counts for these categories.
 178                 $dbw = $this->getDB( DB_PRIMARY );
 179                 $res = $dbw->select( 'category',
 180                         [ 'cat_id', 'count' => "($countingSubquery)" ],
 181                         [ 'cat_id' => $idsToUpdate ],
 182                         __METHOD__ );
 183
 184                 // Update the category counts on the rows we just identified.
 185                 // This logic is equivalent to Category::refreshCounts, except here, we
 186                 // don't remove rows when cat_pages is zero and the category description page
 187                 // doesn't exist - instead we print a suggestion to run
 188                 // cleanupEmptyCategories.php.
 189                 $affectedRows = 0;
 190                 foreach ( $res as $row ) {
 191                         $dbw->update( 'category',
 192                                 [ "cat_{$mode}" => $row->count ],
 193                                 [
 194                                         'cat_id' => $row->cat_id,
 195                                         "cat_{$mode} != " . (int)( $row->count ),
 196                                 ],
 197                                 __METHOD__ );
 198                         $affectedRows += $dbw->affectedRows();
 199                 }
 200
 201                 MediaWikiServices::getInstance()->getDBLoadBalancerFactory()->waitForReplication();
 202
 203                 return $affectedRows;
 204         }
 205 }
 206
 207 $maintClass = RecountCategories::class;
 208 require_once RUN_MAINTENANCE_IF_MAIN;