maintenance/migrateComments.php

   1 <?php
   2 /**
   3  * Migrate comments from pre-1.30 columns to the 'comment' table
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  * @ingroup Maintenance
  22  */
  23
  24 use MediaWiki\MediaWikiServices;
  25 use Wikimedia\Rdbms\IDatabase;
  26
  27 require_once __DIR__ . '/Maintenance.php';
  28
  29 /**
  30  * Maintenance script that migrates comments from pre-1.30 columns to the
  31  * 'comment' table
  32  *
  33  * @ingroup Maintenance
  34  */
  35 class MigrateComments extends LoggedUpdateMaintenance {
  36         public function __construct() {
  37                 parent::__construct();
  38                 $this->addDescription( 'Migrates comments from pre-1.30 columns to the \'comment\' table' );
  39                 $this->setBatchSize( 100 );
  40         }
  41
  42         protected function getUpdateKey() {
  43                 return __CLASS__;
  44         }
  45
  46         protected function updateSkippedMessage() {
  47                 return 'comments already migrated.';
  48         }
  49
  50         protected function doDBUpdates() {
  51                 $this->migrateToTemp(
  52                         'revision', 'rev_id', 'rev_comment', 'revcomment_rev', 'revcomment_comment_id'
  53                 );
  54                 $this->migrate( 'archive', 'ar_id', 'ar_comment' );
  55                 $this->migrate( 'ipblocks', 'ipb_id', 'ipb_reason' );
  56                 $this->migrate( 'image', 'img_name', 'img_description' );
  57                 $this->migrate( 'oldimage', [ 'oi_name', 'oi_timestamp' ], 'oi_description' );
  58                 $this->migrate( 'filearchive', 'fa_id', 'fa_deleted_reason' );
  59                 $this->migrate( 'filearchive', 'fa_id', 'fa_description' );
  60                 $this->migrate( 'recentchanges', 'rc_id', 'rc_comment' );
  61                 $this->migrate( 'logging', 'log_id', 'log_comment' );
  62                 $this->migrate( 'protected_titles', [ 'pt_namespace', 'pt_title' ], 'pt_reason' );
  63                 return true;
  64         }
  65
  66         /**
  67          * Fetch comment IDs for a set of comments
  68          * @param IDatabase $dbw
  69          * @param array &$comments Keys are comment names, values will be set to IDs.
  70          * @return int Count of added comments
  71          */
  72         private function loadCommentIDs( IDatabase $dbw, array &$comments ) {
  73                 $count = 0;
  74                 $needComments = $comments;
  75
  76                 while ( true ) {
  77                         $where = [];
  78                         foreach ( $needComments as $need => $dummy ) {
  79                                 $where[] = $dbw->makeList(
  80                                         [
  81                                                 'comment_hash' => CommentStore::hash( $need, null ),
  82                                                 'comment_text' => $need,
  83                                         ],
  84                                         LIST_AND
  85                                 );
  86                         }
  87
  88                         $res = $dbw->select(
  89                                 'comment',
  90                                 [ 'comment_id', 'comment_text' ],
  91                                 [
  92                                         $dbw->makeList( $where, LIST_OR ),
  93                                         'comment_data' => null,
  94                                 ],
  95                                 __METHOD__
  96                         );
  97                         foreach ( $res as $row ) {
  98                                 $comments[$row->comment_text] = $row->comment_id;
  99                                 unset( $needComments[$row->comment_text] );
 100                         }
 101
 102                         if ( !$needComments ) {
 103                                 break;
 104                         }
 105
 106                         $dbw->insert(
 107                                 'comment',
 108                                 array_map( function ( $v ) {
 109                                         return [
 110                                                 'comment_hash' => CommentStore::hash( $v, null ),
 111                                                 'comment_text' => $v,
 112                                         ];
 113                                 }, array_keys( $needComments ) ),
 114                                 __METHOD__
 115                         );
 116                         $count += $dbw->affectedRows();
 117                 }
 118                 return $count;
 119         }
 120
 121         /**
 122          * Migrate comments in a table.
 123          *
 124          * Assumes any row with the ID field non-zero have already been migrated.
 125          * Assumes the new field name is the same as the old with '_id' appended.
 126          * Blanks the old fields while migrating.
 127          *
 128          * @param string $table Table to migrate
 129          * @param string|string[] $primaryKey Primary key of the table.
 130          * @param string $oldField Old comment field name
 131          */
 132         protected function migrate( $table, $primaryKey, $oldField ) {
 133                 $dbw = $this->getDB( DB_MASTER );
 134                 if ( !$dbw->fieldExists( $table, $oldField, __METHOD__ ) ) {
 135                         $this->output( "No need to migrate $table.$oldField, field does not exist\n" );
 136                         return;
 137                 }
 138
 139                 $newField = $oldField . '_id';
 140                 $primaryKey = (array)$primaryKey;
 141                 $pkFilter = array_flip( $primaryKey );
 142                 $this->output( "Beginning migration of $table.$oldField to $table.$newField\n" );
 143                 $lbFactory = MediaWikiServices::getInstance()->getDBLoadBalancerFactory();
 144                 $lbFactory->waitForReplication();
 145
 146                 $next = '1=1';
 147                 $countUpdated = 0;
 148                 $countComments = 0;
 149                 while ( true ) {
 150                         // Fetch the rows needing update
 151                         $res = $dbw->select(
 152                                 $table,
 153                                 array_merge( $primaryKey, [ $oldField ] ),
 154                                 [
 155                                         $newField => 0,
 156                                         $next,
 157                                 ],
 158                                 __METHOD__,
 159                                 [
 160                                         'ORDER BY' => $primaryKey,
 161                                         'LIMIT' => $this->getBatchSize(),
 162                                 ]
 163                         );
 164                         if ( !$res->numRows() ) {
 165                                 break;
 166                         }
 167
 168                         // Collect the distinct comments from those rows
 169                         $comments = [];
 170                         foreach ( $res as $row ) {
 171                                 $comments[$row->$oldField] = 0;
 172                         }
 173                         $countComments += $this->loadCommentIDs( $dbw, $comments );
 174
 175                         // Update the existing rows
 176                         foreach ( $res as $row ) {
 177                                 $dbw->update(
 178                                         $table,
 179                                         [
 180                                                 $newField => $comments[$row->$oldField],
 181                                                 $oldField => '',
 182                                         ],
 183                                         array_intersect_key( (array)$row, $pkFilter ) + [
 184                                                 $newField => 0
 185                                         ],
 186                                         __METHOD__
 187                                 );
 188                                 $countUpdated += $dbw->affectedRows();
 189                         }
 190
 191                         // Calculate the "next" condition
 192                         $next = '';
 193                         $prompt = [];
 194                         for ( $i = count( $primaryKey ) - 1; $i >= 0; $i-- ) {
 195                                 $field = $primaryKey[$i];
 196                                 $prompt[] = $row->$field;
 197                                 $value = $dbw->addQuotes( $row->$field );
 198                                 if ( $next === '' ) {
 199                                         $next = "$field > $value";
 200                                 } else {
 201                                         $next = "$field > $value OR $field = $value AND ($next)";
 202                                 }
 203                         }
 204                         $prompt = implode( ' ', array_reverse( $prompt ) );
 205                         $this->output( "... $prompt\n" );
 206                         $lbFactory->waitForReplication();
 207                 }
 208
 209                 $this->output(
 210                         "Completed migration, updated $countUpdated row(s) with $countComments new comment(s)\n"
 211                 );
 212         }
 213
 214         /**
 215          * Migrate comments in a table to a temporary table.
 216          *
 217          * Assumes any row with the ID field non-zero have already been migrated.
 218          * Assumes the new table is named "{$table}_comment_temp", and it has two
 219          * columns, in order, being the primary key of the original table and the
 220          * comment ID field.
 221          * Blanks the old fields while migrating.
 222          *
 223          * @param string $table Table to migrate
 224          * @param string $primaryKey Primary key of the table.
 225          * @param string $oldField Old comment field name
 226          * @param string $newPrimaryKey Primary key of the new table.
 227          * @param string $newField New comment field name
 228          */
 229         protected function migrateToTemp( $table, $primaryKey, $oldField, $newPrimaryKey, $newField ) {
 230                 $dbw = $this->getDB( DB_MASTER );
 231                 if ( !$dbw->fieldExists( $table, $oldField, __METHOD__ ) ) {
 232                         $this->output( "No need to migrate $table.$oldField, field does not exist\n" );
 233                         return;
 234                 }
 235
 236                 $newTable = $table . '_comment_temp';
 237                 $this->output( "Beginning migration of $table.$oldField to $newTable.$newField\n" );
 238                 MediaWikiServices::getInstance()->getDBLoadBalancerFactory()->waitForReplication();
 239
 240                 $dbw = $this->getDB( DB_MASTER );
 241                 $next = [];
 242                 $countUpdated = 0;
 243                 $countComments = 0;
 244                 while ( true ) {
 245                         // Fetch the rows needing update
 246                         $res = $dbw->select(
 247                                 [ $table, $newTable ],
 248                                 [ $primaryKey, $oldField ],
 249                                 // @phan-suppress-next-line PhanSuspiciousBinaryAddLists
 250                                 [ $newPrimaryKey => null ] + $next,
 251                                 __METHOD__,
 252                                 [
 253                                         'ORDER BY' => $primaryKey,
 254                                         'LIMIT' => $this->getBatchSize(),
 255                                 ],
 256                                 [ $newTable => [ 'LEFT JOIN', "{$primaryKey}={$newPrimaryKey}" ] ]
 257                         );
 258                         if ( !$res->numRows() ) {
 259                                 break;
 260                         }
 261
 262                         // Collect the distinct comments from those rows
 263                         $comments = [];
 264                         foreach ( $res as $row ) {
 265                                 $comments[$row->$oldField] = 0;
 266                         }
 267                         $countComments += $this->loadCommentIDs( $dbw, $comments );
 268
 269                         // Update rows
 270                         $inserts = [];
 271                         $updates = [];
 272                         foreach ( $res as $row ) {
 273                                 $inserts[] = [
 274                                         $newPrimaryKey => $row->$primaryKey,
 275                                         $newField => $comments[$row->$oldField]
 276                                 ];
 277                                 $updates[] = $row->$primaryKey;
 278                         }
 279                         $this->beginTransaction( $dbw, __METHOD__ );
 280                         $dbw->insert( $newTable, $inserts, __METHOD__ );
 281                         $dbw->update( $table, [ $oldField => '' ], [ $primaryKey => $updates ], __METHOD__ );
 282                         $countUpdated += $dbw->affectedRows();
 283                         $this->commitTransaction( $dbw, __METHOD__ );
 284
 285                         // Calculate the "next" condition
 286                         $next = [ $primaryKey . ' > ' . $dbw->addQuotes( $row->$primaryKey ) ];
 287                         $this->output( "... {$row->$primaryKey}\n" );
 288                 }
 289
 290                 $this->output(
 291                         "Completed migration, updated $countUpdated row(s) with $countComments new comment(s)\n"
 292                 );
 293         }
 294 }
 295
 296 $maintClass = MigrateComments::class;
 297 require_once RUN_MAINTENANCE_IF_MAIN;