maintenance/storage/trackBlobs.php

   1 <?php
   2 /**
   3  * Adds blobs from a given external storage cluster to the blob_tracking table.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  * @ingroup Maintenance
  22  */
  23
  24 use MediaWiki\Revision\SlotRecord;
  25 use Wikimedia\Rdbms\DBConnectionError;
  26 use Wikimedia\Rdbms\IExpression;
  27 use Wikimedia\Rdbms\LikeValue;
  28
  29 // @codeCoverageIgnoreStart
  30 require_once __DIR__ . '/../Maintenance.php';
  31 // @codeCoverageIgnoreEnd
  32
  33 class TrackBlobs extends Maintenance {
  34         /** @var string[] */
  35         public $clusters;
  36         /** @var IExpression|null */
  37         public $textClause;
  38         /** @var bool */
  39         public $doBlobOrphans;
  40         /** @var array */
  41         public $trackedBlobs = [];
  42
  43         /** @var int */
  44         public $batchSize = 1000;
  45         /** @var int */
  46         public $reportingInterval = 10;
  47
  48         public function __construct() {
  49                 parent::__construct();
  50
  51                 $this->addArg( 'cluster', 'cluster(s) to scan', true, true );
  52
  53                 $this->addDescription(
  54                         'Adds blobs from a given ES cluster to the blob_tracking table. ' .
  55                         'Automatically deletes the tracking table and starts from the start again when restarted.'
  56                 );
  57         }
  58
  59         public function execute() {
  60                 $this->clusters = $this->parameters->getArgs();
  61                 if ( extension_loaded( 'gmp' ) ) {
  62                         $this->doBlobOrphans = true;
  63                         foreach ( $this->clusters as $cluster ) {
  64                                 $this->trackedBlobs[$cluster] = gmp_init( 0 );
  65                         }
  66                 } else {
  67                         echo "Warning: the gmp extension is needed to find orphan blobs\n";
  68                 }
  69
  70                 $this->checkIntegrity();
  71                 $this->initTrackingTable();
  72                 $this->trackRevisions();
  73                 $this->trackOrphanText();
  74                 if ( $this->doBlobOrphans ) {
  75                         $this->findOrphanBlobs();
  76                 }
  77                 $this->output( "All done.\n" );
  78         }
  79
  80         private function checkIntegrity() {
  81                 echo "Doing integrity check...\n";
  82                 $dbr = $this->getReplicaDB();
  83
  84                 // Scan for HistoryBlobStub objects in the text table (T22757)
  85
  86                 $exists = (bool)$dbr->newSelectQueryBuilder()
  87                         ->select( '1' )
  88                         ->from( 'text' )
  89                         ->where(
  90                                 'old_flags LIKE \'%object%\' AND old_flags NOT LIKE \'%external%\' ' .
  91                                 'AND LOWER(CONVERT(LEFT(old_text,22) USING latin1)) = \'o:15:"historyblobstub"\'' )
  92                         ->caller( __METHOD__ )->fetchField();
  93
  94                 if ( $exists ) {
  95                         echo "Integrity check failed: found HistoryBlobStub objects in your text table.\n" .
  96                                 "This script could destroy these objects if it continued. Run resolveStubs.php\n" .
  97                                 "to fix this.\n";
  98                         exit( 1 );
  99                 }
 100
 101                 echo "Integrity check OK\n";
 102         }
 103
 104         private function initTrackingTable() {
 105                 $dbw = $this->getDB( DB_PRIMARY );
 106                 if ( $dbw->tableExists( 'blob_tracking', __METHOD__ ) ) {
 107                         $dbw->query( 'DROP TABLE ' . $dbw->tableName( 'blob_tracking' ), __METHOD__ );
 108                         $dbw->query( 'DROP TABLE ' . $dbw->tableName( 'blob_orphans' ), __METHOD__ );
 109                 }
 110                 $dbw->sourceFile( __DIR__ . '/blob_tracking.sql' );
 111         }
 112
 113         private function getTextClause() {
 114                 if ( !$this->textClause ) {
 115                         $dbr = $this->getReplicaDB();
 116                         $conds = [];
 117                         foreach ( $this->clusters as $cluster ) {
 118                                 $conds[] = $dbr->expr(
 119                                         'old_text',
 120                                         IExpression::LIKE,
 121                                         new LikeValue( "DB://$cluster/", $dbr->anyString() )
 122                                 );
 123                         }
 124                         $this->textClause = $dbr->orExpr( $conds );
 125                 }
 126
 127                 return $this->textClause;
 128         }
 129
 130         private function interpretPointer( $text ) {
 131                 if ( !preg_match( '!^DB://(\w+)/(\d+)(?:/([0-9a-fA-F]+)|)$!', $text, $m ) ) {
 132                         return false;
 133                 }
 134
 135                 return [
 136                         'cluster' => $m[1],
 137                         'id' => intval( $m[2] ),
 138                         'hash' => $m[3] ?? null
 139                 ];
 140         }
 141
 142         /**
 143          *  Scan the revision table for rows stored in the specified clusters
 144          */
 145         private function trackRevisions() {
 146                 $dbw = $this->getPrimaryDB();
 147                 $dbr = $this->getReplicaDB();
 148
 149                 $textClause = $this->getTextClause();
 150                 $startId = 0;
 151                 $endId = (int)$dbr->newSelectQueryBuilder()
 152                         ->select( 'MAX(rev_id)' )
 153                         ->from( 'revision' )
 154                         ->caller( __METHOD__ )->fetchField();
 155                 $batchesDone = 0;
 156                 $rowsInserted = 0;
 157
 158                 echo "Finding revisions...\n";
 159
 160                 $conds = [
 161                         $textClause,
 162                         $dbr->expr(
 163                                 'old_flags',
 164                                 IExpression::LIKE,
 165                                 new LikeValue( $dbr->anyString(), 'external', $dbr->anyString() )
 166                         )
 167                 ];
 168                 $slotRoleStore = $this->getServiceContainer()->getSlotRoleStore();
 169
 170                 $conds = array_merge( [
 171                         'slot_role_id' => $slotRoleStore->getId( SlotRecord::MAIN ),
 172                         'SUBSTRING(content_address, 1, 3)=' . $dbr->addQuotes( 'tt:' ),
 173                 ], $conds );
 174
 175                 while ( true ) {
 176                         $res = $dbr->newSelectQueryBuilder()
 177                                 ->select( [ 'rev_id', 'rev_page', 'old_id', 'old_flags', 'old_text' ] )
 178                                 ->from( 'revision' )
 179                                 ->join( 'slots', null, 'rev_id=slot_revision_id' )
 180                                 ->join( 'content', null, 'content_id=slot_content_id' )
 181                                 ->join( 'text', null, 'SUBSTRING(content_address, 4)=old_id' )
 182                                 ->where( $dbr->expr( 'rev_id', '>', $startId ) )
 183                                 ->andWhere( $conds )
 184                                 ->orderBy( 'rev_id' )
 185                                 ->limit( $this->batchSize )
 186                                 ->caller( __METHOD__ )->fetchResultSet();
 187                         if ( !$res->numRows() ) {
 188                                 break;
 189                         }
 190
 191                         $insertBatch = [];
 192                         foreach ( $res as $row ) {
 193                                 $startId = (int)$row->rev_id;
 194                                 $info = $this->interpretPointer( $row->old_text );
 195                                 if ( !$info ) {
 196                                         echo "Invalid DB:// URL in rev_id {$row->rev_id}\n";
 197                                         continue;
 198                                 }
 199                                 if ( !in_array( $info['cluster'], $this->clusters ) ) {
 200                                         echo "Invalid cluster returned in SQL query: {$info['cluster']}\n";
 201                                         continue;
 202                                 }
 203                                 $insertBatch[] = [
 204                                         'bt_page' => $row->rev_page,
 205                                         'bt_rev_id' => $row->rev_id,
 206                                         'bt_text_id' => $row->old_id,
 207                                         'bt_cluster' => $info['cluster'],
 208                                         'bt_blob_id' => $info['id'],
 209                                         'bt_cgz_hash' => $info['hash']
 210                                 ];
 211                                 if ( $this->doBlobOrphans ) {
 212                                         gmp_setbit( $this->trackedBlobs[$info['cluster']], $info['id'] );
 213                                 }
 214                         }
 215                         $dbw->newInsertQueryBuilder()
 216                                 ->insertInto( 'blob_tracking' )
 217                                 ->rows( $insertBatch )
 218                                 ->caller( __METHOD__ )->execute();
 219                         $rowsInserted += count( $insertBatch );
 220
 221                         ++$batchesDone;
 222                         if ( $batchesDone >= $this->reportingInterval ) {
 223                                 $batchesDone = 0;
 224                                 echo "$startId / $endId\n";
 225                                 $this->waitForReplication();
 226                         }
 227                 }
 228                 echo "Found $rowsInserted revisions\n";
 229         }
 230
 231         /**
 232          * Scan the text table for orphan text
 233          * Orphan text here does not imply DB corruption -- deleted text tracked by the
 234          * archive table counts as orphan for our purposes.
 235          */
 236         private function trackOrphanText() {
 237                 # Wait until the blob_tracking table is available in the replica DB
 238                 $dbw = $this->getPrimaryDB();
 239                 $dbr = $this->getReplicaDB();
 240                 $this->getServiceContainer()->getDBLoadBalancerFactory()->waitForReplication( [ 'timeout' => 100_000 ] );
 241
 242                 $textClause = $this->getTextClause();
 243                 $startId = 0;
 244                 $endId = (int)$dbr->newSelectQueryBuilder()
 245                         ->select( 'MAX(old_id)' )
 246                         ->from( 'text' )
 247                         ->caller( __METHOD__ )->fetchField();
 248                 $rowsInserted = 0;
 249                 $batchesDone = 0;
 250
 251                 echo "Finding orphan text...\n";
 252
 253                 # Scan the text table for orphan text
 254                 while ( true ) {
 255                         $res = $dbr->newSelectQueryBuilder()
 256                                 ->select( [ 'old_id', 'old_flags', 'old_text' ] )
 257                                 ->from( 'text' )
 258                                 ->leftJoin( 'blob_tracking', null, 'bt_text_id=old_id' )
 259                                 ->where( [
 260                                         $dbr->expr( 'old_id', '>', $startId ),
 261                                         $textClause,
 262                                         $dbr->expr(
 263                                                 'old_flags',
 264                                                 IExpression::LIKE,
 265                                                 new LikeValue( $dbr->anyString(), 'external', $dbr->anyString() )
 266                                         ),
 267                                         'bt_text_id' => null,
 268                                 ] )
 269                                 ->orderBy( 'old_id' )
 270                                 ->limit( $this->batchSize )
 271                                 ->caller( __METHOD__ )->fetchResultSet();
 272
 273                         if ( !$res->numRows() ) {
 274                                 break;
 275                         }
 276
 277                         $insertBatch = [];
 278                         foreach ( $res as $row ) {
 279                                 $startId = (int)$row->old_id;
 280                                 $info = $this->interpretPointer( $row->old_text );
 281                                 if ( !$info ) {
 282                                         echo "Invalid DB:// URL in old_id {$row->old_id}\n";
 283                                         continue;
 284                                 }
 285                                 if ( !in_array( $info['cluster'], $this->clusters ) ) {
 286                                         echo "Invalid cluster returned in SQL query\n";
 287                                         continue;
 288                                 }
 289
 290                                 $insertBatch[] = [
 291                                         'bt_page' => 0,
 292                                         'bt_rev_id' => 0,
 293                                         'bt_text_id' => $row->old_id,
 294                                         'bt_cluster' => $info['cluster'],
 295                                         'bt_blob_id' => $info['id'],
 296                                         'bt_cgz_hash' => $info['hash']
 297                                 ];
 298                                 if ( $this->doBlobOrphans ) {
 299                                         gmp_setbit( $this->trackedBlobs[$info['cluster']], $info['id'] );
 300                                 }
 301                         }
 302                         $dbw->newInsertQueryBuilder()
 303                                 ->insertInto( 'blob_tracking' )
 304                                 ->rows( $insertBatch )
 305                                 ->caller( __METHOD__ )->execute();
 306
 307                         $rowsInserted += count( $insertBatch );
 308                         ++$batchesDone;
 309                         if ( $batchesDone >= $this->reportingInterval ) {
 310                                 $batchesDone = 0;
 311                                 echo "$startId / $endId\n";
 312                                 $this->waitForReplication();
 313                         }
 314                 }
 315                 echo "Found $rowsInserted orphan text rows\n";
 316         }
 317
 318         /**
 319          * Scan the blobs table for rows not registered in blob_tracking (and thus not
 320          * registered in the text table).
 321          *
 322          * Orphan blobs are indicative of DB corruption. They are inaccessible and
 323          * should probably be deleted.
 324          */
 325         private function findOrphanBlobs() {
 326                 if ( !extension_loaded( 'gmp' ) ) {
 327                         echo "Can't find orphan blobs, need bitfield support provided by GMP.\n";
 328
 329                         return;
 330                 }
 331
 332                 $dbw = $this->getPrimaryDB();
 333                 $lbFactory = $this->getServiceContainer()->getDBLoadBalancerFactory();
 334                 $dbStore = $this->getServiceContainer()->getExternalStoreFactory()->getStore( 'DB' );
 335                 '@phan-var ExternalStoreDB $dbStore'; /** @var ExternalStoreDB $dbStore */
 336
 337                 foreach ( $this->clusters as $cluster ) {
 338                         echo "Searching for orphan blobs in $cluster...\n";
 339                         $lb = $lbFactory->getExternalLB( $cluster );
 340                         try {
 341                                 $extDB = $lb->getMaintenanceConnectionRef( DB_REPLICA );
 342                         } catch ( DBConnectionError $e ) {
 343                                 if ( strpos( $e->getMessage(), 'Unknown database' ) !== false ) {
 344                                         echo "No database on $cluster\n";
 345                                 } else {
 346                                         echo "Error on $cluster: " . $e->getMessage() . "\n";
 347                                 }
 348                                 continue;
 349                         }
 350                         $table = $dbStore->getTable( $cluster );
 351                         if ( !$extDB->tableExists( $table, __METHOD__ ) ) {
 352                                 echo "No blobs table on cluster $cluster\n";
 353                                 continue;
 354                         }
 355                         $startId = 0;
 356                         $batchesDone = 0;
 357                         $actualBlobs = gmp_init( 0 );
 358                         $endId = (int)$extDB->newSelectQueryBuilder()
 359                                 ->select( 'MAX(blob_id)' )
 360                                 ->from( $table )
 361                                 ->caller( __METHOD__ )->fetchField();
 362
 363                         // Build a bitmap of actual blob rows
 364                         while ( true ) {
 365                                 $res = $extDB->newSelectQueryBuilder()
 366                                         ->select( [ 'blob_id' ] )
 367                                         ->from( $table )
 368                                         ->where( $extDB->expr( 'blob_id', '>', $startId ) )
 369                                         ->orderBy( 'blob_id' )
 370                                         ->limit( $this->batchSize )
 371                                         ->caller( __METHOD__ )->fetchResultSet();
 372
 373                                 if ( !$res->numRows() ) {
 374                                         break;
 375                                 }
 376
 377                                 foreach ( $res as $row ) {
 378                                         gmp_setbit( $actualBlobs, $row->blob_id );
 379                                         $startId = (int)$row->blob_id;
 380                                 }
 381
 382                                 ++$batchesDone;
 383                                 if ( $batchesDone >= $this->reportingInterval ) {
 384                                         $batchesDone = 0;
 385                                         echo "$startId / $endId\n";
 386                                 }
 387                         }
 388
 389                         // Find actual blobs that weren't tracked by the previous passes
 390                         // This is a set-theoretic difference A \ B, or in bitwise terms, A & ~B
 391                         $orphans = gmp_and( $actualBlobs, gmp_com( $this->trackedBlobs[$cluster] ) );
 392
 393                         // Traverse the orphan list
 394                         $insertBatch = [];
 395                         $id = 0;
 396                         $numOrphans = 0;
 397                         while ( true ) {
 398                                 $id = gmp_scan1( $orphans, $id );
 399                                 if ( $id == -1 ) {
 400                                         break;
 401                                 }
 402                                 $insertBatch[] = [
 403                                         'bo_cluster' => $cluster,
 404                                         'bo_blob_id' => $id
 405                                 ];
 406                                 if ( count( $insertBatch ) > $this->batchSize ) {
 407                                         $dbw->newInsertQueryBuilder()
 408                                                 ->insertInto( 'blob_orphans' )
 409                                                 ->rows( $insertBatch )
 410                                                 ->caller( __METHOD__ )->execute();
 411                                         $insertBatch = [];
 412                                 }
 413
 414                                 ++$id;
 415                                 ++$numOrphans;
 416                         }
 417                         if ( $insertBatch ) {
 418                                 $dbw->newInsertQueryBuilder()
 419                                         ->insertInto( 'blob_orphans' )
 420                                         ->rows( $insertBatch )
 421                                         ->caller( __METHOD__ )->execute();
 422                         }
 423                         echo "Found $numOrphans orphan(s) in $cluster\n";
 424                 }
 425         }
 426 }
 427
 428 // @codeCoverageIgnoreStart
 429 $maintClass = TrackBlobs::class;
 430 require_once RUN_MAINTENANCE_IF_MAIN;
 431 // @codeCoverageIgnoreEnd