maintenance/rebuildtextindex.php

   1 <?php
   2 /**
   3  * This program is free software; you can redistribute it and/or modify
   4  * it under the terms of the GNU General Public License as published by
   5  * the Free Software Foundation; either version 2 of the License, or
   6  * (at your option) any later version.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  11  * GNU General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along
  14  * with this program; if not, write to the Free Software Foundation, Inc.,
  15  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  16  * http://www.gnu.org/copyleft/gpl.html
  17  *
  18  * @file
  19  */
  20
  21 // @codeCoverageIgnoreStart
  22 require_once __DIR__ . '/Maintenance.php';
  23 // @codeCoverageIgnoreEnd
  24
  25 use MediaWiki\Maintenance\Maintenance;
  26 use MediaWiki\Revision\SlotRecord;
  27 use MediaWiki\Search\SearchUpdate;
  28 use MediaWiki\Title\Title;
  29 use Wikimedia\Rdbms\DatabaseSqlite;
  30
  31 /**
  32  * Rebuild search index table from scratch.
  33  *
  34  * This may take several hours, depending on the database size and server configuration.
  35  * Postgres is trigger-based and should never need rebuilding.
  36  *
  37  * @ingroup Search
  38  * @ingroup Maintenance
  39  */
  40 class RebuildTextIndex extends Maintenance {
  41         private const RTI_CHUNK_SIZE = 500;
  42
  43         public function __construct() {
  44                 parent::__construct();
  45                 $this->addDescription( 'Rebuild search index table from scratch' );
  46         }
  47
  48         public function getDbType() {
  49                 return Maintenance::DB_ADMIN;
  50         }
  51
  52         public function execute() {
  53                 // Shouldn't be needed for Postgres
  54                 $dbw = $this->getPrimaryDB();
  55                 if ( $dbw->getType() == 'postgres' ) {
  56                         $this->fatalError( "This script is not needed when using Postgres.\n" );
  57                 }
  58
  59                 if ( $dbw->getType() == 'sqlite' ) {
  60                         if ( !DatabaseSqlite::getFulltextSearchModule() ) {
  61                                 $this->fatalError( "Your version of SQLite module for PHP doesn't "
  62                                         . "support full-text search (FTS3).\n" );
  63                         }
  64                 }
  65
  66                 if ( $dbw->getType() == 'mysql' ) {
  67                         $this->dropMysqlTextIndex();
  68                         $this->clearSearchIndex();
  69                         $this->populateSearchIndex();
  70                         $this->createMysqlTextIndex();
  71                 } else {
  72                         $this->clearSearchIndex();
  73                         $this->populateSearchIndex();
  74                 }
  75
  76                 $this->output( "Done.\n" );
  77         }
  78
  79         /**
  80          * Populates the search index with content from all pages
  81          */
  82         protected function populateSearchIndex() {
  83                 $dbw = $this->getPrimaryDB();
  84                 $res = $dbw->newSelectQueryBuilder()
  85                         ->select( [ 'count' => 'MAX(page_id)' ] )
  86                         ->from( 'page' )
  87                         ->caller( __METHOD__ )->fetchResultSet();
  88                 $s = $res->fetchObject();
  89                 $count = $s->count;
  90                 $this->output( "Rebuilding index fields for {$count} pages...\n" );
  91                 $n = 0;
  92
  93                 $revStore = $this->getServiceContainer()->getRevisionStore();
  94                 $queryBuilderTemplate = $revStore->newSelectQueryBuilder( $dbw )
  95                         ->joinPage()
  96                         ->joinComment();
  97
  98                 while ( $n < $count ) {
  99                         if ( $n ) {
 100                                 $this->output( $n . "\n" );
 101                         }
 102                         $end = $n + self::RTI_CHUNK_SIZE - 1;
 103                         $queryBuilder = clone $queryBuilderTemplate;
 104                         $res = $queryBuilder->where( [
 105                                         $dbw->expr( 'page_id', '>=', $n )->and( 'page_id', '<=', $end ),
 106                                         'page_latest = rev_id'
 107                                 ] )->caller( __METHOD__ )->fetchResultSet();
 108
 109                         foreach ( $res as $s ) {
 110
 111                                 // T268673 Prevent failure of WikiPage.php: Invalid or virtual namespace -1 given
 112                                 if ( $s->page_namespace < 0 ) {
 113                                         continue;
 114                                 }
 115
 116                                 $title = Title::makeTitle( $s->page_namespace, $s->page_title );
 117                                 try {
 118                                         $revRecord = $revStore->newRevisionFromRow( $s );
 119                                         $content = $revRecord->getContent( SlotRecord::MAIN );
 120
 121                                         $u = new SearchUpdate( $s->page_id, $title, $content );
 122                                         $u->doUpdate();
 123                                 } catch ( MWContentSerializationException $ex ) {
 124                                         $this->output( "Failed to deserialize content of revision {$s->rev_id} of page "
 125                                                 . "`" . $title->getPrefixedDBkey() . "`!\n" );
 126                                 }
 127                         }
 128                         $n += self::RTI_CHUNK_SIZE;
 129                 }
 130         }
 131
 132         /**
 133          * (MySQL only) Drops fulltext index before populating the table.
 134          */
 135         private function dropMysqlTextIndex() {
 136                 $dbw = $this->getDB( DB_PRIMARY );
 137                 $searchindex = $dbw->tableName( 'searchindex' );
 138                 if ( $dbw->indexExists( 'searchindex', 'si_title', __METHOD__ ) ) {
 139                         $this->output( "Dropping index...\n" );
 140                         $sql = "ALTER TABLE $searchindex DROP INDEX si_title, DROP INDEX si_text";
 141                         $dbw->query( $sql, __METHOD__ );
 142                 }
 143         }
 144
 145         /**
 146          * (MySQL only) Adds back fulltext index after populating the table.
 147          */
 148         private function createMysqlTextIndex() {
 149                 $dbw = $this->getPrimaryDB();
 150                 $searchindex = $dbw->tableName( 'searchindex' );
 151                 $this->output( "\nRebuild the index...\n" );
 152                 foreach ( [ 'si_title', 'si_text' ] as $field ) {
 153                         $sql = "ALTER TABLE $searchindex ADD FULLTEXT $field ($field)";
 154                         $dbw->query( $sql, __METHOD__ );
 155                 }
 156         }
 157
 158         /**
 159          * Deletes everything from search index.
 160          */
 161         private function clearSearchIndex() {
 162                 $dbw = $this->getPrimaryDB();
 163                 $this->output( 'Clearing searchindex table...' );
 164                 $dbw->newDeleteQueryBuilder()
 165                         ->deleteFrom( 'searchindex' )
 166                         ->where( '*' )
 167                         ->caller( __METHOD__ )->execute();
 168                 $this->output( "Done\n" );
 169         }
 170 }
 171
 172 // @codeCoverageIgnoreStart
 173 $maintClass = RebuildTextIndex::class;
 174 require_once RUN_MAINTENANCE_IF_MAIN;
 175 // @codeCoverageIgnoreEnd