Update git submodules
[mediawiki.git] / maintenance / namespaceDupes.php
blob1a0f2fec772f4bf5d51b491fdcc33a7db0a18ec6
1 <?php
2 /**
3 * Check for articles to fix after adding/deleting namespaces
5 * Copyright © 2005-2007 Brion Vibber <brion@pobox.com>
6 * https://www.mediawiki.org/
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
18 * You should have received a copy of the GNU General Public License along
19 * with this program; if not, write to the Free Software Foundation, Inc.,
20 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
21 * http://www.gnu.org/copyleft/gpl.html
23 * @file
24 * @ingroup Maintenance
27 require_once __DIR__ . '/Maintenance.php';
29 use MediaWiki\Deferred\LinksUpdate\LinksDeletionUpdate;
30 use MediaWiki\Linker\LinkTarget;
31 use MediaWiki\MainConfigNames;
32 use MediaWiki\Title\Title;
33 use MediaWiki\Title\TitleValue;
34 use Wikimedia\Rdbms\IDatabase;
35 use Wikimedia\Rdbms\IMaintainableDatabase;
36 use Wikimedia\Rdbms\IResultWrapper;
38 /**
39 * Maintenance script that checks for articles to fix after
40 * adding/deleting namespaces.
42 * @ingroup Maintenance
44 class NamespaceDupes extends Maintenance {
46 /**
47 * @var IMaintainableDatabase
49 protected $db;
51 /**
52 * Total number of pages that need fixing that are automatically resolveable
53 * @var int
55 private $resolvablePages = 0;
57 /**
58 * Total number of pages that need fixing
59 * @var int
61 private $totalPages = 0;
63 /**
64 * Total number of links that need fixing that are automatically resolveable
65 * @var int
67 private $resolvableLinks = 0;
69 /**
70 * Total number of erroneous links
71 * @var int
73 private $totalLinks = 0;
75 /**
76 * Total number of links deleted because they weren't automatically resolveable due to the
77 * target already existing
78 * @var int
80 private $deletedLinks = 0;
82 public function __construct() {
83 parent::__construct();
84 $this->addDescription( 'Find and fix pages affected by namespace addition/removal' );
85 $this->addOption( 'fix', 'Attempt to automatically fix errors and delete broken links' );
86 $this->addOption( 'merge', "Instead of renaming conflicts, do a history merge with " .
87 "the correct title" );
88 $this->addOption( 'add-suffix', "Dupes will be renamed with correct namespace with " .
89 "<text> appended after the article name", false, true );
90 $this->addOption( 'add-prefix', "Dupes will be renamed with correct namespace with " .
91 "<text> prepended before the article name", false, true );
92 $this->addOption( 'source-pseudo-namespace', "Move all pages with the given source " .
93 "prefix (with an implied colon following it). If --dest-namespace is not specified, " .
94 "the colon will be replaced with a hyphen.",
95 false, true );
96 $this->addOption( 'dest-namespace', "In combination with --source-pseudo-namespace, " .
97 "specify the namespace ID of the destination.", false, true );
98 $this->addOption( 'move-talk', "If this is specified, pages in the Talk namespace that " .
99 "begin with a conflicting prefix will be renamed, for example " .
100 "Talk:File:Foo -> File_Talk:Foo" );
103 public function execute() {
104 $options = [
105 'fix' => $this->hasOption( 'fix' ),
106 'merge' => $this->hasOption( 'merge' ),
107 'add-suffix' => $this->getOption( 'add-suffix', '' ),
108 'add-prefix' => $this->getOption( 'add-prefix', '' ),
109 'move-talk' => $this->hasOption( 'move-talk' ),
110 'source-pseudo-namespace' => $this->getOption( 'source-pseudo-namespace', '' ),
111 'dest-namespace' => intval( $this->getOption( 'dest-namespace', 0 ) )
114 if ( $options['source-pseudo-namespace'] !== '' ) {
115 $retval = $this->checkPrefix( $options );
116 } else {
117 $retval = $this->checkAll( $options );
120 if ( $retval ) {
121 $this->output( "\nLooks good!\n" );
122 } else {
123 $this->output( "\nOh noeees\n" );
128 * Check all namespaces
130 * @param array $options Associative array of validated command-line options
132 * @return bool
134 private function checkAll( $options ) {
135 $contLang = $this->getServiceContainer()->getContentLanguage();
136 $spaces = [];
138 // List interwikis first, so they'll be overridden
139 // by any conflicting local namespaces.
140 foreach ( $this->getInterwikiList() as $prefix ) {
141 $name = $contLang->ucfirst( $prefix );
142 $spaces[$name] = 0;
145 // Now pull in all canonical and alias namespaces...
146 foreach (
147 $this->getServiceContainer()->getNamespaceInfo()->getCanonicalNamespaces()
148 as $ns => $name
150 // This includes $wgExtraNamespaces
151 if ( $name !== '' ) {
152 $spaces[$name] = $ns;
155 foreach ( $contLang->getNamespaces() as $ns => $name ) {
156 if ( $name !== '' ) {
157 $spaces[$name] = $ns;
160 foreach ( $contLang->getNamespaceAliases() as $name => $ns ) {
161 $spaces[$name] = $ns;
164 // We'll need to check for lowercase keys as well,
165 // since we're doing case-sensitive searches in the db.
166 $capitalLinks = $this->getConfig()->get( MainConfigNames::CapitalLinks );
167 foreach ( $spaces as $name => $ns ) {
168 $moreNames = [];
169 $moreNames[] = $contLang->uc( $name );
170 $moreNames[] = $contLang->ucfirst( $contLang->lc( $name ) );
171 $moreNames[] = $contLang->ucwords( $name );
172 $moreNames[] = $contLang->ucwords( $contLang->lc( $name ) );
173 $moreNames[] = $contLang->ucwordbreaks( $name );
174 $moreNames[] = $contLang->ucwordbreaks( $contLang->lc( $name ) );
175 if ( !$capitalLinks ) {
176 foreach ( $moreNames as $altName ) {
177 $moreNames[] = $contLang->lcfirst( $altName );
179 $moreNames[] = $contLang->lcfirst( $name );
181 foreach ( array_unique( $moreNames ) as $altName ) {
182 if ( $altName !== $name ) {
183 $spaces[$altName] = $ns;
188 // Sort by namespace index, and if there are two with the same index,
189 // break the tie by sorting by name
190 $origSpaces = $spaces;
191 uksort( $spaces, static function ( $a, $b ) use ( $origSpaces ) {
192 return $origSpaces[$a] <=> $origSpaces[$b]
193 ?: $a <=> $b;
194 } );
196 $ok = true;
197 foreach ( $spaces as $name => $ns ) {
198 $ok = $this->checkNamespace( $ns, $name, $options ) && $ok;
201 $this->output(
202 "{$this->totalPages} pages to fix, " .
203 "{$this->resolvablePages} were resolvable.\n\n"
206 foreach ( $spaces as $name => $ns ) {
207 if ( $ns != 0 ) {
208 /* Fix up link destinations for non-interwiki links only.
210 * For example if a page has [[Foo:Bar]] and then a Foo namespace
211 * is introduced, pagelinks needs to be updated to have
212 * page_namespace = NS_FOO.
214 * If instead an interwiki prefix was introduced called "Foo",
215 * the link should instead be moved to the iwlinks table. If a new
216 * language is introduced called "Foo", or if there is a pagelink
217 * [[fr:Bar]] when interlanguage magic links are turned on, the
218 * link would have to be moved to the langlinks table. Let's put
219 * those cases in the too-hard basket for now. The consequences are
220 * not especially severe.
221 * @fixme Handle interwiki links, and pagelinks to Category:, File:
222 * which probably need reparsing.
225 $this->checkLinkTable( 'pagelinks', 'pl', $ns, $name, $options );
226 $this->checkLinkTable( 'templatelinks', 'tl', $ns, $name, $options );
228 // The redirect table has interwiki links randomly mixed in, we
229 // need to filter those out. For example [[w:Foo:Bar]] would
230 // have rd_interwiki=w and rd_namespace=0, which would match the
231 // query for a conflicting namespace "Foo" if filtering wasn't done.
232 $this->checkLinkTable( 'redirect', 'rd', $ns, $name, $options,
233 [ 'rd_interwiki' => null ] );
234 $this->checkLinkTable( 'redirect', 'rd', $ns, $name, $options,
235 [ 'rd_interwiki' => '' ] );
239 $this->output(
240 "{$this->totalLinks} links to fix, " .
241 "{$this->resolvableLinks} were resolvable, " .
242 "{$this->deletedLinks} were deleted.\n"
245 return $ok;
249 * @return string[]
251 private function getInterwikiList() {
252 $result = $this->getServiceContainer()->getInterwikiLookup()->getAllPrefixes();
253 return array_column( $result, 'iw_prefix' );
257 * Check a given prefix and try to move it into the given destination namespace
259 * @param int $ns Destination namespace id
260 * @param string $name
261 * @param array $options Associative array of validated command-line options
262 * @return bool
264 private function checkNamespace( $ns, $name, $options ) {
265 $targets = $this->getTargetList( $ns, $name, $options );
266 $count = $targets->numRows();
267 $this->totalPages += $count;
268 if ( $count == 0 ) {
269 return true;
272 $dryRunNote = $options['fix'] ? '' : ' DRY RUN ONLY';
274 $ok = true;
275 foreach ( $targets as $row ) {
276 // Find the new title and determine the action to take
278 $newTitle = $this->getDestinationTitle(
279 $ns, $name, $row->page_namespace, $row->page_title );
280 $logStatus = false;
281 if ( !$newTitle ) {
282 if ( $options['add-prefix'] == '' && $options['add-suffix'] == '' ) {
283 $logStatus = 'invalid title and --add-prefix not specified';
284 $action = 'abort';
285 } else {
286 $action = 'alternate';
288 } elseif ( $newTitle->exists() ) {
289 if ( $options['merge'] ) {
290 if ( $this->canMerge( $row->page_id, $newTitle, $logStatus ) ) {
291 $action = 'merge';
292 } else {
293 $action = 'abort';
295 } elseif ( $options['add-prefix'] == '' && $options['add-suffix'] == '' ) {
296 $action = 'abort';
297 $logStatus = 'dest title exists and --add-prefix not specified';
298 } else {
299 $action = 'alternate';
301 } else {
302 $action = 'move';
303 $logStatus = 'no conflict';
305 if ( $action === 'alternate' ) {
306 [ $ns, $dbk ] = $this->getDestination( $ns, $name, $row->page_namespace,
307 $row->page_title );
308 $newTitle = $this->getAlternateTitle( $ns, $dbk, $options );
309 if ( !$newTitle ) {
310 $action = 'abort';
311 $logStatus = 'alternate title is invalid';
312 } elseif ( $newTitle->exists() ) {
313 $action = 'abort';
314 $logStatus = 'alternate title conflicts';
315 } else {
316 $action = 'move';
317 $logStatus = 'alternate';
321 // Take the action or log a dry run message
323 $logTitle = "id={$row->page_id} ns={$row->page_namespace} dbk={$row->page_title}";
324 $pageOK = true;
326 switch ( $action ) {
327 case 'abort':
328 $this->output( "$logTitle *** $logStatus\n" );
329 $pageOK = false;
330 break;
331 case 'move':
332 $this->output( "$logTitle -> " .
333 $newTitle->getPrefixedDBkey() . " ($logStatus)$dryRunNote\n" );
335 if ( $options['fix'] ) {
336 $pageOK = $this->movePage( $row->page_id, $newTitle );
338 break;
339 case 'merge':
340 $this->output( "$logTitle => " .
341 $newTitle->getPrefixedDBkey() . " (merge)$dryRunNote\n" );
343 if ( $options['fix'] ) {
344 $pageOK = $this->mergePage( $row, $newTitle );
346 break;
349 if ( $pageOK ) {
350 $this->resolvablePages++;
351 } else {
352 $ok = false;
356 return $ok;
360 * Check and repair the destination fields in a link table
361 * @param string $table The link table name
362 * @param string $fieldPrefix The field prefix in the link table
363 * @param int $ns Destination namespace id
364 * @param string $name
365 * @param array $options Associative array of validated command-line options
366 * @param array $extraConds Extra conditions for the SQL query
368 private function checkLinkTable( $table, $fieldPrefix, $ns, $name, $options,
369 $extraConds = []
371 $dbw = $this->getDB( DB_PRIMARY );
373 $batchConds = [];
374 $fromField = "{$fieldPrefix}_from";
375 $batchSize = 500;
376 $linksMigration = $this->getServiceContainer()->getLinksMigration();
377 if ( isset( $linksMigration::$mapping[$table] ) ) {
378 $queryInfo = $linksMigration->getQueryInfo( $table );
379 [ $namespaceField, $titleField ] = $linksMigration->getTitleFields( $table );
380 } else {
381 $queryInfo = [
382 'tables' => [ $table ],
383 'fields' => [
384 "{$fieldPrefix}_namespace",
385 "{$fieldPrefix}_title"
387 'joins' => []
389 $namespaceField = "{$fieldPrefix}_namespace";
390 $titleField = "{$fieldPrefix}_title";
393 while ( true ) {
394 $res = $dbw->select(
395 $queryInfo['tables'],
396 array_merge( [ $fromField ], $queryInfo['fields'] ),
397 array_merge(
398 $batchConds,
399 $extraConds,
401 $namespaceField => 0,
402 $titleField . $dbw->buildLike( "$name:", $dbw->anyString() )
405 __METHOD__,
407 'ORDER BY' => [ $titleField, $fromField ],
408 'LIMIT' => $batchSize
410 $queryInfo['joins']
413 if ( $res->numRows() == 0 ) {
414 break;
417 $rowsToDeleteIfStillExists = [];
419 foreach ( $res as $row ) {
420 $logTitle = "from={$row->$fromField} ns={$row->$namespaceField} " .
421 "dbk={$row->$titleField}";
422 $destTitle = $this->getDestinationTitle(
423 $ns, $name, $row->$namespaceField, $row->$titleField );
424 $this->totalLinks++;
425 if ( !$destTitle ) {
426 $this->output( "$table $logTitle *** INVALID\n" );
427 continue;
429 $this->resolvableLinks++;
430 if ( !$options['fix'] ) {
431 $this->output( "$table $logTitle -> " .
432 $destTitle->getPrefixedDBkey() . " DRY RUN\n" );
433 continue;
436 if ( isset( $linksMigration::$mapping[$table] ) ) {
437 $setValue = $linksMigration->getLinksConditions( $table, $destTitle );
438 $whereCondition = $linksMigration->getLinksConditions(
439 $table,
440 new TitleValue( 0, $row->$titleField )
442 $deleteCondition = $linksMigration->getLinksConditions(
443 $table,
444 new TitleValue( (int)$row->$namespaceField, $row->$titleField )
446 } else {
447 $setValue = [
448 $namespaceField => $destTitle->getNamespace(),
449 $titleField => $destTitle->getDBkey()
451 $whereCondition = [
452 $namespaceField => 0,
453 $titleField => $row->$titleField
455 $deleteCondition = [
456 $namespaceField => $row->$namespaceField,
457 $titleField => $row->$titleField,
461 $dbw->update( $table,
462 // SET
463 $setValue,
464 // WHERE
465 array_merge( [ $fromField => $row->$fromField ], $whereCondition ),
466 __METHOD__,
467 [ 'IGNORE' ]
470 $rowsToDeleteIfStillExists[] = $dbw->makeList(
471 array_merge( [ $fromField => $row->$fromField ], $deleteCondition ),
472 IDatabase::LIST_AND
475 $this->output( "$table $logTitle -> " .
476 $destTitle->getPrefixedDBkey() . "\n"
480 if ( $options['fix'] && count( $rowsToDeleteIfStillExists ) > 0 ) {
481 $dbw->delete(
482 $table,
483 $dbw->makeList( $rowsToDeleteIfStillExists, IDatabase::LIST_OR ),
484 __METHOD__
487 $this->deletedLinks += $dbw->affectedRows();
488 $this->resolvableLinks -= $dbw->affectedRows();
491 $batchConds = [
492 $dbw->buildComparison( '>', [
493 // @phan-suppress-next-line PhanPossiblyUndeclaredVariable rows contains at least one item
494 $titleField => $row->$titleField,
495 // @phan-suppress-next-line PhanPossiblyUndeclaredVariable rows contains at least one item
496 $fromField => $row->$fromField,
500 $this->waitForReplication();
505 * Move the given pseudo-namespace, either replacing the colon with a hyphen
506 * (useful for pseudo-namespaces that conflict with interwiki links) or move
507 * them to another namespace if specified.
508 * @param array $options Associative array of validated command-line options
509 * @return bool
511 private function checkPrefix( $options ) {
512 $prefix = $options['source-pseudo-namespace'];
513 $ns = $options['dest-namespace'];
514 $this->output( "Checking prefix \"$prefix\" vs namespace $ns\n" );
516 return $this->checkNamespace( $ns, $prefix, $options );
520 * Find pages in main and talk namespaces that have a prefix of the new
521 * namespace so we know titles that will need migrating
523 * @param int $ns Destination namespace id
524 * @param string $name Prefix that is being made a namespace
525 * @param array $options Associative array of validated command-line options
527 * @return IResultWrapper
529 private function getTargetList( $ns, $name, $options ) {
530 $dbw = $this->getDB( DB_PRIMARY );
532 if (
533 $options['move-talk'] &&
534 $this->getServiceContainer()->getNamespaceInfo()->isSubject( $ns )
536 $checkNamespaces = [ NS_MAIN, NS_TALK ];
537 } else {
538 $checkNamespaces = NS_MAIN;
541 return $dbw->newSelectQueryBuilder()
542 ->select( [ 'page_id', 'page_title', 'page_namespace' ] )
543 ->from( 'page' )
544 ->where( [
545 'page_namespace' => $checkNamespaces,
546 'page_title' . $dbw->buildLike( "$name:", $dbw->anyString() ),
548 ->caller( __METHOD__ )->fetchResultSet();
552 * Get the preferred destination for a given target page.
553 * @param int $ns The destination namespace ID
554 * @param string $name The conflicting prefix
555 * @param int $sourceNs The source namespace
556 * @param string $sourceDbk The source DB key (i.e. page_title)
557 * @return array [ ns, dbkey ], not necessarily valid
559 private function getDestination( $ns, $name, $sourceNs, $sourceDbk ) {
560 $dbk = substr( $sourceDbk, strlen( "$name:" ) );
561 if ( $ns == 0 ) {
562 // An interwiki; try an alternate encoding with '-' for ':'
563 $dbk = "$name-" . $dbk;
565 $destNS = $ns;
566 $nsInfo = $this->getServiceContainer()->getNamespaceInfo();
567 if ( $sourceNs == NS_TALK && $nsInfo->isSubject( $ns ) ) {
568 // This is an associated talk page moved with the --move-talk feature.
569 $destNS = $nsInfo->getTalk( $destNS );
571 return [ $destNS, $dbk ];
575 * Get the preferred destination title for a given target page.
576 * @param int $ns The destination namespace ID
577 * @param string $name The conflicting prefix
578 * @param int $sourceNs The source namespace
579 * @param string $sourceDbk The source DB key (i.e. page_title)
580 * @return Title|false
582 private function getDestinationTitle( $ns, $name, $sourceNs, $sourceDbk ) {
583 [ $destNS, $dbk ] = $this->getDestination( $ns, $name, $sourceNs, $sourceDbk );
584 $newTitle = Title::makeTitleSafe( $destNS, $dbk );
585 if ( !$newTitle || !$newTitle->canExist() ) {
586 return false;
588 return $newTitle;
592 * Get an alternative title to move a page to. This is used if the
593 * preferred destination title already exists.
595 * @param int $ns The destination namespace ID
596 * @param string $dbk The source DB key (i.e. page_title)
597 * @param array $options Associative array of validated command-line options
598 * @return Title|false
600 private function getAlternateTitle( $ns, $dbk, $options ) {
601 $prefix = $options['add-prefix'];
602 $suffix = $options['add-suffix'];
603 if ( $prefix == '' && $suffix == '' ) {
604 return false;
606 $newDbk = $prefix . $dbk . $suffix;
607 return Title::makeTitleSafe( $ns, $newDbk );
611 * Move a page
613 * @param int $id The page_id
614 * @param LinkTarget $newLinkTarget The new title link target
615 * @return bool
617 private function movePage( $id, LinkTarget $newLinkTarget ) {
618 $dbw = $this->getDB( DB_PRIMARY );
620 $dbw->update( 'page',
622 "page_namespace" => $newLinkTarget->getNamespace(),
623 "page_title" => $newLinkTarget->getDBkey(),
626 "page_id" => $id,
628 __METHOD__
631 // Update *_from_namespace in links tables
632 $fromNamespaceTables = [
633 [ 'pagelinks', 'pl' ],
634 [ 'templatelinks', 'tl' ],
635 [ 'imagelinks', 'il' ]
637 foreach ( $fromNamespaceTables as [ $table, $fieldPrefix ] ) {
638 $dbw->update( $table,
639 // SET
640 [ "{$fieldPrefix}_from_namespace" => $newLinkTarget->getNamespace() ],
641 // WHERE
642 [ "{$fieldPrefix}_from" => $id ],
643 __METHOD__
647 return true;
651 * Determine if we can merge a page.
652 * We check if an inaccessible revision would become the latest and
653 * deny the merge if so -- it's theoretically possible to update the
654 * latest revision, but opens a can of worms -- search engine updates,
655 * recentchanges review, etc.
657 * @param int $id The page_id
658 * @param LinkTarget $linkTarget The new link target
659 * @param string &$logStatus This is set to the log status message on failure @phan-output-reference
660 * @return bool
662 private function canMerge( $id, LinkTarget $linkTarget, &$logStatus ) {
663 $revisionLookup = $this->getServiceContainer()->getRevisionLookup();
664 $latestDest = $revisionLookup->getRevisionByTitle( $linkTarget, 0,
665 IDBAccessObject::READ_LATEST );
666 $latestSource = $revisionLookup->getRevisionByPageId( $id, 0,
667 IDBAccessObject::READ_LATEST );
668 if ( $latestSource->getTimestamp() > $latestDest->getTimestamp() ) {
669 $logStatus = 'cannot merge since source is later';
670 return false;
671 } else {
672 return true;
677 * Merge page histories
679 * @param stdClass $row Page row
680 * @param Title $newTitle
681 * @return bool
683 private function mergePage( $row, Title $newTitle ) {
684 $dbw = $this->getDB( DB_PRIMARY );
686 $id = $row->page_id;
688 // Construct the WikiPage object we will need later, while the
689 // page_id still exists. Note that this cannot use makeTitleSafe(),
690 // we are deliberately constructing an invalid title.
691 $sourceTitle = Title::makeTitle( $row->page_namespace, $row->page_title );
692 $sourceTitle->resetArticleID( $id );
693 $wikiPage = $this->getServiceContainer()->getWikiPageFactory()->newFromTitle( $sourceTitle );
694 $wikiPage->loadPageData( WikiPage::READ_LATEST );
696 $destId = $newTitle->getArticleID();
697 $this->beginTransaction( $dbw, __METHOD__ );
698 $dbw->update( 'revision',
699 // SET
700 [ 'rev_page' => $destId ],
701 // WHERE
702 [ 'rev_page' => $id ],
703 __METHOD__
706 $dbw->delete( 'page', [ 'page_id' => $id ], __METHOD__ );
708 $this->commitTransaction( $dbw, __METHOD__ );
710 /* Call LinksDeletionUpdate to delete outgoing links from the old title,
711 * and update category counts.
713 * Calling external code with a fake broken Title is a fairly dubious
714 * idea. It's necessary because it's quite a lot of code to duplicate,
715 * but that also makes it fragile since it would be easy for someone to
716 * accidentally introduce an assumption of title validity to the code we
717 * are calling.
719 DeferredUpdates::addUpdate( new LinksDeletionUpdate( $wikiPage ) );
720 DeferredUpdates::doUpdates();
722 return true;
726 $maintClass = NamespaceDupes::class;
727 require_once RUN_MAINTENANCE_IF_MAIN;